dsh: add Transcribe audio (whisper) + Ask with a file tools, workspace download route, OmniRoute x-opencode-session fix, regrouped toolbar

dsh/app.py

@@ -26,7 +26,7 @@ from datetime import datetime, timezone
 
 import httpx
 from fastapi import FastAPI, File, Form, Request, UploadFile
-from fastapi.responses import HTMLResponse, StreamingResponse
+from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
@@ -49,6 +49,9 @@ SYSTEM_PROMPT = os.getenv(
 # image route is currently unreliable). Sent as a separate provider call. ---
 VISION_BASE = os.getenv("DSH_VISION_BASE", "https://openrouter.ai/api/v1")
 VISION_MODEL = os.getenv("DSH_VISION_MODEL", "openai/gpt-5")
+
+# --- existing speech->text (voice_whisper on .13:5000, faster-whisper). Do NOT edit that service ---
+WHISPER_URL = os.getenv("DSH_WHISPER_URL", "http://192.168.20.13:5000/transcribe")
 VISION_KEY = os.getenv("DSH_VISION_KEY", "")
 IMAGE_MODEL = os.getenv("DSH_IMAGE_MODEL", "openai/gpt-image-1")
 VIDEO_MODEL = os.getenv("DSH_VIDEO_MODEL", "z-ai/glm-5v-turbo")
@@ -145,6 +148,8 @@ async def _complete(messages) -> str:
     """Non-streaming OmniRoute call for summarisation tools."""
     payload = {"model": LLM_MODEL, "messages": messages, "stream": False, "temperature": 0.3}
     headers = {"Authorization": f"Bearer {LLM_KEY}"} if LLM_KEY else {}
+    # OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID")
+    headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}"
     url = f"{LLM_BASE.rstrip('/')}/chat/completions"
     async with httpx.AsyncClient(timeout=180) as client:
         r = await client.post(url, json=payload, headers=headers)
@@ -303,6 +308,9 @@ async def chat(message: Annotated[str, Form()],
                 "temperature": 0.7,
             }
         headers = {"Authorization": f"Bearer {key}"} if key else {}
+        if base == LLM_BASE:
+            # OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID")
+            headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}"
         url = f"{base.rstrip('/')}/chat/completions"
         full: list[str] = []
         try:
@@ -376,6 +384,139 @@ async def analyze_video(file: Annotated[UploadFile, File()]) -> dict:
             os.remove(vid)
 
 
+AUDIO_EXT = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aiff", ".aif", ".opus", ".mp4", ".mov", ".webm", ".mkv"}
+TEXT_EXT = {".txt", ".md", ".json", ".log", ".csv"}
+IMAGE_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".avi"}
+
+
+def _ws() -> Path:
+    """Workspace root (bounded)."""
+    return Path(WORKSPACE_DIR).resolve()
+
+
+def _ws_path(path: str) -> Path:
+    """Resolve a workspace-relative path, refusing traversal."""
+    p = (_ws() / path.lstrip("/")).resolve()
+    if not str(p).startswith(str(_ws())):
+        raise ValueError("Path must stay inside your workspace.")
+    return p
+
+
+async def _whisper(data: bytes, fname: str, mime: str) -> str:
+    """Transcribe audio via the existing voice_whisper service (.13:5000)."""
+    async with httpx.AsyncClient(timeout=300) as client:
+        r = await client.post(WHISPER_URL, files={"audio": (fname, data, mime)}, timeout=300)
+        r.raise_for_status()
+        return (r.json() or {}).get("transcript", "") or ""
+
+
+@app.post("/api/tool/transcribe")
+async def transcribe_audio(file: Annotated[UploadFile, File()]) -> dict:
+    """Speech -> text. Save the audio into the workspace, then whisper it."""
+    data = await file.read()
+    if not data:
+        return {"ok": False, "error": "No audio received."}
+    up = _ws() / "uploads"
+    up.mkdir(parents=True, exist_ok=True)
+    fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'audio'}"
+    audio_path = up / fname
+    audio_path.write_bytes(data)
+    try:
+        transcript = await _whisper(data, file.filename or "audio.wav", file.content_type or "audio/wav")
+    except Exception as exc:  # noqa: BLE001
+        return {"ok": False, "error": f"Whisper failed: {exc}", "path": f"uploads/{fname}"}
+    if not transcript.strip():
+        return {"ok": True, "transcript": "", "path": f"uploads/{fname}", "note": "No speech detected."}
+    tr_dir = _ws() / "transcripts"
+    tr_dir.mkdir(parents=True, exist_ok=True)
+    txt_name = Path(fname).stem + ".txt"
+    (tr_dir / txt_name).write_text(transcript, encoding="utf-8")
+    return {"ok": True, "transcript": transcript, "path": f"uploads/{fname}", "txt": f"transcripts/{txt_name}"}
+
+
+@app.post("/api/tool/upload")
+async def upload_file(file: Annotated[UploadFile, File()]) -> dict:
+    """Save an arbitrary file into the user workspace (bounded)."""
+    data = await file.read()
+    if not data:
+        return {"ok": False, "error": "No file received."}
+    up = _ws() / "uploads"
+    up.mkdir(parents=True, exist_ok=True)
+    fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'file'}"
+    (up / fname).write_bytes(data)
+    return {"ok": True, "path": f"uploads/{fname}"}
+
+
+@app.post("/api/tool/askfile")
+async def ask_file(path: Annotated[str, Form()], question: Annotated[str, Form()]) -> dict:
+    """Generic: pick any workspace file and ask the AI to do something with it.
+    Routes by type: audio -> whisper then reason; image/video -> vision;
+    text -> read then reason. No extra skills needed."""
+    try:
+        fp = _ws_path(path)
+    except ValueError as exc:
+        return {"ok": False, "error": str(exc)}
+    if not fp.is_file():
+        return {"ok": False, "error": f"File not found: {path}"}
+    q = (question or "").strip()[:1000] or "What can you tell me about this file?"
+    ext = "." + fp.name.rsplit(".", 1)[-1].lower() if "." in fp.name else ""
+
+    if ext in AUDIO_EXT and ext not in VIDEO_EXT:
+        try:
+            transcript = await _whisper(fp.read_bytes(), fp.name, "audio/wav")
+        except Exception as exc:  # noqa: BLE001
+            return {"ok": False, "error": f"Whisper failed: {exc}"}
+        if not transcript.strip():
+            return {"ok": True, "transcript": "", "error": "No speech detected in that audio."}
+        out = await _complete([
+            {"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."},
+            {"role": "user", "content": f"AUDIO TRANSCRIPT:\n{transcript[:30000]}\n\nQUESTION: {q}"},
+        ])
+        return {"ok": True, "answer": out, "transcript": transcript}
+
+    if ext in IMAGE_EXT:
+        b64 = "data:image/jpeg;base64," + base64.b64encode(fp.read_bytes()).decode()
+        out = await _vision_complete([{"role": "user", "content": [
+            {"type": "text", "text": q},
+            {"type": "image_url", "image_url": {"url": b64}},
+        ]}])
+        return {"ok": True, "answer": out}
+
+    if ext in VIDEO_EXT:
+        frames = _extract_frames(str(fp))
+        if not frames:
+            return {"ok": False, "error": "Could not extract frames from that video."}
+        content = [{"type": "text", "text": q}]
+        content += [{"type": "image_url", "image_url": {"url": f}} for f in frames]
+        out = await _vision_complete([{"role": "user", "content": content}])
+        return {"ok": True, "answer": out}
+
+    if ext in TEXT_EXT:
+        text = fp.read_text(encoding="utf-8", errors="replace")[:60000]
+        out = await _complete([
+            {"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."},
+            {"role": "user", "content": f"FILE ({fp.name}):\n{text}\n\nQUESTION: {q}"},
+        ])
+        return {"ok": True, "answer": out}
+
+    return {"ok": False, "error": f"Unsupported file type '{ext}'. Supported: audio, image, video, txt/md/json/log/csv."}
+
+
+@app.get("/api/files/{path:path}")
+async def get_file(path: str) -> FileResponse:
+    """Download a workspace file (images, transcripts, uploads). Bounded to workspace."""
+    try:
+        fp = _ws_path(path)
+    except ValueError as exc:
+        from fastapi.responses import JSONResponse
+        return JSONResponse({"error": str(exc)}, status_code=400)  # type: ignore[return-value]
+    if not fp.is_file():
+        from fastapi.responses import JSONResponse
+        return JSONResponse({"error": "Not found"}, status_code=404)  # type: ignore[return-value]
+    return FileResponse(str(fp), filename=fp.name, content_disposition_type="attachment")
+
+
 @app.post("/api/tool/image")
 async def generate_image(prompt: Annotated[str, Form()]) -> dict:
     """Generate an image from a prompt via OpenRouter (gpt-image-1); save to workspace."""