diff --git a/dsh/app.py b/dsh/app.py index 57f64e8..0fbf0e6 100644 --- a/dsh/app.py +++ b/dsh/app.py @@ -26,7 +26,7 @@ from datetime import datetime, timezone import httpx from fastapi import FastAPI, File, Form, Request, UploadFile -from fastapi.responses import HTMLResponse, StreamingResponse +from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -49,6 +49,9 @@ SYSTEM_PROMPT = os.getenv( # image route is currently unreliable). Sent as a separate provider call. --- VISION_BASE = os.getenv("DSH_VISION_BASE", "https://openrouter.ai/api/v1") VISION_MODEL = os.getenv("DSH_VISION_MODEL", "openai/gpt-5") + +# --- existing speech->text (voice_whisper on .13:5000, faster-whisper). Do NOT edit that service --- +WHISPER_URL = os.getenv("DSH_WHISPER_URL", "http://192.168.20.13:5000/transcribe") VISION_KEY = os.getenv("DSH_VISION_KEY", "") IMAGE_MODEL = os.getenv("DSH_IMAGE_MODEL", "openai/gpt-image-1") VIDEO_MODEL = os.getenv("DSH_VIDEO_MODEL", "z-ai/glm-5v-turbo") @@ -145,6 +148,8 @@ async def _complete(messages) -> str: """Non-streaming OmniRoute call for summarisation tools.""" payload = {"model": LLM_MODEL, "messages": messages, "stream": False, "temperature": 0.3} headers = {"Authorization": f"Bearer {LLM_KEY}"} if LLM_KEY else {} + # OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID") + headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}" url = f"{LLM_BASE.rstrip('/')}/chat/completions" async with httpx.AsyncClient(timeout=180) as client: r = await client.post(url, json=payload, headers=headers) @@ -303,6 +308,9 @@ async def chat(message: Annotated[str, Form()], "temperature": 0.7, } headers = {"Authorization": f"Bearer {key}"} if key else {} + if base == LLM_BASE: + # OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID") + headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}" url = f"{base.rstrip('/')}/chat/completions" full: list[str] = [] try: @@ -376,6 +384,139 @@ async def analyze_video(file: Annotated[UploadFile, File()]) -> dict: os.remove(vid) +AUDIO_EXT = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aiff", ".aif", ".opus", ".mp4", ".mov", ".webm", ".mkv"} +TEXT_EXT = {".txt", ".md", ".json", ".log", ".csv"} +IMAGE_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp"} +VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".avi"} + + +def _ws() -> Path: + """Workspace root (bounded).""" + return Path(WORKSPACE_DIR).resolve() + + +def _ws_path(path: str) -> Path: + """Resolve a workspace-relative path, refusing traversal.""" + p = (_ws() / path.lstrip("/")).resolve() + if not str(p).startswith(str(_ws())): + raise ValueError("Path must stay inside your workspace.") + return p + + +async def _whisper(data: bytes, fname: str, mime: str) -> str: + """Transcribe audio via the existing voice_whisper service (.13:5000).""" + async with httpx.AsyncClient(timeout=300) as client: + r = await client.post(WHISPER_URL, files={"audio": (fname, data, mime)}, timeout=300) + r.raise_for_status() + return (r.json() or {}).get("transcript", "") or "" + + +@app.post("/api/tool/transcribe") +async def transcribe_audio(file: Annotated[UploadFile, File()]) -> dict: + """Speech -> text. Save the audio into the workspace, then whisper it.""" + data = await file.read() + if not data: + return {"ok": False, "error": "No audio received."} + up = _ws() / "uploads" + up.mkdir(parents=True, exist_ok=True) + fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'audio'}" + audio_path = up / fname + audio_path.write_bytes(data) + try: + transcript = await _whisper(data, file.filename or "audio.wav", file.content_type or "audio/wav") + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"Whisper failed: {exc}", "path": f"uploads/{fname}"} + if not transcript.strip(): + return {"ok": True, "transcript": "", "path": f"uploads/{fname}", "note": "No speech detected."} + tr_dir = _ws() / "transcripts" + tr_dir.mkdir(parents=True, exist_ok=True) + txt_name = Path(fname).stem + ".txt" + (tr_dir / txt_name).write_text(transcript, encoding="utf-8") + return {"ok": True, "transcript": transcript, "path": f"uploads/{fname}", "txt": f"transcripts/{txt_name}"} + + +@app.post("/api/tool/upload") +async def upload_file(file: Annotated[UploadFile, File()]) -> dict: + """Save an arbitrary file into the user workspace (bounded).""" + data = await file.read() + if not data: + return {"ok": False, "error": "No file received."} + up = _ws() / "uploads" + up.mkdir(parents=True, exist_ok=True) + fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'file'}" + (up / fname).write_bytes(data) + return {"ok": True, "path": f"uploads/{fname}"} + + +@app.post("/api/tool/askfile") +async def ask_file(path: Annotated[str, Form()], question: Annotated[str, Form()]) -> dict: + """Generic: pick any workspace file and ask the AI to do something with it. + Routes by type: audio -> whisper then reason; image/video -> vision; + text -> read then reason. No extra skills needed.""" + try: + fp = _ws_path(path) + except ValueError as exc: + return {"ok": False, "error": str(exc)} + if not fp.is_file(): + return {"ok": False, "error": f"File not found: {path}"} + q = (question or "").strip()[:1000] or "What can you tell me about this file?" + ext = "." + fp.name.rsplit(".", 1)[-1].lower() if "." in fp.name else "" + + if ext in AUDIO_EXT and ext not in VIDEO_EXT: + try: + transcript = await _whisper(fp.read_bytes(), fp.name, "audio/wav") + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"Whisper failed: {exc}"} + if not transcript.strip(): + return {"ok": True, "transcript": "", "error": "No speech detected in that audio."} + out = await _complete([ + {"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."}, + {"role": "user", "content": f"AUDIO TRANSCRIPT:\n{transcript[:30000]}\n\nQUESTION: {q}"}, + ]) + return {"ok": True, "answer": out, "transcript": transcript} + + if ext in IMAGE_EXT: + b64 = "data:image/jpeg;base64," + base64.b64encode(fp.read_bytes()).decode() + out = await _vision_complete([{"role": "user", "content": [ + {"type": "text", "text": q}, + {"type": "image_url", "image_url": {"url": b64}}, + ]}]) + return {"ok": True, "answer": out} + + if ext in VIDEO_EXT: + frames = _extract_frames(str(fp)) + if not frames: + return {"ok": False, "error": "Could not extract frames from that video."} + content = [{"type": "text", "text": q}] + content += [{"type": "image_url", "image_url": {"url": f}} for f in frames] + out = await _vision_complete([{"role": "user", "content": content}]) + return {"ok": True, "answer": out} + + if ext in TEXT_EXT: + text = fp.read_text(encoding="utf-8", errors="replace")[:60000] + out = await _complete([ + {"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."}, + {"role": "user", "content": f"FILE ({fp.name}):\n{text}\n\nQUESTION: {q}"}, + ]) + return {"ok": True, "answer": out} + + return {"ok": False, "error": f"Unsupported file type '{ext}'. Supported: audio, image, video, txt/md/json/log/csv."} + + +@app.get("/api/files/{path:path}") +async def get_file(path: str) -> FileResponse: + """Download a workspace file (images, transcripts, uploads). Bounded to workspace.""" + try: + fp = _ws_path(path) + except ValueError as exc: + from fastapi.responses import JSONResponse + return JSONResponse({"error": str(exc)}, status_code=400) # type: ignore[return-value] + if not fp.is_file(): + from fastapi.responses import JSONResponse + return JSONResponse({"error": "Not found"}, status_code=404) # type: ignore[return-value] + return FileResponse(str(fp), filename=fp.name, content_disposition_type="attachment") + + @app.post("/api/tool/image") async def generate_image(prompt: Annotated[str, Form()]) -> dict: """Generate an image from a prompt via OpenRouter (gpt-image-1); save to workspace.""" diff --git a/dsh/templates/chat.html b/dsh/templates/chat.html index adb0209..75241e6 100644 --- a/dsh/templates/chat.html +++ b/dsh/templates/chat.html @@ -62,8 +62,10 @@ border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; font: 15px/1.4 var(--font-family, sans-serif); resize: vertical; } - .send-row { display: flex; justify-content: flex-end; } - #sendBtn { border: none; border-radius: 8px; padding: 9px 20px; cursor: pointer; background: var(--primary, #0075de); color: #fff; font-size: 14px; } + .send-row { display: flex; align-items: stretch; gap: 6px; } + .send-row #input { flex: 1 1 auto; } + .attach { flex: 0 0 auto; display: inline-flex; align-items: center; justify-content: center; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 0 9px; cursor: pointer; font-size: 16px; color: var(--ink, #111); } + #sendBtn { border: none; border-radius: 8px; padding: 0 18px; cursor: pointer; background: var(--primary, #0075de); color: #fff; font-size: 14px; } #toolsPanel { border-top: 1px solid var(--hairline, #e6e6e6); } #toolsToggle { width: 100%; text-align: left; background: none; border: none; cursor: pointer; @@ -72,6 +74,8 @@ #toolsArrow { font-size: 11px; } #toolsBody { padding: 0 12px 10px; } #toolsBody.hidden { display: none; } + .tool-group { margin-top: 8px; border: 1px solid var(--hairline, #e6e6e6); border-radius: 10px; padding: 6px 10px; background: rgba(0,0,0,.02); } + .tool-group-title { font-size: 11px; font-weight: 600; color: #888; text-transform: uppercase; letter-spacing: .4px; margin-bottom: 2px; } .tools-row { display: flex; gap: 8px; flex-wrap: wrap; margin-top: 6px; } .tools-row input[type=text], .tools-row input[type=file] { flex: 2 1 170px; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 6px 10px; font-size: 13px; } .tools-row button { flex: 0 0 auto; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 6px 10px; cursor: pointer; background: #fff; } @@ -113,24 +117,53 @@
-
+ +
+ -
- + +
+
-
- - - - - - - - - + +
+
Ask with a file
+
+ + + +
+
+ +
+
Transcribe audio → text
+
+ + +
+
+ +
+
Generate image
+
+ + +
+
+ +
+
Workspace file / web page (legacy)
+
+ + + + +
@@ -249,20 +282,54 @@ } document.getElementById('sumWeb').addEventListener('click', () => runTool('/api/summarize/web', 'urlInput', 'Summarise page', 'url')); document.getElementById('sumDocs').addEventListener('click', () => runTool('/api/summarize/docs', 'fileInput', 'Summarise file', 'path')); - document.getElementById('genVid').addEventListener('click', async () => { - const f = document.getElementById('vidIn').files[0]; + + function downloadLink(path, label) { + const a = document.createElement('a'); a.href = '/api/files/' + path; a.download = ''; a.textContent = label; + a.style.cssText = 'font-size:12px;color:var(--accent,#6db3f2);text-decoration:underline'; return a; + } + async function postForm(url, body) { + const res = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams(body) }); + return res.json(); + } + function fail(bubble, data) { bubble.textContent = '[error: ' + (data.error || 'unknown') + ']'; } + document.getElementById('askBtn').addEventListener('click', async () => { + const f = document.getElementById('askIn').files[0]; const q = document.getElementById('askQ').value.trim(); if (!f) return; - addBubble('user', 'Analyze video: ' + f.name); addBubble('bot', ''); - const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…analyzing'; + addBubble('user', (q ? 'Do: ' + q + ' — ' : '') + f.name); addBubble('bot', ''); + const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…working on ' + f.name; const fd = new FormData(); fd.append('file', f); + let path = null; try { - const res = await fetch('/api/tool/video', { method: 'POST', body: fd }); - const d = await res.json(); - b.textContent = d.ok ? (d.summary || '') : '[error: ' + (d.error || 'unknown') + ']'; + const up = await fetch('/api/tool/upload', { method: 'POST', body: fd }); const u = await up.json(); + path = u.path; if (!u.ok) return fail(b, u); + const d = await postForm('/api/tool/askfile', { path, question: q }); + if (!d.ok) return fail(b, d); + b.textContent = d.error ? '' : (d.answer || ''); + if (d.error) { b.textContent = d.error; return; } + if (d.transcript && !d.answer) b.textContent = d.transcript; + } catch (err) { b.textContent = '[error: ' + err + ']'; } + logEl.scrollTop = logEl.scrollHeight; + }); + document.getElementById('transcribeBtn').addEventListener('click', async () => { + const f = document.getElementById('audioIn').files[0]; if (!f) return; + addBubble('user', 'Transcribe: ' + f.name); addBubble('bot', ''); + const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…transcribing (whisper)'; + const fd = new FormData(); fd.append('file', f); + try { + const res = await fetch('/api/tool/transcribe', { method: 'POST', body: fd }); + const d = await res.json(); + if (!d.ok) return fail(b, d); + b.textContent = d.transcript ? d.transcript : (d.note || 'No speech detected.'); + if (d.transcript) { + const dl = document.createElement('div'); + dl.appendChild(downloadLink(d.txt, '⬇ download transcript (.txt)')); + dl.appendChild(document.createTextNode(' · ')); + dl.appendChild(downloadLink(d.path, '⬇ audio')); + b.appendChild(dl); + } } catch (err) { b.textContent = '[error: ' + err + ']'; } logEl.scrollTop = logEl.scrollHeight; }); - document.getElementById('genImg').addEventListener('click', async () => { const val = document.getElementById('genInput').value.trim(); if (!val) return; addBubble('user', 'Generate: ' + val); addBubble('bot', ''); @@ -270,7 +337,7 @@ try { const res = await fetch('/api/tool/image', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ prompt: val }) }); const data = await res.json(); - if (data.ok) { b.textContent = ''; const im = document.createElement('img'); im.src = data.b64; im.style.cssText = 'max-width:320px;border-radius:8px;display:block'; b.appendChild(im); const cap = document.createElement('div'); cap.textContent = 'saved: ' + data.path; b.appendChild(cap); } + if (data.ok) { b.textContent = ''; const im = document.createElement('img'); im.src = data.b64; im.style.cssText = 'max-width:320px;border-radius:8px;display:block'; b.appendChild(im); const cap = document.createElement('div'); cap.appendChild(downloadLink(data.path, '⬇ download image')); b.appendChild(cap); } else b.textContent = '[error: ' + (data.error || 'unknown') + ']'; } catch (err) { b.textContent = '[error: ' + err + ']'; } logEl.scrollTop = logEl.scrollHeight;