dsh: add Transcribe audio (whisper) + Ask with a file tools, workspace download route, OmniRoute x-opencode-session fix, regrouped toolbar
This commit is contained in:
143
dsh/app.py
143
dsh/app.py
@@ -26,7 +26,7 @@ from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, File, Form, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse, StreamingResponse
|
||||
from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
@@ -49,6 +49,9 @@ SYSTEM_PROMPT = os.getenv(
|
||||
# image route is currently unreliable). Sent as a separate provider call. ---
|
||||
VISION_BASE = os.getenv("DSH_VISION_BASE", "https://openrouter.ai/api/v1")
|
||||
VISION_MODEL = os.getenv("DSH_VISION_MODEL", "openai/gpt-5")
|
||||
|
||||
# --- existing speech->text (voice_whisper on .13:5000, faster-whisper). Do NOT edit that service ---
|
||||
WHISPER_URL = os.getenv("DSH_WHISPER_URL", "http://192.168.20.13:5000/transcribe")
|
||||
VISION_KEY = os.getenv("DSH_VISION_KEY", "")
|
||||
IMAGE_MODEL = os.getenv("DSH_IMAGE_MODEL", "openai/gpt-image-1")
|
||||
VIDEO_MODEL = os.getenv("DSH_VIDEO_MODEL", "z-ai/glm-5v-turbo")
|
||||
@@ -145,6 +148,8 @@ async def _complete(messages) -> str:
|
||||
"""Non-streaming OmniRoute call for summarisation tools."""
|
||||
payload = {"model": LLM_MODEL, "messages": messages, "stream": False, "temperature": 0.3}
|
||||
headers = {"Authorization": f"Bearer {LLM_KEY}"} if LLM_KEY else {}
|
||||
# OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID")
|
||||
headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}"
|
||||
url = f"{LLM_BASE.rstrip('/')}/chat/completions"
|
||||
async with httpx.AsyncClient(timeout=180) as client:
|
||||
r = await client.post(url, json=payload, headers=headers)
|
||||
@@ -303,6 +308,9 @@ async def chat(message: Annotated[str, Form()],
|
||||
"temperature": 0.7,
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {key}"} if key else {}
|
||||
if base == LLM_BASE:
|
||||
# OmniRoute requires a session id for opencode-go models (else 400 "MissingSessionID")
|
||||
headers["x-opencode-session"] = f"dsh-{os.getpid()}-{uuid4().hex[:8]}"
|
||||
url = f"{base.rstrip('/')}/chat/completions"
|
||||
full: list[str] = []
|
||||
try:
|
||||
@@ -376,6 +384,139 @@ async def analyze_video(file: Annotated[UploadFile, File()]) -> dict:
|
||||
os.remove(vid)
|
||||
|
||||
|
||||
AUDIO_EXT = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aiff", ".aif", ".opus", ".mp4", ".mov", ".webm", ".mkv"}
|
||||
TEXT_EXT = {".txt", ".md", ".json", ".log", ".csv"}
|
||||
IMAGE_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
|
||||
VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".avi"}
|
||||
|
||||
|
||||
def _ws() -> Path:
|
||||
"""Workspace root (bounded)."""
|
||||
return Path(WORKSPACE_DIR).resolve()
|
||||
|
||||
|
||||
def _ws_path(path: str) -> Path:
|
||||
"""Resolve a workspace-relative path, refusing traversal."""
|
||||
p = (_ws() / path.lstrip("/")).resolve()
|
||||
if not str(p).startswith(str(_ws())):
|
||||
raise ValueError("Path must stay inside your workspace.")
|
||||
return p
|
||||
|
||||
|
||||
async def _whisper(data: bytes, fname: str, mime: str) -> str:
|
||||
"""Transcribe audio via the existing voice_whisper service (.13:5000)."""
|
||||
async with httpx.AsyncClient(timeout=300) as client:
|
||||
r = await client.post(WHISPER_URL, files={"audio": (fname, data, mime)}, timeout=300)
|
||||
r.raise_for_status()
|
||||
return (r.json() or {}).get("transcript", "") or ""
|
||||
|
||||
|
||||
@app.post("/api/tool/transcribe")
|
||||
async def transcribe_audio(file: Annotated[UploadFile, File()]) -> dict:
|
||||
"""Speech -> text. Save the audio into the workspace, then whisper it."""
|
||||
data = await file.read()
|
||||
if not data:
|
||||
return {"ok": False, "error": "No audio received."}
|
||||
up = _ws() / "uploads"
|
||||
up.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'audio'}"
|
||||
audio_path = up / fname
|
||||
audio_path.write_bytes(data)
|
||||
try:
|
||||
transcript = await _whisper(data, file.filename or "audio.wav", file.content_type or "audio/wav")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"ok": False, "error": f"Whisper failed: {exc}", "path": f"uploads/{fname}"}
|
||||
if not transcript.strip():
|
||||
return {"ok": True, "transcript": "", "path": f"uploads/{fname}", "note": "No speech detected."}
|
||||
tr_dir = _ws() / "transcripts"
|
||||
tr_dir.mkdir(parents=True, exist_ok=True)
|
||||
txt_name = Path(fname).stem + ".txt"
|
||||
(tr_dir / txt_name).write_text(transcript, encoding="utf-8")
|
||||
return {"ok": True, "transcript": transcript, "path": f"uploads/{fname}", "txt": f"transcripts/{txt_name}"}
|
||||
|
||||
|
||||
@app.post("/api/tool/upload")
|
||||
async def upload_file(file: Annotated[UploadFile, File()]) -> dict:
|
||||
"""Save an arbitrary file into the user workspace (bounded)."""
|
||||
data = await file.read()
|
||||
if not data:
|
||||
return {"ok": False, "error": "No file received."}
|
||||
up = _ws() / "uploads"
|
||||
up.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{file.filename or 'file'}"
|
||||
(up / fname).write_bytes(data)
|
||||
return {"ok": True, "path": f"uploads/{fname}"}
|
||||
|
||||
|
||||
@app.post("/api/tool/askfile")
|
||||
async def ask_file(path: Annotated[str, Form()], question: Annotated[str, Form()]) -> dict:
|
||||
"""Generic: pick any workspace file and ask the AI to do something with it.
|
||||
Routes by type: audio -> whisper then reason; image/video -> vision;
|
||||
text -> read then reason. No extra skills needed."""
|
||||
try:
|
||||
fp = _ws_path(path)
|
||||
except ValueError as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
if not fp.is_file():
|
||||
return {"ok": False, "error": f"File not found: {path}"}
|
||||
q = (question or "").strip()[:1000] or "What can you tell me about this file?"
|
||||
ext = "." + fp.name.rsplit(".", 1)[-1].lower() if "." in fp.name else ""
|
||||
|
||||
if ext in AUDIO_EXT and ext not in VIDEO_EXT:
|
||||
try:
|
||||
transcript = await _whisper(fp.read_bytes(), fp.name, "audio/wav")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"ok": False, "error": f"Whisper failed: {exc}"}
|
||||
if not transcript.strip():
|
||||
return {"ok": True, "transcript": "", "error": "No speech detected in that audio."}
|
||||
out = await _complete([
|
||||
{"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."},
|
||||
{"role": "user", "content": f"AUDIO TRANSCRIPT:\n{transcript[:30000]}\n\nQUESTION: {q}"},
|
||||
])
|
||||
return {"ok": True, "answer": out, "transcript": transcript}
|
||||
|
||||
if ext in IMAGE_EXT:
|
||||
b64 = "data:image/jpeg;base64," + base64.b64encode(fp.read_bytes()).decode()
|
||||
out = await _vision_complete([{"role": "user", "content": [
|
||||
{"type": "text", "text": q},
|
||||
{"type": "image_url", "image_url": {"url": b64}},
|
||||
]}])
|
||||
return {"ok": True, "answer": out}
|
||||
|
||||
if ext in VIDEO_EXT:
|
||||
frames = _extract_frames(str(fp))
|
||||
if not frames:
|
||||
return {"ok": False, "error": "Could not extract frames from that video."}
|
||||
content = [{"type": "text", "text": q}]
|
||||
content += [{"type": "image_url", "image_url": {"url": f}} for f in frames]
|
||||
out = await _vision_complete([{"role": "user", "content": content}])
|
||||
return {"ok": True, "answer": out}
|
||||
|
||||
if ext in TEXT_EXT:
|
||||
text = fp.read_text(encoding="utf-8", errors="replace")[:60000]
|
||||
out = await _complete([
|
||||
{"role": "system", "content": "You help with a file the user uploaded. Quote where useful, be concise."},
|
||||
{"role": "user", "content": f"FILE ({fp.name}):\n{text}\n\nQUESTION: {q}"},
|
||||
])
|
||||
return {"ok": True, "answer": out}
|
||||
|
||||
return {"ok": False, "error": f"Unsupported file type '{ext}'. Supported: audio, image, video, txt/md/json/log/csv."}
|
||||
|
||||
|
||||
@app.get("/api/files/{path:path}")
|
||||
async def get_file(path: str) -> FileResponse:
|
||||
"""Download a workspace file (images, transcripts, uploads). Bounded to workspace."""
|
||||
try:
|
||||
fp = _ws_path(path)
|
||||
except ValueError as exc:
|
||||
from fastapi.responses import JSONResponse
|
||||
return JSONResponse({"error": str(exc)}, status_code=400) # type: ignore[return-value]
|
||||
if not fp.is_file():
|
||||
from fastapi.responses import JSONResponse
|
||||
return JSONResponse({"error": "Not found"}, status_code=404) # type: ignore[return-value]
|
||||
return FileResponse(str(fp), filename=fp.name, content_disposition_type="attachment")
|
||||
|
||||
|
||||
@app.post("/api/tool/image")
|
||||
async def generate_image(prompt: Annotated[str, Form()]) -> dict:
|
||||
"""Generate an image from a prompt via OpenRouter (gpt-image-1); save to workspace."""
|
||||
|
||||
@@ -62,8 +62,10 @@
|
||||
border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px;
|
||||
font: 15px/1.4 var(--font-family, sans-serif); resize: vertical;
|
||||
}
|
||||
.send-row { display: flex; justify-content: flex-end; }
|
||||
#sendBtn { border: none; border-radius: 8px; padding: 9px 20px; cursor: pointer; background: var(--primary, #0075de); color: #fff; font-size: 14px; }
|
||||
.send-row { display: flex; align-items: stretch; gap: 6px; }
|
||||
.send-row #input { flex: 1 1 auto; }
|
||||
.attach { flex: 0 0 auto; display: inline-flex; align-items: center; justify-content: center; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 0 9px; cursor: pointer; font-size: 16px; color: var(--ink, #111); }
|
||||
#sendBtn { border: none; border-radius: 8px; padding: 0 18px; cursor: pointer; background: var(--primary, #0075de); color: #fff; font-size: 14px; }
|
||||
#toolsPanel { border-top: 1px solid var(--hairline, #e6e6e6); }
|
||||
#toolsToggle {
|
||||
width: 100%; text-align: left; background: none; border: none; cursor: pointer;
|
||||
@@ -72,6 +74,8 @@
|
||||
#toolsArrow { font-size: 11px; }
|
||||
#toolsBody { padding: 0 12px 10px; }
|
||||
#toolsBody.hidden { display: none; }
|
||||
.tool-group { margin-top: 8px; border: 1px solid var(--hairline, #e6e6e6); border-radius: 10px; padding: 6px 10px; background: rgba(0,0,0,.02); }
|
||||
.tool-group-title { font-size: 11px; font-weight: 600; color: #888; text-transform: uppercase; letter-spacing: .4px; margin-bottom: 2px; }
|
||||
.tools-row { display: flex; gap: 8px; flex-wrap: wrap; margin-top: 6px; }
|
||||
.tools-row input[type=text], .tools-row input[type=file] { flex: 2 1 170px; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 6px 10px; font-size: 13px; }
|
||||
.tools-row button { flex: 0 0 auto; border: 1px solid var(--hairline, #e6e6e6); border-radius: 8px; padding: 6px 10px; cursor: pointer; background: #fff; }
|
||||
@@ -113,24 +117,53 @@
|
||||
<div class="chat-col">
|
||||
<main id="log" class="log" aria-live="polite"></main>
|
||||
|
||||
<form id="chat" class="composer" autocomplete="off">
|
||||
<form id="chat" class="composer" autocomplete="off">
|
||||
<div class="send-row">
|
||||
<label class="attach" title="Attach an image to your message">🖼
|
||||
<input type="file" id="imgIn" accept="image/*" style="display:none">
|
||||
</label>
|
||||
<textarea id="input" rows="2" placeholder="Type your message…" autofocus></textarea>
|
||||
<div class="send-row"><button type="submit" id="sendBtn">Send</button></div>
|
||||
</form>
|
||||
<button type="submit" id="sendBtn">Send</button>
|
||||
</div>
|
||||
</form>
|
||||
<div id="toolsPanel">
|
||||
<button type="button" id="toolsToggle"><span id="toolsArrow">▾</span> Model & tools</button>
|
||||
<div id="toolsBody">
|
||||
<div class="modelrow"><label for="modelSel">Model:</label><select id="modelSel"></select></div>
|
||||
<div class="tools-row">
|
||||
<input id="urlInput" type="text" placeholder="Paste a URL to summarise…">
|
||||
<button type="button" id="sumWeb">Summarise page</button>
|
||||
<input id="fileInput" type="text" placeholder="Workspace file (e.g. notes.md)…">
|
||||
<button type="button" id="sumDocs">Summarise file</button>
|
||||
<input id="genInput" type="text" placeholder="Describe an image to generate…">
|
||||
<button type="button" id="genImg">Generate image</button>
|
||||
<input type="file" id="imgIn" accept="image/*" title="Attach an image">
|
||||
<input type="file" id="vidIn" accept="video/*" title="Analyze a video / screen recording (GLM)">
|
||||
<button type="button" id="genVid">Analyze video</button>
|
||||
|
||||
<div class="tool-group">
|
||||
<div class="tool-group-title">Ask with a file</div>
|
||||
<div class="tools-row">
|
||||
<input type="file" id="askIn" accept="audio/*,video/*,image/*,.txt,.md,.json,.log,.csv" title="Any file: audio, video, image, text">
|
||||
<input id="askQ" type="text" placeholder="What should I do? e.g. transcribe this, summarise, what's in this…">
|
||||
<button type="button" id="askBtn">Ask</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-group">
|
||||
<div class="tool-group-title">Transcribe audio → text</div>
|
||||
<div class="tools-row">
|
||||
<input type="file" id="audioIn" accept="audio/*,.mp4,.mov,.m4a" title="Audio or video with speech">
|
||||
<button type="button" id="transcribeBtn">Transcribe</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-group">
|
||||
<div class="tool-group-title">Generate image</div>
|
||||
<div class="tools-row">
|
||||
<input id="genInput" type="text" placeholder="Describe an image to generate…">
|
||||
<button type="button" id="genImg">Generate</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tool-group">
|
||||
<div class="tool-group-title">Workspace file / web page (legacy)</div>
|
||||
<div class="tools-row">
|
||||
<input id="urlInput" type="text" placeholder="Paste a URL to summarise…">
|
||||
<button type="button" id="sumWeb">Summarise page</button>
|
||||
<input id="fileInput" type="text" placeholder="Workspace file (e.g. notes.md)…">
|
||||
<button type="button" id="sumDocs">Summarise file</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -249,20 +282,54 @@
|
||||
}
|
||||
document.getElementById('sumWeb').addEventListener('click', () => runTool('/api/summarize/web', 'urlInput', 'Summarise page', 'url'));
|
||||
document.getElementById('sumDocs').addEventListener('click', () => runTool('/api/summarize/docs', 'fileInput', 'Summarise file', 'path'));
|
||||
document.getElementById('genVid').addEventListener('click', async () => {
|
||||
const f = document.getElementById('vidIn').files[0];
|
||||
|
||||
function downloadLink(path, label) {
|
||||
const a = document.createElement('a'); a.href = '/api/files/' + path; a.download = ''; a.textContent = label;
|
||||
a.style.cssText = 'font-size:12px;color:var(--accent,#6db3f2);text-decoration:underline'; return a;
|
||||
}
|
||||
async function postForm(url, body) {
|
||||
const res = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams(body) });
|
||||
return res.json();
|
||||
}
|
||||
function fail(bubble, data) { bubble.textContent = '[error: ' + (data.error || 'unknown') + ']'; }
|
||||
document.getElementById('askBtn').addEventListener('click', async () => {
|
||||
const f = document.getElementById('askIn').files[0]; const q = document.getElementById('askQ').value.trim();
|
||||
if (!f) return;
|
||||
addBubble('user', 'Analyze video: ' + f.name); addBubble('bot', '');
|
||||
const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…analyzing';
|
||||
addBubble('user', (q ? 'Do: ' + q + ' — ' : '') + f.name); addBubble('bot', '');
|
||||
const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…working on ' + f.name;
|
||||
const fd = new FormData(); fd.append('file', f);
|
||||
let path = null;
|
||||
try {
|
||||
const res = await fetch('/api/tool/video', { method: 'POST', body: fd });
|
||||
const d = await res.json();
|
||||
b.textContent = d.ok ? (d.summary || '') : '[error: ' + (d.error || 'unknown') + ']';
|
||||
const up = await fetch('/api/tool/upload', { method: 'POST', body: fd }); const u = await up.json();
|
||||
path = u.path; if (!u.ok) return fail(b, u);
|
||||
const d = await postForm('/api/tool/askfile', { path, question: q });
|
||||
if (!d.ok) return fail(b, d);
|
||||
b.textContent = d.error ? '' : (d.answer || '');
|
||||
if (d.error) { b.textContent = d.error; return; }
|
||||
if (d.transcript && !d.answer) b.textContent = d.transcript;
|
||||
} catch (err) { b.textContent = '[error: ' + err + ']'; }
|
||||
logEl.scrollTop = logEl.scrollHeight;
|
||||
});
|
||||
document.getElementById('transcribeBtn').addEventListener('click', async () => {
|
||||
const f = document.getElementById('audioIn').files[0]; if (!f) return;
|
||||
addBubble('user', 'Transcribe: ' + f.name); addBubble('bot', '');
|
||||
const b = logEl.lastElementChild.querySelector('.bubble'); b.textContent = '…transcribing (whisper)';
|
||||
const fd = new FormData(); fd.append('file', f);
|
||||
try {
|
||||
const res = await fetch('/api/tool/transcribe', { method: 'POST', body: fd });
|
||||
const d = await res.json();
|
||||
if (!d.ok) return fail(b, d);
|
||||
b.textContent = d.transcript ? d.transcript : (d.note || 'No speech detected.');
|
||||
if (d.transcript) {
|
||||
const dl = document.createElement('div');
|
||||
dl.appendChild(downloadLink(d.txt, '⬇ download transcript (.txt)'));
|
||||
dl.appendChild(document.createTextNode(' · '));
|
||||
dl.appendChild(downloadLink(d.path, '⬇ audio'));
|
||||
b.appendChild(dl);
|
||||
}
|
||||
} catch (err) { b.textContent = '[error: ' + err + ']'; }
|
||||
logEl.scrollTop = logEl.scrollHeight;
|
||||
});
|
||||
|
||||
document.getElementById('genImg').addEventListener('click', async () => {
|
||||
const val = document.getElementById('genInput').value.trim(); if (!val) return;
|
||||
addBubble('user', 'Generate: ' + val); addBubble('bot', '');
|
||||
@@ -270,7 +337,7 @@
|
||||
try {
|
||||
const res = await fetch('/api/tool/image', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ prompt: val }) });
|
||||
const data = await res.json();
|
||||
if (data.ok) { b.textContent = ''; const im = document.createElement('img'); im.src = data.b64; im.style.cssText = 'max-width:320px;border-radius:8px;display:block'; b.appendChild(im); const cap = document.createElement('div'); cap.textContent = 'saved: ' + data.path; b.appendChild(cap); }
|
||||
if (data.ok) { b.textContent = ''; const im = document.createElement('img'); im.src = data.b64; im.style.cssText = 'max-width:320px;border-radius:8px;display:block'; b.appendChild(im); const cap = document.createElement('div'); cap.appendChild(downloadLink(data.path, '⬇ download image')); b.appendChild(cap); }
|
||||
else b.textContent = '[error: ' + (data.error || 'unknown') + ']';
|
||||
} catch (err) { b.textContent = '[error: ' + err + ']'; }
|
||||
logEl.scrollTop = logEl.scrollHeight;
|
||||
|
||||
Reference in New Issue
Block a user