Transcriber: accept video uploads (mp4/mov/webm/mkv) -> ffmpeg-extract audio -> MIDI; ffmpeg in workers

portal/main.py

@@ -528,6 +528,8 @@ async def admin_rename_user(
 
 
 _AUDIO_EXT = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aiff", ".aif", ".opus"}
+_VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".m4v", ".avi", ".mpg", ".mpeg"}
+_TRANS_EXT = _AUDIO_EXT | _VIDEO_EXT
 
 
 @app.get("/transcriber", response_class=HTMLResponse)
@@ -538,7 +540,7 @@ async def transcriber_page(request: Request) -> HTMLResponse:
     bucket = s3_bucket_name(user.username)
     all_keys = await run_in_threadpool(list_prefix, bucket, "", 400)
     audio_files = [k for k in all_keys
-                   if k.lower().endswith(tuple(_AUDIO_EXT))]
+                   if k.lower().endswith(tuple(_TRANS_EXT))]
     transcriptions = [k for k in all_keys
                       if k.startswith("transcriptions/") or k.startswith("sheetmusic/")]
     return templates.TemplateResponse(

portal/tasks.py

@@ -7,6 +7,7 @@ Run with:
     celery -A portal.tasks.celery_app worker --loglevel=info
 """
 from __future__ import annotations
+import os
 
 import logging
 
@@ -34,6 +35,22 @@ celery_app.conf.update(
 )
 
 
+_VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".m4v", ".avi", ".mpg", ".mpeg"}
+
+
+def _audio_wav(path: str, suffix: str) -> str:
+    """If the uploaded file is a video, extract its audio track as a 16k mono WAV."""
+    if suffix.lower() in _VIDEO_EXT:
+        import subprocess
+        out = os.path.splitext(path)[0] + ".wav"
+        subprocess.run(
+            ["ffmpeg", "-v", "error", "-y", "-i", path, "-ac", "1", "-ar", "16000", out],
+            check=True,
+        )
+        return out
+    return path
+
+
 @celery_app.task(bind=True, max_retries=3, default_retry_delay=30)
 def ensure_user_bucket(self, username: str) -> dict:
     """Create (idempotently) the per-user S3 bucket in Garage."""
@@ -69,6 +86,7 @@ def transcribe_sheetmusic(self, bucket: str, key: str, user: str = "") -> dict:
         base = pathlib.Path(key).stem or "score"
         audio_path = os.path.join(workdir, base + pathlib.Path(key).suffix)
         client.download_file(bucket, key, audio_path)
+        audio_path = _audio_wav(audio_path, pathlib.Path(key).suffix)
 
         if not os.environ.get("HUGGINGFACE_TOKEN"):
             return {"ok": False, "error": "HUGGINGFACE_TOKEN not set (CC BY-NC model is gated)"}
@@ -119,6 +137,7 @@ def transcribe_audio(self, bucket: str, key: str, user: str = "") -> dict:
         base = pathlib.Path(key).stem or "input"
         audio_path = os.path.join(workdir, base + pathlib.Path(key).suffix)
         client.download_file(bucket, key, audio_path)
+        audio_path = _audio_wav(audio_path, pathlib.Path(key).suffix)
 
         outdir = os.path.join(workdir, "out")
         os.makedirs(outdir, exist_ok=True)
@@ -168,4 +187,4 @@ def transcribe_audio(self, bucket: str, key: str, user: str = "") -> dict:
 def notify_backup(scope: str = "family-home-lab") -> dict:
     """Hook point for backup/health notifications (currently a no-op stub)."""
     logger.info("Backup notify stub fired for %s", scope)
-    return {"ok": True, "scope": scope}
+    return {"ok": True, "scope": scope}

portal/templates/transcriber.html

@@ -41,7 +41,7 @@
       <h2 class="section-title">Upload audio</h2>
       <form method="post" action="/transcriber/upload" enctype="multipart/form-data">
         <div class="form-field">
-          <input class="text-input" type="file" name="file" accept="audio/*,.wav,.mp3,.flac,.m4a" required>
+          <input class="text-input" type="file" name="file" accept="audio/*,.wav,.mp3,.flac,.m4a,.mp4,.mov,.webm,.mkv" required>
         </div>
         <button class="button button-primary width-full" type="submit">Upload & transcribe</button>
       </form>

transcriber-mus/Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /app
 
 # torch/transformers need libgomp; MuseScore does the engraving for --format sheets
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates libgomp1 musescore3 \
+    && apt-get install -y --no-install-recommends ca-certificates libgomp1 musescore3 ffmpeg \
     && ln -sf /usr/bin/musescore3 /usr/local/bin/musescore \
     && rm -rf /var/lib/apt/lists/*
 

transcriber/Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /app
 
 # tensorflow needs libgomp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates libgomp1 \
+    && apt-get install -y --no-install-recommends ca-certificates libgomp1 ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 COPY portal/requirements.txt /tmp/req.txt