"""Background workers (Celery + Redis). Handles async work that shouldn't block the web request: S3 bucket operations, backup notifications, and (future) media processing. Run with: celery -A portal.tasks.celery_app worker --loglevel=info """ from __future__ import annotations import os import logging from celery import Celery from .config import settings from .s3 import s3_bucket_name, s3_client logger = logging.getLogger(__name__) celery_app = Celery( "family_home_lab", broker=settings.CELERY_BROKER_URL, backend=settings.REDIS_URL, include=["portal.tasks"], ) celery_app.conf.update( task_serializer="json", result_serializer="json", accept_content=["json"], timezone="UTC", enable_utc=True, broker_connection_retry_on_startup=True, ) _VIDEO_EXT = {".mp4", ".mov", ".webm", ".mkv", ".m4v", ".avi", ".mpg", ".mpeg"} def _audio_wav(path: str, suffix: str) -> str: """If the uploaded file is a video, extract its audio track as a 16k mono WAV.""" if suffix.lower() in _VIDEO_EXT: import subprocess out = os.path.splitext(path)[0] + ".wav" subprocess.run( ["ffmpeg", "-v", "error", "-y", "-i", path, "-ac", "1", "-ar", "16000", out], check=True, ) return out return path @celery_app.task(bind=True, max_retries=3, default_retry_delay=30) def ensure_user_bucket(self, username: str) -> dict: """Create (idempotently) the per-user S3 bucket in Garage.""" try: client = s3_client() for bucket in ("shared-media", s3_bucket_name(username)): try: client.head_bucket(Bucket=bucket) logger.info("Bucket %s exists", bucket) except Exception: client.create_bucket(Bucket=bucket) logger.info("Created bucket %s", bucket) return {"ok": True, "buckets": ["shared-media", s3_bucket_name(username)]} except Exception as exc: # noqa: BLE001 logger.exception("ensure_user_bucket failed") raise self.retry(exc=exc) from exc @celery_app.task(bind=True, queue="transcription-mus", max_retries=1, default_retry_delay=120) def transcribe_sheetmusic(self, bucket: str, key: str, user: str = "") -> dict: """Transcribe audio to MIDI + MusicXML + engraved PDF / tabs (Kyutai MuScriptor). Runs on the dedicated `transcriber-mus` worker. Requires HUGGINGFACE_TOKEN (the CC BY-NC model weights are gated) and is slow on CPU. Uses the muscriptor CLI: muscriptor transcribe