feat: serve melody .bsm binaries over plain HTTP instead of Firebase Storage

ESP32 devices can't spare the 40KB+ RAM a TLS client needs, so Firebase
Storage's HTTPS-only download URLs were blocking melody downloads. Binaries
are now written to local disk (./data/melody_binaries) and served through a
new unauthenticated /api/melodies/download/{pid} route, exposed publicly on
a separate melodies.bellsystems.net vhost (plain HTTP, no TLS) so the main
console domain can stay HTTPS-only with no exceptions. Preview audio still
uses Firebase Storage since it's only ever fetched by the HTTPS admin UI.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

backend/config.py

@@ -33,6 +33,8 @@ class Settings(BaseSettings):
     built_melodies_storage_path: str = "./storage/built_melodies"
     firmware_storage_path: str = "./storage/firmware"
     flash_assets_storage_path: str = "./storage/flash_assets"
+    melody_binaries_storage_path: str = "./storage/melody_binaries"
+    melody_download_base_url: str = "http://melodies.bellsystems.net/download"
 
     # Email (Resend)
     resend_api_key: str = "re_placeholder_change_me"

backend/melodies/router.py

@@ -1,4 +1,5 @@
 from fastapi import APIRouter, Depends, UploadFile, File, Query, HTTPException, Response
+from fastapi.responses import FileResponse
 from typing import Optional
 from sqlalchemy.ext.asyncio import AsyncSession
 from auth.models import TokenPayload
@@ -13,6 +14,22 @@ from shared.audit import log_action
 router = APIRouter(prefix="/api/melodies", tags=["melodies"])
 
 
+@router.get("/download/{pid}")
+async def download_binary_by_pid(pid: str):
+    """Download a melody's .bsm binary by PID over plain HTTP.
+
+    No auth — ESP32 devices call this directly and can't afford a TLS client's
+    memory footprint, so this is served from a plain-HTTP-only host
+    (melodies.bellsystems.net) rather than the HTTPS console domain.
+    """
+    path = await service.get_binary_path_by_pid(pid)
+    return FileResponse(
+        path=str(path),
+        media_type="application/octet-stream",
+        filename=f"{pid}.bsm",
+    )
+
+
 @router.get("", response_model=MelodyListResponse)
 async def list_melodies(
     search: Optional[str] = Query(None),
@@ -134,15 +151,20 @@ async def upload_file(
 
     if file_type == "binary":
         content_type = "application/octet-stream"
-
-    url = service.upload_file_for_melody(
-        melody_id=melody_id,
-        melody_uid=melody.uid,
-        melody_pid=melody.pid,
-        file_bytes=contents,
-        filename=file.filename,
-        content_type=content_type,
-    )
+        url = service.save_binary_for_melody(
+            melody_uid=melody.uid,
+            pid=melody.pid,
+            file_bytes=contents,
+        )
+    else:
+        url = service.upload_file_for_melody(
+            melody_id=melody_id,
+            melody_uid=melody.uid,
+            melody_pid=melody.pid,
+            file_bytes=contents,
+            filename=file.filename,
+            content_type=content_type,
+        )
 
     # Update the melody document with the file URL
     if file_type == "preview":
@@ -179,7 +201,7 @@ async def get_files(
 ):
     """Get storage file URLs for a melody."""
     melody = await service.get_melody(melody_id)
-    return service.get_storage_files(melody_id, melody.uid)
+    return service.get_storage_files(melody_id, melody.uid, melody.pid)
 
 
 @router.patch("/{melody_id}/set-outdated", response_model=MelodyInDB)

backend/melodies/service.py

@@ -2,15 +2,31 @@ import json
 import uuid
 import logging
 from datetime import datetime
+from pathlib import Path
+
+from fastapi import HTTPException
 
 from shared.firebase import get_db as get_firestore, get_bucket
 from shared.exceptions import NotFoundError
 from melodies.models import MelodyCreate, MelodyUpdate, MelodyInDB
 from melodies import database as melody_db
+from config import settings
 
 COLLECTION = "melodies"
 logger = logging.getLogger("melodies.service")
 
+# Local disk storage for melody .bsm binaries — served over plain HTTP for ESP32 compatibility.
+# The audio preview file still goes to Firebase Storage (only ever fetched by the browser admin UI).
+BINARY_STORAGE_DIR = Path(settings.melody_binaries_storage_path)
+
+
+def _ensure_binary_storage_dir():
+    BINARY_STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def _binary_file_path(melody_uid: str) -> Path:
+    return BINARY_STORAGE_DIR / f"{melody_uid}.bsm"
+
 
 def _parse_localized_string(value: str) -> dict:
     """Parse a JSON-encoded localized string into a dict. Returns {} on failure."""
@@ -127,8 +143,25 @@ def _sanitize_metadata_for_update(existing: dict | None, incoming: dict | None,
     return merged
 
 
+async def _check_pid_unique(pid: str, exclude_melody_id: str | None = None) -> None:
+    """Raise 409 if another melody already uses this PID.
+
+    PIDs are the public lookup key for the /melodies/download/{pid} device-facing
+    route, so two melodies sharing a PID would make that route resolve ambiguously.
+    """
+    if not pid:
+        return
+    rows = await melody_db.list_melodies()
+    for row in rows:
+        if exclude_melody_id and row["id"] == exclude_melody_id:
+            continue
+        if row["data"].get("pid") == pid:
+            raise HTTPException(status_code=409, detail=f"A melody with PID '{pid}' already exists.")
+
+
 async def create_melody(data: MelodyCreate, publish: bool = False, actor_name: str | None = None) -> MelodyInDB:
     """Create a new melody. If publish=True, also push to Firestore."""
+    await _check_pid_unique(data.pid)
     melody_id = str(uuid.uuid4())
     doc_data = data.model_dump()
     doc_data["metadata"] = _sanitize_metadata_for_create(doc_data.get("metadata"), actor_name)
@@ -151,6 +184,9 @@ async def update_melody(melody_id: str, data: MelodyUpdate, actor_name: str | No
     if not row:
         raise NotFoundError("Melody")
 
+    if data.pid is not None and data.pid != row["data"].get("pid"):
+        await _check_pid_unique(data.pid, exclude_melody_id=melody_id)
+
     existing_data = row["data"]
     update_data = data.model_dump(exclude_none=True)
 
@@ -238,6 +274,41 @@ async def delete_melody(melody_id: str) -> None:
     await melody_db.delete_melody(melody_id)
 
 
+def save_binary_for_melody(melody_uid: str, pid: str, file_bytes: bytes) -> str:
+    """Save a melody's .bsm binary to local disk, replacing any previous binary for this melody.
+
+    Returns the plain-HTTP download URL devices use (served from melody_download_base_url,
+    not Firebase — ESP32 devices can't afford the RAM for a TLS client).
+    """
+    _ensure_binary_storage_dir()
+    path = _binary_file_path(melody_uid)
+    path.write_bytes(file_bytes)
+    return f"{settings.melody_download_base_url}/{pid}"
+
+
+async def get_binary_path_by_pid(pid: str) -> Path:
+    """Resolve a melody's PID to its local .bsm file path. Used by the unauthenticated
+    device-facing download route."""
+    rows = await melody_db.list_melodies()
+    for row in rows:
+        if row["data"].get("pid") == pid:
+            melody_uid = row["data"].get("uid")
+            path = _binary_file_path(melody_uid)
+            if path.exists():
+                return path
+            raise NotFoundError("Binary file")
+    raise NotFoundError("Melody")
+
+
+def delete_local_binary(melody_uid: str) -> None:
+    """Delete the local .bsm binary file for a melody, if present."""
+    if not melody_uid:
+        return
+    path = _binary_file_path(melody_uid)
+    if path.exists():
+        path.unlink()
+
+
 def upload_file(melody_id: str, file_bytes: bytes, filename: str, content_type: str) -> str:
     """Upload a file to Firebase Storage under melodies/{melody_id}/."""
     bucket = get_bucket()
@@ -335,31 +406,19 @@ def upload_file_for_melody(melody_id: str, melody_uid: str | None, melody_pid: s
 
 
 def get_binary_file_bytes(melody_id: str, melody_uid: str | None = None) -> tuple[bytes, str]:
-    """Fetch current binary bytes for a melody from Firebase Storage."""
-    bucket = get_bucket()
-    if not bucket:
-        raise RuntimeError("Firebase Storage not initialized")
-
-    prefixes = _storage_prefixes(melody_id, melody_uid)
-    blobs = [b for b in _list_blobs_for_prefixes(bucket, prefixes) if _is_binary_blob_name(b.name)]
-    if not blobs:
+    """Fetch current binary bytes for a melody from local disk storage."""
+    path = _binary_file_path(melody_uid) if melody_uid else None
+    if not path or not path.exists():
         raise NotFoundError("Binary file")
-
-    # Prefer explicit binary.* naming, then newest.
-    blobs.sort(
-        key=lambda b: (
-            0 if "binary" in b.name.rsplit("/", 1)[-1].lower() else 1,
-            -(int(b.time_created.timestamp()) if getattr(b, "time_created", None) else 0),
-        )
-    )
-    chosen = blobs[0]
-    data = chosen.download_as_bytes()
-    content_type = chosen.content_type or "application/octet-stream"
-    return data, content_type
+    return path.read_bytes(), "application/octet-stream"
 
 
 def delete_file(melody_id: str, file_type: str, melody_uid: str | None = None) -> None:
     """Delete a specific file from storage. file_type is 'binary' or 'preview'."""
+    if file_type == "binary":
+        delete_local_binary(melody_uid)
+        return
+
     bucket = get_bucket()
     if not bucket:
         return
@@ -368,14 +427,14 @@ def delete_file(melody_id: str, file_type: str, melody_uid: str | None = None) -
     blobs = _list_blobs_for_prefixes(bucket, prefixes)
 
     for blob in blobs:
-        if file_type == "binary" and "binary" in blob.name:
-            blob.delete()
-        elif file_type == "preview" and "preview" in blob.name:
+        if file_type == "preview" and "preview" in blob.name:
             blob.delete()
 
 
 def _delete_storage_files(melody_id: str, melody_uid: str | None = None) -> None:
-    """Delete all storage files for a melody."""
+    """Delete all storage files for a melody (local binary + Firebase preview)."""
+    delete_local_binary(melody_uid)
+
     bucket = get_bucket()
     if not bucket:
         return
@@ -383,24 +442,29 @@ def _delete_storage_files(melody_id: str, melody_uid: str | None = None) -> None
     prefixes = _storage_prefixes(melody_id, melody_uid)
     blobs = _list_blobs_for_prefixes(bucket, prefixes)
     for blob in blobs:
+        if _is_binary_blob_name(blob.name):
+            continue  # legacy Firebase binaries, if any, are no longer authoritative
         blob.delete()
 
 
-def get_storage_files(melody_id: str, melody_uid: str | None = None) -> dict:
-    """List storage files for a melody, returning URLs."""
+def get_storage_files(melody_id: str, melody_uid: str | None = None, melody_pid: str | None = None) -> dict:
+    """List storage files for a melody, returning URLs. Binary comes from local disk,
+    preview still comes from Firebase Storage."""
+    result = {"binary_url": None, "preview_url": None}
+
+    if melody_uid and melody_pid and _binary_file_path(melody_uid).exists():
+        result["binary_url"] = f"{settings.melody_download_base_url}/{melody_pid}"
+
     bucket = get_bucket()
     if not bucket:
-        return {"binary_url": None, "preview_url": None}
+        return result
 
     prefixes = _storage_prefixes(melody_id, melody_uid)
     blobs = _list_blobs_for_prefixes(bucket, prefixes)
 
-    result = {"binary_url": None, "preview_url": None}
     for blob in blobs:
-        blob.make_public()
-        if _is_binary_blob_name(blob.name):
-            result["binary_url"] = blob.public_url
-        elif "preview" in blob.name:
+        if "preview" in blob.name:
+            blob.make_public()
             result["preview_url"] = blob.public_url
 
     return result

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - ./data/built_melodies:/app/storage/built_melodies
       - ./data/firmware:/app/storage/firmware
       - ./data/flash_assets:/app/storage/flash_assets
+      - ./data/melody_binaries:/app/storage/melody_binaries
       - ./data/firebase-service-account.json:/app/firebase-service-account.json:ro
     ports:
       - "8000:8000"

docs/melody-binary-serving.md

@@ -0,0 +1,71 @@
+# Melody Binary Serving (Plain HTTP)
+
+## Why this exists
+
+ESP32 devices download `.bsm` melody files directly, using a URL supplied through the
+Android app. The download previously pointed at a Firebase Storage public URL (HTTPS
+only). ESP32's TLS client needs 40KB+ of RAM to open an HTTPS connection, which these
+devices can't reliably spare. To fix this, melody `.bsm` binaries are now stored and
+served by our own backend over plain HTTP, instead of Firebase Storage.
+
+This does **not** affect the audio preview file (`information.previewURL`) — that's
+only ever fetched by the browser admin UI, which is already HTTPS, so it stays on
+Firebase Storage unchanged.
+
+## What changed
+
+- **Storage**: `.bsm` binaries are written to `./data/melody_binaries/{melody_uid}.bsm`
+  on the host (mounted into the backend container at `/app/storage/melody_binaries`,
+  same pattern as `./data/firmware` and `./data/built_melodies`).
+- **Upload path**: `SelectArchetypeModal` / `BuildOnTheFlyModal` still call
+  `POST /api/melodies/{melodyId}/upload/binary` exactly as before. The backend
+  (`backend/melodies/service.py::save_binary_for_melody`) now writes the bytes to
+  local disk instead of uploading to Firebase Storage, and returns a URL like
+  `http://melodies.bellsystems.net/download/{pid}`.
+- **Stored URL**: the melody's `url` field (Firestore + SQLite) now holds that
+  plain-HTTP URL instead of a Firebase public URL. No schema change — `url` was
+  already a plain string field.
+- **Download route**: `GET /api/melodies/download/{pid}` (`backend/melodies/router.py`)
+  is unauthenticated (devices have no login token) and resolves `pid` → the melody's
+  `{uid}.bsm` file on disk, same pattern as the existing firmware download route
+  (`backend/firmware/router.py::download_firmware`).
+- **PID uniqueness**: since `pid` is now the public lookup key for the download route,
+  `backend/melodies/service.py::_check_pid_unique` rejects creating/renaming a melody
+  to a PID already used by another melody (409 Conflict).
+- **Cleanup**: deleting a melody or its binary file now also deletes the local
+  `.bsm` file (`service.delete_local_binary`).
+
+## Infrastructure (outside this repo)
+
+The plain-HTTP requirement is handled by keeping `melodies.bellsystems.net` on a
+**separate** hostname from `console.bellsystems.net`, so the console's NPM proxy host
+can stay HTTPS-only with no per-path exceptions.
+
+Required, one-time setup outside this repository:
+
+1. **DNS**: add an A/CNAME record for `melodies.bellsystems.net` pointing at the same
+   host as `console.bellsystems.net`.
+2. **NPM (Nginx Proxy Manager) proxy host**: create a new proxy host for
+   `melodies.bellsystems.net` forwarding to the `nginx` container's exposed port
+   (`90` per `docker-compose.yml`). **Do not force SSL / do not enable the "Force
+   SSL" redirect** on this proxy host — it must remain reachable over plain HTTP.
+
+The in-repo `nginx/nginx.conf` already has a dedicated `server_name
+melodies.bellsystems.net` block that maps `/download/{pid}` to the backend's
+`/api/melodies/download/{pid}` route, so no further nginx changes are needed once
+the NPM proxy host exists.
+
+## Verification
+
+1. Build/select an archetype for a melody → confirm a file appears at
+   `./data/melody_binaries/{uid}.bsm` and the melody's `url` becomes
+   `http://melodies.bellsystems.net/download/{pid}`.
+2. `curl http://localhost:8000/api/melodies/download/{pid}` (direct to backend,
+   bypassing nginx) → returns the `.bsm` bytes, no auth header needed.
+3. `curl http://localhost:90/api/melodies/download/{pid}` (through nginx on the
+   console server block) → same result.
+4. Once NPM is configured, `curl http://melodies.bellsystems.net/download/{pid}`
+   → same result, over plain HTTP.
+5. Delete the melody → confirm the local `.bsm` file is removed.
+6. Upload a preview audio file → confirm it still lands in Firebase Storage and
+   `previewURL` still populates (regression check).

nginx/nginx.conf

@@ -54,4 +54,25 @@ http {
             proxy_set_header Connection "upgrade";
         }
     }
+
+    # Plain-HTTP-only host for ESP32 melody downloads. Kept separate from the
+    # console.bellsystems.net server block above so console can stay HTTPS-only
+    # in NPM with no path-based TLS exceptions. ESP32 devices can't afford the
+    # RAM for a TLS client, so this host must never be forced to HTTPS upstream.
+    server {
+        listen 80;
+        server_name melodies.bellsystems.net;
+
+        resolver 127.0.0.11 valid=5s;
+        set $backend_upstream http://backend:8000;
+
+        location /download/ {
+            rewrite ^/download/(.*)$ /api/melodies/download/$1 break;
+            proxy_pass $backend_upstream$uri;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
 }