From 72f7e00990bacc51bc22897f7e3a0dd73b5a13e7 Mon Sep 17 00:00:00 2001 From: bonamin Date: Mon, 6 Jul 2026 09:41:24 +0300 Subject: [PATCH] feat: serve melody .bsm binaries over plain HTTP instead of Firebase Storage ESP32 devices can't spare the 40KB+ RAM a TLS client needs, so Firebase Storage's HTTPS-only download URLs were blocking melody downloads. Binaries are now written to local disk (./data/melody_binaries) and served through a new unauthenticated /api/melodies/download/{pid} route, exposed publicly on a separate melodies.bellsystems.net vhost (plain HTTP, no TLS) so the main console domain can stay HTTPS-only with no exceptions. Preview audio still uses Firebase Storage since it's only ever fetched by the HTTPS admin UI. Co-Authored-By: Claude Sonnet 5 --- backend/config.py | 2 + backend/melodies/router.py | 42 ++++++++--- backend/melodies/service.py | 128 +++++++++++++++++++++++++--------- docker-compose.yml | 1 + docs/melody-binary-serving.md | 71 +++++++++++++++++++ nginx/nginx.conf | 21 ++++++ 6 files changed, 223 insertions(+), 42 deletions(-) create mode 100644 docs/melody-binary-serving.md diff --git a/backend/config.py b/backend/config.py index eb2fc6c..6cfe866 100644 --- a/backend/config.py +++ b/backend/config.py @@ -33,6 +33,8 @@ class Settings(BaseSettings): built_melodies_storage_path: str = "./storage/built_melodies" firmware_storage_path: str = "./storage/firmware" flash_assets_storage_path: str = "./storage/flash_assets" + melody_binaries_storage_path: str = "./storage/melody_binaries" + melody_download_base_url: str = "http://melodies.bellsystems.net/download" # Email (Resend) resend_api_key: str = "re_placeholder_change_me" diff --git a/backend/melodies/router.py b/backend/melodies/router.py index 085cfd2..0b65aa8 100644 --- a/backend/melodies/router.py +++ b/backend/melodies/router.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, Depends, UploadFile, File, Query, HTTPException, Response +from fastapi.responses import FileResponse from typing import Optional from sqlalchemy.ext.asyncio import AsyncSession from auth.models import TokenPayload @@ -13,6 +14,22 @@ from shared.audit import log_action router = APIRouter(prefix="/api/melodies", tags=["melodies"]) +@router.get("/download/{pid}") +async def download_binary_by_pid(pid: str): + """Download a melody's .bsm binary by PID over plain HTTP. + + No auth — ESP32 devices call this directly and can't afford a TLS client's + memory footprint, so this is served from a plain-HTTP-only host + (melodies.bellsystems.net) rather than the HTTPS console domain. + """ + path = await service.get_binary_path_by_pid(pid) + return FileResponse( + path=str(path), + media_type="application/octet-stream", + filename=f"{pid}.bsm", + ) + + @router.get("", response_model=MelodyListResponse) async def list_melodies( search: Optional[str] = Query(None), @@ -134,15 +151,20 @@ async def upload_file( if file_type == "binary": content_type = "application/octet-stream" - - url = service.upload_file_for_melody( - melody_id=melody_id, - melody_uid=melody.uid, - melody_pid=melody.pid, - file_bytes=contents, - filename=file.filename, - content_type=content_type, - ) + url = service.save_binary_for_melody( + melody_uid=melody.uid, + pid=melody.pid, + file_bytes=contents, + ) + else: + url = service.upload_file_for_melody( + melody_id=melody_id, + melody_uid=melody.uid, + melody_pid=melody.pid, + file_bytes=contents, + filename=file.filename, + content_type=content_type, + ) # Update the melody document with the file URL if file_type == "preview": @@ -179,7 +201,7 @@ async def get_files( ): """Get storage file URLs for a melody.""" melody = await service.get_melody(melody_id) - return service.get_storage_files(melody_id, melody.uid) + return service.get_storage_files(melody_id, melody.uid, melody.pid) @router.patch("/{melody_id}/set-outdated", response_model=MelodyInDB) diff --git a/backend/melodies/service.py b/backend/melodies/service.py index 5f51f6d..543d539 100644 --- a/backend/melodies/service.py +++ b/backend/melodies/service.py @@ -2,15 +2,31 @@ import json import uuid import logging from datetime import datetime +from pathlib import Path + +from fastapi import HTTPException from shared.firebase import get_db as get_firestore, get_bucket from shared.exceptions import NotFoundError from melodies.models import MelodyCreate, MelodyUpdate, MelodyInDB from melodies import database as melody_db +from config import settings COLLECTION = "melodies" logger = logging.getLogger("melodies.service") +# Local disk storage for melody .bsm binaries — served over plain HTTP for ESP32 compatibility. +# The audio preview file still goes to Firebase Storage (only ever fetched by the browser admin UI). +BINARY_STORAGE_DIR = Path(settings.melody_binaries_storage_path) + + +def _ensure_binary_storage_dir(): + BINARY_STORAGE_DIR.mkdir(parents=True, exist_ok=True) + + +def _binary_file_path(melody_uid: str) -> Path: + return BINARY_STORAGE_DIR / f"{melody_uid}.bsm" + def _parse_localized_string(value: str) -> dict: """Parse a JSON-encoded localized string into a dict. Returns {} on failure.""" @@ -127,8 +143,25 @@ def _sanitize_metadata_for_update(existing: dict | None, incoming: dict | None, return merged +async def _check_pid_unique(pid: str, exclude_melody_id: str | None = None) -> None: + """Raise 409 if another melody already uses this PID. + + PIDs are the public lookup key for the /melodies/download/{pid} device-facing + route, so two melodies sharing a PID would make that route resolve ambiguously. + """ + if not pid: + return + rows = await melody_db.list_melodies() + for row in rows: + if exclude_melody_id and row["id"] == exclude_melody_id: + continue + if row["data"].get("pid") == pid: + raise HTTPException(status_code=409, detail=f"A melody with PID '{pid}' already exists.") + + async def create_melody(data: MelodyCreate, publish: bool = False, actor_name: str | None = None) -> MelodyInDB: """Create a new melody. If publish=True, also push to Firestore.""" + await _check_pid_unique(data.pid) melody_id = str(uuid.uuid4()) doc_data = data.model_dump() doc_data["metadata"] = _sanitize_metadata_for_create(doc_data.get("metadata"), actor_name) @@ -151,6 +184,9 @@ async def update_melody(melody_id: str, data: MelodyUpdate, actor_name: str | No if not row: raise NotFoundError("Melody") + if data.pid is not None and data.pid != row["data"].get("pid"): + await _check_pid_unique(data.pid, exclude_melody_id=melody_id) + existing_data = row["data"] update_data = data.model_dump(exclude_none=True) @@ -238,6 +274,41 @@ async def delete_melody(melody_id: str) -> None: await melody_db.delete_melody(melody_id) +def save_binary_for_melody(melody_uid: str, pid: str, file_bytes: bytes) -> str: + """Save a melody's .bsm binary to local disk, replacing any previous binary for this melody. + + Returns the plain-HTTP download URL devices use (served from melody_download_base_url, + not Firebase — ESP32 devices can't afford the RAM for a TLS client). + """ + _ensure_binary_storage_dir() + path = _binary_file_path(melody_uid) + path.write_bytes(file_bytes) + return f"{settings.melody_download_base_url}/{pid}" + + +async def get_binary_path_by_pid(pid: str) -> Path: + """Resolve a melody's PID to its local .bsm file path. Used by the unauthenticated + device-facing download route.""" + rows = await melody_db.list_melodies() + for row in rows: + if row["data"].get("pid") == pid: + melody_uid = row["data"].get("uid") + path = _binary_file_path(melody_uid) + if path.exists(): + return path + raise NotFoundError("Binary file") + raise NotFoundError("Melody") + + +def delete_local_binary(melody_uid: str) -> None: + """Delete the local .bsm binary file for a melody, if present.""" + if not melody_uid: + return + path = _binary_file_path(melody_uid) + if path.exists(): + path.unlink() + + def upload_file(melody_id: str, file_bytes: bytes, filename: str, content_type: str) -> str: """Upload a file to Firebase Storage under melodies/{melody_id}/.""" bucket = get_bucket() @@ -335,31 +406,19 @@ def upload_file_for_melody(melody_id: str, melody_uid: str | None, melody_pid: s def get_binary_file_bytes(melody_id: str, melody_uid: str | None = None) -> tuple[bytes, str]: - """Fetch current binary bytes for a melody from Firebase Storage.""" - bucket = get_bucket() - if not bucket: - raise RuntimeError("Firebase Storage not initialized") - - prefixes = _storage_prefixes(melody_id, melody_uid) - blobs = [b for b in _list_blobs_for_prefixes(bucket, prefixes) if _is_binary_blob_name(b.name)] - if not blobs: + """Fetch current binary bytes for a melody from local disk storage.""" + path = _binary_file_path(melody_uid) if melody_uid else None + if not path or not path.exists(): raise NotFoundError("Binary file") - - # Prefer explicit binary.* naming, then newest. - blobs.sort( - key=lambda b: ( - 0 if "binary" in b.name.rsplit("/", 1)[-1].lower() else 1, - -(int(b.time_created.timestamp()) if getattr(b, "time_created", None) else 0), - ) - ) - chosen = blobs[0] - data = chosen.download_as_bytes() - content_type = chosen.content_type or "application/octet-stream" - return data, content_type + return path.read_bytes(), "application/octet-stream" def delete_file(melody_id: str, file_type: str, melody_uid: str | None = None) -> None: """Delete a specific file from storage. file_type is 'binary' or 'preview'.""" + if file_type == "binary": + delete_local_binary(melody_uid) + return + bucket = get_bucket() if not bucket: return @@ -368,14 +427,14 @@ def delete_file(melody_id: str, file_type: str, melody_uid: str | None = None) - blobs = _list_blobs_for_prefixes(bucket, prefixes) for blob in blobs: - if file_type == "binary" and "binary" in blob.name: - blob.delete() - elif file_type == "preview" and "preview" in blob.name: + if file_type == "preview" and "preview" in blob.name: blob.delete() def _delete_storage_files(melody_id: str, melody_uid: str | None = None) -> None: - """Delete all storage files for a melody.""" + """Delete all storage files for a melody (local binary + Firebase preview).""" + delete_local_binary(melody_uid) + bucket = get_bucket() if not bucket: return @@ -383,24 +442,29 @@ def _delete_storage_files(melody_id: str, melody_uid: str | None = None) -> None prefixes = _storage_prefixes(melody_id, melody_uid) blobs = _list_blobs_for_prefixes(bucket, prefixes) for blob in blobs: + if _is_binary_blob_name(blob.name): + continue # legacy Firebase binaries, if any, are no longer authoritative blob.delete() -def get_storage_files(melody_id: str, melody_uid: str | None = None) -> dict: - """List storage files for a melody, returning URLs.""" +def get_storage_files(melody_id: str, melody_uid: str | None = None, melody_pid: str | None = None) -> dict: + """List storage files for a melody, returning URLs. Binary comes from local disk, + preview still comes from Firebase Storage.""" + result = {"binary_url": None, "preview_url": None} + + if melody_uid and melody_pid and _binary_file_path(melody_uid).exists(): + result["binary_url"] = f"{settings.melody_download_base_url}/{melody_pid}" + bucket = get_bucket() if not bucket: - return {"binary_url": None, "preview_url": None} + return result prefixes = _storage_prefixes(melody_id, melody_uid) blobs = _list_blobs_for_prefixes(bucket, prefixes) - result = {"binary_url": None, "preview_url": None} for blob in blobs: - blob.make_public() - if _is_binary_blob_name(blob.name): - result["binary_url"] = blob.public_url - elif "preview" in blob.name: + if "preview" in blob.name: + blob.make_public() result["preview_url"] = blob.public_url return result diff --git a/docker-compose.yml b/docker-compose.yml index cd2da27..6f75280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - ./data/built_melodies:/app/storage/built_melodies - ./data/firmware:/app/storage/firmware - ./data/flash_assets:/app/storage/flash_assets + - ./data/melody_binaries:/app/storage/melody_binaries - ./data/firebase-service-account.json:/app/firebase-service-account.json:ro ports: - "8000:8000" diff --git a/docs/melody-binary-serving.md b/docs/melody-binary-serving.md new file mode 100644 index 0000000..f54160c --- /dev/null +++ b/docs/melody-binary-serving.md @@ -0,0 +1,71 @@ +# Melody Binary Serving (Plain HTTP) + +## Why this exists + +ESP32 devices download `.bsm` melody files directly, using a URL supplied through the +Android app. The download previously pointed at a Firebase Storage public URL (HTTPS +only). ESP32's TLS client needs 40KB+ of RAM to open an HTTPS connection, which these +devices can't reliably spare. To fix this, melody `.bsm` binaries are now stored and +served by our own backend over plain HTTP, instead of Firebase Storage. + +This does **not** affect the audio preview file (`information.previewURL`) — that's +only ever fetched by the browser admin UI, which is already HTTPS, so it stays on +Firebase Storage unchanged. + +## What changed + +- **Storage**: `.bsm` binaries are written to `./data/melody_binaries/{melody_uid}.bsm` + on the host (mounted into the backend container at `/app/storage/melody_binaries`, + same pattern as `./data/firmware` and `./data/built_melodies`). +- **Upload path**: `SelectArchetypeModal` / `BuildOnTheFlyModal` still call + `POST /api/melodies/{melodyId}/upload/binary` exactly as before. The backend + (`backend/melodies/service.py::save_binary_for_melody`) now writes the bytes to + local disk instead of uploading to Firebase Storage, and returns a URL like + `http://melodies.bellsystems.net/download/{pid}`. +- **Stored URL**: the melody's `url` field (Firestore + SQLite) now holds that + plain-HTTP URL instead of a Firebase public URL. No schema change — `url` was + already a plain string field. +- **Download route**: `GET /api/melodies/download/{pid}` (`backend/melodies/router.py`) + is unauthenticated (devices have no login token) and resolves `pid` → the melody's + `{uid}.bsm` file on disk, same pattern as the existing firmware download route + (`backend/firmware/router.py::download_firmware`). +- **PID uniqueness**: since `pid` is now the public lookup key for the download route, + `backend/melodies/service.py::_check_pid_unique` rejects creating/renaming a melody + to a PID already used by another melody (409 Conflict). +- **Cleanup**: deleting a melody or its binary file now also deletes the local + `.bsm` file (`service.delete_local_binary`). + +## Infrastructure (outside this repo) + +The plain-HTTP requirement is handled by keeping `melodies.bellsystems.net` on a +**separate** hostname from `console.bellsystems.net`, so the console's NPM proxy host +can stay HTTPS-only with no per-path exceptions. + +Required, one-time setup outside this repository: + +1. **DNS**: add an A/CNAME record for `melodies.bellsystems.net` pointing at the same + host as `console.bellsystems.net`. +2. **NPM (Nginx Proxy Manager) proxy host**: create a new proxy host for + `melodies.bellsystems.net` forwarding to the `nginx` container's exposed port + (`90` per `docker-compose.yml`). **Do not force SSL / do not enable the "Force + SSL" redirect** on this proxy host — it must remain reachable over plain HTTP. + +The in-repo `nginx/nginx.conf` already has a dedicated `server_name +melodies.bellsystems.net` block that maps `/download/{pid}` to the backend's +`/api/melodies/download/{pid}` route, so no further nginx changes are needed once +the NPM proxy host exists. + +## Verification + +1. Build/select an archetype for a melody → confirm a file appears at + `./data/melody_binaries/{uid}.bsm` and the melody's `url` becomes + `http://melodies.bellsystems.net/download/{pid}`. +2. `curl http://localhost:8000/api/melodies/download/{pid}` (direct to backend, + bypassing nginx) → returns the `.bsm` bytes, no auth header needed. +3. `curl http://localhost:90/api/melodies/download/{pid}` (through nginx on the + console server block) → same result. +4. Once NPM is configured, `curl http://melodies.bellsystems.net/download/{pid}` + → same result, over plain HTTP. +5. Delete the melody → confirm the local `.bsm` file is removed. +6. Upload a preview audio file → confirm it still lands in Firebase Storage and + `previewURL` still populates (regression check). diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 9674f17..d3d65a3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -54,4 +54,25 @@ http { proxy_set_header Connection "upgrade"; } } + + # Plain-HTTP-only host for ESP32 melody downloads. Kept separate from the + # console.bellsystems.net server block above so console can stay HTTPS-only + # in NPM with no path-based TLS exceptions. ESP32 devices can't afford the + # RAM for a TLS client, so this host must never be forced to HTTPS upstream. + server { + listen 80; + server_name melodies.bellsystems.net; + + resolver 127.0.0.11 valid=5s; + set $backend_upstream http://backend:8000; + + location /download/ { + rewrite ^/download/(.*)$ /api/melodies/download/$1 break; + proxy_pass $backend_upstream$uri; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } }