Initial Switch to V2. Completely Overhauled Backend, Frontend and General Structure.

strategies/AUTOMATION_ENGINE_STRATEGY.md

@@ -0,0 +1,395 @@
+# BellSystems CP — Automation & Notification Engine Strategy
+
+## Overview
+
+This document defines the architecture and implementation plan for a three-tier intelligence layer built on top of the existing BellSystems Control Panel. The system consists of:
+
+1. **Event Logs** — passive, timestamped record of notable system events
+2. **Notifications** — real-time or near-real-time alerts surfaced in the UI
+3. **Automation Rules** — trigger → condition → action pipelines, configurable via UI
+
+The existing tech stack is unchanged: **FastAPI + SQLite (aiosqlite) + Firestore + React**. Everything new slots in as additional tables in `mqtt_data.db`, new backend modules, and new frontend pages/components.
+
+---
+
+## Architecture
+
+```
+┌──────────────────────────────────────────────────┐
+│  Scheduler Loop (runs inside existing FastAPI    │
+│  startup, alongside email_sync_loop)             │
+│                                                  │
+│  Every 60s: evaluate_rules()                     │
+│    ↓                                             │
+│  Rules Engine                                    │
+│    → loads enabled rules from DB                 │
+│    → evaluates conditions against live data      │
+│    → fires Action Executor on match              │
+│                                                  │
+│  Action Executor                                 │
+│    → create_event_log()                          │
+│    → create_notification()                       │
+│    → send_email()           (existing)           │
+│    → mqtt_publish_command() (existing)           │
+│    → update_field()                              │
+└──────────────────────────────────────────────────┘
+                     ↕ REST / WebSocket
+┌──────────────────────────────────────────────────┐
+│  React Frontend                                  │
+│    - Bell icon in Header (unread count badge)    │
+│    - Notifications dropdown/panel                │
+│    - /automations page (rule CRUD)               │
+│    - Event Log viewer (filterable)               │
+└──────────────────────────────────────────────────┘
+```
+
+---
+
+## Database Schema (additions to `mqtt_data.db`)
+
+### `event_log`
+Permanent, append-only record of things that happened.
+
+```sql
+CREATE TABLE IF NOT EXISTS event_log (
+    id          TEXT PRIMARY KEY,
+    category    TEXT NOT NULL,  -- 'device' | 'crm' | 'quotation' | 'user' | 'system'
+    entity_type TEXT,           -- 'device' | 'customer' | 'quotation' | 'user'
+    entity_id   TEXT,           -- the ID of the affected record
+    title       TEXT NOT NULL,
+    detail      TEXT,
+    severity    TEXT NOT NULL DEFAULT 'info',  -- 'info' | 'warning' | 'error'
+    rule_id     TEXT,           -- which automation rule triggered this (nullable)
+    created_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE INDEX IF NOT EXISTS idx_event_log_category ON event_log(category, created_at);
+CREATE INDEX IF NOT EXISTS idx_event_log_entity ON event_log(entity_type, entity_id);
+```
+
+### `notifications`
+Short-lived, user-facing alerts. Cleared once read or after TTL.
+
+```sql
+CREATE TABLE IF NOT EXISTS notifications (
+    id          TEXT PRIMARY KEY,
+    title       TEXT NOT NULL,
+    body        TEXT,
+    link        TEXT,           -- optional frontend route, e.g. "/crm/customers/abc123"
+    severity    TEXT NOT NULL DEFAULT 'info',  -- 'info' | 'warning' | 'error' | 'success'
+    is_read     INTEGER NOT NULL DEFAULT 0,
+    rule_id     TEXT,
+    entity_type TEXT,
+    entity_id   TEXT,
+    created_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE INDEX IF NOT EXISTS idx_notifications_read ON notifications(is_read, created_at);
+```
+
+### `automation_rules`
+Stores user-defined rules. Evaluated by the scheduler.
+
+```sql
+CREATE TABLE IF NOT EXISTS automation_rules (
+    id              TEXT PRIMARY KEY,
+    name            TEXT NOT NULL,
+    description     TEXT,
+    enabled         INTEGER NOT NULL DEFAULT 1,
+    trigger_type    TEXT NOT NULL,  -- 'schedule' | 'mqtt_alert' | 'email_received'
+    trigger_config  TEXT NOT NULL DEFAULT '{}',  -- JSON
+    conditions      TEXT NOT NULL DEFAULT '[]',  -- JSON array of condition objects
+    actions         TEXT NOT NULL DEFAULT '[]',  -- JSON array of action objects
+    cooldown_hours  REAL NOT NULL DEFAULT 0,     -- min hours between firing on same entity
+    last_run_at     TEXT,
+    run_count       INTEGER NOT NULL DEFAULT 0,
+    created_at      TEXT NOT NULL DEFAULT (datetime('now')),
+    updated_at      TEXT NOT NULL DEFAULT (datetime('now'))
+);
+```
+
+### `automation_run_log`
+Deduplication and audit trail for rule executions.
+
+```sql
+CREATE TABLE IF NOT EXISTS automation_run_log (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    rule_id     TEXT NOT NULL,
+    entity_type TEXT,
+    entity_id   TEXT,
+    status      TEXT NOT NULL,  -- 'fired' | 'skipped_cooldown' | 'error'
+    detail      TEXT,
+    fired_at    TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE INDEX IF NOT EXISTS idx_run_log_rule ON automation_run_log(rule_id, fired_at);
+CREATE INDEX IF NOT EXISTS idx_run_log_entity ON automation_run_log(entity_type, entity_id, fired_at);
+```
+
+---
+
+## Backend Module: `automation/`
+
+New module at `backend/automation/`, registered in `main.py`.
+
+```
+backend/automation/
+├── __init__.py
+├── router.py        # CRUD for rules, event_log GET, notifications GET/PATCH
+├── models.py        # Pydantic schemas for rules, conditions, actions
+├── engine.py        # evaluate_rules(), condition evaluators, action executors
+├── scheduler.py     # automation_loop() async task, wired into main.py startup
+└── database.py      # DB helpers for all 4 new tables
+```
+
+### Wiring into `main.py`
+
+```python
+from automation.router import router as automation_router
+from automation.scheduler import automation_loop
+
+app.include_router(automation_router)
+
+# In startup():
+asyncio.create_task(automation_loop())
+```
+
+---
+
+## Rule Object Structure (JSON, stored in DB)
+
+```json
+{
+  "id": "rule_abc123",
+  "name": "Quotation follow-up after 7 days",
+  "enabled": true,
+  "trigger_type": "schedule",
+  "trigger_config": { "interval_hours": 24 },
+  "conditions": [
+    { "entity": "quotation", "field": "status",              "op": "eq",  "value": "sent" },
+    { "entity": "quotation", "field": "days_since_updated",  "op": "gte", "value": 7 },
+    { "entity": "quotation", "field": "has_reply",           "op": "eq",  "value": false }
+  ],
+  "actions": [
+    {
+      "type": "send_email",
+      "template_key": "quotation_followup",
+      "to": "{{quotation.client_email}}",
+      "subject": "Following up on Quotation {{quotation.quotation_number}}",
+      "body": "Hi {{customer.name}}, did you have a chance to review our quotation?"
+    },
+    {
+      "type": "create_notification",
+      "title": "Follow-up sent to {{customer.name}}",
+      "link": "/crm/customers/{{quotation.customer_id}}"
+    },
+    {
+      "type": "create_event_log",
+      "category": "quotation",
+      "severity": "info",
+      "title": "Auto follow-up sent for {{quotation.quotation_number}}"
+    }
+  ],
+  "cooldown_hours": 168
+}
+```
+
+---
+
+## Supported Trigger Types
+
+| Trigger | How it works |
+|---|---|
+| `schedule` | Evaluated every N hours by the background loop |
+| `mqtt_alert` | Fires immediately when `_handle_alerts()` in `mqtt/logger.py` upserts an alert — hook into that function |
+| `email_received` | Fires inside `sync_emails()` in `crm/email_sync.py` after a new inbound email is stored |
+
+> **Note:** `mqtt_alert` and `email_received` triggers bypass the scheduler loop — they are called directly from the relevant handler functions, giving near-real-time response.
+
+---
+
+## Supported Condition Operators
+
+| op | Meaning |
+|---|---|
+| `eq` | equals |
+| `neq` | not equals |
+| `gt` / `gte` / `lt` / `lte` | numeric comparisons |
+| `contains` | string contains |
+| `is_null` / `not_null` | field presence |
+| `days_since` | computed: (now - field_datetime) in days |
+
+---
+
+## Supported Action Types
+
+| Action | What it does | Notes |
+|---|---|---|
+| `create_event_log` | Writes to `event_log` table | Always safe to fire |
+| `create_notification` | Writes to `notifications` table | Surfaces in UI bell icon |
+| `send_email` | Calls existing `crm.email_sync.send_email()` | Uses existing mail accounts |
+| `update_field` | Updates a field on an entity in DB/Firestore | Use carefully — define allowed fields explicitly |
+| `mqtt_publish` | Calls `mqtt_manager.publish_command()` | For device auto-actions |
+| `webhook` | HTTP POST to an external URL | Future / optional |
+
+---
+
+## Notification System (Frontend)
+
+### Bell Icon in Header
+
+- Polling endpoint: `GET /api/notifications?unread=true&limit=20`
+- Poll interval: 30 seconds (or switch to WebSocket push — the WS infrastructure already exists via `mqtt_manager`)
+- Badge shows unread count
+- Click opens a dropdown panel listing recent notifications with title, time, severity color, and optional link
+
+### Notification Panel
+- Mark as read: `PATCH /api/notifications/{id}/read`
+- Mark all read: `PATCH /api/notifications/read-all`
+- Link field navigates to the relevant page on click
+
+### Toast Popups (optional, Phase 3 polish)
+- Triggered by polling detecting new unread notifications since last check
+- Use an existing toast component if one exists, otherwise add a lightweight one
+
+---
+
+## Automation Rules UI (`/automations`)
+
+A new sidebar entry under Settings (sysadmin/admin only).
+
+### Rule List Page
+- Table: name, enabled toggle, trigger type, last run, run count, edit/delete
+- "New Rule" button
+
+### Rule Editor (modal or full page)
+- **Name & description** — free text
+- **Trigger** — dropdown: Schedule / MQTT Alert / Email Received
+  - Schedule: interval hours input
+  - MQTT Alert: subsystem filter (optional)
+  - Email Received: from address filter (optional)
+- **Conditions** — dynamic list, each row:
+  - Entity selector (Quotation / Device / Customer / User)
+  - Field selector (populated based on entity)
+  - Operator dropdown
+  - Value input
+- **Actions** — dynamic list, each row:
+  - Action type dropdown
+  - Type-specific fields (to address, subject, body for email; notification title/body; etc.)
+  - Template variables hint: `{{quotation.quotation_number}}`, `{{customer.name}}`, etc.
+- **Cooldown** — hours between firings on the same entity
+- **Enabled** toggle
+
+### Rule Run History
+- Per-rule log: when it fired, on which entity, success/error
+
+---
+
+## Event Log UI
+
+Accessible from `/event-log` route, linked from Dashboard.
+
+- Filterable by: category, severity, entity type, date range
+- Columns: time, category, severity badge, title, entity link
+- Append-only (no deletion from UI)
+- Retention: purge entries older than configurable days (e.g. 180 days) via the existing `purge_loop` pattern in `mqtt/database.py`
+
+---
+
+## Pre-Built Rules (Seeded on First Run, All Disabled)
+
+These are created on first startup — the admin enables and customizes them.
+
+| Rule | Trigger | Condition | Action |
+|---|---|---|---|
+| Quotation follow-up | Schedule 24h | status=sent AND days_since_updated ≥ 7 AND no reply | Send follow-up email + notify |
+| Device offline warning | Schedule 1h | no heartbeat for > 2h | Create notification + event log |
+| New unknown email | email_received | customer_id IS NULL | Create notification |
+| Subscription expiring soon | Schedule 24h | subscription.expiry_date within 7 days | Notify + send email |
+| Device critical alert | mqtt_alert | state = CRITICAL | Notify + event log + optional MQTT restart |
+| Quotation expired | Schedule 24h | status=sent AND days_since_updated ≥ 30 | Update status → expired + notify |
+
+---
+
+## Implementation Phases
+
+### Phase 1 — Foundation (DB + API)
+- [ ] Add 4 new tables to `mqtt/database.py` schema + migrations
+- [ ] Create `automation/database.py` with all DB helpers
+- [ ] Create `automation/models.py` — Pydantic schemas for rules, conditions, actions, notifications, event_log
+- [ ] Create `automation/router.py` — CRUD for rules, GET event_log, GET/PATCH notifications
+- [ ] Wire router into `main.py`
+
+### Phase 2 — Rules Engine + Scheduler
+- [ ] Create `automation/engine.py` — condition evaluator, template renderer, action executor
+- [ ] Create `automation/scheduler.py` — `automation_loop()` async task
+- [ ] Hook `email_received` trigger into `crm/email_sync.sync_emails()`
+- [ ] Hook `mqtt_alert` trigger into `mqtt/logger._handle_alerts()`
+- [ ] Seed pre-built (disabled) rules on first startup
+- [ ] Wire `automation_loop()` into `main.py` startup
+
+### Phase 3 — Notification UI
+- [ ] Bell icon with unread badge in `Header.jsx`
+- [ ] Notifications dropdown panel component
+- [ ] 30s polling hook in React
+- [ ] Mark read / mark all read
+
+### Phase 4 — Automation Rules UI
+- [ ] `/automations` route and rule list page
+- [ ] Rule editor form (conditions + actions dynamic builder)
+- [ ] Enable/disable toggle
+- [ ] Run history per rule
+- [ ] Add "Automations" entry to Sidebar under Settings
+
+### Phase 5 — Event Log UI
+- [ ] `/event-log` route with filterable table
+- [ ] Purge policy wired into existing `purge_loop`
+- [ ] Dashboard widget showing recent high-severity events
+
+### Phase 6 — Polish
+- [ ] Toast notifications on new unread detection
+- [ ] Template variable previewer in rule editor
+- [ ] "Run now" button per rule (for testing without waiting for scheduler)
+- [ ] Named email templates stored in DB (reusable across rules)
+
+---
+
+## Key Design Decisions
+
+| Decision | Choice | Reason |
+|---|---|---|
+| Storage | SQLite (same `mqtt_data.db`) | Consistent with existing pattern; no new infra |
+| Scheduler | `asyncio` task in FastAPI startup | Same pattern as `email_sync_loop` and `purge_loop` already in `main.py` |
+| Rule format | JSON columns in DB | Flexible, UI-editable, no schema migrations per new rule type |
+| Template variables | `{{entity.field}}` string interpolation | Simple to implement, readable in UI |
+| Cooldown dedup | `automation_run_log` per (rule_id, entity_id) | Prevents repeat firing on same quotation/device within cooldown window |
+| Notification delivery | DB polling (30s) initially | The WS infra exists (`mqtt_manager._ws_subscribers`) — easy to upgrade later |
+| Pre-built rules | Seeded as disabled | Non-intrusive — admin must consciously enable each one |
+| `update_field` safety | Explicit allowlist of permitted fields | Prevents accidental data corruption from misconfigured rules |
+
+---
+
+## Template Variables Reference
+
+Available inside action `body`, `subject`, `title`, `link` fields:
+
+| Variable | Source |
+|---|---|
+| `{{customer.name}}` | Firestore `crm_customers` |
+| `{{customer.organization}}` | Firestore `crm_customers` |
+| `{{quotation.quotation_number}}` | SQLite `crm_quotations` |
+| `{{quotation.final_total}}` | SQLite `crm_quotations` |
+| `{{quotation.status}}` | SQLite `crm_quotations` |
+| `{{quotation.client_email}}` | SQLite `crm_quotations` |
+| `{{device.serial}}` | Firestore `devices` |
+| `{{device.label}}` | Firestore `devices` |
+| `{{alert.subsystem}}` | MQTT alert payload |
+| `{{alert.state}}` | MQTT alert payload |
+| `{{user.email}}` | Firestore `users` |
+
+---
+
+## Notes
+
+- `crm/email_sync.send_email()` is reused as-is for the `send_email` action type. The engine constructs the call parameters.
+- `update_field` actions start with an allowlist of: `quotation.status`, `user.status`. Expand deliberately.
+- For MQTT auto-restart, `mqtt_manager.publish_command(serial, "restart", {})` already works — the engine just calls it.
+- Firestore is read-only from the automation engine (for customer/device lookups). All writes go to SQLite, consistent with the existing architecture.
+- The `has_reply` condition on quotations is computed by checking whether any `crm_comms_log` entry exists with `direction='inbound'` and `customer_id` matching the quotation's customer, dated after the quotation's `updated_at`.

strategies/BellSystems_AdminPanel_Strategy.md

@@ -0,0 +1,311 @@
+# BellSystems Admin Panel — Strategy Guide
+
+## Project Overview
+
+A self-hosted web-based admin panel for managing BellSystems devices, melodies, users, and MQTT communications. Built with **Python (FastAPI)** backend and **React** frontend, deployed on a **VPS** behind **Nginx**.
+
+---
+
+## Deployment Architecture
+
+```
+[Browser] → [Nginx (VPS)]
+                ├── / → React Frontend (static files)
+                └── /api → FastAPI Backend
+                              ├── Firestore (Google Cloud) via Admin SDK
+                              ├── Mosquitto (localhost on VPS)
+                              └── Firebase Storage (for melody files)
+```
+
+**Everything lives on the VPS:**
+
+- Nginx as reverse proxy + static file server
+- FastAPI as the API backend
+- Mosquitto MQTT broker (already running there)
+- React frontend served as static files
+- All containerized with Docker Compose
+
+---
+
+## Tech Stack
+
+| Layer       | Technology                        |
+| ----------- | --------------------------------- |
+| Frontend    | React + Tailwind CSS + Vite       |
+| Backend     | Python 3.11+ / FastAPI            |
+| Database    | Firestore (via Firebase Admin SDK) |
+| File Storage| Firebase Storage                  |
+| MQTT        | Mosquitto (local) + paho-mqtt     |
+| Auth        | JWT (FastAPI) with role-based access |
+| Deployment  | Docker Compose + Nginx            |
+| VPS OS      | Linux (existing VPS in Germany)   |
+
+---
+
+## Project Folder Structure
+
+```
+bellsystems-admin/
+├── docker-compose.yml
+├── .env                          # Secrets (gitignored)
+├── .env.example                  # Template for env vars
+├── README.md
+│
+├── backend/
+│   ├── Dockerfile
+│   ├── requirements.txt
+│   ├── main.py                   # FastAPI app entry point
+│   ├── config.py                 # Settings / env loading
+│   │
+│   ├── auth/
+│   │   ├── __init__.py
+│   │   ├── router.py             # Login / token endpoints
+│   │   ├── models.py             # User/token schemas
+│   │   ├── dependencies.py       # JWT verification, role checks
+│   │   └── utils.py              # Password hashing, token creation
+│   │
+│   ├── melodies/
+│   │   ├── __init__.py
+│   │   ├── router.py             # CRUD endpoints for melodies
+│   │   ├── models.py             # Pydantic schemas
+│   │   └── service.py            # Firestore operations
+│   │
+│   ├── devices/
+│   │   ├── __init__.py
+│   │   ├── router.py             # CRUD endpoints for devices
+│   │   ├── models.py             # Pydantic schemas
+│   │   └── service.py            # Firestore operations
+│   │
+│   ├── users/
+│   │   ├── __init__.py
+│   │   ├── router.py             # CRUD endpoints for users
+│   │   ├── models.py             # Pydantic schemas
+│   │   └── service.py            # Firestore operations
+│   │
+│   ├── mqtt/
+│   │   ├── __init__.py
+│   │   ├── router.py             # Command endpoints + WebSocket for live data
+│   │   ├── client.py             # MQTT client wrapper (paho-mqtt)
+│   │   ├── logger.py             # Log storage and retrieval
+│   │   └── mosquitto.py          # Mosquitto password file management
+│   │
+│   ├── equipment/
+│   │   ├── __init__.py
+│   │   ├── router.py             # Complementary devices / notes
+│   │   ├── models.py
+│   │   └── service.py
+│   │
+│   └── shared/
+│       ├── __init__.py
+│       ├── firebase.py           # Firebase Admin SDK initialization
+│       └── exceptions.py         # Custom error handlers
+│
+├── frontend/
+│   ├── Dockerfile
+│   ├── package.json
+│   ├── vite.config.js
+│   ├── tailwind.config.js
+│   ├── index.html
+│   │
+│   └── src/
+│       ├── main.jsx
+│       ├── App.jsx
+│       │
+│       ├── api/
+│       │   └── client.js         # Axios/fetch wrapper with JWT
+│       │
+│       ├── auth/
+│       │   ├── LoginPage.jsx
+│       │   └── AuthContext.jsx    # JWT token state management
+│       │
+│       ├── layout/
+│       │   ├── Sidebar.jsx       # Navigation menu
+│       │   ├── Header.jsx
+│       │   └── MainLayout.jsx
+│       │
+│       ├── melodies/
+│       │   ├── MelodyList.jsx
+│       │   ├── MelodyForm.jsx    # Add / Edit form
+│       │   └── MelodyDetail.jsx
+│       │
+│       ├── devices/
+│       │   ├── DeviceList.jsx
+│       │   ├── DeviceForm.jsx
+│       │   └── DeviceDetail.jsx
+│       │
+│       ├── users/
+│       │   ├── UserList.jsx
+│       │   ├── UserForm.jsx
+│       │   └── UserDetail.jsx
+│       │
+│       ├── mqtt/
+│       │   ├── MqttDashboard.jsx # Live data, logs, charts
+│       │   └── CommandPanel.jsx  # Send commands to devices
+│       │
+│       └── components/
+│           ├── DataTable.jsx     # Reusable table component
+│           ├── SearchBar.jsx
+│           ├── ConfirmDialog.jsx
+│           └── StatusBadge.jsx
+│
+└── nginx/
+    └── nginx.conf                # Nginx reverse proxy config
+```
+
+---
+
+## Auth & Roles
+
+| Role            | Access                                      |
+| --------------- | ------------------------------------------- |
+| `superadmin`    | Everything — full CRUD on all resources      |
+| `melody_editor` | Melodies: full CRUD                          |
+| `device_manager`| Devices + MQTT: full CRUD + commands         |
+| `user_manager`  | Users: full CRUD                             |
+| `viewer`        | Read-only access to everything               |
+
+Admin users are stored in a separate Firestore collection (`admin_users`) or a local database (SQLite/PostgreSQL). JWT tokens carry the role, and FastAPI dependencies enforce permissions per endpoint.
+
+---
+
+## Build Phases
+
+### Phase 0 — Project Scaffolding
+
+- [ ] Initialize backend (FastAPI project structure)
+- [ ] Initialize frontend (React + Vite + Tailwind)
+- [ ] Docker Compose setup (backend + frontend + nginx)
+- [ ] Environment variables / config
+- [ ] Firebase Admin SDK setup (service account key)
+- [ ] Verify Firestore connectivity
+
+### Phase 1 — Auth System
+
+- [ ] Admin user model (Firestore or local DB)
+- [ ] Login endpoint (email + password → JWT)
+- [ ] JWT middleware for protected routes
+- [ ] Role-based access control (RBAC) dependencies
+- [ ] Login page (frontend)
+- [ ] Auth context + protected routes (frontend)
+
+### Phase 2 — Melody Editor (Priority #1)
+
+- [ ] Firestore melody document schema review
+- [ ] Backend: CRUD endpoints (list, get, create, update, delete)
+- [ ] Backend: Firebase Storage integration (upload/manage bin files)
+- [ ] Frontend: Melody list with search/filter
+- [ ] Frontend: Add/Edit melody form (all attributes)
+- [ ] Frontend: Delete with confirmation
+- [ ] Frontend: File upload for online melodies
+
+### Phase 3 — Device Editor
+
+- [ ] Firestore device document schema review
+- [ ] Backend: CRUD endpoints
+- [ ] Backend: Serial number generation (unique)
+- [ ] Backend: Mosquitto password registration (auto)
+- [ ] Frontend: Device list with status indicators
+- [ ] Frontend: Add/Edit device form
+- [ ] Frontend: Delete with confirmation
+
+### Phase 4 — User Editor
+
+- [ ] Firestore user document schema review
+- [ ] Backend: CRUD endpoints
+- [ ] Backend: User-device assignment management
+- [ ] Frontend: User list with search/filter
+- [ ] Frontend: Add/Edit user form
+- [ ] Frontend: Block/unblock functionality
+- [ ] Frontend: Device assignment UI
+
+### Phase 5 — MQTT Integration
+
+- [ ] Backend: MQTT client (paho-mqtt) connecting to Mosquitto
+- [ ] Backend: Send commands to devices via MQTT
+- [ ] Backend: Receive and store device logs/heartbeats
+- [ ] Backend: WebSocket endpoint for live MQTT data streaming
+- [ ] Frontend: MQTT dashboard (live device status)
+- [ ] Frontend: Command panel (send commands)
+- [ ] Frontend: Log viewer with filtering
+- [ ] Frontend: Basic charts/histograms (playback events, etc.)
+
+### Phase 6 — Equipment & Notes Tracking
+
+- [ ] Backend: CRUD for complementary equipment notes
+- [ ] Frontend: Simple note/log interface per device/user
+
+### Phase 7 — Polish & Deployment
+
+- [ ] SSL certificates (Let's Encrypt via Certbot)
+- [ ] Production Docker Compose config
+- [ ] Nginx hardening
+- [ ] Backup strategy for logs/data
+- [ ] Error handling & loading states (frontend)
+- [ ] Mobile responsiveness (basic)
+
+---
+
+## Key Decisions
+
+| Decision                   | Choice                           | Reason                                       |
+| -------------------------- | -------------------------------- | -------------------------------------------- |
+| Backend framework          | FastAPI                          | Easy to learn, async, auto-docs              |
+| Frontend framework         | React                            | Huge ecosystem, great for admin UIs          |
+| CSS framework              | Tailwind CSS                     | Fast styling, no custom CSS needed           |
+| Build tool                 | Vite                             | Fast dev server, instant HMR                 |
+| MQTT library               | paho-mqtt (Python)               | Standard, reliable, well-documented          |
+| Mosquitto auth             | Password file (direct management)| Simple, no plugins needed                    |
+| Deployment                 | Docker Compose on VPS            | Single server, easy to manage                |
+| Admin auth storage         | Firestore `admin_users` collection| Consistent with existing data layer         |
+
+---
+
+## Environment Variables (.env)
+
+```env
+# Firebase
+FIREBASE_SERVICE_ACCOUNT_PATH=./firebase-service-account.json
+
+# JWT
+JWT_SECRET_KEY=your-secret-key-here
+JWT_ALGORITHM=HS256
+JWT_EXPIRATION_MINUTES=480
+
+# MQTT
+MQTT_BROKER_HOST=localhost
+MQTT_BROKER_PORT=1883
+MQTT_ADMIN_USERNAME=admin
+MQTT_ADMIN_PASSWORD=your-mqtt-admin-password
+MOSQUITTO_PASSWORD_FILE=/etc/mosquitto/passwd
+
+# App
+BACKEND_CORS_ORIGINS=["http://localhost:5173"]
+DEBUG=true
+```
+
+---
+
+## First Steps for Claude Code
+
+Once this strategy is reviewed and approved, the first command for Claude Code:
+
+```
+Initialize the bellsystems-admin project:
+1. Create the folder structure as defined in the strategy guide
+2. Set up the backend with FastAPI (main.py, config.py, requirements.txt)
+3. Set up the frontend with React + Vite + Tailwind
+4. Create docker-compose.yml with backend, frontend, and nginx services
+5. Create the nginx.conf for reverse proxying
+6. Create .env.example with all required variables
+7. Wire up Firebase Admin SDK initialization (backend/shared/firebase.py)
+8. Create a basic health-check endpoint to verify everything runs
+```
+
+---
+
+## Notes
+
+- All Firestore document schemas will be reviewed together before building each module
+- MQTT log storage strategy (local DB vs Firestore) to be decided in Phase 5
+- The existing Flutter app (main product) remains unchanged — this admin panel is a separate tool
+- OTA update management could be added as a future phase

strategies/DATABASE_MIGRATION.md

@@ -0,0 +1,475 @@
+# Database Migration Strategy
+# BellSystems CP v2 — Firestore + SQLite → Postgres
+
+> This is the living plan. Update it as phases complete.
+> Never start a phase without reading the notes from the previous one.
+
+---
+
+## Database Split — Target State
+
+| Data                        | Target       | Source       | Flutter uses? |
+|-----------------------------|--------------|--------------|---------------|
+| Devices                     | Firestore    | Firestore    | YES — keep    |
+| App users (device owners)   | Firestore    | Firestore    | YES — keep    |
+| Published melodies          | Firestore    | Firestore    | YES — keep    |
+| Draft melodies              | Postgres     | SQLite       | No            |
+| Built melodies              | Postgres     | SQLite       | No            |
+| CRM customers               | Postgres     | Firestore    | No            |
+| CRM products                | Postgres     | Firestore    | No            |
+| CRM orders                  | Postgres     | Firestore (subcollection) | No |
+| Console settings            | Postgres     | Firestore    | No            |
+| Public features settings    | Postgres     | Firestore    | No            |
+| Staff / admin users         | Postgres     | Firestore    | No            |
+| Firmware versions           | Postgres     | Firestore    | No            |
+| Notes / Issues              | Postgres     | New (done)   | No            |
+| Support tickets             | Postgres     | New (done)   | No            |
+| CRM comms log               | Postgres     | SQLite       | No            |
+| CRM media references        | Postgres     | SQLite       | No            |
+| CRM sync state              | Postgres     | SQLite       | No            |
+| CRM quotations + items      | Postgres     | SQLite       | No            |
+| Mfg audit log               | Postgres     | SQLite       | No            |
+| Device alerts               | Postgres     | SQLite       | No            |
+| MQTT commands               | Postgres     | SQLite       | No            |
+| MQTT heartbeats             | Postgres     | SQLite       | No            |
+| Device logs                 | Postgres (partitioned) | SQLite | No         |
+| Staff audit log             | Postgres     | New          | No            |
+
+**Rule:** Everything that FlutterFlow touches directly stays in Firestore forever.
+The Console backend continues to write to those Firestore collections exactly as today.
+We only stop *reading* from Firestore in the Console — never stop writing to it.
+
+---
+
+## Deployment Context — Critical
+
+**This project runs in two environments:**
+
+| Environment | SQLite data | Firestore data | Where migrations run |
+|-------------|-------------|----------------|----------------------|
+| Local (Windows + Docker for Desktop) | Empty / stale test data | Live (correct) | Development & testing only |
+| VPS (production Docker) | Live correct data | Live (correct) | **All Phase 1 migrations run here** |
+
+**What this means for each phase:**
+
+- **Phase 0 (schema):** Alembic migrations can be developed and tested locally, then the same migrations are run on the VPS via `docker compose exec backend alembic upgrade head`. The VPS is authoritative.
+- **Phase 1 (SQLite → Postgres):** Migration scripts must be run **on the VPS only**. The local SQLite is not a valid source. Do not run Phase 1 migration scripts locally and assume they reflect real data.
+- **Phase 2 (Firestore → Postgres):** Can be run on either environment (Firestore is the same), but the VPS run is the one that matters. Run locally first to verify the scripts work, then run on the VPS.
+- **Phase 3–5:** All service cutover and testing happens on the VPS.
+
+**The deployment workflow:**
+1. Develop and test code locally
+2. Push code to VPS (git pull or equivalent)
+3. Run `docker compose exec backend alembic upgrade head` on the VPS to apply schema changes
+4. Run migration scripts on the VPS when Phase 1 begins
+5. Verify everything on the VPS before marking a phase complete
+
+---
+
+## Non-negotiable Safety Rules
+
+1. **Never touch a Firestore collection** — only read from it during migration. Never delete, update, or rename documents until you have personally verified the Postgres data is complete and correct.
+2. **Every migration script runs in a transaction** — if any row fails, the entire script rolls back cleanly.
+3. **Idempotent scripts** — every script uses `ON CONFLICT DO NOTHING` or equivalent. Safe to run twice.
+4. **Count verification before commit** — each script prints `Source: N docs/rows → Postgres: N rows ✓` and aborts if counts don't match.
+5. **Migration run log** — a `_migration_runs` table in Postgres records what ran, when, how many rows, and success/failure. Check it after each script.
+6. **One domain at a time** — complete and verify a full domain (schema + migration script + service cutover + smoke test) before starting the next.
+7. **No data loss = no rushing** — downtime during migration is acceptable. Data loss is not.
+
+---
+
+## Phase 0 — Schema Foundation
+**Status: COMPLETE** — Alembic revision `b1c2d3e4f5a6` applied locally. Apply on VPS with `docker compose exec backend alembic upgrade head` before starting Phase 1.
+
+### What exists already in Postgres
+- `entries` + `entry_links` (notes/issues module)
+- `support_tickets` + `ticket_messages` (tickets module)
+- Alembic version history in `alembic_version`
+
+### What Phase 0 adds
+Add the `_migration_runs` tracking table and all new table definitions via Alembic before any data moves.
+
+New tables to create in this phase (schema only, no data yet):
+- `_migration_runs` — tracks what migration scripts have run
+- `crm_products` — flat columns, no JSONB needed
+- `crm_customers` — core columns + JSONB for `contacts`, `notes`, `owned_items`, `location`, `tags`, `technical_issues`, `install_support`, `transaction_history`, `crm_summary`
+- `crm_orders` — core columns + JSONB for `items`, `discount`, `shipping`, `payment_status`, `timeline`
+- `staff` — replaces `admin_users` Firestore collection
+- `console_settings` — key/value or typed columns, replaces Firestore `settings` doc
+- `public_features` — typed columns, replaces Firestore `public_features` doc
+- `crm_comms_log` — mirrors current SQLite schema, adds proper TIMESTAMPTZ columns
+- `crm_media` — mirrors current SQLite schema
+- `crm_sync_state` — key/value
+- `crm_quotations` + `crm_quotation_items` — mirrors current SQLite schema
+- `mfg_audit_log` — mirrors current SQLite schema
+- `device_alerts` — mirrors current SQLite schema
+- `commands` — mirrors current SQLite schema
+- `heartbeats` — mirrors current SQLite schema
+- `melody_drafts` — mirrors current SQLite schema
+- `built_melodies` — mirrors current SQLite schema
+- `device_logs` — **partitioned by month** on `received_at`
+- `audit_log` — new staff action audit system (see schema below)
+
+### Key schema decisions
+
+#### `device_logs` — monthly partitioning
+```sql
+CREATE TABLE device_logs (
+    id            BIGSERIAL,
+    device_serial TEXT NOT NULL,
+    level         TEXT NOT NULL,
+    message       TEXT NOT NULL,
+    device_timestamp BIGINT,
+    received_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (id, received_at)
+) PARTITION BY RANGE (received_at);
+
+-- Partitions created monthly by a background job or manually:
+CREATE TABLE device_logs_2025_01 PARTITION OF device_logs
+    FOR VALUES FROM ('2025-01-01') TO ('2025-02-01');
+-- etc.
+
+CREATE INDEX idx_device_logs_serial_time ON device_logs(device_serial, received_at DESC);
+CREATE INDEX idx_device_logs_level       ON device_logs(level, received_at DESC);
+```
+Dropping a partition to purge old data: `DROP TABLE device_logs_2024_06;` — instant, no DELETE scan.
+
+#### `crm_customers` — JSONB for flexible arrays
+```sql
+CREATE TABLE crm_customers (
+    id                   TEXT PRIMARY KEY,           -- keep Firestore UUID as-is
+    firestore_id         TEXT UNIQUE,                -- same value during transition, null-able later
+    title                TEXT,
+    name                 TEXT NOT NULL,
+    surname              TEXT,
+    organization         TEXT,
+    religion             TEXT,
+    language             TEXT NOT NULL DEFAULT 'el',
+    folder_id            TEXT UNIQUE NOT NULL,
+    relationship_status  TEXT NOT NULL DEFAULT 'lead',
+    nextcloud_folder     TEXT,
+    contacts             JSONB NOT NULL DEFAULT '[]',
+    notes                JSONB NOT NULL DEFAULT '[]',
+    location             JSONB,
+    tags                 TEXT[] NOT NULL DEFAULT '{}',
+    owned_items          JSONB NOT NULL DEFAULT '[]',
+    linked_user_ids      TEXT[] NOT NULL DEFAULT '{}',
+    technical_issues     JSONB NOT NULL DEFAULT '[]',
+    install_support      JSONB NOT NULL DEFAULT '[]',
+    transaction_history  JSONB NOT NULL DEFAULT '[]',
+    crm_summary          JSONB,
+    created_at           TIMESTAMPTZ NOT NULL,
+    updated_at           TIMESTAMPTZ NOT NULL
+);
+CREATE INDEX idx_crm_customers_rel_status ON crm_customers(relationship_status);
+CREATE INDEX idx_crm_customers_tags       ON crm_customers USING GIN(tags);
+CREATE INDEX idx_crm_customers_name       ON crm_customers(name, surname);
+```
+
+#### `crm_orders` — separate table (was Firestore subcollection)
+```sql
+CREATE TABLE crm_orders (
+    id                   TEXT PRIMARY KEY,
+    customer_id          TEXT NOT NULL REFERENCES crm_customers(id) ON DELETE CASCADE,
+    order_number         TEXT UNIQUE NOT NULL,
+    title                TEXT,
+    created_by           TEXT,
+    status               TEXT NOT NULL DEFAULT 'negotiating',
+    status_updated_date  TIMESTAMPTZ,
+    status_updated_by    TEXT,
+    items                JSONB NOT NULL DEFAULT '[]',
+    subtotal             NUMERIC(12,2) NOT NULL DEFAULT 0,
+    discount             JSONB,
+    total_price          NUMERIC(12,2) NOT NULL DEFAULT 0,
+    currency             TEXT NOT NULL DEFAULT 'EUR',
+    shipping             JSONB,
+    payment_status       JSONB NOT NULL DEFAULT '{}',
+    invoice_path         TEXT,
+    notes                TEXT,
+    timeline             JSONB NOT NULL DEFAULT '[]',
+    created_at           TIMESTAMPTZ NOT NULL,
+    updated_at           TIMESTAMPTZ NOT NULL
+);
+CREATE INDEX idx_crm_orders_customer ON crm_orders(customer_id);
+CREATE INDEX idx_crm_orders_status   ON crm_orders(status);
+```
+
+#### `staff` — replaces Firestore `admin_users`
+```sql
+CREATE TABLE staff (
+    id               TEXT PRIMARY KEY,    -- keep Firestore doc ID as-is during transition
+    firestore_id     TEXT UNIQUE,         -- same as id during transition
+    email            TEXT UNIQUE NOT NULL,
+    name             TEXT NOT NULL,
+    role             TEXT NOT NULL DEFAULT 'staff',
+    permissions      JSONB NOT NULL DEFAULT '{}',
+    hashed_password  TEXT NOT NULL,
+    is_active        BOOLEAN NOT NULL DEFAULT TRUE,
+    created_at       TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at       TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+```
+
+#### `audit_log` — new system, no migration source
+```sql
+CREATE TABLE audit_log (
+    id           BIGSERIAL PRIMARY KEY,
+    occurred_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+    actor_id     TEXT NOT NULL,
+    actor_name   TEXT NOT NULL,
+    action       TEXT NOT NULL,   -- CREATE | UPDATE | DELETE | COMMAND | LOGIN | LOGOUT | etc.
+    entity_type  TEXT NOT NULL,   -- customer | order | device | melody | product | staff | ticket | note | quotation | etc.
+    entity_id    TEXT NOT NULL,
+    entity_label TEXT,            -- denormalized human name: "Church of St. George", "SN-0042", etc.
+    changes      JSONB,           -- {"field": {"old": x, "new": y}, ...} — null for CREATE/DELETE/COMMAND
+    meta         JSONB            -- extra context: ip_address, command_name, etc.
+);
+-- Indexes covering the exact filter combos we need:
+CREATE INDEX idx_audit_actor      ON audit_log(actor_id,    occurred_at DESC);
+CREATE INDEX idx_audit_entity     ON audit_log(entity_type, entity_id,  occurred_at DESC);
+CREATE INDEX idx_audit_action     ON audit_log(action,                  occurred_at DESC);
+CREATE INDEX idx_audit_occurred   ON audit_log(occurred_at DESC);
+```
+
+---
+
+## Phase 1 — SQLite → Postgres (Data Migration)
+**Status: NOT STARTED**
+**Prerequisite:** Phase 0 complete (all tables exist in Postgres)
+
+No downtime required — SQLite is local, can read it while the app is running.
+After migration is verified, services are switched to read from Postgres.
+
+### Migration order (least dependencies first)
+
+| Step | Table | Script |
+|------|-------|--------|
+| 1.1  | `melody_drafts` | `migration/migrate_melody_drafts.py` |
+| 1.2  | `built_melodies` | `migration/migrate_built_melodies.py` |
+| 1.3  | `mfg_audit_log` | `migration/migrate_mfg_audit_log.py` |
+| 1.4  | `device_alerts` | `migration/migrate_device_alerts.py` |
+| 1.5  | `crm_sync_state` | `migration/migrate_crm_sync_state.py` |
+| 1.6  | `crm_quotations` | `migration/migrate_crm_quotations.py` |
+| 1.7  | `crm_quotation_items` | `migration/migrate_crm_quotation_items.py` |
+| 1.8  | `crm_media` | `migration/migrate_crm_media.py` |
+| 1.9  | `crm_comms_log` | `migration/migrate_crm_comms_log.py` |
+| 1.10 | `commands` | `migration/migrate_commands.py` |
+| 1.11 | `heartbeats` | `migration/migrate_heartbeats.py` |
+| 1.12 | `device_logs` | `migration/migrate_device_logs.py` (largest — batched) |
+
+### Per-script pattern
+```python
+# Every script follows this structure
+async def run():
+    sqlite_rows = await read_all_from_sqlite("table_name")
+    source_count = len(sqlite_rows)
+    print(f"Source: {source_count} rows")
+
+    async with pg_session() as session:
+        async with session.begin():
+            await session.execute(
+                insert(PgModel).values(rows).on_conflict_do_nothing()
+            )
+            pg_count = await session.scalar(select(func.count()).select_from(PgModel))
+
+    if pg_count < source_count:
+        raise RuntimeError(f"Count mismatch: source={source_count} pg={pg_count}")
+    print(f"Postgres: {pg_count} rows ✓")
+    await log_migration_run("table_name", source_count, pg_count)
+```
+
+### Service cutover per domain
+After each group is migrated and verified:
+1. Update service to import from `database.postgres` instead of `database.core`
+2. Replace `aiosqlite` queries with SQLAlchemy async queries
+3. Smoke test via the Console UI — verify the page loads correctly
+4. Leave SQLite file untouched for 48h as a fallback
+
+---
+
+## Phase 2 — Firestore → Postgres (Data Migration)
+**Status: NOT STARTED**
+**Prerequisite:** Phase 1 complete
+
+Requires `shared.firebase.get_db()` to read from Firestore.
+Scripts run with Firebase Admin SDK — same SDK already initialized in the backend.
+
+### Migration order
+
+| Step | Collection | Script | Notes |
+|------|-----------|--------|-------|
+| 2.1  | `settings` (doc) | `migration/migrate_settings.py` | Single document |
+| 2.2  | `public_features` (doc) | `migration/migrate_public_features.py` | Single document |
+| 2.3  | `crm_products` | `migration/migrate_crm_products.py` | No dependencies |
+| 2.4  | `crm_customers` | `migration/migrate_crm_customers.py` | Strip legacy `negotiating`/`has_problem` fields |
+| 2.5  | `orders` (subcollection) | `migration/migrate_crm_orders.py` | Uses `collection_group("orders")` |
+
+### Converting Firestore types
+Use the existing `_convert_firestore_value` helpers in `devices/service.py` — copy into a shared `migration/utils.py`. Key conversions:
+- `DatetimeWithNanoseconds` → `.isoformat()` string
+- `GeoPoint` → `{"lat": x, "lng": y}` dict
+- `DocumentReference` → `.id` string (just the doc ID, no path)
+
+### Cutover
+After each Firestore collection is migrated and verified:
+1. Switch service to read/write Postgres
+2. **Keep all Firestore write calls** — continue writing to Firestore on every mutation so the data stays current there for any emergency rollback
+3. After 48h of stable operation, remove the redundant Firestore writes (one service at a time)
+
+---
+
+## Phase 3 — Staff Auth Cutover
+**Status: NOT STARTED**
+**Prerequisite:** Phase 2 step 2.5 complete, staff table verified
+
+This is the highest-risk phase because auth affects every request.
+
+### Steps
+1. Migrate `admin_users` Firestore collection → `staff` Postgres table (script: `migration/migrate_staff.py`)
+2. Verify: compare email list, role list, permission maps between Firestore and Postgres
+3. Update `auth/dependencies.py` to query Postgres `staff` table instead of Firestore
+4. Update `staff/service.py` to read/write Postgres
+5. Update `seed_admin.py` to write to Postgres (keep old Firestore version as `seed_admin_firestore_legacy.py`)
+6. Test: log in as each role, verify permissions work
+7. Only after 24h stable — remove Firestore reads from auth
+
+### Rollback plan
+The JWT token payload doesn't change — it still contains `sub` (staff ID) and `permissions`.
+Rolling back is just reverting the two files (`auth/dependencies.py` and `staff/service.py`).
+
+---
+
+## Phase 4 — Audit Log System
+**Status: NOT STARTED**
+**Prerequisite:** Phase 0 (`audit_log` table created)
+
+The audit log system can be built and wired in incrementally — it doesn't block other phases.
+Wire it into each service as that service is cut over to Postgres.
+
+### The logging utility
+`backend/shared/audit.py` — a single async function all services call:
+
+```python
+async def log_action(
+    db: AsyncSession,
+    actor_id: str,
+    actor_name: str,
+    action: str,           # "CREATE" | "UPDATE" | "DELETE" | "COMMAND" | ...
+    entity_type: str,      # "customer" | "order" | "device" | ...
+    entity_id: str,
+    entity_label: str | None = None,
+    changes: dict | None = None,   # {"field": {"old": x, "new": y}}
+    meta: dict | None = None,      # {"ip": ..., "command_name": ...}
+) -> None
+```
+
+### How to capture diffs
+In service update functions:
+```python
+old_data = existing_record.to_dict()          # before
+await session.execute(update_stmt)
+new_data = updated_record.to_dict()           # after
+changes = {
+    k: {"old": old_data[k], "new": new_data[k]}
+    for k in new_data
+    if old_data.get(k) != new_data.get(k)
+}
+await log_action(db, actor_id, actor_name, "UPDATE", "customer", id, label, changes)
+```
+
+### Action types
+| Action | When |
+|--------|------|
+| `CREATE` | Any new record created |
+| `UPDATE` | Any field changed |
+| `DELETE` | Any record deleted |
+| `COMMAND` | MQTT command sent to device |
+| `PUBLISH` | Melody published to Firestore |
+| `UNPUBLISH` | Melody unpublished |
+| `LOGIN` | Staff login |
+| `LOGOUT` | Staff logout |
+| `PERMISSION_CHANGE` | Staff permissions updated |
+| `STATUS_CHANGE` | Order/customer/ticket status changed (convenience — also captured as UPDATE) |
+
+### API endpoint
+`GET /api/audit-log` with query params:
+- `actor_id` — filter by staff member
+- `entity_type` + `entity_id` — filter by a specific record
+- `action` — filter by action type
+- `from_date` / `to_date` — date range
+- `limit` / `offset` — pagination (default limit: 50, max: 200)
+
+---
+
+## Phase 5 — MQTT Live Data Cutover
+**Status: NOT STARTED**
+**Prerequisite:** Phase 1 complete (device_logs in Postgres)
+
+This phase switches the **live MQTT ingestion** from SQLite to Postgres.
+
+### Steps
+1. Update `database/core.py` `insert_log`, `insert_heartbeat`, `insert_command` to write to Postgres
+2. Update read functions (`get_logs`, `get_heartbeats`, etc.) similarly
+3. The partition management background job: each month, at startup or via a cron, ensure next month's partition exists:
+```python
+async def ensure_current_partitions(db: AsyncSession):
+    for month_offset in [0, 1]:  # current + next month
+        d = date.today().replace(day=1) + relativedelta(months=month_offset)
+        partition_name = f"device_logs_{d.strftime('%Y_%m')}"
+        start = d.isoformat()
+        end = (d + relativedelta(months=1)).isoformat()
+        await db.execute(text(f"""
+            CREATE TABLE IF NOT EXISTS {partition_name}
+            PARTITION OF device_logs
+            FOR VALUES FROM ('{start}') TO ('{end}')
+        """))
+```
+
+### Log retention
+- Keep last 6 months of partitions
+- Cron job runs monthly: checks for partitions older than 6 months and drops them
+- Dropping a partition = `DROP TABLE device_logs_2024_09;` — instantaneous, no row-by-row delete
+
+---
+
+## Verification Checklist (run after each phase)
+
+- [ ] `SELECT COUNT(*)` in Postgres matches source count for every migrated table
+- [ ] Sample 10 random records — compare field by field against source
+- [ ] Timestamps are stored as TIMESTAMPTZ, not TEXT strings
+- [ ] All JSONB columns parse correctly (no `null` where arrays expected)
+- [ ] Relevant Console pages load without errors
+- [ ] API endpoints return correct data
+- [ ] `_migration_runs` table shows success for all scripts
+
+---
+
+## Files & Locations
+
+```
+backend/
+├── migration/                  ← all migration scripts live here
+│   ├── utils.py                ← shared helpers (Firestore type converters, PG connection, etc.)
+│   ├── migrate_melody_drafts.py
+│   ├── migrate_crm_customers.py
+│   ├── migrate_crm_orders.py
+│   └── ... (one file per table)
+├── shared/
+│   └── audit.py                ← audit log utility (Phase 4)
+└── alembic/versions/           ← never edit by hand
+```
+
+---
+
+## Current Status Summary
+
+| Phase | Description | Status |
+|-------|-------------|--------|
+| 0     | Schema foundation (all tables in Postgres) | **COMPLETE** (local) — run `alembic upgrade head` on VPS |
+| 1     | SQLite → Postgres (data migration) | NOT STARTED |
+| 2     | Firestore → Postgres (data migration) | NOT STARTED |
+| 3     | Staff auth cutover | NOT STARTED |
+| 4     | Audit log system | NOT STARTED |
+| 5     | MQTT live data cutover | NOT STARTED |
+
+Update this table as each phase completes.

strategies/NOTES_ISSUES_TICKETS_BUILD.md

@@ -0,0 +1,114 @@
+# Notes, Issues & Support Tickets — UI Remaining Work
+
+> Backend is fully built and deployed. Alembic migrations applied.
+> This file tracks ONLY the remaining frontend work.
+> Migration strategy has moved to DATABASE_MIGRATION.md.
+
+## What's done
+
+- [x] `backend/notes/` — ORM, models, service, router (mounted at `/api/notes`)
+- [x] `backend/tickets/` — ORM, models, service, router (mounted at `/api/tickets`)
+- [x] Alembic migrations applied: `entries`, `entry_links`, `support_tickets`, `ticket_messages`
+- [x] `frontend/src/pages/crm/comms/helpdesk/TicketsTab.jsx` — tickets tab on comms page
+- [x] `frontend/src/modals/crm/helpdesk/CreateTicketModal.jsx`
+
+---
+
+## What's remaining
+
+### 7.1 Global Notes & Issues page
+
+Page at `frontend/src/pages/crm/notes/NotesPage.jsx`, route `/crm/notes`.
+
+Features:
+- `<Tabs>` with "Notes" and "Issues" tabs — filters `type` on `GET /api/notes`
+- `<DataTable>` columns: title, type badge, status badge, severity badge, linked entities (chips), author, date
+- Create button → opens `<EntryFormModal>`
+- Click row → opens `<EntryDetailModal>`
+- Filters: status, severity
+- `<Pagination>`
+
+### 7.2 Entry form modal
+
+Modal at `frontend/src/modals/crm/notes/EntryFormModal.jsx`.
+
+Fields:
+- Type selector (note / issue) — conditionally shows status + severity when type = issue
+- Title (`<FormField type="text">`)
+- Body (`<FormField type="textarea">`)
+- Status selector (issue only, `<FormField type="select">`)
+- Severity selector (issue only, `<FormField type="select">`)
+- Linked entities: searchable selects for devices, app users, customers
+
+### 7.3 Device detail page — Notes & Issues tab
+
+Add a "Notes & Issues" tab to the existing device detail page.
+Calls `GET /api/notes/by-entity/device/:deviceId`.
+Shows compact entry list. Inline "Add note" / "Add issue" buttons that pre-fill the device link.
+
+### 7.4 Customer detail page — Notes, Issues & Tickets tabs
+
+**Notes & Issues tab:** calls `GET /api/notes/by-entity/customer/:customerId`
+
+**Support Tickets tab:**
+- Calls `GET /api/tickets/by-customer/:customerId`
+- Ticket list with status + priority badges
+- Click row → opens `<TicketThreadModal>`
+
+### 7.5 Global Support Tickets page
+
+Page at `frontend/src/pages/crm/tickets/TicketsPage.jsx`, route `/crm/tickets`.
+
+Features:
+- `<DataTable>` columns: subject, customer name, device serial, status badge, priority badge, opened via, last updated
+- Click row → opens ticket thread view
+- Create ticket button
+- Filters: status, priority, customer, device
+- `<Pagination>`
+
+### 7.6 Ticket thread modal
+
+Modal at `frontend/src/modals/crm/tickets/TicketThreadModal.jsx`.
+
+- Ticket subject, customer, device, status, priority at the top
+- Message thread — chronological, oldest first
+- Internal notes visually distinct (different background, lock icon) — never shown to customers
+- Reply form at the bottom: toggle between "Reply to customer" and "Internal note"
+- Status change accessible from the thread view
+- "Escalate to issue" button — links to an existing issue entry
+
+---
+
+## API quick reference
+
+```
+GET  /api/notes                          list notes/issues (type, status, severity, page, limit)
+GET  /api/notes/by-entity/:type/:id      notes for a device or customer
+POST /api/notes                          create note or issue
+PATCH /api/notes/:id                     update
+PATCH /api/notes/:id/links               replace entity links
+DELETE /api/notes/:id
+
+GET  /api/tickets                        list tickets (status, priority, customer_id, page, limit)
+GET  /api/tickets/by-customer/:id
+GET  /api/tickets/by-device/:id
+POST /api/tickets
+PATCH /api/tickets/:id
+POST /api/tickets/:id/messages
+POST /api/tickets/:id/escalate
+```
+
+## Entry type rules
+
+| Field      | Note         | Issue                    |
+|------------|--------------|--------------------------|
+| `type`     | `'note'`     | `'issue'`                |
+| `status`   | always null  | required: open / in_progress / resolved |
+| `severity` | always null  | optional: low / medium / high / critical |
+
+## Ticket status flow
+
+```
+open → waiting_on_customer → waiting_on_staff → resolved → closed
+         (staff replied)        (customer replied)
+```

strategies/crm_full_erd.html

@@ -0,0 +1,127 @@
+
+<style>
+#erd svg { width: 100%; height: auto; }
+#erd svg.erDiagram .divider path { stroke-opacity: 0.4; }
+#erd svg.erDiagram .row-rect-odd path,
+#erd svg.erDiagram .row-rect-odd rect,
+#erd svg.erDiagram .row-rect-even path,
+#erd svg.erDiagram .row-rect-even rect { stroke: none !important; }
+</style>
+<div id="erd" style="padding: 1rem 0;"></div>
+<script type="module">
+import mermaid from 'https://esm.sh/mermaid@11/dist/mermaid.esm.min.mjs';
+const dark = matchMedia('(prefers-color-scheme: dark)').matches;
+await document.fonts.ready;
+mermaid.initialize({
+  startOnLoad: false,
+  theme: 'base',
+  fontFamily: '"Anthropic Sans", sans-serif',
+  themeVariables: {
+    darkMode: dark,
+    fontSize: '13px',
+    fontFamily: '"Anthropic Sans", sans-serif',
+    lineColor: dark ? '#9c9a92' : '#73726c',
+    textColor: dark ? '#c2c0b6' : '#3d3d3a',
+    primaryColor: dark ? '#3C3489' : '#EEEDFE',
+    primaryTextColor: dark ? '#CECBF6' : '#3C3489',
+    primaryBorderColor: dark ? '#7F77DD' : '#534AB7',
+    secondaryColor: dark ? '#085041' : '#E1F5EE',
+    tertiaryColor: dark ? '#712B13' : '#FAECE7',
+  },
+});
+const diagram = `erDiagram
+  staff_users {
+    uuid id PK
+    string name
+    string email
+    string role
+  }
+  devices {
+    uuid id PK
+    string serial_number
+    string firmware_version
+    string firestore_doc_id
+  }
+  app_users {
+    uuid id PK
+    string email
+    string display_name
+    string firestore_uid
+  }
+  customers {
+    uuid id PK
+    string full_name
+    string email
+    string phone
+    timestamp created_at
+  }
+  entries {
+    uuid id PK
+    uuid author_id FK
+    varchar type
+    varchar status
+    varchar severity
+    string title
+    text body
+    timestamp created_at
+    timestamp updated_at
+  }
+  entry_links {
+    uuid id PK
+    uuid entry_id FK
+    varchar entity_type
+    uuid entity_id
+  }
+  support_tickets {
+    uuid id PK
+    uuid customer_id FK
+    uuid device_id FK
+    uuid linked_entry_id FK
+    varchar status
+    varchar priority
+    varchar opened_via
+    timestamp created_at
+    timestamp updated_at
+  }
+  ticket_messages {
+    uuid id PK
+    uuid ticket_id FK
+    uuid sender_id
+    varchar sender_type
+    text body
+    timestamp created_at
+  }
+
+  staff_users ||--o{ entries : "authors"
+  entries ||--o{ entry_links : "has links"
+  support_tickets ||--o{ ticket_messages : "has messages"
+  customers ||--o{ support_tickets : "opens"
+  devices ||--o{ support_tickets : "subject of"
+  entries ||--o| support_tickets : "escalated to"
+`;
+const { svg } = await mermaid.render('erd-svg', diagram);
+document.getElementById('erd').innerHTML = svg;
+
+document.querySelectorAll('#erd svg.erDiagram .node').forEach(node => {
+  const firstPath = node.querySelector('path[d]');
+  if (!firstPath) return;
+  const d = firstPath.getAttribute('d');
+  const nums = d.match(/-?[\d.]+/g)?.map(Number);
+  if (!nums || nums.length < 8) return;
+  const xs = [nums[0], nums[2], nums[4], nums[6]];
+  const ys = [nums[1], nums[3], nums[5], nums[7]];
+  const x = Math.min(...xs), y = Math.min(...ys);
+  const w = Math.max(...xs) - x, h = Math.max(...ys) - y;
+  const rect = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
+  rect.setAttribute('x', x); rect.setAttribute('y', y);
+  rect.setAttribute('width', w); rect.setAttribute('height', h);
+  rect.setAttribute('rx', '8');
+  for (const a of ['fill', 'stroke', 'stroke-width', 'class', 'style']) {
+    if (firstPath.hasAttribute(a)) rect.setAttribute(a, firstPath.getAttribute(a));
+  }
+  firstPath.replaceWith(rect);
+});
+document.querySelectorAll('#erd svg.erDiagram .row-rect-odd path, #erd svg.erDiagram .row-rect-even path').forEach(p => {
+  p.setAttribute('stroke', 'none');
+});
+</script>