fix: trying to fix auto-pulls 12 !

.claude/settings.local.json

@@ -0,0 +1,16 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npm create:*)",
+      "Bash(npm install:*)",
+      "Bash(npm run build:*)",
+      "Bash(python -c:*)",
+      "Bash(npx vite build:*)",
+      "Bash(wc:*)",
+      "Bash(ls:*)",
+      "Bash(node -c:*)",
+      "Bash(npm run lint:*)",
+      "Bash(python:*)"
+    ]
+  }
+}

.env.example

@@ -13,8 +13,6 @@ MQTT_BROKER_PORT=1883
 MQTT_ADMIN_USERNAME=admin
 MQTT_ADMIN_PASSWORD=your-mqtt-admin-password
 MOSQUITTO_PASSWORD_FILE=/etc/mosquitto/passwd
-# Must be unique per running instance (VPS vs local dev)
-MQTT_CLIENT_ID=bellsystems-admin-panel
 # HMAC secret used to derive per-device MQTT passwords (must match firmware)
 MQTT_SECRET=change-me-in-production
 
@@ -25,13 +23,6 @@ DEBUG=true
 NGINX_PORT=80
 
 # Local file storage (override if you want to store data elsewhere)
-SQLITE_DB_PATH=./data/database.db
+SQLITE_DB_PATH=./mqtt_data.db
 BUILT_MELODIES_STORAGE_PATH=./storage/built_melodies
 FIRMWARE_STORAGE_PATH=./storage/firmware
-
-# Nextcloud WebDAV
-NEXTCLOUD_URL=https://cloud.example.com
-NEXTCLOUD_USERNAME=service-account@example.com
-NEXTCLOUD_PASSWORD=your-password-here
-NEXTCLOUD_DAV_USER=admin
-NEXTCLOUD_BASE_PATH=BellSystems/Console

.gitignore

@@ -1,8 +1,3 @@
-# Auto-deploy generated files
-deploy.sh
-deploy.log
-.deploy-trigger
-
 # Secrets
 .env
 firebase-service-account.json
@@ -12,11 +7,6 @@ firebase-service-account.json
 !/data/.gitkeep
 !/data/built_melodies/.gitkeep
 
-# SQLite databases
-*.db
-*.db-shm
-*.db-wal
-
 # Python
 __pycache__/
 *.pyc
@@ -38,6 +28,3 @@ Thumbs.db
 .MAIN-APP-REFERENCE/
 
 .project-vesper-plan.md
-
-# claude
-.claude/

.stitch/DESIGN.md

@@ -1,133 +0,0 @@
-# Design System: BellSystems Console Design
-**Project ID:** 18406618574074411899
-
----
-
-## 1. Visual Theme & Atmosphere
-
-**"The Digital Observatory"** — A high-fidelity, immersive enterprise command center aesthetic that rejects the typical boxed-in SaaS feel. The mood is best described as **Atmospheric Depth**: dark, spacious, and precision-focused. Inspired by high-end editorial design and command center interfaces, data is treated as a premium asset surfaced through tonal layering rather than structural lines.
-
-The base is built on **Midnight Navy** — a deep blue-black that evokes an infinite canvas — with UI elements that appear to float and glow rather than sit flat. Hierarchy is established exclusively through surface tone shifts; hard borders are forbidden. The result feels like peering through a high-resolution window into enterprise data, where clarity comes from contrast in depth rather than weight.
-
-Overall density is **medium-high** — information-rich layouts with generous vertical whitespace between elements, but no wasted screen real estate.
-
----
-
-## 2. Color Palette & Roles
-
-### Core Surfaces (darkest to lightest)
-| Name | Hex | Role |
-|---|---|---|
-| **Abyss** | `#0a0e14` | Deepest well; nested content backgrounds |
-| **Midnight** | `#10141a` | Main application viewport / page background |
-| **Void Navy** | `#181c22` | Sidebar, header, and secondary navigation surfaces |
-| **Deep Slate** | `#1c2026` | Default card and information module background |
-| **Elevated Slate** | `#262a31` | High cards, hovered table rows |
-| **Island** | `#31353c` | Active states, selected rows, top-layer containers |
-| **Frosted Glass** | `#353940` | Floating modals and dropdowns (at 80% opacity with blur) |
-
-### Accent & Brand Colors
-| Name | Hex | Role |
-|---|---|---|
-| **Indigo Glow** | `#c0c1ff` | Primary accent; CTA text, key metrics, active nav indicator |
-| **Lavender Soft** | `#8083ff` | Primary container fills |
-| **Violet Pulse** | `#d2bbff` | Secondary accent; gradient pair to Indigo Glow |
-| **Deep Indigo** | `#6001d1` | Secondary container fills |
-| **Royal Indigo** | `#494bd6` | Inverse primary; used on light-over-dark contexts |
-| **Aqua Sky** | `#7bd0ff` | Tertiary accent; data visualization, device status indicators |
-| **Ocean** | `#009bd1` | Tertiary container |
-
-### Semantic / State Colors
-| Name | Hex | Role |
-|---|---|---|
-| **Coral Error** | `#ffb4ab` | Error text and destructive action labels |
-| **Crimson** | `#93000a` | Error container backgrounds |
-| **Emerald** (Tailwind) | `emerald-*` | Online / active device status badges |
-| **Amber** (Tailwind) | `amber-*` | Warning / pending device status badges |
-
-### Text Colors
-| Name | Hex | Role |
-|---|---|---|
-| **Cloud** | `#dfe2eb` | Primary text; body copy, data values, headings |
-| **Mist** | `#c7c4d7` | Secondary / muted text; labels, metadata |
-| **Ghost** | `#908fa0` | Placeholder text, disabled states, divider fill |
-| **Boundary** | `#464554` | Ghost border fallback for inputs (used at low opacity) |
-
----
-
-## 3. Typography Rules
-
-**Single font family throughout: Inter** (geometric sans-serif). Used for headlines, body, and labels alike — unity is achieved through weight and tracking variation rather than font switching.
-
-| Level | Size | Weight | Tracking | Usage |
-|---|---|---|---|---|
-| **Display** | 56px / 3.5rem | Bold (700) | Tight (negative) | Hero KPI metrics, total counts |
-| **Headline** | 24px / 1.5rem | SemiBold (600) | Normal | Page titles, major section headings |
-| **Title** | 16px / 1.0rem | Medium (500) | Normal | Card titles, module headers, tab labels |
-| **Body** | 14px / 0.875rem | Regular (400) | Normal | Default text, table rows, descriptions |
-| **Label** | 11px / 0.6875rem | SemiBold (600) | Wide (+0.1em) | Sidebar category headers, metadata chips — All Caps |
-
-**The Editorial Rule:** Sidebar category headers use the Label style in all-caps with wide letter-spacing (+0.1em). This creates a "system-level" authority that contrasts with fluid body text and signals structural navigation. Body text must never be all-caps.
-
----
-
-## 4. Component Stylings
-
-### Buttons
-- **Primary CTA:** Gradient fill from Indigo Glow (`#c0c1ff`) to Violet Pulse (`#d2bbff`). White text. Softly rounded corners (matching `lg` radius — just barely rounded, not pill). On hover, a high-glow indigo shadow emanates beneath the button.
-- **Secondary / Outline:** Island background (`#31353c`) with an extremely faint ghost border — outline-variant (`#464554`) at 20–30% opacity. Mist text (`#c7c4d7`).
-- **Tertiary / Ghost:** Fully transparent background. Mist text. On hover: Cloud text with elevated background tint.
-- **Destructive:** Coral Error text (`#ffb4ab`) on a Crimson container (`#93000a` at low opacity). Used for delete and irreversible actions. Never bright red.
-
-### Cards & Containers
-- **Default Card:** Deep Slate background (`#1c2026`). No visible border — separation is purely tonal. Features a **subtle top-edge inner glow**: `inset 0px 1px 0px rgba(192, 193, 255, 0.05)` — mimics a ceiling light reflecting off the card's glass surface.
-- **Corner Rounding:** Minimal — just enough to soften, not enough to feel playful. Approximately 4px (the `lg` token = 0.25rem). Cards feel rectangular and purposeful, not bubbly.
-- **Elevation principle:** No drop shadows on standard cards. Hierarchy comes from the surface color step (Void Navy → Deep Slate → Elevated Slate).
-- **Floating Modals / Dropdowns:** Frosted Glass background (`rgba(53, 57, 64, 0.8)`) with `backdrop-filter: blur(12px)`. This "Glassmorphism" effect keeps the user contextually anchored to the underlying page while the modal floats above. Shadow: `0px 8px 24px rgba(13, 17, 23, 0.6)` — navy-tinted, never pure black.
-
-### Inputs & Forms
-- **Default state:** Deep Slate background. Ghost border at very low opacity (near invisible). Text in Cloud color.
-- **Focus state:** Indigo Glow (`#c0c1ff`) border at 40% alpha creates a soft halo glow — not a hard ring. The inner glow on the input also subtly strengthens.
-- **Placeholder text:** Ghost color (`#908fa0`).
-- **Select / Dropdown:** Same surface as inputs; opens as a Frosted Glass panel with blur.
-- **No hard outlines at rest** — inputs feel embedded in the surface until interacted with.
-
-### Status Badges
-- **Shape:** Fully pill-shaped (maximum border-radius / `full` = 0.75rem).
-- **Style:** Functional color (Emerald, Amber, Coral) at approximately 15% background opacity, with the same color at full 100% opacity for the label text. Creates a "glowing ink" effect — the badge appears illuminated from within.
-- **Examples:** Online → soft emerald glow; Warning → soft amber glow; Error/Offline → soft coral glow.
-
-### Navigation Sidebar (224px wide)
-- **Background:** Void Navy (`#181c22`) — one step lighter than the main viewport.
-- **Active item indicator:** A `3px` vertical bar on the far-left edge using Indigo Glow (`#c0c1ff`). The text weight also increases slightly. No background highlight on active items — the light bar IS the indicator.
-- **Inactive items:** Mist text (`#c7c4d7`) at normal weight.
-- **Category headers:** Label-style — all-caps, 11px, SemiBold, wide tracking. Ghost color (`#908fa0`).
-- **Item padding:** Comfortable vertical padding (~9.6px / 0.6rem) for breathing room between items.
-- **No dividers** between nav sections — spacing does the work.
-
-### Data Tables
-- **Row separation:** No horizontal dividers. Alternating subtle tonal rows (Island `#31353c` on hover) and consistent vertical gap rhythm.
-- **Header row:** Mist text (`#c7c4d7`), Label-style capitalization, slightly smaller than body.
-- **Hovered row:** Elevated Slate (`#262a31`) or Island (`#31353c`) background.
-- **Selected row:** Island background with Indigo Glow left border accent.
-
-### Scrollbars
-- Slim — 4px wide track.
-- Thumb: Boundary color (`#464554`), with 2px border-radius.
-- Track: Transparent.
-- Overall feel: Nearly invisible unless sought out.
-
----
-
-## 5. Layout Principles
-
-**Whitespace is structure.** The design never uses lines or dividers to separate sections — space does that job. When content feels disconnected, the solution is always to add vertical breathing room, never to draw a border.
-
-- **Page content area:** Fills the viewport to the right of the 224px sidebar. Padding inside the content area is generous — approximately 24–32px on all sides.
-- **Section spacing:** Major sections within a page are separated by approximately 24px of vertical space. Sub-sections by 16px.
-- **Card grid:** Cards sit in fluid grids with 16px gaps. Cards never touch each other.
-- **Alignment:** Strong left-edge alignment for all content. Data tables, card headers, and page titles all share the same left origin point.
-- **No horizontal rules / `<hr>` elements:** Surface color transitions and whitespace define the visual structure entirely.
-- **Modals:** Centered in the viewport, overlaid on a dark scrim. The page content behind is still readable through the frosted glass effect, maintaining spatial context.
-- **The "No Raw Border" rule:** Any element requiring a visible boundary for accessibility (e.g., an active input) must use the ghost border approach — Boundary color (`#464554`) at 20% opacity maximum. Full-opacity borders are strictly prohibited.
-- **Mobile / responsive:** The sidebar collapses to a drawer on narrow viewports. Cards reflow to single-column. The design's depth relies on background layers, so it translates naturally to smaller screens.

CLAUDE.md

@@ -1,383 +0,0 @@
-# CLAUDE.md — BellSystems Control Panel v2
-# Instructions for Claude Code
-
-> Read this file at the start of every session.
-> This is the v2 project — a clean rebuild. The old v1 code lives in `frontend/src/_archive/` for reference only.
-> Also read `DESIGN.md` before writing any UI code.
-
----
-
-## Project Structure
-
-```
-C:\development\bellsystems-cp-v2\
-│   CLAUDE.md                    ← you are here
-│   DESIGN.md                    ← design rules, component contracts, page layout spec
-│   docker-compose.yml
-│
-├── backend/                     ← FastAPI backend — DO NOT MODIFY
-│
-└── frontend/
-    ├── src/
-    │   ├── _archive/            ← v1 reference code — READ ONLY, never import from here except auth
-    │   ├── assets/
-    │   │   ├── global-icons/    ← action SVGs (edit, delete, download, etc.)
-    │   │   ├── side-menu-icons/ ← sidebar navigation SVGs
-    │   │   ├── comms/           ← communication type SVGs
-    │   │   ├── other-icons/     ← misc SVGs
-    │   │   └── customer-status/ ← CRM status SVGs
-    │   ├── components/
-    │   │   ├── ui/              ← design system components (the ONLY place to source UI)
-    │   │   ├── layout/          ← Sidebar, Header, MainLayout
-    │   │   └── shared/
-    │   ├── hooks/
-    │   ├── lib/
-    │   ├── modals/              ← all modal components live here, grouped by domain
-    │   ├── pages/               ← one file per page, grouped by domain
-    │   ├── providers/
-    │   ├── router/
-    │   │   └── index.jsx        ← all routes defined here
-    │   ├── styles/
-    │   │   ├── tokens.css       ← ALL design tokens (colors, fonts, spacing, shadows)
-    │   │   ├── components.css   ← ALL component-level styles
-    │   │   └── global.css       ← base resets, typography, scrollbar, .page-wrapper
-    │   └── main.jsx             ← app entry point
-    └── vite.config.js
-```
-
----
-
-## Project Overview
-
-Bespoke SaaS Admin Console for BellSystems. Manages Devices, Customers (CRM),
-Manufacturing, Firmware, MQTT, Melodies, Staff, and more.
-
-- **Backend:** FastAPI at `backend/` — never modify
-- **Archive:** `frontend/src/_archive/` — v1 reference, read-only
-- **Active code:** `frontend/src/` (everything except `_archive/`)
-- **Design rules:** `DESIGN.md` at project root — read before writing any UI
-- **Style Guide:** live at `/dev/styleguide` — shows every component with every variant
-- **API client:** `frontend/src/lib/api.js` wraps `_archive/api/client.js`
-
----
-
-## Import Alias
-
-`@/` maps to `frontend/src/`:
-
-```js
-import Button from '@/components/ui/Button'             ✅
-import Select from '@/components/ui/Select'             ✅
-import { useAuth } from '@/hooks/useAuth'               ✅
-import MainLayout from '@/components/layout/MainLayout' ✅
-```
-
-Never use relative `../` paths except inside `providers/` and `hooks/` when referencing `_archive/`.
-
----
-
-## The Golden Rules
-
-- **Never modify `_archive/`** — it is read-only reference material
-- **All new code goes in `frontend/src/`** — no exceptions
-- **No `/v2/` prefix anywhere** — routes start from `/`, imports start from `@/`
-- **Read `DESIGN.md` before writing any UI code**
-- **Source every UI element from `@/components/ui/`** — no raw HTML elements for styled things
-- **Use only CSS tokens** — never raw hex, rgb, or pixel values in component or page files
-- **Use `.masonry-grid` for all content pages with multiple variable-height sections** — never `display: grid` with fixed columns for card layouts. See DESIGN.md §11.
-
----
-
-## Available UI Components
-
-Every component lives in `frontend/src/components/ui/`. These are the ONLY components to use.
-Check the live Style Guide at `/dev/styleguide` to see all variants and states.
-
-| Component       | Import path                          | Purpose                                      |
-|-----------------|--------------------------------------|----------------------------------------------|
-| `Button`        | `@/components/ui/Button`             | All interactive actions                       |
-| `StatusBadge`   | `@/components/ui/StatusBadge`        | Coloured status pills                         |
-| `FormField`     | `@/components/ui/FormField`          | Every text/email/password/textarea input      |
-| `Select`        | `@/components/ui/Select`             | Custom dropdown (used inside FormField type="select") |
-| `Modal`         | `@/components/ui/Modal`              | All overlay dialogs                           |
-| `ConfirmDialog` | `@/components/ui/ConfirmDialog`      | Destructive / confirmation prompts            |
-| `DataTable`     | `@/components/ui/DataTable`          | All tabular data with sorting/selection       |
-| `Pagination`    | `@/components/ui/Pagination`         | Page controls beneath DataTable               |
-| `Spinner`       | `@/components/ui/Spinner`            | Loading indicators                            |
-| `PageHeader`    | `@/components/ui/PageHeader`         | Page title block — every page starts with this |
-| `Card`          | `@/components/ui/Card`               | Contained content sections                   |
-| `Tabs`          | `@/components/ui/Tabs`               | Tabbed navigation within a page              |
-| `Toast`         | `@/components/ui/Toast`              | Transient notifications (via `useToast`)     |
-| `SearchBar`     | `@/components/ui/SearchBar`          | Search inputs with debounce                  |
-| `Breadcrumbs`   | `@/components/ui/Breadcrumbs`        | Navigation trail on detail pages             |
-| `Icon`          | `@/components/ui/Icon`               | Inline SVG icons by name                     |
-
----
-
-## Folder Structure — Pages & Modals
-
-Folders mirror the sidebar section hierarchy exactly.
-
-```
-frontend/src/pages/
-├── auth/                        ← unauthenticated routes (login)
-├── dashboard/                   ← General section
-├── bellcloud/                   ← Bell Cloud section
-│   ├── devices/
-│   │   └── notes/
-│   ├── users/
-│   ├── melodies/
-│   │   └── archetypes/
-│   └── mqtt/
-├── crm/                         ← Headquarters section
-│   ├── comms/
-│   │   └── mail/
-│   ├── customers/
-│   │   └── tabs/
-│   ├── orders/
-│   ├── quotations/
-│   └── products/
-├── engineering/                 ← Engineering section
-│   ├── manufacturing/
-│   ├── firmware/
-│   └── developer/
-├── public/                      ← Public / unauthenticated pages
-│   ├── cloudflash/
-│   └── serial/
-├── settings/                    ← Console Settings section
-│   └── staff/
-└── dev/                         ← Internal dev tools (StyleGuide)
-```
-
-```
-frontend/src/modals/
-├── bellcloud/
-│   ├── devices/
-│   ├── melodies/
-│   └── users/
-├── crm/
-│   └── products/
-├── engineering/
-│   └── manufacturing/
-└── shared/
-```
-
----
-
-## Page Layout — How Every Page Is Structured
-
-Every authenticated page is rendered inside `MainLayout`, which provides:
-- **Sidebar** — fixed left, `224px` wide (`--sidebar-width`)
-- **Header** — fixed top, `56px` tall (`--header-height`)
-- **Content area** — the remaining viewport space
-
-Inside the content area, every page uses the `.page-wrapper` class (defined in `global.css`):
-
-```css
-.page-wrapper {
-  flex: 1;
-  display: flex;
-  flex-direction: column;
-  padding: var(--space-12);     /* 48px on all sides — desktop */
-  gap: var(--space-6);          /* 24px between top-level sections */
-  min-width: 0;
-}
-
-@media (max-width: 768px) {
-  .page-wrapper {
-    padding: var(--space-8);    /* 32px on mobile */
-    gap: var(--space-4);
-  }
-}
-```
-
-**This is the consistency guarantee**: because every page uses `.page-wrapper`, the `<PageHeader>` title on every page starts at exactly the same position — 48px from the top and 48px from the left edge of the content area. Never override these paddings. Never add extra wrappers around `page-wrapper` that introduce additional offset.
-
-### Content width modes
-
-Pages fall into two modes — choose based on how much content the page has:
-
-**Full-width** (default — lists, tables, dashboards):
-```jsx
-<div className="page-wrapper">
-```
-
-**Centered** (forms, settings, pages with very few items):
-```jsx
-<div className="page-wrapper page-wrapper--centered">
-```
-Centered mode caps each direct child at `--content-max-width-sm` (640px) by default and centers it horizontally. Override when needed:
-```jsx
-<div className="page-wrapper page-wrapper--centered"
-     style={{ '--page-content-max-width': 'var(--content-max-width-md)' }}>
-```
-Available tokens: `--content-max-width-xs` (480px), `--content-max-width-sm` (640px), `--content-max-width-md` (800px), `--content-max-width-lg` (1024px).
-
----
-
-## Rules for Every Page
-
-### Before writing any code
-1. Read `DESIGN.md` — confirm tokens and components to use
-2. Check `frontend/src/components/ui/` — use existing components only
-3. Check the Style Guide at `/dev/styleguide` for the correct variant/props
-4. Check `frontend/src/_archive/` for the equivalent v1 page — copy API calls and data shape only, never styling
-
-### Page template
-
-```jsx
-// frontend/src/pages/[domain]/PageName.jsx
-
-import PageHeader from '@/components/ui/PageHeader'
-import { useAuth } from '@/hooks/useAuth'
-// Import ONLY from @/components/ui/ — never raw HTML elements for styled things
-
-export default function PageName() {
-  // 1. Auth
-  const { user } = useAuth()
-
-  // 2. State & data fetching
-
-  // 3. Event handlers
-
-  // 4. Render — always handle: loading, error, empty, data states
-  return (
-    <div className="page-wrapper">
-      <PageHeader title="Page Title" subtitle="Optional description">
-        {/* Action buttons — use <Button variant="primary"> etc. */}
-      </PageHeader>
-
-      {/* Page content — use Card, DataTable, Tabs, etc. */}
-    </div>
-  )
-}
-```
-
-### Styling rules
-- **Tailwind** for layout only: `flex`, `grid`, `items-center`, `min-w-0`, etc.
-- **CSS token variables** for ALL colors, spacing, typography — `var(--token-name)`
-- **No** `.module.css` or per-page scoped CSS files
-- **No** `style={{ }}` inline styles except for genuinely dynamic values (e.g. calculated widths)
-- **No** raw hex, rgb, or pixel values anywhere
-
-### Toolbar buttons — matching SearchBar height
-
-`.btn` has `line-height: 1` while `.searchbar-input` has `line-height: var(--line-height-base)` (1.5). This makes buttons shorter than the search bar by default. Whenever a `<Button>`, `<SegmentedControl>`, or `<IconButtonGroup>` sits in the same toolbar row as a `<SearchBar>`, all buttons must use `padding-top/bottom: var(--space-3)` and `line-height: var(--line-height-base)`.
-
-**Already handled automatically (no extra props needed):**
-- `SegmentedControl` — `.seg-ctrl__btn` in `components.css` enforces `--space-3` padding and `var(--line-height-base)` globally.
-- `IconButtonGroup` — `.icon-btn-group__btn` in `components.css` enforces `--space-3` padding globally.
-
-**Must be overridden manually — standalone `<Button>` in the same row as a `<SearchBar>`:**
-
-```jsx
-// Option A — inline style prop on the button:
-<Button
-  size="md"
-  style={{ paddingTop: 'var(--space-3)', paddingBottom: 'var(--space-3)', lineHeight: 'var(--line-height-base)' }}
->
-  Label
-</Button>
-
-// Option B (preferred when multiple buttons share a toolbar) — scoped CSS block:
-<>
-  <style>{`
-    .my-toolbar .btn {
-      padding-top: var(--space-3) !important;
-      padding-bottom: var(--space-3) !important;
-      line-height: var(--line-height-base) !important;
-    }
-  `}</style>
-  <div className="my-toolbar" style={{ display: 'flex', alignItems: 'center', gap: 'var(--space-2)' }}>
-    <IconButtonGroup … />
-    <Button variant="primary" size="md">Compose</Button>
-  </div>
-</>
-```
-
-### Date & time formatting
-- **Greek/European date style everywhere** — DD/MM/YYYY, never US-style MM/DD/YYYY
-- **All date/time formatting must use `@/lib/formatters`** — never use raw `toLocaleDateString()`, `Intl.DateTimeFormat`, `toLocaleString()`, or `toISOString().slice()` in pages or modals
-- **For `datetime-local` input values** — use `toDatetimeLocal(iso)` and `nowLocal()` from formatters. Never use `new Date(x).toISOString().slice(0, 16)` — it converts to UTC and shifts the time (timezone bug)
-- **Currency** — use `fmtEuro(n)` from formatters (Greek locale: `1.250,00 €`)
-
-Available formatters (`import { ... } from '@/lib/formatters'`):
-
-| Function            | Output example                     | Use for                              |
-|---------------------|------------------------------------|--------------------------------------|
-| `fmtDate`           | `05/03/2026`                       | Short numeric dates (tables, lists)  |
-| `fmtDateMedium`     | `5 Mar 2026`                       | Medium dates (cards, details)        |
-| `fmtDateLong`       | `5 March 2026`                     | Long dates (headings, summaries)     |
-| `fmtDateFull`       | `Wednesday, 5 March 2026`          | Dashboard, full context              |
-| `fmtDateTime`       | `5 March 2026, 2:30 pm`            | Date + 12h time                      |
-| `fmtDateTimeMedium` | `5 Mar 2026, 14:30`                | Date + 24h time (compact)            |
-| `fmtDateTimeFull`   | `Wed, 5 Mar 2026, 2:30 pm`         | Emails, comms                        |
-| `fmtTime24`         | `14:30:05`                         | Time with seconds                    |
-| `fmtRelative`       | `5 minutes ago`                    | Relative timestamps                  |
-| `toDatetimeLocal`   | `2026-03-05T14:30`                 | `datetime-local` input values        |
-| `nowLocal`          | `2026-03-05T14:30`                 | Current time for form defaults       |
-| `toDateInput`       | `2026-03-05`                       | `date` input values                  |
-| `fmtEuro`           | `1.250,00 €`                       | Euro currency                        |
-
-### Data fetching
-- Use `frontend/src/lib/api.js` (wraps `_archive/api/client.js`)
-- Every data-fetching component must handle **loading**, **error**, and **empty** states
-
-### Modals
-- Never defined inside page files
-- Live in `frontend/src/modals/[sidebar-section]/[domain]/ModalName.jsx` — mirror the pages folder structure
-- Pass data via props, actions via callbacks
-
----
-
-## Missing Components — Stop and Ask
-
-If building a page requires a UI component that does not exist in
-frontend/src/components/ui/, Claude Code must STOP and say:
-
-"I need a [ComponentName] component which doesn't exist yet.
-Please build it and add it to the StyleGuide before I continue."
-
-Do NOT:
-- Invent an inline one-off component inside a page file
-- Use raw HTML elements styled with inline CSS as a substitute
-- Proceed and leave a placeholder
-
-The StyleGuide at frontend/src/pages/dev/StyleGuide.jsx is the source
-of truth for what components exist and how they look. Every component
-used in a page must have a visible example there first.
-
-## Building a New Page — Checklist
-
-- [ ] File in correct `frontend/src/pages/[domain]/` folder
-- [ ] Root element is `<div className="page-wrapper">` — nothing else, nothing wrapping it
-- [ ] First child inside `page-wrapper` is `<PageHeader title="...">`
-- [ ] Only components from `frontend/src/components/ui/` used
-- [ ] No raw hex colors or pixel spacing values anywhere
-- [ ] Loading state implemented
-- [ ] Error state implemented
-- [ ] Empty state implemented
-- [ ] Mobile responsive (375px minimum)
-- [ ] Modals in `frontend/src/modals/`
-- [ ] Route added to `frontend/src/router/index.jsx`
-
----
-
-## API Client
-
-```js
-// frontend/src/lib/api.js
-export { default } from '../_archive/api/client'
-```
-
-All pages import from `@/lib/api`, never directly from `_archive`.
-
----
-
-## Auth
-
-`frontend/src/hooks/useAuth.js` re-exports from the archive AuthContext.
-`frontend/src/providers/AuthProvider.jsx` re-exports the AuthProvider.
-
-These are the ONLY two files permitted to import from `_archive/auth/`.
-All other files use `@/hooks/useAuth`.

DESIGN.md

@@ -1,627 +0,0 @@
-# DESIGN SYSTEM — BellSystems Control Panel v2
-
-> Single source of truth for all UI/UX decisions.
-> Read before writing any page, component, or modal.
-> Never override these rules inline. Change the rule here first, then propagate.
->
-> Live reference: `/dev/styleguide` — every component, every variant, every state.
-
----
-
-## 1. Core Philosophy
-
-- **Consistency over creativity.** Every page must feel like it belongs to the same product.
-- **Tokens over hardcoded values.** Never write a raw color, spacing value, or font size. Always `var(--token)`.
-- **Components over repetition.** If you write the same pattern twice, it becomes a shared component.
-- **Page layout is global.** All pages share the same padding and spacing anchors. Content always starts at the same position.
-- **Accessible by default.** ARIA labels, keyboard navigation, visible focus states on all interactive elements.
-
----
-
-## 2. Page Layout Anatomy
-
-Every authenticated page lives inside `MainLayout`, which provides:
-
-```
-┌─────────────┬────────────────────────────────────────────┐
-│             │  HEADER  (height: 56px / --header-height)  │
-│             │────────────────────────────────────────────┤
-│             │                                            │
-│   SIDEBAR   │          CONTENT AREA                      │
-│  (224px /   │  ┌──────────────────────────────────────┐  │
-│ --sidebar-  │  │  .page-wrapper                       │  │
-│   width)    │  │  padding: 48px (--space-12)          │  │
-│             │  │  gap: 24px between sections          │  │
-│             │  │                                      │  │
-│             │  │  <PageHeader>   ← always first       │  │
-│             │  │  <content...>                        │  │
-│             │  └──────────────────────────────────────┘  │
-└─────────────┴────────────────────────────────────────────┘
-```
-
-### The consistency guarantee
-
-`.page-wrapper` is defined once in `global.css`. It is the only wrapper used on every page:
-
-```css
-.page-wrapper {
-  flex: 1;
-  display: flex;
-  flex-direction: column;
-  padding: var(--space-12);  /* 48px — desktop */
-  gap: var(--space-6);       /* 24px between direct children */
-  min-width: 0;
-}
-/* Mobile: padding drops to --space-8 (32px), gap to --space-4 (16px) */
-```
-
-**Rules:**
-- Every page's root element is `<div className="page-wrapper">` — no exceptions
-- Never add extra padding, margin, or wrapper divs that shift content relative to `.page-wrapper`
-- Never override `.page-wrapper` padding per-page
-- The `<PageHeader>` is always the first child inside `.page-wrapper`
-- This ensures that on every page, the title starts at exactly 48px from the top-left corner of the content area
-
-### Content width modes
-
-Every page falls into one of two modes:
-
-#### 1. Full-width (default)
-Content expands to fill the entire available area. Use this for all data-heavy pages: lists, tables, dashboards, detail views.
-
-```jsx
-<div className="page-wrapper">
-  {/* content fills the full content area */}
-</div>
-```
-
-#### 2. Centered (narrow content)
-For pages with a small number of elements that would look lost spanning the full viewport — e.g. settings forms, auth pages, single-entity configuration screens.
-
-```jsx
-<div className="page-wrapper page-wrapper--centered">
-  {/* every direct child is capped at --content-max-width-sm (640px) and centered */}
-</div>
-```
-
-Default max-width is `--content-max-width-sm` (640px). Override per-page only when necessary:
-
-```jsx
-<div
-  className="page-wrapper page-wrapper--centered"
-  style={{ '--page-content-max-width': 'var(--content-max-width-md)' }}
->
-```
-
-Available width tokens:
-
-| Token                       | Value  | Use                                     |
-|-----------------------------|--------|-----------------------------------------|
-| `--content-max-width-xs`    | 480px  | Tiny forms, login, auth                 |
-| `--content-max-width-sm`    | 640px  | Small forms, simple settings (default)  |
-| `--content-max-width-md`    | 800px  | Medium forms, detail-light pages        |
-| `--content-max-width-lg`    | 1024px | Moderate-width constrained pages        |
-
-**Rules:**
-- Never use `page-wrapper--centered` on a list page, data table page, or any page where content should grow with the viewport
-- Never hardcode a pixel `max-width` in a page file — always use a token
-
----
-
-## 3. Color Tokens
-
-All colors are CSS custom properties defined in `frontend/src/styles/tokens.css`.
-The system is **dark-first**: `:root` = dark theme. `[data-theme="light"]` overrides exist as a placeholder.
-
-### Rule: never write a raw color value in any component or page file. Always `var(--token)`.
-
-### Background Surfaces (7-step tonal ladder)
-
-| Token                  | Value        | Use                                                    |
-|------------------------|--------------|--------------------------------------------------------|
-| `--color-bg-abyss`     | `#0a0e14`    | Deepest well: code blocks, input backgrounds           |
-| `--color-bg-base`      | `#10141a`    | Page background (viewport fill)                        |
-| `--color-bg-void`      | `#181c22`    | Sidebar, header                                        |
-| `--color-bg-surface`   | `#1c2026`    | Default card / panel background                        |
-| `--color-bg-elevated`  | `#262a31`    | Raised cards, hovered rows, dropdowns                  |
-| `--color-bg-island`    | `#31353c`    | Active states, selected rows, pressed buttons          |
-| `--color-bg-float`     | `rgba(53,57,64,0.80)` | Glassmorphism: modals, floating panels      |
-
-### Brand / Primary (Indigo Glow)
-
-| Token                      | Value                        | Use                                  |
-|----------------------------|------------------------------|--------------------------------------|
-| `--color-primary`          | `#c0c1ff`                    | CTAs, active nav, key accent          |
-| `--color-primary-hover`    | `#d2bbff`                    | Hover, gradient endpoint              |
-| `--color-primary-container`| `#8083ff`                    | Container fills                       |
-| `--color-primary-subtle`   | `rgba(128,131,255,0.12)`     | Hover backgrounds, tinted areas       |
-| `--gradient-primary`       | `linear-gradient(135deg, #c0c1ff, #d2bbff)` | Primary button fill   |
-
-### Semantic / State Colors
-
-| Token                  | Value                     | Use                                            |
-|------------------------|---------------------------|------------------------------------------------|
-| `--color-success`      | `#4ade80`                 | Online, active, confirmed                      |
-| `--color-success-bg`   | `rgba(74,222,128,0.12)`   | Success badge / button resting background      |
-| `--color-warning`      | `#fbbf24`                 | Pending, needs attention                       |
-| `--color-warning-bg`   | `rgba(251,191,36,0.12)`   | Warning badge / button resting background      |
-| `--color-danger`       | `#ff5c5c`                 | Error text, destructive actions                |
-| `--color-danger-bg`    | `rgba(255,92,92,0.12)`    | Danger badge / button resting background       |
-| `--color-info`         | `#7bd0ff`                 | Informational, aqua-sky accent                 |
-| `--color-info-bg`      | `rgba(123,208,255,0.12)`  | Info badge background                          |
-
-### Text Colors (4-step hierarchy)
-
-| Token                     | Value       | Use                                               |
-|---------------------------|-------------|---------------------------------------------------|
-| `--color-text-primary`    | `#dfe2eb`   | Body copy, data values, headings                  |
-| `--color-text-secondary`  | `#c7c4d7`   | Labels, metadata, inactive nav                    |
-| `--color-text-muted`      | `#908fa0`   | Placeholders, disabled, category headers          |
-| `--color-text-inverse`    | `#10141a`   | Text on primary/accent backgrounds (dark on light)|
-| `--color-text-accent`     | `#c0c1ff`   | Active nav items, links                           |
-
-### Borders
-
-| Token                   | Value                      | Use                                        |
-|-------------------------|----------------------------|--------------------------------------------|
-| `--color-border`        | `rgba(70,69,84,0.20)`      | Resting inputs, card outlines              |
-| `--color-border-strong` | `rgba(70,69,84,0.45)`      | Secondary buttons, stronger dividers       |
-| `--color-border-focus`  | `rgba(192,193,255,0.40)`   | Focus ring halo on inputs                  |
-
----
-
-## 4. Typography
-
-Two-font system. Three families total.
-
-### Font Families
-
-| Token                    | Font                | Role                                              |
-|--------------------------|---------------------|---------------------------------------------------|
-| `--font-family-display`  | `Barlow Condensed`  | H1, H2, page titles, modal titles                 |
-| `--font-family-base`     | `Onest`             | All UI text, body, labels, buttons, table rows    |
-| `--font-family-mono`     | `JetBrains Mono`    | Serial numbers, IDs, code, API keys, terminal     |
-
-**Why this pairing:**
-- `Barlow Condensed` has an industrial/engineering quality — feels like instrument panel labelling. Makes page titles immediately distinctive.
-- `Onest` is a Ukrainian geometric grotesque with slightly unusual proportions and excellent numerics. Clean at 14px. Not the overused Inter/Space Grotesk.
-- `JetBrains Mono` is the standard for developer-facing data.
-
-### Font Sizes
-
-| Token              | Value      | Use                                          |
-|--------------------|------------|----------------------------------------------|
-| `--font-size-xs`   | `0.6875rem` (11px) | Labels, sidebar category headers, chips |
-| `--font-size-sm`   | `0.75rem` (12px)   | Captions, helper text, table headers    |
-| `--font-size-base` | `0.875rem` (14px)  | Body text, table rows (default)         |
-| `--font-size-md`   | `1rem` (16px)      | Card titles, module headers             |
-| `--font-size-lg`   | `1.125rem` (18px)  | Section subheadings                     |
-| `--font-size-xl`   | `1.5rem` (24px)    | Page headings (h1/h2)                   |
-| `--font-size-2xl`  | `3.5rem` (56px)    | Hero KPI numbers, dashboard metrics     |
-
-### Font Weights
-
-| Token                     | Value | Use                                    |
-|---------------------------|-------|----------------------------------------|
-| `--font-weight-normal`    | 400   | Body copy                              |
-| `--font-weight-medium`    | 500   | Emphasized body, table values          |
-| `--font-weight-semibold`  | 600   | Headings, button labels, field labels  |
-| `--font-weight-bold`      | 700   | Strong emphasis, hero metrics          |
-
-### Typography Usage Rules
-
-- **Page titles (`<PageHeader>`):** `Barlow Condensed`, `1.75rem`, weight 600 (handled by `.v2-page-header-title`)
-- **Modal titles:** `Barlow Condensed`, `1.125rem`, weight 600 (handled by `.v2-modal-title`)
-- **H1, H2 globally:** `Barlow Condensed`, `var(--font-size-xl)`, weight 600 — set in `global.css`
-- **H3–H6:** `Onest` (body font), normal heading weights
-- **Card titles:** `Onest`, `--font-size-base`, weight 600
-- **Table headers:** `Onest`, `--font-size-sm`, weight 600, uppercase, `--tracking-wide`
-- **Body / cell text:** `Onest`, `--font-size-base`, weight 400
-- **Muted / helper text:** `Onest`, `--font-size-sm`, `--color-text-muted`
-- **Serials, IDs, codes:** `JetBrains Mono`, `--font-size-sm`
-
-### Letter Spacing
-
-| Token               | Value     | Use                                        |
-|---------------------|-----------|--------------------------------------------|
-| `--tracking-normal` | `0em`     | Default                                    |
-| `--tracking-tight`  | `-0.01em` | Barlow Condensed headings                  |
-| `--tracking-wide`   | `0.08em`  | Uppercase labels, sidebar category headers |
-| `--tracking-display`| `-0.02em` | Hero KPI numbers at 56px                   |
-
----
-
-## 4b. Date, Time & Currency Formatting
-
-All dates use **Greek/European style** (day-first). Never use US-style MM/DD/YYYY anywhere in the app.
-
-All formatting is centralized in `frontend/src/lib/formatters.js`. Never use raw `toLocaleDateString()`, `Intl.DateTimeFormat`, `toLocaleString()`, or `toISOString().slice()` in pages or modals — always import from `@/lib/formatters`.
-
-### Available formatters
-
-| Function            | Output example                     | Use for                              |
-|---------------------|------------------------------------|--------------------------------------|
-| `fmtDate`           | `05/03/2026`                       | Short numeric dates (tables, lists)  |
-| `fmtDateMedium`     | `5 Mar 2026`                       | Medium dates (cards, details)        |
-| `fmtDateLong`       | `5 March 2026`                     | Long dates (headings, summaries)     |
-| `fmtDateFull`       | `Wednesday, 5 March 2026`          | Dashboard, full context              |
-| `fmtDateTime`       | `5 March 2026, 2:30 pm`            | Date + 12h time                      |
-| `fmtDateTimeMedium` | `5 Mar 2026, 14:30`                | Date + 24h time (compact)            |
-| `fmtDateTimeFull`   | `Wed, 5 Mar 2026, 2:30 pm`         | Emails, comms                        |
-| `fmtRelative`       | `5 minutes ago`                    | Relative timestamps                  |
-| `fmtEuro`           | `1.250,00 €`                       | Euro currency (Greek locale)         |
-
-### Form input helpers
-
-| Function            | Output example          | Use for                                          |
-|---------------------|-------------------------|--------------------------------------------------|
-| `toDatetimeLocal`   | `2026-03-05T14:30`      | Populating `datetime-local` inputs (local time)  |
-| `nowLocal`          | `2026-03-05T14:30`      | Current time for form defaults                   |
-| `toDateInput`       | `2026-03-05`            | Populating `date` inputs                         |
-
-### Critical rule: no `toISOString().slice()` for form inputs
-
-`toISOString()` converts to **UTC**, which shifts the time by the user's timezone offset (e.g. 3 hours for Greece). Always use `toDatetimeLocal()` or `nowLocal()` instead.
-
----
-
-## 5. Spacing System
-
-4px base unit. All spacing must use tokens — no arbitrary pixel values.
-
-| Token        | Value  | Common use                                       |
-|--------------|--------|--------------------------------------------------|
-| `--space-1`  | 4px    | Tight gaps, icon padding                         |
-| `--space-2`  | 8px    | Between label and input, inline gaps             |
-| `--space-3`  | 12px   | Table cell padding, compact button padding       |
-| `--space-4`  | 16px   | Between form fields, mobile page padding         |
-| `--space-5`  | 20px   | Tab item spacing                                 |
-| `--space-6`  | 24px   | **Page padding**, card padding, section gap      |
-| `--space-8`  | 32px   | Between major sections                           |
-| `--space-10` | 40px   | Large section gap                                |
-| `--space-12` | 48px   | Extra large spacing                              |
-| `--space-16` | 64px   | Maximum spacing, hero sections                   |
-
----
-
-## 6. Border Radius & Shadows
-
-### Border Radius
-
-| Token          | Value    | Use                                       |
-|----------------|----------|-------------------------------------------|
-| `--radius-sm`  | 4px      | Tags, small chips, select option rows     |
-| `--radius-md`  | 6px      | Buttons, inputs, table badges             |
-| `--radius-lg`  | 8px      | Cards, panels, dropdown menus             |
-| `--radius-xl`  | 12px     | Modals, large containers                  |
-| `--radius-full`| 9999px   | Status badge pills, avatars               |
-
-### Shadows
-
-| Token                   | Value                                    | Use                                |
-|-------------------------|------------------------------------------|------------------------------------|
-| `--shadow-card`         | `inset 0 1px 0 rgba(192,193,255,0.05)`  | Card top-edge glass reflection     |
-| `--shadow-sm`           | `0 2px 8px rgba(10,14,20,0.40)`         | Subtle lift                        |
-| `--shadow-md`           | `0 4px 16px rgba(10,14,20,0.50)`        | Elevated cards                     |
-| `--shadow-lg`           | `0 8px 24px rgba(13,17,23,0.60)`        | Modals, dropdowns                  |
-| `--shadow-focus`        | `0 0 0 3px rgba(192,193,255,0.20)`      | Focus ring glow                    |
-| `--shadow-primary-glow` | `0 4px 16px rgba(192,193,255,0.28)`     | Primary button hover halo          |
-| `--shadow-danger-glow`  | `0 4px 16px rgba(255,92,92,0.40)`       | Danger button hover halo           |
-| `--shadow-success-glow` | `0 4px 16px rgba(74,222,128,0.35)`      | Success button hover halo          |
-
----
-
-## 7. Component Rules
-
-### Button
-
-Import: `@/components/ui/Button`
-
-**Variants:**
-
-| Variant          | Resting state                         | Hover state                                              |
-|------------------|---------------------------------------|----------------------------------------------------------|
-| `primary`        | Indigo→violet gradient, dark text     | `+brightness(1.06)` + `--shadow-primary-glow` halo      |
-| `secondary`      | Island bg, ghost border               | Elevated bg + focus border + subtle indigo glow          |
-| `ghost`          | Transparent                           | Elevated bg + whisper indigo glow                        |
-| `danger`         | Coral tint bg, coral text             | **Solid coral fill**, dark text + `--shadow-danger-glow` |
-| `success`        | Emerald tint bg, emerald text         | **Solid emerald fill**, dark text + `--shadow-success-glow` |
-| `table-actions`  | Fully transparent, muted text         | Island bg + strong border (identical to `secondary`) — also activates on `tr:hover` |
-
-**Sizes:** `sm`, `md` (default), `lg`
-
-**Rules:**
-- Never use a raw `<button>` element for a styled action
-- Always pass `loading` prop for async actions (shows spinner, disables interaction)
-- Icon-only buttons must have `aria-label`
-- Active/press state: `filter: brightness(0.94)`, glow removed
-
----
-
-### FormField
-
-Import: `@/components/ui/FormField`
-
-Wraps every form control: label + input/textarea/select + hint + error message.
-Never place a raw `<input>` on a page.
-
-**Types:** `text`, `email`, `password`, `number`, `tel`, `url`, `textarea`, `select`
-
-**For `type="select"`**: pass `<option>` elements as children. FormField uses the custom `Select` component internally — the native `<select>` is never rendered.
-
-```jsx
-<FormField label="Status" name="status" type="select" value={val} onChange={handleChange}>
-  <option value="">Choose…</option>
-  <option value="active">Active</option>
-  <option value="inactive">Inactive</option>
-</FormField>
-```
-
-**Input appearance:** "cutout" inset-shadow treatment — the field appears recessed into the surface. Background: `--color-bg-abyss`. Focus: `--color-border-focus` ring.
-
----
-
-### Select (standalone)
-
-Import: `@/components/ui/Select`
-
-Fully custom dropdown replacing native `<select>`. Floating menu via portal, keyboard navigation, checkmark on selected item. Usually consumed via `FormField type="select"`. Use directly when you need a select outside a form label context.
-
----
-
-### DataTable
-
-Import: `@/components/ui/DataTable`
-
-**Always include:** column headers, loading skeleton, empty state, pagination.
-**Rows:** alternate tint via `--color-tint-row` (`rgba(192,193,255,0.015)`). Hover: `--color-bg-island`.
-**Status columns:** always `<StatusBadge>` — never raw text.
-**Row actions:** last column, right-aligned, use portal-based action menu.
-
----
-
-### Modal
-
-Import: `@/components/ui/Modal`
-
-Sizes: `sm` (480px), `md` (640px — default), `lg` (800px), `xl` (60vw/60vh), `xxl` (85vw/85vh), `full` (calc(100vw/100vh − 64px)).
-
-**Rules:**
-- Always has: title, close (×) button, footer action buttons
-- Closes on Escape + backdrop click unless `persistent={true}`
-- Destructive prompts use `<ConfirmDialog>` instead
-- Modal JSX never lives inside a page file — always in `frontend/src/modals/[domain]/`
-
----
-
-### ConfirmDialog
-
-Import: `@/components/ui/ConfirmDialog`
-
-Wraps `<Modal size="sm">` with a centred icon + message. Use for any action that is destructive or hard to reverse.
-
-Variants: `danger` (coral circle + triangle icon), `primary` (indigo circle + info icon).
-
----
-
-### PageHeader
-
-Import: `@/components/ui/PageHeader`
-
-**Always the first element inside `.page-wrapper`.** Creates the page title block.
-
-Props: `title` (required), `subtitle`, `breadcrumbs`, `children` (action buttons slot).
-
-The title renders as `<h1>` with class `.v2-page-header-title` — uses `Barlow Condensed` at `1.75rem` / weight 600.
-
-```jsx
-<PageHeader title="Device Inventory" subtitle="All registered Bell units">
-  <Button variant="primary">Add Device</Button>
-</PageHeader>
-```
-
----
-
-### Card
-
-Import: `@/components/ui/Card`
-
-Variants: `flat` (default), `elevated`, `outlined`.
-
-Props: `title`, `subtitle`, `footer`, `padding` (bool, default true), `children`.
-
-Card header has a faint indigo gradient ceiling (`linear-gradient` from top).
-
----
-
-### Tabs
-
-Import: `@/components/ui/Tabs`
-
-Variants: `line` (default — underline indicator), `pill` (filled background).
-
-Props: `tabs` (array of `{key, label, icon?, count?}`), `active`, `onChange`, `variant`.
-
-Line variant uses a sliding indicator measured with `useLayoutEffect`. Pill variant uses filled backgrounds.
-
-Spacing: line tabs have `gap: --space-5` between items, pill tabs `gap: --space-4`.
-
----
-
-### Toast
-
-Import: `@/components/ui/Toast` → `{ ToastProvider, useToast }`
-
-Setup: wrap the app (or router) with `<ToastProvider>`. Then in any component:
-
-```jsx
-const toast = useToast()
-toast.success('Saved', 'Device updated successfully.')
-toast.danger('Error', 'Failed to connect.')
-toast.warning('Warning', 'Firmware is outdated.')
-toast.info('Info', 'Sync in progress.')
-```
-
-Toasts auto-dismiss after 4000ms. Hover pauses the timer. Stack appears in the bottom-right corner.
-
----
-
-### SearchBar
-
-Import: `@/components/ui/SearchBar`
-
-Supports controlled (`value` + `onChange`) or uncontrolled mode.
-Debounced by default (300ms). Clear button appears when text is present.
-Appearance matches the `FormField` cutout treatment.
-
----
-
-### Breadcrumbs
-
-Import: `@/components/ui/Breadcrumbs`
-
-Use on detail pages only (not list pages). Items: array of `{ label, href? }`. Last item has no href — it is the current page.
-
----
-
-### Spinner
-
-Import: `@/components/ui/Spinner`
-
-Props: `size` (`sm`, `md`, `lg`), `color` (defaults to `--color-primary`).
-
-Use inside loading states. Buttons show their own spinner via `loading` prop — do not add a separate `<Spinner>` inside buttons.
-
----
-
-### StatusBadge
-
-Import: `@/components/ui/StatusBadge`
-
-Never use a raw `<span>` with a background color for status. Always `<StatusBadge>`.
-
-Variants: `success`, `warning`, `danger`, `info`, `neutral`.
-
----
-
-### Icon
-
-Import: `@/components/ui/Icon`
-
-Renders an inline SVG by name. 35 named icons available (see Style Guide `/dev/styleguide` → Icon section for the full list).
-
-```jsx
-<Icon name="edit" size={16} />
-<Icon name="delete" size={20} color="var(--color-danger)" />
-```
-
-**Asset SVGs** (from `/assets/` folders) are displayed via `<img>` tags in the Style Guide, not via `<Icon>`. These are pre-rendered SVG files used for sidebar icons, comms icons, customer status icons, etc. Use them as image sources, not as Icon component names.
-
----
-
-## 8. Icons
-
-Three sources:
-
-| Source                              | Use case                              | How to render               |
-|-------------------------------------|---------------------------------------|-----------------------------|
-| `<Icon name="..." />`               | Action icons, UI chrome               | `@/components/ui/Icon`      |
-| `assets/side-menu-icons/*.svg`      | Sidebar navigation                    | `<img src={...} />`         |
-| `assets/comms/*.svg`                | Communication type indicators         | `<img src={...} />`         |
-| `assets/customer-status/*.svg`      | CRM status icons                      | `<img src={...} />`         |
-| `assets/global-icons/*.svg`         | Legacy action icons (prefer `<Icon>`) | `<img src={...} />`         |
-| `assets/other-icons/*.svg`          | Misc UI icons                         | `<img src={...} />`         |
-
-Never add a new icon library (e.g. heroicons, lucide). Use the existing sources.
-
----
-
-## 9. Theming Rules
-
-- Theme is controlled by `data-theme` attribute on `<html>`
-- Default is dark (`:root` = dark theme)
-- **Never** use Tailwind's `dark:` prefix — theming is handled entirely via CSS tokens
-- `[data-theme="light"]` overrides exist in `tokens.css` as a future placeholder
-
----
-
-## 10. Responsive Breakpoints
-
-| Token              | Value  | Behaviour                                              |
-|--------------------|--------|--------------------------------------------------------|
-| `--breakpoint-sm`  | 640px  |                                                        |
-| `--breakpoint-md`  | 768px  | Sidebar collapses; page padding drops to `--space-4`  |
-| `--breakpoint-lg`  | 1024px | Full sidebar shown                                     |
-| `--breakpoint-xl`  | 1280px |                                                        |
-
-Mobile (`< 768px`): single column, sidebar hidden (drawer), tables may become card lists.
-
----
-
-## 11. Section Layout — Masonry Grid
-
-**Default layout for ALL content pages with multiple variable-height sections.**
-
-Sections on a content page must flow like physical objects stacked in columns — the next section always drops into the shortest column. This is CSS column masonry.
-
-### How it works
-
-```
-Column 1    | Column 2    | Column 3
-────────────┼─────────────┼────────────
-Section A   | Section B   | Section C
-(300px)     | (250px)     | (350px)
-            │             │
-Section E   | Section D   |
-(200px)     | (200px)     |
-```
-
-Sections fill left-to-right across the top, then each new section drops into whichever column is currently shortest. This is automatic — the browser handles placement via CSS `columns`.
-
-### Usage
-
-```jsx
-{/* 2 columns */}
-<div className="masonry-grid masonry-grid--2">
-  <Card title="Account Info">…</Card>
-  <Card title="Profile">…</Card>
-  <Card title="Security">…</Card>   {/* auto-drops into shortest column */}
-</div>
-
-{/* 3 columns */}
-<div className="masonry-grid masonry-grid--3">
-  {sections.map(s => <Card key={s.id}>…</Card>)}
-</div>
-```
-
-Available variants: `masonry-grid--2`, `masonry-grid--3`, `masonry-grid--4`
-
-Responsive behaviour:
-- `--4` collapses to 3 cols at 1024px, 1 col at 768px
-- `--3` collapses to 2 cols at 1024px, 1 col at 768px
-- `--2` collapses to 1 col at 768px
-
-### Rules
-
-- **Use `.masonry-grid` by default** on all content pages with 2+ variable-height sections
-- **Do NOT** use `display: grid` with `gridTemplateColumns` for variable-height card layouts — this creates uneven whitespace when cards differ in height
-- **Do NOT** use `.masonry-grid` for DataTable pages — tables span full width on their own
-- **Do NOT** use `.masonry-grid` when sections must align horizontally (e.g. two fields that are semantically paired side-by-side within a card) — that's an internal card layout, not page-level masonry
-- The `Card` component already has `break-inside: avoid` so it will never be split across columns
-
----
-
-## 12. What Claude Code Must NEVER Do
-
-- ❌ Write a hex color, `rgb()`, or `hsl()` value directly in any component or page file
-- ❌ Write a pixel spacing or size value that isn't a `--space-*` token
-- ❌ Use a raw `<button>`, `<input>`, or `<select>` for anything styled — always use the wrapper component
-- ❌ Create a `.module.css` or any per-page CSS file
-- ❌ Use Tailwind's `dark:` prefix — theming is via CSS tokens only
-- ❌ Place modal JSX inside a page file — modals live in `frontend/src/modals/`
-- ❌ Wrap `.page-wrapper` in additional divs that shift content alignment
-- ❌ Override `.page-wrapper`'s padding to make a single page "different"
-- ❌ Skip loading, error, and empty states on any data-fetching component
-- ❌ Import from `_archive/` anywhere except `@/lib/api.js`, `@/hooks/useAuth.js`, and `@/providers/AuthProvider.jsx`
-- ❌ Install a new icon library or introduce new SVG icons outside of `assets/`
-- ❌ Invent new color values not in `tokens.css`

backend/Dockerfile

@@ -1,16 +1,16 @@
 FROM python:3.11-slim
 
-# System dependencies: WeasyPrint (pango/cairo), ffmpeg (video thumbs), poppler (pdf2image)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libpango-1.0-0 \
+    git \
-    libpangocairo-1.0-0 \
+    curl \
-    libgdk-pixbuf-2.0-0 \
+    && curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz \
-    libffi-dev \
+    | tar -xz --strip-components=1 -C /usr/local/bin docker/docker \
-    shared-mime-info \
+    && curl -fsSL "https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64" \
-    fonts-dejavu-core \
+    -o /usr/local/bin/docker-compose \
-    ffmpeg \
+    && chmod +x /usr/local/bin/docker-compose \
-    poppler-utils \
+    && mkdir -p /usr/local/lib/docker/cli-plugins \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+    && ln -s /usr/local/bin/docker-compose /usr/local/lib/docker/cli-plugins/docker-compose \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 

backend/admin/router.py

@@ -42,12 +42,29 @@ async def deploy(request: Request):
 
     logger.info("Auto-deploy triggered via Gitea webhook")
 
-    # Write a trigger file to the host-mounted project path.
+    project_path = settings.deploy_project_path
-    # A host-side watcher service (bellsystems-deploy-watcher) polls for this
+    cmd = (
-    # file and runs deploy-host.sh as the bellsystems user when it appears.
+        f"git config --global --add safe.directory {project_path} && "
-    trigger_path = f"{settings.deploy_project_path}/.deploy-trigger"
+        f"cd {project_path} && "
-    with open(trigger_path, "w") as f:
+        f"git fetch origin main && "
-        f.write("deploy\n")
+        f"git reset --hard origin/main && "
+        f"docker-compose up -d --build"
+    )
+    try:
+        proc = await asyncio.create_subprocess_shell(
+            cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+        stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=300)
+        output = stdout.decode(errors="replace") if stdout else ""
 
-    logger.info("Auto-deploy trigger file written")
+        if proc.returncode != 0:
-    return {"ok": True, "message": "Deploy started"}
+            logger.error(f"Deploy failed (exit {proc.returncode}):\n{output}")
+            raise HTTPException(status_code=500, detail=f"Deploy script failed:\n{output[-500:]}")
+
+        logger.info(f"Deploy succeeded:\n{output[-300:]}")
+        return {"ok": True, "output": output[-1000:]}
+
+    except asyncio.TimeoutError:
+        raise HTTPException(status_code=504, detail="Deploy timed out after 300 seconds")

backend/alembic.ini

@@ -1,113 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# path to migration scripts
-script_location = alembic
-
-# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
-# Uncomment the line below if you want the files to be prepended with date and time
-# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
-
-# sys.path path, will be prepended to sys.path if present.
-# defaults to the current working directory.
-prepend_sys_path = .
-
-# timezone to use when rendering the date within the migration file
-# as well as the filename.
-# If specified, requires the python>=3.9 or backports.zoneinfo library.
-# Any required deps can installed by adding `tzdata` to the `[alembic]` section
-# of pyproject.toml.
-# timezone =
-
-# max length of characters to apply to the "slug" field
-# truncate_slug_length = 40
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-# set to 'true' to allow .pyc and .pyo files without
-# a source .py file to be detected as revisions in the
-# versions/ directory
-# sourceless = false
-
-# version location specification; This defaults
-# to alembic/versions.  When using multiple version
-# directories, initial revisions must be specified with --version-path.
-# The path separator used here should be the separator specified by "version_path_separator" below.
-# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
-
-# version path separator; As mentioned above, this is the character used to split
-# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
-# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
-# Valid values for version_path_separator are:
-#
-# version_path_separator = :
-# version_path_separator = ;
-# version_path_separator = space
-version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
-
-# set to 'true' to search source files recursively
-# in "version_locations" directory
-# New in Alembic version 1.10
-# recursive_version_locations = false
-
-# the output encoding used when revision files
-# are written from script.py.mako
-# output_encoding = utf-8
-
-# NOTE: The database URL is set programmatically in env.py from settings.
-# Do not set sqlalchemy.url here.
-
-
-[post_write_hooks]
-# post_write_hooks defines scripts or Python functions that are run
-# on newly generated revision scripts.  See the documentation for further
-# detail and examples
-
-# format using "black" - use the console_scripts runner, against the "black" entrypoint
-# hooks = black
-# black.type = console_scripts
-# black.entrypoint = black
-# black.options = -l 79 REVISION_SCRIPT_FILENAME
-
-# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
-# hooks = ruff
-# ruff.type = exec
-# ruff.executable = %(here)s/.venv/bin/ruff
-# ruff.options = --fix REVISION_SCRIPT_FILENAME
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

backend/alembic/env.py

@@ -1,44 +0,0 @@
-import asyncio
-from logging.config import fileConfig
-from sqlalchemy.ext.asyncio import create_async_engine
-from alembic import context
-from config import settings
-
-# Import all models so Alembic can see them
-from database.models import Base  # noqa: F401 — triggers all ORM imports
-
-config = context.config
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
-target_metadata = Base.metadata
-
-
-def run_migrations_offline() -> None:
-    url = settings.database_url
-    context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def do_run_migrations(connection):
-    context.configure(connection=connection, target_metadata=target_metadata)
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-async def run_async_migrations() -> None:
-    engine = create_async_engine(settings.database_url)
-    async with engine.begin() as conn:
-        await conn.run_sync(do_run_migrations)
-    await engine.dispose()
-
-
-def run_migrations_online() -> None:
-    asyncio.run(run_async_migrations())
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

backend/alembic/script.py.mako

@@ -1,26 +0,0 @@
-"""${message}
-
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
-Create Date: ${create_date}
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
-
-# revision identifiers, used by Alembic.
-revision: str = ${repr(up_revision)}
-down_revision: Union[str, None] = ${repr(down_revision)}
-branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
-depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
-
-
-def upgrade() -> None:
-    ${upgrades if upgrades else "pass"}
-
-
-def downgrade() -> None:
-    ${downgrades if downgrades else "pass"}

backend/alembic/versions/244a0b0f35be_rename_entries_to_crm_entries.py

@@ -1,83 +0,0 @@
-"""rename_entries_to_crm_entries
-
-Revision ID: 244a0b0f35be
-Revises: 485d40e86e4b
-Create Date: 2026-04-15 20:05:20.835281
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision: str = '244a0b0f35be'
-down_revision: Union[str, None] = '485d40e86e4b'
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # 1. Drop FK from support_tickets -> entries before touching the entries table
-    op.drop_constraint('support_tickets_linked_entry_id_fkey', 'support_tickets', type_='foreignkey')
-    # 2. Drop dependent table first, then parent
-    op.drop_table('entry_links')
-    op.drop_table('entries')
-    # 3. Create new tables with crm_ prefix
-    op.create_table('crm_entries',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('type', sa.String(length=10), nullable=False),
-    sa.Column('title', sa.String(length=500), nullable=False),
-    sa.Column('body', sa.Text(), nullable=True),
-    sa.Column('status', sa.String(length=20), nullable=True),
-    sa.Column('severity', sa.String(length=10), nullable=True),
-    sa.Column('author_id', sa.String(length=128), nullable=False),
-    sa.Column('author_name', sa.String(length=255), nullable=True),
-    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
-    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('crm_entry_links',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('entry_id', sa.UUID(), nullable=False),
-    sa.Column('entity_type', sa.String(length=20), nullable=False),
-    sa.Column('entity_id', sa.String(length=128), nullable=False),
-    sa.ForeignKeyConstraint(['entry_id'], ['crm_entries.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('entry_id', 'entity_type', 'entity_id')
-    )
-    # 4. Recreate FK on support_tickets pointing at new table
-    op.create_foreign_key(None, 'support_tickets', 'crm_entries', ['linked_entry_id'], ['id'], ondelete='SET NULL')
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, 'support_tickets', type_='foreignkey')
-    op.create_foreign_key('support_tickets_linked_entry_id_fkey', 'support_tickets', 'entries', ['linked_entry_id'], ['id'], ondelete='SET NULL')
-    op.create_table('entries',
-    sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
-    sa.Column('type', sa.VARCHAR(length=10), autoincrement=False, nullable=False),
-    sa.Column('title', sa.VARCHAR(length=500), autoincrement=False, nullable=False),
-    sa.Column('body', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.Column('status', sa.VARCHAR(length=20), autoincrement=False, nullable=True),
-    sa.Column('severity', sa.VARCHAR(length=10), autoincrement=False, nullable=True),
-    sa.Column('author_id', sa.VARCHAR(length=128), autoincrement=False, nullable=False),
-    sa.Column('author_name', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
-    sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
-    sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('id', name='entries_pkey'),
-    postgresql_ignore_search_path=False
-    )
-    op.create_table('entry_links',
-    sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
-    sa.Column('entry_id', sa.UUID(), autoincrement=False, nullable=False),
-    sa.Column('entity_type', sa.VARCHAR(length=20), autoincrement=False, nullable=False),
-    sa.Column('entity_id', sa.VARCHAR(length=128), autoincrement=False, nullable=False),
-    sa.ForeignKeyConstraint(['entry_id'], ['entries.id'], name='entry_links_entry_id_fkey', ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id', name='entry_links_pkey'),
-    sa.UniqueConstraint('entry_id', 'entity_type', 'entity_id', name='entry_links_entry_id_entity_type_entity_id_key')
-    )
-    op.drop_table('crm_entry_links')
-    op.drop_table('crm_entries')
-    # ### end Alembic commands ###

backend/alembic/versions/485d40e86e4b_initial_notes_and_tickets.py

@@ -1,82 +0,0 @@
-"""initial_notes_and_tickets
-
-Revision ID: 485d40e86e4b
-Revises: 
-Create Date: 2026-04-15 20:01:04.225959
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision: str = '485d40e86e4b'
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('entries',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('type', sa.String(length=10), nullable=False),
-    sa.Column('title', sa.String(length=500), nullable=False),
-    sa.Column('body', sa.Text(), nullable=True),
-    sa.Column('status', sa.String(length=20), nullable=True),
-    sa.Column('severity', sa.String(length=10), nullable=True),
-    sa.Column('author_id', sa.String(length=128), nullable=False),
-    sa.Column('author_name', sa.String(length=255), nullable=True),
-    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
-    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('entry_links',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('entry_id', sa.UUID(), nullable=False),
-    sa.Column('entity_type', sa.String(length=20), nullable=False),
-    sa.Column('entity_id', sa.String(length=128), nullable=False),
-    sa.ForeignKeyConstraint(['entry_id'], ['entries.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('entry_id', 'entity_type', 'entity_id')
-    )
-    op.create_table('support_tickets',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('customer_id', sa.String(length=128), nullable=False),
-    sa.Column('customer_name', sa.String(length=255), nullable=True),
-    sa.Column('device_id', sa.String(length=128), nullable=True),
-    sa.Column('device_serial', sa.String(length=64), nullable=True),
-    sa.Column('subject', sa.String(length=500), nullable=False),
-    sa.Column('status', sa.String(length=30), nullable=False),
-    sa.Column('priority', sa.String(length=10), nullable=True),
-    sa.Column('opened_via', sa.String(length=20), nullable=True),
-    sa.Column('linked_entry_id', sa.UUID(), nullable=True),
-    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
-    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
-    sa.ForeignKeyConstraint(['linked_entry_id'], ['entries.id'], ondelete='SET NULL'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('ticket_messages',
-    sa.Column('id', sa.UUID(), nullable=False),
-    sa.Column('ticket_id', sa.UUID(), nullable=False),
-    sa.Column('sender_type', sa.String(length=10), nullable=False),
-    sa.Column('sender_id', sa.String(length=128), nullable=False),
-    sa.Column('sender_name', sa.String(length=255), nullable=True),
-    sa.Column('body', sa.Text(), nullable=False),
-    sa.Column('is_internal', sa.Boolean(), nullable=False),
-    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
-    sa.ForeignKeyConstraint(['ticket_id'], ['support_tickets.id'], ondelete='CASCADE'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('ticket_messages')
-    op.drop_table('support_tickets')
-    op.drop_table('entry_links')
-    op.drop_table('entries')
-    # ### end Alembic commands ###

backend/alembic/versions/a1b2c3d4e5f6_add_category_to_crm_entries.py

@@ -1,23 +0,0 @@
-"""add_category_to_crm_entries
-
-Revision ID: a1b2c3d4e5f6
-Revises: 244a0b0f35be
-Create Date: 2026-04-16 09:00:00.000000
-
-"""
-from typing import Sequence, Union
-from alembic import op
-import sqlalchemy as sa
-
-revision: str = 'a1b2c3d4e5f6'
-down_revision: Union[str, None] = '244a0b0f35be'
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column('crm_entries', sa.Column('category', sa.String(length=30), nullable=True))
-
-
-def downgrade() -> None:
-    op.drop_column('crm_entries', 'category')

backend/alembic/versions/b1c2d3e4f5a6_phase_0_schema_foundation.py

@@ -1,507 +0,0 @@
-"""phase_0_schema_foundation
-
-Adds all Phase 0 tables:
-  - _migration_runs        (migration tracking)
-  - audit_log              (staff action audit trail)
-  - crm_products
-  - crm_customers
-  - crm_orders
-  - crm_comms_log
-  - crm_media
-  - crm_sync_state
-  - crm_quotations
-  - crm_quotation_items
-  - staff
-  - console_settings
-  - public_features
-  - melody_drafts
-  - built_melodies
-  - mfg_audit_log
-  - device_alerts
-  - commands               (raw SQL — no ORM model)
-  - heartbeats             (raw SQL — no ORM model)
-  - device_logs            (partitioned by month — raw SQL)
-  - device_logs_2025_01 … device_logs_2026_06  (initial partitions)
-
-Revision ID: b1c2d3e4f5a6
-Revises: a1b2c3d4e5f6
-Create Date: 2026-04-17 00:00:00.000000
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects.postgresql import ARRAY, JSONB
-
-# revision identifiers
-revision: str = "b1c2d3e4f5a6"
-down_revision: Union[str, None] = "a1b2c3d4e5f6"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # ------------------------------------------------------------------ #
-    # _migration_runs
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "_migration_runs",
-        sa.Column("id",          sa.BigInteger(), primary_key=True, autoincrement=True),
-        sa.Column("script_name", sa.String(256), nullable=False),
-        sa.Column("ran_at",      sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("source_rows", sa.BigInteger(), nullable=False, server_default="0"),
-        sa.Column("dest_rows",   sa.BigInteger(), nullable=False, server_default="0"),
-        sa.Column("success",     sa.String(8),  nullable=False, server_default="ok"),
-        sa.Column("notes",       sa.Text(),     nullable=True),
-    )
-
-    # ------------------------------------------------------------------ #
-    # audit_log
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "audit_log",
-        sa.Column("id",           sa.BigInteger(), primary_key=True, autoincrement=True),
-        sa.Column("occurred_at",  sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("actor_id",     sa.String(128), nullable=False),
-        sa.Column("actor_name",   sa.String(255), nullable=False),
-        sa.Column("action",       sa.String(64),  nullable=False),
-        sa.Column("entity_type",  sa.String(64),  nullable=False),
-        sa.Column("entity_id",    sa.String(128), nullable=False),
-        sa.Column("entity_label", sa.String(500), nullable=True),
-        sa.Column("changes",      JSONB,           nullable=True),
-        sa.Column("meta",         JSONB,           nullable=True),
-    )
-    op.create_index("idx_audit_actor",    "audit_log", ["actor_id",   "occurred_at"])
-    op.create_index("idx_audit_entity",   "audit_log", ["entity_type","entity_id", "occurred_at"])
-    op.create_index("idx_audit_action",   "audit_log", ["action",     "occurred_at"])
-    op.create_index("idx_audit_occurred", "audit_log", ["occurred_at"])
-
-    # ------------------------------------------------------------------ #
-    # staff
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "staff",
-        sa.Column("id",              sa.String(128), primary_key=True),
-        sa.Column("firestore_id",    sa.String(128), nullable=True,  unique=True),
-        sa.Column("email",           sa.String(256), nullable=False, unique=True),
-        sa.Column("name",            sa.String(255), nullable=False),
-        sa.Column("role",            sa.String(64),  nullable=False, server_default="staff"),
-        sa.Column("permissions",     JSONB,           nullable=False, server_default="{}"),
-        sa.Column("hashed_password", sa.String(256), nullable=False),
-        sa.Column("is_active",       sa.Boolean(),   nullable=False, server_default="true"),
-        sa.Column("created_at",      sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("updated_at",      sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-
-    # ------------------------------------------------------------------ #
-    # console_settings  &  public_features
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "console_settings",
-        sa.Column("key",        sa.String(128), primary_key=True),
-        sa.Column("value",      JSONB,           nullable=True),
-        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-    op.create_table(
-        "public_features",
-        sa.Column("key",        sa.String(128), primary_key=True),
-        sa.Column("value",      JSONB,           nullable=True),
-        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-
-    # ------------------------------------------------------------------ #
-    # crm_products
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_products",
-        sa.Column("id",           sa.String(128), primary_key=True),
-        sa.Column("firestore_id", sa.String(128), nullable=True, unique=True),
-        sa.Column("name",         sa.String(500), nullable=False),
-        sa.Column("sku",          sa.String(128), nullable=True),
-        sa.Column("category",     sa.String(128), nullable=True),
-        sa.Column("description",  sa.Text(),      nullable=True),
-        sa.Column("unit_cost",    sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("currency",     sa.String(10),  nullable=False, server_default="EUR"),
-        sa.Column("unit_type",    sa.String(32),  nullable=False, server_default="pcs"),
-        sa.Column("is_active",    sa.Boolean(),   nullable=False, server_default="true"),
-        sa.Column("created_at",   sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("updated_at",   sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-
-    # ------------------------------------------------------------------ #
-    # crm_customers
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_customers",
-        sa.Column("id",                  sa.String(128), primary_key=True),
-        sa.Column("firestore_id",        sa.String(128), nullable=True, unique=True),
-        sa.Column("title",               sa.String(32),  nullable=True),
-        sa.Column("name",                sa.String(255), nullable=False),
-        sa.Column("surname",             sa.String(255), nullable=True),
-        sa.Column("organization",        sa.String(500), nullable=True),
-        sa.Column("religion",            sa.String(64),  nullable=True),
-        sa.Column("language",            sa.String(10),  nullable=False, server_default="el"),
-        sa.Column("folder_id",           sa.String(128), nullable=False, unique=True),
-        sa.Column("relationship_status", sa.String(64),  nullable=False, server_default="lead"),
-        sa.Column("nextcloud_folder",    sa.String(500), nullable=True),
-        sa.Column("contacts",            JSONB, nullable=False, server_default="[]"),
-        sa.Column("notes",               JSONB, nullable=False, server_default="[]"),
-        sa.Column("location",            JSONB, nullable=True),
-        sa.Column("tags",                ARRAY(sa.String()), nullable=False, server_default="{}"),
-        sa.Column("owned_items",         JSONB, nullable=False, server_default="[]"),
-        sa.Column("linked_user_ids",     ARRAY(sa.String()), nullable=False, server_default="{}"),
-        sa.Column("technical_issues",    JSONB, nullable=False, server_default="[]"),
-        sa.Column("install_support",     JSONB, nullable=False, server_default="[]"),
-        sa.Column("transaction_history", JSONB, nullable=False, server_default="[]"),
-        sa.Column("crm_summary",         JSONB, nullable=True),
-        sa.Column("created_at",          sa.DateTime(timezone=True), nullable=False),
-        sa.Column("updated_at",          sa.DateTime(timezone=True), nullable=False),
-    )
-    op.create_index("idx_crm_customers_rel_status", "crm_customers", ["relationship_status"])
-    op.create_index("idx_crm_customers_name",       "crm_customers", ["name", "surname"])
-    op.create_index("idx_crm_customers_tags",       "crm_customers", ["tags"],
-                    postgresql_using="gin")
-
-    # ------------------------------------------------------------------ #
-    # crm_orders
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_orders",
-        sa.Column("id",                  sa.String(128), primary_key=True),
-        sa.Column("customer_id",         sa.String(128),
-                  sa.ForeignKey("crm_customers.id", ondelete="CASCADE"), nullable=False),
-        sa.Column("order_number",        sa.String(64),  nullable=False, unique=True),
-        sa.Column("title",               sa.String(500), nullable=True),
-        sa.Column("created_by",          sa.String(128), nullable=True),
-        sa.Column("status",              sa.String(64),  nullable=False,
-                  server_default="negotiating"),
-        sa.Column("status_updated_date", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("status_updated_by",   sa.String(128), nullable=True),
-        sa.Column("items",               JSONB, nullable=False, server_default="[]"),
-        sa.Column("subtotal",            sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("discount",            JSONB, nullable=True),
-        sa.Column("total_price",         sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("currency",            sa.String(10), nullable=False, server_default="EUR"),
-        sa.Column("shipping",            JSONB, nullable=True),
-        sa.Column("payment_status",      JSONB, nullable=False, server_default="{}"),
-        sa.Column("invoice_path",        sa.String(500), nullable=True),
-        sa.Column("notes",               sa.Text(), nullable=True),
-        sa.Column("timeline",            JSONB, nullable=False, server_default="[]"),
-        sa.Column("created_at",          sa.DateTime(timezone=True), nullable=False),
-        sa.Column("updated_at",          sa.DateTime(timezone=True), nullable=False),
-    )
-    op.create_index("idx_crm_orders_customer", "crm_orders", ["customer_id"])
-    op.create_index("idx_crm_orders_status",   "crm_orders", ["status"])
-
-    # ------------------------------------------------------------------ #
-    # crm_comms_log
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_comms_log",
-        sa.Column("id",             sa.String(128), primary_key=True),
-        sa.Column("customer_id",    sa.String(128),
-                  sa.ForeignKey("crm_customers.id", ondelete="SET NULL"), nullable=True),
-        sa.Column("type",           sa.String(32),  nullable=False),
-        sa.Column("mail_account",   sa.String(256), nullable=True),
-        sa.Column("direction",      sa.String(16),  nullable=False),
-        sa.Column("subject",        sa.String(500), nullable=True),
-        sa.Column("body",           sa.Text(),      nullable=True),
-        sa.Column("body_html",      sa.Text(),      nullable=True),
-        sa.Column("attachments",    JSONB, nullable=False, server_default="[]"),
-        sa.Column("ext_message_id", sa.String(500), nullable=True),
-        sa.Column("from_addr",      sa.String(500), nullable=True),
-        sa.Column("to_addrs",       sa.Text(),      nullable=True),
-        sa.Column("logged_by",      sa.String(128), nullable=True),
-        sa.Column("is_important",   sa.Boolean(), nullable=False, server_default="false"),
-        sa.Column("is_read",        sa.Boolean(), nullable=False, server_default="true"),
-        sa.Column("occurred_at",    sa.DateTime(timezone=True), nullable=False),
-        sa.Column("created_at",     sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-    op.create_index("idx_crm_comms_customer", "crm_comms_log", ["customer_id", "occurred_at"])
-
-    # ------------------------------------------------------------------ #
-    # crm_media
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_media",
-        sa.Column("id",             sa.String(128), primary_key=True),
-        sa.Column("customer_id",    sa.String(128),
-                  sa.ForeignKey("crm_customers.id", ondelete="SET NULL"), nullable=True),
-        sa.Column("order_id",       sa.String(128), nullable=True),
-        sa.Column("filename",       sa.String(500),  nullable=False),
-        sa.Column("nextcloud_path", sa.String(1000), nullable=False),
-        sa.Column("thumbnail_path", sa.String(1000), nullable=True),
-        sa.Column("mime_type",      sa.String(128),  nullable=True),
-        sa.Column("direction",      sa.String(16),   nullable=True),
-        sa.Column("tags",           JSONB, nullable=False, server_default="[]"),
-        sa.Column("uploaded_by",    sa.String(128),  nullable=True),
-        sa.Column("created_at",     sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-    op.create_index("idx_crm_media_customer", "crm_media", ["customer_id"])
-    op.create_index("idx_crm_media_order",    "crm_media", ["order_id"])
-
-    # ------------------------------------------------------------------ #
-    # crm_sync_state
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_sync_state",
-        sa.Column("key",   sa.String(128), primary_key=True),
-        sa.Column("value", sa.Text(),      nullable=True),
-    )
-
-    # ------------------------------------------------------------------ #
-    # crm_quotations
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_quotations",
-        sa.Column("id",                       sa.String(128), primary_key=True),
-        sa.Column("quotation_number",         sa.String(64),  nullable=False, unique=True),
-        sa.Column("title",                    sa.String(500), nullable=True),
-        sa.Column("subtitle",                 sa.String(500), nullable=True),
-        sa.Column("customer_id",              sa.String(128),
-                  sa.ForeignKey("crm_customers.id", ondelete="CASCADE"), nullable=False),
-        sa.Column("language",                 sa.String(10),  nullable=False, server_default="en"),
-        sa.Column("status",                   sa.String(32),  nullable=False, server_default="draft"),
-        sa.Column("order_type",               sa.String(64),  nullable=True),
-        sa.Column("shipping_method",          sa.String(64),  nullable=True),
-        sa.Column("estimated_shipping_date",  sa.String(32),  nullable=True),
-        sa.Column("global_discount_label",    sa.String(128), nullable=True),
-        sa.Column("global_discount_percent",  sa.Numeric(8, 4), nullable=False, server_default="0"),
-        sa.Column("vat_percent",              sa.Numeric(8, 4), nullable=False, server_default="24"),
-        sa.Column("global_vat_percent",       sa.Numeric(8, 4), nullable=False, server_default="24"),
-        sa.Column("shipping_cost",            sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("shipping_cost_discount",   sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("install_cost",             sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("install_cost_discount",    sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("extras_label",             sa.String(256), nullable=True),
-        sa.Column("extras_cost",              sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("comments",                 JSONB, nullable=False, server_default="[]"),
-        sa.Column("quick_notes",              JSONB, nullable=False, server_default="{}"),
-        sa.Column("subtotal_before_discount", sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("global_discount_amount",   sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("new_subtotal",             sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("vat_amount",               sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("final_total",              sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("nextcloud_pdf_path",       sa.String(1000), nullable=True),
-        sa.Column("nextcloud_pdf_url",        sa.String(1000), nullable=True),
-        sa.Column("client_org",               sa.String(500), nullable=True),
-        sa.Column("client_name",              sa.String(500), nullable=True),
-        sa.Column("client_location",          sa.String(500), nullable=True),
-        sa.Column("client_phone",             sa.String(64),  nullable=True),
-        sa.Column("client_email",             sa.String(256), nullable=True),
-        sa.Column("is_legacy",                sa.Boolean(), nullable=False, server_default="false"),
-        sa.Column("legacy_date",              sa.String(32),   nullable=True),
-        sa.Column("legacy_pdf_path",          sa.String(1000), nullable=True),
-        sa.Column("created_at",               sa.DateTime(timezone=True), nullable=False),
-        sa.Column("updated_at",               sa.DateTime(timezone=True), nullable=False),
-    )
-    op.create_index("idx_crm_quotations_customer", "crm_quotations", ["customer_id"])
-
-    # ------------------------------------------------------------------ #
-    # crm_quotation_items
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "crm_quotation_items",
-        sa.Column("id",               sa.String(128), primary_key=True),
-        sa.Column("quotation_id",     sa.String(128),
-                  sa.ForeignKey("crm_quotations.id", ondelete="CASCADE"), nullable=False),
-        sa.Column("product_id",       sa.String(128),  nullable=True),
-        sa.Column("description",      sa.Text(),       nullable=True),
-        sa.Column("description_en",   sa.Text(),       nullable=True),
-        sa.Column("description_gr",   sa.Text(),       nullable=True),
-        sa.Column("unit_type",        sa.String(32),   nullable=False, server_default="pcs"),
-        sa.Column("unit_cost",        sa.Numeric(12, 4), nullable=False, server_default="0"),
-        sa.Column("discount_percent", sa.Numeric(8, 4),  nullable=False, server_default="0"),
-        sa.Column("vat_percent",      sa.Numeric(8, 4),  nullable=False, server_default="24"),
-        sa.Column("quantity",         sa.Numeric(12, 4), nullable=False, server_default="1"),
-        sa.Column("line_total",       sa.Numeric(12, 2), nullable=False, server_default="0"),
-        sa.Column("sort_order",       sa.Integer(),      nullable=False, server_default="0"),
-    )
-    op.create_index("idx_crm_quotation_items_quotation", "crm_quotation_items",
-                    ["quotation_id", "sort_order"])
-
-    # ------------------------------------------------------------------ #
-    # melody_drafts
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "melody_drafts",
-        sa.Column("id",         sa.String(128), primary_key=True),
-        sa.Column("status",     sa.String(32),  nullable=False, server_default="draft"),
-        sa.Column("data",       JSONB,           nullable=False),
-        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-    op.create_index("idx_melody_drafts_status", "melody_drafts", ["status"])
-
-    # ------------------------------------------------------------------ #
-    # built_melodies
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "built_melodies",
-        sa.Column("id",                  sa.String(128), primary_key=True),
-        sa.Column("name",                sa.String(500), nullable=False),
-        sa.Column("pid",                 sa.String(128), nullable=False),
-        sa.Column("steps",               JSONB,           nullable=False),
-        sa.Column("binary_path",         sa.String(1000), nullable=True),
-        sa.Column("progmem_code",        sa.Text(),       nullable=True),
-        sa.Column("assigned_melody_ids", JSONB, nullable=False, server_default="[]"),
-        sa.Column("is_builtin",          sa.Boolean(), nullable=False, server_default="false"),
-        sa.Column("created_at",          sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("updated_at",          sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-    )
-
-    # ------------------------------------------------------------------ #
-    # mfg_audit_log
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "mfg_audit_log",
-        sa.Column("id",            sa.BigInteger(), primary_key=True, autoincrement=True),
-        sa.Column("timestamp",     sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("admin_user",    sa.String(256), nullable=False),
-        sa.Column("action",        sa.String(128), nullable=False),
-        sa.Column("serial_number", sa.String(128), nullable=True),
-        sa.Column("detail",        sa.Text(),      nullable=True),
-    )
-    op.create_index("idx_mfg_audit_time",   "mfg_audit_log", ["timestamp"])
-    op.create_index("idx_mfg_audit_action", "mfg_audit_log", ["action"])
-
-    # ------------------------------------------------------------------ #
-    # device_alerts
-    # ------------------------------------------------------------------ #
-    op.create_table(
-        "device_alerts",
-        sa.Column("id",            sa.BigInteger(), primary_key=True, autoincrement=True),
-        sa.Column("device_serial", sa.String(128), nullable=False),
-        sa.Column("subsystem",     sa.String(128), nullable=False),
-        sa.Column("state",         sa.String(64),  nullable=False),
-        sa.Column("message",       sa.Text(),      nullable=True),
-        sa.Column("updated_at",    sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.UniqueConstraint("device_serial", "subsystem", name="uq_device_alerts_serial_subsystem"),
-    )
-    op.create_index("idx_device_alerts_serial", "device_alerts", ["device_serial"])
-
-    # ------------------------------------------------------------------ #
-    # commands  (raw SQL — mirrors SQLite schema, no ORM model)
-    # ------------------------------------------------------------------ #
-    op.execute("""
-        CREATE TABLE commands (
-            id               BIGSERIAL PRIMARY KEY,
-            device_serial    TEXT NOT NULL,
-            command_name     TEXT NOT NULL,
-            command_payload  TEXT,
-            status           TEXT NOT NULL DEFAULT 'pending',
-            response_payload TEXT,
-            sent_at          TIMESTAMPTZ NOT NULL DEFAULT now(),
-            responded_at     TIMESTAMPTZ
-        )
-    """)
-    op.execute("CREATE INDEX idx_commands_serial_time ON commands(device_serial, sent_at DESC)")
-    op.execute("CREATE INDEX idx_commands_status ON commands(status)")
-
-    # ------------------------------------------------------------------ #
-    # heartbeats  (raw SQL — mirrors SQLite schema, no ORM model)
-    # ------------------------------------------------------------------ #
-    op.execute("""
-        CREATE TABLE heartbeats (
-            id               BIGSERIAL PRIMARY KEY,
-            device_serial    TEXT NOT NULL,
-            device_id        TEXT,
-            firmware_version TEXT,
-            ip_address       TEXT,
-            gateway          TEXT,
-            uptime_ms        BIGINT,
-            uptime_display   TEXT,
-            received_at      TIMESTAMPTZ NOT NULL DEFAULT now()
-        )
-    """)
-    op.execute("CREATE INDEX idx_heartbeats_serial_time ON heartbeats(device_serial, received_at DESC)")
-
-    # ------------------------------------------------------------------ #
-    # device_logs — partitioned by month on received_at
-    # ------------------------------------------------------------------ #
-    op.execute("""
-        CREATE TABLE device_logs (
-            id               BIGSERIAL,
-            device_serial    TEXT NOT NULL,
-            level            TEXT NOT NULL,
-            message          TEXT NOT NULL,
-            device_timestamp BIGINT,
-            received_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
-            PRIMARY KEY (id, received_at)
-        ) PARTITION BY RANGE (received_at)
-    """)
-    op.execute("""
-        CREATE INDEX idx_device_logs_serial_time
-        ON device_logs(device_serial, received_at DESC)
-    """)
-    op.execute("""
-        CREATE INDEX idx_device_logs_level
-        ON device_logs(level, received_at DESC)
-    """)
-
-    # Create partitions: 2025-01 through 2026-06 (covers all existing data + near future)
-    partitions = [
-        ("2025_01", "2025-01-01", "2025-02-01"),
-        ("2025_02", "2025-02-01", "2025-03-01"),
-        ("2025_03", "2025-03-01", "2025-04-01"),
-        ("2025_04", "2025-04-01", "2025-05-01"),
-        ("2025_05", "2025-05-01", "2025-06-01"),
-        ("2025_06", "2025-06-01", "2025-07-01"),
-        ("2025_07", "2025-07-01", "2025-08-01"),
-        ("2025_08", "2025-08-01", "2025-09-01"),
-        ("2025_09", "2025-09-01", "2025-10-01"),
-        ("2025_10", "2025-10-01", "2025-11-01"),
-        ("2025_11", "2025-11-01", "2025-12-01"),
-        ("2025_12", "2025-12-01", "2026-01-01"),
-        ("2026_01", "2026-01-01", "2026-02-01"),
-        ("2026_02", "2026-02-01", "2026-03-01"),
-        ("2026_03", "2026-03-01", "2026-04-01"),
-        ("2026_04", "2026-04-01", "2026-05-01"),
-        ("2026_05", "2026-05-01", "2026-06-01"),
-        ("2026_06", "2026-06-01", "2026-07-01"),
-    ]
-    for suffix, start, end in partitions:
-        op.execute(f"""
-            CREATE TABLE device_logs_{suffix} PARTITION OF device_logs
-            FOR VALUES FROM ('{start}') TO ('{end}')
-        """)
-
-
-def downgrade() -> None:
-    # Drop in reverse dependency order
-    op.execute("DROP TABLE IF EXISTS device_logs CASCADE")   # drops all partitions too
-    op.execute("DROP TABLE IF EXISTS heartbeats CASCADE")
-    op.execute("DROP TABLE IF EXISTS commands CASCADE")
-    op.drop_table("device_alerts")
-    op.drop_table("mfg_audit_log")
-    op.drop_table("built_melodies")
-    op.drop_table("melody_drafts")
-    op.drop_table("crm_quotation_items")
-    op.drop_table("crm_quotations")
-    op.drop_table("crm_sync_state")
-    op.drop_table("crm_media")
-    op.drop_table("crm_comms_log")
-    op.drop_table("crm_orders")
-    op.drop_table("crm_customers")
-    op.drop_table("crm_products")
-    op.drop_table("public_features")
-    op.drop_table("console_settings")
-    op.drop_table("staff")
-    op.drop_table("audit_log")
-    op.drop_table("_migration_runs")

backend/alembic/versions/c3d4e5f6a7b8_phase_3_staff_ui_prefs.py

@@ -1,32 +0,0 @@
-"""phase_3_staff_ui_prefs
-
-Adds ui_prefs JSONB column to the staff table (Phase 3 — staff auth cutover).
-Also corrects permissions to be nullable (sysadmin/admin have NULL permissions).
-
-Revision ID: c3d4e5f6a7b8
-Revises: b1c2d3e4f5a6
-Create Date: 2026-04-17 00:00:00.000000
-"""
-from typing import Sequence, Union
-import sqlalchemy as sa
-from sqlalchemy.dialects.postgresql import JSONB
-from alembic import op
-
-revision: str = "c3d4e5f6a7b8"
-down_revision: Union[str, None] = "b1c2d3e4f5a6"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "staff",
-        sa.Column("ui_prefs", JSONB, nullable=False, server_default="{}"),
-    )
-    # permissions was NOT NULL DEFAULT '{}' — relax to nullable for sysadmin/admin
-    op.alter_column("staff", "permissions", nullable=True)
-
-
-def downgrade() -> None:
-    op.drop_column("staff", "ui_prefs")
-    op.alter_column("staff", "permissions", nullable=False)

backend/audit/router.py

@@ -1,74 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from fastapi import APIRouter, Depends, Query
-from sqlalchemy import select, and_
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from database.postgres import get_pg_session
-from shared.orm import AuditLog
-from auth.dependencies import require_sysadmin
-from auth.models import TokenPayload
-
-router = APIRouter(prefix="/api/audit-log", tags=["audit-log"])
-
-_MAX_LIMIT = 200
-_DEFAULT_LIMIT = 50
-
-
-@router.get("")
-async def list_audit_log(
-    actor_id: Optional[str] = Query(None),
-    entity_type: Optional[str] = Query(None),
-    entity_id: Optional[str] = Query(None),
-    action: Optional[str] = Query(None),
-    from_date: Optional[datetime] = Query(None),
-    to_date: Optional[datetime] = Query(None),
-    limit: int = Query(_DEFAULT_LIMIT, ge=1, le=_MAX_LIMIT),
-    offset: int = Query(0, ge=0),
-    _user: TokenPayload = Depends(require_sysadmin),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    filters = []
-    if actor_id:
-        filters.append(AuditLog.actor_id == actor_id)
-    if entity_type:
-        filters.append(AuditLog.entity_type == entity_type)
-    if entity_id:
-        filters.append(AuditLog.entity_id == entity_id)
-    if action:
-        filters.append(AuditLog.action == action)
-    if from_date:
-        filters.append(AuditLog.occurred_at >= from_date)
-    if to_date:
-        filters.append(AuditLog.occurred_at <= to_date)
-
-    stmt = (
-        select(AuditLog)
-        .where(and_(*filters) if filters else True)
-        .order_by(AuditLog.occurred_at.desc())
-        .offset(offset)
-        .limit(limit)
-    )
-    result = await db.execute(stmt)
-    rows = result.scalars().all()
-
-    return {
-        "entries": [
-            {
-                "id":           r.id,
-                "occurred_at":  r.occurred_at.isoformat(),
-                "actor_id":     r.actor_id,
-                "actor_name":   r.actor_name,
-                "action":       r.action,
-                "entity_type":  r.entity_type,
-                "entity_id":    r.entity_id,
-                "entity_label": r.entity_label,
-                "changes":      r.changes,
-                "meta":         r.meta,
-            }
-            for r in rows
-        ],
-        "limit":  limit,
-        "offset": offset,
-    }

backend/auth/dependencies.py

@@ -1,14 +1,10 @@
 from fastapi import Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from jose import JWTError
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
 from auth.utils import decode_access_token
 from auth.models import TokenPayload, Role
-from database.postgres import get_pg_session
-from staff.orm import Staff
 from shared.exceptions import AuthenticationError, AuthorizationError
+from shared.firebase import get_db
 
 security = HTTPBearer()
 
@@ -41,15 +37,18 @@ def require_roles(*allowed_roles: Role):
     return role_checker
 
 
-async def _get_user_permissions(user: TokenPayload, db: AsyncSession) -> dict | None:
+async def _get_user_permissions(user: TokenPayload) -> dict:
-    """Fetch permissions from Postgres for the given user."""
+    """Fetch permissions from Firestore for the given user."""
     if user.role in (Role.sysadmin, Role.admin):
         return None  # Full access
-    result = await db.execute(select(Staff).where(Staff.id == user.sub).limit(1))
+    db = get_db()
-    staff = result.scalar_one_or_none()
+    if not db:
-    if staff is None:
         raise AuthorizationError()
-    return staff.permissions
+    doc = db.collection("admin_users").document(user.sub).get()
+    if not doc.exists:
+        raise AuthorizationError()
+    data = doc.to_dict()
+    return data.get("permissions")
 
 
 def require_permission(section: str, action: str):
@@ -59,17 +58,17 @@ def require_permission(section: str, action: str):
     """
     async def permission_checker(
         current_user: TokenPayload = Depends(get_current_user),
-        db: AsyncSession = Depends(get_pg_session),
     ) -> TokenPayload:
+        # sysadmin and admin have full access
         if current_user.role in (Role.sysadmin, Role.admin):
             return current_user
 
-        permissions = await _get_user_permissions(current_user, db)
+        permissions = await _get_user_permissions(current_user)
         if not permissions:
             raise AuthorizationError()
 
         if section == "mqtt":
-            if not permissions.get("mqtt", {}).get("access", False):
+            if not permissions.get("mqtt", False):
                 raise AuthorizationError()
             return current_user
 
@@ -90,7 +89,11 @@ def require_permission(section: str, action: str):
 # Pre-built convenience dependencies
 require_sysadmin = require_roles(Role.sysadmin)
 require_admin_or_above = require_roles(Role.sysadmin, Role.admin)
+
+# Staff management: only sysadmin and admin
 require_staff_management = require_roles(Role.sysadmin, Role.admin)
+
+# Viewer-level: any authenticated user (actual permission check per-action)
 require_any_authenticated = require_roles(
     Role.sysadmin, Role.admin, Role.editor, Role.user,
 )

backend/auth/models.py

@@ -10,141 +10,45 @@ class Role(str, Enum):
     user = "user"
 
 
-class MelodiesPermissions(BaseModel):
+class SectionPermissions(BaseModel):
-    view: bool = False
-    add: bool = False
-    delete: bool = False
-    safe_edit: bool = False
-    full_edit: bool = False
-    archetype_access: bool = False
-    settings_access: bool = False
-    compose_access: bool = False
-
-
-class DevicesPermissions(BaseModel):
-    view: bool = False
-    add: bool = False
-    delete: bool = False
-    safe_edit: bool = False
-    edit_bells: bool = False
-    edit_clock: bool = False
-    edit_warranty: bool = False
-    full_edit: bool = False
-    control: bool = False
-
-
-class AppUsersPermissions(BaseModel):
-    view: bool = False
-    add: bool = False
-    delete: bool = False
-    safe_edit: bool = False
-    full_edit: bool = False
-
-
-class IssuesNotesPermissions(BaseModel):
-    view: bool = False
-    add: bool = False
-    delete: bool = False
-    edit: bool = False
-
-
-class MailPermissions(BaseModel):
-    view: bool = False
-    compose: bool = False
-    reply: bool = False
-
-
-class CrmPermissions(BaseModel):
-    activity_log: bool = False
-
-
-class CrmCustomersPermissions(BaseModel):
-    full_access: bool = False
-    overview: bool = False
-    orders_view: bool = False
-    orders_edit: bool = False
-    quotations_view: bool = False
-    quotations_edit: bool = False
-    comms_view: bool = False
-    comms_log: bool = False
-    comms_edit: bool = False
-    comms_compose: bool = False
-    add: bool = False
-    delete: bool = False
-    files_view: bool = False
-    files_edit: bool = False
-    devices_view: bool = False
-    devices_edit: bool = False
-
-
-class CrmProductsPermissions(BaseModel):
     view: bool = False
     add: bool = False
     edit: bool = False
-
+    delete: bool = False
-
-class MfgPermissions(BaseModel):
-    view_inventory: bool = False
-    edit: bool = False
-    provision: bool = False
-    firmware_view: bool = False
-    firmware_edit: bool = False
-
-
-class ApiReferencePermissions(BaseModel):
-    access: bool = False
-
-
-class MqttPermissions(BaseModel):
-    access: bool = False
 
 
 class StaffPermissions(BaseModel):
-    melodies: MelodiesPermissions = MelodiesPermissions()
+    melodies: SectionPermissions = SectionPermissions()
-    devices: DevicesPermissions = DevicesPermissions()
+    devices: SectionPermissions = SectionPermissions()
-    app_users: AppUsersPermissions = AppUsersPermissions()
+    app_users: SectionPermissions = SectionPermissions()
-    issues_notes: IssuesNotesPermissions = IssuesNotesPermissions()
+    equipment: SectionPermissions = SectionPermissions()
-    mail: MailPermissions = MailPermissions()
+    manufacturing: SectionPermissions = SectionPermissions()
-    crm: CrmPermissions = CrmPermissions()
+    mqtt: bool = False
-    crm_customers: CrmCustomersPermissions = CrmCustomersPermissions()
-    crm_products: CrmProductsPermissions = CrmProductsPermissions()
-    mfg: MfgPermissions = MfgPermissions()
-    api_reference: ApiReferencePermissions = ApiReferencePermissions()
-    mqtt: MqttPermissions = MqttPermissions()
 
 
+# Default permissions per role
 def default_permissions_for_role(role: str) -> Optional[dict]:
     if role in ("sysadmin", "admin"):
         return None  # Full access, permissions field not used
-
+    full = {"view": True, "add": True, "edit": True, "delete": True}
+    view_only = {"view": True, "add": False, "edit": False, "delete": False}
     if role == "editor":
         return {
-            "melodies": {"view": True, "add": True, "delete": True, "safe_edit": True, "full_edit": True, "archetype_access": True, "settings_access": True, "compose_access": True},
+            "melodies": full,
-            "devices": {"view": True, "add": True, "delete": True, "safe_edit": True, "edit_bells": True, "edit_clock": True, "edit_warranty": True, "full_edit": True, "control": True},
+            "devices": full,
-            "app_users": {"view": True, "add": True, "delete": True, "safe_edit": True, "full_edit": True},
+            "app_users": full,
-            "issues_notes": {"view": True, "add": True, "delete": True, "edit": True},
+            "equipment": full,
-            "mail": {"view": True, "compose": True, "reply": True},
+            "manufacturing": view_only,
-            "crm": {"activity_log": True},
+            "mqtt": True,
-            "crm_customers": {"full_access": True, "overview": True, "orders_view": True, "orders_edit": True, "quotations_view": True, "quotations_edit": True, "comms_view": True, "comms_log": True, "comms_edit": True, "comms_compose": True, "add": True, "delete": True, "files_view": True, "files_edit": True, "devices_view": True, "devices_edit": True},
-            "crm_products": {"view": True, "add": True, "edit": True},
-            "mfg": {"view_inventory": True, "edit": True, "provision": True, "firmware_view": True, "firmware_edit": True},
-            "api_reference": {"access": True},
-            "mqtt": {"access": True},
         }
-
     # user role - view only
     return {
-        "melodies": {"view": True, "add": False, "delete": False, "safe_edit": False, "full_edit": False, "archetype_access": False, "settings_access": False, "compose_access": False},
+        "melodies": view_only,
-        "devices": {"view": True, "add": False, "delete": False, "safe_edit": False, "edit_bells": False, "edit_clock": False, "edit_warranty": False, "full_edit": False, "control": False},
+        "devices": view_only,
-        "app_users": {"view": True, "add": False, "delete": False, "safe_edit": False, "full_edit": False},
+        "app_users": view_only,
-        "issues_notes": {"view": True, "add": False, "delete": False, "edit": False},
+        "equipment": view_only,
-        "mail": {"view": True, "compose": False, "reply": False},
+        "manufacturing": {"view": False, "add": False, "edit": False, "delete": False},
-        "crm": {"activity_log": False},
+        "mqtt": False,
-        "crm_customers": {"full_access": False, "overview": True, "orders_view": True, "orders_edit": False, "quotations_view": True, "quotations_edit": False, "comms_view": True, "comms_log": False, "comms_edit": False, "comms_compose": False, "add": False, "delete": False, "files_view": True, "files_edit": False, "devices_view": True, "devices_edit": False},
-        "crm_products": {"view": True, "add": False, "edit": False},
-        "mfg": {"view_inventory": True, "edit": False, "provision": False, "firmware_view": True, "firmware_edit": False},
-        "api_reference": {"access": False},
-        "mqtt": {"access": False},
     }
 
 

backend/auth/router.py

@@ -1,74 +1,59 @@
-from fastapi import APIRouter, Depends, Request
+from fastapi import APIRouter
-from sqlalchemy import select
+from shared.firebase import get_db
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from database.postgres import get_pg_session
-from staff.orm import Staff
 from auth.models import LoginRequest, TokenResponse
 from auth.utils import verify_password, create_access_token
-from shared.audit import log_action
 from shared.exceptions import AuthenticationError
 
 router = APIRouter(prefix="/api/auth", tags=["auth"])
 
-_ROLE_MAP = {
-    "superadmin":     "sysadmin",
-    "melody_editor":  "editor",
-    "device_manager": "editor",
-    "user_manager":   "editor",
-    "viewer":         "user",
-    "staff":          "user",
-}
-
 
 @router.post("/login", response_model=TokenResponse)
-async def login(
+async def login(body: LoginRequest):
-    body: LoginRequest,
+    db = get_db()
-    request: Request,
+    if not db:
-    db: AsyncSession = Depends(get_pg_session),
+        raise AuthenticationError("Service unavailable")
-):
-    result = await db.execute(
-        select(Staff).where(Staff.email == body.email).limit(1)
-    )
-    staff = result.scalar_one_or_none()
 
-    if staff is None:
+    users_ref = db.collection("admin_users")
+    query = users_ref.where("email", "==", body.email).limit(1).get()
+
+    if not query:
         raise AuthenticationError("Invalid email or password")
 
-    if not staff.is_active:
+    doc = query[0]
+    user_data = doc.to_dict()
+
+    if not user_data.get("is_active", True):
         raise AuthenticationError("Account is disabled")
 
-    if not verify_password(body.password, staff.hashed_password):
+    if not verify_password(body.password, user_data["hashed_password"]):
         raise AuthenticationError("Invalid email or password")
 
-    role = _ROLE_MAP.get(staff.role, staff.role)
+    role = user_data["role"]
+    # Map legacy roles to new roles
+    role_mapping = {
+        "superadmin": "sysadmin",
+        "melody_editor": "editor",
+        "device_manager": "editor",
+        "user_manager": "editor",
+        "viewer": "user",
+    }
+    role = role_mapping.get(role, role)
 
     token = create_access_token({
-        "sub":   staff.id,
+        "sub": doc.id,
-        "email": staff.email,
+        "email": user_data["email"],
-        "role":  role,
+        "role": role,
-        "name":  staff.name,
+        "name": user_data["name"],
     })
 
+    # Get permissions for editor/user roles
     permissions = None
     if role in ("editor", "user"):
-        permissions = staff.permissions
+        permissions = user_data.get("permissions")
-
-    await log_action(
-        db,
-        actor_id=staff.id,
-        actor_name=staff.name,
-        action="LOGIN",
-        entity_type="staff",
-        entity_id=staff.id,
-        entity_label=staff.email,
-        meta={"ip": request.client.host if request.client else None},
-    )
-    await db.commit()
 
     return TokenResponse(
         access_token=token,
         role=role,
-        name=staff.name,
+        name=user_data["name"],
         permissions=permissions,
     )

backend/builder/database.py

@@ -1,38 +1,27 @@
 import json
 import logging
-from database import get_db
+from mqtt.database import get_db
 
 logger = logging.getLogger("builder.database")
 
 
-async def insert_built_melody(melody_id: str, name: str, pid: str, steps: str, is_builtin: bool = False) -> None:
+async def insert_built_melody(melody_id: str, name: str, pid: str, steps: str) -> None:
     db = await get_db()
     await db.execute(
-        """INSERT INTO built_melodies (id, name, pid, steps, assigned_melody_ids, is_builtin)
+        """INSERT INTO built_melodies (id, name, pid, steps, assigned_melody_ids)
-           VALUES (?, ?, ?, ?, ?, ?)""",
+           VALUES (?, ?, ?, ?, ?)""",
-        (melody_id, name, pid, steps, json.dumps([]), 1 if is_builtin else 0),
+        (melody_id, name, pid, steps, json.dumps([])),
     )
     await db.commit()
 
 
-async def update_built_melody(melody_id: str, name: str, pid: str, steps: str, is_builtin: bool = False) -> None:
+async def update_built_melody(melody_id: str, name: str, pid: str, steps: str) -> None:
     db = await get_db()
     await db.execute(
         """UPDATE built_melodies
-           SET name = ?, pid = ?, steps = ?, is_builtin = ?, updated_at = datetime('now')
+           SET name = ?, pid = ?, steps = ?, updated_at = datetime('now')
            WHERE id = ?""",
-        (name, pid, steps, 1 if is_builtin else 0, melody_id),
+        (name, pid, steps, melody_id),
-    )
-    await db.commit()
-
-
-async def update_builtin_flag(melody_id: str, is_builtin: bool) -> None:
-    db = await get_db()
-    await db.execute(
-        """UPDATE built_melodies
-           SET is_builtin = ?, updated_at = datetime('now')
-           WHERE id = ?""",
-        (1 if is_builtin else 0, melody_id),
     )
     await db.commit()
 
@@ -79,7 +68,6 @@ async def get_built_melody(melody_id: str) -> dict | None:
         return None
     row = dict(rows[0])
     row["assigned_melody_ids"] = json.loads(row["assigned_melody_ids"] or "[]")
-    row["is_builtin"] = bool(row.get("is_builtin", 0))
     return row
 
 
@@ -92,7 +80,6 @@ async def list_built_melodies() -> list[dict]:
     for row in rows:
         r = dict(row)
         r["assigned_melody_ids"] = json.loads(r["assigned_melody_ids"] or "[]")
-        r["is_builtin"] = bool(r.get("is_builtin", 0))
         results.append(r)
     return results
 

backend/builder/models.py

@@ -6,14 +6,12 @@ class BuiltMelodyCreate(BaseModel):
     name: str
     pid: str
     steps: str  # raw step string e.g. "1,2,2+1,1,2,3+1"
-    is_builtin: bool = False
 
 
 class BuiltMelodyUpdate(BaseModel):
     name: Optional[str] = None
     pid: Optional[str] = None
     steps: Optional[str] = None
-    is_builtin: Optional[bool] = None
 
 
 class BuiltMelodyInDB(BaseModel):
@@ -21,7 +19,6 @@ class BuiltMelodyInDB(BaseModel):
     name: str
     pid: str
     steps: str
-    is_builtin: bool = False
     binary_path: Optional[str] = None
     binary_url: Optional[str] = None
     progmem_code: Optional[str] = None

backend/builder/router.py

@@ -1,6 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException
-from fastapi.responses import FileResponse, PlainTextResponse
+from fastapi.responses import FileResponse
-from sqlalchemy.ext.asyncio import AsyncSession
 from auth.models import TokenPayload
 from auth.dependencies import require_permission
 from builder.models import (
@@ -10,8 +9,6 @@ from builder.models import (
     BuiltMelodyListResponse,
 )
 from builder import service
-from database.postgres import get_pg_session
-from shared.audit import log_action
 
 router = APIRouter(prefix="/api/builder/melodies", tags=["builder"])
 
@@ -23,7 +20,6 @@ async def list_built_melodies(
     melodies = await service.list_built_melodies()
     return BuiltMelodyListResponse(melodies=melodies, total=len(melodies))
 
-
 @router.get("/for-melody/{firestore_melody_id}")
 async def get_for_firestore_melody(
     firestore_melody_id: str,
@@ -36,14 +32,6 @@ async def get_for_firestore_melody(
     return result.model_dump()
 
 
-@router.get("/generate-builtin-list")
-async def generate_builtin_list(
-    _user: TokenPayload = Depends(require_permission("melodies", "view")),
-):
-    """Generate a C++ header with PROGMEM arrays for all is_builtin archetypes."""
-    code = await service.generate_builtin_list()
-    return PlainTextResponse(content=code, media_type="text/plain")
-
 
 @router.get("/{melody_id}", response_model=BuiltMelodyInDB)
 async def get_built_melody(
@@ -57,12 +45,8 @@ async def get_built_melody(
 async def create_built_melody(
     body: BuiltMelodyCreate,
     _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.create_built_melody(body)
+    return await service.create_built_melody(body)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "archetype",
-                     str(melody.id), melody.name or str(melody.id))
-    return melody
 
 
 @router.put("/{melody_id}", response_model=BuiltMelodyInDB)
@@ -70,40 +54,16 @@ async def update_built_melody(
     melody_id: str,
     body: BuiltMelodyUpdate,
     _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    old = await service.get_built_melody(melody_id)
+    return await service.update_built_melody(melody_id, body)
-    melody = await service.update_built_melody(melody_id, body)
-    _SKIP = {"updated_at", "id", "steps", "builtin_code"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(melody, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(melody, k, None)
-    }
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "archetype",
-                     melody_id, melody.name or melody_id, changes=changes or None)
-    return melody
 
 
 @router.delete("/{melody_id}", status_code=204)
 async def delete_built_melody(
     melody_id: str,
     _user: TokenPayload = Depends(require_permission("melodies", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.get_built_melody(melody_id)
     await service.delete_built_melody(melody_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "archetype",
-                     melody_id, melody.name if melody else melody_id)
-
-
-@router.post("/{melody_id}/toggle-builtin", response_model=BuiltMelodyInDB)
-async def toggle_builtin(
-    melody_id: str,
-    _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-):
-    """Toggle the is_builtin flag for an archetype."""
-    return await service.toggle_builtin(melody_id)
 
 
 @router.post("/{melody_id}/build-binary", response_model=BuiltMelodyInDB)

backend/builder/service.py

@@ -32,7 +32,6 @@ def _row_to_built_melody(row: dict) -> BuiltMelodyInDB:
         name=row["name"],
         pid=row["pid"],
         steps=row["steps"],
-        is_builtin=row.get("is_builtin", False),
         binary_path=binary_path,
         binary_url=binary_url,
         progmem_code=row.get("progmem_code"),
@@ -152,12 +151,8 @@ async def create_built_melody(data: BuiltMelodyCreate) -> BuiltMelodyInDB:
         name=data.name,
         pid=data.pid,
         steps=data.steps,
-        is_builtin=data.is_builtin,
     )
-    # Auto-build binary and builtin code on creation
+    return await get_built_melody(melody_id)
-    result = await get_built_melody(melody_id)
-    result = await _do_build(melody_id)
-    return result
 
 
 async def update_built_melody(melody_id: str, data: BuiltMelodyUpdate) -> BuiltMelodyInDB:
@@ -168,22 +163,11 @@ async def update_built_melody(melody_id: str, data: BuiltMelodyUpdate) -> BuiltM
     new_name = data.name if data.name is not None else row["name"]
     new_pid = data.pid if data.pid is not None else row["pid"]
     new_steps = data.steps if data.steps is not None else row["steps"]
-    new_is_builtin = data.is_builtin if data.is_builtin is not None else row.get("is_builtin", False)
 
     await _check_unique(new_name, new_pid or "", exclude_id=melody_id)
 
-    steps_changed = (data.steps is not None) and (data.steps != row["steps"])
+    await db.update_built_melody(melody_id, name=new_name, pid=new_pid, steps=new_steps)
-
+    return await get_built_melody(melody_id)
-    await db.update_built_melody(melody_id, name=new_name, pid=new_pid, steps=new_steps, is_builtin=new_is_builtin)
-
-    # If steps changed, flag all assigned melodies as outdated, then rebuild
-    if steps_changed:
-        assigned_ids = row.get("assigned_melody_ids", [])
-        if assigned_ids:
-            await _flag_melodies_outdated(assigned_ids, True)
-
-    # Auto-rebuild binary and builtin code on every save
-    return await _do_build(melody_id)
 
 
 async def delete_built_melody(melody_id: str) -> None:
@@ -191,11 +175,6 @@ async def delete_built_melody(melody_id: str) -> None:
     if not row:
         raise HTTPException(status_code=404, detail=f"Built melody '{melody_id}' not found")
 
-    # Flag all assigned melodies as outdated before deleting
-    assigned_ids = row.get("assigned_melody_ids", [])
-    if assigned_ids:
-        await _flag_melodies_outdated(assigned_ids, True)
-
     # Delete the .bsm file if it exists
     if row.get("binary_path"):
         bsm_path = Path(row["binary_path"])
@@ -205,26 +184,10 @@ async def delete_built_melody(melody_id: str) -> None:
     await db.delete_built_melody(melody_id)
 
 
-async def toggle_builtin(melody_id: str) -> BuiltMelodyInDB:
-    """Toggle the is_builtin flag for an archetype."""
-    row = await db.get_built_melody(melody_id)
-    if not row:
-        raise HTTPException(status_code=404, detail=f"Built melody '{melody_id}' not found")
-    new_value = not row.get("is_builtin", False)
-    await db.update_builtin_flag(melody_id, new_value)
-    return await get_built_melody(melody_id)
-
-
 # ============================================================================
 # Build Actions
 # ============================================================================
 
-async def _do_build(melody_id: str) -> BuiltMelodyInDB:
-    """Internal: build both binary and PROGMEM code, return updated record."""
-    await build_binary(melody_id)
-    return await build_builtin_code(melody_id)
-
-
 async def build_binary(melody_id: str) -> BuiltMelodyInDB:
     """Parse steps and write a .bsm binary file to storage."""
     row = await db.get_built_melody(melody_id)
@@ -273,48 +236,6 @@ async def get_binary_path(melody_id: str) -> Optional[Path]:
     return path
 
 
-async def generate_builtin_list() -> str:
-    """Generate a C++ header with PROGMEM arrays for all is_builtin archetypes."""
-    rows = await db.list_built_melodies()
-    builtin_rows = [r for r in rows if r.get("is_builtin")]
-
-    if not builtin_rows:
-        return "// No built-in archetypes defined.\n"
-
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    parts = [
-        f"// Auto-generated Built-in Archetype List",
-        f"// Generated: {timestamp}",
-        f"// Total built-ins: {len(builtin_rows)}",
-        "",
-        "#pragma once",
-        "#include <avr/pgmspace.h>",
-        "",
-    ]
-
-    entry_refs = []
-    for row in builtin_rows:
-        values = steps_string_to_values(row["steps"])
-        array_name = f"melody_builtin_{row['name'].lower().replace(' ', '_')}"
-        display_name = row["name"].replace("_", " ").title()
-        pid = row.get("pid") or f"builtin_{row['name'].lower()}"
-
-        parts.append(f"// {display_name}  |  PID: {pid}  |  Steps: {len(values)}")
-        parts.append(format_melody_array(row["name"].lower().replace(" ", "_"), values))
-        parts.append("")
-        entry_refs.append((display_name, pid, array_name, len(values)))
-
-    # Generate MELODY_LIBRARY array
-    parts.append("// --- MELODY_LIBRARY entries ---")
-    parts.append("// Add these to your firmware's MELODY_LIBRARY[] array:")
-    parts.append("// {")
-    for display_name, pid, array_name, step_count in entry_refs:
-        parts.append(f'//     {{ "{display_name}", "{pid}", {array_name}, {step_count} }},')
-    parts.append("// };")
-
-    return "\n".join(parts)
-
-
 # ============================================================================
 # Assignment
 # ============================================================================
@@ -330,9 +251,6 @@ async def assign_to_melody(built_id: str, firestore_melody_id: str) -> BuiltMelo
         assigned.append(firestore_melody_id)
         await db.update_assigned_melody_ids(built_id, assigned)
 
-    # Clear outdated flag on the melody being assigned
-    await _flag_melodies_outdated([firestore_melody_id], False)
-
     return await get_built_melody(built_id)
 
 
@@ -344,10 +262,6 @@ async def unassign_from_melody(built_id: str, firestore_melody_id: str) -> Built
 
     assigned = [mid for mid in row.get("assigned_melody_ids", []) if mid != firestore_melody_id]
     await db.update_assigned_melody_ids(built_id, assigned)
-
-    # Flag the melody as outdated since it no longer has an archetype
-    await _flag_melodies_outdated([firestore_melody_id], True)
-
     return await get_built_melody(built_id)
 
 
@@ -358,48 +272,3 @@ async def get_built_melody_for_firestore_id(firestore_melody_id: str) -> Optiona
         if firestore_melody_id in row.get("assigned_melody_ids", []):
             return _row_to_built_melody(row)
     return None
-
-
-# ============================================================================
-# Outdated Flag Helpers
-# ============================================================================
-
-async def _flag_melodies_outdated(melody_ids: List[str], outdated: bool) -> None:
-    """Set or clear the outdated_archetype flag on a list of Firestore melody IDs.
-
-    This updates both SQLite (melody_drafts) and Firestore (published melodies).
-    We import inline to avoid circular imports.
-    """
-    if not melody_ids:
-        return
-
-    try:
-        from melodies import database as melody_db
-        from shared.firebase import get_db as get_firestore
-    except ImportError:
-        logger.warning("Could not import melody/firebase modules — skipping outdated flag update")
-        return
-
-    firestore_db = get_firestore()
-
-    for melody_id in melody_ids:
-        try:
-            row = await melody_db.get_melody(melody_id)
-            if not row:
-                continue
-
-            data = row["data"]
-            info = dict(data.get("information", {}))
-            info["outdated_archetype"] = outdated
-            data["information"] = info
-
-            await melody_db.update_melody(melody_id, data)
-
-            # If published, also update Firestore
-            if row.get("status") == "published":
-                doc_ref = firestore_db.collection("melodies").document(melody_id)
-                doc_ref.update({"information.outdated_archetype": outdated})
-
-            logger.info(f"Set outdated_archetype={outdated} on melody {melody_id}")
-        except Exception as e:
-            logger.error(f"Failed to set outdated flag on melody {melody_id}: {e}")

backend/config.py

@@ -1,5 +1,5 @@
 from pydantic_settings import BaseSettings
-from typing import List, Dict, Any
+from typing import List
 import json
 
 
@@ -20,19 +20,14 @@ class Settings(BaseSettings):
     mqtt_admin_password: str = ""
     mqtt_secret: str = "change-me-in-production"
     mosquitto_password_file: str = "/etc/mosquitto/passwd"
-    mqtt_client_id: str = "bellsystems-admin-panel"
 
-    # SQLite (local application database)
+    # SQLite (MQTT data storage)
-    sqlite_db_path: str = "./data/database.db"
+    sqlite_db_path: str = "./mqtt_data.db"
     mqtt_data_retention_days: int = 90
 
-    # Postgres
-    database_url: str = "postgresql+asyncpg://bellsystems_user:password@postgres:5432/bellsystems_db"
-
     # Local file storage
     built_melodies_storage_path: str = "./storage/built_melodies"
     firmware_storage_path: str = "./storage/firmware"
-    flash_assets_storage_path: str = "./storage/flash_assets"
 
     # Email (Resend)
     resend_api_key: str = "re_placeholder_change_me"
@@ -42,30 +37,6 @@ class Settings(BaseSettings):
     backend_cors_origins: str = '["http://localhost:5173"]'
     debug: bool = True
 
-    # Nextcloud WebDAV
-    nextcloud_url: str = ""
-    nextcloud_username: str = ""       # WebDAV login & URL path username
-    nextcloud_password: str = ""       # Use an app password for better security
-    nextcloud_dav_user: str = ""       # Override URL path username if different from login
-    nextcloud_base_path: str = "BellSystems"
-
-    # IMAP/SMTP Email
-    imap_host: str = ""
-    imap_port: int = 993
-    imap_username: str = ""
-    imap_password: str = ""
-    imap_use_ssl: bool = True
-    smtp_host: str = ""
-    smtp_port: int = 587
-    smtp_username: str = ""
-    smtp_password: str = ""
-    smtp_use_tls: bool = True
-    email_sync_interval_minutes: int = 15
-    # Multi-mailbox config (JSON array). If empty, legacy single-account IMAP/SMTP is used.
-    # Example item:
-    # {"key":"sales","label":"Sales","email":"sales@bellsystems.gr","imap_host":"...","imap_username":"...","imap_password":"...","smtp_host":"...","smtp_username":"...","smtp_password":"...","sync_inbound":true,"allow_send":true}
-    mail_accounts_json: str = "[]"
-
     # Auto-deploy (Gitea webhook)
     deploy_secret: str = ""
     deploy_project_path: str = "/app"
@@ -74,14 +45,6 @@ class Settings(BaseSettings):
     def cors_origins(self) -> List[str]:
         return json.loads(self.backend_cors_origins)
 
-    @property
-    def mail_accounts(self) -> List[Dict[str, Any]]:
-        try:
-            raw = json.loads(self.mail_accounts_json or "[]")
-            return raw if isinstance(raw, list) else []
-        except Exception:
-            return []
-
     model_config = {"env_file": ".env", "extra": "ignore"}
 
 

backend/crm/comms_router.py

@@ -1,427 +0,0 @@
-import base64
-import json
-from fastapi import APIRouter, Depends, HTTPException, Query, Form, File, UploadFile
-from pydantic import BaseModel
-from typing import List, Optional
-
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from config import settings
-from crm.models import CommCreate, CommUpdate, CommInDB, CommListResponse, MediaCreate, MediaDirection
-from crm import service
-from crm import email_sync
-from crm.mail_accounts import get_mail_accounts
-
-router = APIRouter(prefix="/api/crm/comms", tags=["crm-comms"])
-
-
-class EmailSendResponse(BaseModel):
-    entry: dict
-
-
-class EmailSyncResponse(BaseModel):
-    new_count: int
-
-
-class MailListResponse(BaseModel):
-    entries: list
-    total: int
-
-
-@router.get("/latest-batch", response_model=dict)
-async def latest_comm_batch(
-    ids: str = Query(..., description="Comma-separated customer IDs"),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Return the latest comm summary (id, type, occurred_at) keyed by customer_id."""
-    customer_ids = [i.strip() for i in ids.split(",") if i.strip()]
-    return await service.get_latest_comm_batch(customer_ids)
-
-
-@router.get("/all", response_model=CommListResponse)
-async def list_all_comms(
-    type: Optional[str] = Query(None),
-    direction: Optional[str] = Query(None),
-    limit: int = Query(200, le=500),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    entries = await service.list_all_comms(type=type, direction=direction, limit=limit)
-    return CommListResponse(entries=entries, total=len(entries))
-
-
-@router.get("", response_model=CommListResponse)
-async def list_comms(
-    customer_id: str = Query(...),
-    type: Optional[str] = Query(None),
-    direction: Optional[str] = Query(None),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    entries = await service.list_comms(customer_id=customer_id, type=type, direction=direction)
-    return CommListResponse(entries=entries, total=len(entries))
-
-
-@router.post("", response_model=CommInDB, status_code=201)
-async def create_comm(
-    body: CommCreate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return await service.create_comm(body)
-
-
-@router.get("/email/all", response_model=MailListResponse)
-async def list_all_emails(
-    direction: Optional[str] = Query(None),
-    customers_only: bool = Query(False),
-    mailbox: Optional[str] = Query(None, description="sales|support|both|all or account key"),
-    limit: int = Query(500, le=1000),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Return all email comms (all senders + unmatched), for the Mail page."""
-    selected_accounts = None
-    if mailbox and mailbox not in {"all", "both"}:
-      if mailbox == "sales":
-          selected_accounts = ["sales"]
-      elif mailbox == "support":
-          selected_accounts = ["support"]
-      else:
-          selected_accounts = [mailbox]
-    entries = await service.list_all_emails(
-        direction=direction,
-        customers_only=customers_only,
-        mail_accounts=selected_accounts,
-        limit=limit,
-    )
-    return MailListResponse(entries=entries, total=len(entries))
-
-
-@router.get("/email/accounts")
-async def list_mail_accounts(
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    accounts = get_mail_accounts()
-    return {
-        "accounts": [
-            {
-                "key": a["key"],
-                "label": a["label"],
-                "email": a["email"],
-                "sync_inbound": bool(a.get("sync_inbound")),
-                "allow_send": bool(a.get("allow_send")),
-            }
-            for a in accounts
-        ]
-    }
-
-
-@router.get("/email/check")
-async def check_new_emails(
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Lightweight check: returns how many emails are on the server vs. stored locally."""
-    return await email_sync.check_new_emails()
-
-
-# Email endpoints — must be before /{comm_id} wildcard routes
-@router.post("/email/send", response_model=EmailSendResponse)
-async def send_email_endpoint(
-    customer_id: Optional[str] = Form(None),
-    from_account: Optional[str] = Form(None),
-    to: str = Form(...),
-    subject: str = Form(...),
-    body: str = Form(...),
-    body_html: str = Form(""),
-    cc: str = Form("[]"),  # JSON-encoded list of strings
-    files: List[UploadFile] = File(default=[]),
-    user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    if not get_mail_accounts():
-        raise HTTPException(status_code=503, detail="SMTP not configured")
-    try:
-        cc_list: List[str] = json.loads(cc) if cc else []
-    except Exception:
-        cc_list = []
-
-    # Read all uploaded files into memory
-    file_attachments = []
-    for f in files:
-        content = await f.read()
-        mime_type = f.content_type or "application/octet-stream"
-        file_attachments.append((f.filename, content, mime_type))
-
-    from crm.email_sync import send_email
-    try:
-        entry = await send_email(
-            customer_id=customer_id or None,
-            from_account=from_account,
-            to=to,
-            subject=subject,
-            body=body,
-            body_html=body_html,
-            cc=cc_list,
-            sent_by=user.name or user.sub,
-            file_attachments=file_attachments if file_attachments else None,
-        )
-    except RuntimeError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-    return EmailSendResponse(entry=entry)
-
-
-@router.post("/email/sync", response_model=EmailSyncResponse)
-async def sync_email_endpoint(
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    if not get_mail_accounts():
-        raise HTTPException(status_code=503, detail="IMAP not configured")
-    from crm.email_sync import sync_emails
-    new_count = await sync_emails()
-    return EmailSyncResponse(new_count=new_count)
-
-
-class SaveInlineRequest(BaseModel):
-    data_uri: str
-    filename: str
-    subfolder: str = "received_media"
-    mime_type: Optional[str] = None
-
-
-async def _resolve_customer_folder(customer_id: str) -> str:
-    """Return the Nextcloud folder_id for a customer (falls back to customer_id)."""
-    from shared.firebase import get_db as get_firestore
-    firestore_db = get_firestore()
-    doc = firestore_db.collection("crm_customers").document(customer_id).get()
-    if not doc.exists:
-        raise HTTPException(status_code=404, detail="Customer not found")
-    data = doc.to_dict()
-    return data.get("folder_id") or customer_id
-
-
-async def _upload_to_nc(folder_id: str, subfolder: str, filename: str,
-                        content: bytes, mime_type: str, customer_id: str,
-                        uploaded_by: str, tags: list[str]) -> dict:
-    from crm import nextcloud
-    target_folder = f"customers/{folder_id}/{subfolder}"
-    file_path = f"{target_folder}/{filename}"
-    await nextcloud.ensure_folder(target_folder)
-    await nextcloud.upload_file(file_path, content, mime_type)
-    media = await service.create_media(MediaCreate(
-        customer_id=customer_id,
-        filename=filename,
-        nextcloud_path=file_path,
-        mime_type=mime_type,
-        direction=MediaDirection.received,
-        tags=tags,
-        uploaded_by=uploaded_by,
-    ))
-    return {"ok": True, "media_id": media.id, "nextcloud_path": file_path}
-
-
-@router.post("/email/{comm_id}/save-inline")
-async def save_email_inline_image(
-    comm_id: str,
-    body: SaveInlineRequest,
-    user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """Save an inline image (data-URI from email HTML body) to Nextcloud."""
-    comm = await service.get_comm(comm_id)
-    customer_id = comm.customer_id
-    if not customer_id:
-        raise HTTPException(status_code=400, detail="This email is not linked to a customer")
-
-    folder_id = await _resolve_customer_folder(customer_id)
-
-    # Parse data URI
-    data_uri = body.data_uri
-    mime_type = body.mime_type or "image/png"
-    if "," in data_uri:
-        header, encoded = data_uri.split(",", 1)
-        try:
-            mime_type = header.split(":")[1].split(";")[0]
-        except Exception:
-            pass
-    else:
-        encoded = data_uri
-    try:
-        content = base64.b64decode(encoded)
-    except Exception:
-        raise HTTPException(status_code=400, detail="Invalid base64 data")
-
-    return await _upload_to_nc(
-        folder_id, body.subfolder, body.filename,
-        content, mime_type, customer_id,
-        user.name or user.sub, ["email-inline-image"],
-    )
-
-
-@router.post("/email/{comm_id}/save-attachment/{attachment_index}")
-async def save_email_attachment(
-    comm_id: str,
-    attachment_index: int,
-    filename: str = Form(...),
-    subfolder: str = Form("received_media"),
-    user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Re-fetch a specific attachment from IMAP (by index in the email's attachment list)
-    and save it to the customer's Nextcloud media folder.
-    """
-    import asyncio
-    comm = await service.get_comm(comm_id)
-    customer_id = comm.customer_id
-    if not customer_id:
-        raise HTTPException(status_code=400, detail="This email is not linked to a customer")
-
-    ext_message_id = comm.ext_message_id
-    if not ext_message_id:
-        raise HTTPException(status_code=400, detail="No message ID stored for this email")
-
-    attachments_meta = comm.attachments or []
-    if attachment_index < 0 or attachment_index >= len(attachments_meta):
-        raise HTTPException(status_code=400, detail="Attachment index out of range")
-
-    att_meta = attachments_meta[attachment_index]
-    mime_type = att_meta.content_type or "application/octet-stream"
-    from crm.mail_accounts import account_by_key, account_by_email
-    account = account_by_key(comm.mail_account) or account_by_email(comm.from_addr)
-    if not account:
-        raise HTTPException(status_code=400, detail="Email account config not found for this message")
-
-    # Re-fetch from IMAP in executor
-    def _fetch_attachment():
-        import imaplib, email as _email
-        if account.get("imap_use_ssl"):
-            imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
-        else:
-            imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
-        imap.login(account["imap_username"], account["imap_password"])
-        imap.select(account.get("imap_inbox", "INBOX"))
-
-        # Search by Message-ID header
-        _, data = imap.search(None, f'HEADER Message-ID "{ext_message_id}"')
-        uids = data[0].split() if data[0] else []
-        if not uids:
-            raise ValueError(f"Message not found on IMAP server: {ext_message_id}")
-
-        _, msg_data = imap.fetch(uids[0], "(RFC822)")
-        raw = msg_data[0][1]
-        msg = _email.message_from_bytes(raw)
-        imap.logout()
-
-        # Walk attachments in order — find the one at attachment_index
-        found_idx = 0
-        for part in msg.walk():
-            cd = str(part.get("Content-Disposition", ""))
-            if "attachment" not in cd:
-                continue
-            if found_idx == attachment_index:
-                payload = part.get_payload(decode=True)
-                if payload is None:
-                    raise ValueError("Attachment payload is empty")
-                return payload
-            found_idx += 1
-
-        raise ValueError(f"Attachment index {attachment_index} not found in message")
-
-    loop = asyncio.get_event_loop()
-    try:
-        content = await loop.run_in_executor(None, _fetch_attachment)
-    except ValueError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    except Exception as e:
-        raise HTTPException(status_code=502, detail=f"IMAP fetch failed: {e}")
-
-    folder_id = await _resolve_customer_folder(customer_id)
-    return await _upload_to_nc(
-        folder_id, subfolder, filename,
-        content, mime_type, customer_id,
-        user.name or user.sub, ["email-attachment"],
-    )
-
-
-class BulkDeleteRequest(BaseModel):
-    ids: List[str]
-
-
-class ToggleImportantRequest(BaseModel):
-    important: bool
-
-
-class ToggleReadRequest(BaseModel):
-    read: bool
-
-
-@router.post("/bulk-delete", status_code=200)
-async def bulk_delete_comms(
-    body: BulkDeleteRequest,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    # Try remote IMAP delete for email rows first (best-effort), then local delete.
-    for comm_id in body.ids:
-        try:
-            comm = await service.get_comm(comm_id)
-            if comm.type == "email" and comm.ext_message_id:
-                await email_sync.delete_remote_email(
-                    comm.ext_message_id,
-                    comm.mail_account,
-                    comm.from_addr,
-                )
-        except Exception:
-            # Keep delete resilient; local delete still proceeds.
-            pass
-    count = await service.delete_comms_bulk(body.ids)
-    return {"deleted": count}
-
-
-@router.patch("/{comm_id}/important", response_model=CommInDB)
-async def set_comm_important(
-    comm_id: str,
-    body: ToggleImportantRequest,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return await service.set_comm_important(comm_id, body.important)
-
-
-@router.patch("/{comm_id}/read", response_model=CommInDB)
-async def set_comm_read(
-    comm_id: str,
-    body: ToggleReadRequest,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    try:
-        comm = await service.get_comm(comm_id)
-        if comm.type == "email" and comm.ext_message_id:
-            await email_sync.set_remote_read(
-                comm.ext_message_id,
-                comm.mail_account,
-                comm.from_addr,
-                body.read,
-            )
-    except Exception:
-        pass
-    return await service.set_comm_read(comm_id, body.read)
-
-
-@router.put("/{comm_id}", response_model=CommInDB)
-async def update_comm(
-    comm_id: str,
-    body: CommUpdate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return await service.update_comm(comm_id, body)
-
-
-@router.delete("/{comm_id}", status_code=204)
-async def delete_comm(
-    comm_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    try:
-        comm = await service.get_comm(comm_id)
-        if comm.type == "email" and comm.ext_message_id:
-            await email_sync.delete_remote_email(
-                comm.ext_message_id,
-                comm.mail_account,
-                comm.from_addr,
-            )
-    except Exception:
-        pass
-    await service.delete_comm(comm_id)

backend/crm/customers_router.py

@@ -1,336 +0,0 @@
-import asyncio
-import logging
-from fastapi import APIRouter, Depends, Query, BackgroundTasks, Body
-from typing import Optional
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from crm.models import CustomerCreate, CustomerUpdate, CustomerInDB, CustomerListResponse, TransactionEntry
-from crm import service, nextcloud
-from config import settings
-from database.postgres import get_pg_session
-from shared.audit import log_action
-
-router = APIRouter(prefix="/api/crm/customers", tags=["crm-customers"])
-logger = logging.getLogger(__name__)
-
-# ── Diff helpers ──────────────────────────────────────────────────────────────
-
-_SCALAR_FIELDS = {
-    "name", "surname", "title", "organization", "religion", "language",
-    "relationship_status", "nextcloud_folder",
-}
-_SKIP_FIELDS = {"updated_at", "firestore_id", "id"}
-
-
-def _scalar_diff(old, new) -> dict:
-    result = {}
-    for f in _SCALAR_FIELDS:
-        ov = getattr(old, f, None)
-        nv = getattr(new, f, None)
-        if ov != nv:
-            result[f] = {"old": ov, "new": nv}
-    return result
-
-
-def _list_diff(field: str, old_list: list, new_list: list, label_fn) -> dict:
-    old_labels = {label_fn(i) for i in (old_list or [])}
-    new_labels = {label_fn(i) for i in (new_list or [])}
-    added   = new_labels - old_labels
-    removed = old_labels - new_labels
-    result = {}
-    if added:
-        result[f"{field}.added"] = {"old": None, "new": sorted(added)}
-    if removed:
-        result[f"{field}.removed"] = {"old": sorted(removed), "new": None}
-    return result
-
-
-def _customer_diff(old, new, changed_fields: set) -> dict:
-    changes = _scalar_diff(old, new)
-
-    # contacts — keyed by type+value string
-    if "contacts" in changed_fields:
-        changes.update(_list_diff(
-            "contacts",
-            old.contacts or [],
-            new.contacts or [],
-            lambda c: f"{c.get('type','?')}:{c.get('value','?')}" if isinstance(c, dict)
-                      else f"{getattr(c,'type','?')}:{getattr(c,'value','?')}",
-        ))
-
-    # location — flatten to individual sub-fields
-    if "location" in changed_fields:
-        old_loc = old.location or {}
-        new_loc = new.location or {}
-        if isinstance(old_loc, object) and not isinstance(old_loc, dict):
-            old_loc = old_loc.model_dump() if hasattr(old_loc, "model_dump") else {}
-        if isinstance(new_loc, object) and not isinstance(new_loc, dict):
-            new_loc = new_loc.model_dump() if hasattr(new_loc, "model_dump") else {}
-        for k in set(old_loc) | set(new_loc):
-            ov, nv = old_loc.get(k), new_loc.get(k)
-            if ov != nv:
-                changes[f"location.{k}"] = {"old": ov, "new": nv}
-
-    # tags
-    if "tags" in changed_fields:
-        old_tags = set(old.tags or [])
-        new_tags = set(new.tags or [])
-        if old_tags != new_tags:
-            changes["tags"] = {"old": sorted(old_tags), "new": sorted(new_tags)}
-
-    return changes
-
-
-@router.get("", response_model=CustomerListResponse)
-async def list_customers(
-    search: Optional[str] = Query(None),
-    tag: Optional[str] = Query(None),
-    sort: Optional[str] = Query(None),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    customers = service.list_customers(search=search, tag=tag, sort=sort)
-    if sort == "latest_comm":
-        customers = await service.list_customers_sorted_by_latest_comm(customers)
-    return CustomerListResponse(customers=customers, total=len(customers))
-
-
-@router.get("/tags", response_model=list[str])
-def list_tags(
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return service.list_all_tags()
-
-
-@router.get("/{customer_id}", response_model=CustomerInDB)
-def get_customer(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return service.get_customer(customer_id)
-
-
-@router.post("", response_model=CustomerInDB, status_code=201)
-async def create_customer(
-    body: CustomerCreate,
-    background_tasks: BackgroundTasks,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    customer = service.create_customer(body)
-    if settings.nextcloud_url:
-        background_tasks.add_task(_init_nextcloud_folder, customer)
-    label = " ".join(filter(None, [customer.name, customer.surname])) or customer.organization or customer.id
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "customer",
-                     customer.id, label)
-    return customer
-
-
-async def _init_nextcloud_folder(customer) -> None:
-    try:
-        nc_path = service.get_customer_nc_path(customer)
-        base = f"customers/{nc_path}"
-        for sub in ("media", "documents", "sent", "received"):
-            await nextcloud.ensure_folder(f"{base}/{sub}")
-        await nextcloud.write_info_file(base, customer.name, customer.id)
-    except Exception as e:
-        logger.warning("Nextcloud folder init failed for customer %s: %s", customer.id, e)
-
-
-@router.put("/{customer_id}", response_model=CustomerInDB)
-async def update_customer(
-    customer_id: str,
-    body: CustomerUpdate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    old = service.get_customer(customer_id)
-    customer = service.update_customer(customer_id, body)
-    label = " ".join(filter(None, [customer.name, customer.surname])) or customer.organization or customer_id
-    changes = _customer_diff(old, customer, body.model_fields_set)
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "customer",
-                     customer_id, label, changes=changes or None)
-    return customer
-
-
-@router.delete("/{customer_id}", status_code=204)
-async def delete_customer(
-    customer_id: str,
-    wipe_comms: bool = Query(False),
-    wipe_files: bool = Query(False),
-    wipe_nextcloud: bool = Query(False),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    customer = service.delete_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-
-    if wipe_comms or wipe_nextcloud:
-        await service.delete_customer_comms(customer_id)
-
-    if wipe_files or wipe_nextcloud:
-        await service.delete_customer_media_entries(customer_id)
-
-    if settings.nextcloud_url:
-        folder = f"customers/{nc_path}"
-        if wipe_nextcloud:
-            try:
-                await nextcloud.delete_file(folder)
-            except Exception as e:
-                logger.warning("Could not delete NC folder for customer %s: %s", customer_id, e)
-        elif wipe_files:
-            stale_folder = f"customers/STALE_{nc_path}"
-            try:
-                await nextcloud.rename_folder(folder, stale_folder)
-            except Exception as e:
-                logger.warning("Could not rename NC folder for customer %s: %s", customer_id, e)
-
-    label = " ".join(filter(None, [customer.name, customer.surname])) or customer.organization or customer_id
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "customer",
-                     customer_id, label)
-
-
-@router.get("/{customer_id}/last-comm-direction")
-async def get_last_comm_direction(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    result = await service.get_last_comm_direction(customer_id)
-    return result
-
-
-# ── Relationship Status ───────────────────────────────────────────────────────
-
-@router.patch("/{customer_id}/relationship-status", response_model=CustomerInDB)
-async def update_relationship_status(
-    customer_id: str,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    customer = service.update_relationship_status(customer_id, body.get("status", ""))
-    label = " ".join(filter(None, [customer.name, customer.surname])) or customer.organization or customer_id
-    await log_action(db, _user.sub, _user.name or _user.email, "STATUS_CHANGE", "customer",
-                     customer_id, label, meta={"status": body.get("status", "")})
-    return customer
-
-
-# ── Technical Issues ──────────────────────────────────────────────────────────
-
-@router.post("/{customer_id}/technical-issues", response_model=CustomerInDB)
-def add_technical_issue(
-    customer_id: str,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.add_technical_issue(
-        customer_id,
-        note=body.get("note", ""),
-        opened_by=body.get("opened_by", ""),
-        date=body.get("date"),
-    )
-
-
-@router.patch("/{customer_id}/technical-issues/{index}/resolve", response_model=CustomerInDB)
-def resolve_technical_issue(
-    customer_id: str,
-    index: int,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.resolve_technical_issue(customer_id, index, body.get("resolved_by", ""))
-
-
-@router.patch("/{customer_id}/technical-issues/{index}", response_model=CustomerInDB)
-def edit_technical_issue(
-    customer_id: str,
-    index: int,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.edit_technical_issue(customer_id, index, body.get("note", ""), body.get("opened_date"))
-
-
-@router.delete("/{customer_id}/technical-issues/{index}", response_model=CustomerInDB)
-def delete_technical_issue(
-    customer_id: str,
-    index: int,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.delete_technical_issue(customer_id, index)
-
-
-# ── Install Support ───────────────────────────────────────────────────────────
-
-@router.post("/{customer_id}/install-support", response_model=CustomerInDB)
-def add_install_support(
-    customer_id: str,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.add_install_support(
-        customer_id,
-        note=body.get("note", ""),
-        opened_by=body.get("opened_by", ""),
-        date=body.get("date"),
-    )
-
-
-@router.patch("/{customer_id}/install-support/{index}/resolve", response_model=CustomerInDB)
-def resolve_install_support(
-    customer_id: str,
-    index: int,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.resolve_install_support(customer_id, index, body.get("resolved_by", ""))
-
-
-@router.patch("/{customer_id}/install-support/{index}", response_model=CustomerInDB)
-def edit_install_support(
-    customer_id: str,
-    index: int,
-    body: dict = Body(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.edit_install_support(customer_id, index, body.get("note", ""), body.get("opened_date"))
-
-
-@router.delete("/{customer_id}/install-support/{index}", response_model=CustomerInDB)
-def delete_install_support(
-    customer_id: str,
-    index: int,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.delete_install_support(customer_id, index)
-
-
-# ── Transactions ──────────────────────────────────────────────────────────────
-
-@router.post("/{customer_id}/transactions", response_model=CustomerInDB)
-def add_transaction(
-    customer_id: str,
-    body: TransactionEntry,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.add_transaction(customer_id, body)
-
-
-@router.patch("/{customer_id}/transactions/{index}", response_model=CustomerInDB)
-def update_transaction(
-    customer_id: str,
-    index: int,
-    body: TransactionEntry,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.update_transaction(customer_id, index, body)
-
-
-@router.delete("/{customer_id}/transactions/{index}", response_model=CustomerInDB)
-def delete_transaction(
-    customer_id: str,
-    index: int,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.delete_transaction(customer_id, index)

backend/crm/email_sync.py

@@ -1,837 +0,0 @@
-"""
-IMAP email sync and SMTP email send for CRM.
-Uses only stdlib imaplib/smtplib — no extra dependencies.
-Sync is run in an executor to avoid blocking the event loop.
-"""
-import asyncio
-import base64
-import email
-import email.header
-import email.utils
-import html.parser
-import imaplib
-import json
-import logging
-import re
-import smtplib
-import uuid
-from datetime import datetime, timezone
-from email.mime.base import MIMEBase
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-from email import encoders
-from typing import List, Optional, Tuple
-
-from config import settings
-import database as mqtt_db
-from crm.mail_accounts import get_mail_accounts, account_by_key, account_by_email
-
-logger = logging.getLogger("crm.email_sync")
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _decode_header(raw: str) -> str:
-    """Decode an RFC2047-encoded email header value."""
-    if not raw:
-        return ""
-    parts = email.header.decode_header(raw)
-    decoded = []
-    for part, enc in parts:
-        if isinstance(part, bytes):
-            decoded.append(part.decode(enc or "utf-8", errors="replace"))
-        else:
-            decoded.append(part)
-    return " ".join(decoded)
-
-
-class _HTMLStripper(html.parser.HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self._text = []
-
-    def handle_data(self, data):
-        self._text.append(data)
-
-    def get_text(self):
-        return " ".join(self._text)
-
-
-def _strip_html(html_str: str) -> str:
-    s = _HTMLStripper()
-    s.feed(html_str)
-    return s.get_text()
-
-
-def _extract_inline_data_images(html_body: str) -> tuple[str, list[tuple[str, bytes, str]]]:
-    """Replace data-URI images in HTML with cid: references and return inline parts.
-    Returns: (new_html, [(cid, image_bytes, mime_type), ...])
-    """
-    if not html_body:
-        return "", []
-
-    inline_parts: list[tuple[str, bytes, str]] = []
-    seen: dict[str, str] = {}  # data-uri -> cid
-
-    src_pattern = re.compile(r"""src=(['"])(data:image/[^'"]+)\1""", re.IGNORECASE)
-    data_pattern = re.compile(r"^data:(image/[a-zA-Z0-9.+-]+);base64,(.+)$", re.IGNORECASE | re.DOTALL)
-
-    def _replace(match: re.Match) -> str:
-        quote = match.group(1)
-        data_uri = match.group(2)
-
-        if data_uri in seen:
-            cid = seen[data_uri]
-            return f"src={quote}cid:{cid}{quote}"
-
-        parsed = data_pattern.match(data_uri)
-        if not parsed:
-            return match.group(0)
-
-        mime_type = parsed.group(1).lower()
-        b64_data = parsed.group(2).strip()
-        try:
-            payload = base64.b64decode(b64_data, validate=False)
-        except Exception:
-            return match.group(0)
-
-        cid = f"inline-{uuid.uuid4().hex}"
-        seen[data_uri] = cid
-        inline_parts.append((cid, payload, mime_type))
-        return f"src={quote}cid:{cid}{quote}"
-
-    return src_pattern.sub(_replace, html_body), inline_parts
-
-
-def _load_customer_email_map() -> dict[str, str]:
-    """Build a lookup of customer email -> customer_id from Firestore."""
-    from shared.firebase import get_db as get_firestore
-    firestore_db = get_firestore()
-    addr_to_customer: dict[str, str] = {}
-    for doc in firestore_db.collection("crm_customers").stream():
-        data = doc.to_dict() or {}
-        for contact in (data.get("contacts") or []):
-            if contact.get("type") == "email" and contact.get("value"):
-                addr_to_customer[str(contact["value"]).strip().lower()] = doc.id
-    return addr_to_customer
-
-
-def _get_body(msg: email.message.Message) -> tuple[str, str]:
-    """Extract (plain_text, html_body) from an email message.
-    Inline images (cid: references) are substituted with data-URIs so they
-    render correctly in a sandboxed iframe without external requests.
-    """
-    import base64 as _b64
-    plain = None
-    html_body = None
-    # Map Content-ID → data-URI for inline images
-    cid_map: dict[str, str] = {}
-
-    if msg.is_multipart():
-        for part in msg.walk():
-            ct = part.get_content_type()
-            cd = str(part.get("Content-Disposition", ""))
-            cid = part.get("Content-ID", "").strip().strip("<>")
-
-            if "attachment" in cd:
-                continue
-
-            if ct == "text/plain" and plain is None:
-                raw = part.get_payload(decode=True)
-                charset = part.get_content_charset() or "utf-8"
-                plain = raw.decode(charset, errors="replace")
-            elif ct == "text/html" and html_body is None:
-                raw = part.get_payload(decode=True)
-                charset = part.get_content_charset() or "utf-8"
-                html_body = raw.decode(charset, errors="replace")
-            elif ct.startswith("image/") and cid:
-                raw = part.get_payload(decode=True)
-                if raw:
-                    b64 = _b64.b64encode(raw).decode("ascii")
-                    cid_map[cid] = f"data:{ct};base64,{b64}"
-    else:
-        ct = msg.get_content_type()
-        payload = msg.get_payload(decode=True)
-        charset = msg.get_content_charset() or "utf-8"
-        if payload:
-            text = payload.decode(charset, errors="replace")
-            if ct == "text/plain":
-                plain = text
-            elif ct == "text/html":
-                html_body = text
-
-    # Substitute cid: references with data-URIs
-    if html_body and cid_map:
-        for cid, data_uri in cid_map.items():
-            html_body = html_body.replace(f"cid:{cid}", data_uri)
-
-    plain_text = (plain or (html_body and _strip_html(html_body)) or "").strip()
-    return plain_text, (html_body or "").strip()
-
-
-def _get_attachments(msg: email.message.Message) -> list[dict]:
-    """Extract attachment info (filename, content_type, size) without storing content."""
-    attachments = []
-    if msg.is_multipart():
-        for part in msg.walk():
-            cd = str(part.get("Content-Disposition", ""))
-            if "attachment" in cd:
-                filename = part.get_filename() or "attachment"
-                filename = _decode_header(filename)
-                ct = part.get_content_type() or "application/octet-stream"
-                payload = part.get_payload(decode=True)
-                size = len(payload) if payload else 0
-                attachments.append({"filename": filename, "content_type": ct, "size": size})
-    return attachments
-
-
-# ---------------------------------------------------------------------------
-# IMAP sync (synchronous — called via run_in_executor)
-# ---------------------------------------------------------------------------
-
-def _sync_account_emails_sync(account: dict) -> tuple[list[dict], bool]:
-    if not account.get("imap_host") or not account.get("imap_username") or not account.get("imap_password"):
-        return [], False
-    if account.get("imap_use_ssl"):
-        imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
-    else:
-        imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
-    imap.login(account["imap_username"], account["imap_password"])
-    # readonly=True prevents marking messages as \Seen while syncing.
-    imap.select(account.get("imap_inbox", "INBOX"), readonly=True)
-    _, data = imap.search(None, "ALL")
-    uids = data[0].split() if data[0] else []
-
-    results = []
-    complete = True
-    for uid in uids:
-        try:
-            _, msg_data = imap.fetch(uid, "(FLAGS RFC822)")
-            meta = msg_data[0][0] if msg_data and isinstance(msg_data[0], tuple) else b""
-            raw = msg_data[0][1]
-            msg = email.message_from_bytes(raw)
-            message_id = msg.get("Message-ID", "").strip()
-            from_addr = email.utils.parseaddr(msg.get("From", ""))[1]
-            to_addrs_raw = msg.get("To", "")
-            to_addrs = [a for _, a in email.utils.getaddresses([to_addrs_raw])]
-            subject = _decode_header(msg.get("Subject", ""))
-            date_str = msg.get("Date", "")
-            try:
-                occurred_at = email.utils.parsedate_to_datetime(date_str).isoformat()
-            except Exception:
-                occurred_at = datetime.now(timezone.utc).isoformat()
-            is_read = b"\\Seen" in (meta or b"")
-            try:
-                body, body_html = _get_body(msg)
-            except Exception:
-                body, body_html = "", ""
-            try:
-                file_attachments = _get_attachments(msg)
-            except Exception:
-                file_attachments = []
-            results.append({
-                "mail_account": account["key"],
-                "message_id": message_id,
-                "from_addr": from_addr,
-                "to_addrs": to_addrs,
-                "subject": subject,
-                "body": body,
-                "body_html": body_html,
-                "attachments": file_attachments,
-                "occurred_at": occurred_at,
-                "is_read": bool(is_read),
-            })
-        except Exception as e:
-            complete = False
-            logger.warning(f"[EMAIL SYNC] Failed to parse message uid={uid} account={account['key']}: {e}")
-    imap.logout()
-    return results, complete
-
-
-def _sync_emails_sync() -> tuple[list[dict], bool]:
-    all_msgs: list[dict] = []
-    all_complete = True
-    # Deduplicate by physical inbox source. Aliases often share the same mailbox.
-    seen_sources: set[tuple] = set()
-    for acc in get_mail_accounts():
-        if not acc.get("sync_inbound"):
-            continue
-        source = (
-            (acc.get("imap_host") or "").lower(),
-            int(acc.get("imap_port") or 0),
-            (acc.get("imap_username") or "").lower(),
-            (acc.get("imap_inbox") or "INBOX").upper(),
-        )
-        if source in seen_sources:
-            continue
-        seen_sources.add(source)
-        msgs, complete = _sync_account_emails_sync(acc)
-        all_msgs.extend(msgs)
-        all_complete = all_complete and complete
-    return all_msgs, all_complete
-
-
-async def sync_emails() -> int:
-    """
-    Pull emails from IMAP, match against CRM customers, store new ones.
-    Returns count of new entries created.
-    """
-    if not get_mail_accounts():
-        return 0
-
-    loop = asyncio.get_event_loop()
-    try:
-        messages, fetch_complete = await loop.run_in_executor(None, _sync_emails_sync)
-    except Exception as e:
-        logger.error(f"[EMAIL SYNC] IMAP connect/fetch failed: {e}")
-        raise
-
-    db = await mqtt_db.get_db()
-
-    # Load all customer email contacts into a flat lookup: email -> customer_id
-    addr_to_customer = _load_customer_email_map()
-
-    # Load already-synced message-ids from DB
-    rows = await db.execute_fetchall(
-        "SELECT id, ext_message_id, COALESCE(mail_account, '') as mail_account, direction, is_read, customer_id "
-        "FROM crm_comms_log WHERE type='email' AND ext_message_id IS NOT NULL"
-    )
-    known_map = {
-        (r[1], r[2] or ""): {
-            "id": r[0],
-            "direction": r[3],
-            "is_read": int(r[4] or 0),
-            "customer_id": r[5],
-        }
-        for r in rows
-    }
-
-    new_count = 0
-    now = datetime.now(timezone.utc).isoformat()
-    server_ids_by_account: dict[str, set[str]] = {}
-    # Global inbound IDs from server snapshot, used to avoid account-classification delete oscillation.
-    inbound_server_ids: set[str] = set()
-    accounts = get_mail_accounts()
-    accounts_by_email = {a["email"].lower(): a for a in accounts}
-    # Initialize tracked inbound accounts even if inbox is empty.
-    for a in accounts:
-        if a.get("sync_inbound"):
-            server_ids_by_account[a["key"]] = set()
-
-    for msg in messages:
-        mid = msg["message_id"]
-        fetch_account_key = (msg.get("mail_account") or "").strip().lower()
-        from_addr = msg["from_addr"].lower()
-        to_addrs = [a.lower() for a in msg["to_addrs"]]
-
-        sender_acc = accounts_by_email.get(from_addr)
-        if sender_acc:
-            direction = "outbound"
-            resolved_account_key = sender_acc["key"]
-            customer_addrs = to_addrs
-        else:
-            direction = "inbound"
-            target_acc = None
-            for addr in to_addrs:
-                if addr in accounts_by_email:
-                    target_acc = accounts_by_email[addr]
-                    break
-            resolved_account_key = (target_acc["key"] if target_acc else fetch_account_key)
-            customer_addrs = [from_addr]
-            if target_acc and not target_acc.get("sync_inbound"):
-                # Ignore inbound for non-synced aliases (e.g. info/news).
-                continue
-
-        if direction == "inbound" and mid and resolved_account_key in server_ids_by_account:
-            server_ids_by_account[resolved_account_key].add(mid)
-            inbound_server_ids.add(mid)
-        # Find matching customer (may be None - we still store the email)
-        customer_id = None
-        for addr in customer_addrs:
-            if addr in addr_to_customer:
-                customer_id = addr_to_customer[addr]
-                break
-
-        if mid and (mid, resolved_account_key) in known_map:
-            existing = known_map[(mid, resolved_account_key)]
-            # Backfill customer linkage for rows created without customer_id.
-            if customer_id and not existing.get("customer_id"):
-                await db.execute(
-                    "UPDATE crm_comms_log SET customer_id=? WHERE id=?",
-                    (customer_id, existing["id"]),
-                )
-            # Existing inbound message: sync read/unread state from server.
-            if direction == "inbound":
-                server_read = 1 if msg.get("is_read") else 0
-                await db.execute(
-                    "UPDATE crm_comms_log SET is_read=? "
-                    "WHERE type='email' AND direction='inbound' AND ext_message_id=? AND mail_account=?",
-                    (server_read, mid, resolved_account_key),
-                )
-            continue  # already stored
-
-        attachments_json = json.dumps(msg.get("attachments") or [])
-        to_addrs_json = json.dumps(to_addrs)
-
-        entry_id = str(uuid.uuid4())
-        await db.execute(
-            """INSERT INTO crm_comms_log
-               (id, customer_id, type, mail_account, direction, subject, body, body_html, attachments,
-                ext_message_id, from_addr, to_addrs, logged_by, occurred_at, created_at, is_read)
-               VALUES (?, ?, 'email', ?, ?, ?, ?, ?, ?, ?, ?, ?, 'system', ?, ?, ?)""",
-            (entry_id, customer_id, resolved_account_key, direction, msg["subject"], msg["body"],
-             msg.get("body_html", ""), attachments_json,
-             mid, from_addr, to_addrs_json, msg["occurred_at"], now, 1 if msg.get("is_read") else 0),
-        )
-        new_count += 1
-
-    # Mirror remote deletes based on global inbound message-id snapshot.
-    # To avoid transient IMAP inconsistency causing add/remove oscillation,
-    # require two consecutive "missing" syncs before local deletion.
-    sync_keys = [a["key"] for a in accounts if a.get("sync_inbound")]
-    if sync_keys and fetch_complete:
-        placeholders = ",".join("?" for _ in sync_keys)
-        local_rows = await db.execute_fetchall(
-            f"SELECT id, ext_message_id, mail_account FROM crm_comms_log "
-            f"WHERE type='email' AND direction='inbound' AND mail_account IN ({placeholders}) "
-            "AND ext_message_id IS NOT NULL",
-            sync_keys,
-        )
-        to_delete: list[str] = []
-        for row in local_rows:
-            row_id, ext_id, acc_key = row[0], row[1], row[2]
-            if not ext_id:
-                continue
-            state_key = f"missing_email::{acc_key}::{ext_id}"
-            if ext_id in inbound_server_ids:
-                await db.execute("DELETE FROM crm_sync_state WHERE key = ?", (state_key,))
-                continue
-            prev = await db.execute_fetchall("SELECT value FROM crm_sync_state WHERE key = ?", (state_key,))
-            prev_count = int(prev[0][0]) if prev and (prev[0][0] or "").isdigit() else 0
-            new_count = prev_count + 1
-            await db.execute(
-                "INSERT INTO crm_sync_state (key, value) VALUES (?, ?) "
-                "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
-                (state_key, str(new_count)),
-            )
-            if new_count >= 2:
-                to_delete.append(row_id)
-                await db.execute("DELETE FROM crm_sync_state WHERE key = ?", (state_key,))
-        if to_delete:
-            del_ph = ",".join("?" for _ in to_delete)
-            await db.execute(f"DELETE FROM crm_comms_log WHERE id IN ({del_ph})", to_delete)
-
-    if new_count or server_ids_by_account:
-        await db.commit()
-
-    # Update last sync time
-    await db.execute(
-        "INSERT INTO crm_sync_state (key, value) VALUES ('last_email_sync', ?) "
-        "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
-        (now,),
-    )
-    await db.commit()
-
-    logger.info(f"[EMAIL SYNC] Done — {new_count} new emails stored")
-    return new_count
-
-
-# ---------------------------------------------------------------------------
-# Lightweight new-mail check (synchronous — called via run_in_executor)
-# ---------------------------------------------------------------------------
-
-def _check_server_count_sync() -> int:
-    # Keep this for backward compatibility; no longer used by check_new_emails().
-    total = 0
-    seen_sources: set[tuple] = set()
-    for acc in get_mail_accounts():
-        if not acc.get("sync_inbound"):
-            continue
-        source = (
-            (acc.get("imap_host") or "").lower(),
-            int(acc.get("imap_port") or 0),
-            (acc.get("imap_username") or "").lower(),
-            (acc.get("imap_inbox") or "INBOX").upper(),
-        )
-        if source in seen_sources:
-            continue
-        seen_sources.add(source)
-        if acc.get("imap_use_ssl"):
-            imap = imaplib.IMAP4_SSL(acc["imap_host"], int(acc["imap_port"]))
-        else:
-            imap = imaplib.IMAP4(acc["imap_host"], int(acc["imap_port"]))
-        imap.login(acc["imap_username"], acc["imap_password"])
-        imap.select(acc.get("imap_inbox", "INBOX"), readonly=True)
-        _, data = imap.search(None, "ALL")
-        total += len(data[0].split()) if data[0] else 0
-        imap.logout()
-    return total
-
-
-async def check_new_emails() -> dict:
-    """
-    Compare server message count vs. locally stored count.
-    Returns {"new_count": int} — does NOT download or store anything.
-    """
-    if not get_mail_accounts():
-        return {"new_count": 0}
-
-    loop = asyncio.get_event_loop()
-    try:
-        # Reuse same account-resolution logic as sync to avoid false positives.
-        messages, _ = await loop.run_in_executor(None, _sync_emails_sync)
-    except Exception as e:
-        logger.warning(f"[EMAIL CHECK] IMAP check failed: {e}")
-        raise
-
-    accounts = get_mail_accounts()
-    accounts_by_email = {a["email"].lower(): a for a in accounts}
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT ext_message_id, COALESCE(mail_account, '') as mail_account FROM crm_comms_log "
-        "WHERE type='email' AND ext_message_id IS NOT NULL"
-    )
-    known_ids = {(r[0], r[1] or "") for r in rows}
-
-    new_count = 0
-    for msg in messages:
-        mid = (msg.get("message_id") or "").strip()
-        if not mid:
-            continue
-        fetch_account_key = (msg.get("mail_account") or "").strip().lower()
-        from_addr = (msg.get("from_addr") or "").lower()
-        to_addrs = [(a or "").lower() for a in (msg.get("to_addrs") or [])]
-
-        sender_acc = accounts_by_email.get(from_addr)
-        if sender_acc:
-            # Outbound copy in mailbox; not part of "new inbound mail" banner.
-            continue
-
-        target_acc = None
-        for addr in to_addrs:
-            if addr in accounts_by_email:
-                target_acc = accounts_by_email[addr]
-                break
-        resolved_account_key = (target_acc["key"] if target_acc else fetch_account_key)
-        if target_acc and not target_acc.get("sync_inbound"):
-            continue
-        if (mid, resolved_account_key) not in known_ids:
-            new_count += 1
-
-    return {"new_count": new_count}
-
-
-# ---------------------------------------------------------------------------
-# SMTP send (synchronous — called via run_in_executor)
-# ---------------------------------------------------------------------------
-
-def _append_to_sent_sync(account: dict, raw_message: bytes) -> None:
-    """Best-effort append of sent MIME message to IMAP Sent folder."""
-    if not raw_message:
-        return
-    try:
-        if account.get("imap_use_ssl"):
-            imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
-        else:
-            imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
-        imap.login(account["imap_username"], account["imap_password"])
-
-        preferred = str(account.get("imap_sent") or "Sent").strip() or "Sent"
-        candidates = [preferred, "Sent", "INBOX.Sent", "Sent Items", "INBOX.Sent Items"]
-        seen = set()
-        ordered_candidates = []
-        for name in candidates:
-            key = name.lower()
-            if key not in seen:
-                seen.add(key)
-                ordered_candidates.append(name)
-
-        appended = False
-        for mailbox in ordered_candidates:
-            try:
-                status, _ = imap.append(mailbox, "\\Seen", None, raw_message)
-                if status == "OK":
-                    appended = True
-                    break
-            except Exception:
-                continue
-
-        if not appended:
-            logger.warning("[EMAIL SEND] Sent copy append failed for account=%s", account.get("key"))
-        imap.logout()
-    except Exception as e:
-        logger.warning("[EMAIL SEND] IMAP append to Sent failed for account=%s: %s", account.get("key"), e)
-
-
-def _send_email_sync(
-    account: dict,
-    to: str,
-    subject: str,
-    body: str,
-    body_html: str,
-    cc: List[str],
-    file_attachments: Optional[List[Tuple[str, bytes, str]]] = None,
-) -> str:
-    """Send via SMTP. Returns the Message-ID header.
-    file_attachments: list of (filename, content_bytes, mime_type)
-    """
-    html_with_cids, inline_images = _extract_inline_data_images(body_html or "")
-
-    # Build body tree:
-    # - with inline images: related(alternative(text/plain, text/html), image parts)
-    # - without inline images: alternative(text/plain, text/html)
-    if inline_images:
-        body_part = MIMEMultipart("related")
-        alt_part = MIMEMultipart("alternative")
-        alt_part.attach(MIMEText(body, "plain", "utf-8"))
-        if html_with_cids:
-            alt_part.attach(MIMEText(html_with_cids, "html", "utf-8"))
-        body_part.attach(alt_part)
-
-        for idx, (cid, content, mime_type) in enumerate(inline_images, start=1):
-            maintype, _, subtype = mime_type.partition("/")
-            img_part = MIMEBase(maintype or "image", subtype or "png")
-            img_part.set_payload(content)
-            encoders.encode_base64(img_part)
-            img_part.add_header("Content-ID", f"<{cid}>")
-            img_part.add_header("Content-Disposition", "inline", filename=f"inline-{idx}.{subtype or 'png'}")
-            body_part.attach(img_part)
-    else:
-        body_part = MIMEMultipart("alternative")
-        body_part.attach(MIMEText(body, "plain", "utf-8"))
-        if body_html:
-            body_part.attach(MIMEText(body_html, "html", "utf-8"))
-
-    # Wrap with mixed only when classic file attachments exist.
-    if file_attachments:
-        msg = MIMEMultipart("mixed")
-        msg.attach(body_part)
-    else:
-        msg = body_part
-
-    from_addr = account["email"]
-    msg["From"] = from_addr
-    msg["To"] = to
-    msg["Subject"] = subject
-    if cc:
-        msg["Cc"] = ", ".join(cc)
-
-    msg_id = f"<{uuid.uuid4()}@bellsystems>"
-    msg["Message-ID"] = msg_id
-
-    # Attach files
-    for filename, content, mime_type in (file_attachments or []):
-        maintype, _, subtype = mime_type.partition("/")
-        part = MIMEBase(maintype or "application", subtype or "octet-stream")
-        part.set_payload(content)
-        encoders.encode_base64(part)
-        part.add_header("Content-Disposition", "attachment", filename=filename)
-        msg.attach(part)
-
-    recipients = [to] + cc
-    raw_for_append = msg.as_bytes()
-    if account.get("smtp_use_tls"):
-        server = smtplib.SMTP(account["smtp_host"], int(account["smtp_port"]))
-        server.starttls()
-    else:
-        server = smtplib.SMTP_SSL(account["smtp_host"], int(account["smtp_port"]))
-
-    server.login(account["smtp_username"], account["smtp_password"])
-    server.sendmail(from_addr, recipients, msg.as_string())
-    server.quit()
-    _append_to_sent_sync(account, raw_for_append)
-
-    return msg_id
-
-
-async def send_email(
-    customer_id: str | None,
-    from_account: str | None,
-    to: str,
-    subject: str,
-    body: str,
-    body_html: str,
-    cc: List[str],
-    sent_by: str,
-    file_attachments: Optional[List[Tuple[str, bytes, str]]] = None,
-) -> dict:
-    """Send an email and record it in crm_comms_log. Returns the new log entry.
-    file_attachments: list of (filename, content_bytes, mime_type)
-    """
-    accounts = get_mail_accounts()
-    if not accounts:
-        raise RuntimeError("SMTP not configured")
-    account = account_by_key(from_account) if from_account else None
-    if not account:
-        raise RuntimeError("Please select a valid sender account")
-    if not account.get("allow_send"):
-        raise RuntimeError("Selected account is not allowed to send")
-    if not account.get("smtp_host") or not account.get("smtp_username") or not account.get("smtp_password"):
-        raise RuntimeError("SMTP not configured for selected account")
-
-    # If the caller did not provide a customer_id (e.g. compose from Mail page),
-    # auto-link by matching recipient addresses against CRM customer emails.
-    resolved_customer_id = customer_id
-    if not resolved_customer_id:
-        addr_to_customer = _load_customer_email_map()
-        rcpts = [to, *cc]
-        parsed_rcpts = [addr for _, addr in email.utils.getaddresses(rcpts) if addr]
-        for addr in parsed_rcpts:
-            key = (addr or "").strip().lower()
-            if key in addr_to_customer:
-                resolved_customer_id = addr_to_customer[key]
-                break
-
-    loop = asyncio.get_event_loop()
-    import functools
-    msg_id = await loop.run_in_executor(
-        None,
-        functools.partial(_send_email_sync, account, to, subject, body, body_html, cc, file_attachments or []),
-    )
-
-    # Upload attachments to Nextcloud and register in crm_media
-    comm_attachments = []
-    if file_attachments and resolved_customer_id:
-        from crm import nextcloud, service
-        from crm.models import MediaCreate, MediaDirection
-        from shared.firebase import get_db as get_firestore
-        firestore_db = get_firestore()
-        doc = firestore_db.collection("crm_customers").document(resolved_customer_id).get()
-        if doc.exists:
-            data = doc.to_dict()
-            # Build a minimal CustomerInDB-like object for get_customer_nc_path
-            folder_id = data.get("folder_id") or resolved_customer_id
-            nc_path = folder_id
-
-            for filename, content, mime_type in file_attachments:
-                # images/video → sent_media, everything else → documents
-                if mime_type.startswith("image/") or mime_type.startswith("video/"):
-                    subfolder = "sent_media"
-                else:
-                    subfolder = "documents"
-                target_folder = f"customers/{nc_path}/{subfolder}"
-                file_path = f"{target_folder}/{filename}"
-                try:
-                    await nextcloud.ensure_folder(target_folder)
-                    await nextcloud.upload_file(file_path, content, mime_type)
-                    await service.create_media(MediaCreate(
-                        customer_id=resolved_customer_id,
-                        filename=filename,
-                        nextcloud_path=file_path,
-                        mime_type=mime_type,
-                        direction=MediaDirection.sent,
-                        tags=["email-attachment"],
-                        uploaded_by=sent_by,
-                    ))
-                    comm_attachments.append({"filename": filename, "nextcloud_path": file_path})
-                except Exception as e:
-                    logger.warning(f"[EMAIL SEND] Failed to upload attachment {filename}: {e}")
-
-    now = datetime.now(timezone.utc).isoformat()
-    entry_id = str(uuid.uuid4())
-    db = await mqtt_db.get_db()
-    our_addr = account["email"].lower()
-    to_addrs_json = json.dumps([to] + cc)
-    attachments_json = json.dumps(comm_attachments)
-    await db.execute(
-        """INSERT INTO crm_comms_log
-           (id, customer_id, type, mail_account, direction, subject, body, body_html, attachments,
-            ext_message_id, from_addr, to_addrs, logged_by, occurred_at, created_at)
-           VALUES (?, ?, 'email', ?, 'outbound', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-        (entry_id, resolved_customer_id, account["key"], subject, body, body_html, attachments_json, msg_id,
-         our_addr, to_addrs_json, sent_by, now, now),
-    )
-    await db.commit()
-
-    return {
-        "id": entry_id,
-        "customer_id": resolved_customer_id,
-        "type": "email",
-        "mail_account": account["key"],
-        "direction": "outbound",
-        "subject": subject,
-        "body": body,
-        "body_html": body_html,
-        "attachments": comm_attachments,
-        "ext_message_id": msg_id,
-        "from_addr": our_addr,
-        "to_addrs": [to] + cc,
-        "logged_by": sent_by,
-        "occurred_at": now,
-        "created_at": now,
-    }
-
-
-def _delete_remote_email_sync(account: dict, ext_message_id: str) -> bool:
-    if not ext_message_id:
-        return False
-    if account.get("imap_use_ssl"):
-        imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
-    else:
-        imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
-    imap.login(account["imap_username"], account["imap_password"])
-    imap.select(account.get("imap_inbox", "INBOX"))
-    _, data = imap.search(None, f'HEADER Message-ID "{ext_message_id}"')
-    uids = data[0].split() if data and data[0] else []
-    if not uids:
-        imap.logout()
-        return False
-    for uid in uids:
-        imap.store(uid, "+FLAGS", "\\Deleted")
-    imap.expunge()
-    imap.logout()
-    return True
-
-
-async def delete_remote_email(ext_message_id: str, mail_account: str | None, from_addr: str | None = None) -> bool:
-    account = account_by_key(mail_account) if mail_account else None
-    if not account:
-        account = account_by_email(from_addr)
-    if not account or not account.get("imap_host"):
-        return False
-    loop = asyncio.get_event_loop()
-    try:
-        return await loop.run_in_executor(None, lambda: _delete_remote_email_sync(account, ext_message_id))
-    except Exception as e:
-        logger.warning(f"[EMAIL DELETE] Failed remote delete for {ext_message_id}: {e}")
-        return False
-
-
-def _set_remote_read_sync(account: dict, ext_message_id: str, read: bool) -> bool:
-    if not ext_message_id:
-        return False
-    if account.get("imap_use_ssl"):
-        imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
-    else:
-        imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
-    imap.login(account["imap_username"], account["imap_password"])
-    imap.select(account.get("imap_inbox", "INBOX"))
-    _, data = imap.search(None, f'HEADER Message-ID "{ext_message_id}"')
-    uids = data[0].split() if data and data[0] else []
-    if not uids:
-        imap.logout()
-        return False
-    flag_op = "+FLAGS" if read else "-FLAGS"
-    for uid in uids:
-        imap.store(uid, flag_op, "\\Seen")
-    imap.logout()
-    return True
-
-
-async def set_remote_read(ext_message_id: str, mail_account: str | None, from_addr: str | None, read: bool) -> bool:
-    account = account_by_key(mail_account) if mail_account else None
-    if not account:
-        account = account_by_email(from_addr)
-    if not account or not account.get("imap_host"):
-        return False
-    loop = asyncio.get_event_loop()
-    try:
-        return await loop.run_in_executor(None, lambda: _set_remote_read_sync(account, ext_message_id, read))
-    except Exception as e:
-        logger.warning(f"[EMAIL READ] Failed remote read update for {ext_message_id}: {e}")
-        return False
-
-
-

backend/crm/mail_accounts.py

@@ -1,104 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from config import settings
-
-
-def _bool(v: Any, default: bool) -> bool:
-    if isinstance(v, bool):
-        return v
-    if isinstance(v, str):
-        return v.strip().lower() in {"1", "true", "yes", "on"}
-    if v is None:
-        return default
-    return bool(v)
-
-
-def get_mail_accounts() -> list[dict]:
-    """
-    Returns normalized account dictionaries.
-    Falls back to legacy single-account config if MAIL_ACCOUNTS_JSON is empty.
-    """
-    configured = settings.mail_accounts
-    normalized: list[dict] = []
-
-    for idx, raw in enumerate(configured):
-        if not isinstance(raw, dict):
-            continue
-        key = str(raw.get("key") or "").strip().lower()
-        email = str(raw.get("email") or "").strip().lower()
-        if not key or not email:
-            continue
-        normalized.append(
-            {
-                "key": key,
-                "label": str(raw.get("label") or key.title()),
-                "email": email,
-                "imap_host": raw.get("imap_host") or settings.imap_host,
-                "imap_port": int(raw.get("imap_port") or settings.imap_port or 993),
-                "imap_username": raw.get("imap_username") or email,
-                "imap_password": raw.get("imap_password") or settings.imap_password,
-                "imap_use_ssl": _bool(raw.get("imap_use_ssl"), settings.imap_use_ssl),
-                "imap_inbox": str(raw.get("imap_inbox") or "INBOX"),
-                "imap_sent": str(raw.get("imap_sent") or "Sent"),
-                "smtp_host": raw.get("smtp_host") or settings.smtp_host,
-                "smtp_port": int(raw.get("smtp_port") or settings.smtp_port or 587),
-                "smtp_username": raw.get("smtp_username") or email,
-                "smtp_password": raw.get("smtp_password") or settings.smtp_password,
-                "smtp_use_tls": _bool(raw.get("smtp_use_tls"), settings.smtp_use_tls),
-                "sync_inbound": _bool(raw.get("sync_inbound"), True),
-                "allow_send": _bool(raw.get("allow_send"), True),
-            }
-        )
-
-    if normalized:
-        return normalized
-
-    # Legacy single-account fallback
-    if settings.imap_host or settings.smtp_host:
-        legacy_email = (settings.smtp_username or settings.imap_username or "").strip().lower()
-        if legacy_email:
-            return [
-                {
-                    "key": "default",
-                    "label": "Default",
-                    "email": legacy_email,
-                    "imap_host": settings.imap_host,
-                    "imap_port": settings.imap_port,
-                    "imap_username": settings.imap_username,
-                    "imap_password": settings.imap_password,
-                    "imap_use_ssl": settings.imap_use_ssl,
-                    "imap_inbox": "INBOX",
-                    "imap_sent": "Sent",
-                    "smtp_host": settings.smtp_host,
-                    "smtp_port": settings.smtp_port,
-                    "smtp_username": settings.smtp_username,
-                    "smtp_password": settings.smtp_password,
-                    "smtp_use_tls": settings.smtp_use_tls,
-                    "sync_inbound": True,
-                    "allow_send": True,
-                }
-            ]
-
-    return []
-
-
-def account_by_key(key: str | None) -> dict | None:
-    k = (key or "").strip().lower()
-    if not k:
-        return None
-    for acc in get_mail_accounts():
-        if acc["key"] == k:
-            return acc
-    return None
-
-
-def account_by_email(email_addr: str | None) -> dict | None:
-    e = (email_addr or "").strip().lower()
-    if not e:
-        return None
-    for acc in get_mail_accounts():
-        if acc["email"] == e:
-            return acc
-    return None

backend/crm/media_router.py

@@ -1,35 +0,0 @@
-from fastapi import APIRouter, Depends, Query
-from typing import Optional
-
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from crm.models import MediaCreate, MediaInDB, MediaListResponse
-from crm import service
-
-router = APIRouter(prefix="/api/crm/media", tags=["crm-media"])
-
-
-@router.get("", response_model=MediaListResponse)
-async def list_media(
-    customer_id: Optional[str] = Query(None),
-    order_id: Optional[str] = Query(None),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    items = await service.list_media(customer_id=customer_id, order_id=order_id)
-    return MediaListResponse(items=items, total=len(items))
-
-
-@router.post("", response_model=MediaInDB, status_code=201)
-async def create_media(
-    body: MediaCreate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return await service.create_media(body)
-
-
-@router.delete("/{media_id}", status_code=204)
-async def delete_media(
-    media_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    await service.delete_media(media_id)

backend/crm/models.py

@@ -1,443 +0,0 @@
-from enum import Enum
-from typing import Any, Dict, List, Optional
-from pydantic import BaseModel
-
-
-class ProductCategory(str, Enum):
-    controller = "controller"
-    striker = "striker"
-    clock = "clock"
-    part = "part"
-    repair_service = "repair_service"
-
-
-class CostLineItem(BaseModel):
-    name: str
-    quantity: float = 1
-    price: float = 0.0
-
-
-class ProductCosts(BaseModel):
-    labor_hours: Optional[float] = None
-    labor_rate: Optional[float] = None
-    items: List[CostLineItem] = []
-    total: Optional[float] = None
-
-
-class ProductStock(BaseModel):
-    on_hand: int = 0
-    reserved: int = 0
-    available: int = 0
-
-
-class ProductCreate(BaseModel):
-    name: str
-    sku: Optional[str] = None
-    category: ProductCategory
-    description: Optional[str] = None
-    name_en: Optional[str] = None
-    name_gr: Optional[str] = None
-    description_en: Optional[str] = None
-    description_gr: Optional[str] = None
-    price: float
-    currency: str = "EUR"
-    costs: Optional[ProductCosts] = None
-    stock: Optional[ProductStock] = None
-    active: bool = True
-    status: str = "active"  # active | discontinued | planned
-    photo_url: Optional[str] = None
-
-
-class ProductUpdate(BaseModel):
-    name: Optional[str] = None
-    sku: Optional[str] = None
-    category: Optional[ProductCategory] = None
-    description: Optional[str] = None
-    name_en: Optional[str] = None
-    name_gr: Optional[str] = None
-    description_en: Optional[str] = None
-    description_gr: Optional[str] = None
-    price: Optional[float] = None
-    currency: Optional[str] = None
-    costs: Optional[ProductCosts] = None
-    stock: Optional[ProductStock] = None
-    active: Optional[bool] = None
-    status: Optional[str] = None
-    photo_url: Optional[str] = None
-
-
-class ProductInDB(ProductCreate):
-    id: str
-    created_at: str
-    updated_at: str
-
-
-class ProductListResponse(BaseModel):
-    products: List[ProductInDB]
-    total: int
-
-
-# ── Customers ────────────────────────────────────────────────────────────────
-
-class ContactType(str, Enum):
-    email = "email"
-    phone = "phone"
-    whatsapp = "whatsapp"
-    other = "other"
-
-
-class CustomerContact(BaseModel):
-    type: ContactType
-    label: str
-    value: str
-    primary: bool = False
-
-
-class CustomerNote(BaseModel):
-    text: str
-    by: str
-    at: str
-
-
-class OwnedItemType(str, Enum):
-    console_device = "console_device"
-    product = "product"
-    freetext = "freetext"
-
-
-class OwnedItem(BaseModel):
-    type: OwnedItemType
-    # console_device fields
-    device_id: Optional[str] = None
-    label: Optional[str] = None
-    # product fields
-    product_id: Optional[str] = None
-    product_name: Optional[str] = None
-    quantity: Optional[int] = None
-    serial_numbers: Optional[List[str]] = None
-    # freetext fields
-    description: Optional[str] = None
-    serial_number: Optional[str] = None
-    notes: Optional[str] = None
-
-
-class CustomerLocation(BaseModel):
-    address: Optional[str] = None
-    city: Optional[str] = None
-    postal_code: Optional[str] = None
-    region: Optional[str] = None
-    country: Optional[str] = None
-
-
-# ── New customer status models ────────────────────────────────────────────────
-
-class TechnicalIssue(BaseModel):
-    active: bool = True
-    opened_date: str  # ISO string
-    resolved_date: Optional[str] = None
-    note: str
-    opened_by: str
-    resolved_by: Optional[str] = None
-
-
-class InstallSupportEntry(BaseModel):
-    active: bool = True
-    opened_date: str  # ISO string
-    resolved_date: Optional[str] = None
-    note: str
-    opened_by: str
-    resolved_by: Optional[str] = None
-
-
-class TransactionEntry(BaseModel):
-    date: str  # ISO string
-    flow: str  # "invoice" | "payment" | "refund" | "credit"
-    payment_type: Optional[str] = None  # "cash" | "bank_transfer" | "card" | "paypal" — null for invoices
-    category: str  # "full_payment" | "advance" | "installment"
-    amount: float
-    currency: str = "EUR"
-    invoice_ref: Optional[str] = None
-    order_ref: Optional[str] = None
-    recorded_by: str
-    note: str = ""
-
-
-# Lightweight summary stored on customer doc for fast CustomerList expanded view
-class CrmSummary(BaseModel):
-    active_order_status: Optional[str] = None
-    active_order_status_date: Optional[str] = None
-    active_order_title: Optional[str] = None
-    active_issues_count: int = 0
-    latest_issue_date: Optional[str] = None
-    active_support_count: int = 0
-    latest_support_date: Optional[str] = None
-
-
-class CustomerCreate(BaseModel):
-    title: Optional[str] = None
-    name: str
-    surname: Optional[str] = None
-    organization: Optional[str] = None
-    religion: Optional[str] = None
-    contacts: List[CustomerContact] = []
-    notes: List[CustomerNote] = []
-    location: Optional[CustomerLocation] = None
-    language: str = "el"
-    tags: List[str] = []
-    owned_items: List[OwnedItem] = []
-    linked_user_ids: List[str] = []
-    nextcloud_folder: Optional[str] = None
-    folder_id: Optional[str] = None
-    relationship_status: str = "lead"
-    technical_issues: List[Dict[str, Any]] = []
-    install_support: List[Dict[str, Any]] = []
-    transaction_history: List[Dict[str, Any]] = []
-    crm_summary: Optional[Dict[str, Any]] = None
-
-
-class CustomerUpdate(BaseModel):
-    title: Optional[str] = None
-    name: Optional[str] = None
-    surname: Optional[str] = None
-    organization: Optional[str] = None
-    religion: Optional[str] = None
-    contacts: Optional[List[CustomerContact]] = None
-    notes: Optional[List[CustomerNote]] = None
-    location: Optional[CustomerLocation] = None
-    language: Optional[str] = None
-    tags: Optional[List[str]] = None
-    owned_items: Optional[List[OwnedItem]] = None
-    linked_user_ids: Optional[List[str]] = None
-    nextcloud_folder: Optional[str] = None
-    relationship_status: Optional[str] = None
-    # folder_id intentionally excluded from update — set once at creation
-
-
-class CustomerInDB(CustomerCreate):
-    id: str
-    created_at: str
-    updated_at: str
-
-
-class CustomerListResponse(BaseModel):
-    customers: List[CustomerInDB]
-    total: int
-
-
-# ── Orders ───────────────────────────────────────────────────────────────────
-
-class OrderStatus(str, Enum):
-    negotiating = "negotiating"
-    awaiting_quotation = "awaiting_quotation"
-    awaiting_customer_confirmation = "awaiting_customer_confirmation"
-    awaiting_fulfilment = "awaiting_fulfilment"
-    awaiting_payment = "awaiting_payment"
-    manufacturing = "manufacturing"
-    shipped = "shipped"
-    installed = "installed"
-    declined = "declined"
-    complete = "complete"
-
-
-class OrderPaymentStatus(BaseModel):
-    required_amount: float = 0
-    received_amount: float = 0
-    balance_due: float = 0
-    advance_required: bool = False
-    advance_amount: Optional[float] = None
-    payment_complete: bool = False
-
-
-class OrderTimelineEvent(BaseModel):
-    date: str  # ISO string
-    type: str  # "quote_request" | "quote_sent" | "quote_accepted" | "quote_declined"
-               # | "mfg_started" | "mfg_complete" | "order_shipped" | "installed"
-               # | "payment_received" | "invoice_sent" | "note"
-    note: str = ""
-    updated_by: str
-
-
-class OrderDiscount(BaseModel):
-    type: str  # "percentage" | "fixed"
-    value: float = 0
-    reason: Optional[str] = None
-
-
-class OrderShipping(BaseModel):
-    method: Optional[str] = None
-    tracking_number: Optional[str] = None
-    carrier: Optional[str] = None
-    shipped_at: Optional[str] = None
-    delivered_at: Optional[str] = None
-    destination: Optional[str] = None
-
-
-class OrderItem(BaseModel):
-    type: str  # console_device | product | freetext
-    product_id: Optional[str] = None
-    product_name: Optional[str] = None
-    description: Optional[str] = None
-    quantity: int = 1
-    unit_price: float = 0.0
-    serial_numbers: List[str] = []
-
-
-class OrderCreate(BaseModel):
-    customer_id: str
-    order_number: Optional[str] = None
-    title: Optional[str] = None
-    created_by: Optional[str] = None
-    status: OrderStatus = OrderStatus.negotiating
-    status_updated_date: Optional[str] = None
-    status_updated_by: Optional[str] = None
-    items: List[OrderItem] = []
-    subtotal: float = 0
-    discount: Optional[OrderDiscount] = None
-    total_price: float = 0
-    currency: str = "EUR"
-    shipping: Optional[OrderShipping] = None
-    payment_status: Optional[Dict[str, Any]] = None
-    invoice_path: Optional[str] = None
-    notes: Optional[str] = None
-    timeline: List[Dict[str, Any]] = []
-
-
-class OrderUpdate(BaseModel):
-    order_number: Optional[str] = None
-    title: Optional[str] = None
-    status: Optional[OrderStatus] = None
-    status_updated_date: Optional[str] = None
-    status_updated_by: Optional[str] = None
-    items: Optional[List[OrderItem]] = None
-    subtotal: Optional[float] = None
-    discount: Optional[OrderDiscount] = None
-    total_price: Optional[float] = None
-    currency: Optional[str] = None
-    shipping: Optional[OrderShipping] = None
-    payment_status: Optional[Dict[str, Any]] = None
-    invoice_path: Optional[str] = None
-    notes: Optional[str] = None
-
-
-class OrderInDB(OrderCreate):
-    id: str
-    created_at: str
-    updated_at: str
-
-
-class OrderListResponse(BaseModel):
-    orders: List[OrderInDB]
-    total: int
-
-
-# ── Comms Log ─────────────────────────────────────────────────────────────────
-
-class CommType(str, Enum):
-    email = "email"
-    whatsapp = "whatsapp"
-    call = "call"
-    sms = "sms"
-    note = "note"
-    in_person = "in_person"
-
-
-class CommDirection(str, Enum):
-    inbound = "inbound"
-    outbound = "outbound"
-    internal = "internal"
-
-
-class CommAttachment(BaseModel):
-    filename: str
-    nextcloud_path: Optional[str] = None
-    content_type: Optional[str] = None
-    size: Optional[int] = None
-
-
-class CommCreate(BaseModel):
-    customer_id: Optional[str] = None
-    type: CommType
-    mail_account: Optional[str] = None
-    direction: CommDirection
-    subject: Optional[str] = None
-    body: Optional[str] = None
-    body_html: Optional[str] = None
-    attachments: List[CommAttachment] = []
-    ext_message_id: Optional[str] = None
-    from_addr: Optional[str] = None
-    to_addrs: Optional[List[str]] = None
-    logged_by: Optional[str] = None
-    occurred_at: Optional[str] = None  # defaults to now if not provided
-
-
-class CommUpdate(BaseModel):
-    type: Optional[CommType] = None
-    direction: Optional[CommDirection] = None
-    subject: Optional[str] = None
-    body: Optional[str] = None
-    logged_by: Optional[str] = None
-    occurred_at: Optional[str] = None
-
-
-class CommInDB(BaseModel):
-    id: str
-    customer_id: Optional[str] = None
-    type: CommType
-    mail_account: Optional[str] = None
-    direction: CommDirection
-    subject: Optional[str] = None
-    body: Optional[str] = None
-    body_html: Optional[str] = None
-    attachments: List[CommAttachment] = []
-    ext_message_id: Optional[str] = None
-    from_addr: Optional[str] = None
-    to_addrs: Optional[List[str]] = None
-    logged_by: Optional[str] = None
-    occurred_at: str
-    created_at: str
-    is_important: bool = False
-    is_read: bool = False
-
-
-class CommListResponse(BaseModel):
-    entries: List[CommInDB]
-    total: int
-
-
-# ── Media ─────────────────────────────────────────────────────────────────────
-
-class MediaDirection(str, Enum):
-    received = "received"
-    sent = "sent"
-    internal = "internal"
-
-
-class MediaCreate(BaseModel):
-    customer_id: Optional[str] = None
-    order_id: Optional[str] = None
-    filename: str
-    nextcloud_path: str
-    mime_type: Optional[str] = None
-    direction: Optional[MediaDirection] = None
-    tags: List[str] = []
-    uploaded_by: Optional[str] = None
-    thumbnail_path: Optional[str] = None
-
-
-class MediaInDB(BaseModel):
-    id: str
-    customer_id: Optional[str] = None
-    order_id: Optional[str] = None
-    filename: str
-    nextcloud_path: str
-    mime_type: Optional[str] = None
-    direction: Optional[MediaDirection] = None
-    tags: List[str] = []
-    uploaded_by: Optional[str] = None
-    created_at: str
-    thumbnail_path: Optional[str] = None
-
-
-class MediaListResponse(BaseModel):
-    items: List[MediaInDB]
-    total: int

backend/crm/nextcloud.py

@@ -1,329 +0,0 @@
-"""
-Nextcloud WebDAV client.
-
-All paths passed to these functions are relative to `settings.nextcloud_base_path`.
-The full WebDAV URL is:
-  {nextcloud_url}/remote.php/dav/files/{username}/{base_path}/{relative_path}
-"""
-import xml.etree.ElementTree as ET
-from typing import List
-from urllib.parse import unquote
-
-import httpx
-from fastapi import HTTPException
-
-from config import settings
-
-DAV_NS = "DAV:"
-
-# Default timeout for all Nextcloud WebDAV requests (seconds)
-_TIMEOUT = 60.0
-
-# Shared async client — reuses TCP connections across requests so Nextcloud
-# doesn't see rapid connection bursts that trigger brute-force throttling.
-_http_client: httpx.AsyncClient | None = None
-
-
-def _get_client() -> httpx.AsyncClient:
-    global _http_client
-    if _http_client is None or _http_client.is_closed:
-        _http_client = httpx.AsyncClient(
-            timeout=_TIMEOUT,
-            follow_redirects=True,
-            headers={"User-Agent": "BellSystems-CP/1.0"},
-        )
-    return _http_client
-
-
-async def close_client() -> None:
-    """Close the shared HTTP client. Call this on application shutdown."""
-    global _http_client
-    if _http_client and not _http_client.is_closed:
-        await _http_client.aclose()
-    _http_client = None
-
-
-async def keepalive_ping() -> None:
-    """
-    Send a lightweight PROPFIND Depth:0 to the Nextcloud base folder to keep
-    the TCP connection alive. Safe to call even if Nextcloud is not configured.
-    """
-    if not settings.nextcloud_url:
-        return
-    try:
-        url = _base_url()
-        client = _get_client()
-        await client.request(
-            "PROPFIND",
-            url,
-            auth=_auth(),
-            headers={"Depth": "0", "Content-Type": "application/xml"},
-            content=_PROPFIND_BODY,
-        )
-    except Exception as e:
-        print(f"[NEXTCLOUD KEEPALIVE] ping failed: {e}")
-
-
-def _dav_user() -> str:
-    """The username used in the WebDAV URL path (may differ from the login username)."""
-    return settings.nextcloud_dav_user or settings.nextcloud_username
-
-
-def _base_url() -> str:
-    if not settings.nextcloud_url:
-        raise HTTPException(status_code=503, detail="Nextcloud not configured")
-    return (
-        f"{settings.nextcloud_url.rstrip('/')}"
-        f"/remote.php/dav/files/{_dav_user()}"
-        f"/{settings.nextcloud_base_path}"
-    )
-
-
-def _auth() -> tuple[str, str]:
-    return (settings.nextcloud_username, settings.nextcloud_password)
-
-
-def _full_url(relative_path: str) -> str:
-    """Build full WebDAV URL for a relative path."""
-    path = relative_path.strip("/")
-    base = _base_url()
-    return f"{base}/{path}" if path else base
-
-
-def _parse_propfind(xml_bytes: bytes, base_path_prefix: str) -> List[dict]:
-    """
-    Parse a PROPFIND XML response.
-    Returns list of file/folder entries, skipping the root itself.
-    """
-    root = ET.fromstring(xml_bytes)
-    results = []
-
-    # The prefix we need to strip from D:href to get the relative path back
-    # href looks like: /remote.php/dav/files/user/BellSystems/Console/customers/abc/
-    dav_prefix = (
-        f"/remote.php/dav/files/{_dav_user()}"
-        f"/{settings.nextcloud_base_path}/"
-    )
-
-    for response in root.findall(f"{{{DAV_NS}}}response"):
-        href_el = response.find(f"{{{DAV_NS}}}href")
-        if href_el is None:
-            continue
-        href = unquote(href_el.text or "")
-
-        # Strip DAV prefix to get relative path within base_path
-        if href.startswith(dav_prefix):
-            rel = href[len(dav_prefix):].rstrip("/")
-        else:
-            rel = href
-
-        # Skip the folder itself (the root of the PROPFIND request)
-        if rel == base_path_prefix.strip("/"):
-            continue
-
-        propstat = response.find(f"{{{DAV_NS}}}propstat")
-        if propstat is None:
-            continue
-        prop = propstat.find(f"{{{DAV_NS}}}prop")
-        if prop is None:
-            continue
-
-        # is_dir: resourcetype contains D:collection
-        resource_type = prop.find(f"{{{DAV_NS}}}resourcetype")
-        is_dir = resource_type is not None and resource_type.find(f"{{{DAV_NS}}}collection") is not None
-
-        content_type_el = prop.find(f"{{{DAV_NS}}}getcontenttype")
-        mime_type = content_type_el.text if content_type_el is not None else (
-            "inode/directory" if is_dir else "application/octet-stream"
-        )
-
-        size_el = prop.find(f"{{{DAV_NS}}}getcontentlength")
-        size = int(size_el.text) if size_el is not None and size_el.text else 0
-
-        modified_el = prop.find(f"{{{DAV_NS}}}getlastmodified")
-        last_modified = modified_el.text if modified_el is not None else None
-
-        filename = rel.split("/")[-1] if rel else ""
-
-        results.append({
-            "filename": filename,
-            "path": rel,
-            "mime_type": mime_type,
-            "size": size,
-            "last_modified": last_modified,
-            "is_dir": is_dir,
-        })
-
-    return results
-
-
-async def ensure_folder(relative_path: str) -> None:
-    """
-    Create a folder (and all parents) in Nextcloud via MKCOL.
-    Includes the base_path segments so the full hierarchy is created from scratch.
-    Silently succeeds if folders already exist.
-    """
-    # Build the complete path list: base_path segments + relative_path segments
-    base_parts = settings.nextcloud_base_path.strip("/").split("/")
-    rel_parts = relative_path.strip("/").split("/") if relative_path.strip("/") else []
-    all_parts = base_parts + rel_parts
-
-    dav_root = f"{settings.nextcloud_url.rstrip('/')}/remote.php/dav/files/{_dav_user()}"
-    client = _get_client()
-    built = ""
-    for part in all_parts:
-        built = f"{built}/{part}" if built else part
-        url = f"{dav_root}/{built}"
-        resp = await client.request("MKCOL", url, auth=_auth())
-        # 201 = created, 405/409 = already exists — both are fine
-        if resp.status_code not in (201, 405, 409):
-            raise HTTPException(
-                status_code=502,
-                detail=f"Failed to create Nextcloud folder '{built}': {resp.status_code}",
-            )
-
-
-async def write_info_file(customer_folder: str, customer_name: str, customer_id: str) -> None:
-    """Write a _info.txt stub into a new customer folder for human browsability."""
-    content = f"Customer: {customer_name}\nID: {customer_id}\n"
-    await upload_file(
-        f"{customer_folder}/_info.txt",
-        content.encode("utf-8"),
-        "text/plain",
-    )
-
-
-_PROPFIND_BODY = b"""<?xml version="1.0"?>
-<D:propfind xmlns:D="DAV:">
-  <D:prop>
-    <D:resourcetype/>
-    <D:getcontenttype/>
-    <D:getcontentlength/>
-    <D:getlastmodified/>
-  </D:prop>
-</D:propfind>"""
-
-
-async def list_folder(relative_path: str) -> List[dict]:
-    """
-    PROPFIND at depth=1 to list a folder's immediate children.
-    relative_path is relative to nextcloud_base_path.
-    """
-    url = _full_url(relative_path)
-    client = _get_client()
-    resp = await client.request(
-        "PROPFIND",
-        url,
-        auth=_auth(),
-        headers={"Depth": "1", "Content-Type": "application/xml"},
-        content=_PROPFIND_BODY,
-    )
-    if resp.status_code == 404:
-        return []
-    if resp.status_code not in (207, 200):
-        raise HTTPException(status_code=502, detail=f"Nextcloud PROPFIND failed: {resp.status_code}")
-    return _parse_propfind(resp.content, relative_path)
-
-
-async def list_folder_recursive(relative_path: str) -> List[dict]:
-    """
-    Recursively list ALL files under a folder (any depth).
-    Tries Depth:infinity first (single call). Falls back to manual recursion
-    via Depth:1 if the server returns 403/400 (some servers disable infinity).
-    Returns only file entries (is_dir=False).
-    """
-    url = _full_url(relative_path)
-    client = _get_client()
-    resp = await client.request(
-        "PROPFIND",
-        url,
-        auth=_auth(),
-        headers={"Depth": "infinity", "Content-Type": "application/xml"},
-        content=_PROPFIND_BODY,
-    )
-
-    if resp.status_code in (207, 200):
-        all_items = _parse_propfind(resp.content, relative_path)
-        return [item for item in all_items if not item["is_dir"]]
-
-    # Depth:infinity not supported — fall back to recursive Depth:1
-    if resp.status_code in (403, 400, 412):
-        return await _list_recursive_fallback(relative_path)
-
-    if resp.status_code == 404:
-        return []
-
-    raise HTTPException(status_code=502, detail=f"Nextcloud PROPFIND failed: {resp.status_code}")
-
-
-async def _list_recursive_fallback(relative_path: str) -> List[dict]:
-    """Manually recurse via Depth:1 calls when Depth:infinity is blocked."""
-    items = await list_folder(relative_path)
-    files = []
-    dirs = []
-    for item in items:
-        if item["is_dir"]:
-            dirs.append(item["path"])
-        else:
-            files.append(item)
-    for dir_path in dirs:
-        child_files = await _list_recursive_fallback(dir_path)
-        files.extend(child_files)
-    return files
-
-
-async def upload_file(relative_path: str, content: bytes, mime_type: str) -> str:
-    """
-    PUT a file to Nextcloud. Returns the relative_path on success.
-    relative_path includes filename, e.g. "customers/abc123/media/photo.jpg"
-    """
-    url = _full_url(relative_path)
-    client = _get_client()
-    resp = await client.put(
-        url,
-        auth=_auth(),
-        content=content,
-        headers={"Content-Type": mime_type},
-    )
-    if resp.status_code not in (200, 201, 204):
-        raise HTTPException(status_code=502, detail=f"Nextcloud upload failed: {resp.status_code}")
-    return relative_path
-
-
-async def download_file(relative_path: str) -> tuple[bytes, str]:
-    """
-    GET a file from Nextcloud. Returns (bytes, mime_type).
-    """
-    url = _full_url(relative_path)
-    client = _get_client()
-    resp = await client.get(url, auth=_auth())
-    if resp.status_code == 404:
-        raise HTTPException(status_code=404, detail="File not found in Nextcloud")
-    if resp.status_code != 200:
-        raise HTTPException(status_code=502, detail=f"Nextcloud download failed: {resp.status_code}")
-    mime = resp.headers.get("content-type", "application/octet-stream").split(";")[0].strip()
-    return resp.content, mime
-
-
-async def delete_file(relative_path: str) -> None:
-    """DELETE a file from Nextcloud."""
-    url = _full_url(relative_path)
-    client = _get_client()
-    resp = await client.request("DELETE", url, auth=_auth())
-    if resp.status_code not in (200, 204, 404):
-        raise HTTPException(status_code=502, detail=f"Nextcloud delete failed: {resp.status_code}")
-
-
-async def rename_folder(old_relative_path: str, new_relative_path: str) -> None:
-    """Rename/move a folder in Nextcloud using WebDAV MOVE."""
-    url = _full_url(old_relative_path)
-    destination = _full_url(new_relative_path)
-    client = _get_client()
-    resp = await client.request(
-        "MOVE",
-        url,
-        auth=_auth(),
-        headers={"Destination": destination, "Overwrite": "F"},
-    )
-    if resp.status_code not in (201, 204):
-        raise HTTPException(status_code=502, detail=f"Nextcloud rename failed: {resp.status_code}")

backend/crm/nextcloud_router.py

@@ -1,490 +0,0 @@
-"""
-Nextcloud WebDAV proxy endpoints.
-
-Folder convention (all paths relative to nextcloud_base_path = BellSystems/Console):
-  customers/{folder_id}/media/
-  customers/{folder_id}/documents/
-  customers/{folder_id}/sent/
-  customers/{folder_id}/received/
-
-folder_id = customer.folder_id if set, else customer.id (legacy fallback).
-"""
-from fastapi import APIRouter, Depends, Query, UploadFile, File, Form, Response, HTTPException, Request
-from fastapi.responses import StreamingResponse
-from typing import Optional
-
-from jose import JWTError
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from auth.utils import decode_access_token
-from crm import nextcloud, service
-from config import settings
-from crm.models import MediaCreate, MediaDirection
-from crm.thumbnails import generate_thumbnail
-
-router = APIRouter(prefix="/api/crm/nextcloud", tags=["crm-nextcloud"])
-
-DIRECTION_MAP = {
-    "sent": MediaDirection.sent,
-    "received": MediaDirection.received,
-    "internal": MediaDirection.internal,
-    "media": MediaDirection.internal,
-    "documents": MediaDirection.internal,
-}
-
-
-@router.get("/web-url")
-async def get_web_url(
-    path: str = Query(..., description="Path relative to nextcloud_base_path"),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """
-    Return the Nextcloud Files web-UI URL for a given file path.
-    Opens the parent folder with the file highlighted.
-    """
-    if not settings.nextcloud_url:
-        raise HTTPException(status_code=503, detail="Nextcloud not configured")
-    base = settings.nextcloud_base_path.strip("/")
-    # path is relative to base, e.g. "customers/abc/media/photo.jpg"
-    parts = path.rsplit("/", 1)
-    folder_rel = parts[0] if len(parts) == 2 else ""
-    filename = parts[-1]
-    nc_dir = f"/{base}/{folder_rel}" if folder_rel else f"/{base}"
-    from urllib.parse import urlencode, quote
-    qs = urlencode({"dir": nc_dir, "scrollto": filename})
-    url = f"{settings.nextcloud_url.rstrip('/')}/index.php/apps/files/?{qs}"
-    return {"url": url}
-
-
-@router.get("/browse")
-async def browse(
-    path: str = Query(..., description="Path relative to nextcloud_base_path"),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """List immediate children of a Nextcloud folder."""
-    items = await nextcloud.list_folder(path)
-    return {"path": path, "items": items}
-
-
-@router.get("/browse-all")
-async def browse_all(
-    customer_id: str = Query(..., description="Customer ID"),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """
-    Recursively list ALL files for a customer across all subfolders and any depth.
-    Uses Depth:infinity (one WebDAV call) with automatic fallback to recursive Depth:1.
-    Each file item includes a 'subfolder' key derived from its path.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-
-    all_files = await nextcloud.list_folder_recursive(base)
-
-    # Exclude _info.txt stubs — human-readable only, should never appear in the UI.
-    # .thumbs/ files are kept: the frontend needs them to build the thumbnail map
-    # (it already filters them out of the visible file list itself).
-    all_files = [
-        f for f in all_files
-        if not f["path"].endswith("/_info.txt")
-    ]
-
-    # Tag each file with the top-level subfolder it lives under
-    for item in all_files:
-        parts = item["path"].split("/")
-        # path looks like: customers/{nc_path}/{subfolder}/[...]/filename
-        # parts[0]=customers, parts[1]={nc_path}, parts[2]={subfolder}
-        item["subfolder"] = parts[2] if len(parts) > 2 else "other"
-
-    return {"items": all_files}
-
-
-@router.get("/file")
-async def proxy_file(
-    request: Request,
-    path: str = Query(..., description="Path relative to nextcloud_base_path"),
-    token: Optional[str] = Query(None, description="JWT token for browser-native requests (img src, video src, a href) that cannot send an Authorization header"),
-):
-    """
-    Stream a file from Nextcloud through the backend (proxy).
-    Supports HTTP Range requests so videos can be seeked and start playing immediately.
-    Accepts auth via Authorization: Bearer header OR ?token= query param.
-    """
-    if token is None:
-        raise HTTPException(status_code=403, detail="Not authenticated")
-    try:
-        decode_access_token(token)
-    except (JWTError, KeyError):
-        raise HTTPException(status_code=403, detail="Invalid token")
-
-    # Forward the Range header to Nextcloud so we get a true partial response
-    # without buffering the whole file into memory.
-    nc_url = nextcloud._full_url(path)
-    nc_auth = nextcloud._auth()
-    forward_headers = {}
-    range_header = request.headers.get("range")
-    if range_header:
-        forward_headers["Range"] = range_header
-
-    import httpx as _httpx
-
-    # Use a dedicated streaming client — httpx.stream() keeps the connection open
-    # for the lifetime of the generator, so we can't reuse the shared persistent client.
-    # We enter the stream context here to get headers immediately (no body buffering),
-    # then hand the body iterator to StreamingResponse.
-    stream_client = _httpx.AsyncClient(timeout=None, follow_redirects=True)
-    nc_resp_ctx = stream_client.stream("GET", nc_url, auth=nc_auth, headers=forward_headers)
-    nc_resp = await nc_resp_ctx.__aenter__()
-
-    if nc_resp.status_code == 404:
-        await nc_resp_ctx.__aexit__(None, None, None)
-        await stream_client.aclose()
-        raise HTTPException(status_code=404, detail="File not found in Nextcloud")
-    if nc_resp.status_code not in (200, 206):
-        await nc_resp_ctx.__aexit__(None, None, None)
-        await stream_client.aclose()
-        raise HTTPException(status_code=502, detail=f"Nextcloud returned {nc_resp.status_code}")
-
-    mime_type = nc_resp.headers.get("content-type", "application/octet-stream").split(";")[0].strip()
-
-    resp_headers = {"Accept-Ranges": "bytes"}
-    for h in ("content-range", "content-length"):
-        if h in nc_resp.headers:
-            resp_headers[h.title()] = nc_resp.headers[h]
-
-    async def _stream():
-        try:
-            async for chunk in nc_resp.aiter_bytes(chunk_size=64 * 1024):
-                yield chunk
-        finally:
-            await nc_resp_ctx.__aexit__(None, None, None)
-            await stream_client.aclose()
-
-    return StreamingResponse(
-        _stream(),
-        status_code=nc_resp.status_code,
-        media_type=mime_type,
-        headers=resp_headers,
-    )
-
-
-@router.put("/file-put")
-async def put_file(
-    request: Request,
-    path: str = Query(..., description="Path relative to nextcloud_base_path"),
-    token: Optional[str] = Query(None),
-):
-    """
-    Overwrite a file in Nextcloud with a new body (used for TXT in-browser editing).
-    Auth via ?token= query param (same pattern as /file GET).
-    """
-    if token is None:
-        raise HTTPException(status_code=403, detail="Not authenticated")
-    try:
-        decode_access_token(token)
-    except (JWTError, KeyError):
-        raise HTTPException(status_code=403, detail="Invalid token")
-
-    body = await request.body()
-    content_type = request.headers.get("content-type", "text/plain")
-    await nextcloud.upload_file(path, body, content_type)
-    return {"updated": path}
-
-
-@router.post("/upload")
-async def upload_file(
-    file: UploadFile = File(...),
-    customer_id: str = Form(...),
-    subfolder: str = Form("media"),  # "media" | "documents" | "sent" | "received"
-    direction: Optional[str] = Form(None),
-    tags: Optional[str] = Form(None),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Upload a file to the customer's Nextcloud folder and record it in crm_media.
-    Uses the customer's folder_id as the NC path (falls back to UUID for legacy records).
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-
-    target_folder = f"customers/{nc_path}/{subfolder}"
-    file_path = f"{target_folder}/{file.filename}"
-
-    # Ensure the target subfolder exists (idempotent, fast for existing folders)
-    await nextcloud.ensure_folder(target_folder)
-
-    # Read and upload
-    content = await file.read()
-    mime_type = file.content_type or "application/octet-stream"
-    await nextcloud.upload_file(file_path, content, mime_type)
-
-    # Generate and upload thumbnail (best-effort, non-blocking)
-    # Always stored as {stem}.jpg regardless of source extension so the thumb
-    # filename is unambiguous and the existence check can never false-positive.
-    thumb_path = None
-    try:
-        thumb_bytes = generate_thumbnail(content, mime_type, file.filename)
-        if thumb_bytes:
-            thumb_folder = f"{target_folder}/.thumbs"
-            stem = file.filename.rsplit(".", 1)[0] if "." in file.filename else file.filename
-            thumb_filename = f"{stem}.jpg"
-            thumb_nc_path = f"{thumb_folder}/{thumb_filename}"
-            await nextcloud.ensure_folder(thumb_folder)
-            await nextcloud.upload_file(thumb_nc_path, thumb_bytes, "image/jpeg")
-            thumb_path = thumb_nc_path
-    except Exception as e:
-        import logging
-        logging.getLogger(__name__).warning("Thumbnail generation failed for %s: %s", file.filename, e)
-
-    # Resolve direction
-    resolved_direction = None
-    if direction:
-        try:
-            resolved_direction = MediaDirection(direction)
-        except ValueError:
-            resolved_direction = DIRECTION_MAP.get(subfolder, MediaDirection.internal)
-    else:
-        resolved_direction = DIRECTION_MAP.get(subfolder, MediaDirection.internal)
-
-    # Save metadata record
-    tag_list = [t.strip() for t in tags.split(",")] if tags else []
-    media_record = await service.create_media(MediaCreate(
-        customer_id=customer_id,
-        filename=file.filename,
-        nextcloud_path=file_path,
-        mime_type=mime_type,
-        direction=resolved_direction,
-        tags=tag_list,
-        uploaded_by=_user.name,
-        thumbnail_path=thumb_path,
-    ))
-
-    return media_record
-
-
-@router.delete("/file")
-async def delete_file(
-    path: str = Query(..., description="Path relative to nextcloud_base_path"),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """Delete a file from Nextcloud and remove the matching crm_media record if found."""
-    await nextcloud.delete_file(path)
-
-    # Best-effort: delete the DB record if one matches this path
-    media_list = await service.list_media()
-    for m in media_list:
-        if m.nextcloud_path == path:
-            try:
-                await service.delete_media(m.id)
-            except Exception:
-                pass
-            break
-
-    return {"deleted": path}
-
-
-@router.post("/init-customer-folder")
-async def init_customer_folder(
-    customer_id: str = Form(...),
-    customer_name: str = Form(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Create the standard folder structure for a customer in Nextcloud
-    and write an _info.txt stub for human readability.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-    for sub in ("media", "documents", "sent", "received"):
-        await nextcloud.ensure_folder(f"{base}/{sub}")
-    await nextcloud.write_info_file(base, customer_name, customer_id)
-    return {"initialized": base}
-
-
-@router.post("/sync")
-async def sync_nextcloud_files(
-    customer_id: str = Form(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Scan the customer's Nextcloud folder and register any files not yet tracked in the DB.
-    Returns counts of newly synced and skipped (already tracked) files.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-
-    # Collect all NC files recursively (handles nested folders at any depth)
-    all_nc_files = await nextcloud.list_folder_recursive(base)
-    # Skip .thumbs/ folder contents and the _info.txt stub — these are internal
-    all_nc_files = [
-        f for f in all_nc_files
-        if "/.thumbs/" not in f["path"] and not f["path"].endswith("/_info.txt")
-    ]
-    for item in all_nc_files:
-        parts = item["path"].split("/")
-        item["_subfolder"] = parts[2] if len(parts) > 2 else "media"
-
-    # Get existing DB records for this customer
-    existing = await service.list_media(customer_id=customer_id)
-    tracked_paths = {m.nextcloud_path for m in existing}
-
-    synced = 0
-    skipped = 0
-    for f in all_nc_files:
-        if f["path"] in tracked_paths:
-            skipped += 1
-            continue
-        sub = f["_subfolder"]
-        direction = DIRECTION_MAP.get(sub, MediaDirection.internal)
-        await service.create_media(MediaCreate(
-            customer_id=customer_id,
-            filename=f["filename"],
-            nextcloud_path=f["path"],
-            mime_type=f.get("mime_type") or "application/octet-stream",
-            direction=direction,
-            tags=[],
-            uploaded_by="nextcloud-sync",
-        ))
-        synced += 1
-
-    return {"synced": synced, "skipped": skipped}
-
-
-@router.post("/generate-thumbs")
-async def generate_thumbs(
-    customer_id: str = Form(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Scan all customer files in Nextcloud and generate thumbnails for any file
-    that doesn't already have one in the corresponding .thumbs/ sub-folder.
-    Skips files inside .thumbs/ itself and file types that can't be thumbnailed.
-    Returns counts of generated, skipped (already exists), and failed files.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-
-    all_nc_files = await nextcloud.list_folder_recursive(base)
-
-    # Build a set of existing thumb paths for O(1) lookup
-    existing_thumbs = {
-        f["path"] for f in all_nc_files if "/.thumbs/" in f["path"]
-    }
-
-    # Only process real files (not thumbs themselves)
-    candidates = [f for f in all_nc_files if "/.thumbs/" not in f["path"]]
-
-    generated = 0
-    skipped = 0
-    failed = 0
-
-    for f in candidates:
-        # Derive where the thumb would live
-        path = f["path"]  # e.g. customers/{nc_path}/{subfolder}/photo.jpg
-        parts = path.rsplit("/", 1)
-        if len(parts) != 2:
-            skipped += 1
-            continue
-        parent_folder, filename = parts
-        stem = filename.rsplit(".", 1)[0] if "." in filename else filename
-        thumb_filename = f"{stem}.jpg"
-        thumb_nc_path = f"{parent_folder}/.thumbs/{thumb_filename}"
-
-        if thumb_nc_path in existing_thumbs:
-            skipped += 1
-            continue
-
-        # Download the file, generate thumb, upload
-        try:
-            content, mime_type = await nextcloud.download_file(path)
-            thumb_bytes = generate_thumbnail(content, mime_type, filename)
-            if not thumb_bytes:
-                skipped += 1  # unsupported file type
-                continue
-            thumb_folder = f"{parent_folder}/.thumbs"
-            await nextcloud.ensure_folder(thumb_folder)
-            await nextcloud.upload_file(thumb_nc_path, thumb_bytes, "image/jpeg")
-            generated += 1
-        except Exception as e:
-            import logging
-            logging.getLogger(__name__).warning("Thumb gen failed for %s: %s", path, e)
-            failed += 1
-
-    return {"generated": generated, "skipped": skipped, "failed": failed}
-
-
-@router.post("/clear-thumbs")
-async def clear_thumbs(
-    customer_id: str = Form(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Delete all .thumbs sub-folders for a customer across all subfolders.
-    This lets you regenerate thumbnails from scratch.
-    Returns count of .thumbs folders deleted.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-
-    all_nc_files = await nextcloud.list_folder_recursive(base)
-
-    # Collect unique .thumbs folder paths
-    thumb_folders = set()
-    for f in all_nc_files:
-        if "/.thumbs/" in f["path"]:
-            folder = f["path"].split("/.thumbs/")[0] + "/.thumbs"
-            thumb_folders.add(folder)
-
-    deleted = 0
-    for folder in thumb_folders:
-        try:
-            await nextcloud.delete_file(folder)
-            deleted += 1
-        except Exception as e:
-            import logging
-            logging.getLogger(__name__).warning("Failed to delete .thumbs folder %s: %s", folder, e)
-
-    return {"deleted_folders": deleted}
-
-
-@router.post("/untrack-deleted")
-async def untrack_deleted_files(
-    customer_id: str = Form(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """
-    Remove DB records for files that no longer exist in Nextcloud.
-    Returns count of untracked records.
-    """
-    customer = service.get_customer(customer_id)
-    nc_path = service.get_customer_nc_path(customer)
-    base = f"customers/{nc_path}"
-
-    # Collect all NC file paths recursively (excluding thumbs and info stub)
-    all_nc_files = await nextcloud.list_folder_recursive(base)
-    nc_paths = {
-        item["path"] for item in all_nc_files
-        if "/.thumbs/" not in item["path"] and not item["path"].endswith("/_info.txt")
-    }
-
-    # Find DB records whose NC path no longer exists, OR that are internal files
-    # (_info.txt / .thumbs/) which should never have been tracked in the first place.
-    existing = await service.list_media(customer_id=customer_id)
-    untracked = 0
-    for m in existing:
-        is_internal = m.nextcloud_path and (
-            "/.thumbs/" in m.nextcloud_path or m.nextcloud_path.endswith("/_info.txt")
-        )
-        if m.nextcloud_path and (is_internal or m.nextcloud_path not in nc_paths):
-            try:
-                await service.delete_media(m.id)
-                untracked += 1
-            except Exception:
-                pass
-
-    return {"untracked": untracked}

backend/crm/orders_router.py

@@ -1,177 +0,0 @@
-from fastapi import APIRouter, Depends, Query
-from typing import Optional
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from crm.models import OrderCreate, OrderUpdate, OrderInDB, OrderListResponse
-from crm import service
-from database.postgres import get_pg_session
-from shared.audit import log_action
-
-router = APIRouter(prefix="/api/crm/customers/{customer_id}/orders", tags=["crm-orders"])
-
-
-@router.get("", response_model=OrderListResponse)
-def list_orders(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    orders = service.list_orders(customer_id)
-    return OrderListResponse(orders=orders, total=len(orders))
-
-
-# IMPORTANT: specific sub-paths must come before /{order_id}
-@router.get("/next-order-number")
-def get_next_order_number(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Return the next globally unique order number (ORD-DDMMYY-NNN across all customers)."""
-    return {"order_number": service._generate_order_number(customer_id)}
-
-
-@router.post("/init-negotiations", response_model=OrderInDB, status_code=201)
-async def init_negotiations(
-    customer_id: str,
-    body: dict,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    order = service.init_negotiations(
-        customer_id=customer_id,
-        title=body.get("title", ""),
-        note=body.get("note", ""),
-        date=body.get("date"),
-        created_by=body.get("created_by", ""),
-    )
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "order",
-                     order.id, order.order_number or order.id, meta={"action_detail": "negotiations_started"})
-    return order
-
-
-@router.post("", response_model=OrderInDB, status_code=201)
-async def create_order(
-    customer_id: str,
-    body: OrderCreate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    order = service.create_order(customer_id, body)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "order",
-                     order.id, order.order_number or order.id)
-    return order
-
-
-@router.get("/{order_id}", response_model=OrderInDB)
-def get_order(
-    customer_id: str,
-    order_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return service.get_order(customer_id, order_id)
-
-
-@router.patch("/{order_id}", response_model=OrderInDB)
-async def update_order(
-    customer_id: str,
-    order_id: str,
-    body: OrderUpdate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    old = service.get_order(customer_id, order_id)
-    order = service.update_order(customer_id, order_id, body)
-    action = "STATUS_CHANGE" if body.status is not None else "UPDATE"
-    _SKIP = {"updated_at", "id", "customer_id", "items", "timeline", "discount", "shipping", "payment_status"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(order, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(order, k, None)
-    }
-    await log_action(db, _user.sub, _user.name or _user.email, action, "order",
-                     order_id, order.order_number or order_id, changes=changes or None)
-    return order
-
-
-@router.delete("/{order_id}", status_code=204)
-async def delete_order(
-    customer_id: str,
-    order_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    service.delete_order(customer_id, order_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "order",
-                     order_id, order_id)
-
-
-@router.post("/{order_id}/timeline", response_model=OrderInDB)
-def append_timeline_event(
-    customer_id: str,
-    order_id: str,
-    body: dict,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.append_timeline_event(customer_id, order_id, body)
-
-
-@router.patch("/{order_id}/timeline/{index}", response_model=OrderInDB)
-def update_timeline_event(
-    customer_id: str,
-    order_id: str,
-    index: int,
-    body: dict,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.update_timeline_event(customer_id, order_id, index, body)
-
-
-@router.delete("/{order_id}/timeline/{index}", response_model=OrderInDB)
-def delete_timeline_event(
-    customer_id: str,
-    order_id: str,
-    index: int,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.delete_timeline_event(customer_id, order_id, index)
-
-
-@router.patch("/{order_id}/payment-status", response_model=OrderInDB)
-def update_payment_status(
-    customer_id: str,
-    order_id: str,
-    body: dict,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    return service.update_order_payment_status(customer_id, order_id, body)
-
-
-# ── Global order list (collection group) ─────────────────────────────────────
-# Separate router registered at /api/crm/orders for the global OrderList page
-
-global_router = APIRouter(prefix="/api/crm/orders", tags=["crm-orders-global"])
-
-
-@global_router.get("")
-def list_all_orders(
-    status: Optional[str] = Query(None),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    orders = service.list_all_orders(status=status)
-    # Enrich with customer names
-    customer_ids = list({o.customer_id for o in orders if o.customer_id})
-    customer_names: dict[str, str] = {}
-    for cid in customer_ids:
-        try:
-            c = service.get_customer(cid)
-            parts = [c.name, c.organization] if c.organization else [c.name]
-            customer_names[cid] = " / ".join(filter(None, parts))
-        except Exception:
-            pass
-    enriched = []
-    for o in orders:
-        d = o.model_dump()
-        d["customer_name"] = customer_names.get(o.customer_id)
-        enriched.append(d)
-    return {"orders": enriched, "total": len(enriched)}

backend/crm/orm.py

@@ -1,239 +0,0 @@
-from datetime import datetime, timezone
-from sqlalchemy import (
-    BigInteger, Boolean, Column, DateTime, ForeignKey, Index, Integer,
-    Numeric, String, Text, UniqueConstraint,
-)
-from sqlalchemy.dialects.postgresql import ARRAY, JSONB
-from sqlalchemy.orm import relationship
-from database.postgres import Base
-
-
-def _now():
-    return datetime.now(timezone.utc)
-
-
-class CrmProduct(Base):
-    __tablename__ = "crm_products"
-
-    id           = Column(String(128), primary_key=True)   # Firestore doc ID
-    firestore_id = Column(String(128), unique=True)        # same as id during transition
-    name         = Column(String(500), nullable=False)
-    sku          = Column(String(128))
-    category     = Column(String(128))
-    description  = Column(Text)
-    unit_cost    = Column(Numeric(12, 2), nullable=False, default=0)
-    currency     = Column(String(10), nullable=False, default="EUR")
-    unit_type    = Column(String(32), nullable=False, default="pcs")
-    is_active    = Column(Boolean, nullable=False, default=True)
-    created_at   = Column(DateTime(timezone=True), nullable=False, default=_now)
-    updated_at   = Column(DateTime(timezone=True), nullable=False, default=_now, onupdate=_now)
-
-
-class CrmCustomer(Base):
-    __tablename__ = "crm_customers"
-    __table_args__ = (
-        Index("idx_crm_customers_rel_status", "relationship_status"),
-        Index("idx_crm_customers_name", "name", "surname"),
-        Index("idx_crm_customers_tags", "tags", postgresql_using="gin"),
-    )
-
-    id                  = Column(String(128), primary_key=True)   # Firestore doc ID
-    firestore_id        = Column(String(128), unique=True)
-    title               = Column(String(32))
-    name                = Column(String(255), nullable=False)
-    surname             = Column(String(255))
-    organization        = Column(String(500))
-    religion            = Column(String(64))
-    language            = Column(String(10), nullable=False, default="el")
-    folder_id           = Column(String(128), unique=True, nullable=False)
-    relationship_status = Column(String(64), nullable=False, default="lead")
-    nextcloud_folder    = Column(String(500))
-    contacts            = Column(JSONB, nullable=False, default=list)
-    notes               = Column(JSONB, nullable=False, default=list)
-    location            = Column(JSONB)
-    tags                = Column(ARRAY(String), nullable=False, default=list)
-    owned_items         = Column(JSONB, nullable=False, default=list)
-    linked_user_ids     = Column(ARRAY(String), nullable=False, default=list)
-    technical_issues    = Column(JSONB, nullable=False, default=list)
-    install_support     = Column(JSONB, nullable=False, default=list)
-    transaction_history = Column(JSONB, nullable=False, default=list)
-    crm_summary         = Column(JSONB)
-    created_at          = Column(DateTime(timezone=True), nullable=False)
-    updated_at          = Column(DateTime(timezone=True), nullable=False)
-
-    orders     = relationship("CrmOrder", back_populates="customer",
-                              cascade="all, delete-orphan", lazy="noload")
-    quotations = relationship("CrmQuotation", back_populates="customer",
-                              cascade="all, delete-orphan", lazy="noload")
-    comms      = relationship("CrmCommsLog", back_populates="customer",
-                              cascade="all, delete-orphan", lazy="noload")
-    media      = relationship("CrmMedia", back_populates="customer", lazy="noload")
-
-
-class CrmOrder(Base):
-    __tablename__ = "crm_orders"
-    __table_args__ = (
-        Index("idx_crm_orders_customer", "customer_id"),
-        Index("idx_crm_orders_status", "status"),
-    )
-
-    id                 = Column(String(128), primary_key=True)   # Firestore doc ID
-    customer_id        = Column(String(128), ForeignKey("crm_customers.id", ondelete="CASCADE"),
-                                nullable=False)
-    order_number       = Column(String(64), unique=True, nullable=False)
-    title              = Column(String(500))
-    created_by         = Column(String(128))
-    status             = Column(String(64), nullable=False, default="negotiating")
-    status_updated_date = Column(DateTime(timezone=True))
-    status_updated_by  = Column(String(128))
-    items              = Column(JSONB, nullable=False, default=list)
-    subtotal           = Column(Numeric(12, 2), nullable=False, default=0)
-    discount           = Column(JSONB)
-    total_price        = Column(Numeric(12, 2), nullable=False, default=0)
-    currency           = Column(String(10), nullable=False, default="EUR")
-    shipping           = Column(JSONB)
-    payment_status     = Column(JSONB, nullable=False, default=dict)
-    invoice_path       = Column(String(500))
-    notes              = Column(Text)
-    timeline           = Column(JSONB, nullable=False, default=list)
-    created_at         = Column(DateTime(timezone=True), nullable=False)
-    updated_at         = Column(DateTime(timezone=True), nullable=False)
-
-    customer = relationship("CrmCustomer", back_populates="orders")
-
-
-class CrmCommsLog(Base):
-    __tablename__ = "crm_comms_log"
-    __table_args__ = (
-        Index("idx_crm_comms_customer", "customer_id", "occurred_at"),
-    )
-
-    id             = Column(String(128), primary_key=True)
-    customer_id    = Column(String(128), ForeignKey("crm_customers.id", ondelete="SET NULL"),
-                            nullable=True)
-    type           = Column(String(32), nullable=False)       # email | sms | call | note | ...
-    mail_account   = Column(String(256))
-    direction      = Column(String(16), nullable=False)       # inbound | outbound
-    subject        = Column(String(500))
-    body           = Column(Text)
-    body_html      = Column(Text)
-    attachments    = Column(JSONB, nullable=False, default=list)
-    ext_message_id = Column(String(500))
-    from_addr      = Column(String(500))
-    to_addrs       = Column(Text)                             # JSON array as text or comma-sep
-    logged_by      = Column(String(128))
-    is_important   = Column(Boolean, nullable=False, default=False)
-    is_read        = Column(Boolean, nullable=False, default=True)
-    occurred_at    = Column(DateTime(timezone=True), nullable=False)
-    created_at     = Column(DateTime(timezone=True), nullable=False, default=_now)
-
-    customer = relationship("CrmCustomer", back_populates="comms")
-
-
-class CrmMedia(Base):
-    __tablename__ = "crm_media"
-    __table_args__ = (
-        Index("idx_crm_media_customer", "customer_id"),
-        Index("idx_crm_media_order", "order_id"),
-    )
-
-    id             = Column(String(128), primary_key=True)
-    customer_id    = Column(String(128), ForeignKey("crm_customers.id", ondelete="SET NULL"),
-                            nullable=True)
-    order_id       = Column(String(128))
-    filename       = Column(String(500), nullable=False)
-    nextcloud_path = Column(String(1000), nullable=False)
-    thumbnail_path = Column(String(1000))
-    mime_type      = Column(String(128))
-    direction      = Column(String(16))
-    tags           = Column(JSONB, nullable=False, default=list)
-    uploaded_by    = Column(String(128))
-    created_at     = Column(DateTime(timezone=True), nullable=False, default=_now)
-
-    customer = relationship("CrmCustomer", back_populates="media")
-
-
-class CrmSyncState(Base):
-    __tablename__ = "crm_sync_state"
-
-    key   = Column(String(128), primary_key=True)
-    value = Column(Text)
-
-
-class CrmQuotation(Base):
-    __tablename__ = "crm_quotations"
-    __table_args__ = (
-        Index("idx_crm_quotations_customer", "customer_id"),
-    )
-
-    id                       = Column(String(128), primary_key=True)
-    quotation_number         = Column(String(64), unique=True, nullable=False)
-    title                    = Column(String(500))
-    subtitle                 = Column(String(500))
-    customer_id              = Column(String(128), ForeignKey("crm_customers.id", ondelete="CASCADE"),
-                                      nullable=False)
-    language                 = Column(String(10), nullable=False, default="en")
-    status                   = Column(String(32), nullable=False, default="draft")
-    order_type               = Column(String(64))
-    shipping_method          = Column(String(64))
-    estimated_shipping_date  = Column(String(32))            # stored as DATE string
-    global_discount_label    = Column(String(128))
-    global_discount_percent  = Column(Numeric(8, 4), nullable=False, default=0)
-    vat_percent              = Column(Numeric(8, 4), nullable=False, default=24)
-    global_vat_percent       = Column(Numeric(8, 4), nullable=False, default=24)
-    shipping_cost            = Column(Numeric(12, 2), nullable=False, default=0)
-    shipping_cost_discount   = Column(Numeric(12, 2), nullable=False, default=0)
-    install_cost             = Column(Numeric(12, 2), nullable=False, default=0)
-    install_cost_discount    = Column(Numeric(12, 2), nullable=False, default=0)
-    extras_label             = Column(String(256))
-    extras_cost              = Column(Numeric(12, 2), nullable=False, default=0)
-    comments                 = Column(JSONB, nullable=False, default=list)
-    quick_notes              = Column(JSONB, nullable=False, default=dict)
-    subtotal_before_discount = Column(Numeric(12, 2), nullable=False, default=0)
-    global_discount_amount   = Column(Numeric(12, 2), nullable=False, default=0)
-    new_subtotal             = Column(Numeric(12, 2), nullable=False, default=0)
-    vat_amount               = Column(Numeric(12, 2), nullable=False, default=0)
-    final_total              = Column(Numeric(12, 2), nullable=False, default=0)
-    nextcloud_pdf_path       = Column(String(1000))
-    nextcloud_pdf_url        = Column(String(1000))
-    # Client snapshot fields (denormalised for PDF generation)
-    client_org               = Column(String(500))
-    client_name              = Column(String(500))
-    client_location          = Column(String(500))
-    client_phone             = Column(String(64))
-    client_email             = Column(String(256))
-    # Legacy quotation fields
-    is_legacy                = Column(Boolean, nullable=False, default=False)
-    legacy_date              = Column(String(32))
-    legacy_pdf_path          = Column(String(1000))
-    created_at               = Column(DateTime(timezone=True), nullable=False)
-    updated_at               = Column(DateTime(timezone=True), nullable=False)
-
-    customer = relationship("CrmCustomer", back_populates="quotations")
-    items    = relationship("CrmQuotationItem", back_populates="quotation",
-                            cascade="all, delete-orphan",
-                            order_by="CrmQuotationItem.sort_order", lazy="noload")
-
-
-class CrmQuotationItem(Base):
-    __tablename__ = "crm_quotation_items"
-    __table_args__ = (
-        Index("idx_crm_quotation_items_quotation", "quotation_id", "sort_order"),
-    )
-
-    id               = Column(String(128), primary_key=True)
-    quotation_id     = Column(String(128), ForeignKey("crm_quotations.id", ondelete="CASCADE"),
-                              nullable=False)
-    product_id       = Column(String(128))
-    description      = Column(Text)
-    description_en   = Column(Text)
-    description_gr   = Column(Text)
-    unit_type        = Column(String(32), nullable=False, default="pcs")
-    unit_cost        = Column(Numeric(12, 4), nullable=False, default=0)
-    discount_percent = Column(Numeric(8, 4), nullable=False, default=0)
-    vat_percent      = Column(Numeric(8, 4), nullable=False, default=24)
-    quantity         = Column(Numeric(12, 4), nullable=False, default=1)
-    line_total       = Column(Numeric(12, 2), nullable=False, default=0)
-    sort_order       = Column(Integer, nullable=False, default=0)
-
-    quotation = relationship("CrmQuotation", back_populates="items")

backend/crm/quotation_models.py

@@ -1,162 +0,0 @@
-from enum import Enum
-from typing import Any, Dict, List, Optional
-from pydantic import BaseModel
-
-
-class QuotationStatus(str, Enum):
-    draft = "draft"
-    built = "built"
-    sent = "sent"
-    accepted = "accepted"
-    declined = "declined"
-
-
-class QuotationItemCreate(BaseModel):
-    product_id: Optional[str] = None
-    description: Optional[str] = None
-    description_en: Optional[str] = None
-    description_gr: Optional[str] = None
-    unit_type: str = "pcs"  # pcs / kg / m
-    unit_cost: float = 0.0
-    discount_percent: float = 0.0
-    quantity: float = 1.0
-    vat_percent: float = 24.0
-    sort_order: int = 0
-
-
-class QuotationItemInDB(QuotationItemCreate):
-    id: str
-    quotation_id: str
-    line_total: float = 0.0
-
-
-class QuotationCreate(BaseModel):
-    customer_id: str
-    title: Optional[str] = None
-    subtitle: Optional[str] = None
-    language: str = "en"  # en / gr
-    order_type: Optional[str] = None
-    shipping_method: Optional[str] = None
-    estimated_shipping_date: Optional[str] = None
-    global_discount_label: Optional[str] = None
-    global_discount_percent: float = 0.0
-    global_vat_percent: float = 24.0
-    shipping_cost: float = 0.0
-    shipping_cost_discount: float = 0.0
-    install_cost: float = 0.0
-    install_cost_discount: float = 0.0
-    extras_label: Optional[str] = None
-    extras_cost: float = 0.0
-    comments: List[str] = []
-    quick_notes: Optional[Dict[str, Any]] = None
-    items: List[QuotationItemCreate] = []
-    # Client override fields (for this quotation only; customer record is not modified)
-    client_org: Optional[str] = None
-    client_name: Optional[str] = None
-    client_location: Optional[str] = None
-    client_phone: Optional[str] = None
-    client_email: Optional[str] = None
-    # Legacy quotation fields
-    is_legacy: bool = False
-    legacy_date: Optional[str] = None  # ISO date string, manually set
-    legacy_pdf_path: Optional[str] = None  # Nextcloud path to uploaded PDF
-
-
-class QuotationUpdate(BaseModel):
-    title: Optional[str] = None
-    subtitle: Optional[str] = None
-    language: Optional[str] = None
-    status: Optional[QuotationStatus] = None
-    order_type: Optional[str] = None
-    shipping_method: Optional[str] = None
-    estimated_shipping_date: Optional[str] = None
-    global_discount_label: Optional[str] = None
-    global_discount_percent: Optional[float] = None
-    global_vat_percent: Optional[float] = None
-    shipping_cost: Optional[float] = None
-    shipping_cost_discount: Optional[float] = None
-    install_cost: Optional[float] = None
-    install_cost_discount: Optional[float] = None
-    extras_label: Optional[str] = None
-    extras_cost: Optional[float] = None
-    comments: Optional[List[str]] = None
-    quick_notes: Optional[Dict[str, Any]] = None
-    items: Optional[List[QuotationItemCreate]] = None
-    # Client override fields
-    client_org: Optional[str] = None
-    client_name: Optional[str] = None
-    client_location: Optional[str] = None
-    client_phone: Optional[str] = None
-    client_email: Optional[str] = None
-    # Legacy quotation fields
-    is_legacy: Optional[bool] = None
-    legacy_date: Optional[str] = None
-    legacy_pdf_path: Optional[str] = None
-
-
-class QuotationInDB(BaseModel):
-    id: str
-    quotation_number: str
-    customer_id: str
-    title: Optional[str] = None
-    subtitle: Optional[str] = None
-    language: str = "en"
-    status: QuotationStatus = QuotationStatus.draft
-    order_type: Optional[str] = None
-    shipping_method: Optional[str] = None
-    estimated_shipping_date: Optional[str] = None
-    global_discount_label: Optional[str] = None
-    global_discount_percent: float = 0.0
-    global_vat_percent: float = 24.0
-    shipping_cost: float = 0.0
-    shipping_cost_discount: float = 0.0
-    install_cost: float = 0.0
-    install_cost_discount: float = 0.0
-    extras_label: Optional[str] = None
-    extras_cost: float = 0.0
-    comments: List[str] = []
-    quick_notes: Dict[str, Any] = {}
-    subtotal_before_discount: float = 0.0
-    global_discount_amount: float = 0.0
-    new_subtotal: float = 0.0
-    vat_amount: float = 0.0
-    final_total: float = 0.0
-    nextcloud_pdf_path: Optional[str] = None
-    nextcloud_pdf_url: Optional[str] = None
-    created_at: str
-    updated_at: str
-    items: List[QuotationItemInDB] = []
-    # Client override fields
-    client_org: Optional[str] = None
-    client_name: Optional[str] = None
-    client_location: Optional[str] = None
-    client_phone: Optional[str] = None
-    client_email: Optional[str] = None
-    # Legacy quotation fields
-    is_legacy: bool = False
-    legacy_date: Optional[str] = None
-    legacy_pdf_path: Optional[str] = None
-
-
-class QuotationListItem(BaseModel):
-    id: str
-    quotation_number: str
-    title: Optional[str] = None
-    customer_id: str
-    status: QuotationStatus
-    final_total: float
-    created_at: str
-    updated_at: str
-    nextcloud_pdf_url: Optional[str] = None
-    is_legacy: bool = False
-    legacy_date: Optional[str] = None
-    legacy_pdf_path: Optional[str] = None
-
-
-class QuotationListResponse(BaseModel):
-    quotations: List[QuotationListItem]
-    total: int
-
-
-class NextNumberResponse(BaseModel):
-    next_number: str

backend/crm/quotations_router.py

@@ -1,143 +0,0 @@
-from fastapi import APIRouter, Depends, Query, UploadFile, File
-from fastapi.responses import StreamingResponse
-from typing import Optional
-import io
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from auth.dependencies import require_permission
-from auth.models import TokenPayload
-from crm.quotation_models import (
-    NextNumberResponse,
-    QuotationCreate,
-    QuotationInDB,
-    QuotationListResponse,
-    QuotationUpdate,
-)
-from crm import quotations_service as svc
-from database.postgres import get_pg_session
-from shared.audit import log_action
-
-router = APIRouter(prefix="/api/crm/quotations", tags=["crm-quotations"])
-
-
-# IMPORTANT: Static paths must come BEFORE /{id} to avoid route collision in FastAPI
-
-@router.get("/next-number", response_model=NextNumberResponse)
-async def get_next_number(
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Returns the next available quotation number (preview only — does not commit)."""
-    next_num = await svc.get_next_number()
-    return NextNumberResponse(next_number=next_num)
-
-
-@router.get("/all", response_model=list[dict])
-async def list_all_quotations(
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Returns all quotations across all customers, each including customer_name."""
-    return await svc.list_all_quotations()
-
-
-@router.get("/customer/{customer_id}", response_model=QuotationListResponse)
-async def list_quotations_for_customer(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    quotations = await svc.list_quotations(customer_id)
-    return QuotationListResponse(quotations=quotations, total=len(quotations))
-
-
-@router.get("/{quotation_id}/pdf")
-async def proxy_quotation_pdf(
-    quotation_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    """Proxy the quotation PDF from Nextcloud to bypass browser cookie restrictions."""
-    pdf_bytes = await svc.get_quotation_pdf_bytes(quotation_id)
-    return StreamingResponse(
-        io.BytesIO(pdf_bytes),
-        media_type="application/pdf",
-        headers={"Content-Disposition": "inline"},
-    )
-
-
-@router.get("/{quotation_id}", response_model=QuotationInDB)
-async def get_quotation(
-    quotation_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return await svc.get_quotation(quotation_id)
-
-
-@router.post("", response_model=QuotationInDB, status_code=201)
-async def create_quotation(
-    body: QuotationCreate,
-    generate_pdf: bool = Query(False),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """
-    Create a quotation. Pass ?generate_pdf=true to immediately generate and upload the PDF.
-    """
-    q = await svc.create_quotation(body, generate_pdf=generate_pdf)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "quotation",
-                     str(q.id), q.quotation_number or str(q.id))
-    return q
-
-
-@router.put("/{quotation_id}", response_model=QuotationInDB)
-async def update_quotation(
-    quotation_id: str,
-    body: QuotationUpdate,
-    generate_pdf: bool = Query(False),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """
-    Update a quotation. Pass ?generate_pdf=true to regenerate the PDF.
-    """
-    old = await svc.get_quotation(quotation_id)
-    q = await svc.update_quotation(quotation_id, body, generate_pdf=generate_pdf)
-    _SKIP = {"updated_at", "id", "items", "pdf_path"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(q, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(q, k, None)
-    }
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "quotation",
-                     quotation_id, q.quotation_number or quotation_id, changes=changes or None)
-    return q
-
-
-@router.delete("/{quotation_id}", status_code=204)
-async def delete_quotation(
-    quotation_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    q = await svc.get_quotation(quotation_id)
-    await svc.delete_quotation(quotation_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "quotation",
-                     quotation_id, q.quotation_number if q else quotation_id)
-
-
-@router.post("/{quotation_id}/regenerate-pdf", response_model=QuotationInDB)
-async def regenerate_pdf(
-    quotation_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """Force PDF regeneration and re-upload to Nextcloud."""
-    return await svc.regenerate_pdf(quotation_id)
-
-
-@router.post("/{quotation_id}/legacy-pdf", response_model=QuotationInDB)
-async def upload_legacy_pdf(
-    quotation_id: str,
-    file: UploadFile = File(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """Upload a PDF file for a legacy quotation and store its Nextcloud path."""
-    pdf_bytes = await file.read()
-    filename = file.filename or f"legacy-{quotation_id}.pdf"
-    return await svc.upload_legacy_pdf(quotation_id, pdf_bytes, filename)

backend/crm/quotations_service.py

@@ -1,566 +0,0 @@
-import json
-import logging
-import os
-import uuid
-from datetime import datetime
-from decimal import Decimal, ROUND_HALF_UP
-from pathlib import Path
-from typing import Optional
-
-from fastapi import HTTPException
-
-from crm import nextcloud
-from crm.quotation_models import (
-    QuotationCreate,
-    QuotationInDB,
-    QuotationItemCreate,
-    QuotationItemInDB,
-    QuotationListItem,
-    QuotationUpdate,
-)
-from crm.service import get_customer
-import database as mqtt_db
-
-logger = logging.getLogger(__name__)
-
-# Path to Jinja2 templates directory (relative to this file)
-_TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-
-
-# ── Helpers ───────────────────────────────────────────────────────────────────
-
-def _d(value) -> Decimal:
-    """Convert to Decimal safely."""
-    return Decimal(str(value if value is not None else 0))
-
-
-def _float(d: Decimal) -> float:
-    """Round Decimal to 2dp and return as float for storage."""
-    return float(d.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP))
-
-
-def _calculate_totals(
-    items: list,
-    global_discount_percent: float,
-    global_vat_percent: float,
-    shipping_cost: float,
-    shipping_cost_discount: float,
-    install_cost: float,
-    install_cost_discount: float,
-    extras_cost: float,
-) -> dict:
-    """
-    Calculate all monetary totals using Decimal arithmetic (ROUND_HALF_UP).
-    VAT is a single global rate applied to items only (not shipping or install).
-    Shipping and install costs carry 0% VAT.
-    Returns a dict of floats ready for DB storage.
-    """
-    # Per-line totals (items only)
-    item_totals = []
-    for item in items:
-        cost = _d(item.get("unit_cost", 0))
-        qty = _d(item.get("quantity", 1))
-        disc = _d(item.get("discount_percent", 0))
-        net = cost * qty * (1 - disc / 100)
-        item_totals.append(net)
-
-    items_net = sum(item_totals, Decimal(0))
-
-    # Shipping net (VAT = 0%)
-    ship_gross = _d(shipping_cost)
-    ship_disc = _d(shipping_cost_discount)
-    ship_net = ship_gross * (1 - ship_disc / 100)
-
-    # Install net (VAT = 0%)
-    install_gross = _d(install_cost)
-    install_disc = _d(install_cost_discount)
-    install_net = install_gross * (1 - install_disc / 100)
-
-    subtotal = items_net + ship_net + install_net
-
-    global_disc_pct = _d(global_discount_percent)
-    global_disc_amount = subtotal * (global_disc_pct / 100)
-    new_subtotal = subtotal - global_disc_amount
-
-    # VAT applies only to items portion, scaled by the global discount ratio
-    vat_pct = _d(global_vat_percent)
-    if subtotal > 0 and items_net > 0:
-        items_ratio = items_net / subtotal
-        vat_amount = new_subtotal * items_ratio * (vat_pct / 100)
-    else:
-        vat_amount = Decimal(0)
-
-    extras = _d(extras_cost)
-    final_total = new_subtotal + vat_amount + extras
-
-    return {
-        "subtotal_before_discount": _float(subtotal),
-        "global_discount_amount": _float(global_disc_amount),
-        "new_subtotal": _float(new_subtotal),
-        "vat_amount": _float(vat_amount),
-        "final_total": _float(final_total),
-    }
-
-
-def _calc_line_total(item) -> float:
-    cost = _d(item.get("unit_cost", 0))
-    qty = _d(item.get("quantity", 1))
-    disc = _d(item.get("discount_percent", 0))
-    return _float(cost * qty * (1 - disc / 100))
-
-
-async def _generate_quotation_number(db) -> str:
-    now = datetime.utcnow()
-    yy = now.strftime("%y")
-    mm = now.strftime("%m")
-    prefix = f"QT-{yy}-{mm}-"
-    rows = await db.execute_fetchall(
-        "SELECT quotation_number FROM crm_quotations WHERE quotation_number LIKE ? ORDER BY quotation_number DESC LIMIT 1",
-        (f"{prefix}%",),
-    )
-    if rows:
-        last_num = rows[0][0]  # e.g. "QT-26-04-012"
-        try:
-            seq = int(last_num[len(prefix):]) + 1
-        except ValueError:
-            seq = 1
-    else:
-        seq = 1
-    return f"{prefix}{seq:03d}"
-
-
-def _row_to_quotation(row: dict, items: list[dict]) -> QuotationInDB:
-    row = dict(row)
-    row["comments"] = json.loads(row.get("comments") or "[]")
-    row["quick_notes"] = json.loads(row.get("quick_notes") or "{}")
-    item_models = [QuotationItemInDB(**{k: v for k, v in i.items() if k in QuotationItemInDB.model_fields}) for i in items]
-    return QuotationInDB(**{k: v for k, v in row.items() if k in QuotationInDB.model_fields}, items=item_models)
-
-
-def _row_to_list_item(row: dict) -> QuotationListItem:
-    return QuotationListItem(**{k: v for k, v in dict(row).items() if k in QuotationListItem.model_fields})
-
-
-async def _fetch_items(db, quotation_id: str) -> list[dict]:
-    rows = await db.execute_fetchall(
-        "SELECT * FROM crm_quotation_items WHERE quotation_id = ? ORDER BY sort_order ASC",
-        (quotation_id,),
-    )
-    return [dict(r) for r in rows]
-
-
-# ── Public API ────────────────────────────────────────────────────────────────
-
-async def get_next_number() -> str:
-    db = await mqtt_db.get_db()
-    return await _generate_quotation_number(db)
-
-
-async def list_all_quotations() -> list[dict]:
-    """Return all quotations across all customers, with customer_name injected."""
-    from shared.firebase import get_db as get_firestore
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT id, quotation_number, title, customer_id, status, final_total, created_at, updated_at, "
-        "nextcloud_pdf_url, is_legacy, legacy_date, legacy_pdf_path "
-        "FROM crm_quotations ORDER BY created_at DESC",
-        (),
-    )
-    items = [dict(r) for r in rows]
-    # Fetch unique customer names from Firestore in one pass
-    customer_ids = {i["customer_id"] for i in items if i.get("customer_id")}
-    customer_names: dict[str, str] = {}
-    if customer_ids:
-        fstore = get_firestore()
-        for cid in customer_ids:
-            try:
-                doc = fstore.collection("crm_customers").document(cid).get()
-                if doc.exists:
-                    d = doc.to_dict()
-                    name_parts = [d.get("name", ""), d.get("surname", "")]
-                    full_name = " ".join(p for p in name_parts if p).strip()
-                    org = (d.get("organization", "") or "").strip()
-                    customer_names[cid] = {"name": full_name or cid, "org": org}
-            except Exception:
-                customer_names[cid] = {"name": cid, "org": ""}
-    for item in items:
-        info = customer_names.get(item["customer_id"], {"name": "", "org": ""})
-        item["customer_name"] = info["name"]
-        item["customer_org"] = info["org"]
-    return items
-
-
-async def list_quotations(customer_id: str) -> list[QuotationListItem]:
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT id, quotation_number, title, customer_id, status, final_total, created_at, updated_at, "
-        "nextcloud_pdf_url, is_legacy, legacy_date, legacy_pdf_path "
-        "FROM crm_quotations WHERE customer_id = ? ORDER BY created_at DESC",
-        (customer_id,),
-    )
-    return [_row_to_list_item(dict(r)) for r in rows]
-
-
-async def get_quotation(quotation_id: str) -> QuotationInDB:
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT * FROM crm_quotations WHERE id = ?", (quotation_id,)
-    )
-    if not rows:
-        raise HTTPException(status_code=404, detail="Quotation not found")
-    items = await _fetch_items(db, quotation_id)
-    return _row_to_quotation(dict(rows[0]), items)
-
-
-async def create_quotation(data: QuotationCreate, generate_pdf: bool = False) -> QuotationInDB:
-    db = await mqtt_db.get_db()
-    now = datetime.utcnow().isoformat()
-    qid = str(uuid.uuid4())
-    quotation_number = await _generate_quotation_number(db)
-
-    # Build items list for calculation
-    items_raw = [item.model_dump() for item in data.items]
-
-    # Calculate per-item line totals
-    for item in items_raw:
-        item["line_total"] = _calc_line_total(item)
-
-    totals = _calculate_totals(
-        items_raw,
-        data.global_discount_percent,
-        data.global_vat_percent,
-        data.shipping_cost,
-        data.shipping_cost_discount,
-        data.install_cost,
-        data.install_cost_discount,
-        data.extras_cost,
-    )
-
-    comments_json = json.dumps(data.comments)
-    quick_notes_json = json.dumps(data.quick_notes or {})
-
-    await db.execute(
-        """INSERT INTO crm_quotations (
-            id, quotation_number, title, subtitle, customer_id,
-            language, status, order_type, shipping_method, estimated_shipping_date,
-            global_discount_label, global_discount_percent, global_vat_percent,
-            shipping_cost, shipping_cost_discount, install_cost, install_cost_discount,
-            extras_label, extras_cost, comments, quick_notes,
-            subtotal_before_discount, global_discount_amount, new_subtotal, vat_amount, final_total,
-            nextcloud_pdf_path, nextcloud_pdf_url,
-            client_org, client_name, client_location, client_phone, client_email,
-            is_legacy, legacy_date, legacy_pdf_path,
-            created_at, updated_at
-        ) VALUES (
-            ?, ?, ?, ?, ?,
-            ?, 'draft', ?, ?, ?,
-            ?, ?, ?,
-            ?, ?, ?, ?,
-            ?, ?, ?, ?,
-            ?, ?, ?, ?, ?,
-            NULL, NULL,
-            ?, ?, ?, ?, ?,
-            ?, ?, ?,
-            ?, ?
-        )""",
-        (
-            qid, quotation_number, data.title, data.subtitle, data.customer_id,
-            data.language, data.order_type, data.shipping_method, data.estimated_shipping_date,
-            data.global_discount_label, data.global_discount_percent, data.global_vat_percent,
-            data.shipping_cost, data.shipping_cost_discount, data.install_cost, data.install_cost_discount,
-            data.extras_label, data.extras_cost, comments_json, quick_notes_json,
-            totals["subtotal_before_discount"], totals["global_discount_amount"],
-            totals["new_subtotal"], totals["vat_amount"], totals["final_total"],
-            data.client_org, data.client_name, data.client_location, data.client_phone, data.client_email,
-            1 if data.is_legacy else 0, data.legacy_date, data.legacy_pdf_path,
-            now, now,
-        ),
-    )
-
-    # Insert items
-    for i, item in enumerate(items_raw):
-        item_id = str(uuid.uuid4())
-        await db.execute(
-            """INSERT INTO crm_quotation_items
-               (id, quotation_id, product_id, description, description_en, description_gr,
-                unit_type, unit_cost, discount_percent, quantity, vat_percent, line_total, sort_order)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-            (
-                item_id, qid, item.get("product_id"), item.get("description"),
-                item.get("description_en"), item.get("description_gr"),
-                item.get("unit_type", "pcs"), item.get("unit_cost", 0),
-                item.get("discount_percent", 0), item.get("quantity", 1),
-                item.get("vat_percent", 24), item["line_total"], item.get("sort_order", i),
-            ),
-        )
-
-    await db.commit()
-
-    quotation = await get_quotation(qid)
-
-    if generate_pdf and not data.is_legacy:
-        quotation = await _do_generate_and_upload_pdf(quotation)
-
-    return quotation
-
-
-async def update_quotation(quotation_id: str, data: QuotationUpdate, generate_pdf: bool = False) -> QuotationInDB:
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT * FROM crm_quotations WHERE id = ?", (quotation_id,)
-    )
-    if not rows:
-        raise HTTPException(status_code=404, detail="Quotation not found")
-
-    existing = dict(rows[0])
-    now = datetime.utcnow().isoformat()
-
-    # Merge update into existing values
-    update_fields = data.model_dump(exclude_none=True)
-
-    # Build SET clause — handle comments JSON separately
-    set_parts = []
-    params = []
-
-    scalar_fields = [
-        "title", "subtitle", "language", "status", "order_type", "shipping_method",
-        "estimated_shipping_date", "global_discount_label", "global_discount_percent", "global_vat_percent",
-        "shipping_cost", "shipping_cost_discount", "install_cost",
-        "install_cost_discount", "extras_label", "extras_cost",
-        "client_org", "client_name", "client_location", "client_phone", "client_email",
-        "legacy_date", "legacy_pdf_path",
-    ]
-
-    for field in scalar_fields:
-        if field in update_fields:
-            set_parts.append(f"{field} = ?")
-            params.append(update_fields[field])
-
-    if "comments" in update_fields:
-        set_parts.append("comments = ?")
-        params.append(json.dumps(update_fields["comments"]))
-
-    if "quick_notes" in update_fields:
-        set_parts.append("quick_notes = ?")
-        params.append(json.dumps(update_fields["quick_notes"] or {}))
-
-    # Recalculate totals using merged values
-    merged = {**existing, **{k: update_fields.get(k, existing.get(k)) for k in scalar_fields}}
-
-    # If items are being updated, recalculate with new items; otherwise use existing items
-    if "items" in update_fields:
-        items_raw = [item.model_dump() for item in data.items]
-        for item in items_raw:
-            item["line_total"] = _calc_line_total(item)
-    else:
-        existing_items = await _fetch_items(db, quotation_id)
-        items_raw = existing_items
-
-    totals = _calculate_totals(
-        items_raw,
-        float(merged.get("global_discount_percent", 0)),
-        float(merged.get("global_vat_percent", 24)),
-        float(merged.get("shipping_cost", 0)),
-        float(merged.get("shipping_cost_discount", 0)),
-        float(merged.get("install_cost", 0)),
-        float(merged.get("install_cost_discount", 0)),
-        float(merged.get("extras_cost", 0)),
-    )
-
-    for field, val in totals.items():
-        set_parts.append(f"{field} = ?")
-        params.append(val)
-
-    set_parts.append("updated_at = ?")
-    params.append(now)
-    params.append(quotation_id)
-
-    if set_parts:
-        await db.execute(
-            f"UPDATE crm_quotations SET {', '.join(set_parts)} WHERE id = ?",
-            params,
-        )
-
-    # Replace items if provided
-    if "items" in update_fields:
-        await db.execute("DELETE FROM crm_quotation_items WHERE quotation_id = ?", (quotation_id,))
-        for i, item in enumerate(items_raw):
-            item_id = str(uuid.uuid4())
-            await db.execute(
-                """INSERT INTO crm_quotation_items
-                   (id, quotation_id, product_id, description, description_en, description_gr,
-                    unit_type, unit_cost, discount_percent, quantity, vat_percent, line_total, sort_order)
-                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-                (
-                    item_id, quotation_id, item.get("product_id"), item.get("description"),
-                    item.get("description_en"), item.get("description_gr"),
-                    item.get("unit_type", "pcs"), item.get("unit_cost", 0),
-                    item.get("discount_percent", 0), item.get("quantity", 1),
-                    item.get("vat_percent", 24), item["line_total"], item.get("sort_order", i),
-                ),
-            )
-
-    await db.commit()
-
-    quotation = await get_quotation(quotation_id)
-
-    if generate_pdf:
-        quotation = await _do_generate_and_upload_pdf(quotation)
-
-    return quotation
-
-
-async def delete_quotation(quotation_id: str) -> None:
-    db = await mqtt_db.get_db()
-    rows = await db.execute_fetchall(
-        "SELECT nextcloud_pdf_path FROM crm_quotations WHERE id = ?", (quotation_id,)
-    )
-    if not rows:
-        raise HTTPException(status_code=404, detail="Quotation not found")
-
-    pdf_path = dict(rows[0]).get("nextcloud_pdf_path")
-
-    await db.execute("DELETE FROM crm_quotation_items WHERE quotation_id = ?", (quotation_id,))
-    await db.execute("DELETE FROM crm_quotations WHERE id = ?", (quotation_id,))
-    await db.commit()
-
-    # Remove PDF from Nextcloud (best-effort)
-    if pdf_path:
-        try:
-            await nextcloud.delete_file(pdf_path)
-        except Exception as e:
-            logger.warning("Failed to delete PDF from Nextcloud (%s): %s", pdf_path, e)
-
-
-# ── PDF Generation ─────────────────────────────────────────────────────────────
-
-async def _do_generate_and_upload_pdf(quotation: QuotationInDB) -> QuotationInDB:
-    """Generate PDF, upload to Nextcloud, update DB record. Returns updated quotation."""
-    try:
-        customer = get_customer(quotation.customer_id)
-    except Exception as e:
-        logger.error("Cannot generate PDF — customer not found: %s", e)
-        return quotation
-
-    try:
-        pdf_bytes = await _generate_pdf_bytes(quotation, customer)
-    except Exception as e:
-        logger.error("PDF generation failed for quotation %s: %s", quotation.id, e)
-        return quotation
-
-    # Delete old PDF if present
-    if quotation.nextcloud_pdf_path:
-        try:
-            await nextcloud.delete_file(quotation.nextcloud_pdf_path)
-        except Exception:
-            pass
-
-    try:
-        pdf_path, pdf_url = await _upload_pdf(customer, quotation, pdf_bytes)
-    except Exception as e:
-        logger.error("PDF upload failed for quotation %s: %s", quotation.id, e)
-        return quotation
-
-    # Persist paths
-    db = await mqtt_db.get_db()
-    await db.execute(
-        "UPDATE crm_quotations SET nextcloud_pdf_path = ?, nextcloud_pdf_url = ? WHERE id = ?",
-        (pdf_path, pdf_url, quotation.id),
-    )
-    await db.commit()
-
-    return await get_quotation(quotation.id)
-
-
-async def _generate_pdf_bytes(quotation: QuotationInDB, customer) -> bytes:
-    """Render Jinja2 template and convert to PDF via WeasyPrint."""
-    from jinja2 import Environment, FileSystemLoader, select_autoescape
-    import weasyprint
-
-    env = Environment(
-        loader=FileSystemLoader(str(_TEMPLATES_DIR)),
-        autoescape=select_autoescape(["html"]),
-    )
-
-    def format_money(value):
-        try:
-            f = float(value)
-            # Greek-style: dot thousands separator, comma decimal
-            formatted = f"{f:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".")
-            return f"{formatted} €"
-        except (TypeError, ValueError):
-            return "0,00 €"
-
-    env.filters["format_money"] = format_money
-
-    template = env.get_template("quotation.html")
-
-    html_str = template.render(
-        quotation=quotation,
-        customer=customer,
-        lang=quotation.language,
-    )
-
-    pdf = weasyprint.HTML(string=html_str, base_url=str(_TEMPLATES_DIR)).write_pdf()
-    return pdf
-
-
-async def _upload_pdf(customer, quotation: QuotationInDB, pdf_bytes: bytes) -> tuple[str, str]:
-    """Upload PDF to Nextcloud, return (relative_path, public_url)."""
-    from crm.service import get_customer_nc_path
-    from config import settings
-
-    nc_folder = get_customer_nc_path(customer)
-    date_str = datetime.utcnow().strftime("%Y-%m-%d")
-    filename = f"Quotation-{quotation.quotation_number}-{date_str}.pdf"
-    rel_path = f"customers/{nc_folder}/quotations/{filename}"
-
-    await nextcloud.ensure_folder(f"customers/{nc_folder}/quotations")
-    await nextcloud.upload_file(rel_path, pdf_bytes, "application/pdf")
-
-    # Construct a direct WebDAV download URL
-    from crm.nextcloud import _full_url
-    pdf_url = _full_url(rel_path)
-
-    return rel_path, pdf_url
-
-
-async def regenerate_pdf(quotation_id: str) -> QuotationInDB:
-    quotation = await get_quotation(quotation_id)
-    return await _do_generate_and_upload_pdf(quotation)
-
-
-async def get_quotation_pdf_bytes(quotation_id: str) -> bytes:
-    """Download the PDF for a quotation from Nextcloud and return raw bytes."""
-    from fastapi import HTTPException
-    quotation = await get_quotation(quotation_id)
-    # For legacy quotations, the PDF is at legacy_pdf_path
-    path = quotation.legacy_pdf_path if quotation.is_legacy else quotation.nextcloud_pdf_path
-    if not path:
-        raise HTTPException(status_code=404, detail="No PDF available for this quotation")
-    pdf_bytes, _ = await nextcloud.download_file(path)
-    return pdf_bytes
-
-
-async def upload_legacy_pdf(quotation_id: str, pdf_bytes: bytes, filename: str) -> QuotationInDB:
-    """Upload a legacy PDF to Nextcloud and store its path in the quotation record."""
-    quotation = await get_quotation(quotation_id)
-    if not quotation.is_legacy:
-        raise HTTPException(status_code=400, detail="This quotation is not a legacy quotation")
-
-    from crm.service import get_customer, get_customer_nc_path
-    customer = get_customer(quotation.customer_id)
-    nc_folder = get_customer_nc_path(customer)
-
-    await nextcloud.ensure_folder(f"customers/{nc_folder}/quotations")
-    rel_path = f"customers/{nc_folder}/quotations/{filename}"
-    await nextcloud.upload_file(rel_path, pdf_bytes, "application/pdf")
-
-    db = await mqtt_db.get_db()
-    now = datetime.utcnow().isoformat()
-    await db.execute(
-        "UPDATE crm_quotations SET legacy_pdf_path = ?, updated_at = ? WHERE id = ?",
-        (rel_path, now, quotation_id),
-    )
-    await db.commit()
-    return await get_quotation(quotation_id)

backend/crm/router.py

@@ -1,115 +0,0 @@
-from fastapi import APIRouter, Depends, Query, UploadFile, File, HTTPException
-from fastapi.responses import FileResponse
-from typing import Optional
-import os
-import shutil
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from auth.models import TokenPayload
-from auth.dependencies import require_permission
-from crm.models import ProductCreate, ProductUpdate, ProductInDB, ProductListResponse
-from crm import service
-from database.postgres import get_pg_session
-from shared.audit import log_action
-
-router = APIRouter(prefix="/api/crm/products", tags=["crm-products"])
-
-PHOTO_DIR = os.path.join(os.path.dirname(__file__), "..", "storage", "product_images")
-os.makedirs(PHOTO_DIR, exist_ok=True)
-
-
-@router.get("", response_model=ProductListResponse)
-def list_products(
-    search: Optional[str] = Query(None),
-    category: Optional[str] = Query(None),
-    active_only: bool = Query(False),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    products = service.list_products(search=search, category=category, active_only=active_only)
-    return ProductListResponse(products=products, total=len(products))
-
-
-@router.get("/{product_id}", response_model=ProductInDB)
-def get_product(
-    product_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return service.get_product(product_id)
-
-
-@router.post("", response_model=ProductInDB, status_code=201)
-async def create_product(
-    body: ProductCreate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    product = service.create_product(body)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "product",
-                     product.id, product.name)
-    return product
-
-
-@router.put("/{product_id}", response_model=ProductInDB)
-async def update_product(
-    product_id: str,
-    body: ProductUpdate,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    old = service.get_product(product_id)
-    product = service.update_product(product_id, body)
-    _SKIP = {"updated_at", "id", "photo_url"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(product, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(product, k, None)
-    }
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "product",
-                     product_id, product.name, changes=changes or None)
-    return product
-
-
-@router.delete("/{product_id}", status_code=204)
-async def delete_product(
-    product_id: str,
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    product = service.get_product(product_id)
-    service.delete_product(product_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "product",
-                     product_id, product.name)
-
-
-@router.post("/{product_id}/photo", response_model=ProductInDB)
-async def upload_product_photo(
-    product_id: str,
-    file: UploadFile = File(...),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    """Upload a product photo. Accepts JPG or PNG, stored on disk."""
-    if file.content_type not in ("image/jpeg", "image/png", "image/webp"):
-        raise HTTPException(status_code=400, detail="Only JPG, PNG, or WebP images are accepted.")
-    ext = {"image/jpeg": "jpg", "image/png": "png", "image/webp": "webp"}.get(file.content_type, "jpg")
-    photo_path = os.path.join(PHOTO_DIR, f"{product_id}.{ext}")
-    # Remove any old photo files for this product
-    for old_ext in ("jpg", "png", "webp"):
-        old_path = os.path.join(PHOTO_DIR, f"{product_id}.{old_ext}")
-        if os.path.exists(old_path) and old_path != photo_path:
-            os.remove(old_path)
-    with open(photo_path, "wb") as f:
-        shutil.copyfileobj(file.file, f)
-    photo_url = f"/crm/products/{product_id}/photo"
-    return service.update_product(product_id, ProductUpdate(photo_url=photo_url))
-
-
-@router.get("/{product_id}/photo")
-def get_product_photo(
-    product_id: str,
-):
-    """Serve a product photo from disk."""
-    for ext in ("jpg", "png", "webp"):
-        photo_path = os.path.join(PHOTO_DIR, f"{product_id}.{ext}")
-        if os.path.exists(photo_path):
-            return FileResponse(photo_path)
-    raise HTTPException(status_code=404, detail="No photo found for this product.")

backend/crm/thumbnails.py

@@ -1,125 +0,0 @@
-"""
-Thumbnail generation for uploaded media files.
-
-Supports:
-  - Images (via Pillow): JPEG thumbnail at 300×300 max
-  - Videos (via ffmpeg subprocess): extract first frame as JPEG
-  - PDFs (via pdf2image + Poppler): render first page as JPEG
-
-Returns None if the type is unsupported or if generation fails.
-"""
-import io
-import logging
-import subprocess
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-THUMB_SIZE = (220, 220)  # small enough for gallery tiles; keeps files ~4-6 KB
-
-
-def _thumb_from_image(content: bytes) -> bytes | None:
-    try:
-        from PIL import Image, ImageOps
-        img = Image.open(io.BytesIO(content))
-        img = ImageOps.exif_transpose(img)  # honour EXIF Orientation tag before resizing
-        img = img.convert("RGB")
-        img.thumbnail(THUMB_SIZE, Image.LANCZOS)
-        out = io.BytesIO()
-        # quality=55 + optimize=True + progressive encoding → ~4-6 KB for typical photos
-        img.save(out, format="JPEG", quality=65, optimize=True, progressive=True)
-        return out.getvalue()
-    except Exception as e:
-        logger.warning("Image thumbnail failed: %s", e)
-        return None
-
-
-def _thumb_from_video(content: bytes) -> bytes | None:
-    """
-    Extract the first frame of a video as a JPEG thumbnail.
-
-    We write the video to a temp file instead of piping it to ffmpeg because
-    most video containers (MP4, MOV, MKV …) store their index (moov atom) at
-    an arbitrary offset and ffmpeg cannot seek on a pipe — causing rc≠0 with
-    "moov atom not found" or similar errors when stdin is used.
-    """
-    import tempfile
-    import os
-    try:
-        # Write to a temp file so ffmpeg can seek freely
-        with tempfile.NamedTemporaryFile(suffix=".video", delete=False) as tmp_in:
-            tmp_in.write(content)
-            tmp_in_path = tmp_in.name
-
-        with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_out:
-            tmp_out_path = tmp_out.name
-
-        try:
-            result = subprocess.run(
-                [
-                    "ffmpeg", "-y",
-                    "-i", tmp_in_path,
-                    "-vframes", "1",
-                    "-vf", f"scale={THUMB_SIZE[0]}:-2",
-                    "-q:v", "4",   # JPEG quality 1-31 (lower = better); 4 ≈ ~80% quality
-                    tmp_out_path,
-                ],
-                capture_output=True,
-                timeout=60,
-            )
-            if result.returncode == 0 and os.path.getsize(tmp_out_path) > 0:
-                with open(tmp_out_path, "rb") as f:
-                    return f.read()
-            logger.warning(
-                "ffmpeg video thumb failed (rc=%s): %s",
-                result.returncode,
-                result.stderr[-400:].decode(errors="replace") if result.stderr else "",
-            )
-            return None
-        finally:
-            os.unlink(tmp_in_path)
-            try:
-                os.unlink(tmp_out_path)
-            except FileNotFoundError:
-                pass
-    except FileNotFoundError:
-        logger.warning("ffmpeg not found — video thumbnails unavailable")
-        return None
-    except Exception as e:
-        logger.warning("Video thumbnail failed: %s", e)
-        return None
-
-
-def _thumb_from_pdf(content: bytes) -> bytes | None:
-    try:
-        from pdf2image import convert_from_bytes
-        pages = convert_from_bytes(content, first_page=1, last_page=1, size=THUMB_SIZE)
-        if not pages:
-            return None
-        out = io.BytesIO()
-        pages[0].save(out, format="JPEG", quality=55, optimize=True, progressive=True)
-        return out.getvalue()
-    except ImportError:
-        logger.warning("pdf2image not installed — PDF thumbnails unavailable")
-        return None
-    except Exception as e:
-        logger.warning("PDF thumbnail failed: %s", e)
-        return None
-
-
-def generate_thumbnail(content: bytes, mime_type: str, filename: str) -> bytes | None:
-    """
-    Generate a small JPEG thumbnail for the given file content.
-    Returns JPEG bytes or None if unsupported / generation fails.
-    """
-    mt = (mime_type or "").lower()
-    fn = (filename or "").lower()
-
-    if mt.startswith("image/"):
-        return _thumb_from_image(content)
-    if mt.startswith("video/"):
-        return _thumb_from_video(content)
-    if mt == "application/pdf" or fn.endswith(".pdf"):
-        return _thumb_from_pdf(content)
-
-    return None

backend/database/__init__.py

@@ -1,47 +0,0 @@
-# MQTT live data — Phase 5: all functions now backed by Postgres
-from database.pg_mqtt import (
-    init_db,
-    close_db,
-    purge_loop,
-    purge_old_data,
-    insert_log,
-    insert_heartbeat,
-    insert_command,
-    update_command_response,
-    get_logs,
-    get_heartbeats,
-    get_commands,
-    get_latest_heartbeats,
-    get_pending_command,
-    upsert_alert,
-    delete_alert,
-    get_alerts,
-    partition_manager_loop,
-    ensure_current_partitions,
-)
-
-# SQLite connection — still used by melodies, builder, manufacturing, and crm
-# modules that have not yet been cut over to Postgres.
-from database.core import get_db
-
-__all__ = [
-    "init_db",
-    "close_db",
-    "get_db",
-    "purge_loop",
-    "purge_old_data",
-    "insert_log",
-    "insert_heartbeat",
-    "insert_command",
-    "update_command_response",
-    "get_logs",
-    "get_heartbeats",
-    "get_commands",
-    "get_latest_heartbeats",
-    "get_pending_command",
-    "upsert_alert",
-    "delete_alert",
-    "get_alerts",
-    "partition_manager_loop",
-    "ensure_current_partitions",
-]

backend/database/core.py

@@ -1,455 +0,0 @@
-import aiosqlite
-import asyncio
-import json
-import logging
-import os
-from datetime import datetime, timedelta, timezone
-from config import settings
-
-logger = logging.getLogger("database")
-
-_db: aiosqlite.Connection | None = None
-
-SCHEMA_STATEMENTS = [
-    """CREATE TABLE IF NOT EXISTS device_logs (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        device_serial TEXT NOT NULL,
-        level TEXT NOT NULL,
-        message TEXT NOT NULL,
-        device_timestamp INTEGER,
-        received_at TEXT NOT NULL DEFAULT (datetime('now'))
-    )""",
-    """CREATE TABLE IF NOT EXISTS heartbeats (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        device_serial TEXT NOT NULL,
-        device_id TEXT,
-        firmware_version TEXT,
-        ip_address TEXT,
-        gateway TEXT,
-        uptime_ms INTEGER,
-        uptime_display TEXT,
-        received_at TEXT NOT NULL DEFAULT (datetime('now'))
-    )""",
-    """CREATE TABLE IF NOT EXISTS commands (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        device_serial TEXT NOT NULL,
-        command_name TEXT NOT NULL,
-        command_payload TEXT,
-        status TEXT NOT NULL DEFAULT 'pending',
-        response_payload TEXT,
-        sent_at TEXT NOT NULL DEFAULT (datetime('now')),
-        responded_at TEXT
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_logs_serial_time ON device_logs(device_serial, received_at)",
-    "CREATE INDEX IF NOT EXISTS idx_logs_level ON device_logs(level)",
-    "CREATE INDEX IF NOT EXISTS idx_heartbeats_serial_time ON heartbeats(device_serial, received_at)",
-    "CREATE INDEX IF NOT EXISTS idx_commands_serial_time ON commands(device_serial, sent_at)",
-    "CREATE INDEX IF NOT EXISTS idx_commands_status ON commands(status)",
-    # Melody drafts table
-    """CREATE TABLE IF NOT EXISTS melody_drafts (
-        id TEXT PRIMARY KEY,
-        status TEXT NOT NULL DEFAULT 'draft',
-        data TEXT NOT NULL,
-        created_at TEXT NOT NULL DEFAULT (datetime('now')),
-        updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_melody_drafts_status ON melody_drafts(status)",
-    # Built melodies table (local melody builder)
-    """CREATE TABLE IF NOT EXISTS built_melodies (
-        id TEXT PRIMARY KEY,
-        name TEXT NOT NULL,
-        pid TEXT NOT NULL,
-        steps TEXT NOT NULL,
-        binary_path TEXT,
-        progmem_code TEXT,
-        assigned_melody_ids TEXT NOT NULL DEFAULT '[]',
-        created_at TEXT NOT NULL DEFAULT (datetime('now')),
-        updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-    )""",
-    # Manufacturing audit log
-    """CREATE TABLE IF NOT EXISTS mfg_audit_log (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        timestamp TEXT NOT NULL DEFAULT (datetime('now')),
-        admin_user TEXT NOT NULL,
-        action TEXT NOT NULL,
-        serial_number TEXT,
-        detail TEXT
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_mfg_audit_time ON mfg_audit_log(timestamp)",
-    "CREATE INDEX IF NOT EXISTS idx_mfg_audit_action ON mfg_audit_log(action)",
-    # Active device alerts (current state, not history)
-    """CREATE TABLE IF NOT EXISTS device_alerts (
-        device_serial TEXT NOT NULL,
-        subsystem     TEXT NOT NULL,
-        state         TEXT NOT NULL,
-        message       TEXT,
-        updated_at    TEXT NOT NULL DEFAULT (datetime('now')),
-        PRIMARY KEY (device_serial, subsystem)
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_device_alerts_serial ON device_alerts(device_serial)",
-    # CRM communications log
-    """CREATE TABLE IF NOT EXISTS crm_comms_log (
-        id             TEXT PRIMARY KEY,
-        customer_id    TEXT,
-        type           TEXT NOT NULL,
-        mail_account   TEXT,
-        direction      TEXT NOT NULL,
-        subject        TEXT,
-        body           TEXT,
-        body_html      TEXT,
-        attachments    TEXT NOT NULL DEFAULT '[]',
-        ext_message_id TEXT,
-        from_addr      TEXT,
-        to_addrs       TEXT,
-        logged_by      TEXT,
-        occurred_at    TEXT NOT NULL,
-        created_at     TEXT NOT NULL
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_crm_comms_customer ON crm_comms_log(customer_id, occurred_at)",
-    # CRM media references
-    """CREATE TABLE IF NOT EXISTS crm_media (
-        id             TEXT PRIMARY KEY,
-        customer_id    TEXT,
-        order_id       TEXT,
-        filename       TEXT NOT NULL,
-        nextcloud_path TEXT NOT NULL,
-        mime_type      TEXT,
-        direction      TEXT,
-        tags           TEXT NOT NULL DEFAULT '[]',
-        uploaded_by    TEXT,
-        created_at     TEXT NOT NULL
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_crm_media_customer ON crm_media(customer_id)",
-    "CREATE INDEX IF NOT EXISTS idx_crm_media_order ON crm_media(order_id)",
-    # CRM sync state (last email sync timestamp, etc.)
-    """CREATE TABLE IF NOT EXISTS crm_sync_state (
-        key   TEXT PRIMARY KEY,
-        value TEXT
-    )""",
-    # CRM Quotations
-    """CREATE TABLE IF NOT EXISTS crm_quotations (
-        id                        TEXT PRIMARY KEY,
-        quotation_number          TEXT UNIQUE NOT NULL,
-        title                     TEXT,
-        subtitle                  TEXT,
-        customer_id               TEXT NOT NULL,
-        language                  TEXT NOT NULL DEFAULT 'en',
-        status                    TEXT NOT NULL DEFAULT 'draft',
-        order_type                TEXT,
-        shipping_method           TEXT,
-        estimated_shipping_date   TEXT,
-        global_discount_label     TEXT,
-        global_discount_percent   REAL NOT NULL DEFAULT 0,
-        vat_percent               REAL NOT NULL DEFAULT 24,
-        shipping_cost             REAL NOT NULL DEFAULT 0,
-        shipping_cost_discount    REAL NOT NULL DEFAULT 0,
-        install_cost              REAL NOT NULL DEFAULT 0,
-        install_cost_discount     REAL NOT NULL DEFAULT 0,
-        extras_label              TEXT,
-        extras_cost               REAL NOT NULL DEFAULT 0,
-        comments                  TEXT NOT NULL DEFAULT '[]',
-        subtotal_before_discount  REAL NOT NULL DEFAULT 0,
-        global_discount_amount    REAL NOT NULL DEFAULT 0,
-        new_subtotal              REAL NOT NULL DEFAULT 0,
-        vat_amount                REAL NOT NULL DEFAULT 0,
-        final_total               REAL NOT NULL DEFAULT 0,
-        nextcloud_pdf_path        TEXT,
-        nextcloud_pdf_url         TEXT,
-        created_at                TEXT NOT NULL,
-        updated_at                TEXT NOT NULL
-    )""",
-    """CREATE TABLE IF NOT EXISTS crm_quotation_items (
-        id               TEXT PRIMARY KEY,
-        quotation_id     TEXT NOT NULL,
-        product_id       TEXT,
-        description      TEXT,
-        description_en   TEXT,
-        description_gr   TEXT,
-        unit_type        TEXT NOT NULL DEFAULT 'pcs',
-        unit_cost        REAL NOT NULL DEFAULT 0,
-        discount_percent REAL NOT NULL DEFAULT 0,
-        quantity         REAL NOT NULL DEFAULT 1,
-        line_total       REAL NOT NULL DEFAULT 0,
-        sort_order       INTEGER NOT NULL DEFAULT 0,
-        FOREIGN KEY (quotation_id) REFERENCES crm_quotations(id)
-    )""",
-    "CREATE INDEX IF NOT EXISTS idx_crm_quotations_customer ON crm_quotations(customer_id)",
-    "CREATE INDEX IF NOT EXISTS idx_crm_quotation_items_quotation ON crm_quotation_items(quotation_id, sort_order)",
-]
-
-
-async def init_db():
-    global _db
-    os.makedirs(os.path.dirname(os.path.abspath(settings.sqlite_db_path)), exist_ok=True)
-    _db = await aiosqlite.connect(settings.sqlite_db_path)
-    _db.row_factory = aiosqlite.Row
-    for stmt in SCHEMA_STATEMENTS:
-        await _db.execute(stmt)
-    await _db.commit()
-    # Migrations: add columns that may not exist in older DBs
-    _migrations = [
-        "ALTER TABLE crm_comms_log ADD COLUMN body_html TEXT",
-        "ALTER TABLE crm_comms_log ADD COLUMN mail_account TEXT",
-        "ALTER TABLE crm_comms_log ADD COLUMN from_addr TEXT",
-        "ALTER TABLE crm_comms_log ADD COLUMN to_addrs TEXT",
-        "ALTER TABLE crm_comms_log ADD COLUMN is_important INTEGER NOT NULL DEFAULT 0",
-        "ALTER TABLE crm_comms_log ADD COLUMN is_read INTEGER NOT NULL DEFAULT 0",
-        "ALTER TABLE crm_quotation_items ADD COLUMN vat_percent REAL NOT NULL DEFAULT 24",
-        "ALTER TABLE crm_quotations ADD COLUMN quick_notes TEXT NOT NULL DEFAULT '{}'",
-        "ALTER TABLE crm_quotations ADD COLUMN client_org TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN client_name TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN client_location TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN client_phone TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN client_email TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN is_legacy INTEGER NOT NULL DEFAULT 0",
-        "ALTER TABLE crm_quotations ADD COLUMN legacy_date TEXT",
-        "ALTER TABLE crm_quotations ADD COLUMN legacy_pdf_path TEXT",
-        "ALTER TABLE crm_media ADD COLUMN thumbnail_path TEXT",
-        "ALTER TABLE crm_quotation_items ADD COLUMN description_en TEXT",
-        "ALTER TABLE crm_quotation_items ADD COLUMN description_gr TEXT",
-        "ALTER TABLE built_melodies ADD COLUMN is_builtin INTEGER NOT NULL DEFAULT 0",
-        "ALTER TABLE crm_quotations ADD COLUMN global_vat_percent REAL NOT NULL DEFAULT 24",
-    ]
-    for m in _migrations:
-        try:
-            await _db.execute(m)
-            await _db.commit()
-        except Exception:
-            pass  # column already exists
-
-    # Migration: drop NOT NULL on crm_comms_log.customer_id if it exists.
-    # SQLite doesn't support ALTER COLUMN, so we check via table_info and
-    # rebuild the table if needed.
-    rows = await _db.execute_fetchall("PRAGMA table_info(crm_comms_log)")
-    for row in rows:
-        # row: (cid, name, type, notnull, dflt_value, pk)
-        if row[1] == "customer_id" and row[3] == 1:  # notnull=1
-            logger.info("Migrating crm_comms_log: removing NOT NULL from customer_id")
-            await _db.execute("ALTER TABLE crm_comms_log RENAME TO crm_comms_log_old")
-            await _db.execute("""CREATE TABLE crm_comms_log (
-                id             TEXT PRIMARY KEY,
-                customer_id    TEXT,
-                type           TEXT NOT NULL,
-                mail_account   TEXT,
-                direction      TEXT NOT NULL,
-                subject        TEXT,
-                body           TEXT,
-                body_html      TEXT,
-                attachments    TEXT NOT NULL DEFAULT '[]',
-                ext_message_id TEXT,
-                from_addr      TEXT,
-                to_addrs       TEXT,
-                logged_by      TEXT,
-                occurred_at    TEXT NOT NULL,
-                created_at     TEXT NOT NULL
-            )""")
-            await _db.execute("""INSERT INTO crm_comms_log
-                SELECT id, customer_id, type, NULL, direction, subject, body, body_html,
-                       attachments, ext_message_id, from_addr, to_addrs, logged_by,
-                       occurred_at, created_at
-                FROM crm_comms_log_old""")
-            await _db.execute("DROP TABLE crm_comms_log_old")
-            await _db.execute("CREATE INDEX IF NOT EXISTS idx_crm_comms_customer ON crm_comms_log(customer_id, occurred_at)")
-            await _db.commit()
-            logger.info("Migration complete: crm_comms_log.customer_id is now nullable")
-            break
-    logger.info(f"SQLite database initialized at {settings.sqlite_db_path}")
-
-
-async def close_db():
-    global _db
-    if _db:
-        await _db.close()
-        _db = None
-
-
-async def get_db() -> aiosqlite.Connection:
-    if _db is None:
-        await init_db()
-    return _db
-
-
-# --- Insert Operations ---
-
-async def insert_log(device_serial: str, level: str, message: str,
-                     device_timestamp: int | None = None):
-    db = await get_db()
-    cursor = await db.execute(
-        "INSERT INTO device_logs (device_serial, level, message, device_timestamp) VALUES (?, ?, ?, ?)",
-        (device_serial, level, message, device_timestamp)
-    )
-    await db.commit()
-    return cursor.lastrowid
-
-
-async def insert_heartbeat(device_serial: str, device_id: str,
-                           firmware_version: str, ip_address: str,
-                           gateway: str, uptime_ms: int, uptime_display: str):
-    db = await get_db()
-    cursor = await db.execute(
-        """INSERT INTO heartbeats
-           (device_serial, device_id, firmware_version, ip_address, gateway, uptime_ms, uptime_display)
-           VALUES (?, ?, ?, ?, ?, ?, ?)""",
-        (device_serial, device_id, firmware_version, ip_address, gateway, uptime_ms, uptime_display)
-    )
-    await db.commit()
-    return cursor.lastrowid
-
-
-async def insert_command(device_serial: str, command_name: str,
-                         command_payload: dict) -> int:
-    db = await get_db()
-    cursor = await db.execute(
-        "INSERT INTO commands (device_serial, command_name, command_payload) VALUES (?, ?, ?)",
-        (device_serial, command_name, json.dumps(command_payload))
-    )
-    await db.commit()
-    return cursor.lastrowid
-
-
-async def update_command_response(command_id: int, status: str,
-                                  response_payload: dict | None = None):
-    db = await get_db()
-    await db.execute(
-        """UPDATE commands SET status = ?, response_payload = ?,
-           responded_at = datetime('now') WHERE id = ?""",
-        (status, json.dumps(response_payload) if response_payload else None, command_id)
-    )
-    await db.commit()
-
-
-# --- Query Operations ---
-
-async def get_logs(device_serial: str, level: str | None = None,
-                   search: str | None = None,
-                   limit: int = 100, offset: int = 0) -> tuple[list, int]:
-    db = await get_db()
-    where_clauses = ["device_serial = ?"]
-    params: list = [device_serial]
-
-    if level:
-        where_clauses.append("level = ?")
-        params.append(level)
-    if search:
-        where_clauses.append("message LIKE ?")
-        params.append(f"%{search}%")
-
-    where = " AND ".join(where_clauses)
-
-    count_row = await db.execute_fetchall(
-        f"SELECT COUNT(*) as cnt FROM device_logs WHERE {where}", params
-    )
-    total = count_row[0][0]
-
-    rows = await db.execute_fetchall(
-        f"SELECT * FROM device_logs WHERE {where} ORDER BY received_at DESC LIMIT ? OFFSET ?",
-        params + [limit, offset]
-    )
-    return [dict(r) for r in rows], total
-
-
-async def get_heartbeats(device_serial: str, limit: int = 100,
-                         offset: int = 0) -> tuple[list, int]:
-    db = await get_db()
-    count_row = await db.execute_fetchall(
-        "SELECT COUNT(*) FROM heartbeats WHERE device_serial = ?", (device_serial,)
-    )
-    total = count_row[0][0]
-    rows = await db.execute_fetchall(
-        "SELECT * FROM heartbeats WHERE device_serial = ? ORDER BY received_at DESC LIMIT ? OFFSET ?",
-        (device_serial, limit, offset)
-    )
-    return [dict(r) for r in rows], total
-
-
-async def get_commands(device_serial: str, limit: int = 100,
-                       offset: int = 0) -> tuple[list, int]:
-    db = await get_db()
-    count_row = await db.execute_fetchall(
-        "SELECT COUNT(*) FROM commands WHERE device_serial = ?", (device_serial,)
-    )
-    total = count_row[0][0]
-    rows = await db.execute_fetchall(
-        "SELECT * FROM commands WHERE device_serial = ? ORDER BY sent_at DESC LIMIT ? OFFSET ?",
-        (device_serial, limit, offset)
-    )
-    return [dict(r) for r in rows], total
-
-
-async def get_latest_heartbeats() -> list:
-    db = await get_db()
-    rows = await db.execute_fetchall("""
-        SELECT h.* FROM heartbeats h
-        INNER JOIN (
-            SELECT device_serial, MAX(received_at) as max_time
-            FROM heartbeats GROUP BY device_serial
-        ) latest ON h.device_serial = latest.device_serial
-        AND h.received_at = latest.max_time
-    """)
-    return [dict(r) for r in rows]
-
-
-async def get_pending_command(device_serial: str) -> dict | None:
-    db = await get_db()
-    rows = await db.execute_fetchall(
-        """SELECT * FROM commands WHERE device_serial = ? AND status = 'pending'
-           ORDER BY sent_at DESC LIMIT 1""",
-        (device_serial,)
-    )
-    return dict(rows[0]) if rows else None
-
-
-# --- Cleanup ---
-
-async def purge_old_data(retention_days: int | None = None):
-    days = retention_days or settings.mqtt_data_retention_days
-    cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
-    db = await get_db()
-    await db.execute("DELETE FROM device_logs WHERE received_at < ?", (cutoff,))
-    await db.execute("DELETE FROM heartbeats WHERE received_at < ?", (cutoff,))
-    await db.execute("DELETE FROM commands WHERE sent_at < ?", (cutoff,))
-    await db.commit()
-    logger.info(f"Purged MQTT data older than {days} days")
-
-
-async def purge_loop():
-    while True:
-        await asyncio.sleep(86400)
-        try:
-            await purge_old_data()
-        except Exception as e:
-            logger.error(f"Purge failed: {e}")
-
-
-# --- Device Alerts ---
-
-async def upsert_alert(device_serial: str, subsystem: str, state: str,
-                       message: str | None = None):
-    db = await get_db()
-    await db.execute(
-        """INSERT INTO device_alerts (device_serial, subsystem, state, message, updated_at)
-           VALUES (?, ?, ?, ?, datetime('now'))
-           ON CONFLICT(device_serial, subsystem)
-           DO UPDATE SET state=excluded.state, message=excluded.message,
-                         updated_at=excluded.updated_at""",
-        (device_serial, subsystem, state, message),
-    )
-    await db.commit()
-
-
-async def delete_alert(device_serial: str, subsystem: str):
-    db = await get_db()
-    await db.execute(
-        "DELETE FROM device_alerts WHERE device_serial = ? AND subsystem = ?",
-        (device_serial, subsystem),
-    )
-    await db.commit()
-
-
-async def get_alerts(device_serial: str) -> list:
-    db = await get_db()
-    rows = await db.execute_fetchall(
-        "SELECT * FROM device_alerts WHERE device_serial = ? ORDER BY updated_at DESC",
-        (device_serial,),
-    )
-    return [dict(r) for r in rows]

backend/database/models.py

@@ -1,23 +0,0 @@
-from database.postgres import Base  # noqa: F401 — Base must be imported for Alembic autogenerate
-
-# Import all ORM models here so Alembic autogenerate detects them.
-# Add each new model file as it is created.
-
-# --- Existing ---
-from notes.orm import Entry, EntryLink                              # noqa: F401
-from tickets.orm import SupportTicket, TicketMessage               # noqa: F401
-
-# --- Phase 0 ---
-from shared.orm import MigrationRun, AuditLog                      # noqa: F401
-from crm.orm import (                                              # noqa: F401
-    CrmProduct, CrmCustomer, CrmOrder,
-    CrmCommsLog, CrmMedia, CrmSyncState,
-    CrmQuotation, CrmQuotationItem,
-)
-from staff.orm import Staff                                         # noqa: F401
-from settings.orm import ConsoleSetting, PublicFeature             # noqa: F401
-from melodies.orm import MelodyDraft, BuiltMelody                  # noqa: F401
-from manufacturing.orm import MfgAuditLog                          # noqa: F401
-from devices.orm import DeviceAlert                                 # noqa: F401
-# NOTE: device_logs, commands, heartbeats are partitioned/raw-SQL tables —
-# they are NOT ORM models and are created via op.execute() in the migration.

backend/database/pg_mqtt.py

@@ -1,411 +0,0 @@
-"""
-Phase 5 — MQTT live data functions backed by Postgres.
-
-device_logs is a partitioned table; heartbeats and commands are plain tables.
-All three are accessed via raw SQL (not ORM) because device_logs partitioning
-does not play well with SQLAlchemy's declarative ORM.
-
-device_alerts is an ORM model (devices/orm.py) and is handled here via raw SQL
-to keep a single consistent interface for callers that used to import from database.core.
-"""
-
-import asyncio
-import json
-import logging
-from datetime import date, datetime, timedelta, timezone
-
-from sqlalchemy import text
-from config import settings
-from database.postgres import AsyncSessionLocal
-
-logger = logging.getLogger("database.pg_mqtt")
-
-
-# ---------------------------------------------------------------------------
-# Insert operations
-# ---------------------------------------------------------------------------
-
-async def insert_log(device_serial: str, level: str, message: str,
-                     device_timestamp: int | None = None) -> int:
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(
-            text("""
-                INSERT INTO device_logs (device_serial, level, message, device_timestamp, received_at)
-                VALUES (:serial, :level, :message, :ts, now())
-                RETURNING id
-            """),
-            {"serial": device_serial, "level": level, "message": message, "ts": device_timestamp},
-        )
-        row = result.fetchone()
-        await session.commit()
-        return row[0]
-
-
-async def insert_heartbeat(device_serial: str, device_id: str,
-                           firmware_version: str, ip_address: str,
-                           gateway: str, uptime_ms: int, uptime_display: str) -> int:
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(
-            text("""
-                INSERT INTO heartbeats
-                    (device_serial, device_id, firmware_version, ip_address,
-                     gateway, uptime_ms, uptime_display, received_at)
-                VALUES
-                    (:serial, :device_id, :fw, :ip, :gw, :uptime_ms, :uptime_display, now())
-                RETURNING id
-            """),
-            {
-                "serial": device_serial,
-                "device_id": device_id,
-                "fw": firmware_version,
-                "ip": ip_address,
-                "gw": gateway,
-                "uptime_ms": uptime_ms,
-                "uptime_display": uptime_display,
-            },
-        )
-        row = result.fetchone()
-        await session.commit()
-        return row[0]
-
-
-async def insert_command(device_serial: str, command_name: str,
-                         command_payload: dict) -> int:
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(
-            text("""
-                INSERT INTO commands (device_serial, command_name, command_payload, sent_at)
-                VALUES (:serial, :name, :payload, now())
-                RETURNING id
-            """),
-            {
-                "serial": device_serial,
-                "name": command_name,
-                "payload": json.dumps(command_payload),
-            },
-        )
-        row = result.fetchone()
-        await session.commit()
-        return row[0]
-
-
-async def update_command_response(command_id: int, status: str,
-                                  response_payload: dict | None = None):
-    async with AsyncSessionLocal() as session:
-        await session.execute(
-            text("""
-                UPDATE commands
-                SET status = :status,
-                    response_payload = :payload,
-                    responded_at = now()
-                WHERE id = :id
-            """),
-            {
-                "id": command_id,
-                "status": status,
-                "payload": json.dumps(response_payload) if response_payload else None,
-            },
-        )
-        await session.commit()
-
-
-# ---------------------------------------------------------------------------
-# Query operations
-# ---------------------------------------------------------------------------
-
-async def get_logs(device_serial: str, level: str | None = None,
-                   search: str | None = None,
-                   limit: int = 100, offset: int = 0) -> tuple[list, int]:
-    where = "device_serial = :serial"
-    params: dict = {"serial": device_serial, "limit": limit, "offset": offset}
-
-    if level:
-        where += " AND level = :level"
-        params["level"] = level
-    if search:
-        where += " AND message ILIKE :search"
-        params["search"] = f"%{search}%"
-
-    async with AsyncSessionLocal() as session:
-        count_result = await session.execute(
-            text(f"SELECT COUNT(*) FROM device_logs WHERE {where}"), params
-        )
-        total = count_result.scalar()
-
-        rows_result = await session.execute(
-            text(f"""
-                SELECT id, device_serial, level, message, device_timestamp,
-                       received_at AT TIME ZONE 'UTC' AS received_at
-                FROM device_logs
-                WHERE {where}
-                ORDER BY received_at DESC
-                LIMIT :limit OFFSET :offset
-            """),
-            params,
-        )
-        rows = rows_result.mappings().all()
-
-    return [_row_to_dict(r) for r in rows], total
-
-
-async def get_heartbeats(device_serial: str, limit: int = 100,
-                         offset: int = 0) -> tuple[list, int]:
-    async with AsyncSessionLocal() as session:
-        count_result = await session.execute(
-            text("SELECT COUNT(*) FROM heartbeats WHERE device_serial = :serial"),
-            {"serial": device_serial},
-        )
-        total = count_result.scalar()
-
-        rows_result = await session.execute(
-            text("""
-                SELECT id, device_serial, device_id, firmware_version, ip_address,
-                       gateway, uptime_ms, uptime_display,
-                       received_at AT TIME ZONE 'UTC' AS received_at
-                FROM heartbeats
-                WHERE device_serial = :serial
-                ORDER BY received_at DESC
-                LIMIT :limit OFFSET :offset
-            """),
-            {"serial": device_serial, "limit": limit, "offset": offset},
-        )
-        rows = rows_result.mappings().all()
-
-    return [_row_to_dict(r) for r in rows], total
-
-
-async def get_commands(device_serial: str, limit: int = 100,
-                       offset: int = 0) -> tuple[list, int]:
-    async with AsyncSessionLocal() as session:
-        count_result = await session.execute(
-            text("SELECT COUNT(*) FROM commands WHERE device_serial = :serial"),
-            {"serial": device_serial},
-        )
-        total = count_result.scalar()
-
-        rows_result = await session.execute(
-            text("""
-                SELECT id, device_serial, command_name, command_payload, status,
-                       response_payload,
-                       sent_at AT TIME ZONE 'UTC' AS sent_at,
-                       responded_at AT TIME ZONE 'UTC' AS responded_at
-                FROM commands
-                WHERE device_serial = :serial
-                ORDER BY sent_at DESC
-                LIMIT :limit OFFSET :offset
-            """),
-            {"serial": device_serial, "limit": limit, "offset": offset},
-        )
-        rows = rows_result.mappings().all()
-
-    return [_row_to_dict(r) for r in rows], total
-
-
-async def get_latest_heartbeats() -> list:
-    async with AsyncSessionLocal() as session:
-        rows_result = await session.execute(
-            text("""
-                SELECT DISTINCT ON (device_serial)
-                    id, device_serial, device_id, firmware_version, ip_address,
-                    gateway, uptime_ms, uptime_display,
-                    received_at AT TIME ZONE 'UTC' AS received_at
-                FROM heartbeats
-                ORDER BY device_serial, received_at DESC
-            """)
-        )
-        rows = rows_result.mappings().all()
-
-    return [_row_to_dict(r) for r in rows]
-
-
-async def get_pending_command(device_serial: str) -> dict | None:
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(
-            text("""
-                SELECT id, device_serial, command_name, command_payload, status,
-                       response_payload,
-                       sent_at AT TIME ZONE 'UTC' AS sent_at,
-                       responded_at AT TIME ZONE 'UTC' AS responded_at
-                FROM commands
-                WHERE device_serial = :serial AND status = 'pending'
-                ORDER BY sent_at DESC
-                LIMIT 1
-            """),
-            {"serial": device_serial},
-        )
-        row = result.mappings().fetchone()
-
-    return _row_to_dict(row) if row else None
-
-
-# ---------------------------------------------------------------------------
-# Device alerts
-# ---------------------------------------------------------------------------
-
-async def upsert_alert(device_serial: str, subsystem: str, state: str,
-                       message: str | None = None):
-    async with AsyncSessionLocal() as session:
-        await session.execute(
-            text("""
-                INSERT INTO device_alerts (device_serial, subsystem, state, message, updated_at)
-                VALUES (:serial, :subsystem, :state, :message, now())
-                ON CONFLICT (device_serial, subsystem)
-                DO UPDATE SET
-                    state      = EXCLUDED.state,
-                    message    = EXCLUDED.message,
-                    updated_at = EXCLUDED.updated_at
-            """),
-            {"serial": device_serial, "subsystem": subsystem, "state": state, "message": message},
-        )
-        await session.commit()
-
-
-async def delete_alert(device_serial: str, subsystem: str):
-    async with AsyncSessionLocal() as session:
-        await session.execute(
-            text("DELETE FROM device_alerts WHERE device_serial = :serial AND subsystem = :subsystem"),
-            {"serial": device_serial, "subsystem": subsystem},
-        )
-        await session.commit()
-
-
-async def get_alerts(device_serial: str) -> list:
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(
-            text("""
-                SELECT id, device_serial, subsystem, state, message,
-                       updated_at AT TIME ZONE 'UTC' AS updated_at
-                FROM device_alerts
-                WHERE device_serial = :serial
-                ORDER BY updated_at DESC
-            """),
-            {"serial": device_serial},
-        )
-        rows = result.mappings().all()
-
-    return [_row_to_dict(r) for r in rows]
-
-
-# ---------------------------------------------------------------------------
-# Partition management
-# ---------------------------------------------------------------------------
-
-def _add_months(d: date, months: int) -> date:
-    month = d.month - 1 + months
-    year = d.year + month // 12
-    month = month % 12 + 1
-    return d.replace(year=year, month=month, day=1)
-
-
-async def ensure_current_partitions():
-    """Create device_logs partitions for the current and next month if missing."""
-    async with AsyncSessionLocal() as session:
-        for month_offset in (0, 1):
-            d = _add_months(date.today().replace(day=1), month_offset)
-            partition_name = f"device_logs_{d.strftime('%Y_%m')}"
-            start = d.isoformat()
-            end = _add_months(d, 1).isoformat()
-            await session.execute(text(f"""
-                CREATE TABLE IF NOT EXISTS {partition_name}
-                PARTITION OF device_logs
-                FOR VALUES FROM ('{start}') TO ('{end}')
-            """))
-        await session.commit()
-    logger.info("Partition check complete")
-
-
-async def drop_old_partitions(keep_months: int = 6):
-    """Drop device_logs partitions older than keep_months."""
-    cutoff = _add_months(date.today().replace(day=1), -keep_months)
-    async with AsyncSessionLocal() as session:
-        result = await session.execute(text("""
-            SELECT tablename FROM pg_tables
-            WHERE schemaname = 'public'
-              AND tablename LIKE 'device_logs_%'
-        """))
-        partitions = [r[0] for r in result.fetchall()]
-
-    for name in partitions:
-        # name format: device_logs_YYYY_MM
-        parts = name.split("_")
-        if len(parts) != 4:
-            continue
-        try:
-            partition_date = date(int(parts[2]), int(parts[3]), 1)
-        except ValueError:
-            continue
-        if partition_date < cutoff:
-            async with AsyncSessionLocal() as session:
-                await session.execute(text(f"DROP TABLE IF EXISTS {name}"))
-                await session.commit()
-            logger.info(f"Dropped old partition: {name}")
-
-
-async def partition_manager_loop():
-    """Runs once on startup, then monthly thereafter."""
-    await ensure_current_partitions()
-    while True:
-        # Sleep ~30 days, wake up and ensure next month's partition exists
-        await asyncio.sleep(30 * 24 * 3600)
-        try:
-            await ensure_current_partitions()
-            await drop_old_partitions()
-        except Exception as e:
-            logger.error(f"Partition manager error: {e}")
-
-
-# ---------------------------------------------------------------------------
-# Cleanup (replaces SQLite purge_loop — now a no-op since Postgres uses
-# partition drops instead of row-by-row deletes for device_logs; heartbeats
-# and commands are still purged by row deletion)
-# ---------------------------------------------------------------------------
-
-async def purge_old_data(retention_days: int | None = None):
-    days = retention_days or settings.mqtt_data_retention_days
-    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
-    async with AsyncSessionLocal() as session:
-        await session.execute(
-            text("DELETE FROM heartbeats WHERE received_at < :cutoff"),
-            {"cutoff": cutoff},
-        )
-        await session.execute(
-            text("DELETE FROM commands WHERE sent_at < :cutoff"),
-            {"cutoff": cutoff},
-        )
-        await session.commit()
-    logger.info(f"Purged heartbeats and commands older than {days} days")
-
-
-async def purge_loop():
-    while True:
-        await asyncio.sleep(86400)
-        try:
-            await purge_old_data()
-        except Exception as e:
-            logger.error(f"Purge failed: {e}")
-
-
-# ---------------------------------------------------------------------------
-# Stub — no longer needed but kept so nothing that imports init_db/close_db breaks
-# ---------------------------------------------------------------------------
-
-async def init_db():
-    """No-op: Postgres schema is managed by Alembic, not runtime init."""
-    logger.info("Postgres MQTT backend active — no SQLite init needed")
-
-
-async def close_db():
-    """No-op: SQLAlchemy engine lifecycle is managed by the process."""
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _row_to_dict(row) -> dict:
-    """Convert a SQLAlchemy RowMapping to a plain dict with ISO string timestamps."""
-    d = dict(row)
-    for key, val in d.items():
-        if isinstance(val, datetime):
-            d[key] = val.isoformat()
-    return d

backend/database/postgres.py

@@ -1,16 +0,0 @@
-from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
-from sqlalchemy.orm import DeclarativeBase
-from config import settings
-
-engine = create_async_engine(settings.database_url, pool_size=10, echo=False)
-AsyncSessionLocal = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
-
-
-class Base(DeclarativeBase):
-    pass
-
-
-async def get_pg_session() -> AsyncSession:
-    """FastAPI dependency — yields a DB session and closes it after the request."""
-    async with AsyncSessionLocal() as session:
-        yield session

backend/devices/models.py

@@ -31,11 +31,11 @@ class DeviceTiers(str, Enum):
 class DeviceNetworkSettings(BaseModel):
     hostname: str = ""
     useStaticIP: bool = False
-    ipAddress: Any = []
+    ipAddress: List[str] = []
-    gateway: Any = []
+    gateway: List[str] = []
-    subnet: Any = []
+    subnet: List[str] = []
-    dns1: Any = []
+    dns1: List[str] = []
-    dns2: Any = []
+    dns2: List[str] = []
 
 
 class DeviceClockSettings(BaseModel):
@@ -119,19 +119,13 @@ class DeviceCreate(BaseModel):
     device_subscription: DeviceSubInformation = DeviceSubInformation()
     device_stats: DeviceStatistics = DeviceStatistics()
     events_on: bool = False
-    device_location_coordinates: Any = None  # GeoPoint dict {lat, lng} or legacy str
+    device_location_coordinates: str = ""
     device_melodies_all: List[MelodyMainItem] = []
     device_melodies_favorites: List[str] = []
     user_list: List[str] = []
     websocket_url: str = ""
     churchAssistantURL: str = ""
     staffNotes: str = ""
-    hw_family: str = ""
-    hw_revision: str = ""
-    tags: List[str] = []
-    serial_number: str = ""
-    customer_id: str = ""
-    mfg_status: str = ""
 
 
 class DeviceUpdate(BaseModel):
@@ -144,23 +138,17 @@ class DeviceUpdate(BaseModel):
     device_subscription: Optional[Dict[str, Any]] = None
     device_stats: Optional[Dict[str, Any]] = None
     events_on: Optional[bool] = None
-    device_location_coordinates: Optional[Any] = None  # dict {lat, lng} or legacy str
+    device_location_coordinates: Optional[str] = None
     device_melodies_all: Optional[List[MelodyMainItem]] = None
     device_melodies_favorites: Optional[List[str]] = None
     user_list: Optional[List[str]] = None
     websocket_url: Optional[str] = None
     churchAssistantURL: Optional[str] = None
     staffNotes: Optional[str] = None
-    hw_family: Optional[str] = None
-    hw_revision: Optional[str] = None
-    tags: Optional[List[str]] = None
-    customer_id: Optional[str] = None
-    mfg_status: Optional[str] = None
 
 
 class DeviceInDB(DeviceCreate):
     id: str
-    # Legacy field — kept for backwards compat; new docs use serial_number
     device_id: str = ""
 
 
@@ -169,15 +157,6 @@ class DeviceListResponse(BaseModel):
     total: int
 
 
-class DeviceNoteCreate(BaseModel):
-    content: str
-    created_by: str = ""
-
-
-class DeviceNoteUpdate(BaseModel):
-    content: str
-
-
 class DeviceUserInfo(BaseModel):
     """User info resolved from device_users sub-collection or user_list."""
     user_id: str = ""

backend/devices/orm.py

@@ -1,31 +0,0 @@
-from datetime import datetime, timezone
-from sqlalchemy import BigInteger, Column, DateTime, Index, String, Text, UniqueConstraint
-from sqlalchemy.dialects.postgresql import JSONB
-from database.postgres import Base
-
-
-def _now():
-    return datetime.now(timezone.utc)
-
-
-class DeviceAlert(Base):
-    """Current alert state per device+subsystem (upserted, not appended)."""
-    __tablename__ = "device_alerts"
-    __table_args__ = (
-        UniqueConstraint("device_serial", "subsystem"),
-        Index("idx_device_alerts_serial", "device_serial"),
-    )
-
-    id            = Column(BigInteger, primary_key=True, autoincrement=True)
-    device_serial = Column(String(128), nullable=False)
-    subsystem     = Column(String(128), nullable=False)
-    state         = Column(String(64), nullable=False)
-    message       = Column(Text)
-    updated_at    = Column(DateTime(timezone=True), nullable=False, default=_now, onupdate=_now)
-
-
-# NOTE: device_logs, commands, and heartbeats are NOT declared as ORM models here.
-# device_logs is a partitioned table — SQLAlchemy ORM does not support declarative
-# partitioned tables cleanly. All three tables are created via raw SQL in the
-# Alembic migration and accessed via raw queries in database/core.py (SQLite now)
-# and will be accessed via raw async SQL after Phase 5 cutover.

backend/devices/router.py

@@ -1,28 +1,15 @@
-import uuid
+from fastapi import APIRouter, Depends, Query
-from datetime import datetime
+from typing import Optional
-from fastapi import APIRouter, Depends, Query, HTTPException
-from typing import Optional, List
-from pydantic import BaseModel
-from sqlalchemy.ext.asyncio import AsyncSession
 from auth.models import TokenPayload
 from auth.dependencies import require_permission
 from devices.models import (
     DeviceCreate, DeviceUpdate, DeviceInDB, DeviceListResponse,
     DeviceUsersResponse, DeviceUserInfo,
-    DeviceNoteCreate, DeviceNoteUpdate,
 )
 from devices import service
-import database as mqtt_db
-from mqtt.models import DeviceAlertEntry, DeviceAlertsResponse
-from shared.firebase import get_db as get_firestore
-from database.postgres import get_pg_session
-from shared.audit import log_action
 
 router = APIRouter(prefix="/api/devices", tags=["devices"])
 
-NOTES_COLLECTION = "notes"
-CRM_COLLECTION = "crm_customers"
-
 
 @router.get("", response_model=DeviceListResponse)
 async def list_devices(
@@ -61,12 +48,8 @@ async def get_device_users(
 async def create_device(
     body: DeviceCreate,
     _user: TokenPayload = Depends(require_permission("devices", "add")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    device = service.create_device(body)
+    return service.create_device(body)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "device",
-                     device.device_id, device.device_name or device.device_id)
-    return device
 
 
 @router.put("/{device_id}", response_model=DeviceInDB)
@@ -74,434 +57,13 @@ async def update_device(
     device_id: str,
     body: DeviceUpdate,
     _user: TokenPayload = Depends(require_permission("devices", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    old = service.get_device(device_id)
+    return service.update_device(device_id, body)
-    device = service.update_device(device_id, body)
-    _SKIP = {"updated_at", "device_id", "tags", "user_list"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(device, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(device, k, None)
-    }
-    if "tags" in body.model_fields_set and (old.tags or []) != (device.tags or []):
-        changes["tags"] = {"old": sorted(old.tags or []), "new": sorted(device.tags or [])}
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "device",
-                     device_id, device.device_name or device_id, changes=changes or None)
-    return device
 
 
 @router.delete("/{device_id}", status_code=204)
 async def delete_device(
     device_id: str,
     _user: TokenPayload = Depends(require_permission("devices", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
     service.delete_device(device_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "device",
-                     device_id, device_id)
-
-
-@router.get("/{device_id}/alerts", response_model=DeviceAlertsResponse)
-async def get_device_alerts(
-    device_id: str,
-    _user: TokenPayload = Depends(require_permission("devices", "view")),
-):
-    """Return the current active alert set for a device. Empty list means fully healthy."""
-    rows = await mqtt_db.get_alerts(device_id)
-    return DeviceAlertsResponse(alerts=[DeviceAlertEntry(**r) for r in rows])
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Device Notes
-# ─────────────────────────────────────────────────────────────────────────────
-
-@router.get("/{device_id}/notes")
-async def list_device_notes(
-    device_id: str,
-    _user: TokenPayload = Depends(require_permission("devices", "view")),
-):
-    """List all notes for a device."""
-    db = get_firestore()
-    docs = db.collection(NOTES_COLLECTION).where("device_id", "==", device_id).stream()
-    notes = []
-    for doc in docs:
-        note = doc.to_dict()
-        note["id"] = doc.id
-        for f in ("created_at", "updated_at"):
-            if hasattr(note.get(f), "isoformat"):
-                note[f] = note[f].isoformat()
-        notes.append(note)
-    notes.sort(key=lambda n: n.get("created_at") or "", reverse=False)
-    return {"notes": notes, "total": len(notes)}
-
-
-@router.post("/{device_id}/notes", status_code=201)
-async def create_device_note(
-    device_id: str,
-    body: DeviceNoteCreate,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-):
-    """Create a new note for a device."""
-    db = get_firestore()
-    now = datetime.utcnow()
-    note_id = str(uuid.uuid4())
-    note_data = {
-        "device_id": device_id,
-        "content": body.content,
-        "created_by": body.created_by or _user.name or "",
-        "created_at": now,
-        "updated_at": now,
-    }
-    db.collection(NOTES_COLLECTION).document(note_id).set(note_data)
-    note_data["id"] = note_id
-    note_data["created_at"] = now.isoformat()
-    note_data["updated_at"] = now.isoformat()
-    return note_data
-
-
-@router.put("/{device_id}/notes/{note_id}")
-async def update_device_note(
-    device_id: str,
-    note_id: str,
-    body: DeviceNoteUpdate,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-):
-    """Update an existing device note."""
-    db = get_firestore()
-    doc_ref = db.collection(NOTES_COLLECTION).document(note_id)
-    doc = doc_ref.get()
-    if not doc.exists or doc.to_dict().get("device_id") != device_id:
-        raise HTTPException(status_code=404, detail="Note not found")
-    now = datetime.utcnow()
-    doc_ref.update({"content": body.content, "updated_at": now})
-    updated = doc.to_dict()
-    updated["id"] = note_id
-    updated["content"] = body.content
-    updated["updated_at"] = now.isoformat()
-    if hasattr(updated.get("created_at"), "isoformat"):
-        updated["created_at"] = updated["created_at"].isoformat()
-    return updated
-
-
-@router.delete("/{device_id}/notes/{note_id}", status_code=204)
-async def delete_device_note(
-    device_id: str,
-    note_id: str,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-):
-    """Delete a device note."""
-    db = get_firestore()
-    doc_ref = db.collection(NOTES_COLLECTION).document(note_id)
-    doc = doc_ref.get()
-    if not doc.exists or doc.to_dict().get("device_id") != device_id:
-        raise HTTPException(status_code=404, detail="Note not found")
-    doc_ref.delete()
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Device Tags
-# ─────────────────────────────────────────────────────────────────────────────
-
-class TagsUpdate(BaseModel):
-    tags: List[str]
-
-
-@router.put("/{device_id}/tags", response_model=DeviceInDB)
-async def update_device_tags(
-    device_id: str,
-    body: TagsUpdate,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-):
-    """Replace the tags list for a device."""
-    return service.update_device(device_id, DeviceUpdate(tags=body.tags))
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Assign Device to Customer
-# ─────────────────────────────────────────────────────────────────────────────
-
-class CustomerSearchResult(BaseModel):
-    id: str
-    name: str
-    email: str
-    organization: str = ""
-
-
-class AssignCustomerBody(BaseModel):
-    customer_id: str
-    label: str = ""
-
-
-@router.get("/{device_id}/customer-search")
-async def search_customers_for_device(
-    device_id: str,
-    q: str = Query(""),
-    _user: TokenPayload = Depends(require_permission("devices", "view")),
-):
-    """Search customers by name, email, phone, org, or tags, returning top 20 matches."""
-    db = get_firestore()
-    docs = db.collection(CRM_COLLECTION).stream()
-    results = []
-    q_lower = q.lower().strip()
-    for doc in docs:
-        data = doc.to_dict()
-        name = data.get("name", "") or ""
-        surname = data.get("surname", "") or ""
-        email = data.get("email", "") or ""
-        organization = data.get("organization", "") or ""
-        phone = data.get("phone", "") or ""
-        tags = " ".join(data.get("tags", []) or [])
-        location = data.get("location") or {}
-        city = location.get("city", "") or ""
-        searchable = f"{name} {surname} {email} {organization} {phone} {tags} {city}".lower()
-        if not q_lower or q_lower in searchable:
-            results.append({
-                "id": doc.id,
-                "name": name,
-                "surname": surname,
-                "email": email,
-                "organization": organization,
-                "city": city,
-            })
-        if len(results) >= 20:
-            break
-    return {"results": results}
-
-
-@router.post("/{device_id}/assign-customer")
-async def assign_device_to_customer(
-    device_id: str,
-    body: AssignCustomerBody,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Assign a device to a customer.
-
-    - Sets owner field on the device document.
-    - Adds a console_device entry to the customer's owned_items list.
-    """
-    db = get_firestore()
-
-    # Verify device exists
-    device = service.get_device(device_id)
-
-    # Get customer
-    customer_ref = db.collection(CRM_COLLECTION).document(body.customer_id)
-    customer_doc = customer_ref.get()
-    if not customer_doc.exists:
-        raise HTTPException(status_code=404, detail="Customer not found")
-    customer_data = customer_doc.to_dict()
-    customer_email = customer_data.get("email", "")
-
-    # Update device: owner email + customer_id
-    device_ref = db.collection("devices").document(device_id)
-    device_ref.update({"owner": customer_email, "customer_id": body.customer_id})
-
-    # Add to customer owned_items (avoid duplicates)
-    owned_items = customer_data.get("owned_items", []) or []
-    already_assigned = any(
-        item.get("type") == "console_device" and item.get("console_device", {}).get("device_id") == device_id
-        for item in owned_items
-    )
-    if not already_assigned:
-        owned_items.append({
-            "type": "console_device",
-            "console_device": {
-                "device_id": device_id,
-                "label": body.label or device.device_name or device_id,
-            }
-        })
-        customer_ref.update({"owned_items": owned_items})
-
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "device",
-                     device_id, device_id, meta={"action_detail": "assigned_to_customer",
-                                                  "customer_id": body.customer_id})
-    return {"status": "assigned", "device_id": device_id, "customer_id": body.customer_id}
-
-
-@router.delete("/{device_id}/assign-customer", status_code=204)
-async def unassign_device_from_customer(
-    device_id: str,
-    customer_id: str = Query(...),
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Remove device assignment from a customer."""
-    db = get_firestore()
-
-    # Clear customer_id on device
-    device_ref = db.collection("devices").document(device_id)
-    device_ref.update({"customer_id": ""})
-
-    # Remove from customer owned_items
-    customer_ref = db.collection(CRM_COLLECTION).document(customer_id)
-    customer_doc = customer_ref.get()
-    if customer_doc.exists:
-        customer_data = customer_doc.to_dict()
-        owned_items = [
-            item for item in (customer_data.get("owned_items") or [])
-            if not (item.get("type") == "console_device" and item.get("console_device", {}).get("device_id") == device_id)
-        ]
-        customer_ref.update({"owned_items": owned_items})
-
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "device",
-                     device_id, device_id, meta={"action_detail": "unassigned_from_customer",
-                                                  "customer_id": customer_id})
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Customer detail (for Owner display in fleet)
-# ─────────────────────────────────────────────────────────────────────────────
-
-@router.get("/{device_id}/customer")
-async def get_device_customer(
-    device_id: str,
-    _user: TokenPayload = Depends(require_permission("devices", "view")),
-):
-    """Return basic customer details for a device's assigned customer_id."""
-    db = get_firestore()
-    device_ref = db.collection("devices").document(device_id)
-    device_doc = device_ref.get()
-    if not device_doc.exists:
-        raise HTTPException(status_code=404, detail="Device not found")
-    device_data = device_doc.to_dict() or {}
-    customer_id = device_data.get("customer_id")
-    if not customer_id:
-        return {"customer": None}
-    customer_doc = db.collection(CRM_COLLECTION).document(customer_id).get()
-    if not customer_doc.exists:
-        return {"customer": None}
-    cd = customer_doc.to_dict() or {}
-    return {
-        "customer": {
-            "id": customer_doc.id,
-            "name": cd.get("name") or "",
-            "email": cd.get("email") or "",
-            "organization": cd.get("organization") or "",
-            "phone": cd.get("phone") or "",
-        }
-    }
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# User list management (for Manage tab — assign/remove users from user_list)
-# ─────────────────────────────────────────────────────────────────────────────
-
-class UserSearchResult(BaseModel):
-    id: str
-    display_name: str = ""
-    email: str = ""
-    phone: str = ""
-    photo_url: str = ""
-
-
-@router.get("/{device_id}/user-search")
-async def search_users_for_device(
-    device_id: str,
-    q: str = Query(""),
-    _user: TokenPayload = Depends(require_permission("devices", "view")),
-):
-    """Search the users collection by name, email, or phone."""
-    db = get_firestore()
-    docs = db.collection("users").stream()
-    results = []
-    q_lower = q.lower().strip()
-    for doc in docs:
-        data = doc.to_dict() or {}
-        name = (data.get("display_name") or "").lower()
-        email = (data.get("email") or "").lower()
-        phone = (data.get("phone") or "").lower()
-        if not q_lower or q_lower in name or q_lower in email or q_lower in phone:
-            results.append({
-                "id": doc.id,
-                "display_name": data.get("display_name") or "",
-                "email": data.get("email") or "",
-                "phone": data.get("phone") or "",
-                "photo_url": data.get("photo_url") or "",
-            })
-        if len(results) >= 20:
-            break
-    return {"results": results}
-
-
-class AddUserBody(BaseModel):
-    user_id: str
-
-
-@router.post("/{device_id}/user-list", status_code=200)
-async def add_user_to_device(
-    device_id: str,
-    body: AddUserBody,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Add a user reference to the device's user_list field."""
-    db = get_firestore()
-    device_ref = db.collection("devices").document(device_id)
-    device_doc = device_ref.get()
-    if not device_doc.exists:
-        raise HTTPException(status_code=404, detail="Device not found")
-
-    # Verify user exists
-    user_doc = db.collection("users").document(body.user_id).get()
-    if not user_doc.exists:
-        raise HTTPException(status_code=404, detail="User not found")
-
-    data = device_doc.to_dict() or {}
-    user_list = data.get("user_list", []) or []
-
-    # Avoid duplicates — check both string paths and DocumentReferences
-    from google.cloud.firestore_v1 import DocumentReference as DocRef
-    existing_ids = set()
-    for entry in user_list:
-        if isinstance(entry, DocRef):
-            existing_ids.add(entry.id)
-        elif isinstance(entry, str):
-            existing_ids.add(entry.split("/")[-1])
-
-    if body.user_id not in existing_ids:
-        user_ref = db.collection("users").document(body.user_id)
-        user_list.append(user_ref)
-        device_ref.update({"user_list": user_list})
-
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "device",
-                     device_id, device_id, meta={"action_detail": "user_added",
-                                                  "user_id": body.user_id})
-    return {"status": "added", "user_id": body.user_id}
-
-
-@router.delete("/{device_id}/user-list/{user_id}", status_code=200)
-async def remove_user_from_device(
-    device_id: str,
-    user_id: str,
-    _user: TokenPayload = Depends(require_permission("devices", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Remove a user reference from the device's user_list field."""
-    db = get_firestore()
-    device_ref = db.collection("devices").document(device_id)
-    device_doc = device_ref.get()
-    if not device_doc.exists:
-        raise HTTPException(status_code=404, detail="Device not found")
-
-    data = device_doc.to_dict() or {}
-    user_list = data.get("user_list", []) or []
-
-    from google.cloud.firestore_v1 import DocumentReference as DocRef
-
-    def resolves_to(entry, uid: str) -> bool:
-        if isinstance(entry, DocRef):
-            return entry.id == uid
-        if isinstance(entry, str):
-            return entry.split("/")[-1] == uid
-        return False
-
-    # Remove any entry that resolves to this user_id (handles both DocRef and string paths)
-    new_list = [entry for entry in user_list if not resolves_to(entry, user_id)]
-    device_ref.update({"user_list": new_list})
-
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "device",
-                     device_id, device_id, meta={"action_detail": "user_removed",
-                                                  "user_id": user_id})
-    return {"status": "removed", "user_id": user_id}

backend/devices/service.py

@@ -52,11 +52,10 @@ def _generate_serial_number() -> str:
 def _ensure_unique_serial(db) -> str:
     """Generate a serial number and verify it doesn't already exist in Firestore."""
     existing_sns = set()
-    for doc in db.collection(COLLECTION).select(["serial_number"]).stream():
+    for doc in db.collection(COLLECTION).select(["device_id"]).stream():
         data = doc.to_dict()
-        sn = data.get("serial_number") or data.get("device_id")
+        if data.get("device_id"):
-        if sn:
+            existing_sns.add(data["device_id"])
-            existing_sns.add(sn)
 
     for _ in range(100):  # safety limit
         sn = _generate_serial_number()
@@ -72,7 +71,7 @@ def _convert_firestore_value(val):
         # Firestore DatetimeWithNanoseconds is a datetime subclass
         return val.strftime("%d %B %Y at %H:%M:%S UTC%z")
     if isinstance(val, GeoPoint):
-        return {"lat": val.latitude, "lng": val.longitude}
+        return f"{val.latitude}° N, {val.longitude}° E"
     if isinstance(val, DocumentReference):
         # Store the document path (e.g. "users/abc123")
         return val.path
@@ -96,40 +95,18 @@ def _sanitize_dict(d: dict) -> dict:
     return result
 
 
-def _auto_upgrade_claimed(doc_ref, data: dict) -> dict:
-    """If the device has entries in user_list and isn't already claimed/decommissioned,
-    upgrade mfg_status to 'claimed' automatically and return the updated data dict."""
-    current_status = data.get("mfg_status", "")
-    if current_status in ("claimed", "decommissioned"):
-        return data
-    user_list = data.get("user_list", []) or []
-    if user_list:
-        doc_ref.update({"mfg_status": "claimed"})
-        data = dict(data)
-        data["mfg_status"] = "claimed"
-    return data
-
-
 def _doc_to_device(doc) -> DeviceInDB:
-    """Convert a Firestore document snapshot to a DeviceInDB model.
+    """Convert a Firestore document snapshot to a DeviceInDB model."""
-
+    data = _sanitize_dict(doc.to_dict())
-    Also auto-upgrades mfg_status to 'claimed' if user_list is non-empty.
-    """
-    raw = doc.to_dict()
-    raw = _auto_upgrade_claimed(doc.reference, raw)
-    data = _sanitize_dict(raw)
     return DeviceInDB(id=doc.id, **data)
 
 
-FLEET_STATUSES = {"sold", "claimed"}
-
-
 def list_devices(
     search: str | None = None,
     online_only: bool | None = None,
     subscription_tier: str | None = None,
 ) -> list[DeviceInDB]:
-    """List fleet devices (sold + claimed only) with optional filters."""
+    """List devices with optional filters."""
     db = get_db()
     ref = db.collection(COLLECTION)
     query = ref
@@ -141,14 +118,6 @@ def list_devices(
     results = []
 
     for doc in docs:
-        raw = doc.to_dict() or {}
-
-        # Only include sold/claimed devices in the fleet view.
-        # Legacy devices without mfg_status are included to avoid breaking old data.
-        mfg_status = raw.get("mfg_status")
-        if mfg_status and mfg_status not in FLEET_STATUSES:
-            continue
-
         device = _doc_to_device(doc)
 
         # Client-side filters
@@ -159,7 +128,7 @@ def list_devices(
             search_lower = search.lower()
             name_match = search_lower in (device.device_name or "").lower()
             location_match = search_lower in (device.device_location or "").lower()
-            sn_match = search_lower in (device.serial_number or "").lower()
+            sn_match = search_lower in (device.device_id or "").lower()
             if not (name_match or location_match or sn_match):
                 continue
 
@@ -213,11 +182,6 @@ def update_device(device_doc_id: str, data: DeviceUpdate) -> DeviceInDB:
 
     update_data = data.model_dump(exclude_none=True)
 
-    # Convert {lat, lng} dict to a Firestore GeoPoint
-    coords = update_data.get("device_location_coordinates")
-    if isinstance(coords, dict) and "lat" in coords and "lng" in coords:
-        update_data["device_location_coordinates"] = GeoPoint(coords["lat"], coords["lng"])
-
     # Deep-merge nested structs so unmentioned sub-fields are preserved
     existing = doc.to_dict()
     nested_keys = (

backend/equipment/service.py

@@ -4,7 +4,7 @@ from shared.firebase import get_db
 from shared.exceptions import NotFoundError
 from equipment.models import NoteCreate, NoteUpdate, NoteInDB
 
-COLLECTION = "notes"
+COLLECTION = "equipment_notes"
 
 VALID_CATEGORIES = {"general", "maintenance", "installation", "issue", "action_item", "other"}
 

backend/firmware/models.py

@@ -1,29 +1,18 @@
 from pydantic import BaseModel
 from typing import Optional, List
-from enum import Enum
-
-
-class UpdateType(str, Enum):
-    optional  = "optional"   # user-initiated only
-    mandatory = "mandatory"  # auto-installs on next reboot
-    emergency = "emergency"  # auto-installs on reboot + daily check + MQTT push
 
 
 class FirmwareVersion(BaseModel):
     id: str
-    hw_type: str              # e.g. "vesper", "vesper_plus", "vesper_pro", "bespoke"
+    hw_type: str        # "vs", "vp", "vx"
-    channel: str              # "stable", "beta", "alpha", "testing"
+    channel: str        # "stable", "beta", "alpha", "testing"
-    version: str              # semver e.g. "1.5"
+    version: str        # semver e.g. "1.4.2"
     filename: str
     size_bytes: int
     sha256: str
-    update_type: UpdateType = UpdateType.mandatory
-    min_fw_version: Optional[str] = None  # minimum fw version required to install this
     uploaded_at: str
-    changelog: Optional[str] = None
+    notes: Optional[str] = None
-    release_note: Optional[str] = None
     is_latest: bool = False
-    bespoke_uid: Optional[str] = None    # only set when hw_type == "bespoke"
 
 
 class FirmwareListResponse(BaseModel):
@@ -31,36 +20,12 @@ class FirmwareListResponse(BaseModel):
     total: int
 
 
-class FirmwareMetadataResponse(BaseModel):
+class FirmwareLatestResponse(BaseModel):
-    """Returned by both /latest and /{version}/info endpoints.
-
-    Two orthogonal axes:
-      channel     — the release track the device is subscribed to
-                    ("stable" | "beta" | "development")
-                    Firmware validates this matches the channel it requested.
-      update_type — the urgency of THIS release, set by the publisher
-                    ("optional" | "mandatory" | "emergency")
-                    Firmware reads mandatory/emergency booleans derived from this.
-
-    Additional firmware-compatible fields:
-      size        — binary size in bytes (firmware reads "size", not "size_bytes")
-      mandatory   — True when update_type is mandatory or emergency
-      emergency   — True only when update_type is emergency
-    """
     hw_type: str
-    channel: str                       # release track — firmware validates this
+    channel: str
     version: str
-    size: int                          # firmware reads "size"
+    size_bytes: int
-    size_bytes: int                    # kept for admin-panel consumers
     sha256: str
-    update_type: UpdateType            # urgency enum — for admin panel display
-    mandatory: bool                    # derived: update_type in (mandatory, emergency)
-    emergency: bool                    # derived: update_type == emergency
-    min_fw_version: Optional[str] = None
     download_url: str
     uploaded_at: str
-    release_note: Optional[str] = None
+    notes: Optional[str] = None
-
-
-# Keep backwards-compatible alias
-FirmwareLatestResponse = FirmwareMetadataResponse

backend/firmware/router.py

@@ -1,21 +1,13 @@
-from fastapi import APIRouter, Depends, Query, UploadFile, File, Form, HTTPException
+from fastapi import APIRouter, Depends, Query, UploadFile, File, Form
-from fastapi.responses import FileResponse, PlainTextResponse
+from fastapi.responses import FileResponse
-from pydantic import BaseModel
 from typing import Optional
-import logging
-from sqlalchemy.ext.asyncio import AsyncSession
 
 from auth.models import TokenPayload
 from auth.dependencies import require_permission
-from firmware.models import FirmwareVersion, FirmwareListResponse, FirmwareMetadataResponse, UpdateType
+from firmware.models import FirmwareVersion, FirmwareListResponse, FirmwareLatestResponse
 from firmware import service
-from database.postgres import get_pg_session
-from shared.audit import log_action
-
-logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/firmware", tags=["firmware"])
-ota_router = APIRouter(prefix="/api/ota", tags=["ota-telemetry"])
 
 
 @router.post("/upload", response_model=FirmwareVersion, status_code=201)
@@ -23,30 +15,18 @@ async def upload_firmware(
     hw_type: str = Form(...),
     channel: str = Form(...),
     version: str = Form(...),
-    update_type: UpdateType = Form(UpdateType.mandatory),
+    notes: Optional[str] = Form(None),
-    min_fw_version: Optional[str] = Form(None),
-    changelog: Optional[str] = Form(None),
-    release_note: Optional[str] = Form(None),
-    bespoke_uid: Optional[str] = Form(None),
     file: UploadFile = File(...),
     _user: TokenPayload = Depends(require_permission("manufacturing", "add")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
     file_bytes = await file.read()
-    fw = service.upload_firmware(
+    return service.upload_firmware(
         hw_type=hw_type,
         channel=channel,
         version=version,
         file_bytes=file_bytes,
-        update_type=update_type,
+        notes=notes,
-        min_fw_version=min_fw_version,
-        changelog=changelog,
-        release_note=release_note,
-        bespoke_uid=bespoke_uid,
     )
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "firmware",
-                     fw.id, f"{hw_type} v{version} ({channel})")
-    return fw
 
 
 @router.get("", response_model=FirmwareListResponse)
@@ -59,37 +39,12 @@ def list_firmware(
     return FirmwareListResponse(firmware=items, total=len(items))
 
 
-@router.get("/{hw_type}/{channel}/latest", response_model=FirmwareMetadataResponse)
+@router.get("/{hw_type}/{channel}/latest", response_model=FirmwareLatestResponse)
-def get_latest_firmware(
+def get_latest_firmware(hw_type: str, channel: str):
-    hw_type: str,
-    channel: str,
-    hw_version: Optional[str] = Query(None, description="Hardware revision from NVS, e.g. '1.0'"),
-    current_version: Optional[str] = Query(None, description="Currently running firmware semver, e.g. '1.2.3'"),
-):
     """Returns metadata for the latest firmware for a given hw_type + channel.
     No auth required — devices call this endpoint to check for updates.
     """
-    return service.get_latest(hw_type, channel, hw_version=hw_version, current_version=current_version)
+    return service.get_latest(hw_type, channel)
-
-
-@router.get("/{hw_type}/{channel}/latest/changelog", response_class=PlainTextResponse)
-def get_latest_changelog(hw_type: str, channel: str):
-    """Returns the full changelog for the latest firmware. Plain text."""
-    return service.get_latest_changelog(hw_type, channel)
-
-
-@router.get("/{hw_type}/{channel}/{version}/info/changelog", response_class=PlainTextResponse)
-def get_version_changelog(hw_type: str, channel: str, version: str):
-    """Returns the full changelog for a specific firmware version. Plain text."""
-    return service.get_version_changelog(hw_type, channel, version)
-
-
-@router.get("/{hw_type}/{channel}/{version}/info", response_model=FirmwareMetadataResponse)
-def get_firmware_info(hw_type: str, channel: str, version: str):
-    """Returns metadata for a specific firmware version.
-    No auth required — devices call this to resolve upgrade chains.
-    """
-    return service.get_version_info(hw_type, channel, version)
 
 
 @router.get("/{hw_type}/{channel}/{version}/firmware.bin")
@@ -103,94 +58,9 @@ def download_firmware(hw_type: str, channel: str, version: str):
     )
 
 
-@router.put("/{firmware_id}", response_model=FirmwareVersion)
-async def edit_firmware(
-    firmware_id: str,
-    channel: Optional[str] = Form(None),
-    version: Optional[str] = Form(None),
-    update_type: Optional[UpdateType] = Form(None),
-    min_fw_version: Optional[str] = Form(None),
-    changelog: Optional[str] = Form(None),
-    release_note: Optional[str] = Form(None),
-    bespoke_uid: Optional[str] = Form(None),
-    file: Optional[UploadFile] = File(None),
-    _user: TokenPayload = Depends(require_permission("manufacturing", "add")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    file_bytes = await file.read() if file and file.filename else None
-    fw = service.edit_firmware(
-        doc_id=firmware_id,
-        channel=channel,
-        version=version,
-        update_type=update_type,
-        min_fw_version=min_fw_version,
-        changelog=changelog,
-        release_note=release_note,
-        bespoke_uid=bespoke_uid,
-        file_bytes=file_bytes,
-    )
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "firmware",
-                     firmware_id, f"{fw.hw_type} v{fw.version} ({fw.channel})" if fw else firmware_id)
-    return fw
-
-
 @router.delete("/{firmware_id}", status_code=204)
-async def delete_firmware(
+def delete_firmware(
     firmware_id: str,
     _user: TokenPayload = Depends(require_permission("manufacturing", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    fw = service.get_firmware(firmware_id) if hasattr(service, "get_firmware") else None
     service.delete_firmware(firmware_id)
-    label = f"{fw.hw_type} v{fw.version} ({fw.channel})" if fw else firmware_id
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "firmware",
-                     firmware_id, label)
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# OTA event telemetry — called by devices (no auth, best-effort)
-# ─────────────────────────────────────────────────────────────────────────────
-
-class OtaDownloadEvent(BaseModel):
-    device_uid: str
-    hw_type: str
-    hw_version: str
-    from_version: str
-    to_version: str
-    channel: str
-
-
-class OtaFlashEvent(BaseModel):
-    device_uid: str
-    hw_type: str
-    hw_version: str
-    from_version: str
-    to_version: str
-    channel: str
-    sha256: str
-
-
-@ota_router.post("/events/download", status_code=204)
-def ota_event_download(event: OtaDownloadEvent):
-    """Device reports that firmware was fully written to flash (pre-commit).
-    No auth required — best-effort telemetry from the device.
-    """
-    logger.info(
-        "OTA download event: device=%s hw=%s/%s %s → %s (channel=%s)",
-        event.device_uid, event.hw_type, event.hw_version,
-        event.from_version, event.to_version, event.channel,
-    )
-    service.record_ota_event("download", event.model_dump())
-
-
-@ota_router.post("/events/flash", status_code=204)
-def ota_event_flash(event: OtaFlashEvent):
-    """Device reports that firmware partition was committed and device is rebooting.
-    No auth required — best-effort telemetry from the device.
-    """
-    logger.info(
-        "OTA flash event: device=%s hw=%s/%s %s → %s (channel=%s sha256=%.16s...)",
-        event.device_uid, event.hw_type, event.hw_version,
-        event.from_version, event.to_version, event.channel, event.sha256,
-    )
-    service.record_ota_event("flash", event.model_dump())

backend/firmware/service.py

@@ -1,22 +1,18 @@
 import hashlib
-import logging
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
 
 from fastapi import HTTPException
 
 from config import settings
 from shared.firebase import get_db
 from shared.exceptions import NotFoundError
-from firmware.models import FirmwareVersion, FirmwareMetadataResponse, UpdateType
+from firmware.models import FirmwareVersion, FirmwareLatestResponse
-
-logger = logging.getLogger(__name__)
 
 COLLECTION = "firmware_versions"
 
-VALID_HW_TYPES = {"vesper", "vesper_plus", "vesper_pro", "chronos", "chronos_pro", "agnus", "agnus_mini", "bespoke"}
+VALID_HW_TYPES = {"vs", "vp", "vx"}
 VALID_CHANNELS = {"stable", "beta", "alpha", "testing"}
 
 
@@ -40,34 +36,9 @@ def _doc_to_firmware_version(doc) -> FirmwareVersion:
         filename=data.get("filename", "firmware.bin"),
         size_bytes=data.get("size_bytes", 0),
         sha256=data.get("sha256", ""),
-        update_type=data.get("update_type", UpdateType.mandatory),
-        min_fw_version=data.get("min_fw_version"),
         uploaded_at=uploaded_str,
-        changelog=data.get("changelog"),
+        notes=data.get("notes"),
-        release_note=data.get("release_note"),
         is_latest=data.get("is_latest", False),
-        bespoke_uid=data.get("bespoke_uid"),
-    )
-
-
-def _fw_to_metadata_response(fw: FirmwareVersion) -> FirmwareMetadataResponse:
-    download_url = f"/api/firmware/{fw.hw_type}/{fw.channel}/{fw.version}/firmware.bin"
-    is_emergency = fw.update_type == UpdateType.emergency
-    is_mandatory = fw.update_type in (UpdateType.mandatory, UpdateType.emergency)
-    return FirmwareMetadataResponse(
-        hw_type=fw.hw_type,
-        channel=fw.channel,            # firmware validates this matches requested channel
-        version=fw.version,
-        size=fw.size_bytes,            # firmware reads "size"
-        size_bytes=fw.size_bytes,      # kept for admin-panel consumers
-        sha256=fw.sha256,
-        update_type=fw.update_type,    # urgency enum — for admin panel display
-        mandatory=is_mandatory,        # firmware reads this to decide auto-apply
-        emergency=is_emergency,        # firmware reads this to decide immediate apply
-        min_fw_version=fw.min_fw_version,
-        download_url=download_url,
-        uploaded_at=fw.uploaded_at,
-        release_note=fw.release_note,
     )
 
 
@@ -76,61 +47,33 @@ def upload_firmware(
     channel: str,
     version: str,
     file_bytes: bytes,
-    update_type: UpdateType = UpdateType.mandatory,
+    notes: str | None = None,
-    min_fw_version: str | None = None,
-    changelog: str | None = None,
-    release_note: str | None = None,
-    bespoke_uid: str | None = None,
 ) -> FirmwareVersion:
     if hw_type not in VALID_HW_TYPES:
-        raise HTTPException(status_code=400, detail=f"Invalid hw_type. Must be one of: {', '.join(sorted(VALID_HW_TYPES))}")
+        raise HTTPException(status_code=400, detail=f"Invalid hw_type. Must be one of: {', '.join(VALID_HW_TYPES)}")
     if channel not in VALID_CHANNELS:
-        raise HTTPException(status_code=400, detail=f"Invalid channel. Must be one of: {', '.join(sorted(VALID_CHANNELS))}")
+        raise HTTPException(status_code=400, detail=f"Invalid channel. Must be one of: {', '.join(VALID_CHANNELS)}")
-    if hw_type == "bespoke" and not bespoke_uid:
-        raise HTTPException(status_code=400, detail="bespoke_uid is required when hw_type is 'bespoke'")
-
-    db = get_db()
-    sha256 = hashlib.sha256(file_bytes).hexdigest()
-    now = datetime.now(timezone.utc)
-
-    # For bespoke firmware: if a firmware with the same bespoke_uid already exists,
-    # overwrite it (delete old doc + file, reuse same storage path keyed by uid).
-    if hw_type == "bespoke" and bespoke_uid:
-        existing_docs = list(
-            db.collection(COLLECTION)
-            .where("hw_type", "==", "bespoke")
-            .where("bespoke_uid", "==", bespoke_uid)
-            .stream()
-        )
-        for old_doc in existing_docs:
-            old_data = old_doc.to_dict() or {}
-            old_path = _storage_path("bespoke", old_data.get("channel", channel), old_data.get("version", version))
-            if old_path.exists():
-                old_path.unlink()
-                try:
-                    old_path.parent.rmdir()
-                except OSError:
-                    pass
-            old_doc.reference.delete()
 
     dest = _storage_path(hw_type, channel, version)
     dest.parent.mkdir(parents=True, exist_ok=True)
     dest.write_bytes(file_bytes)
 
+    sha256 = hashlib.sha256(file_bytes).hexdigest()
+    now = datetime.now(timezone.utc)
     doc_id = str(uuid.uuid4())
 
+    db = get_db()
+
     # Mark previous latest for this hw_type+channel as no longer latest
-    # (skip for bespoke — each bespoke_uid is its own independent firmware)
+    prev_docs = (
-    if hw_type != "bespoke":
+        db.collection(COLLECTION)
-        prev_docs = (
+        .where("hw_type", "==", hw_type)
-            db.collection(COLLECTION)
+        .where("channel", "==", channel)
-            .where("hw_type", "==", hw_type)
+        .where("is_latest", "==", True)
-            .where("channel", "==", channel)
+        .stream()
-            .where("is_latest", "==", True)
+    )
-            .stream()
+    for prev in prev_docs:
-        )
+        prev.reference.update({"is_latest": False})
-        for prev in prev_docs:
-            prev.reference.update({"is_latest": False})
 
     doc_ref = db.collection(COLLECTION).document(doc_id)
     doc_ref.set({
@@ -140,13 +83,9 @@ def upload_firmware(
         "filename": "firmware.bin",
         "size_bytes": len(file_bytes),
         "sha256": sha256,
-        "update_type": update_type.value,
-        "min_fw_version": min_fw_version,
         "uploaded_at": now,
-        "changelog": changelog,
+        "notes": notes,
-        "release_note": release_note,
         "is_latest": True,
-        "bespoke_uid": bespoke_uid,
     })
 
     return _doc_to_firmware_version(doc_ref.get())
@@ -169,52 +108,7 @@ def list_firmware(
     return items
 
 
-def get_latest(hw_type: str, channel: str, hw_version: str | None = None, current_version: str | None = None) -> FirmwareMetadataResponse:
+def get_latest(hw_type: str, channel: str) -> FirmwareLatestResponse:
-    if hw_type not in VALID_HW_TYPES:
-        raise HTTPException(status_code=400, detail=f"Invalid hw_type '{hw_type}'")
-    if hw_type == "bespoke":
-        raise HTTPException(status_code=400, detail="Bespoke firmware is not served via auto-update. Use the direct download URL.")
-    if channel not in VALID_CHANNELS:
-        raise HTTPException(status_code=400, detail=f"Invalid channel '{channel}'")
-
-    db = get_db()
-    docs = list(
-        db.collection(COLLECTION)
-        .where("hw_type", "==", hw_type)
-        .where("channel", "==", channel)
-        .where("is_latest", "==", True)
-        .limit(1)
-        .stream()
-    )
-    if not docs:
-        raise NotFoundError("Firmware")
-
-    return _fw_to_metadata_response(_doc_to_firmware_version(docs[0]))
-
-
-def get_version_info(hw_type: str, channel: str, version: str) -> FirmwareMetadataResponse:
-    """Fetch metadata for a specific version. Used by devices resolving upgrade chains."""
-    if hw_type not in VALID_HW_TYPES:
-        raise HTTPException(status_code=400, detail=f"Invalid hw_type '{hw_type}'")
-    if channel not in VALID_CHANNELS:
-        raise HTTPException(status_code=400, detail=f"Invalid channel '{channel}'")
-
-    db = get_db()
-    docs = list(
-        db.collection(COLLECTION)
-        .where("hw_type", "==", hw_type)
-        .where("channel", "==", channel)
-        .where("version", "==", version)
-        .limit(1)
-        .stream()
-    )
-    if not docs:
-        raise NotFoundError("Firmware version")
-
-    return _fw_to_metadata_response(_doc_to_firmware_version(docs[0]))
-
-
-def get_latest_changelog(hw_type: str, channel: str) -> str:
     if hw_type not in VALID_HW_TYPES:
         raise HTTPException(status_code=400, detail=f"Invalid hw_type '{hw_type}'")
     if channel not in VALID_CHANNELS:
@@ -231,33 +125,19 @@ def get_latest_changelog(hw_type: str, channel: str) -> str:
     )
     if not docs:
         raise NotFoundError("Firmware")
+
     fw = _doc_to_firmware_version(docs[0])
-    if not fw.changelog:
+    download_url = f"/api/firmware/{hw_type}/{channel}/{fw.version}/firmware.bin"
-        raise NotFoundError("Changelog")
+    return FirmwareLatestResponse(
-    return fw.changelog
+        hw_type=fw.hw_type,
-
+        channel=fw.channel,
-
+        version=fw.version,
-def get_version_changelog(hw_type: str, channel: str, version: str) -> str:
+        size_bytes=fw.size_bytes,
-    if hw_type not in VALID_HW_TYPES:
+        sha256=fw.sha256,
-        raise HTTPException(status_code=400, detail=f"Invalid hw_type '{hw_type}'")
+        download_url=download_url,
-    if channel not in VALID_CHANNELS:
+        uploaded_at=fw.uploaded_at,
-        raise HTTPException(status_code=400, detail=f"Invalid channel '{channel}'")
+        notes=fw.notes,
-
-    db = get_db()
-    docs = list(
-        db.collection(COLLECTION)
-        .where("hw_type", "==", hw_type)
-        .where("channel", "==", channel)
-        .where("version", "==", version)
-        .limit(1)
-        .stream()
     )
-    if not docs:
-        raise NotFoundError("Firmware version")
-    fw = _doc_to_firmware_version(docs[0])
-    if not fw.changelog:
-        raise NotFoundError("Changelog")
-    return fw.changelog
 
 
 def get_firmware_path(hw_type: str, channel: str, version: str) -> Path:
@@ -267,98 +147,6 @@ def get_firmware_path(hw_type: str, channel: str, version: str) -> Path:
     return path
 
 
-def record_ota_event(event_type: str, payload: dict[str, Any]) -> None:
-    """Persist an OTA telemetry event (download or flash) to Firestore.
-
-    Best-effort — caller should not raise on failure.
-    """
-    try:
-        db = get_db()
-        db.collection("ota_events").add({
-            "event_type": event_type,
-            "received_at": datetime.now(timezone.utc),
-            **payload,
-        })
-    except Exception as exc:
-        logger.warning("Failed to persist OTA event (%s): %s", event_type, exc)
-
-
-def edit_firmware(
-    doc_id: str,
-    channel: str | None = None,
-    version: str | None = None,
-    update_type: UpdateType | None = None,
-    min_fw_version: str | None = None,
-    changelog: str | None = None,
-    release_note: str | None = None,
-    bespoke_uid: str | None = None,
-    file_bytes: bytes | None = None,
-) -> FirmwareVersion:
-    db = get_db()
-    doc_ref = db.collection(COLLECTION).document(doc_id)
-    doc = doc_ref.get()
-    if not doc.exists:
-        raise NotFoundError("Firmware")
-
-    data = doc.to_dict() or {}
-    hw_type = data["hw_type"]
-    old_channel = data.get("channel", "")
-    old_version = data.get("version", "")
-
-    effective_channel = channel if channel is not None else old_channel
-    effective_version = version if version is not None else old_version
-
-    if channel is not None and channel not in VALID_CHANNELS:
-        raise HTTPException(status_code=400, detail=f"Invalid channel. Must be one of: {', '.join(sorted(VALID_CHANNELS))}")
-
-    updates: dict = {}
-    if channel is not None:
-        updates["channel"] = channel
-    if version is not None:
-        updates["version"] = version
-    if update_type is not None:
-        updates["update_type"] = update_type.value
-    if min_fw_version is not None:
-        updates["min_fw_version"] = min_fw_version if min_fw_version else None
-    if changelog is not None:
-        updates["changelog"] = changelog if changelog else None
-    if release_note is not None:
-        updates["release_note"] = release_note if release_note else None
-    if bespoke_uid is not None:
-        updates["bespoke_uid"] = bespoke_uid if bespoke_uid else None
-
-    if file_bytes is not None:
-        # Move binary if path changed
-        old_path = _storage_path(hw_type, old_channel, old_version)
-        new_path = _storage_path(hw_type, effective_channel, effective_version)
-        if old_path != new_path and old_path.exists():
-            old_path.unlink()
-            try:
-                old_path.parent.rmdir()
-            except OSError:
-                pass
-        new_path.parent.mkdir(parents=True, exist_ok=True)
-        new_path.write_bytes(file_bytes)
-        updates["sha256"] = hashlib.sha256(file_bytes).hexdigest()
-        updates["size_bytes"] = len(file_bytes)
-    elif (channel is not None and channel != old_channel) or (version is not None and version != old_version):
-        # Path changed but no new file — move existing binary
-        old_path = _storage_path(hw_type, old_channel, old_version)
-        new_path = _storage_path(hw_type, effective_channel, effective_version)
-        if old_path.exists() and old_path != new_path:
-            new_path.parent.mkdir(parents=True, exist_ok=True)
-            old_path.rename(new_path)
-            try:
-                old_path.parent.rmdir()
-            except OSError:
-                pass
-
-    if updates:
-        doc_ref.update(updates)
-
-    return _doc_to_firmware_version(doc_ref.get())
-
-
 def delete_firmware(doc_id: str) -> None:
     db = get_db()
     doc_ref = db.collection(COLLECTION).document(doc_id)
@@ -390,9 +178,9 @@ def delete_firmware(doc_id: str) -> None:
             db.collection(COLLECTION)
             .where("hw_type", "==", hw_type)
             .where("channel", "==", channel)
+            .order_by("uploaded_at", direction="DESCENDING")
+            .limit(1)
             .stream()
         )
         if remaining:
-            # Sort in Python to avoid needing a composite Firestore index
-            remaining.sort(key=lambda d: d.to_dict().get("uploaded_at", ""), reverse=True)
             remaining[0].reference.update({"is_latest": True})

backend/main.py

@@ -15,24 +15,10 @@ from staff.router import router as staff_router
 from helpdesk.router import router as helpdesk_router
 from builder.router import router as builder_router
 from manufacturing.router import router as manufacturing_router
-from firmware.router import router as firmware_router, ota_router
+from firmware.router import router as firmware_router
 from admin.router import router as admin_router
-from crm.router import router as crm_products_router
-from crm.customers_router import router as crm_customers_router
-from crm.orders_router import router as crm_orders_router, global_router as crm_orders_global_router
-from crm.comms_router import router as crm_comms_router
-from crm.media_router import router as crm_media_router
-from crm.nextcloud_router import router as crm_nextcloud_router
-from crm.quotations_router import router as crm_quotations_router
-from public.router import router as public_router
-from notes.router import router as notes_router
-from tickets.router import router as tickets_router
-from audit.router import router as audit_router
-from search.router import router as search_router
-from crm.nextcloud import close_client as close_nextcloud_client, keepalive_ping as nextcloud_keepalive
-from crm.mail_accounts import get_mail_accounts
 from mqtt.client import mqtt_manager
-import database as db
+from mqtt import database as mqtt_db
 from melodies import service as melody_service
 
 app = FastAPI(
@@ -63,75 +49,22 @@ app.include_router(staff_router)
 app.include_router(builder_router)
 app.include_router(manufacturing_router)
 app.include_router(firmware_router)
-app.include_router(ota_router)
 app.include_router(admin_router)
-app.include_router(crm_products_router)
-app.include_router(crm_customers_router)
-app.include_router(crm_orders_router)
-app.include_router(crm_orders_global_router)
-app.include_router(crm_comms_router)
-app.include_router(crm_media_router)
-app.include_router(crm_nextcloud_router)
-app.include_router(crm_quotations_router)
-app.include_router(public_router)
-app.include_router(notes_router)
-app.include_router(tickets_router)
-app.include_router(audit_router)
-app.include_router(search_router)
-
-
-async def nextcloud_keepalive_loop():
-    await nextcloud_keepalive()  # eager warmup on startup
-    while True:
-        await asyncio.sleep(45)
-        await nextcloud_keepalive()
-
-
-async def email_sync_loop():
-    while True:
-        await asyncio.sleep(settings.email_sync_interval_minutes * 60)
-        try:
-            from crm.email_sync import sync_emails
-            await sync_emails()
-        except Exception as e:
-            print(f"[EMAIL SYNC] Error: {e}")
-
-
-async def crm_poll_loop():
-    while True:
-        await asyncio.sleep(24 * 60 * 60)  # once per day
-        try:
-            from crm.service import poll_crm_customer_statuses
-            poll_crm_customer_statuses()
-        except Exception as e:
-            print(f"[CRM POLL] Error: {e}")
 
 
 @app.on_event("startup")
 async def startup():
     init_firebase()
-    from database.core import init_db as sqlite_init_db
+    await mqtt_db.init_db()
-    await sqlite_init_db()
     await melody_service.migrate_from_firestore()
     mqtt_manager.start(asyncio.get_event_loop())
-    asyncio.create_task(db.partition_manager_loop())
+    asyncio.create_task(mqtt_db.purge_loop())
-    asyncio.create_task(db.purge_loop())
-    asyncio.create_task(nextcloud_keepalive_loop())
-    asyncio.create_task(crm_poll_loop())
-    sync_accounts = [a for a in get_mail_accounts() if a.get("sync_inbound") and a.get("imap_host")]
-    if sync_accounts:
-        print(f"[EMAIL SYNC] IMAP configured for {len(sync_accounts)} account(s) - starting sync loop")
-        asyncio.create_task(email_sync_loop())
-    else:
-        print("[EMAIL SYNC] IMAP not configured - sync loop disabled")
 
 
 @app.on_event("shutdown")
 async def shutdown():
     mqtt_manager.stop()
-    from database.core import close_db as sqlite_close_db
+    await mqtt_db.close_db()
-    await sqlite_close_db()
-    await close_nextcloud_client()
 
 
 @app.get("/api/health")
@@ -141,4 +74,3 @@ async def health_check():
         "firebase": firebase_initialized,
         "mqtt": mqtt_manager.connected,
     }
-

backend/manufacturing/audit.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from database import get_db
+from mqtt.database import get_db
 
 logger = logging.getLogger("manufacturing.audit")
 

backend/manufacturing/models.py

@@ -4,45 +4,15 @@ from enum import Enum
 
 
 class BoardType(str, Enum):
-    vesper      = "vesper"
+    vs = "vs"   # Vesper
-    vesper_plus = "vesper_plus"
+    vp = "vp"   # Vesper+
-    vesper_pro  = "vesper_pro"
+    vx = "vx"   # VesperPro
-    chronos     = "chronos"
-    chronos_pro = "chronos_pro"
-    agnus_mini  = "agnus_mini"
-    agnus       = "agnus"
 
 
 BOARD_TYPE_LABELS = {
-    "vesper":      "Vesper",
+    "vs": "Vesper",
-    "vesper_plus": "Vesper Plus",
+    "vp": "Vesper+",
-    "vesper_pro":  "Vesper Pro",
+    "vx": "VesperPro",
-    "chronos":     "Chronos",
-    "chronos_pro": "Chronos Pro",
-    "agnus_mini":  "Agnus Mini",
-    "agnus":       "Agnus",
-}
-
-# Family codes (BS + 4 chars = segment 1 of serial number)
-BOARD_FAMILY_CODES = {
-    "vesper":      "VSPR",
-    "vesper_plus": "VSPR",
-    "vesper_pro":  "VSPR",
-    "agnus":       "AGNS",
-    "agnus_mini":  "AGNS",
-    "chronos":     "CRNS",
-    "chronos_pro": "CRNS",
-}
-
-# Variant codes (3 chars = first part of segment 3 of serial number)
-BOARD_VARIANT_CODES = {
-    "vesper":      "STD",
-    "vesper_plus": "PLS",
-    "vesper_pro":  "PRO",
-    "agnus":       "STD",
-    "agnus_mini":  "MIN",
-    "chronos":     "STD",
-    "chronos_pro": "PRO",
 }
 
 
@@ -55,20 +25,9 @@ class MfgStatus(str, Enum):
     decommissioned = "decommissioned"
 
 
-class LifecycleEntry(BaseModel):
-    status_id: str
-    date: str  # ISO 8601 UTC string
-    note: Optional[str] = None
-    set_by: Optional[str] = None
-
-
 class BatchCreate(BaseModel):
     board_type: BoardType
-    board_version: str = Field(
+    board_version: str = Field(..., pattern=r"^\d{2}$", description="2-digit zero-padded version, e.g. '01'")
-        ...,
-        pattern=r"^\d+(\.\d+)*$",
-        description="SemVer-style version string, e.g. '1.0' or legacy '01'",
-    )
     quantity: int = Field(..., ge=1, le=100)
 
 
@@ -90,10 +49,6 @@ class DeviceInventoryItem(BaseModel):
     created_at: Optional[str] = None
     owner: Optional[str] = None
     assigned_to: Optional[str] = None
-    device_name: Optional[str] = None
-    lifecycle_history: Optional[List["LifecycleEntry"]] = None
-    customer_id: Optional[str] = None
-    user_list: Optional[List[str]] = None
 
 
 class DeviceInventoryListResponse(BaseModel):
@@ -104,19 +59,11 @@ class DeviceInventoryListResponse(BaseModel):
 class DeviceStatusUpdate(BaseModel):
     status: MfgStatus
     note: Optional[str] = None
-    force_claimed: bool = False
 
 
 class DeviceAssign(BaseModel):
-    customer_id: str
+    customer_email: str
-
+    customer_name: Optional[str] = None
-
-class CustomerSearchResult(BaseModel):
-    id: str
-    name: str = ""
-    email: str = ""
-    organization: str = ""
-    phone: str = ""
 
 
 class RecentActivityItem(BaseModel):

backend/manufacturing/orm.py

@@ -1,22 +0,0 @@
-from datetime import datetime, timezone
-from sqlalchemy import BigInteger, Column, DateTime, Index, String, Text
-from database.postgres import Base
-
-
-def _now():
-    return datetime.now(timezone.utc)
-
-
-class MfgAuditLog(Base):
-    __tablename__ = "mfg_audit_log"
-    __table_args__ = (
-        Index("idx_mfg_audit_time", "timestamp"),
-        Index("idx_mfg_audit_action", "action"),
-    )
-
-    id            = Column(BigInteger, primary_key=True, autoincrement=True)
-    timestamp     = Column(DateTime(timezone=True), nullable=False, default=_now)
-    admin_user    = Column(String(256), nullable=False)
-    action        = Column(String(128), nullable=False)
-    serial_number = Column(String(128))
-    detail        = Column(Text)

backend/manufacturing/router.py

@@ -1,9 +1,7 @@
-from fastapi import APIRouter, Depends, Query, HTTPException, UploadFile, File
+from fastapi import APIRouter, Depends, Query
 from fastapi.responses import Response
 from fastapi.responses import RedirectResponse
 from typing import Optional
-from pydantic import BaseModel
-from sqlalchemy.ext.asyncio import AsyncSession
 
 from auth.models import TokenPayload
 from auth.dependencies import require_permission
@@ -14,26 +12,7 @@ from manufacturing.models import (
     ManufacturingStats,
 )
 from manufacturing import service
-from shared.audit import log_action
+from manufacturing import audit
-from shared.exceptions import NotFoundError
-from shared.firebase import get_db as get_firestore
-from database.postgres import get_pg_session
-
-
-class LifecycleEntryPatch(BaseModel):
-    index: int
-    date: Optional[str] = None
-    note: Optional[str] = None
-
-class LifecycleEntryCreate(BaseModel):
-    status_id: str
-    date: Optional[str] = None
-    note: Optional[str] = None
-
-VALID_FLASH_ASSETS = {"bootloader.bin", "partitions.bin"}
-VALID_HW_TYPES_MFG = {"vesper", "vesper_plus", "vesper_pro", "agnus", "agnus_mini", "chronos", "chronos_pro"}
-# Bespoke UIDs are dynamic — we allow any non-empty slug that doesn't clash with
-# a standard hw_type name. The flash-asset upload endpoint checks this below.
 
 router = APIRouter(prefix="/api/manufacturing", tags=["manufacturing"])
 
@@ -45,21 +24,26 @@ def get_stats(
     return service.get_stats()
 
 
+@router.get("/audit-log")
+async def get_audit_log(
+    limit: int = Query(20, ge=1, le=100),
+    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
+):
+    entries = await audit.get_recent(limit=limit)
+    return {"entries": entries}
+
 
 @router.post("/batch", response_model=BatchResponse, status_code=201)
 async def create_batch(
     body: BatchCreate,
     user: TokenPayload = Depends(require_permission("manufacturing", "add")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
     result = service.create_batch(body)
-    await log_action(
+    await audit.log_action(
-        db, user.sub, user.email,
+        admin_user=user.email,
-        action="CREATE",
+        action="batch_created",
-        entity_type="device_batch",
+        detail={
-        entity_id=result.batch_id,
+            "batch_id": result.batch_id,
-        entity_label=f"Batch {result.batch_id} ({result.board_type}, qty {len(result.serial_numbers)})",
-        meta={
             "board_type": result.board_type,
             "board_version": result.board_version,
             "quantity": len(result.serial_numbers),
@@ -95,207 +79,18 @@ def get_device(
     return service.get_device_by_sn(sn)
 
 
-@router.get("/customers/search")
-def search_customers(
-    q: str = Query(""),
-    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-):
-    """Search CRM customers by name, email, phone, organization, or tags."""
-    results = service.search_customers(q)
-    return {"results": results}
-
-
-@router.get("/customers/{customer_id}")
-def get_customer(
-    customer_id: str,
-    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-):
-    """Get a single CRM customer by ID."""
-    db = get_firestore()
-    doc = db.collection("crm_customers").document(customer_id).get()
-    if not doc.exists:
-        raise HTTPException(status_code=404, detail="Customer not found")
-    data = doc.to_dict() or {}
-    loc = data.get("location") or {}
-    city = loc.get("city") if isinstance(loc, dict) else None
-    return {
-        "id": doc.id,
-        "name": data.get("name") or "",
-        "surname": data.get("surname") or "",
-        "email": data.get("email") or "",
-        "organization": data.get("organization") or "",
-        "phone": data.get("phone") or "",
-        "city": city or "",
-    }
-
-
 @router.patch("/devices/{sn}/status", response_model=DeviceInventoryItem)
 async def update_status(
     sn: str,
     body: DeviceStatusUpdate,
     user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    # Guard: claimed requires at least one user in user_list
+    result = service.update_device_status(sn, body)
-    # (allow if explicitly force_claimed=true, which the mfg UI sets after adding a user manually)
+    await audit.log_action(
-    if body.status.value == "claimed":
+        admin_user=user.email,
-        db = get_firestore()
+        action="status_updated",
-        docs = list(db.collection("devices").where("serial_number", "==", sn).limit(1).stream())
+        serial_number=sn,
-        if docs:
+        detail={"status": body.status.value, "note": body.note},
-            data = docs[0].to_dict() or {}
-            user_list = data.get("user_list", []) or []
-            if not user_list and not getattr(body, "force_claimed", False):
-                raise HTTPException(
-                    status_code=400,
-                    detail="Cannot set status to 'claimed': device has no users in user_list. "
-                           "Assign a user first, then set to Claimed.",
-                )
-
-    # Guard: sold requires a customer assigned
-    if body.status.value == "sold":
-        db = get_firestore()
-        docs = list(db.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-        if docs:
-            data = docs[0].to_dict() or {}
-            if not data.get("customer_id"):
-                raise HTTPException(
-                    status_code=400,
-                    detail="Cannot set status to 'sold' without an assigned customer. "
-                           "Use the 'Assign to Customer' action first.",
-                )
-
-    result = service.update_device_status(sn, body, set_by=user.email)
-    await log_action(
-        db, user.sub, user.email,
-        action="STATUS_CHANGE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"status": body.status.value, "note": body.note},
-    )
-    return result
-
-
-@router.patch("/devices/{sn}/lifecycle", response_model=DeviceInventoryItem)
-async def patch_lifecycle_entry(
-    sn: str,
-    body: LifecycleEntryPatch,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Edit the date and/or note of a lifecycle history entry by index."""
-    fs = get_firestore()
-    docs = list(fs.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise HTTPException(status_code=404, detail="Device not found")
-    doc_ref = docs[0].reference
-    data = docs[0].to_dict() or {}
-    history = data.get("lifecycle_history") or []
-    if body.index < 0 or body.index >= len(history):
-        raise HTTPException(status_code=400, detail="Invalid lifecycle entry index")
-    if body.date is not None:
-        history[body.index]["date"] = body.date
-    if body.note is not None:
-        history[body.index]["note"] = body.note
-    doc_ref.update({"lifecycle_history": history})
-    from manufacturing.service import _doc_to_inventory_item
-    result = _doc_to_inventory_item(doc_ref.get())
-    await log_action(
-        db, user.sub, user.email,
-        action="UPDATE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"lifecycle_index": body.index, "date": body.date, "note": body.note},
-    )
-    return result
-
-
-@router.post("/devices/{sn}/lifecycle", response_model=DeviceInventoryItem, status_code=200)
-async def create_lifecycle_entry(
-    sn: str,
-    body: LifecycleEntryCreate,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Upsert a lifecycle history entry for the given status_id.
-
-    If an entry for this status already exists it is overwritten in-place;
-    otherwise a new entry is appended.  This prevents duplicate entries when
-    a status is visited more than once (max one entry per status).
-    """
-    from datetime import datetime, timezone
-    fs = get_firestore()
-    docs = list(fs.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise HTTPException(status_code=404, detail="Device not found")
-    doc_ref = docs[0].reference
-    data = docs[0].to_dict() or {}
-    history = list(data.get("lifecycle_history") or [])
-
-    new_entry = {
-        "status_id": body.status_id,
-        "date": body.date or datetime.now(timezone.utc).isoformat(),
-        "note": body.note,
-        "set_by": user.email,
-    }
-
-    existing_idx = next(
-        (i for i, e in enumerate(history) if e.get("status_id") == body.status_id),
-        None,
-    )
-    is_update = existing_idx is not None
-    if is_update:
-        history[existing_idx] = new_entry
-    else:
-        history.append(new_entry)
-
-    doc_ref.update({"lifecycle_history": history})
-    from manufacturing.service import _doc_to_inventory_item
-    result = _doc_to_inventory_item(doc_ref.get())
-    await log_action(
-        db, user.sub, user.email,
-        action="UPDATE" if is_update else "CREATE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"lifecycle_status": body.status_id, "date": new_entry["date"], "note": body.note},
-    )
-    return result
-
-
-@router.delete("/devices/{sn}/lifecycle/{index}", response_model=DeviceInventoryItem)
-async def delete_lifecycle_entry(
-    sn: str,
-    index: int,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Delete a lifecycle history entry by index. Cannot delete the entry for the current status."""
-    fs = get_firestore()
-    docs = list(fs.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise HTTPException(status_code=404, detail="Device not found")
-    doc_ref = docs[0].reference
-    data = docs[0].to_dict() or {}
-    history = data.get("lifecycle_history") or []
-    if index < 0 or index >= len(history):
-        raise HTTPException(status_code=400, detail="Invalid lifecycle entry index")
-    current_status = data.get("mfg_status", "")
-    deleted_entry = history[index]
-    if deleted_entry.get("status_id") == current_status:
-        raise HTTPException(status_code=400, detail="Cannot delete the entry for the current status. Change the status first.")
-    history.pop(index)
-    doc_ref.update({"lifecycle_history": history})
-    from manufacturing.service import _doc_to_inventory_item
-    result = _doc_to_inventory_item(doc_ref.get())
-    await log_action(
-        db, user.sub, user.email,
-        action="DELETE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"lifecycle_status": deleted_entry.get("status_id"), "index": index},
     )
     return result
 
@@ -303,20 +98,13 @@ async def delete_lifecycle_entry(
 @router.get("/devices/{sn}/nvs.bin")
 async def download_nvs(
     sn: str,
-    hw_type_override: Optional[str] = Query(None, description="Override hw_type written to NVS (for bespoke firmware)"),
-    hw_revision_override: Optional[str] = Query(None, description="Override hw_revision written to NVS (for bespoke firmware)"),
-    nvs_schema: Optional[str] = Query(None, description="NVS schema to use: 'legacy' or 'new' (default)"),
     user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    binary = service.get_nvs_binary(sn, hw_type_override=hw_type_override, hw_revision_override=hw_revision_override, legacy=(nvs_schema == "legacy"))
+    binary = service.get_nvs_binary(sn)
-    await log_action(
+    await audit.log_action(
-        db, user.sub, user.email,
+        admin_user=user.email,
-        action="COMMAND",
+        action="device_flashed",
-        entity_type="device",
+        serial_number=sn,
-        entity_id=sn,
-        entity_label=sn,
-        meta={"command": "nvs_flash", "hw_type_override": hw_type_override, "nvs_schema": nvs_schema or "new"},
     )
     return Response(
         content=binary,
@@ -330,156 +118,17 @@ async def assign_device(
     sn: str,
     body: DeviceAssign,
     user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    try:
+    result = service.assign_device(sn, body)
-        result = service.assign_device(sn, body)
+    await audit.log_action(
-    except NotFoundError as e:
+        admin_user=user.email,
-        raise HTTPException(status_code=404, detail=str(e))
+        action="device_assigned",
-    await log_action(
+        serial_number=sn,
-        db, user.sub, user.email,
+        detail={"customer_email": body.customer_email, "customer_name": body.customer_name},
-        action="UPDATE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"customer_id": body.customer_id},
     )
     return result
 
 
-@router.delete("/devices/{sn}", status_code=204)
-async def delete_device(
-    sn: str,
-    force: bool = Query(False, description="Required to delete sold/claimed devices"),
-    user: TokenPayload = Depends(require_permission("manufacturing", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Delete a device. Sold/claimed devices require force=true."""
-    try:
-        service.delete_device(sn, force=force)
-    except NotFoundError:
-        raise HTTPException(status_code=404, detail="Device not found")
-    except PermissionError as e:
-        raise HTTPException(status_code=403, detail=str(e))
-    await log_action(
-        db, user.sub, user.email,
-        action="DELETE",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"force": force},
-    )
-
-
-@router.post("/devices/{sn}/email/manufactured", status_code=204)
-async def send_manufactured_email(
-    sn: str,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Send the 'device manufactured' notification to the assigned customer's email."""
-    db = get_firestore()
-    docs = list(db.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise HTTPException(status_code=404, detail="Device not found")
-    data = docs[0].to_dict() or {}
-    customer_id = data.get("customer_id")
-    if not customer_id:
-        raise HTTPException(status_code=400, detail="No customer assigned to this device")
-    customer_doc = db.collection("crm_customers").document(customer_id).get()
-    if not customer_doc.exists:
-        raise HTTPException(status_code=404, detail="Assigned customer not found")
-    cdata = customer_doc.to_dict() or {}
-    email = cdata.get("email")
-    if not email:
-        raise HTTPException(status_code=400, detail="Customer has no email address")
-    name_parts = [cdata.get("name") or "", cdata.get("surname") or ""]
-    customer_name = " ".join(p for p in name_parts if p).strip() or None
-    hw_family = data.get("hw_family") or data.get("hw_type") or ""
-    from utils.emails.device_mfged_mail import send_device_manufactured_email
-    send_device_manufactured_email(
-        customer_email=email,
-        serial_number=sn,
-        device_name=hw_family.replace("_", " ").title(),
-        customer_name=customer_name,
-    )
-    await log_action(
-        db, user.sub, user.email,
-        action="COMMAND",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"command": "email_manufactured", "recipient": email},
-    )
-
-
-@router.post("/devices/{sn}/email/assigned", status_code=204)
-async def send_assigned_email(
-    sn: str,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Send the 'device assigned / app instructions' email to the assigned user(s)."""
-    db = get_firestore()
-    docs = list(db.collection("devices").where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise HTTPException(status_code=404, detail="Device not found")
-    data = docs[0].to_dict() or {}
-    user_list = data.get("user_list") or []
-    if not user_list:
-        raise HTTPException(status_code=400, detail="No users assigned to this device")
-    hw_family = data.get("hw_family") or data.get("hw_type") or ""
-    device_name = hw_family.replace("_", " ").title()
-    from utils.emails.device_assigned_mail import send_device_assigned_email
-    errors = []
-    for uid in user_list:
-        try:
-            user_doc = db.collection("users").document(uid).get()
-            if not user_doc.exists:
-                continue
-            udata = user_doc.to_dict() or {}
-            email = udata.get("email")
-            if not email:
-                continue
-            display_name = udata.get("display_name") or udata.get("name") or None
-            send_device_assigned_email(
-                user_email=email,
-                serial_number=sn,
-                device_name=device_name,
-                user_name=display_name,
-            )
-        except Exception as exc:
-            errors.append(str(exc))
-    if errors:
-        raise HTTPException(status_code=500, detail=f"Some emails failed: {'; '.join(errors)}")
-    await log_action(
-        db, user.sub, user.email,
-        action="COMMAND",
-        entity_type="device",
-        entity_id=sn,
-        entity_label=sn,
-        meta={"command": "email_assigned", "user_count": len(user_list)},
-    )
-
-
-@router.delete("/devices", status_code=200)
-async def delete_unprovisioned(
-    user: TokenPayload = Depends(require_permission("manufacturing", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Delete all devices with status 'manufactured' (never provisioned)."""
-    deleted = service.delete_unprovisioned_devices()
-    await log_action(
-        db, user.sub, user.email,
-        action="DELETE",
-        entity_type="device_batch",
-        entity_id="bulk_unprovisioned",
-        entity_label=f"Bulk delete unprovisioned ({len(deleted)} devices)",
-        meta={"count": len(deleted), "serial_numbers": deleted},
-    )
-    return {"deleted": deleted, "count": len(deleted)}
-
-
 @router.get("/devices/{sn}/firmware.bin")
 def redirect_firmware(
     sn: str,
@@ -490,144 +139,3 @@ def redirect_firmware(
     """
     url = service.get_firmware_url(sn)
     return RedirectResponse(url=url, status_code=302)
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Flash assets — bootloader.bin and partitions.bin per hw_type
-# These are the binaries that must be flashed at fixed addresses during full
-# provisioning (0x1000 bootloader, 0x8000 partition table).
-# They are NOT flashed during OTA updates — only during initial provisioning.
-# Upload once per hw_type after each PlatformIO build that changes the layout.
-# ─────────────────────────────────────────────────────────────────────────────
-
-@router.get("/flash-assets")
-def list_flash_assets(
-    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-):
-    """Return asset status for all known board types (and any discovered bespoke UIDs).
-
-    Checks the filesystem directly — no database involved.
-    Each entry contains: hw_type, bootloader (exists, size, uploaded_at), partitions (same), note.
-    """
-    return {"assets": service.list_flash_assets()}
-
-
-@router.delete("/flash-assets/{hw_type}/{asset}", status_code=204)
-async def delete_flash_asset(
-    hw_type: str,
-    asset: str,
-    user: TokenPayload = Depends(require_permission("manufacturing", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Delete a single flash asset file (bootloader.bin or partitions.bin)."""
-    if asset not in VALID_FLASH_ASSETS:
-        raise HTTPException(status_code=400, detail=f"Invalid asset. Must be one of: {', '.join(sorted(VALID_FLASH_ASSETS))}")
-    try:
-        service.delete_flash_asset(hw_type, asset)
-    except NotFoundError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    await log_action(
-        db, user.sub, user.email,
-        action="DELETE",
-        entity_type="firmware",
-        entity_id=f"{hw_type}/{asset}",
-        entity_label=f"{hw_type} / {asset}",
-        meta={"hw_type": hw_type, "asset": asset},
-    )
-
-
-class FlashAssetNoteBody(BaseModel):
-    note: str
-
-
-@router.put("/flash-assets/{hw_type}/note", status_code=204)
-async def set_flash_asset_note(
-    hw_type: str,
-    body: FlashAssetNoteBody,
-    user: TokenPayload = Depends(require_permission("manufacturing", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Save (or overwrite) the note for a hw_type's flash asset set.
-
-    The note is stored as note.txt next to the binary files.
-    Pass an empty string to clear the note.
-    """
-    service.set_flash_asset_note(hw_type, body.note)
-    await log_action(
-        db, user.sub, user.email,
-        action="UPDATE",
-        entity_type="firmware",
-        entity_id=hw_type,
-        entity_label=hw_type,
-        meta={"note": body.note},
-    )
-
-
-@router.post("/flash-assets/{hw_type}/{asset}", status_code=204)
-async def upload_flash_asset(
-    hw_type: str,
-    asset: str,
-    file: UploadFile = File(...),
-    user: TokenPayload = Depends(require_permission("manufacturing", "add")),
-    db: AsyncSession = Depends(get_pg_session),
-):
-    """Upload a bootloader.bin or partitions.bin for a given hw_type.
-
-    These are build artifacts from PlatformIO (.pio/build/{env}/bootloader.bin
-    and .pio/build/{env}/partitions.bin). Upload them once per hw_type after
-    each PlatformIO build that changes the partition layout.
-    """
-    if not hw_type or len(hw_type) > 128:
-        raise HTTPException(status_code=400, detail="Invalid hw_type/bespoke UID.")
-    if asset not in VALID_FLASH_ASSETS:
-        raise HTTPException(status_code=400, detail=f"Invalid asset. Must be one of: {', '.join(sorted(VALID_FLASH_ASSETS))}")
-    data = await file.read()
-    service.save_flash_asset(hw_type, asset, data)
-    await log_action(
-        db, user.sub, user.email,
-        action="CREATE",
-        entity_type="firmware",
-        entity_id=f"{hw_type}/{asset}",
-        entity_label=f"{hw_type} / {asset}",
-        meta={"hw_type": hw_type, "asset": asset, "size_bytes": len(data)},
-    )
-
-
-@router.get("/devices/{sn}/bootloader.bin")
-def download_bootloader(
-    sn: str,
-    hw_type_override: Optional[str] = Query(None, description="Override hw_type for flash asset lookup (for bespoke firmware)"),
-    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-):
-    """Return the bootloader.bin for this device's hw_type (flashed at 0x1000)."""
-    item = service.get_device_by_sn(sn)
-    hw_type = hw_type_override or item.hw_type
-    try:
-        data = service.get_flash_asset(hw_type, "bootloader.bin")
-    except NotFoundError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    return Response(
-        content=data,
-        media_type="application/octet-stream",
-        headers={"Content-Disposition": f'attachment; filename="bootloader_{hw_type}.bin"'},
-    )
-
-
-@router.get("/devices/{sn}/partitions.bin")
-def download_partitions(
-    sn: str,
-    hw_type_override: Optional[str] = Query(None, description="Override hw_type for flash asset lookup (for bespoke firmware)"),
-    _user: TokenPayload = Depends(require_permission("manufacturing", "view")),
-):
-    """Return the partitions.bin for this device's hw_type (flashed at 0x8000)."""
-    item = service.get_device_by_sn(sn)
-    hw_type = hw_type_override or item.hw_type
-    try:
-        data = service.get_flash_asset(hw_type, "partitions.bin")
-    except NotFoundError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-    return Response(
-        content=data,
-        media_type="application/octet-stream",
-        headers={"Content-Disposition": f'attachment; filename="partitions_{hw_type}.bin"'},
-    )

backend/manufacturing/service.py

@@ -1,17 +1,12 @@
-import logging
 import random
 import string
 from datetime import datetime, timezone
-from pathlib import Path
 
-logger = logging.getLogger(__name__)
-
-from config import settings
 from shared.firebase import get_db
 from shared.exceptions import NotFoundError
 from utils.serial_number import generate_serial
 from utils.nvs_generator import generate as generate_nvs_binary
-from manufacturing.models import BatchCreate, BatchResponse, DeviceInventoryItem, DeviceStatusUpdate, DeviceAssign, ManufacturingStats, RecentActivityItem, BOARD_TYPE_LABELS
+from manufacturing.models import BatchCreate, BatchResponse, DeviceInventoryItem, DeviceStatusUpdate, DeviceAssign, ManufacturingStats, RecentActivityItem
 
 COLLECTION = "devices"
 _BATCH_ID_CHARS = string.ascii_uppercase + string.digits
@@ -33,18 +28,6 @@ def _get_existing_sns(db) -> set:
     return existing
 
 
-def _resolve_user_list(raw_list: list) -> list[str]:
-    """Convert user_list entries (DocumentReferences or path strings) to plain user ID strings."""
-    from google.cloud.firestore_v1 import DocumentReference
-    result = []
-    for entry in raw_list:
-        if isinstance(entry, DocumentReference):
-            result.append(entry.id)
-        elif isinstance(entry, str):
-            result.append(entry.split("/")[-1])
-    return result
-
-
 def _doc_to_inventory_item(doc) -> DeviceInventoryItem:
     data = doc.to_dict() or {}
     created_raw = data.get("created_at")
@@ -63,10 +46,6 @@ def _doc_to_inventory_item(doc) -> DeviceInventoryItem:
         created_at=created_str,
         owner=data.get("owner"),
         assigned_to=data.get("assigned_to"),
-        device_name=data.get("device_name") or None,
-        lifecycle_history=data.get("lifecycle_history") or [],
-        customer_id=data.get("customer_id"),
-        user_list=_resolve_user_list(data.get("user_list") or []),
     )
 
 
@@ -95,19 +74,11 @@ def create_batch(data: BatchCreate) -> BatchResponse:
             "created_at": now,
             "owner": None,
             "assigned_to": None,
-            "user_list": [],
+            "users_list": [],
             # Legacy fields left empty so existing device views don't break
             "device_name": "",
             "device_location": "",
             "is_Online": False,
-            "lifecycle_history": [
-                {
-                    "status_id": "manufactured",
-                    "date": now.isoformat(),
-                    "note": None,
-                    "set_by": None,
-                }
-            ],
         })
         serial_numbers.append(sn)
 
@@ -158,38 +129,14 @@ def get_device_by_sn(sn: str) -> DeviceInventoryItem:
     return _doc_to_inventory_item(docs[0])
 
 
-def update_device_status(sn: str, data: DeviceStatusUpdate, set_by: str | None = None) -> DeviceInventoryItem:
+def update_device_status(sn: str, data: DeviceStatusUpdate) -> DeviceInventoryItem:
     db = get_db()
     docs = list(db.collection(COLLECTION).where("serial_number", "==", sn).limit(1).stream())
     if not docs:
         raise NotFoundError("Device")
 
     doc_ref = docs[0].reference
-    doc_data = docs[0].to_dict() or {}
+    update = {"mfg_status": data.status.value}
-    now = datetime.now(timezone.utc).isoformat()
-
-    history = list(doc_data.get("lifecycle_history") or [])
-
-    # Upsert lifecycle entry — overwrite existing entry for this status if present
-    new_entry = {
-        "status_id": data.status.value,
-        "date": now,
-        "note": data.note if data.note else None,
-        "set_by": set_by,
-    }
-    existing_idx = next(
-        (i for i, e in enumerate(history) if e.get("status_id") == data.status.value),
-        None,
-    )
-    if existing_idx is not None:
-        history[existing_idx] = new_entry
-    else:
-        history.append(new_entry)
-
-    update = {
-        "mfg_status": data.status.value,
-        "lifecycle_history": history,
-    }
     if data.note:
         update["mfg_status_note"] = data.note
     doc_ref.update(update)
@@ -197,115 +144,39 @@ def update_device_status(sn: str, data: DeviceStatusUpdate, set_by: str | None =
     return _doc_to_inventory_item(doc_ref.get())
 
 
-def get_nvs_binary(sn: str, hw_type_override: str | None = None, hw_revision_override: str | None = None, legacy: bool = False) -> bytes:
+def get_nvs_binary(sn: str) -> bytes:
     item = get_device_by_sn(sn)
     return generate_nvs_binary(
         serial_number=item.serial_number,
-        hw_family=hw_type_override if hw_type_override else item.hw_type,
+        hw_type=item.hw_type,
-        hw_revision=hw_revision_override if hw_revision_override else item.hw_version,
+        hw_version=item.hw_version,
-        legacy=legacy,
     )
 
 
 def assign_device(sn: str, data: DeviceAssign) -> DeviceInventoryItem:
-    """Assign a device to a customer by customer_id.
+    from utils.email import send_device_assignment_invite
 
-    - Stores customer_id on the device doc.
-    - Adds the device to the customer's owned_items list.
-    - Sets mfg_status to 'sold' unless device is already 'claimed'.
-    """
     db = get_db()
-    CRM_COLLECTION = "crm_customers"
-
-    # Get device doc
     docs = list(db.collection(COLLECTION).where("serial_number", "==", sn).limit(1).stream())
     if not docs:
         raise NotFoundError("Device")
 
-    doc_data = docs[0].to_dict() or {}
     doc_ref = docs[0].reference
-    current_status = doc_data.get("mfg_status", "manufactured")
-
-    # Get customer doc
-    customer_ref = db.collection(CRM_COLLECTION).document(data.customer_id)
-    customer_doc = customer_ref.get()
-    if not customer_doc.exists:
-        raise NotFoundError("Customer")
-    customer_data = customer_doc.to_dict() or {}
-
-    # Determine new status: don't downgrade claimed → sold
-    new_status = current_status if current_status == "claimed" else "sold"
-
-    now = datetime.now(timezone.utc).isoformat()
-    history = doc_data.get("lifecycle_history") or []
-    history.append({
-        "status_id": new_status,
-        "date": now,
-        "note": "Assigned to customer",
-        "set_by": None,
-    })
-
     doc_ref.update({
-        "customer_id": data.customer_id,
+        "owner": data.customer_email,
-        "mfg_status": new_status,
+        "assigned_to": data.customer_email,
-        "lifecycle_history": history,
+        "mfg_status": "sold",
     })
 
-    # Add to customer's owned_items (avoid duplicates)
+    send_device_assignment_invite(
-    owned_items = customer_data.get("owned_items", []) or []
+        customer_email=data.customer_email,
-    device_doc_id = docs[0].id
+        serial_number=sn,
-    already_assigned = any(
+        customer_name=data.customer_name,
-        item.get("type") == "console_device"
-        and item.get("console_device", {}).get("device_id") == device_doc_id
-        for item in owned_items
     )
-    if not already_assigned:
-        device_name = doc_data.get("device_name") or BOARD_TYPE_LABELS.get(doc_data.get("hw_type", ""), sn)
-        owned_items.append({
-            "type": "console_device",
-            "console_device": {
-                "device_id": device_doc_id,
-                "serial_number": sn,
-                "label": device_name,
-            },
-        })
-        customer_ref.update({"owned_items": owned_items})
 
     return _doc_to_inventory_item(doc_ref.get())
 
 
-def search_customers(q: str) -> list:
-    """Search crm_customers by name, email, phone, organization, or tags."""
-    db = get_db()
-    CRM_COLLECTION = "crm_customers"
-    docs = db.collection(CRM_COLLECTION).stream()
-    results = []
-    q_lower = q.lower().strip()
-    for doc in docs:
-        data = doc.to_dict() or {}
-        loc = data.get("location") or {}
-        loc = loc if isinstance(loc, dict) else {}
-        city = loc.get("city") or ""
-        searchable = " ".join(filter(None, [
-            data.get("name"), data.get("surname"),
-            data.get("email"), data.get("phone"), data.get("organization"),
-            loc.get("address"), loc.get("city"), loc.get("postal_code"),
-            loc.get("region"), loc.get("country"),
-            " ".join(data.get("tags") or []),
-        ])).lower()
-        if not q_lower or q_lower in searchable:
-            results.append({
-                "id": doc.id,
-                "name": data.get("name") or "",
-                "surname": data.get("surname") or "",
-                "email": data.get("email") or "",
-                "organization": data.get("organization") or "",
-                "phone": data.get("phone") or "",
-                "city": city or "",
-            })
-    return results
-
-
 def get_stats() -> ManufacturingStats:
     db = get_db()
     docs = list(db.collection(COLLECTION).stream())
@@ -346,146 +217,6 @@ def get_stats() -> ManufacturingStats:
     return ManufacturingStats(counts=counts, recent_activity=recent)
 
 
-PROTECTED_STATUSES = {"sold", "claimed"}
-
-
-def delete_device(sn: str, force: bool = False) -> None:
-    """Delete a device by serial number.
-
-    Raises PermissionError if the device is sold/claimed and force is not set.
-    The frontend uses force=True only after the user confirms by typing the SN.
-    """
-    db = get_db()
-    docs = list(db.collection(COLLECTION).where("serial_number", "==", sn).limit(1).stream())
-    if not docs:
-        raise NotFoundError("Device")
-
-    data = docs[0].to_dict() or {}
-    status = data.get("mfg_status", "manufactured")
-
-    if status in PROTECTED_STATUSES and not force:
-        raise PermissionError(
-            f"Device {sn} has status '{status}' and cannot be deleted without explicit confirmation."
-        )
-
-    docs[0].reference.delete()
-
-
-def delete_unprovisioned_devices() -> list[str]:
-    """Delete all devices with status 'manufactured' (never flashed/provisioned).
-
-    Returns the list of deleted serial numbers.
-    """
-    db = get_db()
-    docs = list(db.collection(COLLECTION).where("mfg_status", "==", "manufactured").stream())
-    deleted = []
-    for doc in docs:
-        data = doc.to_dict() or {}
-        sn = data.get("serial_number", "")
-        doc.reference.delete()
-        deleted.append(sn)
-    return deleted
-
-
-KNOWN_HW_TYPES = ["vesper", "vesper_plus", "vesper_pro", "agnus", "agnus_mini", "chronos", "chronos_pro"]
-FLASH_ASSET_FILES = ["bootloader.bin", "partitions.bin"]
-
-
-def _flash_asset_path(hw_type: str, asset: str) -> Path:
-    """Return path to a flash asset (bootloader.bin or partitions.bin) for a given hw_type."""
-    return Path(settings.flash_assets_storage_path) / hw_type / asset
-
-
-def _flash_asset_info(hw_type: str) -> dict:
-    """Build the asset info dict for a single hw_type by inspecting the filesystem."""
-    base = Path(settings.flash_assets_storage_path) / hw_type
-    note_path = base / "note.txt"
-    note = note_path.read_text(encoding="utf-8").strip() if note_path.exists() else ""
-
-    files = {}
-    for fname in FLASH_ASSET_FILES:
-        p = base / fname
-        if p.exists():
-            stat = p.stat()
-            files[fname] = {
-                "exists": True,
-                "size_bytes": stat.st_size,
-                "uploaded_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
-            }
-        else:
-            files[fname] = {"exists": False, "size_bytes": None, "uploaded_at": None}
-
-    return {
-        "hw_type": hw_type,
-        "bootloader": files["bootloader.bin"],
-        "partitions": files["partitions.bin"],
-        "note": note,
-    }
-
-
-def list_flash_assets() -> list:
-    """Return asset status for all known board types plus any discovered bespoke directories."""
-    base = Path(settings.flash_assets_storage_path)
-    results = []
-
-    # Always include all known hw types, even if no files uploaded yet
-    seen = set(KNOWN_HW_TYPES)
-    for hw_type in KNOWN_HW_TYPES:
-        results.append(_flash_asset_info(hw_type))
-
-    # Discover bespoke directories (anything in storage/flash_assets/ not in known list)
-    if base.exists():
-        for entry in sorted(base.iterdir()):
-            if entry.is_dir() and entry.name not in seen:
-                seen.add(entry.name)
-                info = _flash_asset_info(entry.name)
-                info["is_bespoke"] = True
-                results.append(info)
-
-    # Mark known types
-    for r in results:
-        r.setdefault("is_bespoke", False)
-
-    return results
-
-
-def save_flash_asset(hw_type: str, asset: str, data: bytes) -> Path:
-    """Persist a flash asset binary. asset must be 'bootloader.bin' or 'partitions.bin'."""
-    if asset not in ("bootloader.bin", "partitions.bin"):
-        raise ValueError(f"Unknown flash asset: {asset}")
-    path = _flash_asset_path(hw_type, asset)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_bytes(data)
-    return path
-
-
-def delete_flash_asset(hw_type: str, asset: str) -> None:
-    """Delete a flash asset file. Raises NotFoundError if not present."""
-    path = _flash_asset_path(hw_type, asset)
-    if not path.exists():
-        raise NotFoundError(f"Flash asset '{asset}' for '{hw_type}' not found")
-    path.unlink()
-
-
-def set_flash_asset_note(hw_type: str, note: str) -> None:
-    """Write (or clear) the note for a hw_type's flash asset directory."""
-    base = Path(settings.flash_assets_storage_path) / hw_type
-    base.mkdir(parents=True, exist_ok=True)
-    note_path = base / "note.txt"
-    if note.strip():
-        note_path.write_text(note.strip(), encoding="utf-8")
-    elif note_path.exists():
-        note_path.unlink()
-
-
-def get_flash_asset(hw_type: str, asset: str) -> bytes:
-    """Load a flash asset binary. Raises NotFoundError if not uploaded yet."""
-    path = _flash_asset_path(hw_type, asset)
-    if not path.exists():
-        raise NotFoundError(f"Flash asset '{asset}' for hw_type '{hw_type}' — upload it first via POST /api/manufacturing/flash-assets/{{hw_type}}/{{asset}}")
-    return path.read_bytes()
-
-
 def get_firmware_url(sn: str) -> str:
     """Return the FastAPI download URL for the latest stable firmware for this device's hw_type."""
     from firmware.service import get_latest

backend/melodies/database.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from database import get_db
+from mqtt.database import get_db
 
 logger = logging.getLogger("melodies.database")
 

backend/melodies/models.py

@@ -30,7 +30,6 @@ class MelodyInfo(BaseModel):
     isTrueRing: bool = False
     previewURL: str = ""
     archetype_csv: Optional[str] = None
-    outdated_archetype: bool = False
 
 
 class MelodyAttributes(BaseModel):

backend/melodies/orm.py

@@ -1,39 +0,0 @@
-from datetime import datetime, timezone
-from sqlalchemy import Boolean, Column, DateTime, Index, String, Text
-from sqlalchemy.dialects.postgresql import JSONB
-from database.postgres import Base
-
-
-def _now():
-    return datetime.now(timezone.utc)
-
-
-class MelodyDraft(Base):
-    __tablename__ = "melody_drafts"
-    __table_args__ = (
-        Index("idx_melody_drafts_status", "status"),
-    )
-
-    id         = Column(String(128), primary_key=True)
-    status     = Column(String(32), nullable=False, default="draft")
-    # 'data' stores the full melody definition as JSON (was TEXT/JSON in SQLite)
-    data       = Column(JSONB, nullable=False)
-    created_at = Column(DateTime(timezone=True), nullable=False, default=_now)
-    updated_at = Column(DateTime(timezone=True), nullable=False, default=_now, onupdate=_now)
-
-
-class BuiltMelody(Base):
-    __tablename__ = "built_melodies"
-
-    id                  = Column(String(128), primary_key=True)
-    name                = Column(String(500), nullable=False)
-    pid                 = Column(String(128), nullable=False)
-    # 'steps' is a JSON array of step definitions
-    steps               = Column(JSONB, nullable=False)
-    binary_path         = Column(String(1000))
-    progmem_code        = Column(Text)
-    # JSON array of melody IDs this built melody is assigned to
-    assigned_melody_ids = Column(JSONB, nullable=False, default=list)
-    is_builtin          = Column(Boolean, nullable=False, default=False)
-    created_at          = Column(DateTime(timezone=True), nullable=False, default=_now)
-    updated_at          = Column(DateTime(timezone=True), nullable=False, default=_now, onupdate=_now)

backend/melodies/router.py

@@ -1,14 +1,11 @@
 from fastapi import APIRouter, Depends, UploadFile, File, Query, HTTPException, Response
 from typing import Optional
-from sqlalchemy.ext.asyncio import AsyncSession
 from auth.models import TokenPayload
 from auth.dependencies import require_permission
 from melodies.models import (
     MelodyCreate, MelodyUpdate, MelodyInDB, MelodyListResponse, MelodyInfo,
 )
 from melodies import service
-from database.postgres import get_pg_session
-from shared.audit import log_action
 
 router = APIRouter(prefix="/api/melodies", tags=["melodies"])
 
@@ -45,12 +42,8 @@ async def create_melody(
     body: MelodyCreate,
     publish: bool = Query(False),
     _user: TokenPayload = Depends(require_permission("melodies", "add")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.create_melody(body, publish=publish, actor_name=_user.name)
+    return await service.create_melody(body, publish=publish, actor_name=_user.name)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "melody",
-                     melody.id, melody.information.name if melody.information else melody.id)
-    return melody
 
 
 @router.put("/{melody_id}", response_model=MelodyInDB)
@@ -58,61 +51,32 @@ async def update_melody(
     melody_id: str,
     body: MelodyUpdate,
     _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    old = await service.get_melody(melody_id)
+    return await service.update_melody(melody_id, body, actor_name=_user.name)
-    melody = await service.update_melody(melody_id, body, actor_name=_user.name)
-    _SKIP = {"updated_at", "id", "metadata", "information", "noteAssignments"}
-    changes = {
-        k: {"old": getattr(old, k, None), "new": getattr(melody, k, None)}
-        for k in body.model_fields_set
-        if k not in _SKIP and getattr(old, k, None) != getattr(melody, k, None)
-    }
-    # Surface the name change from inside the information sub-object
-    old_name = old.information.name if old.information else None
-    new_name = melody.information.name if melody.information else None
-    if old_name != new_name:
-        changes["name"] = {"old": old_name, "new": new_name}
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "melody",
-                     melody_id, new_name or melody_id, changes=changes or None)
-    return melody
 
 
 @router.delete("/{melody_id}", status_code=204)
 async def delete_melody(
     melody_id: str,
     _user: TokenPayload = Depends(require_permission("melodies", "delete")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.get_melody(melody_id)
-    label = melody.information.name if melody.information else melody_id
     await service.delete_melody(melody_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "melody",
-                     melody_id, label)
 
 
 @router.post("/{melody_id}/publish", response_model=MelodyInDB)
 async def publish_melody(
     melody_id: str,
     _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.publish_melody(melody_id)
+    return await service.publish_melody(melody_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "PUBLISH", "melody",
-                     melody_id, melody.information.name if melody.information else melody_id)
-    return melody
 
 
 @router.post("/{melody_id}/unpublish", response_model=MelodyInDB)
 async def unpublish_melody(
     melody_id: str,
     _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-    db: AsyncSession = Depends(get_pg_session),
 ):
-    melody = await service.unpublish_melody(melody_id)
+    return await service.unpublish_melody(melody_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "UNPUBLISH", "melody",
-                     melody_id, melody.information.name if melody.information else melody_id)
-    return melody
 
 
 @router.post("/{melody_id}/upload/{file_type}")
@@ -182,23 +146,6 @@ async def get_files(
     return service.get_storage_files(melody_id, melody.uid)
 
 
-@router.patch("/{melody_id}/set-outdated", response_model=MelodyInDB)
-async def set_outdated(
-    melody_id: str,
-    outdated: bool = Query(...),
-    _user: TokenPayload = Depends(require_permission("melodies", "edit")),
-):
-    """Manually set or clear the outdated_archetype flag on a melody."""
-    melody = await service.get_melody(melody_id)
-    info = melody.information.model_dump()
-    info["outdated_archetype"] = outdated
-    return await service.update_melody(
-        melody_id,
-        MelodyUpdate(information=MelodyInfo(**info)),
-        actor_name=_user.name,
-    )
-
-
 @router.get("/{melody_id}/download/binary")
 async def download_binary_file(
     melody_id: str,

backend/migrate_customer_flags.py

@@ -1,178 +0,0 @@
-"""
-One-time migration script: convert legacy negotiating/has_problem flags to new structure.
-
-Run AFTER deploying the new backend code:
-    cd backend && python migrate_customer_flags.py
-
-What it does:
-  1. For each customer with negotiating=True:
-     - Creates an order subcollection document with status="negotiating"
-     - Sets relationship_status="active" (only if currently "lead" or "prospect")
-  2. For each customer with has_problem=True:
-     - Appends one entry to technical_issues with active=True
-  3. Removes negotiating and has_problem fields from every customer document
-  4. Initialises relationship_status="lead" on any customer missing it
-  5. Recomputes crm_summary for each affected customer
-"""
-
-import sys
-import os
-import uuid
-from datetime import datetime
-
-# Make sure we can import backend modules
-sys.path.insert(0, os.path.dirname(__file__))
-
-from shared.firebase import init_firebase, get_db
-
-init_firebase()
-
-
-def migrate():
-    db = get_db()
-    customers_ref = db.collection("crm_customers")
-    docs = list(customers_ref.stream())
-    print(f"Found {len(docs)} customer documents.")
-
-    migrated_neg = 0
-    migrated_prob = 0
-    now = datetime.utcnow().isoformat()
-
-    for doc in docs:
-        data = doc.to_dict() or {}
-        customer_id = doc.id
-        updates = {}
-        changed = False
-
-        # ── 1. Initialise new fields if missing ──────────────────────────────
-        if "relationship_status" not in data:
-            updates["relationship_status"] = "lead"
-            changed = True
-        if "technical_issues" not in data:
-            updates["technical_issues"] = []
-            changed = True
-        if "install_support" not in data:
-            updates["install_support"] = []
-            changed = True
-        if "transaction_history" not in data:
-            updates["transaction_history"] = []
-            changed = True
-
-        # ── 2. Migrate negotiating flag ───────────────────────────────────────
-        if data.get("negotiating"):
-            order_id = str(uuid.uuid4())
-            order_data = {
-                "customer_id": customer_id,
-                "order_number": f"ORD-{datetime.utcnow().year}-001-migrated",
-                "title": "Migrated from legacy negotiating flag",
-                "created_by": "system",
-                "status": "negotiating",
-                "status_updated_date": now,
-                "status_updated_by": "system",
-                "items": [],
-                "subtotal": 0,
-                "discount": None,
-                "total_price": 0,
-                "currency": "EUR",
-                "shipping": None,
-                "payment_status": {
-                    "required_amount": 0,
-                    "received_amount": 0,
-                    "balance_due": 0,
-                    "advance_required": False,
-                    "advance_amount": None,
-                    "payment_complete": False,
-                },
-                "invoice_path": None,
-                "notes": "Migrated from legacy negotiating flag",
-                "timeline": [{
-                    "date": now,
-                    "type": "note",
-                    "note": "Migrated from legacy negotiating flag",
-                    "updated_by": "system",
-                }],
-                "created_at": now,
-                "updated_at": now,
-            }
-            customers_ref.document(customer_id).collection("orders").document(order_id).set(order_data)
-
-            current_rel = updates.get("relationship_status") or data.get("relationship_status", "lead")
-            if current_rel in ("lead", "prospect"):
-                updates["relationship_status"] = "active"
-
-            migrated_neg += 1
-            print(f"  [{customer_id}] Created negotiating order, set relationship_status=active")
-
-        # ── 3. Migrate has_problem flag ───────────────────────────────────────
-        if data.get("has_problem"):
-            existing_issues = list(updates.get("technical_issues") or data.get("technical_issues") or [])
-            existing_issues.append({
-                "active": True,
-                "opened_date": data.get("updated_at") or now,
-                "resolved_date": None,
-                "note": "Migrated from legacy has_problem flag",
-                "opened_by": "system",
-                "resolved_by": None,
-            })
-            updates["technical_issues"] = existing_issues
-            migrated_prob += 1
-            changed = True
-            print(f"  [{customer_id}] Appended technical issue from has_problem flag")
-
-        # ── 4. Remove legacy fields ───────────────────────────────────────────
-        from google.cloud.firestore_v1 import DELETE_FIELD
-        if "negotiating" in data:
-            updates["negotiating"] = DELETE_FIELD
-            changed = True
-        if "has_problem" in data:
-            updates["has_problem"] = DELETE_FIELD
-            changed = True
-
-        if changed or data.get("negotiating") or data.get("has_problem"):
-            updates["updated_at"] = now
-            customers_ref.document(customer_id).update(updates)
-
-        # ── 5. Recompute crm_summary ──────────────────────────────────────────
-        # Re-read updated doc to compute summary
-        updated_doc = customers_ref.document(customer_id).get()
-        updated_data = updated_doc.to_dict() or {}
-
-        issues = updated_data.get("technical_issues") or []
-        active_issues = [i for i in issues if i.get("active")]
-        support = updated_data.get("install_support") or []
-        active_support = [s for s in support if s.get("active")]
-
-        TERMINAL = {"declined", "complete"}
-        active_order_status = None
-        active_order_status_date = None
-        active_order_title = None
-        latest_date = ""
-        for odoc in customers_ref.document(customer_id).collection("orders").stream():
-            odata = odoc.to_dict() or {}
-            if odata.get("status") not in TERMINAL:
-                upd = odata.get("status_updated_date") or odata.get("created_at") or ""
-                if upd > latest_date:
-                    latest_date = upd
-                    active_order_status = odata.get("status")
-                    active_order_status_date = upd
-                    active_order_title = odata.get("title")
-
-        summary = {
-            "active_order_status": active_order_status,
-            "active_order_status_date": active_order_status_date,
-            "active_order_title": active_order_title,
-            "active_issues_count": len(active_issues),
-            "latest_issue_date": max((i.get("opened_date") or "") for i in active_issues) if active_issues else None,
-            "active_support_count": len(active_support),
-            "latest_support_date": max((s.get("opened_date") or "") for s in active_support) if active_support else None,
-        }
-        customers_ref.document(customer_id).update({"crm_summary": summary})
-
-    print(f"\nMigration complete.")
-    print(f"  Negotiating orders created: {migrated_neg}")
-    print(f"  Technical issues created:   {migrated_prob}")
-    print(f"  Total customers processed:  {len(docs)}")
-
-
-if __name__ == "__main__":
-    migrate()

backend/migration/migrate_built_melodies.py

@@ -1,65 +0,0 @@
-"""
-Phase 1 — Step 1.2: built_melodies  (SQLite → Postgres)
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_built_melodies
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from melodies.orm import BuiltMelody
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, parse_json, log_run, pg_count
-
-SCRIPT = "migrate_built_melodies"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM built_melodies")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} built_melodies rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        records.append({
-            "id":                  r["id"],
-            "name":                r["name"],
-            "pid":                 r["pid"],
-            "steps":               parse_json(r["steps"], default=[]),
-            "binary_path":         r["binary_path"],
-            "progmem_code":        r["progmem_code"],
-            "assigned_melody_ids": parse_json(r["assigned_melody_ids"], default=[]),
-            "is_builtin":          bool(r["is_builtin"]) if r["is_builtin"] is not None else False,
-            "created_at":          parse_dt(r["created_at"]),
-            "updated_at":          parse_dt(r["updated_at"]),
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(BuiltMelody).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "built_melodies")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_commands.py

@@ -1,73 +0,0 @@
-"""
-Phase 1 — Step 1.10: commands  (SQLite → Postgres)
-
-commands is a raw-SQL table (no ORM model). BIGSERIAL PK — SQLite integer IDs
-are NOT preserved; rows are inserted in sent_at order.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_commands
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_commands"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM commands ORDER BY sent_at")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} commands rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = [
-        {
-            "device_serial":    r["device_serial"],
-            "command_name":     r["command_name"],
-            "command_payload":  r["command_payload"],
-            "status":           r["status"] or "pending",
-            "response_payload": r["response_payload"],
-            "sent_at":          parse_dt(r["sent_at"]),
-            "responded_at":     parse_dt(r["responded_at"]),
-        }
-        for r in rows
-    ]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(
-                text("""
-                    INSERT INTO commands
-                        (device_serial, command_name, command_payload, status,
-                         response_payload, sent_at, responded_at)
-                    VALUES
-                        (:device_serial, :command_name, :command_payload, :status,
-                         :response_payload, :sent_at, :responded_at)
-                """),
-                records,
-            )
-            dest_count = await pg_count(session, "commands")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_comms_log.py

@@ -1,84 +0,0 @@
-"""
-Phase 1 — Step 1.9: crm_comms_log  (SQLite → Postgres)
-
-FK to crm_customers(id) (nullable, ON DELETE SET NULL) — FK enforcement
-suppressed until Phase 2 populates crm_customers.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_comms_log
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmCommsLog
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, parse_json, log_run, pg_count
-
-SCRIPT = "migrate_crm_comms_log"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM crm_comms_log ORDER BY occurred_at")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} crm_comms_log rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        # attachments stored as JSON text in SQLite
-        attachments = parse_json(r["attachments"], default=[])
-
-        # is_important / is_read stored as INTEGER (0/1) in SQLite
-        is_important = bool(r["is_important"]) if r["is_important"] is not None else False
-        is_read      = bool(r["is_read"])      if r["is_read"]      is not None else True
-
-        records.append({
-            "id":             r["id"],
-            "customer_id":    r["customer_id"],
-            "type":           r["type"],
-            "mail_account":   r["mail_account"],
-            "direction":      r["direction"],
-            "subject":        r["subject"],
-            "body":           r["body"],
-            "body_html":      r["body_html"],
-            "attachments":    attachments,
-            "ext_message_id": r["ext_message_id"],
-            "from_addr":      r["from_addr"],
-            "to_addrs":       r["to_addrs"],
-            "logged_by":      r["logged_by"],
-            "is_important":   is_important,
-            "is_read":        is_read,
-            "occurred_at":    parse_dt(r["occurred_at"]),
-            "created_at":     parse_dt(r["created_at"]),
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(text("SET LOCAL session_replication_role = replica"))
-            stmt = pg_insert(CrmCommsLog).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_comms_log")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_customers.py

@@ -1,172 +0,0 @@
-"""
-Phase 2 — Step 2.4: crm_customers  (Firestore → Postgres)
-
-Reads the 'crm_customers' Firestore collection.
-- Strips legacy fields: 'negotiating', 'has_problem'
-- Converts Firestore DatetimeWithNanoseconds → UTC datetime
-- Converts nested dicts/lists → JSONB-ready Python objects
-
-After this runs, the FK constraints on crm_quotations, crm_comms_log,
-crm_media, and crm_orders (all inserted in Phase 1 with FK enforcement
-suppressed) become valid.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_customers
-"""
-
-import asyncio
-import sys
-from datetime import datetime, timezone
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmCustomer
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_crm_customers"
-COLLECTION = "crm_customers"
-
-_LEGACY_FIELDS = {"negotiating", "has_problem"}
-
-_VALID_STATUSES = {
-    "lead", "active", "inactive", "archived",
-    "prospect", "churned", "vip",
-}
-
-
-def _now_utc() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _coerce_dt(val) -> datetime | None:
-    """Handle both Firestore DatetimeWithNanoseconds and ISO strings."""
-    if val is None:
-        return None
-    if isinstance(val, datetime):
-        return val.replace(tzinfo=timezone.utc) if val.tzinfo is None else val
-    return parse_dt(str(val))
-
-
-def _coerce_list(val, default=None) -> list:
-    if isinstance(val, list):
-        return val
-    return default if default is not None else []
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised.", file=sys.stderr)
-        sys.exit(1)
-
-    docs = list(fs.collection(COLLECTION).stream())
-    source_count = len(docs)
-    print(f"Source (Firestore): {source_count} crm_customers documents")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    skipped = 0
-    for doc in docs:
-        d = doc.to_dict()
-
-        # Strip legacy fields
-        for f in _LEGACY_FIELDS:
-            d.pop(f, None)
-
-        # folder_id is NOT NULL UNIQUE — skip docs missing it
-        folder_id = d.get("folder_id") or ""
-        if not folder_id:
-            print(f"  WARNING: customer {doc.id} has no folder_id — skipping", file=sys.stderr)
-            skipped += 1
-            continue
-
-        # relationship_status — normalise unknown values to 'lead'
-        rel_status = d.get("relationship_status") or "lead"
-        if rel_status not in _VALID_STATUSES:
-            rel_status = "lead"
-
-        # contacts / notes — Firestore stores as list of maps
-        contacts = _coerce_list(d.get("contacts"))
-        # Serialise nested Pydantic-style objects to plain dicts
-        contacts = [c if isinstance(c, dict) else vars(c) for c in contacts]
-
-        notes = _coerce_list(d.get("notes"))
-        notes = [n if isinstance(n, dict) else vars(n) for n in notes]
-
-        # location — may be a map or None
-        location = d.get("location")
-        if location and not isinstance(location, dict):
-            location = vars(location)
-
-        tags = _coerce_list(d.get("tags"))
-        owned_items = _coerce_list(d.get("owned_items"))
-        owned_items = [o if isinstance(o, dict) else vars(o) for o in owned_items]
-
-        linked_user_ids = _coerce_list(d.get("linked_user_ids"))
-
-        technical_issues    = _coerce_list(d.get("technical_issues"))
-        install_support     = _coerce_list(d.get("install_support"))
-        transaction_history = _coerce_list(d.get("transaction_history"))
-
-        crm_summary = d.get("crm_summary")
-        if crm_summary and not isinstance(crm_summary, dict):
-            crm_summary = vars(crm_summary)
-
-        created_at = _coerce_dt(d.get("created_at")) or _now_utc()
-        updated_at = _coerce_dt(d.get("updated_at")) or _now_utc()
-
-        records.append({
-            "id":                  doc.id,
-            "firestore_id":        doc.id,
-            "title":               d.get("title"),
-            "name":                d.get("name") or "",
-            "surname":             d.get("surname"),
-            "organization":        d.get("organization"),
-            "religion":            d.get("religion"),
-            "language":            d.get("language") or "el",
-            "folder_id":           folder_id,
-            "relationship_status": rel_status,
-            "nextcloud_folder":    d.get("nextcloud_folder"),
-            "contacts":            contacts,
-            "notes":               notes,
-            "location":            location,
-            "tags":                tags,
-            "owned_items":         owned_items,
-            "linked_user_ids":     linked_user_ids,
-            "technical_issues":    technical_issues,
-            "install_support":     install_support,
-            "transaction_history": transaction_history,
-            "crm_summary":         crm_summary,
-            "created_at":          created_at,
-            "updated_at":          updated_at,
-        })
-
-    actual_source = source_count - skipped
-    print(f"  {skipped} skipped (missing folder_id), {actual_source} to insert")
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(CrmCustomer).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_customers")
-
-    if dest_count < actual_source:
-        msg = f"Count mismatch: expected>={actual_source}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count,
-                  notes=f"{skipped} skipped (no folder_id)" if skipped else None)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_media.py

@@ -1,75 +0,0 @@
-"""
-Phase 1 — Step 1.8: crm_media  (SQLite → Postgres)
-
-FK to crm_customers(id) (nullable) — FK enforcement suppressed until Phase 2
-populates crm_customers.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_media
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmMedia
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, parse_json, log_run, pg_count
-
-SCRIPT = "migrate_crm_media"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM crm_media ORDER BY created_at")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} crm_media rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        # SQLite stores tags as JSON text; Postgres column is JSONB
-        tags_raw = r["tags"]
-        tags = parse_json(tags_raw, default=[])
-
-        records.append({
-            "id":             r["id"],
-            "customer_id":    r["customer_id"],
-            "order_id":       r["order_id"],
-            "filename":       r["filename"],
-            "nextcloud_path": r["nextcloud_path"],
-            "thumbnail_path": r["thumbnail_path"],
-            "mime_type":      r["mime_type"],
-            "direction":      r["direction"],
-            "tags":           tags,
-            "uploaded_by":    r["uploaded_by"],
-            "created_at":     parse_dt(r["created_at"]),
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(text("SET LOCAL session_replication_role = replica"))
-            stmt = pg_insert(CrmMedia).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_media")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_orders.py

@@ -1,156 +0,0 @@
-"""
-Phase 2 — Step 2.5: crm_orders  (Firestore → Postgres)
-
-Orders are stored as a subcollection under each customer:
-    crm_customers/{customer_id}/orders/{order_id}
-
-Uses collection_group("orders") to fetch all orders in one pass,
-then inserts into crm_orders. crm_customers MUST already be in Postgres
-(step 2.4) so the FK constraint is satisfied.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_orders
-"""
-
-import asyncio
-import sys
-from datetime import datetime, timezone
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmOrder
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_crm_orders"
-
-
-def _now_utc() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _coerce_dt(val) -> datetime | None:
-    if val is None:
-        return None
-    if isinstance(val, datetime):
-        return val.replace(tzinfo=timezone.utc) if val.tzinfo is None else val
-    return parse_dt(str(val))
-
-
-def _coerce_list(val) -> list:
-    return val if isinstance(val, list) else []
-
-
-def _coerce_dict(val) -> dict:
-    return val if isinstance(val, dict) else {}
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised.", file=sys.stderr)
-        sys.exit(1)
-
-    # collection_group fetches from ALL customers' 'orders' subcollections
-    docs = list(fs.collection_group("orders").stream())
-    source_count = len(docs)
-    print(f"Source (Firestore): {source_count} order documents (via collection_group)")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    # First, collect all customer IDs already in Postgres so we can skip
-    # orphaned orders whose customer didn't migrate (missing folder_id edge case)
-    async with AsyncPgSession() as session:
-        from sqlalchemy import text
-        result = await session.execute(text("SELECT id FROM crm_customers"))
-        valid_customer_ids = {row[0] for row in result.fetchall()}
-
-    records = []
-    skipped = 0
-    seen_order_numbers: set[str] = set()
-    for doc in docs:
-        d = doc.to_dict()
-
-        # Extract customer_id from the document path:
-        # crm_customers/{customer_id}/orders/{order_id}
-        path_parts = doc.reference.path.split("/")
-        # path: crm_customers / <cid> / orders / <oid>
-        try:
-            customer_id = path_parts[1]
-        except IndexError:
-            print(f"  WARNING: cannot parse customer_id from path {doc.reference.path} — skipping")
-            skipped += 1
-            continue
-
-        if customer_id not in valid_customer_ids:
-            print(f"  WARNING: order {doc.id} references unknown customer {customer_id} — skipping")
-            skipped += 1
-            continue
-
-        order_number = d.get("order_number") or f"ORD-LEGACY-{doc.id}"
-        # Deduplicate: if this order_number was already seen in this batch,
-        # make it unique by appending the doc ID suffix.
-        if order_number in seen_order_numbers:
-            order_number = f"{order_number}-{doc.id[:8]}"
-            print(f"  INFO: duplicate order_number — renamed to {order_number}")
-        seen_order_numbers.add(order_number)
-
-        created_at = _coerce_dt(d.get("created_at")) or _now_utc()
-        updated_at = _coerce_dt(d.get("updated_at")) or _now_utc()
-        status_updated_date = _coerce_dt(d.get("status_updated_date"))
-
-        records.append({
-            "id":                  doc.id,
-            "customer_id":         customer_id,
-            "order_number":        order_number,
-            "title":               d.get("title"),
-            "created_by":          d.get("created_by"),
-            "status":              d.get("status") or "negotiating",
-            "status_updated_date": status_updated_date,
-            "status_updated_by":   d.get("status_updated_by"),
-            "items":               _coerce_list(d.get("items")),
-            "subtotal":            float(d.get("subtotal") or 0),
-            "discount":            d.get("discount") if isinstance(d.get("discount"), dict) else None,
-            "total_price":         float(d.get("total_price") or 0),
-            "currency":            d.get("currency") or "EUR",
-            "shipping":            d.get("shipping") if isinstance(d.get("shipping"), dict) else None,
-            "payment_status":      _coerce_dict(d.get("payment_status")),
-            "invoice_path":        d.get("invoice_path"),
-            "notes":               d.get("notes") if isinstance(d.get("notes"), str) else None,
-            "timeline":            _coerce_list(d.get("timeline")),
-            "created_at":          created_at,
-            "updated_at":          updated_at,
-        })
-
-    actual_source = source_count - skipped
-    print(f"  {skipped} skipped (orphaned/bad path), {actual_source} to insert")
-
-    if not records:
-        print("Nothing valid to insert.")
-        await log_run(SCRIPT, source_count, 0, notes=f"{skipped} all skipped")
-        return
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(CrmOrder).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_orders")
-
-    if dest_count < actual_source:
-        msg = f"Count mismatch: expected>={actual_source}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count,
-                  notes=f"{skipped} skipped" if skipped else None)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_products.py

@@ -1,102 +0,0 @@
-"""
-Phase 2 — Step 2.3: crm_products  (Firestore → Postgres)
-
-Reads the 'crm_products' Firestore collection. The Firestore schema is richer
-than the Postgres target (has costs, stock, name_en, etc.) — we extract only
-what the Postgres ORM model covers. The rest stays in Firestore until the
-service is fully cut over.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_products
-"""
-
-import asyncio
-import sys
-from datetime import datetime, timezone
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmProduct
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_crm_products"
-COLLECTION = "crm_products"
-
-_LEGACY_STATUS_MAP = {
-    "active": True,
-    "discontinued": False,
-    "planned": True,
-}
-
-
-def _now_utc() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised.", file=sys.stderr)
-        sys.exit(1)
-
-    docs = list(fs.collection(COLLECTION).stream())
-    source_count = len(docs)
-    print(f"Source (Firestore): {source_count} crm_products documents")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for doc in docs:
-        d = doc.to_dict()
-
-        # is_active: prefer 'active' bool field, fall back to 'status' string
-        if "active" in d:
-            is_active = bool(d["active"])
-        else:
-            is_active = _LEGACY_STATUS_MAP.get(d.get("status", "active"), True)
-
-        # unit_cost: Firestore uses 'price'
-        unit_cost = d.get("unit_cost") or d.get("price") or 0
-
-        created_at = parse_dt(d.get("created_at")) or _now_utc()
-        updated_at = parse_dt(d.get("updated_at")) or _now_utc()
-
-        records.append({
-            "id":           doc.id,
-            "firestore_id": doc.id,
-            "name":         d.get("name") or d.get("name_en") or "",
-            "sku":          d.get("sku"),
-            "category":     d.get("category"),
-            "description":  d.get("description") or d.get("description_en"),
-            "unit_cost":    unit_cost,
-            "currency":     d.get("currency") or "EUR",
-            "unit_type":    d.get("unit_type") or "pcs",
-            "is_active":    is_active,
-            "created_at":   created_at,
-            "updated_at":   updated_at,
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(CrmProduct).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_products")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_quotation_items.py

@@ -1,84 +0,0 @@
-"""
-Phase 1 — Step 1.7: crm_quotation_items  (SQLite → Postgres)
-
-FK to crm_quotations(id) — quotations must be migrated first (step 1.6).
-FK enforcement suppressed via session_replication_role for the same reason
-as in migrate_crm_quotations (parent crm_customers not yet in PG).
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_quotation_items
-"""
-
-import asyncio
-import sys
-from decimal import Decimal
-
-from sqlalchemy import text
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmQuotationItem
-from migration.utils import open_sqlite, AsyncPgSession, log_run, pg_count
-
-SCRIPT = "migrate_crm_quotation_items"
-
-
-def _dec(val, default="0") -> Decimal:
-    try:
-        return Decimal(str(val)) if val is not None else Decimal(default)
-    except Exception:
-        return Decimal(default)
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall(
-        "SELECT * FROM crm_quotation_items ORDER BY quotation_id, sort_order"
-    )
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} crm_quotation_items rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        records.append({
-            "id":               r["id"],
-            "quotation_id":     r["quotation_id"],
-            "product_id":       r["product_id"],
-            "description":      r["description"],
-            "description_en":   r["description_en"],
-            "description_gr":   r["description_gr"],
-            "unit_type":        r["unit_type"] or "pcs",
-            "unit_cost":        _dec(r["unit_cost"]),
-            "discount_percent": _dec(r["discount_percent"]),
-            "vat_percent":      _dec(r["vat_percent"], "24"),
-            "quantity":         _dec(r["quantity"], "1"),
-            "line_total":       _dec(r["line_total"]),
-            "sort_order":       int(r["sort_order"]) if r["sort_order"] is not None else 0,
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(text("SET LOCAL session_replication_role = replica"))
-            stmt = pg_insert(CrmQuotationItem).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_quotation_items")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_quotations.py

@@ -1,118 +0,0 @@
-"""
-Phase 1 — Step 1.6: crm_quotations  (SQLite → Postgres)
-
-NOTE: crm_quotations has a FK to crm_customers(id).
-The customer rows DO NOT exist in Postgres yet (they migrate in Phase 2).
-To avoid FK violations, this script temporarily disables FK checks for the
-session using SET CONSTRAINTS ALL DEFERRED — but since customer_id is a real
-FK with ON DELETE CASCADE, we instead insert with the constraint deferred.
-
-Safer approach used here: insert with `customer_id` as-is and rely on the
-fact that crm_customers will be populated in Phase 2 before any service
-reads join across the two tables. The FK is not deferred — instead we disable
-the FK constraint enforcement for this transaction only via a session-level
-SET session_replication_role = replica; which suppresses FK checks in Postgres.
-We restore it immediately after the transaction.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_quotations
-"""
-
-import asyncio
-import sys
-from decimal import Decimal
-
-from sqlalchemy import text
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmQuotation
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, parse_json, log_run, pg_count
-
-SCRIPT = "migrate_crm_quotations"
-
-
-def _dec(val, default="0") -> Decimal:
-    try:
-        return Decimal(str(val)) if val is not None else Decimal(default)
-    except Exception:
-        return Decimal(default)
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM crm_quotations ORDER BY created_at")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} crm_quotations rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        records.append({
-            "id":                       r["id"],
-            "quotation_number":         r["quotation_number"],
-            "title":                    r["title"],
-            "subtitle":                 r["subtitle"],
-            "customer_id":              r["customer_id"],
-            "language":                 r["language"] or "en",
-            "status":                   r["status"] or "draft",
-            "order_type":               r["order_type"],
-            "shipping_method":          r["shipping_method"],
-            "estimated_shipping_date":  r["estimated_shipping_date"],
-            "global_discount_label":    r["global_discount_label"],
-            "global_discount_percent":  _dec(r["global_discount_percent"]),
-            "vat_percent":              _dec(r["vat_percent"], "24"),
-            "global_vat_percent":       _dec(r["global_vat_percent"], "24"),
-            "shipping_cost":            _dec(r["shipping_cost"]),
-            "shipping_cost_discount":   _dec(r["shipping_cost_discount"]),
-            "install_cost":             _dec(r["install_cost"]),
-            "install_cost_discount":    _dec(r["install_cost_discount"]),
-            "extras_label":             r["extras_label"],
-            "extras_cost":              _dec(r["extras_cost"]),
-            "comments":                 parse_json(r["comments"], default=[]),
-            "quick_notes":              parse_json(r["quick_notes"], default={}),
-            "subtotal_before_discount": _dec(r["subtotal_before_discount"]),
-            "global_discount_amount":   _dec(r["global_discount_amount"]),
-            "new_subtotal":             _dec(r["new_subtotal"]),
-            "vat_amount":               _dec(r["vat_amount"]),
-            "final_total":              _dec(r["final_total"]),
-            "nextcloud_pdf_path":       r["nextcloud_pdf_path"],
-            "nextcloud_pdf_url":        r["nextcloud_pdf_url"],
-            "client_org":               r["client_org"],
-            "client_name":              r["client_name"],
-            "client_location":          r["client_location"],
-            "client_phone":             r["client_phone"],
-            "client_email":             r["client_email"],
-            "is_legacy":                bool(r["is_legacy"]) if r["is_legacy"] is not None else False,
-            "legacy_date":              r["legacy_date"],
-            "legacy_pdf_path":          r["legacy_pdf_path"],
-            "created_at":               parse_dt(r["created_at"]),
-            "updated_at":               parse_dt(r["updated_at"]),
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            # Disable FK enforcement so we can insert before crm_customers arrives in Phase 2.
-            await session.execute(text("SET LOCAL session_replication_role = replica"))
-            stmt = pg_insert(CrmQuotation).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_quotations")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_crm_sync_state.py

@@ -1,54 +0,0 @@
-"""
-Phase 1 — Step 1.5: crm_sync_state  (SQLite → Postgres)
-
-Simple key/value table — small, no FK deps.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_crm_sync_state
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from crm.orm import CrmSyncState
-from migration.utils import open_sqlite, AsyncPgSession, log_run, pg_count
-
-SCRIPT = "migrate_crm_sync_state"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM crm_sync_state")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} crm_sync_state rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = [{"key": r["key"], "value": r["value"]} for r in rows]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(CrmSyncState).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["key"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "crm_sync_state")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_device_alerts.py

@@ -1,69 +0,0 @@
-"""
-Phase 1 — Step 1.4: device_alerts  (SQLite → Postgres)
-
-device_alerts is a "current state" table — one row per (device_serial, subsystem).
-The SQLite PK is (device_serial, subsystem); Postgres adds a BIGSERIAL surrogate PK
-with a unique constraint on the pair.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_device_alerts
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_device_alerts"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM device_alerts")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} device_alerts rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = [
-        {
-            "device_serial": r["device_serial"],
-            "subsystem":     r["subsystem"],
-            "state":         r["state"],
-            "message":       r["message"],
-            "updated_at":    parse_dt(r["updated_at"]),
-        }
-        for r in rows
-    ]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(
-                text("""
-                    INSERT INTO device_alerts (device_serial, subsystem, state, message, updated_at)
-                    VALUES (:device_serial, :subsystem, :state, :message, :updated_at)
-                    ON CONFLICT (device_serial, subsystem) DO NOTHING
-                """),
-                records,
-            )
-            dest_count = await pg_count(session, "device_alerts")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_device_logs.py

@@ -1,93 +0,0 @@
-"""
-Phase 1 — Step 1.12: device_logs  (SQLite → Postgres)
-
-Largest table — migrated in batches of 10,000 rows to avoid memory issues.
-device_logs is a partitioned table; rows route automatically to the correct
-monthly partition based on received_at.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_device_logs
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_device_logs"
-BATCH_SIZE = 10_000
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-
-    # Total count first
-    count_row = await sqlite.execute_fetchall("SELECT COUNT(*) FROM device_logs")
-    source_count = count_row[0][0]
-    print(f"Source (SQLite): {source_count} device_logs rows")
-
-    if source_count == 0:
-        await sqlite.close()
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    offset = 0
-    total_inserted = 0
-
-    while offset < source_count:
-        rows = await sqlite.execute_fetchall(
-            "SELECT * FROM device_logs ORDER BY received_at LIMIT ? OFFSET ?",
-            (BATCH_SIZE, offset),
-        )
-        if not rows:
-            break
-
-        records = [
-            {
-                "device_serial":    r["device_serial"],
-                "level":            r["level"],
-                "message":          r["message"],
-                "device_timestamp": r["device_timestamp"],
-                "received_at":      parse_dt(r["received_at"]),
-            }
-            for r in rows
-        ]
-
-        async with AsyncPgSession() as session:
-            async with session.begin():
-                await session.execute(
-                    text("""
-                        INSERT INTO device_logs
-                            (device_serial, level, message, device_timestamp, received_at)
-                        VALUES
-                            (:device_serial, :level, :message, :device_timestamp, :received_at)
-                    """),
-                    records,
-                )
-
-        total_inserted += len(records)
-        offset += BATCH_SIZE
-        pct = min(100, int(total_inserted / source_count * 100))
-        print(f"  {total_inserted}/{source_count} rows inserted ({pct}%)")
-
-    await sqlite.close()
-
-    # Final count verify
-    async with AsyncPgSession() as session:
-        dest_count = await pg_count(session, "device_logs")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_heartbeats.py

@@ -1,73 +0,0 @@
-"""
-Phase 1 — Step 1.11: heartbeats  (SQLite → Postgres)
-
-Raw-SQL table (no ORM model). BIGSERIAL PK — SQLite IDs not preserved.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_heartbeats
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_heartbeats"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM heartbeats ORDER BY received_at")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} heartbeats rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = [
-        {
-            "device_serial":    r["device_serial"],
-            "device_id":        r["device_id"],
-            "firmware_version": r["firmware_version"],
-            "ip_address":       r["ip_address"],
-            "gateway":          r["gateway"],
-            "uptime_ms":        r["uptime_ms"],
-            "uptime_display":   r["uptime_display"],
-            "received_at":      parse_dt(r["received_at"]),
-        }
-        for r in rows
-    ]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(
-                text("""
-                    INSERT INTO heartbeats
-                        (device_serial, device_id, firmware_version, ip_address,
-                         gateway, uptime_ms, uptime_display, received_at)
-                    VALUES
-                        (:device_serial, :device_id, :firmware_version, :ip_address,
-                         :gateway, :uptime_ms, :uptime_display, :received_at)
-                """),
-                records,
-            )
-            dest_count = await pg_count(session, "heartbeats")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_melody_drafts.py

@@ -1,66 +0,0 @@
-"""
-Phase 1 — Step 1.1: melody_drafts  (SQLite → Postgres)
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_melody_drafts
-"""
-
-import asyncio
-import json
-import sys
-
-from sqlalchemy import text
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from melodies.orm import MelodyDraft
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, parse_json, log_run, pg_count
-
-SCRIPT = "migrate_melody_drafts"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM melody_drafts")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} melody_drafts rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    for r in rows:
-        data_raw = r["data"]
-        # SQLite stores data as JSON text; Postgres column is JSONB
-        data = parse_json(data_raw, default={})
-
-        records.append({
-            "id":         r["id"],
-            "status":     r["status"] or "draft",
-            "data":       data,
-            "created_at": parse_dt(r["created_at"]),
-            "updated_at": parse_dt(r["updated_at"]),
-        })
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(MelodyDraft).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "melody_drafts")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_mfg_audit_log.py

@@ -1,67 +0,0 @@
-"""
-Phase 1 — Step 1.3: mfg_audit_log  (SQLite → Postgres)
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_mfg_audit_log
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy import text
-
-from migration.utils import open_sqlite, AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_mfg_audit_log"
-
-
-async def run() -> None:
-    sqlite = await open_sqlite()
-    rows = await sqlite.execute_fetchall("SELECT * FROM mfg_audit_log ORDER BY id")
-    await sqlite.close()
-
-    source_count = len(rows)
-    print(f"Source (SQLite): {source_count} mfg_audit_log rows")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    # mfg_audit_log uses a BIGSERIAL PK — we don't preserve SQLite integer IDs
-    # because the Postgres sequence will assign new ones. We insert in the same
-    # timestamp order so the audit trail remains coherent.
-    records = [
-        {
-            "timestamp":     parse_dt(r["timestamp"]),
-            "admin_user":    r["admin_user"],
-            "action":        r["action"],
-            "serial_number": r["serial_number"],
-            "detail":        r["detail"],
-        }
-        for r in rows
-    ]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            await session.execute(
-                text("""
-                    INSERT INTO mfg_audit_log (timestamp, admin_user, action, serial_number, detail)
-                    VALUES (:timestamp, :admin_user, :action, :serial_number, :detail)
-                """),
-                records,
-            )
-            dest_count = await pg_count(session, "mfg_audit_log")
-
-    if dest_count < source_count:
-        msg = f"Count mismatch: source={source_count}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_public_features.py

@@ -1,56 +0,0 @@
-"""
-Phase 2 — Step 2.2: public_features  (Firestore → Postgres)
-
-Reads the single 'admin_settings/public_features' doc from Firestore and
-flattens each field into a key/value row in public_features.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_public_features
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from settings.orm import PublicFeature
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, log_run, pg_count
-
-SCRIPT = "migrate_public_features"
-COLLECTION = "admin_settings"
-DOC_ID = "public_features"
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised — check service account path.", file=sys.stderr)
-        sys.exit(1)
-
-    doc = fs.collection(COLLECTION).document(DOC_ID).get()
-    if not doc.exists:
-        print("No public_features document found in Firestore — skipping.")
-        await log_run(SCRIPT, 0, 0, notes="source doc not found")
-        return
-
-    data = doc.to_dict()
-    source_count = len(data)
-    print(f"Source (Firestore): {source_count} fields in {COLLECTION}/{DOC_ID}")
-
-    records = [{"key": k, "value": v} for k, v in data.items()]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(PublicFeature).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["key"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "public_features")
-
-    print(f"Postgres public_features: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_settings.py

@@ -1,56 +0,0 @@
-"""
-Phase 2 — Step 2.1: console_settings  (Firestore → Postgres)
-
-Reads the single 'admin_settings/melody_settings' doc from Firestore and
-flattens each field into a key/value row in console_settings.
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_settings
-"""
-
-import asyncio
-import sys
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from settings.orm import ConsoleSetting
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, log_run, pg_count
-
-SCRIPT = "migrate_settings"
-COLLECTION = "admin_settings"
-DOC_ID = "melody_settings"
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised — check service account path.", file=sys.stderr)
-        sys.exit(1)
-
-    doc = fs.collection(COLLECTION).document(DOC_ID).get()
-    if not doc.exists:
-        print("No melody_settings document found in Firestore — skipping.")
-        await log_run(SCRIPT, 0, 0, notes="source doc not found")
-        return
-
-    data = doc.to_dict()
-    source_count = len(data)
-    print(f"Source (Firestore): {source_count} fields in {COLLECTION}/{DOC_ID}")
-
-    records = [{"key": k, "value": v} for k, v in data.items()]
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(ConsoleSetting).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["key"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "console_settings")
-
-    print(f"Postgres console_settings: {dest_count} rows ✓")
-    await log_run(SCRIPT, source_count, dest_count)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/migrate_staff.py

@@ -1,143 +0,0 @@
-"""
-Phase 3 — Step 3.1: admin_users (Firestore → Postgres staff table)
-
-Reads every document in the 'admin_users' Firestore collection and inserts
-a matching row into the Postgres 'staff' table.
-
-Key transformations:
-  - Legacy role names mapped to canonical roles (superadmin→sysadmin, etc.)
-  - permissions=None stored as JSONB null (sysadmin/admin have no permission map)
-  - ui_prefs column NOT migrated (not part of the Postgres schema — dropped)
-  - Firestore doc ID preserved as staff.id and staff.firestore_id
-  - created_at/updated_at default to now() if missing from Firestore doc
-
-Run on VPS:
-    docker compose exec backend python -m migration.migrate_staff
-"""
-
-import asyncio
-import sys
-from datetime import datetime, timezone
-
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-from staff.orm import Staff
-from shared.firebase import init_firebase, get_db as get_firestore
-from migration.utils import AsyncPgSession, parse_dt, log_run, pg_count
-
-SCRIPT = "migrate_staff"
-COLLECTION = "admin_users"
-
-_ROLE_MAP = {
-    "superadmin":    "sysadmin",
-    "melody_editor": "editor",
-    "device_manager": "editor",
-    "user_manager":  "editor",
-    "viewer":        "user",
-    # canonical roles pass through unchanged
-    "sysadmin": "sysadmin",
-    "admin":    "admin",
-    "editor":   "editor",
-    "user":     "user",
-    "staff":    "user",
-}
-
-
-def _now_utc() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _coerce_dt(val) -> datetime | None:
-    if val is None:
-        return None
-    if isinstance(val, datetime):
-        return val.replace(tzinfo=timezone.utc) if val.tzinfo is None else val
-    return parse_dt(str(val))
-
-
-async def run() -> None:
-    init_firebase()
-    fs = get_firestore()
-    if fs is None:
-        print("ERROR: Firebase not initialised.", file=sys.stderr)
-        sys.exit(1)
-
-    docs = list(fs.collection(COLLECTION).stream())
-    source_count = len(docs)
-    print(f"Source (Firestore): {source_count} admin_users documents")
-
-    if source_count == 0:
-        print("Nothing to migrate.")
-        await log_run(SCRIPT, 0, 0, notes="source empty")
-        return
-
-    records = []
-    skipped = 0
-
-    for doc in docs:
-        d = doc.to_dict()
-
-        hashed_password = d.get("hashed_password") or ""
-        if not hashed_password:
-            print(f"  WARNING: {doc.id} ({d.get('email')}) has no hashed_password — skipping",
-                  file=sys.stderr)
-            skipped += 1
-            continue
-
-        email = d.get("email") or ""
-        if not email:
-            print(f"  WARNING: {doc.id} has no email — skipping", file=sys.stderr)
-            skipped += 1
-            continue
-
-        raw_role = d.get("role") or "user"
-        role = _ROLE_MAP.get(raw_role, "user")
-
-        # sysadmin/admin have no permission map
-        permissions = d.get("permissions")
-        if role in ("sysadmin", "admin"):
-            permissions = None
-
-        now = _now_utc()
-        records.append({
-            "id":              doc.id,
-            "firestore_id":    doc.id,
-            "email":           email,
-            "name":            d.get("name") or "",
-            "role":            role,
-            "permissions":     permissions,
-            "hashed_password": hashed_password,
-            "is_active":       bool(d.get("is_active", True)),
-            "created_at":      _coerce_dt(d.get("created_at")) or now,
-            "updated_at":      _coerce_dt(d.get("updated_at")) or now,
-        })
-
-    actual_source = source_count - skipped
-    print(f"  {skipped} skipped (missing email or password), {actual_source} to insert")
-
-    if not records:
-        print("Nothing to insert after filtering.")
-        await log_run(SCRIPT, source_count, 0, success=False,
-                      notes="all docs skipped — missing required fields")
-        sys.exit(1)
-
-    async with AsyncPgSession() as session:
-        async with session.begin():
-            stmt = pg_insert(Staff).values(records)
-            stmt = stmt.on_conflict_do_nothing(index_elements=["id"])
-            await session.execute(stmt)
-            dest_count = await pg_count(session, "staff")
-
-    if dest_count < actual_source:
-        msg = f"Count mismatch: expected>={actual_source}  postgres={dest_count}"
-        print(f"ERROR: {msg}", file=sys.stderr)
-        await log_run(SCRIPT, source_count, dest_count, success=False, notes=msg)
-        sys.exit(1)
-
-    print(f"Postgres: {dest_count} rows ✓")
-    note = f"{skipped} skipped (missing fields)" if skipped else None
-    await log_run(SCRIPT, source_count, dest_count, notes=note)
-
-
-if __name__ == "__main__":
-    asyncio.run(run())

backend/migration/utils.py

@@ -1,116 +0,0 @@
-"""
-Shared helpers for all Phase 1 SQLite → Postgres migration scripts.
-
-Usage in each script:
-    from migration.utils import open_sqlite, get_pg, log_run, parse_dt, parse_json
-"""
-
-import json
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-import aiosqlite
-from sqlalchemy import text
-from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
-
-from config import settings
-
-
-# ── SQLite ────────────────────────────────────────────────────────────────────
-
-async def open_sqlite() -> aiosqlite.Connection:
-    """Open the SQLite database (read-only; no writes during migration)."""
-    db_path = Path(settings.sqlite_db_path)
-    if not db_path.exists():
-        print(f"ERROR: SQLite database not found at {db_path.resolve()}", file=sys.stderr)
-        sys.exit(1)
-    conn = await aiosqlite.connect(str(db_path))
-    conn.row_factory = aiosqlite.Row
-    return conn
-
-
-# ── Postgres ──────────────────────────────────────────────────────────────────
-
-def _make_pg_session() -> async_sessionmaker:
-    engine = create_async_engine(settings.database_url, pool_size=5, echo=False)
-    return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
-
-
-AsyncPgSession = _make_pg_session()
-
-
-# ── Type helpers ──────────────────────────────────────────────────────────────
-
-def parse_dt(value: str | None) -> datetime | None:
-    """Parse a SQLite TEXT timestamp → timezone-aware datetime (UTC)."""
-    if not value:
-        return None
-    for fmt in (
-        "%Y-%m-%dT%H:%M:%S.%f",
-        "%Y-%m-%dT%H:%M:%S",
-        "%Y-%m-%d %H:%M:%S.%f",
-        "%Y-%m-%d %H:%M:%S",
-        "%Y-%m-%d",
-    ):
-        try:
-            dt = datetime.strptime(value, fmt)
-            return dt.replace(tzinfo=timezone.utc)
-        except ValueError:
-            continue
-    # ISO format with offset — let fromisoformat handle it
-    try:
-        dt = datetime.fromisoformat(value)
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=timezone.utc)
-        return dt
-    except ValueError:
-        pass
-    print(f"WARNING: could not parse timestamp {value!r} — using now()", file=sys.stderr)
-    return datetime.now(timezone.utc)
-
-
-def parse_json(value: str | None, default=None):
-    """Parse a SQLite TEXT JSON column → Python object."""
-    if value is None:
-        return default
-    try:
-        return json.loads(value)
-    except (json.JSONDecodeError, TypeError):
-        return default
-
-
-# ── Migration run log ─────────────────────────────────────────────────────────
-
-async def log_run(
-    script_name: str,
-    source_rows: int,
-    dest_rows: int,
-    success: bool = True,
-    notes: str | None = None,
-) -> None:
-    """Insert a row into _migration_runs recording this script's execution."""
-    async with AsyncPgSession() as session:
-        await session.execute(
-            text("""
-                INSERT INTO _migration_runs
-                    (script_name, ran_at, source_rows, dest_rows, success, notes)
-                VALUES
-                    (:script_name, now(), :source_rows, :dest_rows, :success, :notes)
-            """),
-            {
-                "script_name": script_name,
-                "source_rows": source_rows,
-                "dest_rows":   dest_rows,
-                "success":     "ok" if success else "error",
-                "notes":       notes,
-            },
-        )
-        await session.commit()
-
-
-# ── Count helper ──────────────────────────────────────────────────────────────
-
-async def pg_count(session: AsyncSession, table: str) -> int:
-    row = await session.execute(text(f"SELECT COUNT(*) FROM {table}"))
-    return row.scalar()

backend/mqtt/auth.py

@@ -18,7 +18,7 @@ User types handled:
 - Kiosk users (e.g. "PV25L22BP01R01-kiosk"):
     Same HMAC auth derived from the full kiosk username.
     ACL: allowed to access topics of their base device (suffix stripped).
-- admin, bonamin, NodeRED, and other non-device users:
+- bonamin, NodeRED, and other non-device users:
     These connect via the passwd file backend (go-auth file backend).
     They never reach this HTTP backend — go-auth resolves them first.
     The ACL endpoint below handles them defensively anyway (superuser list).
@@ -35,7 +35,7 @@ LEGACY_PASSWORD = "vesper"
 
 # Users authenticated via passwd file (go-auth file backend).
 # If they somehow reach the HTTP ACL endpoint, grant full access.
-SUPERUSERS = {"admin", "bonamin", "NodeRED"}
+SUPERUSERS = {"bonamin", "NodeRED"}
 
 
 def _derive_password(username: str) -> str:
@@ -86,7 +86,7 @@ async def mqtt_auth_user(
                or kiosk variant: "PV25L22BP01R01-kiosk"
     Password = HMAC-derived (new firmware) or "vesper" (legacy firmware)
 
-    Note: admin, bonamin and NodeRED authenticate via the go-auth passwd file backend
+    Note: bonamin and NodeRED authenticate via the go-auth passwd file backend
     and never reach this endpoint.
     """
     if _is_valid_password(username, password):

backend/mqtt/client.py

@@ -26,7 +26,7 @@ class MqttManager:
 
         self._client = paho_mqtt.Client(
             callback_api_version=paho_mqtt.CallbackAPIVersion.VERSION2,
-            client_id=settings.mqtt_client_id,
+            client_id="bellsystems-admin-panel",
             clean_session=True,
         )
 
@@ -64,8 +64,6 @@ class MqttManager:
             client.subscribe([
                 ("vesper/+/data", 1),
                 ("vesper/+/status/heartbeat", 1),
-                ("vesper/+/status/alerts", 1),
-                ("vesper/+/status/info", 0),
                 ("vesper/+/logs", 1),
             ])
         else:

backend/mqtt/database.py

@@ -0,0 +1,254 @@
+import aiosqlite
+import asyncio
+import json
+import logging
+from datetime import datetime, timedelta, timezone
+from config import settings
+
+logger = logging.getLogger("mqtt.database")
+
+_db: aiosqlite.Connection | None = None
+
+SCHEMA_STATEMENTS = [
+    """CREATE TABLE IF NOT EXISTS device_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        device_serial TEXT NOT NULL,
+        level TEXT NOT NULL,
+        message TEXT NOT NULL,
+        device_timestamp INTEGER,
+        received_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )""",
+    """CREATE TABLE IF NOT EXISTS heartbeats (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        device_serial TEXT NOT NULL,
+        device_id TEXT,
+        firmware_version TEXT,
+        ip_address TEXT,
+        gateway TEXT,
+        uptime_ms INTEGER,
+        uptime_display TEXT,
+        received_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )""",
+    """CREATE TABLE IF NOT EXISTS commands (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        device_serial TEXT NOT NULL,
+        command_name TEXT NOT NULL,
+        command_payload TEXT,
+        status TEXT NOT NULL DEFAULT 'pending',
+        response_payload TEXT,
+        sent_at TEXT NOT NULL DEFAULT (datetime('now')),
+        responded_at TEXT
+    )""",
+    "CREATE INDEX IF NOT EXISTS idx_logs_serial_time ON device_logs(device_serial, received_at)",
+    "CREATE INDEX IF NOT EXISTS idx_logs_level ON device_logs(level)",
+    "CREATE INDEX IF NOT EXISTS idx_heartbeats_serial_time ON heartbeats(device_serial, received_at)",
+    "CREATE INDEX IF NOT EXISTS idx_commands_serial_time ON commands(device_serial, sent_at)",
+    "CREATE INDEX IF NOT EXISTS idx_commands_status ON commands(status)",
+    # Melody drafts table
+    """CREATE TABLE IF NOT EXISTS melody_drafts (
+        id TEXT PRIMARY KEY,
+        status TEXT NOT NULL DEFAULT 'draft',
+        data TEXT NOT NULL,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )""",
+    "CREATE INDEX IF NOT EXISTS idx_melody_drafts_status ON melody_drafts(status)",
+    # Built melodies table (local melody builder)
+    """CREATE TABLE IF NOT EXISTS built_melodies (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        pid TEXT NOT NULL,
+        steps TEXT NOT NULL,
+        binary_path TEXT,
+        progmem_code TEXT,
+        assigned_melody_ids TEXT NOT NULL DEFAULT '[]',
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )""",
+    # Manufacturing audit log
+    """CREATE TABLE IF NOT EXISTS mfg_audit_log (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+        admin_user TEXT NOT NULL,
+        action TEXT NOT NULL,
+        serial_number TEXT,
+        detail TEXT
+    )""",
+    "CREATE INDEX IF NOT EXISTS idx_mfg_audit_time ON mfg_audit_log(timestamp)",
+    "CREATE INDEX IF NOT EXISTS idx_mfg_audit_action ON mfg_audit_log(action)",
+]
+
+
+async def init_db():
+    global _db
+    _db = await aiosqlite.connect(settings.sqlite_db_path)
+    _db.row_factory = aiosqlite.Row
+    for stmt in SCHEMA_STATEMENTS:
+        await _db.execute(stmt)
+    await _db.commit()
+    logger.info(f"SQLite database initialized at {settings.sqlite_db_path}")
+
+
+async def close_db():
+    global _db
+    if _db:
+        await _db.close()
+        _db = None
+
+
+async def get_db() -> aiosqlite.Connection:
+    if _db is None:
+        await init_db()
+    return _db
+
+
+# --- Insert Operations ---
+
+async def insert_log(device_serial: str, level: str, message: str,
+                     device_timestamp: int | None = None):
+    db = await get_db()
+    cursor = await db.execute(
+        "INSERT INTO device_logs (device_serial, level, message, device_timestamp) VALUES (?, ?, ?, ?)",
+        (device_serial, level, message, device_timestamp)
+    )
+    await db.commit()
+    return cursor.lastrowid
+
+
+async def insert_heartbeat(device_serial: str, device_id: str,
+                           firmware_version: str, ip_address: str,
+                           gateway: str, uptime_ms: int, uptime_display: str):
+    db = await get_db()
+    cursor = await db.execute(
+        """INSERT INTO heartbeats
+           (device_serial, device_id, firmware_version, ip_address, gateway, uptime_ms, uptime_display)
+           VALUES (?, ?, ?, ?, ?, ?, ?)""",
+        (device_serial, device_id, firmware_version, ip_address, gateway, uptime_ms, uptime_display)
+    )
+    await db.commit()
+    return cursor.lastrowid
+
+
+async def insert_command(device_serial: str, command_name: str,
+                         command_payload: dict) -> int:
+    db = await get_db()
+    cursor = await db.execute(
+        "INSERT INTO commands (device_serial, command_name, command_payload) VALUES (?, ?, ?)",
+        (device_serial, command_name, json.dumps(command_payload))
+    )
+    await db.commit()
+    return cursor.lastrowid
+
+
+async def update_command_response(command_id: int, status: str,
+                                  response_payload: dict | None = None):
+    db = await get_db()
+    await db.execute(
+        """UPDATE commands SET status = ?, response_payload = ?,
+           responded_at = datetime('now') WHERE id = ?""",
+        (status, json.dumps(response_payload) if response_payload else None, command_id)
+    )
+    await db.commit()
+
+
+# --- Query Operations ---
+
+async def get_logs(device_serial: str, level: str | None = None,
+                   search: str | None = None,
+                   limit: int = 100, offset: int = 0) -> tuple[list, int]:
+    db = await get_db()
+    where_clauses = ["device_serial = ?"]
+    params: list = [device_serial]
+
+    if level:
+        where_clauses.append("level = ?")
+        params.append(level)
+    if search:
+        where_clauses.append("message LIKE ?")
+        params.append(f"%{search}%")
+
+    where = " AND ".join(where_clauses)
+
+    count_row = await db.execute_fetchall(
+        f"SELECT COUNT(*) as cnt FROM device_logs WHERE {where}", params
+    )
+    total = count_row[0][0]
+
+    rows = await db.execute_fetchall(
+        f"SELECT * FROM device_logs WHERE {where} ORDER BY received_at DESC LIMIT ? OFFSET ?",
+        params + [limit, offset]
+    )
+    return [dict(r) for r in rows], total
+
+
+async def get_heartbeats(device_serial: str, limit: int = 100,
+                         offset: int = 0) -> tuple[list, int]:
+    db = await get_db()
+    count_row = await db.execute_fetchall(
+        "SELECT COUNT(*) FROM heartbeats WHERE device_serial = ?", (device_serial,)
+    )
+    total = count_row[0][0]
+    rows = await db.execute_fetchall(
+        "SELECT * FROM heartbeats WHERE device_serial = ? ORDER BY received_at DESC LIMIT ? OFFSET ?",
+        (device_serial, limit, offset)
+    )
+    return [dict(r) for r in rows], total
+
+
+async def get_commands(device_serial: str, limit: int = 100,
+                       offset: int = 0) -> tuple[list, int]:
+    db = await get_db()
+    count_row = await db.execute_fetchall(
+        "SELECT COUNT(*) FROM commands WHERE device_serial = ?", (device_serial,)
+    )
+    total = count_row[0][0]
+    rows = await db.execute_fetchall(
+        "SELECT * FROM commands WHERE device_serial = ? ORDER BY sent_at DESC LIMIT ? OFFSET ?",
+        (device_serial, limit, offset)
+    )
+    return [dict(r) for r in rows], total
+
+
+async def get_latest_heartbeats() -> list:
+    db = await get_db()
+    rows = await db.execute_fetchall("""
+        SELECT h.* FROM heartbeats h
+        INNER JOIN (
+            SELECT device_serial, MAX(received_at) as max_time
+            FROM heartbeats GROUP BY device_serial
+        ) latest ON h.device_serial = latest.device_serial
+        AND h.received_at = latest.max_time
+    """)
+    return [dict(r) for r in rows]
+
+
+async def get_pending_command(device_serial: str) -> dict | None:
+    db = await get_db()
+    rows = await db.execute_fetchall(
+        """SELECT * FROM commands WHERE device_serial = ? AND status = 'pending'
+           ORDER BY sent_at DESC LIMIT 1""",
+        (device_serial,)
+    )
+    return dict(rows[0]) if rows else None
+
+
+# --- Cleanup ---
+
+async def purge_old_data(retention_days: int | None = None):
+    days = retention_days or settings.mqtt_data_retention_days
+    cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
+    db = await get_db()
+    await db.execute("DELETE FROM device_logs WHERE received_at < ?", (cutoff,))
+    await db.execute("DELETE FROM heartbeats WHERE received_at < ?", (cutoff,))
+    await db.execute("DELETE FROM commands WHERE sent_at < ?", (cutoff,))
+    await db.commit()
+    logger.info(f"Purged MQTT data older than {days} days")
+
+
+async def purge_loop():
+    while True:
+        await asyncio.sleep(86400)
+        try:
+            await purge_old_data()
+        except Exception as e:
+            logger.error(f"Purge failed: {e}")

backend/mqtt/logger.py

@@ -1,5 +1,5 @@
 import logging
-import database as db
+from mqtt import database as db
 
 logger = logging.getLogger("mqtt.logger")
 
@@ -18,10 +18,6 @@ async def handle_message(serial: str, topic_type: str, payload: dict):
     try:
         if topic_type == "status/heartbeat":
             await _handle_heartbeat(serial, payload)
-        elif topic_type == "status/alerts":
-            await _handle_alerts(serial, payload)
-        elif topic_type == "status/info":
-            await _handle_info(serial, payload)
         elif topic_type == "logs":
             await _handle_log(serial, payload)
         elif topic_type == "data":
@@ -33,8 +29,6 @@ async def handle_message(serial: str, topic_type: str, payload: dict):
 
 
 async def _handle_heartbeat(serial: str, payload: dict):
-    # Store silently — do not log as a visible event.
-    # The console surfaces an alert only when the device goes silent (no heartbeat for 90s).
     inner = payload.get("payload", {})
     await db.insert_heartbeat(
         device_serial=serial,
@@ -61,31 +55,6 @@ async def _handle_log(serial: str, payload: dict):
     )
 
 
-async def _handle_alerts(serial: str, payload: dict):
-    subsystem = payload.get("subsystem", "")
-    state = payload.get("state", "")
-    if not subsystem or not state:
-        logger.warning(f"Malformed alert payload from {serial}: {payload}")
-        return
-
-    if state == "CLEARED":
-        await db.delete_alert(serial, subsystem)
-    else:
-        await db.upsert_alert(serial, subsystem, state, payload.get("msg"))
-
-
-async def _handle_info(serial: str, payload: dict):
-    event_type = payload.get("type", "")
-    data = payload.get("payload", {})
-
-    if event_type == "playback_started":
-        logger.debug(f"{serial}: playback started — melody_uid={data.get('melody_uid')}")
-    elif event_type == "playback_stopped":
-        logger.debug(f"{serial}: playback stopped")
-    else:
-        logger.debug(f"{serial}: info event '{event_type}'")
-
-
 async def _handle_data_response(serial: str, payload: dict):
     status = payload.get("status", "")
 

backend/mqtt/models.py

@@ -84,15 +84,3 @@ class CommandSendResponse(BaseModel):
     success: bool
     command_id: int
     message: str
-
-
-class DeviceAlertEntry(BaseModel):
-    device_serial: str
-    subsystem: str
-    state: str
-    message: Optional[str] = None
-    updated_at: str
-
-
-class DeviceAlertsResponse(BaseModel):
-    alerts: List[DeviceAlertEntry]

backend/mqtt/router.py

@@ -8,7 +8,7 @@ from mqtt.models import (
     CommandListResponse, HeartbeatEntry,
 )
 from mqtt.client import mqtt_manager
-import database as db
+from mqtt import database as db
 from datetime import datetime, timezone
 
 router = APIRouter(prefix="/api/mqtt", tags=["mqtt"])
@@ -129,29 +129,27 @@ async def mqtt_websocket(websocket: WebSocket):
 
     try:
         from auth.utils import decode_access_token
-        from sqlalchemy import select
+        from shared.firebase import get_db
-        from database.postgres import AsyncSessionLocal
-        from staff.orm import Staff
-
         payload = decode_access_token(token)
         role = payload.get("role", "")
 
         # sysadmin and admin always have MQTT access
         if role not in ("sysadmin", "admin"):
+            # Check MQTT permission for editor/user
             user_sub = payload.get("sub", "")
-            async with AsyncSessionLocal() as session:
+            db_inst = get_db()
-                result = await session.execute(
+            if db_inst:
-                    select(Staff).where(Staff.id == user_sub).limit(1)
+                doc = db_inst.collection("admin_users").document(user_sub).get()
-                )
+                if doc.exists:
-                staff = result.scalar_one_or_none()
+                    perms = doc.to_dict().get("permissions", {})
-
+                    if not perms.get("mqtt", False):
-            if staff is None:
+                        await websocket.close(code=4003, reason="MQTT access denied")
-                await websocket.close(code=4003, reason="User not found")
+                        return
-                return
+                else:
-
+                    await websocket.close(code=4003, reason="User not found")
-            perms = staff.permissions or {}
+                    return
-            if not perms.get("mqtt", {}).get("access", False):
+            else:
-                await websocket.close(code=4003, reason="MQTT access denied")
+                await websocket.close(code=4003, reason="Service unavailable")
                 return
     except Exception:
         await websocket.close(code=4001, reason="Invalid token")

backend/notes/models.py

@@ -1,100 +0,0 @@
-from pydantic import BaseModel, Field, field_validator
-from typing import Optional, List
-from uuid import UUID
-from datetime import datetime
-
-VALID_TYPES      = {"note", "issue"}
-VALID_STATUSES   = {"open", "researching", "resolved"}
-VALID_SEVERITIES = {"low", "medium", "high", "critical"}
-VALID_CATEGORIES = {"technical", "install_support", "general"}
-VALID_ENTITIES   = {"device", "app_user", "customer"}
-
-
-class EntryLinkIn(BaseModel):
-    entity_type: str
-    entity_id: str
-
-    @field_validator("entity_type")
-    @classmethod
-    def check_entity_type(cls, v):
-        if v not in VALID_ENTITIES:
-            raise ValueError(f"entity_type must be one of {VALID_ENTITIES}")
-        return v
-
-
-class EntryCreate(BaseModel):
-    type: str
-    title: str = Field(..., max_length=500)
-    body: Optional[str] = None
-    status: Optional[str] = None
-    severity: Optional[str] = None
-    category: Optional[str] = None
-    links: List[EntryLinkIn] = []
-
-    @field_validator("type")
-    @classmethod
-    def check_type(cls, v):
-        if v not in VALID_TYPES:
-            raise ValueError(f"type must be one of {VALID_TYPES}")
-        return v
-
-    @field_validator("status")
-    @classmethod
-    def check_status(cls, v):
-        if v is not None and v not in VALID_STATUSES:
-            raise ValueError(f"status must be one of {VALID_STATUSES}")
-        return v
-
-    @field_validator("severity")
-    @classmethod
-    def check_severity(cls, v):
-        if v is not None and v not in VALID_SEVERITIES:
-            raise ValueError(f"severity must be one of {VALID_SEVERITIES}")
-        return v
-
-    @field_validator("category")
-    @classmethod
-    def check_category(cls, v):
-        if v is not None and v not in VALID_CATEGORIES:
-            raise ValueError(f"category must be one of {VALID_CATEGORIES}")
-        return v
-
-
-class EntryUpdate(BaseModel):
-    title: Optional[str] = Field(None, max_length=500)
-    body: Optional[str] = None
-    status: Optional[str] = None
-    severity: Optional[str] = None
-    category: Optional[str] = None
-
-
-class EntryLinkOut(BaseModel):
-    id: UUID
-    entity_type: str
-    entity_id: str
-    model_config = {"from_attributes": True}
-
-
-class EntryOut(BaseModel):
-    id: UUID
-    type: str
-    title: str
-    body: Optional[str]
-    status: Optional[str]
-    severity: Optional[str]
-    category: Optional[str]
-    author_id: str
-    author_name: Optional[str]
-    created_at: datetime
-    updated_at: datetime
-    links: List[EntryLinkOut] = []
-    model_config = {"from_attributes": True}
-
-
-class EntryListResponse(BaseModel):
-    data: List[EntryOut]
-    pagination: dict
-
-
-class LinksReplaceIn(BaseModel):
-    links: List[EntryLinkIn]

backend/notes/orm.py

@@ -1,42 +0,0 @@
-import uuid
-from datetime import datetime, timezone
-from sqlalchemy import Column, String, Text, DateTime, ForeignKey, UniqueConstraint
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.orm import relationship
-from database.postgres import Base
-
-
-def _now():
-    return datetime.now(timezone.utc)
-
-
-class Entry(Base):
-    __tablename__ = "crm_entries"
-
-    id          = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    type        = Column(String(10), nullable=False)          # 'note' | 'issue'
-    title       = Column(String(500), nullable=False)
-    body        = Column(Text, nullable=True)
-    status      = Column(String(20), nullable=True)           # null for notes; open/researching/resolved for issues
-    severity    = Column(String(10), nullable=True)           # null | low | medium | high | critical
-    category    = Column(String(30), nullable=True)           # null for notes; technical | install_support | general
-    author_id   = Column(String(128), nullable=False)         # staff user ID from JWT
-    author_name = Column(String(255), nullable=True)          # denormalized for display
-    created_at  = Column(DateTime(timezone=True), nullable=False, default=_now)
-    updated_at  = Column(DateTime(timezone=True), nullable=False, default=_now, onupdate=_now)
-
-    links = relationship("EntryLink", back_populates="entry", cascade="all, delete-orphan", lazy="noload")
-
-
-class EntryLink(Base):
-    __tablename__ = "crm_entry_links"
-    __table_args__ = (
-        UniqueConstraint("entry_id", "entity_type", "entity_id"),
-    )
-
-    id          = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    entry_id    = Column(UUID(as_uuid=True), ForeignKey("crm_entries.id", ondelete="CASCADE"), nullable=False)
-    entity_type = Column(String(20), nullable=False)   # 'device' | 'app_user' | 'customer'
-    entity_id   = Column(String(128), nullable=False)  # Firestore ID or Postgres UUID as string
-
-    entry = relationship("Entry", back_populates="links")

backend/notes/router.py

@@ -1,93 +0,0 @@
-from fastapi import APIRouter, Depends, Query
-from uuid import UUID
-from sqlalchemy.ext.asyncio import AsyncSession
-from database.postgres import get_pg_session
-from auth.dependencies import require_permission
-from auth.models import TokenPayload
-from notes import service
-from notes.models import EntryCreate, EntryUpdate, EntryOut, EntryListResponse, LinksReplaceIn
-from shared.audit import log_action
-
-router = APIRouter(prefix="/api/notes", tags=["notes"])
-
-
-@router.get("", response_model=EntryListResponse)
-async def list_entries(
-    type:     str | None = Query(None),
-    status:   str | None = Query(None),
-    severity: str | None = Query(None),
-    category: str | None = Query(None),
-    page:     int = Query(1, ge=1),
-    limit:    int = Query(25, ge=1, le=100),
-    db:       AsyncSession = Depends(get_pg_session),
-    _user:    TokenPayload = Depends(require_permission("crm", "view")),
-):
-    rows, total = await service.list_entries(db, type, status, severity, category, page, limit)
-    return {"data": rows, "pagination": {"page": page, "limit": limit, "total": total}}
-
-
-@router.get("/by-entity/{entity_type}/{entity_id}", response_model=list[EntryOut])
-async def list_by_entity(
-    entity_type: str, entity_id: str,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return await service.list_entries_for_entity(db, entity_type, entity_id)
-
-
-@router.get("/{entry_id}", response_model=EntryOut)
-async def get_entry(
-    entry_id: UUID,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "view")),
-):
-    return await service.get_entry(db, entry_id)
-
-
-@router.post("", response_model=EntryOut, status_code=201)
-async def create_entry(
-    body:  EntryCreate,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "add")),
-):
-    entry = await service.create_entry(db, body, _user.sub, _user.name or _user.email)
-    await log_action(db, _user.sub, _user.name or _user.email, "CREATE", "note",
-                     str(entry.id), entry.title or entry.type)
-    return entry
-
-
-@router.patch("/{entry_id}", response_model=EntryOut)
-async def update_entry(
-    entry_id: UUID, body: EntryUpdate,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    entry = await service.update_entry(db, entry_id, body)
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "note",
-                     str(entry_id), entry.title or entry.type)
-    return entry
-
-
-@router.patch("/{entry_id}/links", response_model=EntryOut)
-async def replace_links(
-    entry_id: UUID, body: LinksReplaceIn,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "edit")),
-):
-    entry = await service.replace_links(db, entry_id, body.links)
-    await log_action(db, _user.sub, _user.name or _user.email, "UPDATE", "note",
-                     str(entry_id), entry.title or entry.type,
-                     meta={"action_detail": "links_updated"})
-    return entry
-
-
-@router.delete("/{entry_id}", status_code=204)
-async def delete_entry(
-    entry_id: UUID,
-    db:    AsyncSession = Depends(get_pg_session),
-    _user: TokenPayload = Depends(require_permission("crm", "delete")),
-):
-    entry = await service.get_entry(db, entry_id)
-    await service.delete_entry(db, entry_id)
-    await log_action(db, _user.sub, _user.name or _user.email, "DELETE", "note",
-                     str(entry_id), entry.title or entry.type if entry else str(entry_id))