update: Major Overhaul to all subsystems

2026-03-07 11:32:18 +02:00
parent 810e81b323
commit c62188fda6
107 changed files with 20414 additions and 929 deletions
--- a/backend/crm/comms_router.py
+++ b/backend/crm/comms_router.py
@@ -0,0 +1,417 @@
+import base64
+import json
+from fastapi import APIRouter, Depends, HTTPException, Query, Form, File, UploadFile
+from pydantic import BaseModel
+from typing import List, Optional
+
+from auth.models import TokenPayload
+from auth.dependencies import require_permission
+from config import settings
+from crm.models import CommCreate, CommUpdate, CommInDB, CommListResponse, MediaCreate, MediaDirection
+from crm import service
+from crm import email_sync
+from crm.mail_accounts import get_mail_accounts
+
+router = APIRouter(prefix="/api/crm/comms", tags=["crm-comms"])
+
+
+class EmailSendResponse(BaseModel):
+    entry: dict
+
+
+class EmailSyncResponse(BaseModel):
+    new_count: int
+
+
+class MailListResponse(BaseModel):
+    entries: list
+    total: int
+
+
+@router.get("/all", response_model=CommListResponse)
+async def list_all_comms(
+    type: Optional[str] = Query(None),
+    direction: Optional[str] = Query(None),
+    limit: int = Query(200, le=500),
+    _user: TokenPayload = Depends(require_permission("crm", "view")),
+):
+    entries = await service.list_all_comms(type=type, direction=direction, limit=limit)
+    return CommListResponse(entries=entries, total=len(entries))
+
+
+@router.get("", response_model=CommListResponse)
+async def list_comms(
+    customer_id: str = Query(...),
+    type: Optional[str] = Query(None),
+    direction: Optional[str] = Query(None),
+    _user: TokenPayload = Depends(require_permission("crm", "view")),
+):
+    entries = await service.list_comms(customer_id=customer_id, type=type, direction=direction)
+    return CommListResponse(entries=entries, total=len(entries))
+
+
+@router.post("", response_model=CommInDB, status_code=201)
+async def create_comm(
+    body: CommCreate,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    return await service.create_comm(body)
+
+
+@router.get("/email/all", response_model=MailListResponse)
+async def list_all_emails(
+    direction: Optional[str] = Query(None),
+    customers_only: bool = Query(False),
+    mailbox: Optional[str] = Query(None, description="sales|support|both|all or account key"),
+    limit: int = Query(500, le=1000),
+    _user: TokenPayload = Depends(require_permission("crm", "view")),
+):
+    """Return all email comms (all senders + unmatched), for the Mail page."""
+    selected_accounts = None
+    if mailbox and mailbox not in {"all", "both"}:
+      if mailbox == "sales":
+          selected_accounts = ["sales"]
+      elif mailbox == "support":
+          selected_accounts = ["support"]
+      else:
+          selected_accounts = [mailbox]
+    entries = await service.list_all_emails(
+        direction=direction,
+        customers_only=customers_only,
+        mail_accounts=selected_accounts,
+        limit=limit,
+    )
+    return MailListResponse(entries=entries, total=len(entries))
+
+
+@router.get("/email/accounts")
+async def list_mail_accounts(
+    _user: TokenPayload = Depends(require_permission("crm", "view")),
+):
+    accounts = get_mail_accounts()
+    return {
+        "accounts": [
+            {
+                "key": a["key"],
+                "label": a["label"],
+                "email": a["email"],
+                "sync_inbound": bool(a.get("sync_inbound")),
+                "allow_send": bool(a.get("allow_send")),
+            }
+            for a in accounts
+        ]
+    }
+
+
+@router.get("/email/check")
+async def check_new_emails(
+    _user: TokenPayload = Depends(require_permission("crm", "view")),
+):
+    """Lightweight check: returns how many emails are on the server vs. stored locally."""
+    return await email_sync.check_new_emails()
+
+
+# Email endpoints — must be before /{comm_id} wildcard routes
+@router.post("/email/send", response_model=EmailSendResponse)
+async def send_email_endpoint(
+    customer_id: Optional[str] = Form(None),
+    from_account: Optional[str] = Form(None),
+    to: str = Form(...),
+    subject: str = Form(...),
+    body: str = Form(...),
+    body_html: str = Form(""),
+    cc: str = Form("[]"),  # JSON-encoded list of strings
+    files: List[UploadFile] = File(default=[]),
+    user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    if not get_mail_accounts():
+        raise HTTPException(status_code=503, detail="SMTP not configured")
+    try:
+        cc_list: List[str] = json.loads(cc) if cc else []
+    except Exception:
+        cc_list = []
+
+    # Read all uploaded files into memory
+    file_attachments = []
+    for f in files:
+        content = await f.read()
+        mime_type = f.content_type or "application/octet-stream"
+        file_attachments.append((f.filename, content, mime_type))
+
+    from crm.email_sync import send_email
+    try:
+        entry = await send_email(
+            customer_id=customer_id or None,
+            from_account=from_account,
+            to=to,
+            subject=subject,
+            body=body,
+            body_html=body_html,
+            cc=cc_list,
+            sent_by=user.name or user.sub,
+            file_attachments=file_attachments if file_attachments else None,
+        )
+    except RuntimeError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    return EmailSendResponse(entry=entry)
+
+
+@router.post("/email/sync", response_model=EmailSyncResponse)
+async def sync_email_endpoint(
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    if not get_mail_accounts():
+        raise HTTPException(status_code=503, detail="IMAP not configured")
+    from crm.email_sync import sync_emails
+    new_count = await sync_emails()
+    return EmailSyncResponse(new_count=new_count)
+
+
+class SaveInlineRequest(BaseModel):
+    data_uri: str
+    filename: str
+    subfolder: str = "received_media"
+    mime_type: Optional[str] = None
+
+
+async def _resolve_customer_folder(customer_id: str) -> str:
+    """Return the Nextcloud folder_id for a customer (falls back to customer_id)."""
+    from shared.firebase import get_db as get_firestore
+    firestore_db = get_firestore()
+    doc = firestore_db.collection("crm_customers").document(customer_id).get()
+    if not doc.exists:
+        raise HTTPException(status_code=404, detail="Customer not found")
+    data = doc.to_dict()
+    return data.get("folder_id") or customer_id
+
+
+async def _upload_to_nc(folder_id: str, subfolder: str, filename: str,
+                        content: bytes, mime_type: str, customer_id: str,
+                        uploaded_by: str, tags: list[str]) -> dict:
+    from crm import nextcloud
+    target_folder = f"customers/{folder_id}/{subfolder}"
+    file_path = f"{target_folder}/{filename}"
+    await nextcloud.ensure_folder(target_folder)
+    await nextcloud.upload_file(file_path, content, mime_type)
+    media = await service.create_media(MediaCreate(
+        customer_id=customer_id,
+        filename=filename,
+        nextcloud_path=file_path,
+        mime_type=mime_type,
+        direction=MediaDirection.received,
+        tags=tags,
+        uploaded_by=uploaded_by,
+    ))
+    return {"ok": True, "media_id": media.id, "nextcloud_path": file_path}
+
+
+@router.post("/email/{comm_id}/save-inline")
+async def save_email_inline_image(
+    comm_id: str,
+    body: SaveInlineRequest,
+    user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    """Save an inline image (data-URI from email HTML body) to Nextcloud."""
+    comm = await service.get_comm(comm_id)
+    customer_id = comm.customer_id
+    if not customer_id:
+        raise HTTPException(status_code=400, detail="This email is not linked to a customer")
+
+    folder_id = await _resolve_customer_folder(customer_id)
+
+    # Parse data URI
+    data_uri = body.data_uri
+    mime_type = body.mime_type or "image/png"
+    if "," in data_uri:
+        header, encoded = data_uri.split(",", 1)
+        try:
+            mime_type = header.split(":")[1].split(";")[0]
+        except Exception:
+            pass
+    else:
+        encoded = data_uri
+    try:
+        content = base64.b64decode(encoded)
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid base64 data")
+
+    return await _upload_to_nc(
+        folder_id, body.subfolder, body.filename,
+        content, mime_type, customer_id,
+        user.name or user.sub, ["email-inline-image"],
+    )
+
+
+@router.post("/email/{comm_id}/save-attachment/{attachment_index}")
+async def save_email_attachment(
+    comm_id: str,
+    attachment_index: int,
+    filename: str = Form(...),
+    subfolder: str = Form("received_media"),
+    user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    """
+    Re-fetch a specific attachment from IMAP (by index in the email's attachment list)
+    and save it to the customer's Nextcloud media folder.
+    """
+    import asyncio
+    comm = await service.get_comm(comm_id)
+    customer_id = comm.customer_id
+    if not customer_id:
+        raise HTTPException(status_code=400, detail="This email is not linked to a customer")
+
+    ext_message_id = comm.ext_message_id
+    if not ext_message_id:
+        raise HTTPException(status_code=400, detail="No message ID stored for this email")
+
+    attachments_meta = comm.attachments or []
+    if attachment_index < 0 or attachment_index >= len(attachments_meta):
+        raise HTTPException(status_code=400, detail="Attachment index out of range")
+
+    att_meta = attachments_meta[attachment_index]
+    mime_type = att_meta.content_type or "application/octet-stream"
+    from crm.mail_accounts import account_by_key, account_by_email
+    account = account_by_key(comm.mail_account) or account_by_email(comm.from_addr)
+    if not account:
+        raise HTTPException(status_code=400, detail="Email account config not found for this message")
+
+    # Re-fetch from IMAP in executor
+    def _fetch_attachment():
+        import imaplib, email as _email
+        if account.get("imap_use_ssl"):
+            imap = imaplib.IMAP4_SSL(account["imap_host"], int(account["imap_port"]))
+        else:
+            imap = imaplib.IMAP4(account["imap_host"], int(account["imap_port"]))
+        imap.login(account["imap_username"], account["imap_password"])
+        imap.select(account.get("imap_inbox", "INBOX"))
+
+        # Search by Message-ID header
+        _, data = imap.search(None, f'HEADER Message-ID "{ext_message_id}"')
+        uids = data[0].split() if data[0] else []
+        if not uids:
+            raise ValueError(f"Message not found on IMAP server: {ext_message_id}")
+
+        _, msg_data = imap.fetch(uids[0], "(RFC822)")
+        raw = msg_data[0][1]
+        msg = _email.message_from_bytes(raw)
+        imap.logout()
+
+        # Walk attachments in order — find the one at attachment_index
+        found_idx = 0
+        for part in msg.walk():
+            cd = str(part.get("Content-Disposition", ""))
+            if "attachment" not in cd:
+                continue
+            if found_idx == attachment_index:
+                payload = part.get_payload(decode=True)
+                if payload is None:
+                    raise ValueError("Attachment payload is empty")
+                return payload
+            found_idx += 1
+
+        raise ValueError(f"Attachment index {attachment_index} not found in message")
+
+    loop = asyncio.get_event_loop()
+    try:
+        content = await loop.run_in_executor(None, _fetch_attachment)
+    except ValueError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=f"IMAP fetch failed: {e}")
+
+    folder_id = await _resolve_customer_folder(customer_id)
+    return await _upload_to_nc(
+        folder_id, subfolder, filename,
+        content, mime_type, customer_id,
+        user.name or user.sub, ["email-attachment"],
+    )
+
+
+class BulkDeleteRequest(BaseModel):
+    ids: List[str]
+
+
+class ToggleImportantRequest(BaseModel):
+    important: bool
+
+
+class ToggleReadRequest(BaseModel):
+    read: bool
+
+
+@router.post("/bulk-delete", status_code=200)
+async def bulk_delete_comms(
+    body: BulkDeleteRequest,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    # Try remote IMAP delete for email rows first (best-effort), then local delete.
+    for comm_id in body.ids:
+        try:
+            comm = await service.get_comm(comm_id)
+            if comm.type == "email" and comm.ext_message_id:
+                await email_sync.delete_remote_email(
+                    comm.ext_message_id,
+                    comm.mail_account,
+                    comm.from_addr,
+                )
+        except Exception:
+            # Keep delete resilient; local delete still proceeds.
+            pass
+    count = await service.delete_comms_bulk(body.ids)
+    return {"deleted": count}
+
+
+@router.patch("/{comm_id}/important", response_model=CommInDB)
+async def set_comm_important(
+    comm_id: str,
+    body: ToggleImportantRequest,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    return await service.set_comm_important(comm_id, body.important)
+
+
+@router.patch("/{comm_id}/read", response_model=CommInDB)
+async def set_comm_read(
+    comm_id: str,
+    body: ToggleReadRequest,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    try:
+        comm = await service.get_comm(comm_id)
+        if comm.type == "email" and comm.ext_message_id:
+            await email_sync.set_remote_read(
+                comm.ext_message_id,
+                comm.mail_account,
+                comm.from_addr,
+                body.read,
+            )
+    except Exception:
+        pass
+    return await service.set_comm_read(comm_id, body.read)
+
+
+@router.put("/{comm_id}", response_model=CommInDB)
+async def update_comm(
+    comm_id: str,
+    body: CommUpdate,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    return await service.update_comm(comm_id, body)
+
+
+@router.delete("/{comm_id}", status_code=204)
+async def delete_comm(
+    comm_id: str,
+    _user: TokenPayload = Depends(require_permission("crm", "edit")),
+):
+    try:
+        comm = await service.get_comm(comm_id)
+        if comm.type == "email" and comm.ext_message_id:
+            await email_sync.delete_remote_email(
+                comm.ext_message_id,
+                comm.mail_account,
+                comm.from_addr,
+            )
+    except Exception:
+        pass
+    await service.delete_comm(comm_id)