Add db migration and DNS dmarc entries

app/main.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from types import SimpleNamespace
 
 from fastapi import Depends, FastAPI, HTTPException, Request
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from sqlalchemy import desc, func, select
@@ -18,10 +18,11 @@ from app import __version__
 from app.auth import require_admin_csrf, require_dashboard_auth, require_homepage_token
 from app.config import Settings, configure_logging, get_settings
 from app.db import database_ok, get_db, init_db, session_scope
+from app.dns_policy import DomainDnsPolicy, collect_domain_dns_policy
 from app.homepage import domain_homepage_summary, domain_metrics, homepage_summary, latest_summary, resolve_date_range, traffic_distribution
 from app.inbox_locks import InboxRunLease, inbox_run_locks
 from app.jobs import import_jobs
-from app.models import Alert, DailyStat, InboxStatus, LLMReport, Record, Report, SkippedReportPayload, utcnow
+from app.models import Alert, AuthResult, DailyStat, DomainDnsSnapshot, InboxStatus, LLMReport, Record, Report, SkippedReportPayload, utcnow
 from app.scheduler import generate_open_posture_summaries, scheduler_ok, start_scheduler
 from app.schemas import BacklogRequest, ProcessNowRequest
 from app.message_processor import process_inbox
@@ -200,6 +201,12 @@ def _alert_view(alert: Alert, session: Session | None = None) -> SimpleNamespace
     details = _alert_details(alert)
     details = _infer_alert_report_details(session, alert, details)
     date_range = details.get("date_range") if isinstance(details.get("date_range"), dict) else {}
+    published_policy = details.get("published_policy") if isinstance(details.get("published_policy"), dict) else {}
+    receiver_action = details.get("receiver_action") if isinstance(details.get("receiver_action"), dict) else {}
+    effective_policy = published_policy.get("effective")
+    effective_source = published_policy.get("effective_source") or "p"
+    receiver_disposition = receiver_action.get("disposition") or details.get("disposition")
+    override_type = receiver_action.get("override_type")
     report_db_id = details.get("report_db_id")
     report_db_ids = details.get("report_db_ids") if isinstance(details.get("report_db_ids"), list) else []
     if not report_db_id and isinstance(details.get("report_db_ids"), list) and details["report_db_ids"]:
@@ -236,6 +243,11 @@ def _alert_view(alert: Alert, session: Session | None = None) -> SimpleNamespace
         report_db_id=report_db_id,
         report_db_ids=report_db_ids,
         source_ip=details.get("source_ip"),
+        published_policy=published_policy,
+        receiver_action=receiver_action,
+        published_policy_label=f"{effective_source}={effective_policy}" if effective_policy else None,
+        receiver_action_label=f"receiver {receiver_disposition}" if receiver_disposition else None,
+        policy_override_label=f"override {override_type}" if override_type else None,
         source_history=_source_history(session, alert.domain, details.get("source_ip"), alert.type, report_db_id) if session else None,
     )
 
@@ -380,6 +392,83 @@ def _record_auth_tooltip(record: Record, auth_type: str) -> str:
     return "; ".join(items) if items else f"No {auth_type.upper()} auth result domain reported."
 
 
+def _json_list(value: str | None) -> list:
+    try:
+        data = json.loads(value or "[]")
+    except json.JSONDecodeError:
+        return []
+    return data if isinstance(data, list) else []
+
+
+def _observed_dkim_selectors(session: Session, domain: str) -> list[str]:
+    rows = session.execute(
+        select(AuthResult.selector)
+        .select_from(AuthResult)
+        .join(Record)
+        .join(Report)
+        .where(
+            Report.domain == domain,
+            AuthResult.auth_type == "dkim",
+            AuthResult.selector.is_not(None),
+        )
+        .distinct()
+        .order_by(AuthResult.selector)
+    ).scalars().all()
+    return [row for row in rows if row]
+
+
+def _snapshot_model(domain: str, policy: DomainDnsPolicy) -> DomainDnsSnapshot:
+    return DomainDnsSnapshot(
+        domain=domain,
+        dmarc_record=policy.dmarc.raw,
+        dmarc_policy_p=policy.dmarc.p,
+        dmarc_policy_sp=policy.dmarc.sp,
+        dmarc_policy_pct=policy.dmarc.pct,
+        dmarc_adkim=policy.dmarc.adkim,
+        dmarc_aspf=policy.dmarc.aspf,
+        dmarc_fo=policy.dmarc.fo,
+        dmarc_rua=policy.dmarc.rua,
+        dmarc_ruf=policy.dmarc.ruf,
+        spf_record=policy.spf.raw,
+        spf_all=policy.spf.all_mechanism,
+        spf_includes_json=json.dumps(policy.spf.includes, sort_keys=True),
+        dkim_records_json=json.dumps([item.__dict__ for item in policy.dkim], sort_keys=True),
+        mx_records_json=json.dumps(policy.mx_records, sort_keys=True),
+        errors_json=json.dumps(policy.errors, sort_keys=True),
+    )
+
+
+def _latest_dns_snapshot(session: Session, domain: str) -> SimpleNamespace | None:
+    snapshot = session.scalar(
+        select(DomainDnsSnapshot)
+        .where(DomainDnsSnapshot.domain == domain)
+        .order_by(desc(DomainDnsSnapshot.checked_at), desc(DomainDnsSnapshot.id))
+        .limit(1)
+    )
+    if not snapshot:
+        return None
+    return SimpleNamespace(
+        id=snapshot.id,
+        domain=snapshot.domain,
+        checked_at=snapshot.checked_at,
+        dmarc_record=snapshot.dmarc_record,
+        dmarc_policy_p=snapshot.dmarc_policy_p,
+        dmarc_policy_sp=snapshot.dmarc_policy_sp,
+        dmarc_policy_pct=snapshot.dmarc_policy_pct,
+        dmarc_adkim=snapshot.dmarc_adkim,
+        dmarc_aspf=snapshot.dmarc_aspf,
+        dmarc_fo=snapshot.dmarc_fo,
+        dmarc_rua=snapshot.dmarc_rua,
+        dmarc_ruf=snapshot.dmarc_ruf,
+        spf_record=snapshot.spf_record,
+        spf_all=snapshot.spf_all,
+        spf_includes=_json_list(snapshot.spf_includes_json),
+        dkim_records=_json_list(snapshot.dkim_records_json),
+        mx_records=_json_list(snapshot.mx_records_json),
+        errors=_json_list(snapshot.errors_json),
+    )
+
+
 @app.get("/domains/{domain}", response_class=HTMLResponse, dependencies=[Depends(require_dashboard_auth)])
 def domain_page(domain: str, request: Request, source_page: int = 1, alert_page: int = 1, report_page: int = 1, trend_page: int = 1, session: Session = Depends(get_db)):
     metrics = domain_metrics(session, domain)
@@ -443,10 +532,19 @@ def domain_page(domain: str, request: Request, source_page: int = 1, alert_page:
             "dispositions": dispositions,
             "known_unknown": known_unknown,
             "summary": latest_summary(session, domain),
+            "dns_snapshot": _latest_dns_snapshot(session, domain),
         },
     )
 
 
+@app.post("/domains/{domain}/dns/refresh", dependencies=dashboard_post_auth)
+def refresh_domain_dns(domain: str, session: Session = Depends(get_db)):
+    policy = collect_domain_dns_policy(domain, selectors=_observed_dkim_selectors(session, domain))
+    session.add(_snapshot_model(domain, policy))
+    session.commit()
+    return RedirectResponse(url=f"/domains/{domain}", status_code=303)
+
+
 @app.get("/reports/{report_id}", response_class=HTMLResponse, dependencies=[Depends(require_dashboard_auth)])
 def report_page(report_id: int, request: Request, session: Session = Depends(get_db)):
     report = session.scalar(select(Report).options(selectinload(Report.records).selectinload(Record.auth_results)).where(Report.id == report_id))
@@ -915,4 +1013,3 @@ def api_import_job(job_id: str):
 @app.get("/api/admin/inboxes/{inbox_id}/status", dependencies=[Depends(require_dashboard_auth)])
 def api_inbox_status(inbox_id: str, session: Session = Depends(get_db)):
     return _inbox_status_payload(inbox_id, session)
-