Add TLS reports capabilities

app/main.py

@@ -22,7 +22,7 @@ from app.dns_policy import DomainDnsPolicy, collect_domain_dns_policy
 from app.homepage import domain_homepage_summary, domain_metrics, homepage_summary, latest_summary, resolve_date_range, traffic_distribution
 from app.inbox_locks import InboxRunLease, inbox_run_locks
 from app.jobs import import_jobs
-from app.models import Alert, DailyStat, DomainDnsSnapshot, InboxStatus, LLMReport, Record, Report, SkippedReportPayload, utcnow
+from app.models import Alert, DailyStat, DomainDnsSnapshot, InboxStatus, LLMReport, Record, Report, SkippedReportPayload, TLSPolicy, TLSReport, utcnow
 from app.scheduler import generate_open_posture_summaries, scheduler_ok, start_scheduler
 from app.schemas import BacklogRequest, ProcessNowRequest
 from app.message_processor import process_inbox
@@ -100,7 +100,7 @@ def health():
 @app.get("/", response_class=HTMLResponse, dependencies=[Depends(require_dashboard_auth)])
 def index(request: Request, session: Session = Depends(get_db)):
     data = homepage_summary(session, period="all")
-    domains = session.execute(select(Report.domain).distinct().order_by(Report.domain)).scalars().all()
+    domains = _domain_list(session)
     alerts_total = session.scalar(select(func.count(Alert.id)).where(Alert.status == "open")) or 0
     alerts = _alert_views(session.execute(select(Alert).where(Alert.status == "open")).scalars().all(), session)[:5]
     traffic = traffic_distribution(session, period="all")
@@ -209,8 +209,12 @@ def _alert_view(alert: Alert, session: Session | None = None) -> SimpleNamespace
     override_type = receiver_action.get("override_type")
     report_db_id = details.get("report_db_id")
     report_db_ids = details.get("report_db_ids") if isinstance(details.get("report_db_ids"), list) else []
+    tls_report_db_id = report_db_id if details.get("report_type") == "tlsrpt" else None
     if not report_db_id and isinstance(details.get("report_db_ids"), list) and details["report_db_ids"]:
         report_db_id = details["report_db_ids"][-1]
+    if details.get("report_type") == "tlsrpt":
+        report_db_id = None
+        report_db_ids = []
     if alert.type in {"sudden_unknown_failure_spike"}:
         report_db_id = None
     report_time = (
@@ -242,6 +246,8 @@ def _alert_view(alert: Alert, session: Session | None = None) -> SimpleNamespace
         report_time=report_time,
         report_db_id=report_db_id,
         report_db_ids=report_db_ids,
+        tls_report_db_id=tls_report_db_id,
+        source_report_url=f"/tls-reports/{tls_report_db_id}" if tls_report_db_id else (f"/reports/{report_db_id}" if report_db_id else None),
         source_ip=details.get("source_ip"),
         published_policy=published_policy,
         receiver_action=receiver_action,
@@ -327,6 +333,49 @@ def _domain_sources(session: Session, domain: str) -> list[SimpleNamespace]:
     ]
 
 
+def _report_stamp(report: Report | TLSReport) -> datetime:
+    value = report.date_end or report.date_begin or report.created_at
+    return _parse_dt(value.isoformat() if hasattr(value, "isoformat") else str(value)) or datetime.now(timezone.utc)
+
+
+def _domain_report_rows(session: Session, domain: str, page: int, page_size: int) -> tuple[list[SimpleNamespace], int]:
+    dmarc_reports = session.execute(select(Report).where(Report.domain == domain)).scalars().all()
+    tls_reports = session.execute(select(TLSReport).where(TLSReport.domain == domain)).scalars().all()
+    rows = [
+        SimpleNamespace(
+            id=report.id,
+            kind="dmarc",
+            label="DMARC",
+            icon="description",
+            url=f"/reports/{report.id}",
+            org_name=report.org_name,
+            report_id=report.report_id,
+            date_begin=report.date_begin,
+            date_end=report.date_end,
+            stamp=_report_stamp(report),
+        )
+        for report in dmarc_reports
+    ]
+    rows.extend(
+        SimpleNamespace(
+            id=report.id,
+            kind="tlsrpt",
+            label="TLS",
+            icon="lock",
+            url=f"/tls-reports/{report.id}",
+            org_name=report.org_name,
+            report_id=report.report_id,
+            date_begin=report.date_begin,
+            date_end=report.date_end,
+            stamp=_report_stamp(report),
+        )
+        for report in tls_reports
+    )
+    rows.sort(key=lambda item: item.stamp, reverse=True)
+    total = len(rows)
+    return rows[(page - 1) * page_size : page * page_size], total
+
+
 def _source_history(session: Session, domain: str, source_ip: str | None, alert_type: str, report_db_id: int | None) -> str | None:
     if not source_ip:
         return None
@@ -400,6 +449,12 @@ def _json_list(value: str | None) -> list:
     return data if isinstance(data, list) else []
 
 
+def _domain_list(session: Session) -> list[str]:
+    dmarc_domains = set(session.execute(select(Report.domain).distinct()).scalars().all())
+    tls_domains = set(session.execute(select(TLSReport.domain).distinct()).scalars().all())
+    return sorted(dmarc_domains | tls_domains)
+
+
 def _snapshot_model(domain: str, policy: DomainDnsPolicy) -> DomainDnsSnapshot:
     return DomainDnsSnapshot(
         domain=domain,
@@ -472,14 +527,7 @@ def domain_page(domain: str, request: Request, source_page: int = 1, alert_page:
     alerts = all_alerts[(alert_page - 1) * alert_page_size : alert_page * alert_page_size]
     report_page_size = 20
     report_page = max(1, report_page)
-    report_total = session.scalar(select(func.count(Report.id)).where(Report.domain == domain)) or 0
-    reports = session.execute(
-        select(Report)
-        .where(Report.domain == domain)
-        .order_by(desc(func.coalesce(Report.date_end, Report.date_begin, Report.created_at)))
-        .offset((report_page - 1) * report_page_size)
-        .limit(report_page_size)
-    ).scalars().all()
+    reports, report_total = _domain_report_rows(session, domain, report_page, report_page_size)
     reporters = session.execute(
         select(Report.org_name, func.count(Report.id)).where(Report.domain == domain).group_by(Report.org_name).order_by(desc(func.count(Report.id))).limit(10)
     ).all()
@@ -545,6 +593,33 @@ def report_page(report_id: int, request: Request, session: Session = Depends(get
     return templates.TemplateResponse("report.html", {"request": request, "report": report, "alerts": related_alerts})
 
 
+@app.get("/tls-reports/{report_id}", response_class=HTMLResponse, dependencies=[Depends(require_dashboard_auth)])
+def tls_report_page(report_id: int, request: Request, session: Session = Depends(get_db)):
+    report = session.scalar(
+        select(TLSReport)
+        .options(selectinload(TLSReport.policies).selectinload(TLSPolicy.failures))
+        .where(TLSReport.id == report_id)
+    )
+    if not report:
+        raise HTTPException(status_code=404)
+    for policy in report.policies:
+        policy.policy_strings = _json_list(policy.policy_strings_json)
+        policy.mx_hosts = _json_list(policy.mx_hosts_json)
+        policy.total_sessions = policy.total_successful_session_count + policy.total_failure_session_count
+        policy.failure_rate = (
+            policy.total_failure_session_count / policy.total_sessions * 100
+            if policy.total_sessions
+            else 0
+        )
+    related_alerts = []
+    domain_alerts = session.execute(select(Alert).where(Alert.domain == report.domain)).scalars().all()
+    for alert in domain_alerts:
+        details = _alert_details(alert)
+        if details.get("report_type") == "tlsrpt" and details.get("report_db_id") == report.id:
+            related_alerts.append(_alert_view(alert, session))
+    return templates.TemplateResponse("tls_report.html", {"request": request, "report": report, "alerts": related_alerts})
+
+
 @app.get("/alerts", response_class=HTMLResponse, dependencies=[Depends(require_dashboard_auth)])
 def alerts_page(
     request: Request,
@@ -729,7 +804,7 @@ def api_homepage_domain(domain: str, session: Session = Depends(get_db)):
 
 @app.get("/api/domains", dependencies=[Depends(require_dashboard_auth)])
 def api_domains(session: Session = Depends(get_db)):
-    return {"domains": session.execute(select(Report.domain).distinct()).scalars().all()}
+    return {"domains": _domain_list(session)}
 
 
 def _overview_payload(session: Session, period: str = "all", domain: str | None = None, date_from: str | None = None, date_to: str | None = None) -> dict:
@@ -757,6 +832,9 @@ def api_traffic(period: str = "all", domain: str | None = None, date_from: str |
 
 def _latest_report_day(session: Session) -> date | None:
     latest = session.scalar(select(func.max(func.coalesce(Report.date_end, Report.date_begin, Report.created_at))))
+    latest_tls = session.scalar(select(func.max(func.coalesce(TLSReport.date_end, TLSReport.date_begin, TLSReport.created_at))))
+    if latest_tls and (not latest or latest_tls > latest):
+        latest = latest_tls
     if isinstance(latest, str):
         latest = _parse_dt(latest)
     return latest.date() if latest else None