Nautilus/DMARC-Sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add TLS reports capabilities

Browse Source
This commit is contained in:
Beda Schmid
2026-06-23 11:49:58 -03:00
parent 5c1dc95c36
commit f4ae9f8532
20 changed files with 1158 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/scheduler.py
+60 -9
View File
@@ -15,7 +15,7 @@ from app.db import session_scope
from app.inbox_locks import inbox_run_locks
from app.llm import LLMClient
from app.message_processor import process_inbox
from app.models import Alert, DailyStat, LLMReport, Record, Report, utcnow
from app.models import Alert, DailyStat, LLMReport, Record, Report, TLSPolicy, TLSReport, utcnow
logger = logging.getLogger(__name__)
@@ -63,6 +63,53 @@ def _domain_records(session: Session, domain: str, start: datetime, end: datetim
).scalars().all()
def _domain_list(session: Session) -> list[str]:
dmarc_domains = set(session.execute(select(Report.domain).distinct()).scalars().all())
tls_domains = set(session.execute(select(TLSReport.domain).distinct()).scalars().all())
return sorted(dmarc_domains | tls_domains)
def _tls_date_expr():
return func.coalesce(TLSReport.date_end, TLSReport.date_begin, TLSReport.created_at)
def _tls_bounds(session: Session, domain: str | None = None) -> tuple[datetime | None, datetime | None]:
stmt = select(func.min(_tls_date_expr()), func.max(_tls_date_expr()))
if domain:
stmt = stmt.where(TLSReport.domain == domain)
start, end = session.execute(stmt).one()
return _as_utc(start), _as_utc(end)
def _merge_bounds(*bounds: tuple[datetime | None, datetime | None]) -> tuple[datetime, datetime]:
starts = [start for start, _ in bounds if start]
ends = [end for _, end in bounds if end]
period_start = min(starts) if starts else datetime.now(timezone.utc)
period_end = max(ends) if ends else period_start
return period_start, period_end
def _tls_metrics(session: Session, domain: str, start: datetime | None = None, end: datetime | None = None) -> dict:
stmt = select(TLSPolicy, TLSReport).join(TLSReport).where(TLSReport.domain == domain)
if start:
stmt = stmt.where(_tls_date_expr() >= start)
if end:
stmt = stmt.where(_tls_date_expr() <= end)
rows = session.execute(stmt).all()
successes = sum(policy.total_successful_session_count for policy, _ in rows)
failures = sum(policy.total_failure_session_count for policy, _ in rows)
total = successes + failures
reports = {report.id for _, report in rows}
reporters = sorted({report.org_name for _, report in rows if report.org_name})
return {
"tls_reports": len(reports),
"tls_successful_sessions": successes,
"tls_failed_sessions": failures,
"tls_failure_rate": round(failures / total * 100, 2) if total else 0,
"tls_reporters": reporters[:10],
}
def aggregate_daily_stats(session: Session, domain: str, day: date) -> DailyStat:
start = datetime.combine(day, time.min, tzinfo=timezone.utc)
end = start + timedelta(days=1)
@@ -141,6 +188,7 @@ def _summary_payload(session: Session, domain: str, day: date, stat: DailyStat)
"unknown_sources": stat.unknown_source_count,
"critical_alerts": critical,
"warnings": warnings,
**_tls_metrics(session, domain, period_start, period_end),
},
"top_sources": json.loads(stat.top_sources_json or "[]"),
"reporters": [{"org": org or "unknown", "reports": count} for org, count in reports],
@@ -169,8 +217,7 @@ def _posture_payload(session: Session, domain: str) -> tuple[dict, datetime, dat
func.max(func.coalesce(Report.date_end, Report.date_begin, Report.created_at)),
).where(Report.domain == domain)
).one()
period_start = _as_utc(bounds[0]) or datetime.now(timezone.utc)
period_end = _as_utc(bounds[1]) or period_start
period_start, period_end = _merge_bounds((_as_utc(bounds[0]), _as_utc(bounds[1])), _tls_bounds(session, domain))
records = session.execute(select(Record).join(Report).where(Report.domain == domain)).scalars().all()
reports = session.execute(
select(Report.org_name, func.count(Report.id))
@@ -214,6 +261,7 @@ def _posture_payload(session: Session, domain: str) -> tuple[dict, datetime, dat
"unknown_failing_sources": len({row.source_ip for row in failing_unknown}),
"open_critical_alerts": len([alert for alert in alerts if alert.severity == "critical"]),
"open_warnings": len([alert for alert in alerts if alert.severity == "warning"]),
**_tls_metrics(session, domain),
},
"top_sources": [
{
@@ -250,7 +298,7 @@ def _posture_payload(session: Session, domain: str) -> tuple[dict, datetime, dat
def _portfolio_posture_payload(session: Session) -> tuple[dict, datetime, datetime] | None:
domains = session.execute(select(Report.domain).distinct().order_by(Report.domain)).scalars().all()
domains = _domain_list(session)
if not domains:
return None
bounds = session.execute(
@@ -259,8 +307,7 @@ def _portfolio_posture_payload(session: Session) -> tuple[dict, datetime, dateti
func.max(func.coalesce(Report.date_end, Report.date_begin, Report.created_at)),
)
).one()
period_start = _as_utc(bounds[0]) or datetime.now(timezone.utc)
period_end = _as_utc(bounds[1]) or period_start
period_start, period_end = _merge_bounds((_as_utc(bounds[0]), _as_utc(bounds[1])), _tls_bounds(session))
records = session.execute(select(Record).join(Report)).scalars().all()
alerts = session.execute(
select(Alert)
@@ -286,6 +333,7 @@ def _portfolio_posture_payload(session: Session) -> tuple[dict, datetime, dateti
"unknown_sources": len({row.source_ip for row in domain_records if not row.is_known_sender}),
"open_critical_alerts": len([alert for alert in domain_alerts if alert.severity == "critical"]),
"open_warnings": len([alert for alert in domain_alerts if alert.severity == "warning"]),
**_tls_metrics(session, domain),
}
)
top_alerts = [
@@ -322,6 +370,9 @@ def _portfolio_posture_payload(session: Session) -> tuple[dict, datetime, dateti
"unknown_failing_sources": len({row.source_ip for row in failing_unknown}),
"open_critical_alerts": len([alert for alert in alerts if alert.severity == "critical"]),
"open_warnings": len([alert for alert in alerts if alert.severity == "warning"]),
"tls_reports": session.scalar(select(func.count(TLSReport.id))) or 0,
"tls_failed_sessions": sum(row["tls_failed_sessions"] for row in domain_rows),
"tls_successful_sessions": sum(row["tls_successful_sessions"] for row in domain_rows),
},
"domain_posture": domain_rows,
"top_open_alerts": top_alerts,
@@ -376,7 +427,7 @@ def generate_open_posture_summaries(settings: Settings, *, force: bool = True) -
report.plain_text = plain
generated.append(report)
send_digest_email(settings, "DMARC Sentinel portfolio posture summary", plain)
domains = session.execute(select(Report.domain).distinct()).scalars().all()
domains = _domain_list(session)
for domain in domains:
payload, period_start, period_end = _posture_payload(session, domain)
existing = session.scalar(
@@ -421,7 +472,7 @@ def generate_daily_summaries(settings: Settings, target_day: date | None = None,
llm = LLMClient(settings)
generated: list[LLMReport] = []
with session_scope() as session:
domains = session.execute(select(Report.domain).distinct()).scalars().all()
domains = _domain_list(session)
for domain in domains:
stat = aggregate_daily_stats(session, domain, target_day)
payload = _summary_payload(session, domain, target_day, stat)
@@ -473,7 +524,7 @@ def generate_weekly_summaries(settings: Settings) -> list[LLMReport]:
llm = LLMClient(settings)
generated: list[LLMReport] = []
with session_scope() as session:
domains = session.execute(select(Report.domain).distinct()).scalars().all()
domains = _domain_list(session)
for domain in domains:
records = _domain_records(session, domain, start, end)
existing = session.scalar(