Add TLS reports capabilities
This commit is contained in:
+166
-24
@@ -20,7 +20,9 @@ from app.dmarc_parser import DMARCParseError, parse_dmarc_xml
|
||||
from app.imap_client import IMAPClient
|
||||
from app.known_senders import classify_record
|
||||
from app.llm import LLMClient
|
||||
from app.models import Alert, AuthResult, InboxStatus, MailMessage, Record, Report, SkippedReportPayload, utcnow
|
||||
from app.models import Alert, AuthResult, InboxStatus, MailMessage, Record, Report, SkippedReportPayload, TLSFailure, TLSPolicy, TLSReport, utcnow
|
||||
from app.tlsrpt_analyzer import analyze_tls_report
|
||||
from app.tlsrpt_parser import TLSRPTParseError, parse_tlsrpt_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -79,6 +81,8 @@ def is_candidate_message(message: Message, inbox: InboxConfig) -> bool:
|
||||
return (
|
||||
inbox.recipient.lower() in recipients
|
||||
or "dmarc" in subject.lower()
|
||||
or "tls" in subject.lower()
|
||||
or "tlsrpt" in subject.lower()
|
||||
or inbox.domain.lower() in subject.lower()
|
||||
or "report domain" in subject.lower()
|
||||
or message_has_candidate_attachment(message)
|
||||
@@ -164,7 +168,7 @@ def _record_ingestion_rejection(
|
||||
domain=inbox.domain,
|
||||
severity="warning",
|
||||
type="ingestion_rejected",
|
||||
title=f"DMARC payload rejected for {inbox.label}",
|
||||
title=f"Report payload rejected for {inbox.label}",
|
||||
summary=f"A message in {inbox.folder} was rejected during {stage}: {reason}",
|
||||
details_json=json.dumps(details, sort_keys=True, default=str),
|
||||
first_seen_at=now,
|
||||
@@ -186,6 +190,17 @@ def _duplicate_report_sample(existing: Report, mail: MailMessage) -> dict[str, s
|
||||
}
|
||||
|
||||
|
||||
def _duplicate_tls_report_sample(existing: TLSReport, mail: MailMessage) -> dict[str, str | int | None]:
|
||||
return {
|
||||
"existing_report_db_id": None,
|
||||
"existing_report_id": existing.report_id,
|
||||
"reporting_org": existing.org_name,
|
||||
"report_date": (existing.date_end or existing.date_begin).date().isoformat() if (existing.date_end or existing.date_begin) else None,
|
||||
"duplicate_message_uid": mail.imap_uid,
|
||||
"duplicate_message_id": mail.message_id,
|
||||
}
|
||||
|
||||
|
||||
def _record_duplicate_report_payload(session: Session, inbox: InboxConfig, mail: MailMessage, existing: Report, sha256: str) -> None:
|
||||
skipped = session.scalar(
|
||||
select(SkippedReportPayload).where(
|
||||
@@ -216,6 +231,35 @@ def _record_duplicate_report_payload(session: Session, inbox: InboxConfig, mail:
|
||||
skipped.report_date = report_date
|
||||
|
||||
|
||||
def _record_duplicate_tls_report_payload(session: Session, inbox: InboxConfig, mail: MailMessage, existing: TLSReport, sha256: str) -> None:
|
||||
skipped = session.scalar(
|
||||
select(SkippedReportPayload).where(
|
||||
SkippedReportPayload.inbox_id == inbox.id,
|
||||
SkippedReportPayload.folder == mail.folder,
|
||||
SkippedReportPayload.imap_uid == mail.imap_uid,
|
||||
SkippedReportPayload.raw_xml_sha256 == sha256,
|
||||
SkippedReportPayload.reason == "duplicate_tls_report_payload",
|
||||
)
|
||||
)
|
||||
report_date = (existing.date_end or existing.date_begin).date() if (existing.date_end or existing.date_begin) else None
|
||||
if not skipped:
|
||||
skipped = SkippedReportPayload(
|
||||
inbox_id=inbox.id,
|
||||
folder=mail.folder,
|
||||
imap_uid=mail.imap_uid,
|
||||
message_id=mail.message_id,
|
||||
mail_message_id=mail.id,
|
||||
reason="duplicate_tls_report_payload",
|
||||
raw_xml_sha256=sha256,
|
||||
)
|
||||
session.add(skipped)
|
||||
skipped.message_id = mail.message_id
|
||||
skipped.mail_message_id = mail.id
|
||||
skipped.report_identifier = existing.report_id
|
||||
skipped.reporting_org = existing.org_name
|
||||
skipped.report_date = report_date
|
||||
|
||||
|
||||
def _store_report(session: Session, settings: Settings, inbox: InboxConfig, mail: MailMessage, extracted) -> tuple[Report | None, Report | None]:
|
||||
existing = session.scalar(select(Report).where(Report.raw_xml_sha256 == extracted.sha256))
|
||||
if existing:
|
||||
@@ -286,6 +330,75 @@ def _store_report(session: Session, settings: Settings, inbox: InboxConfig, mail
|
||||
return report, None
|
||||
|
||||
|
||||
def _store_tls_report(session: Session, settings: Settings, inbox: InboxConfig, mail: MailMessage, extracted) -> tuple[TLSReport | None, TLSReport | None]:
|
||||
existing = session.scalar(select(TLSReport).where(TLSReport.raw_json_sha256 == extracted.sha256))
|
||||
if existing:
|
||||
return None, existing
|
||||
parsed = parse_tlsrpt_json(
|
||||
extracted.payload,
|
||||
max_policies=settings.app.max_xml_records_per_report,
|
||||
max_failures_per_policy=settings.app.max_xml_records_per_report,
|
||||
max_session_count=settings.app.max_record_count,
|
||||
max_future_days=settings.app.max_report_future_days,
|
||||
max_past_days=settings.app.max_report_past_days,
|
||||
)
|
||||
domains = {policy.policy_domain for policy in parsed.policies}
|
||||
mismatches = sorted(domain for domain in domains if not _domain_equal(domain, inbox.domain))
|
||||
if mismatches:
|
||||
raise TLSRPTParseError(f"TLS report domain {', '.join(mismatches)} does not match inbox domain {inbox.domain}")
|
||||
report = TLSReport(
|
||||
inbox_id=inbox.id,
|
||||
mail_message_id=mail.id,
|
||||
raw_json_sha256=extracted.sha256,
|
||||
report_id=parsed.report_id,
|
||||
org_name=parsed.org_name,
|
||||
contact_info=parsed.contact_info,
|
||||
domain=parsed.domain,
|
||||
date_begin=parsed.date_begin,
|
||||
date_end=parsed.date_end,
|
||||
)
|
||||
session.add(report)
|
||||
session.flush()
|
||||
for parsed_policy in parsed.policies:
|
||||
policy = TLSPolicy(
|
||||
tls_report=report,
|
||||
policy_type=parsed_policy.policy_type,
|
||||
policy_domain=parsed_policy.policy_domain,
|
||||
policy_strings_json=json.dumps(parsed_policy.policy_strings, sort_keys=True),
|
||||
mx_hosts_json=json.dumps(parsed_policy.mx_hosts, sort_keys=True),
|
||||
total_successful_session_count=parsed_policy.total_successful_session_count,
|
||||
total_failure_session_count=parsed_policy.total_failure_session_count,
|
||||
)
|
||||
session.add(policy)
|
||||
session.flush()
|
||||
for parsed_failure in parsed_policy.failures:
|
||||
session.add(
|
||||
TLSFailure(
|
||||
policy=policy,
|
||||
result_type=parsed_failure.result_type,
|
||||
sending_mta_ip=parsed_failure.sending_mta_ip,
|
||||
receiving_mx_hostname=parsed_failure.receiving_mx_hostname,
|
||||
receiving_mx_helo=parsed_failure.receiving_mx_helo,
|
||||
receiving_ip=parsed_failure.receiving_ip,
|
||||
failed_session_count=parsed_failure.failed_session_count,
|
||||
additional_information=parsed_failure.additional_information,
|
||||
failure_reason_code=parsed_failure.failure_reason_code,
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return report, None
|
||||
|
||||
|
||||
def _detect_report_kind(filename: str, payload: bytes) -> str:
|
||||
lower = filename.lower()
|
||||
stripped = payload.lstrip()
|
||||
if stripped.startswith(b"<") or lower.endswith(".xml"):
|
||||
return "dmarc"
|
||||
if stripped.startswith(b"{") or lower.endswith(".json"):
|
||||
return "tlsrpt"
|
||||
raise DMARCParseError("Unsupported report payload: expected DMARC XML or TLSRPT JSON")
|
||||
|
||||
|
||||
def process_inbox(
|
||||
session: Session,
|
||||
settings: Settings,
|
||||
@@ -338,26 +451,49 @@ def process_inbox(
|
||||
)
|
||||
imported_any = False
|
||||
for extracted in reports:
|
||||
report, duplicate_report = _store_report(session, settings, inbox, mail, extracted)
|
||||
if duplicate_report:
|
||||
_record_duplicate_report_payload(session, inbox, mail, duplicate_report, extracted.sha256)
|
||||
summary.duplicate_reports_skipped += 1
|
||||
if summary.duplicate_report_samples is None:
|
||||
summary.duplicate_report_samples = []
|
||||
if len(summary.duplicate_report_samples) < 100:
|
||||
summary.duplicate_report_samples.append(_duplicate_report_sample(duplicate_report, mail))
|
||||
continue
|
||||
if report:
|
||||
imported_any = True
|
||||
summary.valid_reports_imported += 1
|
||||
summary.records_imported += len(report.records)
|
||||
alerts = analyze_report(session, settings, report, llm=llm)
|
||||
new_alerts = [item for item in alerts if item[1]]
|
||||
summary.alerts_created += len(new_alerts)
|
||||
summary.llm_explanations_generated += len([item for item in alerts if item[1] and item[0].severity in {"warning", "critical"}])
|
||||
for alert, is_new, severity_increased in alerts:
|
||||
if is_new or severity_increased:
|
||||
send_alert_email(settings, alert, severity_increased=severity_increased)
|
||||
kind = _detect_report_kind(extracted.filename, extracted.payload)
|
||||
if kind == "dmarc":
|
||||
report, duplicate_report = _store_report(session, settings, inbox, mail, extracted)
|
||||
if duplicate_report:
|
||||
_record_duplicate_report_payload(session, inbox, mail, duplicate_report, extracted.sha256)
|
||||
summary.duplicate_reports_skipped += 1
|
||||
if summary.duplicate_report_samples is None:
|
||||
summary.duplicate_report_samples = []
|
||||
if len(summary.duplicate_report_samples) < 100:
|
||||
summary.duplicate_report_samples.append(_duplicate_report_sample(duplicate_report, mail))
|
||||
continue
|
||||
if report:
|
||||
imported_any = True
|
||||
summary.valid_reports_imported += 1
|
||||
summary.records_imported += len(report.records)
|
||||
alerts = analyze_report(session, settings, report, llm=llm)
|
||||
new_alerts = [item for item in alerts if item[1]]
|
||||
summary.alerts_created += len(new_alerts)
|
||||
summary.llm_explanations_generated += len([item for item in alerts if item[1] and item[0].severity in {"warning", "critical"}])
|
||||
for alert, is_new, severity_increased in alerts:
|
||||
if is_new or severity_increased:
|
||||
send_alert_email(settings, alert, severity_increased=severity_increased)
|
||||
else:
|
||||
tls_report, duplicate_tls_report = _store_tls_report(session, settings, inbox, mail, extracted)
|
||||
if duplicate_tls_report:
|
||||
_record_duplicate_tls_report_payload(session, inbox, mail, duplicate_tls_report, extracted.sha256)
|
||||
summary.duplicate_reports_skipped += 1
|
||||
if summary.duplicate_report_samples is None:
|
||||
summary.duplicate_report_samples = []
|
||||
if len(summary.duplicate_report_samples) < 100:
|
||||
summary.duplicate_report_samples.append(_duplicate_tls_report_sample(duplicate_tls_report, mail))
|
||||
continue
|
||||
if tls_report:
|
||||
imported_any = True
|
||||
summary.valid_reports_imported += 1
|
||||
summary.records_imported += len(tls_report.policies)
|
||||
alerts = analyze_tls_report(session, settings, tls_report, llm=llm)
|
||||
new_alerts = [item for item in alerts if item[1]]
|
||||
summary.alerts_created += len(new_alerts)
|
||||
summary.llm_explanations_generated += len([item for item in alerts if item[1] and item[0].severity in {"warning", "critical"}])
|
||||
for alert, is_new, severity_increased in alerts:
|
||||
if is_new or severity_increased:
|
||||
send_alert_email(settings, alert, severity_increased=severity_increased)
|
||||
mail.status = "success" if imported_any else "skipped"
|
||||
mail.error = None
|
||||
mail.processed_at = utcnow()
|
||||
@@ -376,13 +512,19 @@ def process_inbox(
|
||||
mail.status = "failed"
|
||||
mail.error = str(exc)
|
||||
mail.processed_at = utcnow()
|
||||
if isinstance(exc, (AttachmentExtractionError, DMARCParseError)):
|
||||
if isinstance(exc, (AttachmentExtractionError, DMARCParseError, TLSRPTParseError)):
|
||||
alert, is_new = _record_ingestion_rejection(
|
||||
session,
|
||||
inbox,
|
||||
mail,
|
||||
str(exc),
|
||||
stage="attachment extraction" if isinstance(exc, AttachmentExtractionError) else "DMARC XML validation",
|
||||
stage=(
|
||||
"attachment extraction"
|
||||
if isinstance(exc, AttachmentExtractionError)
|
||||
else "TLSRPT JSON validation"
|
||||
if isinstance(exc, TLSRPTParseError)
|
||||
else "report payload validation"
|
||||
),
|
||||
)
|
||||
summary.rejected_messages += 1
|
||||
if is_new:
|
||||
|
||||
Reference in New Issue
Block a user