{{ alert.llm_summary or alert.summary }}
{% if alert.llm_recommended_action %}{{ alert.llm_recommended_action }}
diff --git a/app/templates/base.html b/app/templates/base.html index f7ba4fa..e2c5b90 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -12,7 +12,7 @@ - +{{ dns_snapshot.dmarc_record or "No DMARC record found." }}
+ {{ dns_snapshot.spf_record or "No SPF record found." }}
+ {{ item.record or item.error or "No DKIM record found." }}
+ No observed DKIM selectors yet.
+ {% endfor %}
+ {{ mx }}
+ {% else %}
+ No MX records found.
+ {% endfor %}
+ {{ alert.llm_summary or alert.summary }}
{% else %} diff --git a/docs/runtime.md b/docs/runtime.md index b780953..79eac22 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -32,7 +32,7 @@ There is no repository-local Ruff configuration or requirement in `requirements. The Docker image starts the FastAPI app with: ```bash -uvicorn app.main:app --host 0.0.0.0 --port 8000 +alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000 ``` The compose service: @@ -45,6 +45,13 @@ The compose service: - attaches to the external `npm_proxy` network with static address `192.168.99.18`. The app initializes database tables on import through `app.main` calling `init_db()`. +Alembic migrations are run first in Docker and are the preservation path for existing SQLite databases. `init_db()` remains as a lightweight fallback for missing tables in local/test workflows. + +Manual migration command inside the container: + +```bash +docker compose exec dmarc-sentinel alembic upgrade head +``` ## CLI Backlog Processing diff --git a/logs/.gitkeep b/logs/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/logs/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..b20bf83 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from app.config import get_settings +from app.db import Base +from app import models # noqa: F401 + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _database_url() -> str: + return get_settings().app.database_url + + +def run_migrations_offline() -> None: + context.configure( + url=_database_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + section = config.get_section(config.config_ini_section, {}) + section["sqlalchemy.url"] = _database_url() + connectable = engine_from_config( + section, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..c6284cb --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/20260520_0001_domain_dns_snapshots.py b/migrations/versions/20260520_0001_domain_dns_snapshots.py new file mode 100644 index 0000000..85f97a3 --- /dev/null +++ b/migrations/versions/20260520_0001_domain_dns_snapshots.py @@ -0,0 +1,66 @@ +"""add domain dns snapshots + +Revision ID: 20260520_0001 +Revises: +Create Date: 2026-05-20 00:00:00.000000+00:00 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260520_0001" +down_revision = None +branch_labels = None +depends_on = None + + +def _domain_dns_snapshots() -> sa.Table: + return sa.Table( + "domain_dns_snapshots", + sa.MetaData(), + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("domain", sa.String(length=255), nullable=False), + sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("dmarc_record", sa.Text(), nullable=True), + sa.Column("dmarc_policy_p", sa.String(length=40), nullable=True), + sa.Column("dmarc_policy_sp", sa.String(length=40), nullable=True), + sa.Column("dmarc_policy_pct", sa.Integer(), nullable=True), + sa.Column("dmarc_adkim", sa.String(length=20), nullable=True), + sa.Column("dmarc_aspf", sa.String(length=20), nullable=True), + sa.Column("dmarc_fo", sa.String(length=80), nullable=True), + sa.Column("dmarc_rua", sa.Text(), nullable=True), + sa.Column("dmarc_ruf", sa.Text(), nullable=True), + sa.Column("spf_record", sa.Text(), nullable=True), + sa.Column("spf_all", sa.String(length=20), nullable=True), + sa.Column("spf_includes_json", sa.Text(), nullable=False, server_default="[]"), + sa.Column("dkim_records_json", sa.Text(), nullable=False, server_default="[]"), + sa.Column("mx_records_json", sa.Text(), nullable=False, server_default="[]"), + sa.Column("errors_json", sa.Text(), nullable=False, server_default="[]"), + ) + + +def upgrade() -> None: + bind = op.get_bind() + table = _domain_dns_snapshots() + table.create(bind=bind, checkfirst=True) + op.create_index( + op.f("ix_domain_dns_snapshots_checked_at"), + "domain_dns_snapshots", + ["checked_at"], + unique=False, + if_not_exists=True, + ) + op.create_index( + op.f("ix_domain_dns_snapshots_domain"), + "domain_dns_snapshots", + ["domain"], + unique=False, + if_not_exists=True, + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_domain_dns_snapshots_domain"), table_name="domain_dns_snapshots", if_exists=True) + op.drop_index(op.f("ix_domain_dns_snapshots_checked_at"), table_name="domain_dns_snapshots", if_exists=True) + _domain_dns_snapshots().drop(bind=op.get_bind(), checkfirst=True) diff --git a/requirements.txt b/requirements.txt index 6848ddc..a12e683 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,9 @@ PyYAML==6.0.2 Jinja2==3.1.5 python-multipart==0.0.20 APScheduler==3.10.4 +alembic==1.14.0 defusedxml==0.7.1 openai==1.58.1 pytest==8.3.4 httpx==0.28.1 +dnspython==2.7.0 diff --git a/tests/test_analyzer.py b/tests/test_analyzer.py index 57a8fa3..15afac0 100644 --- a/tests/test_analyzer.py +++ b/tests/test_analyzer.py @@ -41,6 +41,11 @@ def _report( dkim_aligned: bool | None = None, report_time: datetime | None = None, org_name: str = "google.com", + policy_p: str | None = None, + policy_sp: str | None = None, + policy_pct: int | None = None, + disposition: str = "none", + reason_type: str | None = None, ) -> Report: dkim_aligned = dmarc_pass if dkim_aligned is None else dkim_aligned report_time = report_time or datetime.now(timezone.utc) @@ -52,6 +57,9 @@ def _report( domain="tukutoi.com", date_begin=report_time - timedelta(hours=1), date_end=report_time, + policy_p=policy_p, + policy_sp=policy_sp, + policy_pct=policy_pct, ) session.add(report) session.flush() @@ -60,7 +68,7 @@ def _report( report=report, source_ip=source_ip, count=count, - disposition="none", + disposition=disposition, policy_dkim="pass" if dkim_aligned else "fail", policy_spf="pass" if spf_aligned else "fail", dkim_aligned=dkim_aligned, @@ -70,6 +78,7 @@ def _report( known_sender_id="mailcow" if known else None, known_sender_name="mailcow outbound" if known else None, is_known_sender=known, + reason_type=reason_type, ) ) session.commit() @@ -175,3 +184,28 @@ def test_missing_reporter_gap_does_not_create_alert(): alerts = analyze_report(session, settings, report) assert not any(alert.type == "missing_reporter" for alert, _, _ in alerts) + + +def test_alert_details_include_published_policy_and_receiver_action(): + session = _session() + report = _report( + session, + source_ip="203.0.113.91", + count=25, + known=False, + dmarc_pass=False, + policy_p="reject", + policy_sp="quarantine", + policy_pct=100, + disposition="reject", + ) + + alerts = analyze_report(session, _settings(), report) + + alert = next(alert for alert, _, _ in alerts if alert.type == "unknown_source_failed_both") + details = json.loads(alert.details_json) + assert details["published_policy"]["p"] == "reject" + assert details["published_policy"]["effective"] == "reject" + assert details["published_policy"]["effective_source"] == "p" + assert details["receiver_action"]["disposition"] == "reject" + assert "Published DMARC policy was p=reject; pct=100" in alert.summary diff --git a/tests/test_dns_policy.py b/tests/test_dns_policy.py new file mode 100644 index 0000000..08fdf6d --- /dev/null +++ b/tests/test_dns_policy.py @@ -0,0 +1,47 @@ +from app.dns_policy import collect_domain_dns_policy, parse_dmarc_records, parse_spf_records + + +def test_parse_dmarc_record_extracts_policy_tags(): + parsed, errors = parse_dmarc_records(["v=DMARC1; p=reject; sp=quarantine; pct=50; adkim=s; aspf=r; rua=mailto:d@example.com"]) + + assert errors == [] + assert parsed.p == "reject" + assert parsed.sp == "quarantine" + assert parsed.pct == 50 + assert parsed.adkim == "s" + assert parsed.rua == "mailto:d@example.com" + + +def test_parse_spf_record_extracts_includes_and_all_mechanism(): + parsed, errors = parse_spf_records(["v=spf1 include:_spf.google.com include:mailgun.org -all"]) + + assert errors == [] + assert parsed.includes == ["_spf.google.com", "mailgun.org"] + assert parsed.all_mechanism == "-all" + + +def test_collect_domain_dns_policy_uses_observed_dkim_selectors(): + txt_records = { + "_dmarc.example.com": ["v=DMARC1; p=reject; pct=100"], + "example.com": ["v=spf1 include:_spf.example.net -all"], + "s1._domainkey.example.com": ["v=DKIM1; k=rsa; p=abc"], + } + + def txt_lookup(name: str) -> list[str]: + if name not in txt_records: + raise RuntimeError("not found") + return txt_records[name] + + policy = collect_domain_dns_policy( + "example.com", + selectors=["s1"], + txt_lookup=txt_lookup, + mx_lookup=lambda name: ["10 mail.example.com"], + ) + + assert policy.dmarc.p == "reject" + assert policy.spf.all_mechanism == "-all" + assert policy.mx_records == ["10 mail.example.com"] + assert policy.dkim[0].selector == "s1" + assert policy.dkim[0].record == "v=DKIM1; k=rsa; p=abc" + assert policy.errors == []