Harden security controls from REPORT findings

This commit is contained in:
2026-03-01 13:32:08 -03:00
parent da5cbc2c01
commit bdd97d1c62
20 changed files with 1455 additions and 97 deletions

View File

@@ -59,13 +59,21 @@ def get_request_role(
credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(bearer_auth)],
settings: Annotated[Settings, Depends(get_settings)],
) -> str:
"""Authenticates request token and returns its authorization role."""
"""Authenticates request token and returns its authorization role.
Development environments can optionally allow tokenless user access for non-admin routes to
preserve local workflow compatibility while production remains token-enforced.
"""
if credentials is None:
if settings.allow_development_anonymous_user_access and settings.app_env.strip().lower() in {"development", "dev"}:
return AuthRole.USER
_raise_unauthorized()
token = credentials.credentials.strip()
if not token:
if settings.allow_development_anonymous_user_access and settings.app_env.strip().lower() in {"development", "dev"}:
return AuthRole.USER
_raise_unauthorized()
return _resolve_token_role(token=token, settings=settings)

View File

@@ -14,7 +14,7 @@ from fastapi.responses import FileResponse, Response, StreamingResponse
from sqlalchemy import or_, func, select
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.config import get_settings, is_inline_preview_mime_type_safe
from app.db.base import get_session
from app.models.document import Document, DocumentStatus
from app.schemas.documents import (
@@ -448,14 +448,22 @@ def download_document(document_id: UUID, session: Session = Depends(get_session)
@router.get("/{document_id}/preview")
def preview_document(document_id: UUID, session: Session = Depends(get_session)) -> FileResponse:
"""Streams the original document inline when browser rendering is supported."""
"""Streams trusted-safe MIME types inline and forces attachment for active script-capable types."""
document = session.execute(select(Document).where(Document.id == document_id)).scalar_one_or_none()
if document is None:
raise HTTPException(status_code=404, detail="Document not found")
original_path = absolute_path(document.stored_relative_path)
return FileResponse(path=original_path, media_type=document.mime_type)
common_headers = {"X-Content-Type-Options": "nosniff"}
if not is_inline_preview_mime_type_safe(document.mime_type):
return FileResponse(
path=original_path,
filename=document.original_filename,
media_type="application/octet-stream",
headers=common_headers,
)
return FileResponse(path=original_path, media_type=document.mime_type, headers=common_headers)
@router.get("/{document_id}/thumbnail")

View File

@@ -19,6 +19,9 @@ class Settings(BaseSettings):
app_env: str = "development"
database_url: str = "postgresql+psycopg://dcm:dcm@db:5432/dcm"
redis_url: str = "redis://redis:6379/0"
redis_security_mode: str = "auto"
redis_tls_mode: str = "auto"
allow_development_anonymous_user_access: bool = True
storage_root: Path = Path("/data/storage")
upload_chunk_size: int = 4 * 1024 * 1024
max_upload_files_per_request: int = 50
@@ -26,6 +29,7 @@ class Settings(BaseSettings):
max_upload_request_size_bytes: int = 100 * 1024 * 1024
max_zip_members: int = 250
max_zip_depth: int = 2
max_zip_descendants_per_root: int = 1000
max_zip_member_uncompressed_bytes: int = 25 * 1024 * 1024
max_zip_total_uncompressed_bytes: int = 150 * 1024 * 1024
max_zip_compression_ratio: float = 120.0
@@ -44,12 +48,13 @@ class Settings(BaseSettings):
default_openai_timeout_seconds: int = 45
default_openai_handwriting_enabled: bool = True
default_openai_api_key: str = ""
app_settings_encryption_key: str = ""
default_summary_model: str = "gpt-4.1-mini"
default_routing_model: str = "gpt-4.1-mini"
typesense_protocol: str = "http"
typesense_host: str = "typesense"
typesense_port: int = 8108
typesense_api_key: str = "dcm-typesense-key"
typesense_api_key: str = ""
typesense_collection_name: str = "documents"
typesense_timeout_seconds: int = 120
typesense_num_retries: int = 0
@@ -58,6 +63,111 @@ class Settings(BaseSettings):
LOCAL_HOSTNAME_SUFFIXES = (".local", ".internal", ".home.arpa")
SCRIPT_CAPABLE_INLINE_MIME_TYPES = frozenset(
{
"application/ecmascript",
"application/javascript",
"application/x-javascript",
"application/xhtml+xml",
"image/svg+xml",
"text/ecmascript",
"text/html",
"text/javascript",
}
)
SCRIPT_CAPABLE_XML_MIME_TYPES = frozenset({"application/xml", "text/xml"})
REDIS_SECURITY_MODES = frozenset({"auto", "strict", "compat"})
REDIS_TLS_MODES = frozenset({"auto", "required", "allow_insecure"})
def _is_production_environment(app_env: str) -> bool:
"""Returns whether the runtime environment should enforce production-only security gates."""
normalized = app_env.strip().lower()
return normalized in {"production", "prod"}
def _normalize_redis_security_mode(raw_mode: str) -> str:
"""Normalizes Redis security mode values into one supported mode."""
normalized = raw_mode.strip().lower()
if normalized not in REDIS_SECURITY_MODES:
return "auto"
return normalized
def _normalize_redis_tls_mode(raw_mode: str) -> str:
"""Normalizes Redis TLS mode values into one supported mode."""
normalized = raw_mode.strip().lower()
if normalized not in REDIS_TLS_MODES:
return "auto"
return normalized
def validate_redis_url_security(
redis_url: str,
*,
app_env: str | None = None,
security_mode: str | None = None,
tls_mode: str | None = None,
) -> str:
"""Validates Redis URL security posture with production fail-closed defaults."""
settings = get_settings()
resolved_app_env = app_env if app_env is not None else settings.app_env
resolved_security_mode = (
_normalize_redis_security_mode(security_mode)
if security_mode is not None
else _normalize_redis_security_mode(settings.redis_security_mode)
)
resolved_tls_mode = (
_normalize_redis_tls_mode(tls_mode)
if tls_mode is not None
else _normalize_redis_tls_mode(settings.redis_tls_mode)
)
candidate = redis_url.strip()
if not candidate:
raise ValueError("Redis URL must not be empty")
parsed = urlparse(candidate)
scheme = parsed.scheme.lower()
if scheme not in {"redis", "rediss"}:
raise ValueError("Redis URL must use redis:// or rediss://")
if not parsed.hostname:
raise ValueError("Redis URL must include a hostname")
strict_security = (
resolved_security_mode == "strict"
or (resolved_security_mode == "auto" and _is_production_environment(resolved_app_env))
)
require_tls = (
resolved_tls_mode == "required"
or (resolved_tls_mode == "auto" and strict_security)
)
has_password = bool(parsed.password and parsed.password.strip())
uses_tls = scheme == "rediss"
if strict_security and not has_password:
raise ValueError("Redis URL must include authentication when security mode is strict")
if require_tls and not uses_tls:
raise ValueError("Redis URL must use rediss:// when TLS is required")
return candidate
def is_inline_preview_mime_type_safe(mime_type: str) -> bool:
"""Returns whether a MIME type is safe to serve inline from untrusted document uploads."""
normalized = mime_type.split(";", 1)[0].strip().lower() if mime_type else ""
if not normalized:
return False
if normalized in SCRIPT_CAPABLE_INLINE_MIME_TYPES:
return False
if normalized in SCRIPT_CAPABLE_XML_MIME_TYPES or normalized.endswith("+xml"):
return False
return True
def _normalize_allowlist(allowlist: object) -> tuple[str, ...]:

View File

@@ -1,7 +1,13 @@
"""Persistent single-user application settings service backed by host-mounted storage."""
import base64
import binascii
import hashlib
import hmac
import json
import os
import re
import secrets
from pathlib import Path
from typing import Any
@@ -57,6 +63,172 @@ DEFAULT_ROUTING_PROMPT = (
"Confidence must be between 0 and 1."
)
PROVIDER_API_KEY_CIPHERTEXT_PREFIX = "enc-v1"
PROVIDER_API_KEY_KEYFILE_NAME = ".settings-api-key"
PROVIDER_API_KEY_STREAM_CONTEXT = b"dcm-provider-api-key-stream"
PROVIDER_API_KEY_AUTH_CONTEXT = b"dcm-provider-api-key-auth"
PROVIDER_API_KEY_NONCE_BYTES = 16
PROVIDER_API_KEY_TAG_BYTES = 32
def _settings_api_key_path() -> Path:
"""Returns the storage path used for local symmetric encryption key persistence."""
return settings.storage_root / PROVIDER_API_KEY_KEYFILE_NAME
def _write_private_text_file(path: Path, content: str) -> None:
"""Writes text files with restrictive owner-only permissions for local secret material."""
path.parent.mkdir(parents=True, exist_ok=True)
file_descriptor = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle:
handle.write(content)
os.chmod(path, 0o600)
def _urlsafe_b64encode_no_padding(data: bytes) -> str:
"""Encodes bytes to URL-safe base64 without padding for compact JSON persistence."""
return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
def _urlsafe_b64decode_no_padding(data: str) -> bytes:
"""Decodes URL-safe base64 values that may omit trailing padding characters."""
padded = data + "=" * (-len(data) % 4)
return base64.urlsafe_b64decode(padded.encode("ascii"))
def _derive_provider_api_key_key() -> bytes:
"""Resolves the master key used to encrypt provider API keys for settings storage."""
configured_key = settings.app_settings_encryption_key.strip()
if configured_key:
try:
decoded = _urlsafe_b64decode_no_padding(configured_key)
if len(decoded) >= 32:
return decoded[:32]
except (binascii.Error, ValueError):
pass
return hashlib.sha256(configured_key.encode("utf-8")).digest()
key_path = _settings_api_key_path()
if key_path.exists():
try:
persisted = key_path.read_text(encoding="utf-8").strip()
decoded = _urlsafe_b64decode_no_padding(persisted)
if len(decoded) >= 32:
return decoded[:32]
except (OSError, UnicodeDecodeError, binascii.Error, ValueError):
pass
generated = secrets.token_bytes(32)
_write_private_text_file(key_path, _urlsafe_b64encode_no_padding(generated))
return generated
def _xor_bytes(left: bytes, right: bytes) -> bytes:
"""Applies byte-wise XOR for equal-length byte sequences."""
return bytes(first ^ second for first, second in zip(left, right))
def _derive_stream_cipher_bytes(master_key: bytes, nonce: bytes, length: int) -> bytes:
"""Derives deterministic stream bytes from HMAC-SHA256 blocks for payload masking."""
stream = bytearray()
counter = 0
while len(stream) < length:
counter_bytes = counter.to_bytes(4, "big")
block = hmac.new(
master_key,
PROVIDER_API_KEY_STREAM_CONTEXT + nonce + counter_bytes,
hashlib.sha256,
).digest()
stream.extend(block)
counter += 1
return bytes(stream[:length])
def _encrypt_provider_api_key(value: str) -> str:
"""Encrypts one provider API key for at-rest JSON persistence."""
normalized = value.strip()
if not normalized:
return ""
plaintext = normalized.encode("utf-8")
master_key = _derive_provider_api_key_key()
nonce = secrets.token_bytes(PROVIDER_API_KEY_NONCE_BYTES)
keystream = _derive_stream_cipher_bytes(master_key, nonce, len(plaintext))
ciphertext = _xor_bytes(plaintext, keystream)
tag = hmac.new(
master_key,
PROVIDER_API_KEY_AUTH_CONTEXT + nonce + ciphertext,
hashlib.sha256,
).digest()
payload = nonce + ciphertext + tag
encoded = _urlsafe_b64encode_no_padding(payload)
return f"{PROVIDER_API_KEY_CIPHERTEXT_PREFIX}:{encoded}"
def _decrypt_provider_api_key(value: str) -> str:
"""Decrypts provider API key ciphertext while rejecting tampered payloads."""
normalized = value.strip()
if not normalized:
return ""
if not normalized.startswith(f"{PROVIDER_API_KEY_CIPHERTEXT_PREFIX}:"):
return normalized
encoded_payload = normalized.split(":", 1)[1]
if not encoded_payload:
raise AppSettingsValidationError("Provider API key ciphertext is missing payload bytes")
try:
payload = _urlsafe_b64decode_no_padding(encoded_payload)
except (binascii.Error, ValueError) as error:
raise AppSettingsValidationError("Provider API key ciphertext is not valid base64") from error
minimum_length = PROVIDER_API_KEY_NONCE_BYTES + PROVIDER_API_KEY_TAG_BYTES
if len(payload) < minimum_length:
raise AppSettingsValidationError("Provider API key ciphertext payload is truncated")
nonce = payload[:PROVIDER_API_KEY_NONCE_BYTES]
ciphertext = payload[PROVIDER_API_KEY_NONCE_BYTES:-PROVIDER_API_KEY_TAG_BYTES]
received_tag = payload[-PROVIDER_API_KEY_TAG_BYTES:]
master_key = _derive_provider_api_key_key()
expected_tag = hmac.new(
master_key,
PROVIDER_API_KEY_AUTH_CONTEXT + nonce + ciphertext,
hashlib.sha256,
).digest()
if not hmac.compare_digest(received_tag, expected_tag):
raise AppSettingsValidationError("Provider API key ciphertext integrity check failed")
keystream = _derive_stream_cipher_bytes(master_key, nonce, len(ciphertext))
plaintext = _xor_bytes(ciphertext, keystream)
try:
return plaintext.decode("utf-8").strip()
except UnicodeDecodeError as error:
raise AppSettingsValidationError("Provider API key ciphertext is not valid UTF-8") from error
def _read_provider_api_key(provider_payload: dict[str, Any]) -> str:
"""Reads provider API key values from encrypted or legacy plaintext settings payloads."""
encrypted_value = provider_payload.get("api_key_encrypted")
if isinstance(encrypted_value, str) and encrypted_value.strip():
try:
return _decrypt_provider_api_key(encrypted_value)
except AppSettingsValidationError:
return ""
plaintext_value = provider_payload.get("api_key")
if plaintext_value is None:
return ""
return str(plaintext_value).strip()
def _default_settings() -> dict[str, Any]:
"""Builds default settings including providers and model task bindings."""
@@ -243,8 +415,17 @@ def _normalize_provider(
if provider_type != "openai_compatible":
provider_type = "openai_compatible"
api_key_value = payload.get("api_key", fallback_values.get("api_key", defaults["api_key"]))
api_key = str(api_key_value).strip() if api_key_value is not None else ""
payload_api_key = _read_provider_api_key(payload)
fallback_api_key = _read_provider_api_key(fallback_values)
default_api_key = _read_provider_api_key(defaults)
if "api_key" in payload and payload.get("api_key") is not None:
api_key = str(payload.get("api_key")).strip()
elif payload_api_key:
api_key = payload_api_key
elif fallback_api_key:
api_key = fallback_api_key
else:
api_key = default_api_key
raw_base_url = str(payload.get("base_url", fallback_values.get("base_url", defaults["base_url"]))).strip()
if not raw_base_url:
@@ -266,6 +447,7 @@ def _normalize_provider(
)
),
"api_key": api_key,
"api_key_encrypted": _encrypt_provider_api_key(api_key),
}
@@ -653,6 +835,26 @@ def _sanitize_settings(payload: dict[str, Any]) -> dict[str, Any]:
}
def _serialize_settings_for_storage(payload: dict[str, Any]) -> dict[str, Any]:
"""Converts sanitized runtime payload into storage-safe form without plaintext provider keys."""
storage_payload = dict(payload)
providers_storage: list[dict[str, Any]] = []
for provider in payload.get("providers", []):
if not isinstance(provider, dict):
continue
provider_storage = dict(provider)
plaintext_api_key = str(provider_storage.pop("api_key", "")).strip()
encrypted_api_key = str(provider_storage.get("api_key_encrypted", "")).strip()
if plaintext_api_key:
encrypted_api_key = _encrypt_provider_api_key(plaintext_api_key)
provider_storage["api_key_encrypted"] = encrypted_api_key
providers_storage.append(provider_storage)
storage_payload["providers"] = providers_storage
return storage_payload
def ensure_app_settings() -> None:
"""Creates a settings file with defaults when no persisted settings are present."""
@@ -662,7 +864,7 @@ def ensure_app_settings() -> None:
return
defaults = _sanitize_settings(_default_settings())
path.write_text(json.dumps(defaults, indent=2), encoding="utf-8")
_write_private_text_file(path, json.dumps(_serialize_settings_for_storage(defaults), indent=2))
def _read_raw_settings() -> dict[str, Any]:
@@ -682,7 +884,8 @@ def _write_settings(payload: dict[str, Any]) -> None:
path = _settings_path()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
storage_payload = _serialize_settings_for_storage(payload)
_write_private_text_file(path, json.dumps(storage_payload, indent=2))
def read_app_settings() -> dict[str, Any]:
@@ -879,16 +1082,21 @@ def update_app_settings(
def read_handwriting_provider_settings() -> dict[str, Any]:
"""Returns OCR settings in legacy shape for the handwriting transcription service."""
"""Returns OCR settings in legacy shape with DNS-revalidated provider base URL safety checks."""
runtime = read_task_runtime_settings(TASK_OCR_HANDWRITING)
provider = runtime["provider"]
task = runtime["task"]
raw_base_url = str(provider.get("base_url", settings.default_openai_base_url))
try:
normalized_base_url = normalize_and_validate_provider_base_url(raw_base_url, resolve_dns=True)
except ValueError as error:
raise AppSettingsValidationError(str(error)) from error
return {
"provider": provider["provider_type"],
"enabled": bool(task.get("enabled", True)),
"openai_base_url": str(provider.get("base_url", settings.default_openai_base_url)),
"openai_base_url": normalized_base_url,
"openai_model": str(task.get("model", settings.default_openai_model)),
"openai_timeout_seconds": int(provider.get("timeout_seconds", settings.default_openai_timeout_seconds)),
"openai_api_key": str(provider.get("api_key", "")),

View File

@@ -299,17 +299,24 @@ def extract_text_content(filename: str, data: bytes, mime_type: str) -> Extracti
)
def extract_archive_members(data: bytes, depth: int = 0) -> list[ArchiveMember]:
"""Extracts processable ZIP members within configured decompression safety budgets."""
def extract_archive_members(data: bytes, depth: int = 0, max_members: int | None = None) -> list[ArchiveMember]:
"""Extracts processable ZIP members with depth-aware and decompression safety guardrails."""
members: list[ArchiveMember] = []
if depth > settings.max_zip_depth:
normalized_depth = max(0, depth)
if normalized_depth >= settings.max_zip_depth:
return members
member_limit = settings.max_zip_members
if max_members is not None:
member_limit = max(0, min(settings.max_zip_members, int(max_members)))
if member_limit <= 0:
return members
total_uncompressed_bytes = 0
try:
with zipfile.ZipFile(io.BytesIO(data)) as archive:
infos = [info for info in archive.infolist() if not info.is_dir()][: settings.max_zip_members]
infos = [info for info in archive.infolist() if not info.is_dir()][:member_limit]
for info in infos:
if info.file_size <= 0:
continue

View File

@@ -10,6 +10,7 @@ from typing import Any
from openai import APIConnectionError, APIError, APITimeoutError, OpenAI
from PIL import Image, ImageOps
from app.core.config import normalize_and_validate_provider_base_url
from app.services.app_settings import DEFAULT_OCR_PROMPT, read_handwriting_provider_settings
MAX_IMAGE_SIDE = 2000
@@ -151,12 +152,17 @@ def _normalize_image_bytes(image_data: bytes) -> tuple[bytes, str]:
def _create_client(provider_settings: dict[str, Any]) -> OpenAI:
"""Creates an OpenAI client configured for compatible endpoints and timeouts."""
"""Creates an OpenAI client configured with DNS-revalidated endpoint and request timeout controls."""
api_key = str(provider_settings.get("openai_api_key", "")).strip() or "no-key-required"
raw_base_url = str(provider_settings.get("openai_base_url", "")).strip()
try:
normalized_base_url = normalize_and_validate_provider_base_url(raw_base_url, resolve_dns=True)
except ValueError as error:
raise HandwritingTranscriptionError(f"invalid_provider_base_url:{error}") from error
return OpenAI(
api_key=api_key,
base_url=str(provider_settings["openai_base_url"]),
base_url=normalized_base_url,
timeout=int(provider_settings["openai_timeout_seconds"]),
)

View File

@@ -3,16 +3,17 @@
from redis import Redis
from rq import Queue
from app.core.config import get_settings
from app.core.config import get_settings, validate_redis_url_security
settings = get_settings()
def get_redis() -> Redis:
"""Creates a Redis connection from configured URL."""
"""Creates a Redis connection after enforcing URL security policy checks."""
return Redis.from_url(settings.redis_url)
secure_redis_url = validate_redis_url_security(settings.redis_url)
return Redis.from_url(secure_redis_url)
def get_processing_queue() -> Queue:

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.db.base import SessionLocal
from app.models.document import Document, DocumentStatus
from app.services.app_settings import (
@@ -37,6 +38,13 @@ from app.services.storage import absolute_path, compute_sha256, store_bytes, wri
from app.worker.queue import get_processing_queue
settings = get_settings()
ARCHIVE_ROOT_ID_METADATA_KEY = "archive_root_document_id"
ARCHIVE_DEPTH_METADATA_KEY = "archive_depth"
ARCHIVE_DESCENDANT_COUNT_METADATA_KEY = "archive_descendant_count"
def _cleanup_processing_logs_with_settings(session: Session) -> None:
"""Applies configured processing log retention while trimming old log entries."""
@@ -48,13 +56,80 @@ def _cleanup_processing_logs_with_settings(session: Session) -> None:
)
def _metadata_non_negative_int(value: object, fallback: int = 0) -> int:
"""Parses metadata values as non-negative integers with safe fallback behavior."""
try:
parsed = int(value)
except (TypeError, ValueError):
return fallback
return max(0, parsed)
def _metadata_uuid(value: object) -> uuid.UUID | None:
"""Parses metadata values as UUIDs while tolerating malformed legacy values."""
if not isinstance(value, str) or not value.strip():
return None
try:
return uuid.UUID(value.strip())
except ValueError:
return None
def _resolve_archive_lineage(session: Session, document: Document) -> tuple[uuid.UUID, int]:
"""Resolves archive root document id and depth for metadata propagation compatibility."""
metadata_json = dict(document.metadata_json)
metadata_root = _metadata_uuid(metadata_json.get(ARCHIVE_ROOT_ID_METADATA_KEY))
metadata_depth = _metadata_non_negative_int(metadata_json.get(ARCHIVE_DEPTH_METADATA_KEY), fallback=0)
if metadata_root is not None:
return metadata_root, metadata_depth
if not document.is_archive_member:
return document.id, 0
depth = 0
root_document_id = document.id
parent_document_id = document.parent_document_id
visited: set[uuid.UUID] = {document.id}
while parent_document_id is not None and parent_document_id not in visited:
visited.add(parent_document_id)
parent_document = session.execute(select(Document).where(Document.id == parent_document_id)).scalar_one_or_none()
if parent_document is None:
break
depth += 1
root_document_id = parent_document.id
parent_document_id = parent_document.parent_document_id
return root_document_id, depth
def _merge_archive_metadata(document: Document, **updates: object) -> None:
"""Applies archive metadata updates while preserving unrelated document metadata keys."""
metadata_json = dict(document.metadata_json)
metadata_json.update(updates)
document.metadata_json = metadata_json
def _load_archive_root_for_update(session: Session, root_document_id: uuid.UUID) -> Document | None:
"""Loads archive root row with write lock to serialize descendant-count budget updates."""
return session.execute(
select(Document).where(Document.id == root_document_id).with_for_update()
).scalar_one_or_none()
def _create_archive_member_document(
parent: Document,
member_name: str,
member_data: bytes,
mime_type: str,
archive_root_document_id: uuid.UUID,
archive_depth: int,
) -> Document:
"""Creates a child document entity for a file extracted from an uploaded archive."""
"""Creates child document entities with lineage metadata for recursive archive processing."""
extension = Path(member_name).suffix.lower()
stored_relative_path = store_bytes(member_name, member_data)
@@ -68,7 +143,12 @@ def _create_archive_member_document(
size_bytes=len(member_data),
logical_path=parent.logical_path,
tags=list(parent.tags),
metadata_json={"origin": "archive", "parent": str(parent.id)},
metadata_json={
"origin": "archive",
"parent": str(parent.id),
ARCHIVE_ROOT_ID_METADATA_KEY: str(archive_root_document_id),
ARCHIVE_DEPTH_METADATA_KEY: archive_depth,
},
is_archive_member=True,
archived_member_path=member_name,
parent_document_id=parent.id,
@@ -110,16 +190,46 @@ def process_document_task(document_id: str) -> None:
if document.extension == ".zip":
child_ids: list[str] = []
archive_root_document_id, archive_depth = _resolve_archive_lineage(session=session, document=document)
_merge_archive_metadata(
document,
**{
ARCHIVE_ROOT_ID_METADATA_KEY: str(archive_root_document_id),
ARCHIVE_DEPTH_METADATA_KEY: archive_depth,
},
)
root_document = _load_archive_root_for_update(session=session, root_document_id=archive_root_document_id)
if root_document is None:
root_document = document
root_metadata_json = dict(root_document.metadata_json)
existing_descendant_count = _metadata_non_negative_int(
root_metadata_json.get(ARCHIVE_DESCENDANT_COUNT_METADATA_KEY),
fallback=0,
)
max_descendants_per_root = max(0, int(settings.max_zip_descendants_per_root))
remaining_descendant_budget = max(0, max_descendants_per_root - existing_descendant_count)
extraction_member_cap = remaining_descendant_budget
log_processing_event(
session=session,
stage="archive",
event="Archive extraction started",
level="info",
document=document,
payload_json={"size_bytes": len(data)},
payload_json={
"size_bytes": len(data),
"archive_root_document_id": str(archive_root_document_id),
"archive_depth": archive_depth,
"remaining_descendant_budget": remaining_descendant_budget,
},
)
try:
members = extract_archive_members(data)
members = extract_archive_members(
data,
depth=archive_depth,
max_members=extraction_member_cap,
)
for member in members:
mime_type = sniff_mime(member.data)
child = _create_archive_member_document(
@@ -127,6 +237,8 @@ def process_document_task(document_id: str) -> None:
member_name=member.name,
member_data=member.data,
mime_type=mime_type,
archive_root_document_id=archive_root_document_id,
archive_depth=archive_depth + 1,
)
session.add(child)
session.flush()
@@ -142,8 +254,27 @@ def process_document_task(document_id: str) -> None:
"member_name": member.name,
"member_size_bytes": len(member.data),
"mime_type": mime_type,
"archive_root_document_id": str(archive_root_document_id),
"archive_depth": archive_depth + 1,
},
)
updated_root_metadata = dict(root_document.metadata_json)
updated_root_metadata[ARCHIVE_ROOT_ID_METADATA_KEY] = str(archive_root_document_id)
updated_root_metadata[ARCHIVE_DEPTH_METADATA_KEY] = 0
updated_root_metadata[ARCHIVE_DESCENDANT_COUNT_METADATA_KEY] = existing_descendant_count + len(child_ids)
root_document.metadata_json = updated_root_metadata
limit_flags: dict[str, object] = {}
if archive_depth >= settings.max_zip_depth:
limit_flags["max_depth_reached"] = True
if remaining_descendant_budget <= 0:
limit_flags["max_descendants_reached"] = True
elif len(child_ids) >= remaining_descendant_budget:
limit_flags["max_descendants_reached"] = True
if limit_flags:
_merge_archive_metadata(document, **limit_flags)
document.status = DocumentStatus.PROCESSED
document.extracted_text = f"archive with {len(members)} files"
log_processing_event(
@@ -152,7 +283,13 @@ def process_document_task(document_id: str) -> None:
event="Archive extraction completed",
level="info",
document=document,
payload_json={"member_count": len(members)},
payload_json={
"member_count": len(members),
"archive_root_document_id": str(archive_root_document_id),
"archive_depth": archive_depth,
"descendant_count": existing_descendant_count + len(child_ids),
"remaining_descendant_budget": max(0, remaining_descendant_budget - len(child_ids)),
},
)
except Exception as exc:
document.status = DocumentStatus.ERROR
@@ -231,7 +368,10 @@ def process_document_task(document_id: str) -> None:
event="Archive child job enqueued",
level="info",
document_id=uuid.UUID(child_id),
payload_json={"parent_document_id": str(document.id)},
payload_json={
"parent_document_id": str(document.id),
"archive_root_document_id": str(archive_root_document_id),
},
)
session.commit()
return