"""Celery application + periodic schedule (spec §45 background workers, §40 Daily Global Scan). AUDIT (StockLab overhaul, Part 25 — Redis/Celery production review): before this pass, Celery ran entirely on library defaults — no worker concurrency limit (defaults to CPU count, which can over-commit a small server), no `acks_late` (a worker killed mid-task silently drops that task instead of requeuing it), unlimited task prefetch (one worker can hoard many queued tasks instead of leaving them for other workers), and no task time limit (a hung task runs forever, tying up a worker slot indefinitely). All five are now sourced from `Settings`, conservative-by-default for the target small single-server deployment (docs/DEPLOYMENT.md §9), and overridable via env. """ from __future__ import annotations from celery import Celery from celery.schedules import crontab from app.core.config import get_settings settings = get_settings() celery_app = Celery("stocklab", broker=settings.REDIS_URL, backend=settings.REDIS_URL) celery_app.conf.update( task_serializer="json", accept_content=["json"], result_serializer="json", timezone="UTC", enable_utc=True, task_track_started=True, worker_concurrency=settings.CELERY_WORKER_CONCURRENCY, worker_prefetch_multiplier=settings.CELERY_WORKER_PREFETCH_MULTIPLIER, task_acks_late=settings.CELERY_TASK_ACKS_LATE, task_time_limit=settings.CELERY_TASK_TIME_LIMIT_SECONDS, task_soft_time_limit=settings.CELERY_TASK_SOFT_TIME_LIMIT_SECONDS, imports=( "app.workers.discovery", "app.workers.ingest", "app.workers.peer_groups", "app.workers.recompute", "app.workers.token_cleanup", ), ) celery_app.conf.beat_schedule = { "daily-universe-ingestion": { "task": "app.workers.ingest.ingest_universe_task", "schedule": crontab(hour=2, minute=0), # spec §52: universe refresh minimum daily }, # AUDIT FIX (StockLab final engineering pass, Part B2): a universe-wide recompute with a real # peer universe, scheduled between ingestion (02:00) and the global scan (04:00). Before this, # nothing ever computed peer groups, so every percentile score was the neutral 50 that # percentile_rank() returns for an empty peer list -- see docs/AUDIT_PEER_GROUPS_B2.md. The # per-security recompute that ingestion triggers also gets real peers now, but rebuilds the # universe per call; this pass loads it once for the whole set. "daily-peer-universe-recompute": { "task": "app.workers.peer_groups.recompute_universe_task", "schedule": crontab(hour=3, minute=0), }, "daily-global-scan": { "task": "app.workers.discovery.daily_global_scan_task", "schedule": crontab(hour=4, minute=0), # after ingestion + recompute have settled }, # AUDIT FIX (StockLab overhaul, final engineering pass, Part A4): keeps the revoked_tokens # table (app/models/governance.py::RevokedToken) from growing forever -- see # app/core/token_revocation.py::prune_expired_revoked_tokens() for why deleting rows past # their own expiry is safe. Runs off-peak, independent of the ingestion/scan pair above (no # ordering dependency on either). "prune-expired-revoked-tokens": { "task": "app.workers.token_cleanup.prune_expired_revoked_tokens_task", "schedule": crontab(hour=3, minute=30), }, }