from __future__ import annotations import json from dataclasses import dataclass from datetime import UTC, datetime, timedelta from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import get_settings from app.core.exceptions import NotFoundError from app.core.ids import new_id from app.models.entities import Job from app.repositories.backend_repository import BackendRepository from app.repositories.blob_replica_repository import BlobReplicaRepository from app.repositories.file_repository import FileRepository from app.repositories.job_repository import JobRepository from app.services.policy_service import PolicyService from app.services.preview_service import PreviewService from app.services.storage_service import StorageService class JobNotFoundError(NotFoundError): code = "job_not_found" message = "Job does not exist." @dataclass class JobRunResult: processed: int completed: int failed: int class JobService: def __init__(self, session: AsyncSession) -> None: self.session = session self.settings = get_settings() self.repository = JobRepository(session) self.file_repository = FileRepository(session) self.backend_repository = BackendRepository(session) self.blob_replica_repository = BlobReplicaRepository(session) self.storage_service = StorageService(session) self.policy_service = PolicyService(session) self.preview_service = PreviewService(session) async def list_jobs(self) -> list[Job]: return await self.repository.list_all() async def get_job(self, job_id: str) -> Job: job = await self.repository.get_by_id(job_id) if job is None: raise JobNotFoundError() return job async def enqueue_blob_replication(self, blob_id: str, backend_id: str) -> Job: payload_json = json.dumps({"blob_id": blob_id, "backend_id": backend_id}, sort_keys=True) existing = await self.repository.find_active_by_kind_and_payload("replicate_blob", payload_json) if existing is not None: return existing job = Job( id=new_id("job"), kind="replicate_blob", status="queued", payload_json=payload_json, run_after=datetime.now(UTC), last_error=None, ) created = await self.repository.create(job) await self.session.commit() return created async def enqueue_file_reconcile(self, file_id: str) -> Job: payload_json = json.dumps({"file_id": file_id}, sort_keys=True) existing = await self.repository.find_active_by_kind_and_payload("reconcile_file_replicas", payload_json) if existing is not None: return existing job = Job( id=new_id("job"), kind="reconcile_file_replicas", status="queued", payload_json=payload_json, run_after=datetime.now(UTC), last_error=None, ) created = await self.repository.create(job) await self.session.commit() return created async def enqueue_full_reconcile(self) -> Job: payload_json = json.dumps({}, sort_keys=True) existing = await self.repository.find_active_by_kind_and_payload("reconcile_all_files", payload_json) if existing is not None: return existing job = Job( id=new_id("job"), kind="reconcile_all_files", status="queued", payload_json=payload_json, run_after=datetime.now(UTC), last_error=None, ) created = await self.repository.create(job) await self.session.commit() return created async def enqueue_backend_health_check(self, backend_id: str) -> Job: payload_json = json.dumps({"backend_id": backend_id}, sort_keys=True) existing = await self.repository.find_active_by_kind_and_payload("check_backend_health", payload_json) if existing is not None: return existing job = Job( id=new_id("job"), kind="check_backend_health", status="queued", payload_json=payload_json, run_after=datetime.now(UTC), last_error=None, ) created = await self.repository.create(job) await self.session.commit() return created async def enqueue_preview_generation(self, file_id: str) -> Job: payload_json = json.dumps({"file_id": file_id}, sort_keys=True) existing = await self.repository.find_active_by_kind_and_payload("generate_preview_artifacts", payload_json) if existing is not None: return existing job = Job( id=new_id("job"), kind="generate_preview_artifacts", status="queued", payload_json=payload_json, run_after=datetime.now(UTC), last_error=None, ) created = await self.repository.create(job) await self.session.commit() return created async def retry_job(self, job_id: str) -> Job: job = await self.get_job(job_id) job.status = "retryable" job.run_after = datetime.now(UTC) job.last_error = None saved = await self.repository.save(job) await self.session.commit() return saved async def run_pending_jobs(self, limit: int | None = None) -> JobRunResult: now = datetime.now(UTC) runnable = await self.repository.list_runnable(now, limit or self.settings.job_batch_size) completed = 0 failed = 0 for job in runnable: job.status = "running" await self.repository.save(job) await self.session.commit() try: await self._execute(job) job.status = "completed" job.last_error = None completed += 1 except Exception as exc: job.attempt_count += 1 job.last_error = str(exc) if job.attempt_count >= job.max_attempts: job.status = "failed" else: job.status = "retryable" job.run_after = datetime.now(UTC) + timedelta(seconds=min(60, 2**job.attempt_count)) failed += 1 await self.repository.save(job) await self.session.commit() return JobRunResult(processed=len(runnable), completed=completed, failed=failed) async def _execute(self, job: Job) -> None: payload = json.loads(job.payload_json) if job.kind == "reconcile_all_files": files = await self.file_repository.list_active() for file_entry in files: await self.enqueue_file_reconcile(file_entry.id) return if job.kind == "reconcile_file_replicas": file_id = payload["file_id"] file_entry = await self.file_repository.get_by_id(file_id) if file_entry is None or file_entry.is_deleted or file_entry.deleted_at is not None: raise ValueError("File is unavailable.") blob = await self.file_repository.get_primary_blob_for_file(file_id) if blob is None: raise ValueError("Primary blob does not exist.") decision = await self.policy_service.get_placement_decision(file_entry.mime_type, size_bytes=blob.size_bytes) desired_backends = decision.backends existing_replicas = await self.blob_replica_repository.list_by_blob(blob.id) ready_backend_ids = {replica.backend_id for replica in existing_replicas if replica.status == "ready"} missing_backends = [backend for backend in desired_backends if backend.id not in ready_backend_ids] if not missing_backends: return source_path = await self.storage_service.resolve_best_local_path(blob) failed_targets: list[str] = [] for backend in missing_backends: try: status, _detail = await self.storage_service.check_backend(backend) if status != "healthy": failed_targets.append(backend.name) continue await self.storage_service.persist_blob_to_backend(blob, backend, str(source_path)) except Exception: failed_targets.append(backend.name) if failed_targets: raise ValueError( "Reconciliation is still waiting on backends: " + ", ".join(sorted(set(failed_targets))) ) return if job.kind == "replicate_blob": blob_id = payload["blob_id"] backend_id = payload["backend_id"] blob = await self.file_repository.get_blob_by_id(blob_id) if blob is None: raise ValueError("Blob does not exist.") backend = await self.backend_repository.get_by_id(backend_id) if backend is None or not backend.is_enabled: raise ValueError("Backend is unavailable.") source_path = await self.storage_service.resolve_best_local_path(blob) await self.storage_service.persist_blob_to_backend(blob, backend, str(source_path)) return if job.kind == "check_backend_health": backend_id = payload["backend_id"] backend = await self.backend_repository.get_by_id(backend_id) if backend is None or not backend.is_enabled: raise ValueError("Backend is unavailable.") status, _detail = await self.storage_service.check_backend(backend) backend.last_health_status = status backend.last_health_checked_at = datetime.now(UTC) await self.backend_repository.save(backend) await self.session.commit() return if job.kind == "generate_preview_artifacts": file_id = payload["file_id"] await self.preview_service.generate_preview_artifacts_for_file(file_id) return raise ValueError(f"Unsupported job kind: {job.kind}")