You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
254 lines
10 KiB
254 lines
10 KiB
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.config import get_settings
|
|
from app.core.exceptions import NotFoundError
|
|
from app.core.ids import new_id
|
|
from app.models.entities import Job
|
|
from app.repositories.backend_repository import BackendRepository
|
|
from app.repositories.blob_replica_repository import BlobReplicaRepository
|
|
from app.repositories.file_repository import FileRepository
|
|
from app.repositories.job_repository import JobRepository
|
|
from app.services.policy_service import PolicyService
|
|
from app.services.preview_service import PreviewService
|
|
from app.services.storage_service import StorageService
|
|
|
|
|
|
class JobNotFoundError(NotFoundError):
|
|
code = "job_not_found"
|
|
message = "Job does not exist."
|
|
|
|
|
|
@dataclass
|
|
class JobRunResult:
|
|
processed: int
|
|
completed: int
|
|
failed: int
|
|
|
|
|
|
class JobService:
|
|
def __init__(self, session: AsyncSession) -> None:
|
|
self.session = session
|
|
self.settings = get_settings()
|
|
self.repository = JobRepository(session)
|
|
self.file_repository = FileRepository(session)
|
|
self.backend_repository = BackendRepository(session)
|
|
self.blob_replica_repository = BlobReplicaRepository(session)
|
|
self.storage_service = StorageService(session)
|
|
self.policy_service = PolicyService(session)
|
|
self.preview_service = PreviewService(session)
|
|
|
|
async def list_jobs(self) -> list[Job]:
|
|
return await self.repository.list_all()
|
|
|
|
async def get_job(self, job_id: str) -> Job:
|
|
job = await self.repository.get_by_id(job_id)
|
|
if job is None:
|
|
raise JobNotFoundError()
|
|
return job
|
|
|
|
async def enqueue_blob_replication(self, blob_id: str, backend_id: str) -> Job:
|
|
payload_json = json.dumps({"blob_id": blob_id, "backend_id": backend_id}, sort_keys=True)
|
|
existing = await self.repository.find_active_by_kind_and_payload("replicate_blob", payload_json)
|
|
if existing is not None:
|
|
return existing
|
|
job = Job(
|
|
id=new_id("job"),
|
|
kind="replicate_blob",
|
|
status="queued",
|
|
payload_json=payload_json,
|
|
run_after=datetime.now(UTC),
|
|
last_error=None,
|
|
)
|
|
created = await self.repository.create(job)
|
|
await self.session.commit()
|
|
return created
|
|
|
|
async def enqueue_file_reconcile(self, file_id: str) -> Job:
|
|
payload_json = json.dumps({"file_id": file_id}, sort_keys=True)
|
|
existing = await self.repository.find_active_by_kind_and_payload("reconcile_file_replicas", payload_json)
|
|
if existing is not None:
|
|
return existing
|
|
job = Job(
|
|
id=new_id("job"),
|
|
kind="reconcile_file_replicas",
|
|
status="queued",
|
|
payload_json=payload_json,
|
|
run_after=datetime.now(UTC),
|
|
last_error=None,
|
|
)
|
|
created = await self.repository.create(job)
|
|
await self.session.commit()
|
|
return created
|
|
|
|
async def enqueue_full_reconcile(self) -> Job:
|
|
payload_json = json.dumps({}, sort_keys=True)
|
|
existing = await self.repository.find_active_by_kind_and_payload("reconcile_all_files", payload_json)
|
|
if existing is not None:
|
|
return existing
|
|
job = Job(
|
|
id=new_id("job"),
|
|
kind="reconcile_all_files",
|
|
status="queued",
|
|
payload_json=payload_json,
|
|
run_after=datetime.now(UTC),
|
|
last_error=None,
|
|
)
|
|
created = await self.repository.create(job)
|
|
await self.session.commit()
|
|
return created
|
|
|
|
async def enqueue_backend_health_check(self, backend_id: str) -> Job:
|
|
payload_json = json.dumps({"backend_id": backend_id}, sort_keys=True)
|
|
existing = await self.repository.find_active_by_kind_and_payload("check_backend_health", payload_json)
|
|
if existing is not None:
|
|
return existing
|
|
job = Job(
|
|
id=new_id("job"),
|
|
kind="check_backend_health",
|
|
status="queued",
|
|
payload_json=payload_json,
|
|
run_after=datetime.now(UTC),
|
|
last_error=None,
|
|
)
|
|
created = await self.repository.create(job)
|
|
await self.session.commit()
|
|
return created
|
|
|
|
async def enqueue_preview_generation(self, file_id: str) -> Job:
|
|
payload_json = json.dumps({"file_id": file_id}, sort_keys=True)
|
|
existing = await self.repository.find_active_by_kind_and_payload("generate_preview_artifacts", payload_json)
|
|
if existing is not None:
|
|
return existing
|
|
job = Job(
|
|
id=new_id("job"),
|
|
kind="generate_preview_artifacts",
|
|
status="queued",
|
|
payload_json=payload_json,
|
|
run_after=datetime.now(UTC),
|
|
last_error=None,
|
|
)
|
|
created = await self.repository.create(job)
|
|
await self.session.commit()
|
|
return created
|
|
|
|
async def retry_job(self, job_id: str) -> Job:
|
|
job = await self.get_job(job_id)
|
|
job.status = "retryable"
|
|
job.run_after = datetime.now(UTC)
|
|
job.last_error = None
|
|
saved = await self.repository.save(job)
|
|
await self.session.commit()
|
|
return saved
|
|
|
|
async def run_pending_jobs(self, limit: int | None = None) -> JobRunResult:
|
|
now = datetime.now(UTC)
|
|
runnable = await self.repository.list_runnable(now, limit or self.settings.job_batch_size)
|
|
completed = 0
|
|
failed = 0
|
|
for job in runnable:
|
|
job.status = "running"
|
|
await self.repository.save(job)
|
|
await self.session.commit()
|
|
try:
|
|
await self._execute(job)
|
|
job.status = "completed"
|
|
job.last_error = None
|
|
completed += 1
|
|
except Exception as exc:
|
|
job.attempt_count += 1
|
|
job.last_error = str(exc)
|
|
if job.attempt_count >= job.max_attempts:
|
|
job.status = "failed"
|
|
else:
|
|
job.status = "retryable"
|
|
job.run_after = datetime.now(UTC) + timedelta(seconds=min(60, 2**job.attempt_count))
|
|
failed += 1
|
|
await self.repository.save(job)
|
|
await self.session.commit()
|
|
return JobRunResult(processed=len(runnable), completed=completed, failed=failed)
|
|
|
|
async def _execute(self, job: Job) -> None:
|
|
payload = json.loads(job.payload_json)
|
|
if job.kind == "reconcile_all_files":
|
|
files = await self.file_repository.list_active()
|
|
for file_entry in files:
|
|
await self.enqueue_file_reconcile(file_entry.id)
|
|
return
|
|
|
|
if job.kind == "reconcile_file_replicas":
|
|
file_id = payload["file_id"]
|
|
file_entry = await self.file_repository.get_by_id(file_id)
|
|
if file_entry is None or file_entry.is_deleted or file_entry.deleted_at is not None:
|
|
raise ValueError("File is unavailable.")
|
|
|
|
blob = await self.file_repository.get_primary_blob_for_file(file_id)
|
|
if blob is None:
|
|
raise ValueError("Primary blob does not exist.")
|
|
|
|
decision = await self.policy_service.get_placement_decision(file_entry.mime_type, size_bytes=blob.size_bytes)
|
|
desired_backends = decision.backends
|
|
existing_replicas = await self.blob_replica_repository.list_by_blob(blob.id)
|
|
ready_backend_ids = {replica.backend_id for replica in existing_replicas if replica.status == "ready"}
|
|
|
|
missing_backends = [backend for backend in desired_backends if backend.id not in ready_backend_ids]
|
|
if not missing_backends:
|
|
return
|
|
|
|
source_path = await self.storage_service.resolve_best_local_path(blob)
|
|
failed_targets: list[str] = []
|
|
for backend in missing_backends:
|
|
try:
|
|
status, _detail = await self.storage_service.check_backend(backend)
|
|
if status != "healthy":
|
|
failed_targets.append(backend.name)
|
|
continue
|
|
await self.storage_service.persist_blob_to_backend(blob, backend, str(source_path))
|
|
except Exception:
|
|
failed_targets.append(backend.name)
|
|
|
|
if failed_targets:
|
|
raise ValueError(
|
|
"Reconciliation is still waiting on backends: " + ", ".join(sorted(set(failed_targets)))
|
|
)
|
|
return
|
|
|
|
if job.kind == "replicate_blob":
|
|
blob_id = payload["blob_id"]
|
|
backend_id = payload["backend_id"]
|
|
|
|
blob = await self.file_repository.get_blob_by_id(blob_id)
|
|
if blob is None:
|
|
raise ValueError("Blob does not exist.")
|
|
|
|
backend = await self.backend_repository.get_by_id(backend_id)
|
|
if backend is None or not backend.is_enabled:
|
|
raise ValueError("Backend is unavailable.")
|
|
|
|
source_path = await self.storage_service.resolve_best_local_path(blob)
|
|
await self.storage_service.persist_blob_to_backend(blob, backend, str(source_path))
|
|
return
|
|
|
|
if job.kind == "check_backend_health":
|
|
backend_id = payload["backend_id"]
|
|
backend = await self.backend_repository.get_by_id(backend_id)
|
|
if backend is None or not backend.is_enabled:
|
|
raise ValueError("Backend is unavailable.")
|
|
status, _detail = await self.storage_service.check_backend(backend)
|
|
backend.last_health_status = status
|
|
backend.last_health_checked_at = datetime.now(UTC)
|
|
await self.backend_repository.save(backend)
|
|
await self.session.commit()
|
|
return
|
|
|
|
if job.kind == "generate_preview_artifacts":
|
|
file_id = payload["file_id"]
|
|
await self.preview_service.generate_preview_artifacts_for_file(file_id)
|
|
return
|
|
|
|
raise ValueError(f"Unsupported job kind: {job.kind}")
|
|
|