from pathlib import Path import pytest from sqlalchemy import select from app.adapters.storage.s3 import S3StorageAdapter from app.models.entities import Backend, BlobReplica, CacheEntry @pytest.mark.anyio async def test_finalize_enqueues_file_reconciliation_job(api_client, session_factory, fake_s3) -> None: create_backend_response = await api_client.post( "/api/backends", json={ "name": "s3-main", "type": "s3", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": {"bucket": "iron-test"}, }, ) assert create_backend_response.status_code == 200 s3_backend_id = create_backend_response.json()["backend"]["id"] create_response = await api_client.post( "/files", headers={"Tus-Resumable": "1.0.0", "Upload-Length": "11"}, ) upload_id = create_response.headers["Location"].rsplit("/", 1)[-1] await api_client.patch( f"/files/{upload_id}", headers={ "Tus-Resumable": "1.0.0", "Upload-Offset": "0", "Content-Type": "application/offset+octet-stream", }, content=b"hello world", ) finalize_response = await api_client.post( f"/api/uploads/{upload_id}/finalize", json={"directory_id": "dir_root", "filename": "movie.mp4"}, ) assert finalize_response.status_code == 200 jobs_response = await api_client.get("/api/jobs") assert jobs_response.status_code == 200 assert any(item["kind"] == "reconcile_file_replicas" for item in jobs_response.json()["items"]) run_response = await api_client.post("/api/jobs/run-pending") assert run_response.status_code == 200 assert run_response.json()["completed"] >= 1 file_id = finalize_response.json()["file"]["id"] async with session_factory() as session: backend = await session.get(Backend, s3_backend_id) blob = ( await session.execute( select(BlobReplica).join(Backend, BlobReplica.backend_id == Backend.id).where(Backend.id == backend.id) ) ).scalars().all() assert len(blob) == 1 download_response = await api_client.get(f"/api/files/{file_id}/download") assert download_response.status_code == 200 assert download_response.content == b"hello world" @pytest.mark.anyio async def test_remote_replica_can_be_fetched_when_local_copy_is_missing( api_client, session_factory, runtime_paths, fake_s3, ) -> None: await api_client.post( "/api/backends", json={ "name": "s3-main", "type": "s3", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": {"bucket": "iron-test"}, }, ) create_response = await api_client.post( "/files", headers={"Tus-Resumable": "1.0.0", "Upload-Length": "12"}, ) upload_id = create_response.headers["Location"].rsplit("/", 1)[-1] await api_client.patch( f"/files/{upload_id}", headers={ "Tus-Resumable": "1.0.0", "Upload-Offset": "0", "Content-Type": "application/offset+octet-stream", }, content=b"hello remote", ) finalize_response = await api_client.post( f"/api/uploads/{upload_id}/finalize", json={"directory_id": "dir_root", "filename": "movie.mp4"}, ) file_id = finalize_response.json()["file"]["id"] await api_client.post("/api/jobs/run-pending") local_storage_dir = Path(runtime_paths["local_storage_dir"]) for item in local_storage_dir.rglob("*"): if item.is_file(): item.unlink() download_response = await api_client.get(f"/api/files/{file_id}/download") assert download_response.status_code == 200 assert download_response.content == b"hello remote" async with session_factory() as session: cached_entries = (await session.execute(select(CacheEntry))).scalars().all() assert len(cached_entries) == 1 assert Path(cached_entries[0].local_path).exists() @pytest.mark.anyio async def test_file_reconcile_enqueues_convergence_job(api_client, fake_s3) -> None: await api_client.post( "/api/backends", json={ "name": "s3-main", "type": "s3", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": {"bucket": "iron-test"}, }, ) create_response = await api_client.post( "/files", headers={"Tus-Resumable": "1.0.0", "Upload-Length": "5"}, ) upload_id = create_response.headers["Location"].rsplit("/", 1)[-1] await api_client.patch( f"/files/{upload_id}", headers={ "Tus-Resumable": "1.0.0", "Upload-Offset": "0", "Content-Type": "application/offset+octet-stream", }, content=b"hello", ) finalize_response = await api_client.post( f"/api/uploads/{upload_id}/finalize", json={"directory_id": "dir_root", "filename": "repair.txt"}, ) file_id = finalize_response.json()["file"]["id"] await api_client.post("/api/jobs/run-pending") jobs_before = await api_client.get("/api/jobs") completed_jobs = [item for item in jobs_before.json()["items"] if item["kind"] == "reconcile_file_replicas"] assert completed_jobs reconcile_response = await api_client.post(f"/api/files/{file_id}/reconcile") assert reconcile_response.status_code == 200 assert reconcile_response.json()["enqueued_jobs"] == 1 assert reconcile_response.json()["reconciliation_job_kind"] == "reconcile_file_replicas" @pytest.mark.anyio async def test_enqueue_health_checks_creates_jobs(api_client, fake_s3) -> None: await api_client.post( "/api/backends", json={ "name": "s3-main", "type": "s3", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": {"bucket": "iron-test"}, }, ) enqueue_response = await api_client.post("/api/jobs/enqueue-health-checks") assert enqueue_response.status_code == 200 assert enqueue_response.json()["enqueued_jobs"] >= 2 run_response = await api_client.post("/api/jobs/run-pending") assert run_response.status_code == 200 assert run_response.json()["completed"] >= 2 @pytest.mark.anyio async def test_webdav_replica_can_be_fetched_when_local_copy_is_missing( api_client, runtime_paths, fake_webdav, ) -> None: await api_client.post( "/api/backends", json={ "name": "openlist-main", "type": "webdav", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": { "endpoint_url": fake_webdav["endpoint_url"], "username": fake_webdav["username"], "password": fake_webdav["password"], "root_path": "iron", }, }, ) create_response = await api_client.post( "/files", headers={"Tus-Resumable": "1.0.0", "Upload-Length": "12"}, ) upload_id = create_response.headers["Location"].rsplit("/", 1)[-1] await api_client.patch( f"/files/{upload_id}", headers={ "Tus-Resumable": "1.0.0", "Upload-Offset": "0", "Content-Type": "application/offset+octet-stream", }, content=b"hello webdav", ) finalize_response = await api_client.post( f"/api/uploads/{upload_id}/finalize", json={"directory_id": "dir_root", "filename": "movie.mp4"}, ) file_id = finalize_response.json()["file"]["id"] await api_client.post("/api/jobs/run-pending") local_storage_dir = Path(runtime_paths["local_storage_dir"]) for item in local_storage_dir.rglob("*"): if item.is_file(): item.unlink() download_response = await api_client.get(f"/api/files/{file_id}/download") assert download_response.status_code == 200 assert download_response.content == b"hello webdav" @pytest.mark.anyio async def test_enqueue_full_reconcile_creates_system_job(api_client) -> None: response = await api_client.post("/api/jobs/enqueue-full-reconcile") assert response.status_code == 200 assert response.json()["enqueued_jobs"] == 1 jobs_response = await api_client.get("/api/jobs") assert jobs_response.status_code == 200 assert any(item["kind"] == "reconcile_all_files" for item in jobs_response.json()["items"]) @pytest.mark.anyio async def test_reconcile_job_retries_until_backend_recovers( api_client, monkeypatch: pytest.MonkeyPatch, fake_s3, ) -> None: await api_client.post( "/api/backends", json={ "name": "s3-main", "type": "s3", "stability_class": "stable", "read_priority": 90, "write_priority": 90, "config": {"bucket": "iron-test"}, }, ) async def unhealthy_check(self): return {"status": "unhealthy", "detail": "simulated outage"} monkeypatch.setattr(S3StorageAdapter, "check", unhealthy_check) create_response = await api_client.post( "/files", headers={"Tus-Resumable": "1.0.0", "Upload-Length": "5"}, ) upload_id = create_response.headers["Location"].rsplit("/", 1)[-1] await api_client.patch( f"/files/{upload_id}", headers={ "Tus-Resumable": "1.0.0", "Upload-Offset": "0", "Content-Type": "application/offset+octet-stream", }, content=b"hello", ) finalize_response = await api_client.post( f"/api/uploads/{upload_id}/finalize", json={"directory_id": "dir_root", "filename": "retry.txt"}, ) file_id = finalize_response.json()["file"]["id"] first_run = await api_client.post("/api/jobs/run-pending") assert first_run.status_code == 200 assert first_run.json()["failed"] >= 1 jobs_response = await api_client.get("/api/jobs") reconcile_job = next(item for item in jobs_response.json()["items"] if item["kind"] == "reconcile_file_replicas") assert reconcile_job["status"] == "retryable" async def healthy_check(self): return {"status": "healthy", "detail": self.config["bucket"]} monkeypatch.setattr(S3StorageAdapter, "check", healthy_check) retry_response = await api_client.post(f"/api/jobs/{reconcile_job['id']}/retry") assert retry_response.status_code == 200 second_run = await api_client.post("/api/jobs/run-pending") assert second_run.status_code == 200 assert second_run.json()["completed"] >= 1 download_response = await api_client.get(f"/api/files/{file_id}/download") assert download_response.status_code == 200 assert download_response.content == b"hello"