You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
271 lines
9.1 KiB
271 lines
9.1 KiB
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.adapters.storage.s3 import S3StorageAdapter
|
|
from app.models.entities import Backend, BlobReplica, CacheEntry
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_finalize_enqueues_file_reconciliation_job(api_client, session_factory, fake_s3) -> None:
|
|
create_backend_response = await api_client.post(
|
|
"/api/backends",
|
|
json={
|
|
"name": "s3-main",
|
|
"type": "s3",
|
|
"stability_class": "stable",
|
|
"read_priority": 90,
|
|
"write_priority": 90,
|
|
"config": {"bucket": "iron-test"},
|
|
},
|
|
)
|
|
assert create_backend_response.status_code == 200
|
|
s3_backend_id = create_backend_response.json()["backend"]["id"]
|
|
|
|
create_response = await api_client.post(
|
|
"/files",
|
|
headers={"Tus-Resumable": "1.0.0", "Upload-Length": "11"},
|
|
)
|
|
upload_id = create_response.headers["Location"].rsplit("/", 1)[-1]
|
|
|
|
await api_client.patch(
|
|
f"/files/{upload_id}",
|
|
headers={
|
|
"Tus-Resumable": "1.0.0",
|
|
"Upload-Offset": "0",
|
|
"Content-Type": "application/offset+octet-stream",
|
|
},
|
|
content=b"hello world",
|
|
)
|
|
finalize_response = await api_client.post(
|
|
f"/api/uploads/{upload_id}/finalize",
|
|
json={"directory_id": "dir_root", "filename": "movie.mp4"},
|
|
)
|
|
assert finalize_response.status_code == 200
|
|
|
|
jobs_response = await api_client.get("/api/jobs")
|
|
assert jobs_response.status_code == 200
|
|
assert any(item["kind"] == "reconcile_file_replicas" for item in jobs_response.json()["items"])
|
|
|
|
run_response = await api_client.post("/api/jobs/run-pending")
|
|
assert run_response.status_code == 200
|
|
assert run_response.json()["completed"] >= 1
|
|
|
|
file_id = finalize_response.json()["file"]["id"]
|
|
async with session_factory() as session:
|
|
backend = await session.get(Backend, s3_backend_id)
|
|
blob = (
|
|
await session.execute(
|
|
select(BlobReplica).join(Backend, BlobReplica.backend_id == Backend.id).where(Backend.id == backend.id)
|
|
)
|
|
).scalars().all()
|
|
assert len(blob) == 1
|
|
|
|
download_response = await api_client.get(f"/api/files/{file_id}/download")
|
|
assert download_response.status_code == 200
|
|
assert download_response.content == b"hello world"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_remote_replica_can_be_fetched_when_local_copy_is_missing(
|
|
api_client,
|
|
session_factory,
|
|
runtime_paths,
|
|
fake_s3,
|
|
) -> None:
|
|
await api_client.post(
|
|
"/api/backends",
|
|
json={
|
|
"name": "s3-main",
|
|
"type": "s3",
|
|
"stability_class": "stable",
|
|
"read_priority": 90,
|
|
"write_priority": 90,
|
|
"config": {"bucket": "iron-test"},
|
|
},
|
|
)
|
|
|
|
create_response = await api_client.post(
|
|
"/files",
|
|
headers={"Tus-Resumable": "1.0.0", "Upload-Length": "12"},
|
|
)
|
|
upload_id = create_response.headers["Location"].rsplit("/", 1)[-1]
|
|
await api_client.patch(
|
|
f"/files/{upload_id}",
|
|
headers={
|
|
"Tus-Resumable": "1.0.0",
|
|
"Upload-Offset": "0",
|
|
"Content-Type": "application/offset+octet-stream",
|
|
},
|
|
content=b"hello remote",
|
|
)
|
|
finalize_response = await api_client.post(
|
|
f"/api/uploads/{upload_id}/finalize",
|
|
json={"directory_id": "dir_root", "filename": "movie.mp4"},
|
|
)
|
|
file_id = finalize_response.json()["file"]["id"]
|
|
|
|
await api_client.post("/api/jobs/run-pending")
|
|
|
|
local_storage_dir = Path(runtime_paths["local_storage_dir"])
|
|
for item in local_storage_dir.rglob("*"):
|
|
if item.is_file():
|
|
item.unlink()
|
|
|
|
download_response = await api_client.get(f"/api/files/{file_id}/download")
|
|
assert download_response.status_code == 200
|
|
assert download_response.content == b"hello remote"
|
|
|
|
async with session_factory() as session:
|
|
cached_entries = (await session.execute(select(CacheEntry))).scalars().all()
|
|
assert len(cached_entries) == 1
|
|
assert Path(cached_entries[0].local_path).exists()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_file_reconcile_enqueues_convergence_job(api_client, fake_s3) -> None:
|
|
await api_client.post(
|
|
"/api/backends",
|
|
json={
|
|
"name": "s3-main",
|
|
"type": "s3",
|
|
"stability_class": "stable",
|
|
"read_priority": 90,
|
|
"write_priority": 90,
|
|
"config": {"bucket": "iron-test"},
|
|
},
|
|
)
|
|
|
|
create_response = await api_client.post(
|
|
"/files",
|
|
headers={"Tus-Resumable": "1.0.0", "Upload-Length": "5"},
|
|
)
|
|
upload_id = create_response.headers["Location"].rsplit("/", 1)[-1]
|
|
await api_client.patch(
|
|
f"/files/{upload_id}",
|
|
headers={
|
|
"Tus-Resumable": "1.0.0",
|
|
"Upload-Offset": "0",
|
|
"Content-Type": "application/offset+octet-stream",
|
|
},
|
|
content=b"hello",
|
|
)
|
|
finalize_response = await api_client.post(
|
|
f"/api/uploads/{upload_id}/finalize",
|
|
json={"directory_id": "dir_root", "filename": "repair.txt"},
|
|
)
|
|
file_id = finalize_response.json()["file"]["id"]
|
|
await api_client.post("/api/jobs/run-pending")
|
|
|
|
jobs_before = await api_client.get("/api/jobs")
|
|
completed_jobs = [item for item in jobs_before.json()["items"] if item["kind"] == "reconcile_file_replicas"]
|
|
assert completed_jobs
|
|
|
|
reconcile_response = await api_client.post(f"/api/files/{file_id}/reconcile")
|
|
assert reconcile_response.status_code == 200
|
|
assert reconcile_response.json()["enqueued_jobs"] == 1
|
|
assert reconcile_response.json()["reconciliation_job_kind"] == "reconcile_file_replicas"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_enqueue_health_checks_creates_jobs(api_client, fake_s3) -> None:
|
|
await api_client.post(
|
|
"/api/backends",
|
|
json={
|
|
"name": "s3-main",
|
|
"type": "s3",
|
|
"stability_class": "stable",
|
|
"read_priority": 90,
|
|
"write_priority": 90,
|
|
"config": {"bucket": "iron-test"},
|
|
},
|
|
)
|
|
|
|
enqueue_response = await api_client.post("/api/jobs/enqueue-health-checks")
|
|
assert enqueue_response.status_code == 200
|
|
assert enqueue_response.json()["enqueued_jobs"] >= 2
|
|
|
|
run_response = await api_client.post("/api/jobs/run-pending")
|
|
assert run_response.status_code == 200
|
|
assert run_response.json()["completed"] >= 2
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_enqueue_full_reconcile_creates_system_job(api_client) -> None:
|
|
response = await api_client.post("/api/jobs/enqueue-full-reconcile")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["enqueued_jobs"] == 1
|
|
|
|
jobs_response = await api_client.get("/api/jobs")
|
|
assert jobs_response.status_code == 200
|
|
assert any(item["kind"] == "reconcile_all_files" for item in jobs_response.json()["items"])
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_reconcile_job_retries_until_backend_recovers(
|
|
api_client,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
fake_s3,
|
|
) -> None:
|
|
await api_client.post(
|
|
"/api/backends",
|
|
json={
|
|
"name": "s3-main",
|
|
"type": "s3",
|
|
"stability_class": "stable",
|
|
"read_priority": 90,
|
|
"write_priority": 90,
|
|
"config": {"bucket": "iron-test"},
|
|
},
|
|
)
|
|
|
|
async def unhealthy_check(self):
|
|
return {"status": "unhealthy", "detail": "simulated outage"}
|
|
|
|
monkeypatch.setattr(S3StorageAdapter, "check", unhealthy_check)
|
|
|
|
create_response = await api_client.post(
|
|
"/files",
|
|
headers={"Tus-Resumable": "1.0.0", "Upload-Length": "5"},
|
|
)
|
|
upload_id = create_response.headers["Location"].rsplit("/", 1)[-1]
|
|
await api_client.patch(
|
|
f"/files/{upload_id}",
|
|
headers={
|
|
"Tus-Resumable": "1.0.0",
|
|
"Upload-Offset": "0",
|
|
"Content-Type": "application/offset+octet-stream",
|
|
},
|
|
content=b"hello",
|
|
)
|
|
finalize_response = await api_client.post(
|
|
f"/api/uploads/{upload_id}/finalize",
|
|
json={"directory_id": "dir_root", "filename": "retry.txt"},
|
|
)
|
|
file_id = finalize_response.json()["file"]["id"]
|
|
|
|
first_run = await api_client.post("/api/jobs/run-pending")
|
|
assert first_run.status_code == 200
|
|
assert first_run.json()["failed"] >= 1
|
|
|
|
jobs_response = await api_client.get("/api/jobs")
|
|
reconcile_job = next(item for item in jobs_response.json()["items"] if item["kind"] == "reconcile_file_replicas")
|
|
assert reconcile_job["status"] == "retryable"
|
|
|
|
async def healthy_check(self):
|
|
return {"status": "healthy", "detail": self.config["bucket"]}
|
|
|
|
monkeypatch.setattr(S3StorageAdapter, "check", healthy_check)
|
|
|
|
retry_response = await api_client.post(f"/api/jobs/{reconcile_job['id']}/retry")
|
|
assert retry_response.status_code == 200
|
|
|
|
second_run = await api_client.post("/api/jobs/run-pending")
|
|
assert second_run.status_code == 200
|
|
assert second_run.json()["completed"] >= 1
|
|
|
|
download_response = await api_client.get(f"/api/files/{file_id}/download")
|
|
assert download_response.status_code == 200
|
|
assert download_response.content == b"hello"
|
|
|