from __future__ import annotations import hashlib import os import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from scripts.real_openlist_chaos_acceptance import default_cache_root, local_backend_base_path from scripts.real_openlist_multi_backend_acceptance import MultiBackendConfig, ensure_backends from scripts.real_openlist_acceptance import IronClient def main() -> int: config = MultiBackendConfig.from_env() client = IronClient(config.base) file_class = os.getenv("IRON_MULTI_CHAOS_FILE_CLASS", config.file_class) size_bytes = int(os.getenv("IRON_MULTI_CHAOS_FILE_SIZE_BYTES", str(2 * 1024 * 1024))) print(f"[1/11] logging into Iron at {config.base.iron_base_url}") client.login() print("[2/11] ensuring three OpenList/Aliyun backends") backends = ensure_backends(client, config) for backend in backends[:3]: check = client.check_backend(backend["id"]) if check["status"] != "healthy": raise RuntimeError(f"backend {backend['name']} unhealthy: {check}") print(f" {backend['name']}: healthy") backends = backends[:3] backend_ids = [backend["id"] for backend in backends] original_policy = next(policy for policy in client.list_policies() if policy["file_class"] == file_class) try: print("[3/11] forcing two stable remote replicas across three backends") client.upsert_policy( file_class, require_local=True, stable_replica_count=2, opportunistic_replica_count=0, preferred_backend_ids=backend_ids, excluded_backend_ids=[], max_non_local_size_bytes=None, ) print(f"[4/11] uploading degraded-mode probe file ({size_bytes} bytes)") content = build_deterministic_bytes(size_bytes) file_id = client.upload_bytes( filename=f"multi-chaos-{int(time.time())}.bin", content=content, directory_id=config.base.directory_id, ) detail = client.wait_for_ready_replica_set(file_id, {"bkd_local_default", backend_ids[0], backend_ids[1]}) print(f" initial replicas: {detail['ready_replica_backend_ids']}") print("[5/11] disabling the first remote backend and confirming it stays out of placement") client.disable_backend(backend_ids[0]) disabled_detail = client.file_detail(file_id) print(f" disabled backend: {backends[0]['name']}") print("[6/11] forcing remote recovery while one backend is disabled") local_replica = next(replica for replica in disabled_detail["replicas"] if replica["backend_id"] == "bkd_local_default") local_path = Path(local_backend_base_path(client)) / local_replica["storage_key"] cache_path = default_cache_root() / local_replica["storage_key"] if local_path.exists(): local_path.unlink() if cache_path.exists(): cache_path.unlink() recovered = client.download_file(file_id) if recovered != content: raise RuntimeError("recovered bytes did not match original content") print(" remote recovery succeeded from surviving replicas") print("[7/11] uploading a second file while the first remote backend is disabled") degraded_file_id = client.upload_bytes( filename=f"multi-chaos-degraded-{int(time.time())}.bin", content=content[::-1], directory_id=config.base.directory_id, ) degraded_detail = client.wait_for_ready_replica_set( degraded_file_id, {"bkd_local_default", backend_ids[1], backend_ids[2]}, ) ready_ids = set(degraded_detail["ready_replica_backend_ids"]) if backend_ids[0] in ready_ids: raise RuntimeError("disabled backend unexpectedly received a replica") print(f" degraded upload replicas: {degraded_detail['ready_replica_backend_ids']}") print("[8/11] re-enabling the first backend") client.enable_backend(backend_ids[0]) check = client.check_backend(backend_ids[0]) if check["status"] != "healthy": raise RuntimeError(f"re-enabled backend unhealthy: {check}") print(" backend healthy again") print("[9/11] raising policy to three stable remote replicas and reconciling") client.upsert_policy( file_class, require_local=True, stable_replica_count=3, opportunistic_replica_count=0, preferred_backend_ids=backend_ids, excluded_backend_ids=[], max_non_local_size_bytes=None, ) healed_detail = client.wait_for_ready_replica_set(degraded_file_id, {"bkd_local_default", *backend_ids}) print(f" healed replicas: {healed_detail['ready_replica_backend_ids']}") print("[10/11] deleting the healed file to recycle bin and restoring it") client.delete_file(degraded_file_id) recycle_items = client.list_recycle_bin() if not any(item["id"] == degraded_file_id for item in recycle_items): raise RuntimeError("healed file not found in recycle bin") client.restore_file(degraded_file_id) restored_detail = client.wait_for_ready_replica_set(degraded_file_id, {"bkd_local_default", *backend_ids}) restored_bytes = client.download_file(degraded_file_id) if restored_bytes != content[::-1]: raise RuntimeError("restored multi-backend file bytes did not match") print(f" restored replicas: {restored_detail['ready_replica_backend_ids']}") print("[11/11] final integrity digests") print(f" probe sha256: {hashlib.sha256(content).hexdigest()}") print(f" degraded sha256: {hashlib.sha256(content[::-1]).hexdigest()}") print("Multi-backend chaos acceptance passed.") return 0 finally: for backend_id in backend_ids: backend = next((item for item in client.list_backends() if item["id"] == backend_id), None) if backend is not None and not backend.get("enabled", True): client.enable_backend(backend_id) client.upsert_policy( file_class, require_local=bool(original_policy["require_local"]), stable_replica_count=int(original_policy["stable_replica_count"]), opportunistic_replica_count=int(original_policy["opportunistic_replica_count"]), preferred_backend_ids=list(original_policy["preferred_backend_ids"]), excluded_backend_ids=list(original_policy["excluded_backend_ids"]), max_non_local_size_bytes=original_policy["max_non_local_size_bytes"], ) def build_deterministic_bytes(size_bytes: int) -> bytes: chunk = hashlib.sha256(b"iron-multi-chaos-seed").digest() repeats = (size_bytes // len(chunk)) + 1 return (chunk * repeats)[:size_bytes] if __name__ == "__main__": raise SystemExit(main())