iron/scripts/real_openlist_multi_backend_chaos.py

from __future__ import annotations

import hashlib
import os
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

from scripts.real_openlist_chaos_acceptance import default_cache_root, local_backend_base_path
from scripts.real_openlist_multi_backend_acceptance import MultiBackendConfig, ensure_backends
from scripts.real_openlist_acceptance import IronClient


def main() -> int:
    config = MultiBackendConfig.from_env()
    client = IronClient(config.base)
    file_class = os.getenv("IRON_MULTI_CHAOS_FILE_CLASS", config.file_class)
    size_bytes = int(os.getenv("IRON_MULTI_CHAOS_FILE_SIZE_BYTES", str(2 * 1024 * 1024)))

    print(f"[1/11] logging into Iron at {config.base.iron_base_url}")
    client.login()

    print("[2/11] ensuring three OpenList/Aliyun backends")
    backends = ensure_backends(client, config)
    for backend in backends[:3]:
        check = client.check_backend(backend["id"])
        if check["status"] != "healthy":
            raise RuntimeError(f"backend {backend['name']} unhealthy: {check}")
        print(f"       {backend['name']}: healthy")

    backends = backends[:3]
    backend_ids = [backend["id"] for backend in backends]
    original_policy = next(policy for policy in client.list_policies() if policy["file_class"] == file_class)

    try:
        print("[3/11] forcing two stable remote replicas across three backends")
        client.upsert_policy(
            file_class,
            require_local=True,
            stable_replica_count=2,
            opportunistic_replica_count=0,
            preferred_backend_ids=backend_ids,
            excluded_backend_ids=[],
            max_non_local_size_bytes=None,
        )

        print(f"[4/11] uploading degraded-mode probe file ({size_bytes} bytes)")
        content = build_deterministic_bytes(size_bytes)
        file_id = client.upload_bytes(
            filename=f"multi-chaos-{int(time.time())}.bin",
            content=content,
            directory_id=config.base.directory_id,
        )
        detail = client.wait_for_ready_replica_set(file_id, {"bkd_local_default", backend_ids[0], backend_ids[1]})
        print(f"       initial replicas: {detail['ready_replica_backend_ids']}")

        print("[5/11] disabling the first remote backend and confirming it stays out of placement")
        client.disable_backend(backend_ids[0])
        disabled_detail = client.file_detail(file_id)
        print(f"       disabled backend: {backends[0]['name']}")

        print("[6/11] forcing remote recovery while one backend is disabled")
        local_replica = next(replica for replica in disabled_detail["replicas"] if replica["backend_id"] == "bkd_local_default")
        local_path = Path(local_backend_base_path(client)) / local_replica["storage_key"]
        cache_path = default_cache_root() / local_replica["storage_key"]
        if local_path.exists():
            local_path.unlink()
        if cache_path.exists():
            cache_path.unlink()
        recovered = client.download_file(file_id)
        if recovered != content:
            raise RuntimeError("recovered bytes did not match original content")
        print("       remote recovery succeeded from surviving replicas")

        print("[7/11] uploading a second file while the first remote backend is disabled")
        degraded_file_id = client.upload_bytes(
            filename=f"multi-chaos-degraded-{int(time.time())}.bin",
            content=content[::-1],
            directory_id=config.base.directory_id,
        )
        degraded_detail = client.wait_for_ready_replica_set(
            degraded_file_id,
            {"bkd_local_default", backend_ids[1], backend_ids[2]},
        )
        ready_ids = set(degraded_detail["ready_replica_backend_ids"])
        if backend_ids[0] in ready_ids:
            raise RuntimeError("disabled backend unexpectedly received a replica")
        print(f"       degraded upload replicas: {degraded_detail['ready_replica_backend_ids']}")

        print("[8/11] re-enabling the first backend")
        client.enable_backend(backend_ids[0])
        check = client.check_backend(backend_ids[0])
        if check["status"] != "healthy":
            raise RuntimeError(f"re-enabled backend unhealthy: {check}")
        print("       backend healthy again")

        print("[9/11] raising policy to three stable remote replicas and reconciling")
        client.upsert_policy(
            file_class,
            require_local=True,
            stable_replica_count=3,
            opportunistic_replica_count=0,
            preferred_backend_ids=backend_ids,
            excluded_backend_ids=[],
            max_non_local_size_bytes=None,
        )
        healed_detail = client.wait_for_ready_replica_set(degraded_file_id, {"bkd_local_default", *backend_ids})
        print(f"       healed replicas: {healed_detail['ready_replica_backend_ids']}")

        print("[10/11] deleting the healed file to recycle bin and restoring it")
        client.delete_file(degraded_file_id)
        recycle_items = client.list_recycle_bin()
        if not any(item["id"] == degraded_file_id for item in recycle_items):
            raise RuntimeError("healed file not found in recycle bin")
        client.restore_file(degraded_file_id)
        restored_detail = client.wait_for_ready_replica_set(degraded_file_id, {"bkd_local_default", *backend_ids})
        restored_bytes = client.download_file(degraded_file_id)
        if restored_bytes != content[::-1]:
            raise RuntimeError("restored multi-backend file bytes did not match")
        print(f"       restored replicas: {restored_detail['ready_replica_backend_ids']}")

        print("[11/11] final integrity digests")
        print(f"       probe sha256: {hashlib.sha256(content).hexdigest()}")
        print(f"       degraded sha256: {hashlib.sha256(content[::-1]).hexdigest()}")
        print("Multi-backend chaos acceptance passed.")
        return 0
    finally:
        for backend_id in backend_ids:
            backend = next((item for item in client.list_backends() if item["id"] == backend_id), None)
            if backend is not None and not backend.get("enabled", True):
                client.enable_backend(backend_id)
        client.upsert_policy(
            file_class,
            require_local=bool(original_policy["require_local"]),
            stable_replica_count=int(original_policy["stable_replica_count"]),
            opportunistic_replica_count=int(original_policy["opportunistic_replica_count"]),
            preferred_backend_ids=list(original_policy["preferred_backend_ids"]),
            excluded_backend_ids=list(original_policy["excluded_backend_ids"]),
            max_non_local_size_bytes=original_policy["max_non_local_size_bytes"],
        )


def build_deterministic_bytes(size_bytes: int) -> bytes:
    chunk = hashlib.sha256(b"iron-multi-chaos-seed").digest()
    repeats = (size_bytes // len(chunk)) + 1
    return (chunk * repeats)[:size_bytes]


if __name__ == "__main__":
    raise SystemExit(main())