feat(mainnet): harden NetUID 36 validator launch

Dockerfile

@@ -3,24 +3,29 @@
 # The validator scans on-chain reveals, dispatches duels to the GPU eval server,
 # crowns winners and sets weights. It never loads a model, so it needs no torch.
 # The GPU eval server uses Dockerfile.eval instead.
-FROM python:3.12-slim
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
 
 # ffprobe/ffmpeg for the video utilities; curl for healthchecks;
 # build-essential for any deps that need compiling.
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ffmpeg curl build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+RUN python -m venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH="/opt/venv/bin:${PATH}" \
+    LEOMA_UV_LOCK_PATH=/app/uv.lock
+
 WORKDIR /app
-RUN pip install --no-cache-dir uv
+RUN pip install --no-cache-dir uv==0.11.6
 
-COPY pyproject.toml README.md ./
+COPY pyproject.toml uv.lock README.md ./
 COPY leoma ./leoma
 
 # Install the package (non-editable for a production image). chain.toml ships
 # INSIDE the package via [tool.setuptools.package-data] — it is consensus-critical
 # and is read at import time, so it must exist in site-packages, not just in git.
-RUN uv pip install --system --no-cache .
+RUN uv sync --active --frozen --no-dev --no-editable --no-cache
 
 # Fail at BUILD time if the package cannot import. This is exactly the bug class
 # that shipped before (a missing chain.toml / missing numpy only surfaced when the

Dockerfile.eval

@@ -6,7 +6,7 @@
 #
 # It needs the [eval] extra (torch / diffusers / lpips / opencv / open_clip).
 # The validator image (Dockerfile) deliberately does NOT.
-FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04@sha256:2fcc4280646484290cc50dce5e65f388dd04352b07cbe89a635703bd1f9aedb6
 
 ENV DEBIAN_FRONTEND=noninteractive \
     PYTHONUNBUFFERED=1 \
@@ -22,13 +22,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  && rm -rf /var/lib/apt/lists/*
 
 RUN python3.12 -m venv /opt/venv
-ENV PATH="/opt/venv/bin:${PATH}"
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH="/opt/venv/bin:${PATH}"
 
 WORKDIR /app
-RUN pip install --no-cache-dir uv
+RUN pip install --no-cache-dir uv==0.11.6
 
 COPY pyproject.toml uv.lock README.md ./
 COPY leoma ./leoma
+ENV LEOMA_UV_LOCK_PATH=/app/uv.lock
 
 # The [eval] extra is the whole point of this image. Frozen sync makes rebuilds use
 # the exact tested CUDA/Diffusers/Transformers graph from uv.lock; an open-ended GPU

README.md

@@ -70,7 +70,9 @@ The validator scans reveals, dispatches duels to an eval server, crowns winners,
 
 - **Bittensor wallet** (coldkey + hotkey) registered as a validator on the subnet.
 - **A reachable eval server** (`EVAL_SERVER_URL`), typically an SSH tunnel to a GPU box.
-- **An own bucket** (`R2_OWN_BUCKET` + `R2_OWN_WRITE_*`) for durable king state.
+- **A private Hippius state bucket** (`HIPPIUS_OWN_BUCKET` +
+  `HIPPIUS_OWN_WRITE_*`) for durable king state. Legacy `R2_OWN_*` aliases remain
+  supported.
 - **A separate public-read dashboard bucket** (`LEOMA_DASHBOARD_BUCKET`) defaulting
   to the same owner endpoint/write credentials. Providers with bucket-scoped keys
   can use the optional `LEOMA_DASHBOARD_*` overrides. Expose only
@@ -81,12 +83,16 @@ The validator scans reveals, dispatches duels to an eval server, crowns winners,
 ```bash
 # Requires Python 3.12+
 pip install -e .            # or: uv pip install -e .
-cp env.validator.example .env   # fill in wallet, EVAL_SERVER_URL, R2_OWN_*
+cp env.validator.example .env   # fill in wallet, EVAL_SERVER_URL, HIPPIUS_OWN_*
 leoma serve                 # scan reveals -> duel -> crown -> set weights
 ```
 
-Or with Docker: `cp env.validator.example .env && docker compose up -d validator`. Mount your
-Bittensor wallets so the container can sign weight-setting transactions.
+For a local image build, use
+`cp env.validator.example .env && docker compose up -d validator`. Production is
+pull-only and digest-addressed: set `LEOMA_VALIDATOR_IMAGE_DIGEST` and
+`BITTENSOR_WALLETS_DIR`, then run
+`docker compose -f docker-compose.validator.production.yml up -d`. A mutable tag
+cannot be supplied to that production file.
 
 ---
 
@@ -98,13 +104,26 @@ downloaded anonymously; Hippius Hub credentials are optional for private genesis
 registry rate limits.
 
 ```bash
-pip install -e '.[eval]'
+uv sync --frozen --extra eval --no-dev
 cp env.eval.example .env    # fill in HIPPIUS_VIDEOS_READ_*
-leoma servers eval-server   # FastAPI on EVAL_SERVER_PORT (default 9000)
+uv run leoma servers eval-server   # FastAPI on EVAL_SERVER_PORT (default 9000)
 ```
 
 The validator reaches it over `EVAL_SERVER_URL` (default `http://localhost:9000`, usually an SSH
 tunnel). One duel runs at a time. See `ecosystem.eval.config.js` for a PM2 launcher.
+The evaluator refuses jobs unless its Python, CUDA, H100 capability, uv lock, and
+installed generation/scoring packages match the runtime pinned in `chain.toml`.
+The production 8xH100 layout runs four digest-pinned containers from
+`docker-compose.eval.8xh100.production.yml`, one isolated process per GPU pair.
+See `docs/PRODUCTION_8XH100_RUNBOOK.md` for image publication and rollout.
+
+For a live-chain rehearsal on a non-production validator host, run
+`leoma rehearse --ticks N --state-bucket <test-bucket> --state-prefix
+rehearsals/<run-id>`. It exercises
+the real reveal, evaluator, verdict, persistence, and dashboard paths while the
+central weight boundary suppresses every `set_weights` extrinsic. The rehearsal
+bucket must differ from the configured validator state bucket, and the process exits
+after `N` ticks.
 
 Before a new eval box is allowed to duel, prove it decodes the pinned corpus byte-identically:
 

docker-compose.eval.8xh100.production.yml

@@ -0,0 +1,74 @@
+# Leoma production evaluator fleet (run on the 8xH100 host).
+#
+# Four isolated processes each see exactly two physical GPUs. Inside a container the
+# assigned pair is remapped to cuda:0/cuda:1. Host ports bind loopback only and are
+# reached from the validator through the authenticated SSH tunnel in the runbook.
+x-eval-common: &eval-common
+  image: rendixnetwork/leoma@${LEOMA_EVAL_IMAGE_DIGEST:?set LEOMA_EVAL_IMAGE_DIGEST to sha256:<64hex>}
+  pull_policy: always
+  restart: unless-stopped
+  command: leoma servers eval-server
+  env_file:
+    - .env
+  environment: &eval-environment
+    PYTHONUNBUFFERED: "1"
+    EVAL_SERVER_HOST: "0.0.0.0"
+    EVAL_SERVER_PORT: "9000"
+    LEOMA_EVAL_TOKEN: ${LEOMA_EVAL_TOKEN:?set LEOMA_EVAL_TOKEN in .env}
+    HIPPIUS_HUB_TOKEN: ${HIPPIUS_HUB_TOKEN:-}
+    LEOMA_MODEL_CACHE_DIR: ${LEOMA_MODEL_CACHE_DIR:-/var/lib/leoma/models}
+    LEOMA_MAX_CACHED_SNAPSHOTS: ${LEOMA_MAX_CACHED_SNAPSHOTS:-8}
+    LEOMA_MIN_FREE_BYTES: ${LEOMA_MIN_FREE_BYTES:-322122547200}
+    OBJECT_STORAGE_BACKEND: ${OBJECT_STORAGE_BACKEND:-hippius}
+    HIPPIUS_ENDPOINT: ${HIPPIUS_ENDPOINT:-s3.hippius.com}
+    HIPPIUS_REGION: ${HIPPIUS_REGION:-decentralized}
+    HIPPIUS_SOURCE_BUCKET: ${HIPPIUS_SOURCE_BUCKET:-leoma-source}
+    HIPPIUS_VIDEOS_READ_ACCESS_KEY: ${HIPPIUS_VIDEOS_READ_ACCESS_KEY:-}
+    HIPPIUS_VIDEOS_READ_SECRET_KEY: ${HIPPIUS_VIDEOS_READ_SECRET_KEY:-}
+    LEOMA_CONCURRENT_GENERATION: "1"
+    LEOMA_KING_DEVICE: cuda:0
+    LEOMA_CHALLENGER_DEVICE: cuda:1
+  volumes:
+    - ${LEOMA_MODEL_CACHE_HOST_DIR:-/var/lib/leoma/models}:${LEOMA_MODEL_CACHE_DIR:-/var/lib/leoma/models}
+    - ${LEOMA_CALIBRATION_HOST_DIR:-./calibration}:/var/lib/leoma/calibration
+
+services:
+  leoma-eval-0:
+    <<: *eval-common
+    container_name: leoma-eval-0
+    ports:
+      - "127.0.0.1:9000:9000"
+    gpus:
+      - driver: nvidia
+        device_ids: ["0", "1"]
+        capabilities: [gpu]
+
+  leoma-eval-1:
+    <<: *eval-common
+    container_name: leoma-eval-1
+    ports:
+      - "127.0.0.1:9001:9000"
+    gpus:
+      - driver: nvidia
+        device_ids: ["2", "3"]
+        capabilities: [gpu]
+
+  leoma-eval-2:
+    <<: *eval-common
+    container_name: leoma-eval-2
+    ports:
+      - "127.0.0.1:9002:9000"
+    gpus:
+      - driver: nvidia
+        device_ids: ["4", "5"]
+        capabilities: [gpu]
+
+  leoma-eval-3:
+    <<: *eval-common
+    container_name: leoma-eval-3
+    ports:
+      - "127.0.0.1:9003:9000"
+    gpus:
+      - driver: nvidia
+        device_ids: ["6", "7"]
+        capabilities: [gpu]

docker-compose.validator.production.yml

@@ -0,0 +1,50 @@
+# Leoma production validator (run on the validator host).
+#
+# The image is pull-only and addressed by registry digest. Set the variable to the
+# raw `sha256:<64 hex>` manifest digest; a mutable image tag cannot be supplied.
+services:
+  validator:
+    image: rendixnetwork/leoma@${LEOMA_VALIDATOR_IMAGE_DIGEST:?set LEOMA_VALIDATOR_IMAGE_DIGEST to sha256:<64hex>}
+    pull_policy: always
+    container_name: leoma-validator
+    restart: unless-stopped
+    command: leoma serve
+    env_file:
+      - .env
+    environment:
+      PYTHONUNBUFFERED: "1"
+      NETWORK: ${NETWORK:-finney}
+      NETUID: ${NETUID:-36}
+      LEOMA_TESTNET_SUBNET_NAME: ${LEOMA_TESTNET_SUBNET_NAME:-}
+      WALLET_NAME: ${WALLET_NAME:-default}
+      HOTKEY_NAME: ${HOTKEY_NAME:-default}
+      EVAL_SERVER_URL: ${EVAL_SERVER_URL:-http://127.0.0.1:9000}
+      EVAL_SERVER_URLS: ${EVAL_SERVER_URLS:-}
+      LEOMA_EVAL_TOKEN: ${LEOMA_EVAL_TOKEN:?set LEOMA_EVAL_TOKEN in .env}
+      OBJECT_STORAGE_BACKEND: ${OBJECT_STORAGE_BACKEND:-hippius}
+      HIPPIUS_HUB_TOKEN: ${HIPPIUS_HUB_TOKEN:-}
+      HIPPIUS_ENDPOINT: ${HIPPIUS_ENDPOINT:-s3.hippius.com}
+      HIPPIUS_REGION: ${HIPPIUS_REGION:-decentralized}
+      HIPPIUS_SOURCE_BUCKET: ${HIPPIUS_SOURCE_BUCKET:-leoma-source}
+      HIPPIUS_VIDEOS_READ_ACCESS_KEY: ${HIPPIUS_VIDEOS_READ_ACCESS_KEY:-}
+      HIPPIUS_VIDEOS_READ_SECRET_KEY: ${HIPPIUS_VIDEOS_READ_SECRET_KEY:-}
+      HIPPIUS_OWN_BUCKET: ${HIPPIUS_OWN_BUCKET:-}
+      HIPPIUS_OWN_WRITE_ACCESS_KEY: ${HIPPIUS_OWN_WRITE_ACCESS_KEY:-}
+      HIPPIUS_OWN_WRITE_SECRET_KEY: ${HIPPIUS_OWN_WRITE_SECRET_KEY:-}
+      HIPPIUS_OWN_ENDPOINT: ${HIPPIUS_OWN_ENDPOINT:-}
+      HIPPIUS_OWN_REGION: ${HIPPIUS_OWN_REGION:-decentralized}
+      # Legacy Cloudflare R2 aliases.
+      R2_OWN_BUCKET: ${R2_OWN_BUCKET:-}
+      R2_OWN_WRITE_ACCESS_KEY: ${R2_OWN_WRITE_ACCESS_KEY:-}
+      R2_OWN_WRITE_SECRET_KEY: ${R2_OWN_WRITE_SECRET_KEY:-}
+      R2_OWN_ENDPOINT: ${R2_OWN_ENDPOINT:-}
+      R2_OWN_REGION: ${R2_OWN_REGION:-auto}
+      LEOMA_DASHBOARD_BUCKET: ${LEOMA_DASHBOARD_BUCKET:-}
+      LEOMA_DASHBOARD_ENDPOINT: ${LEOMA_DASHBOARD_ENDPOINT:-}
+      LEOMA_DASHBOARD_REGION: ${LEOMA_DASHBOARD_REGION:-}
+      LEOMA_DASHBOARD_WRITE_ACCESS_KEY: ${LEOMA_DASHBOARD_WRITE_ACCESS_KEY:-}
+      LEOMA_DASHBOARD_WRITE_SECRET_KEY: ${LEOMA_DASHBOARD_WRITE_SECRET_KEY:-}
+    volumes:
+      # Preflight reads the real hotkey and coldkeypub. Never bake wallet files
+      # into an image or make the host wallet writable by the container.
+      - ${BITTENSOR_WALLETS_DIR:?set BITTENSOR_WALLETS_DIR to the host wallets directory}:/root/.bittensor/wallets:ro

docker-compose.yml

@@ -1,13 +1,16 @@
-# Leoma Subnet — king of the hill.
+# Leoma Subnet — local/dev builds only.
 #
 # Two roles: the validator (scans reveals, dispatches duels, sets weights) and
 # the eval server (GPU box that downloads + runs miner models). They are usually
-# on separate hosts; this compose is a convenience for co-locating them.
+# on separate hosts; this compose is a convenience for co-locating and rebuilding
+# them. Production MUST use docker-compose.validator.production.yml and
+# docker-compose.eval.8xh100.production.yml, which accept only image digests and
+# contain no build instructions.
 services:
   # Validator: scan reveals -> duel via eval server -> crown -> set weights.
   validator:
     build: .
-    image: rendixnetwork/leoma:latest
+    image: rendixnetwork/leoma:dev-validator
     container_name: leoma-validator
     restart: unless-stopped
     command: leoma serve
@@ -16,7 +19,7 @@ services:
     environment:
       PYTHONUNBUFFERED: "1"
       NETWORK: ${NETWORK:-finney}
-      NETUID: ${NETUID:-99}
+      NETUID: ${NETUID:-36}
       WALLET_NAME: ${WALLET_NAME:-default}
       HOTKEY_NAME: ${HOTKEY_NAME:-default}
       EVAL_SERVER_URL: ${EVAL_SERVER_URL:-http://eval-server:9000}
@@ -48,7 +51,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.eval
-    image: rendixnetwork/leoma:latest-eval
+    image: rendixnetwork/leoma:dev-eval
     container_name: leoma-eval-server
     restart: unless-stopped
     command: leoma servers eval-server

docs/DEPENDENCY_SECURITY.md

@@ -0,0 +1,58 @@
+# Dependency security and numerical-runtime policy
+
+Last reviewed: 2026-08-02.
+
+Leoma has two different upgrade classes. Validator, network, HTTP, crypto, and CLI
+packages may be patched after the normal test/build gate. Packages that decode
+frames, load weights, generate video, or calculate scores are consensus-sensitive:
+their upgrade also requires a coordinated runtime-digest release and H100
+calibration. An unattended dependency bot must never merge the latter class.
+
+## Current audit result
+
+The locked validator graph has no known advisory after upgrading FastAPI/Starlette,
+Pillow, cryptography, urllib3, idna, msgpack, pyasn1, Pygments, Click, setuptools,
+and yt-dlp. Pillow 12.3.0 is a frame-processing change, so this release still needs
+the calibration procedure below before mainnet rollout.
+
+The evaluator intentionally retains Torch 2.6.0 and Diffusers 0.35.2 until the
+replacement stack passes a real Wan2.2 compatibility/calibration run. The current
+advisory database reports findings in both. Their most serious miner-reachable class
+is malicious repository code or pickle deserialization. Leoma applies independent
+controls before either library sees a miner snapshot:
+
+- the registry request allows only JSON/config/tokenizer data and `.safetensors`;
+- the materialized directory is rechecked before every load, including completed
+  cache entries; executable files, pickle weights, symlinks, and special files fail;
+- Diffusers is forced offline with `local_files_only=True`,
+  `trust_remote_code=False`, and `use_safetensors=True`;
+- the pipeline, component libraries/classes, shape-critical config, and total model
+  size must match the immutable Wan2.2 base architecture.
+
+These controls substantially reduce reachability but do not make an old ML runtime
+equivalent to a patched one. The evaluator remains a dedicated, authenticated box;
+its ports bind loopback, model repositories are treated as hostile, and no wallet or
+state-bucket write credentials belong on that host.
+
+The dashboard lock has all available non-breaking fixes. npm currently reports one
+React Router RSC action-CSRF advisory with no non-vulnerable published version; its
+suggested downgrade reintroduces older advisories. Leoma is a client-rendered Vite
+SPA using `BrowserRouter`/`Routes`, with no RSC server, route actions, loaders, or
+document request handler, so the affected server path is absent. Recheck and remove
+this exception as soon as a fixed release is published.
+
+## Consensus-sensitive upgrade gate
+
+For Pillow, Torch, Diffusers, Transformers, Accelerate, safetensors, NumPy, SciPy,
+OpenCV, torchvision, LPIPS, or OpenCLIP changes:
+
+1. Update exact versions in `uv.lock` and `leoma/eval/runtime_lock.py`.
+2. Set `[runtime].eval_lock_digest` in `chain.toml` to the new `uv.lock` SHA-256.
+3. Build and publish the eval image, then record its immutable registry digest.
+4. On every physical H100, run two same-model control records and analyze all 16.
+5. Run a complete seed-versus-seed duel and a known real challenger regression.
+6. Only after every check passes, deploy that exact image digest to all four eval
+   pairs and the matching consensus release to every validator.
+
+Do not mix old/new runtime digests. Evaluator health and validator dispatch are
+fail-closed specifically to prevent that partial rollout from producing verdicts.