From 10d141bd66f044d57dccb6ac0c1c71fbdb36e555 Mon Sep 17 00:00:00 2001 From: breken-ai Date: Tue, 8 Sep 2026 23:23:19 -0700 Subject: [PATCH] fix: resolve preset env before claiming the agent session create_preset wraps everything in a BaseException handler that marks the session failed, and _resolve_preset_env ran inside that try. A resume from a shell missing a passthrough variable raises ConfigurationError before the agent is spawned, so the session is written failed, the workspace deleted, and load_resumable_session refuses it forever. Hoist env resolution above the session claim and the try: a pre-spawn failure leaves an interrupted session resumable, and a fresh attempt leaves no dead record behind. Fixes #4164 --- .../_internal/cli/services/presets/create.py | 6 ++- .../cli/services/presets/test_create.py | 49 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index e2617098be..96f3e2c537 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -411,12 +411,16 @@ def create_preset( allowed_fleets: Optional[tuple[str, ...]] = None, previous: Sequence[PresetSession] = (), ) -> PresetCreateResult: + # Resolve env before the session is claimed and before the try that marks + # it failed: a missing variable here means nothing was spawned and nothing + # changed, so a resumable session must stay interrupted - a failure this + # early is not a failed creation. + resolved_configuration = _resolve_preset_env(configuration) session = resume_session or create_preset_session( configuration, previous=tuple(session.preset_id for session in previous), ) try: - resolved_configuration = _resolve_preset_env(configuration) # A creation session runs unattended for hours; an idling machine would # freeze the agent and the process supervising it alike. with prevent_idle_sleep(): diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index e3ae148d88..225f543b6a 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -1,5 +1,6 @@ import asyncio import json +from datetime import datetime, timezone import os import uuid from types import SimpleNamespace @@ -49,7 +50,7 @@ create_agent_workspace, remove_agent_workspace, ) -from dstack._internal.core.errors import CLIError +from dstack._internal.core.errors import CLIError, ConfigurationError from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.runs import Run, RunStatus @@ -392,6 +393,52 @@ async def run_agent(**kwargs): assert creation_context.run_apis.stopped_names == stopped_names + def test_env_resolution_failure_keeps_resume_session_interrupted(self, tmp_path, monkeypatch): + # Regression for #4164: resuming an interrupted preset from a shell + # missing a passthrough env var raised before the agent was spawned, + # yet the blanket `except BaseException` marked the session "failed" and + # deleted the workspace, making it unresumable forever. A failure before + # anything started must leave the session interrupted. + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.delenv("MODEL_LABEL", raising=False) + + session_path = tmp_path / ".dstack" / "presets" / "ab12cd34" + session_path.mkdir(parents=True) + (session_path / "agent.log").touch() + (session_path / "trace.jsonl").touch() + session = PresetSession(path=session_path, preset_id="ab12cd34") + session.write_state( + PresetSessionState( + id="ab12cd34", + name="qwen", + model="Qwen/Qwen3.5-27B", + trials_num=None, + previous=[], + created_at=datetime.now(timezone.utc), + status="interrupted", + owner=None, + run=None, + ) + ) + + with pytest.raises(ConfigurationError, match="MODEL_LABEL"): + create_preset( + api=SimpleNamespace(), + configuration=PresetConfiguration( + name="qwen", + base="Qwen/Qwen3.5-27B", + env=["MODEL_LABEL"], + ), + store=PresetStore(tmp_path / "presets"), + resume_session=session, + ) + + assert session.read_state().status == "interrupted" + assert session_path.is_dir() + + + class TestResolvePreviousSessions: def _store(self, tmp_path, monkeypatch, *ids): store = tmp_path / "presets-store"