Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/dstack/_internal/cli/services/presets/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,12 +411,16 @@ def create_preset(
allowed_fleets: Optional[tuple[str, ...]] = None,
previous: Sequence[PresetSession] = (),
) -> PresetCreateResult:
# Resolve env before the session is claimed and before the try that marks
# it failed: a missing variable here means nothing was spawned and nothing
# changed, so a resumable session must stay interrupted - a failure this
# early is not a failed creation.
resolved_configuration = _resolve_preset_env(configuration)
session = resume_session or create_preset_session(
configuration,
previous=tuple(session.preset_id for session in previous),
)
try:
resolved_configuration = _resolve_preset_env(configuration)
# A creation session runs unattended for hours; an idling machine would
# freeze the agent and the process supervising it alike.
with prevent_idle_sleep():
Expand Down
49 changes: 48 additions & 1 deletion src/tests/_internal/cli/services/presets/test_create.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
import json
from datetime import datetime, timezone
import os
import uuid
from types import SimpleNamespace
Expand Down Expand Up @@ -49,7 +50,7 @@
create_agent_workspace,
remove_agent_workspace,
)
from dstack._internal.core.errors import CLIError
from dstack._internal.core.errors import CLIError, ConfigurationError
from dstack._internal.core.models.configurations import PresetConfiguration
from dstack._internal.core.models.envs import EnvSentinel
from dstack._internal.core.models.runs import Run, RunStatus
Expand Down Expand Up @@ -392,6 +393,52 @@ async def run_agent(**kwargs):
assert creation_context.run_apis.stopped_names == stopped_names


def test_env_resolution_failure_keeps_resume_session_interrupted(self, tmp_path, monkeypatch):
# Regression for #4164: resuming an interrupted preset from a shell
# missing a passthrough env var raised before the agent was spawned,
# yet the blanket `except BaseException` marked the session "failed" and
# deleted the workspace, making it unresumable forever. A failure before
# anything started must leave the session interrupted.
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("USERPROFILE", str(tmp_path))
monkeypatch.delenv("MODEL_LABEL", raising=False)

session_path = tmp_path / ".dstack" / "presets" / "ab12cd34"
session_path.mkdir(parents=True)
(session_path / "agent.log").touch()
(session_path / "trace.jsonl").touch()
session = PresetSession(path=session_path, preset_id="ab12cd34")
session.write_state(
PresetSessionState(
id="ab12cd34",
name="qwen",
model="Qwen/Qwen3.5-27B",
trials_num=None,
previous=[],
created_at=datetime.now(timezone.utc),
status="interrupted",
owner=None,
run=None,
)
)

with pytest.raises(ConfigurationError, match="MODEL_LABEL"):
create_preset(
api=SimpleNamespace(),
configuration=PresetConfiguration(
name="qwen",
base="Qwen/Qwen3.5-27B",
env=["MODEL_LABEL"],
),
store=PresetStore(tmp_path / "presets"),
resume_session=session,
)

assert session.read_state().status == "interrupted"
assert session_path.is_dir()



class TestResolvePreviousSessions:
def _store(self, tmp_path, monkeypatch, *ids):
store = tmp_path / "presets-store"
Expand Down