[Bug]: Service with replica IP refs stuck in provisioning when instances are newly provisioned

Steps to reproduce

Create a fleet with nodes: min: 0 on a VM backend (reproduced on jarvislabs, india-noida-01), so that instances are provisioned for the run rather than reused.
Apply a service whose group commands reference replica IPs, e.g. (reproduced with a 4-replica GPU service with the same refs; this is a minimal equivalent):

type: service
name: replica-ip-refs

groups:
  - name: workers
    replicas: 2
    commands:
      - echo "head=${{ groups[0].replicas[0].IP_ADDRESS }}"
      - python3 -m http.server 8000
    resources:
      cpu: 2

port: 8000

Actual behaviour

The run stays in provisioning forever. The instances come up and the instances rows get hostname and internal_ip set, but the jobs' job_provisioning_data keeps hostname=None, internal_ip=None, and every job logs waiting for referenced group IPs on each iteration.

For a PROVISIONING job, _prepare_startup_context runs before _process_provisioning_status and returns None when a referenced IP is missing:

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 491 to 497 in 70c2616

    
           if context.job_model.status == JobStatus.PROVISIONING: 
        
               startup_context = await _prepare_startup_context(context=context, result=result) 
        
               if startup_context is None: 
        
                   return result 
        
               await _process_provisioning_status( 
        
                   context=context, startup_context=startup_context, result=result

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 653 to 663 in 70c2616

    
           commands = context.job.job_spec.commands 
        
           try: 
        
               for c in commands: 
        
                   validate_groups_refs(c) 
        
               if not _substitute_groups_ip_refs(commands, context): 
        
                   logger.debug( 
        
                       "%s: waiting for referenced group IPs", 
        
                       fmt(context.job_model), 
        
                   ) 
        
                   return None

But copying the instance's provisioning data (with hostname/internal_ip) into the job only happens in _process_provisioning_status, which is therefore never reached:

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 816 to 818 in 70c2616

    
           if job_provisioning_data.hostname is None: 
        
               _wait_for_instance_provisioning_data(context.job_model, result)

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 1200 to 1226 in 70c2616

    
           def _wait_for_instance_provisioning_data( 
        
               job_model: JobModel, 
        
               result: _ProcessResult, 
        
           ) -> None: 
        
               if job_model.instance is None: 
        
                   logger.error( 
        
                       "%s: cannot update job_provisioning_data. job_model.instance is None.", 
        
                       fmt(job_model), 
        
                   ) 
        
                   return 
        
               if job_model.instance.job_provisioning_data is None: 
        
                   logger.error( 
        
                       "%s: cannot update job_provisioning_data. job_model.job_provisioning_data is None.", 
        
                       fmt(job_model), 
        
                   ) 
        
                   return 
        
               if job_model.instance.status == InstanceStatus.TERMINATED: 
        
                   _terminate_job( 
        
                       job_model=job_model, 
        
                       job_update_map=result.job_update_map, 
        
                       termination_reason=JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED, 
        
                       termination_reason_message="Instance is terminated", 
        
                   ) 
        
                   return 
        
               result.job_update_map["job_provisioning_data"] = job_model.instance.job_provisioning_data

The IP view reads internal_ip from the jobs' own provisioning data, so the referenced IPs never appear and all jobs wait on each other:

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 1932 to 1957 in 70c2616

    
           def _build_replica_groups_ip_view( 
        
               jobs: list[Job], configuration: ServiceConfiguration 
        
           ) -> list[list[str]]: 
        
               """Fixed-length rows of replicas.min; slot j is the j-th live job in that group.""" 
        
               view: list[list[str]] = [] 
        
               for group_index, group in enumerate(configuration.replica_groups): 
        
                   size = group.replicas.min or 0 
        
                   row = [""] * size 
        
                   group_name = group.name if group.name is not None else str(group_index) 
        
                   group_jobs = [ 
        
                       job 
        
                       for job in jobs 
        
                       if job.job_spec.replica_group == group_name and not _job_is_finished(job) 
        
                   ] 
        
                   group_jobs.sort(key=lambda job: job.job_spec.replica_num) 
        
                   for slot, job in enumerate(group_jobs[:size]): 
        
                       row[slot] = _job_internal_ip(job) 
        
                   view.append(row) 
        
               return view 
        
           def _job_internal_ip(job: Job) -> str: 
        
               if job.job_submissions: 
        
                   jpd = job.job_submissions[-1].job_provisioning_data 
        
                   if jpd is not None: 
        
                       return jpd.internal_ip or ""

It works only when jobs land on already-provisioned idle fleet instances, since their provisioning data is copied with IPs at assignment. Stopping the stuck run and re-applying within the idle window works around it. groups[i].nodes[j].IP_ADDRESS refs in multi-node tasks go through the same gate and are likely affected too (not verified).

Expected behaviour

Jobs pick up their instance's provisioning data while waiting, and the run starts once all referenced replicas have internal IPs.

dstack version

master (70c2616)

Another consequence: if an instance fails to provision while its job is waiting on the IP gate, the job is never failed. Seen on JarvisLabs: the instance went TERMINATED with Error while waiting for instance to become running, but the job stayed PROVISIONING indefinitely, so the run neither failed nor retried. The check that terminates the job when its instance is terminated lives in the same skipped _wait_for_instance_provisioning_data:

dstack/src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

Lines 1217 to 1224 in 70c2616

    
           if job_model.instance.status == InstanceStatus.TERMINATED: 
        
               _terminate_job( 
        
                   job_model=job_model, 
        
                   job_update_map=result.job_update_map, 
        
                   termination_reason=JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED, 
        
                   termination_reason_message="Instance is terminated", 
        
               ) 
        
               return

Navigation Menu

Repository navigation

[Bug]: Service with replica IP refs stuck in provisioning when instances are newly provisioned #4331

Steps to reproduce

Actual behaviour

Expected behaviour

dstack version

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development


	if context.job_model.status == JobStatus.PROVISIONING:
	startup_context = await _prepare_startup_context(context=context, result=result)
	if startup_context is None:
	return result
	await _process_provisioning_status(
	context=context, startup_context=startup_context, result=result


	commands = context.job.job_spec.commands
	try:
	for c in commands:
	validate_groups_refs(c)
	if not _substitute_groups_ip_refs(commands, context):
	logger.debug(
	"%s: waiting for referenced group IPs",
	fmt(context.job_model),
	)
	return None


	if job_provisioning_data.hostname is None:
	_wait_for_instance_provisioning_data(context.job_model, result)

	def _wait_for_instance_provisioning_data(
	job_model: JobModel,
	result: _ProcessResult,
	) -> None:
	if job_model.instance is None:
	logger.error(
	"%s: cannot update job_provisioning_data. job_model.instance is None.",
	fmt(job_model),
	)
	return
	if job_model.instance.job_provisioning_data is None:
	logger.error(
	"%s: cannot update job_provisioning_data. job_model.job_provisioning_data is None.",
	fmt(job_model),
	)
	return

	if job_model.instance.status == InstanceStatus.TERMINATED:
	_terminate_job(
	job_model=job_model,
	job_update_map=result.job_update_map,
	termination_reason=JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED,
	termination_reason_message="Instance is terminated",
	)
	return

	result.job_update_map["job_provisioning_data"] = job_model.instance.job_provisioning_data

	def _build_replica_groups_ip_view(
	jobs: list[Job], configuration: ServiceConfiguration
	) -> list[list[str]]:
	"""Fixed-length rows of replicas.min; slot j is the j-th live job in that group."""
	view: list[list[str]] = []
	for group_index, group in enumerate(configuration.replica_groups):
	size = group.replicas.min or 0
	row = [""] * size
	group_name = group.name if group.name is not None else str(group_index)
	group_jobs = [
	job
	for job in jobs
	if job.job_spec.replica_group == group_name and not _job_is_finished(job)
	]
	group_jobs.sort(key=lambda job: job.job_spec.replica_num)
	for slot, job in enumerate(group_jobs[:size]):
	row[slot] = _job_internal_ip(job)
	view.append(row)
	return view


	def _job_internal_ip(job: Job) -> str:
	if job.job_submissions:
	jpd = job.job_submissions[-1].job_provisioning_data
	if jpd is not None:
	return jpd.internal_ip or ""

[Bug]: Service with replica IP refs stuck in provisioning when instances are newly provisioned #4331

Description

Steps to reproduce

Actual behaviour

Expected behaviour

dstack version

Activity

r4victor commented on Oct 2, 2026

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions