[Bug]: A task is marked as aborted after completion (instead of done) #885

type: task

python: "3.11"

commands:
  - echo "Hello, World!"
  - sleep 30
  - echo "Goodbye, World!"

dstack run . -f issues/tasks-get-aborted/alpine.dstack.yml -y
 Configuration  issues/tasks-get-aborted/alpine.dstack.yml
 Project        main
 User           admin
 Pool name      default-pool
 Min resources  2..xCPU, 8GB..
 Max price      -
 Max duration   72h
 Spot policy    auto
 Retry policy   no

 #  BACKEND  REGION    INSTANCE       RESOURCES                 SPOT  PRICE
 1  gcp      us-west4  e2-standard-2  2xCPU, 8GB, 100GB (disk)  yes   $0.009104  Ready
 2  gcp      us-west4  e2-standard-2  2xCPU, 8GB, 100GB (disk)  yes   $0.009104  Ready
 3  gcp      us-west4  e2-standard-2  2xCPU, 8GB, 100GB (disk)  yes   $0.009104
    ...
 Shown 3 of 4649 offers, $65.54 max

lovely-snail-1 provisioning completed (running)
Hello, World!
Goodbye, World!
Aborted

ime=2024-02-10T12:03:05.358463-05:00 level=info msg=Run job log_level=trace
time=2024-02-10T12:03:05.358746-05:00 level=trace msg=Fetching git repository
time=2024-02-10T12:03:05.358831-05:00 level=trace msg=Credentials is not empty
time=2024-02-10T12:03:05.358875-05:00 level=trace msg=Select HTTPS protocol
time=2024-02-10T12:03:05.358916-05:00 level=warning msg=OAuth token is empty
time=2024-02-10T12:03:05.358985-05:00 level=trace msg=Checking out remote repo GIT URL=https://github.com/dstackai/dstack-examples.git
time=2024-02-10T12:03:05.359044-05:00 level=info msg=git checkout hash=912648c38482c35b4d0d173f2044f5798feecfb2 url=https://github.com/dstackai/dstack-examples.git auth=<nil> branch=main path=/workflow
time=2024-02-10T12:03:05.968668-05:00 level=trace msg=Applying diff
time=2024-02-10T12:03:05.968752-05:00 level=info msg=apply diff start dir=/workflow
time=2024-02-10T12:03:05.96885-05:00 level=trace msg=apply diff file file=.dstack.yml text_fragments_cnt=1
time=2024-02-10T12:03:05.97066-05:00 level=trace msg=apply diff file text_fragments_cnt=2 file=README.md
time=2024-02-10T12:03:05.971082-05:00 level=trace msg=apply diff file text_fragments_cnt=1 file=
time=2024-02-10T12:03:05.971443-05:00 level=trace msg=apply diff file file= text_fragments_cnt=1
time=2024-02-10T12:03:05.97174-05:00 level=trace msg=apply diff file text_fragments_cnt=1 file=
time=2024-02-10T12:03:05.972-05:00 level=info msg=Job state changed new=running
time=2024-02-10T12:03:05.972146-05:00 level=trace msg=Starting exec cmd=/bin/bash -i -c echo "Hello, World!" && sleep 30 && echo "Goodbye, World!" working_dir=/workflow env=[NV_CUDA_COMPAT_PACKAGE=cuda-compat-12-1 PIP_ROOT_USER_ACTION=ignore HOSTNAME=main-purple-firefox-1-0 LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 HOME=/root PYTHON=3.11 CUDA_VERSION=12.1.0 NVIDIA_REQUIRE_CUDA=cuda>=12.1 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526 NVIDIA_DRIVER_CAPABILITIES=compute,utility NV_CUDA_CUDART_VERSION=12.1.55-1 PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin NVARCH=x86_64 LANG=C.UTF-8 LC_ALL=C.UTF-8 PWD=/ NVIDIA_VISIBLE_DEVICES=all RUN_NAME=lovely-snail-1 REPO_ID=dstack-examples-2aut8wsk HOME=/root]
time=2024-02-10T12:03:36.173195-05:00 level=info msg=Job state changed new=done

DEBUG 2024-02-10T18:02:53.485 dstack._internal.server.background.tasks.process_submitted_jobs (896c50)lovely-snail-1-0: provisioning
INFO 2024-02-10T18:02:53.502 dstack._internal.server.background.tasks.process_submitted_jobs (896c50)lovely-snail-1-0: now is provisioning
INFO 2024-02-10T18:02:57.756 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
DEBUG 2024-02-10T18:02:57.781 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process provisioning job with shim, age=0:00:05.059315
INFO 2024-02-10T18:02:59.794 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:02:59.817 dstack._internal.server.app Processed request POST http://127.0.0.1:3000/api/project/main/runs/get in 2.050039s
INFO 2024-02-10T18:03:00.627 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: now is pulling
DEBUG 2024-02-10T18:03:01.462 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process pulling job with shim, age=0:00:08.740524
DEBUG 2024-02-10T18:03:03.931 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: submitting job spec
DEBUG 2024-02-10T18:03:03.932 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: repo credentials are https
DEBUG 2024-02-10T18:03:04.420 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: uploading code
DEBUG 2024-02-10T18:03:04.881 dstack._internal.server.app Processed request POST http://127.0.0.1:3000/api/project/main/runs/get in 0.024585s
DEBUG 2024-02-10T18:03:04.909 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: starting job
INFO 2024-02-10T18:03:07.513 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
DEBUG 2024-02-10T18:03:07.541 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:14.819726
INFO 2024-02-10T18:03:09.733 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: now is running
INFO 2024-02-10T18:03:09.882 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:03:09.905 dstack._internal.server.app Processed request POST http://127.0.0.1:3000/api/project/main/runs/get in 0.012254s
DEBUG 2024-02-10T18:03:11.478 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:18.756386
INFO 2024-02-10T18:03:17.517 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
DEBUG 2024-02-10T18:03:17.538 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:24.816911
INFO 2024-02-10T18:03:19.646 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:03:21.487 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:28.765353
INFO 2024-02-10T18:03:27.515 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
DEBUG 2024-02-10T18:03:27.542 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:34.820591
INFO 2024-02-10T18:03:29.574 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:03:31.496 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:38.775097
INFO 2024-02-10T18:03:37.533 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
DEBUG 2024-02-10T18:03:37.561 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: process running job, age=0:00:44.839384
INFO 2024-02-10T18:03:39.697 dstack._internal.server.background.tasks.process_running_jobs (896c50)lovely-snail-1-0: now is done
INFO 2024-02-10T18:03:39.706 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:03:39.738 dstack._internal.server.app Processed request POST http://127.0.0.1:3000/api/project/main/runs/get in 2.193675s
DEBUG 2024-02-10T18:03:39.777 dstack._internal.server.app Processed request POST http://127.0.0.1:3000/api/project/main/runs/stop in 0.010103s
INFO 2024-02-10T18:03:47.562 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
INFO 2024-02-10T18:03:49.807 dstack._internal.server.background.tasks.process_pools check instance selfish-wasp-1-0 status: True
DEBUG 2024-02-10T18:03:55.475 dstack._internal.server.services.jobs Terminating runner instance 34.125.224.72
INFO 2024-02-10T18:03:57.485 dstack._internal.server.background.tasks.process_pools check instance purple-firefox-1-0 status: True
INFO 2024-02-10T18:03:57.487 dstack._internal.server.background.tasks.process_pools instance selfish-wasp-1-0 terminated by termination policy: idle time 307s
INFO 2024-02-10T18:03:57.502 dstack._internal.server.background.tasks.process_finished_jobs (896c50)lovely-snail-1-0: marked as removed