From be62c5014e7519317a4203fe1f0a360bb13b9fa0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 12 Oct 2024 10:02:02 +0800 Subject: [PATCH 001/435] Add poky as submodule --- .gitmodules | 3 +++ poky | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 poky diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..370d73243 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "poky"] + path = poky + url = https://github.com/yoctoproject/poky.git diff --git a/poky b/poky new file mode 160000 index 000000000..cd44e6bd4 --- /dev/null +++ b/poky @@ -0,0 +1 @@ +Subproject commit cd44e6bd40b0c1f498b3feaeb5e9b72f8bf32d41 From 3d48162ef43e860f46d34b7b38dac1a02c2b165d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 12 Oct 2024 10:25:07 +0800 Subject: [PATCH 002/435] Add submodules --- .gitmodules | 6 ++++++ meta-confidential-compute | 1 + meta-virtualization | 1 + 3 files changed, 8 insertions(+) create mode 160000 meta-confidential-compute create mode 160000 meta-virtualization diff --git a/.gitmodules b/.gitmodules index 370d73243..421b33cdc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "poky"] path = poky url = https://github.com/yoctoproject/poky.git +[submodule "meta-confidential-compute"] + path = meta-confidential-compute + url = https://github.com/Phala-Network/meta-confidential-compute.git +[submodule "meta-virtualization"] + path = meta-virtualization + url = https://github.com/lgirdk/meta-virtualization diff --git a/meta-confidential-compute b/meta-confidential-compute new file mode 160000 index 000000000..6d1355e0c --- /dev/null +++ b/meta-confidential-compute @@ -0,0 +1 @@ +Subproject commit 6d1355e0c684f7fdfb688ecfef0ed63feb2aac4c diff --git a/meta-virtualization b/meta-virtualization new file mode 160000 index 000000000..1665d1e2a --- /dev/null +++ b/meta-virtualization @@ -0,0 +1 @@ +Subproject commit 1665d1e2ad2244171dcdb580df4f54739d6444bd From 5cc94c609801d1b0bd6e2f5d60d24b974c889ed0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 12 Oct 2024 11:34:22 +0800 Subject: [PATCH 003/435] Add meta-dstack --- Makefile | 7 ++++ meta-dstack/conf/distro/dstack.conf | 6 ++++ meta-dstack/conf/layer.conf | 8 +++++ .../recipes/images/dstack-initramfs.bb | 34 ++++++++++++++++++ setup | 36 +++++++++++++++++++ 5 files changed, 91 insertions(+) create mode 100644 Makefile create mode 100644 meta-dstack/conf/distro/dstack.conf create mode 100644 meta-dstack/conf/layer.conf create mode 100644 meta-dstack/recipes/images/dstack-initramfs.bb create mode 100755 setup diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..d3bd987dd --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: build gen-measurements + +build: + cd srcs/poky/ && bitbake dstack-initramfs + +gen-measurements: + echo "Not implemented" \ No newline at end of file diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf new file mode 100644 index 000000000..afedd890a --- /dev/null +++ b/meta-dstack/conf/distro/dstack.conf @@ -0,0 +1,6 @@ +require conf/distro/cvm.conf + +DISTRO = "dstack" +DISTRO_NAME = "DStack" +DISTROOVERRIDES = "poky:dstack" +INITRAMFS_IMAGE = "dstack-initramfs" diff --git a/meta-dstack/conf/layer.conf b/meta-dstack/conf/layer.conf new file mode 100644 index 000000000..19f4ef0cd --- /dev/null +++ b/meta-dstack/conf/layer.conf @@ -0,0 +1,8 @@ +BBPATH .= ":${LAYERDIR}" +BBFILES += "${LAYERDIR}/recipes/*/*.bb ${LAYERDIR}/recipes/*/*.bbappend" + +BBFILE_COLLECTIONS += "dstack" +BBFILE_PATTERN_dstack := "^${LAYERDIR}/" +BBFILE_PRIORITY_dstack = "20" +LAYERVERSION_dstack = "4" +LAYERSERIES_COMPAT_dstack = "scarthgap" diff --git a/meta-dstack/recipes/images/dstack-initramfs.bb b/meta-dstack/recipes/images/dstack-initramfs.bb new file mode 100644 index 000000000..337070476 --- /dev/null +++ b/meta-dstack/recipes/images/dstack-initramfs.bb @@ -0,0 +1,34 @@ +# Simple initramfs image artifact generation for tiny images. +DESCRIPTION = "Tiny image capable of booting a device. The kernel includes \ +the Minimal RAM-based Initial Root Filesystem (initramfs), which finds the \ +first 'init' program more efficiently. core-image-tiny-initramfs doesn't \ +actually generate an image but rather generates boot and rootfs artifacts \ +that can subsequently be picked up by external image generation tools such as wic." + +CVM_DEPS = "busybox-mdev init-ifupdown initscripts base-files base-passwd netbase busybox-udhcpd" + +PACKAGE_INSTALL = "ca-certificates sysvinit busybox-udhcpd dropbear ${CVM_DEPS} ${VIRTUAL-RUNTIME_base-utils} ${ROOTFS_BOOTSTRAP_INSTALL}" + +INITRAMFS_MAXSIZE = "1000000" + +# Do not pollute the initrd image with rootfs features +IMAGE_FEATURES = "debug-tweaks" + +export IMAGE_BASENAME = "dstack-initramfs" +IMAGE_NAME_SUFFIX ?= "" +IMAGE_LINGUAS = "" + +LICENSE = "MIT" + +# don't actually generate an image, just the artifacts needed for one +IMAGE_FSTYPES = "${INITRAMFS_FSTYPES}" + +inherit core-image + +IMAGE_ROOTFS_SIZE = "8192" +IMAGE_ROOTFS_EXTRA_SPACE = "0" + +# Use the same restriction as initramfs-live-install +COMPATIBLE_HOST = "x86_64.*-linux" + +# QB_KERNEL_CMDLINE_APPEND += "debugshell=3 init=/bin/busybox sh init" diff --git a/setup b/setup new file mode 100755 index 000000000..b7453e61d --- /dev/null +++ b/setup @@ -0,0 +1,36 @@ +#!/bin/bash + +# Check if the first argument is -v or --verbose +verbose=0 +if [[ $1 == "-v" ]] || [[ $1 == "--verbose" ]]; then + verbose=1 + # Shift the positional parameters to the left, effectively removing the first argument + shift +fi + +# Define a function for verbose output +function verbose_output { + if [[ $verbose -eq 1 ]]; then + echo "$1" + fi +} + +# Set CURRENT_PATH to the current working directory +CURRENT_PATH=$(pwd) +verbose_output "Set CURRENT_PATH to $CURRENT_PATH" + +# Switch to the oe-init-build-env directory +pushd $CURRENT_PATH/poky/ +verbose_output "Switched to $(pwd)" + +# Source the oe-init-build-env script +source oe-init-build-env +verbose_output "Sourced the oe-init-build-env script" + +# Add the meta-evm, meta-confidential-compute, meta-secure-core/meta-tpm2, meta-openembedded/meta-python, meta-openembedded/meta-oe layers meta-rust-bin meta-clang to bblayers.conf +bitbake-layers add-layer ../../meta-confidential-compute +bitbake-layers add-layer ../../meta-dstack + +# Return to the original directory +popd +verbose_output "Returned to $(pwd)" From b1d8bb742154b8593c19eb5e94cf8e1807059831 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 12 Oct 2024 15:29:13 +0800 Subject: [PATCH 004/435] Add meta-openembedded and meta-virtualization --- .gitignore | 1 + .gitmodules | 3 + build/conf/local.conf | 288 ++++++++++++++++++ .../recipes/images/dstack-initramfs.bb | 16 +- meta-openembedded | 1 + meta-virtualization | 2 +- setup | 40 +-- 7 files changed, 316 insertions(+), 35 deletions(-) create mode 100644 .gitignore create mode 100644 build/conf/local.conf create mode 160000 meta-openembedded diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/build diff --git a/.gitmodules b/.gitmodules index 421b33cdc..0b7abaecf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "meta-virtualization"] path = meta-virtualization url = https://github.com/lgirdk/meta-virtualization +[submodule "meta-openembedded"] + path = meta-openembedded + url = https://github.com/openembedded/meta-openembedded diff --git a/build/conf/local.conf b/build/conf/local.conf new file mode 100644 index 000000000..204ff0796 --- /dev/null +++ b/build/conf/local.conf @@ -0,0 +1,288 @@ +# +# This file is your local configuration file and is where all local user settings +# are placed. The comments in this file give some guide to the options a new user +# to the system might want to change but pretty much any configuration option can +# be set in this file. More adventurous users can look at +# local.conf.sample.extended which contains other examples of configuration which +# can be placed in this file but new users likely won't need any of them +# initially. There's also site.conf.sample which contains examples of site specific +# information such as proxy server addresses. +# +# Lines starting with the '#' character are commented out and in some cases the +# default values are provided as comments to show people example syntax. Enabling +# the option is a question of removing the # character and making any change to the +# variable as required. + +# +# Machine Selection +# +# You need to select a specific machine to target the build with. There are a selection +# of emulated machines available which can boot and run in the QEMU emulator: +# +#MACHINE ?= "qemuarm" +#MACHINE ?= "qemuarm64" +#MACHINE ?= "qemumips" +#MACHINE ?= "qemumips64" +#MACHINE ?= "qemuppc" +#MACHINE ?= "qemux86" +#MACHINE ?= "qemux86-64" +# +# There are also the following hardware board target machines included for +# demonstration purposes: +# +#MACHINE ?= "beaglebone-yocto" +#MACHINE ?= "genericarm64" +#MACHINE ?= "genericx86" +#MACHINE ?= "genericx86-64" +# +# This sets the default machine to be qemux86-64 if no other machine is selected: +MACHINE ??= "tdx" + +# These are some of the more commonly used values. Looking at the files in the +# meta/conf/machine directory, or the conf/machine directory of any additional layers +# you add in will show all the available machines. + +# +# Where to place downloads +# +# During a first build the system will download many different source code tarballs +# from various upstream projects. This can take a while, particularly if your network +# connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you +# can preserve this directory to speed up this part of subsequent builds. This directory +# is safe to share between multiple builds on the same machine too. +# +# The default is a downloads directory under TOPDIR which is the build directory. +# +#DL_DIR ?= "${TOPDIR}/downloads" + +# +# Where to place shared-state files +# +# BitBake has the capability to accelerate builds based on previously built output. +# This is done using "shared state" files which can be thought of as cache objects +# and this option determines where those files are placed. +# +# You can wipe out TMPDIR leaving this directory intact and the build would regenerate +# from these files if no changes were made to the configuration. If changes were made +# to the configuration, only shared state files where the state was still valid would +# be used (done using checksums). +# +# The default is a sstate-cache directory under TOPDIR. +# +#SSTATE_DIR ?= "${TOPDIR}/sstate-cache" + +# +# Where to place the build output +# +# This option specifies where the bulk of the building work should be done and +# where BitBake should place its temporary files and output. Keep in mind that +# this includes the extraction and compilation of many applications and the toolchain +# which can use Gigabytes of hard disk space. +# +# The default is a tmp directory under TOPDIR. +# +#TMPDIR = "${TOPDIR}/tmp" + +# +# Default policy config +# +# The distribution setting controls which policy settings are used as defaults. +# The default value is fine for general Yocto project use, at least initially. +# Ultimately when creating custom policy, people will likely end up subclassing +# these defaults. +# +DISTRO ?= "dstack" +# As an example of a subclass there is a "bleeding" edge policy configuration +# where many versions are set to the absolute latest code from the upstream +# source control systems. This is just mentioned here as an example, its not +# useful to most new users. +# DISTRO ?= "poky-bleeding" + +# +# Package Management configuration +# +# This variable lists which packaging formats to enable. Multiple package backends +# can be enabled at once and the first item listed in the variable will be used +# to generate the root filesystems. +# Options are: +# - 'package_deb' for debian style deb files +# - 'package_ipk' for ipk files are used by opkg (a debian style embedded package manager) +# - 'package_rpm' for rpm style packages +# E.g.: PACKAGE_CLASSES ?= "package_rpm package_deb package_ipk" +# OE-Core defaults to ipkg, whilst Poky defaults to rpm: +# PACKAGE_CLASSES ?= "package_rpm" + +# +# SDK target architecture +# +# This variable specifies the architecture to build SDK items for and means +# you can build the SDK packages for architectures other than the machine you are +# running the build on (i.e. building i686 packages on an x86_64 host). +# Supported values are i686, x86_64, aarch64 +#SDKMACHINE ?= "i686" + +# +# Extra image configuration defaults +# +# The EXTRA_IMAGE_FEATURES variable allows extra packages to be added to the generated +# images. Some of these options are added to certain image types automatically. The +# variable can contain the following options: +# "dbg-pkgs" - add -dbg packages for all installed packages +# (adds symbol information for debugging/profiling) +# "src-pkgs" - add -src packages for all installed packages +# (adds source code for debugging) +# "dev-pkgs" - add -dev packages for all installed packages +# (useful if you want to develop against libs in the image) +# "ptest-pkgs" - add -ptest packages for all ptest-enabled packages +# (useful if you want to run the package test suites) +# "tools-sdk" - add development tools (gcc, make, pkgconfig etc.) +# "tools-debug" - add debugging tools (gdb, strace) +# "eclipse-debug" - add Eclipse remote debugging support +# "tools-profile" - add profiling tools (oprofile, lttng, valgrind) +# "tools-testapps" - add useful testing tools (ts_print, aplay, arecord etc.) +# "debug-tweaks" - make an image suitable for development +# e.g. ssh root access has a blank password +# There are other application targets that can be used here too, see +# meta/classes-recipe/image.bbclass and +# meta/classes-recipe/core-image.bbclass for more details. +# We default to enabling the debugging tweaks. +EXTRA_IMAGE_FEATURES ?= "debug-tweaks" + +# +# Additional image features +# +# The following is a list of additional classes to use when building images which +# enable extra features. Some available options which can be included in this variable +# are: +# - 'buildstats' collect build statistics +USER_CLASSES ?= "buildstats" + +# +# Runtime testing of images +# +# The build system can test booting virtual machine images under qemu (an emulator) +# after any root filesystems are created and run tests against those images. It can also +# run tests against any SDK that are built. To enable this uncomment these lines. +# See meta/classes-recipe/test{image,sdk}.bbclass for further details. +#IMAGE_CLASSES += "testimage testsdk" +#TESTIMAGE_AUTO:qemuall = "1" + +# +# Interactive shell configuration +# +# Under certain circumstances the system may need input from you and to do this it +# can launch an interactive shell. It needs to do this since the build is +# multithreaded and needs to be able to handle the case where more than one parallel +# process may require the user's attention. The default is iterate over the available +# terminal types to find one that works. +# +# Examples of the occasions this may happen are when resolving patches which cannot +# be applied, to use the devshell or the kernel menuconfig +# +# Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none +# Note: currently, Konsole support only works for KDE 3.x due to the way +# newer Konsole versions behave +#OE_TERMINAL = "auto" +# By default disable interactive patch resolution (tasks will just fail instead): +PATCHRESOLVE = "noop" + +# +# Disk Space Monitoring during the build +# +# Monitor the disk space during the build. If there is less that 1GB of space or less +# than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully +# shutdown the build. If there is less than 100MB or 1K inodes, perform a hard halt +# of the build. The reason for this is that running completely out of space can corrupt +# files and damages the build in ways which may not be easily recoverable. +# It's necessary to monitor /tmp, if there is no space left the build will fail +# with very exotic errors. +BB_DISKMON_DIRS ??= "\ + STOPTASKS,${TMPDIR},1G,100K \ + STOPTASKS,${DL_DIR},1G,100K \ + STOPTASKS,${SSTATE_DIR},1G,100K \ + STOPTASKS,/tmp,100M,100K \ + HALT,${TMPDIR},100M,1K \ + HALT,${DL_DIR},100M,1K \ + HALT,${SSTATE_DIR},100M,1K \ + HALT,/tmp,10M,1K" + +# +# Shared-state files from other locations +# +# As mentioned above, shared state files are prebuilt cache data objects which can be +# used to accelerate build time. This variable can be used to configure the system +# to search other mirror locations for these objects before it builds the data itself. +# +# This can be a filesystem directory, or a remote url such as https or ftp. These +# would contain the sstate-cache results from previous builds (possibly from other +# machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the +# cache locations to check for the shared objects. +# NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH +# at the end as shown in the examples below. This will be substituted with the +# correct path within the directory structure. +#SSTATE_MIRRORS ?= "\ +#file://.* https://someserver.tld/share/sstate/PATH;downloadfilename=PATH \ +#file://.* file:///some/local/dir/sstate/PATH" + +# +# Yocto Project SState Mirror +# +# The Yocto Project has prebuilt artefacts available for its releases, you can enable +# use of these by uncommenting some of the following lines. This will mean the build uses +# the network to check for artefacts at the start of builds, which does slow it down +# initially but it will then speed up the builds by not having to build things if they are +# present in the cache. It assumes you can download something faster than you can build it +# which will depend on your network. +# Note: For this to work you also need hash-equivalence passthrough to the matching server +# There is a choice between our sstate server directly and a faster content delivery network +# (CDN) kindly provided by JSDelivr, uncomment one of the SSTATE_MIRRORS lines, not both. +# Using the CDN rather than the yoctoproject.org address is suggested/preferred. +# +#BB_HASHSERVE_UPSTREAM = 'wss://hashserv.yoctoproject.org/ws' +#SSTATE_MIRRORS ?= "file://.* https://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH" +# +###SSTATE_MIRRORS ?= "file://.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" + + +# +# Qemu configuration +# +# By default native qemu will build with a builtin VNC server where graphical output can be +# seen. The line below enables the SDL UI frontend too. +PACKAGECONFIG:append:pn-qemu-system-native = " sdl" +# By default libsdl2-native will be built, if you want to use your host's libSDL instead of +# the minimal libsdl built by libsdl2-native then uncomment the ASSUME_PROVIDED line below. +#ASSUME_PROVIDED += "libsdl2-native" + +# You can also enable the Gtk UI frontend, which takes somewhat longer to build, but adds +# a handy set of menus for controlling the emulator. +#PACKAGECONFIG:append:pn-qemu-system-native = " gtk+" + +# +# Hash Equivalence +# +# Enable support for automatically running a local hash equivalence server and +# instruct bitbake to use a hash equivalence aware signature generator. Hash +# equivalence improves reuse of sstate by detecting when a given sstate +# artifact can be reused as equivalent, even if the current task hash doesn't +# match the one that generated the artifact. +# +# A shared hash equivalent server can be set with ":" format +# +#BB_HASHSERVE = "auto" +#BB_SIGNATURE_HANDLER = "OEEquivHash" + +# +# Memory Resident Bitbake +# +# Bitbake's server component can stay in memory after the UI for the current command +# has completed. This means subsequent commands can run faster since there is no need +# for bitbake to reload cache files and so on. Number is in seconds, after which the +# server will shut down. +# +#BB_SERVER_TIMEOUT = "60" + +# CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to +# track the version of this file when it was generated. This can safely be ignored if +# this doesn't mean anything to you. +CONF_VERSION = "2" diff --git a/meta-dstack/recipes/images/dstack-initramfs.bb b/meta-dstack/recipes/images/dstack-initramfs.bb index 337070476..97e3bebaf 100644 --- a/meta-dstack/recipes/images/dstack-initramfs.bb +++ b/meta-dstack/recipes/images/dstack-initramfs.bb @@ -5,9 +5,19 @@ first 'init' program more efficiently. core-image-tiny-initramfs doesn't \ actually generate an image but rather generates boot and rootfs artifacts \ that can subsequently be picked up by external image generation tools such as wic." -CVM_DEPS = "busybox-mdev init-ifupdown initscripts base-files base-passwd netbase busybox-udhcpd" - -PACKAGE_INSTALL = "ca-certificates sysvinit busybox-udhcpd dropbear ${CVM_DEPS} ${VIRTUAL-RUNTIME_base-utils} ${ROOTFS_BOOTSTRAP_INSTALL}" +PACKAGE_INSTALL = "busybox-mdev \ + init-ifupdown \ + initscripts \ + base-files \ + base-passwd \ + netbase \ + busybox-udhcpd \ + ca-certificates \ + sysvinit \ + dropbear \ + docker-moby \ + ${VIRTUAL-RUNTIME_base-utils} \ + ${ROOTFS_BOOTSTRAP_INSTALL}" INITRAMFS_MAXSIZE = "1000000" diff --git a/meta-openembedded b/meta-openembedded new file mode 160000 index 000000000..72018ca1b --- /dev/null +++ b/meta-openembedded @@ -0,0 +1 @@ +Subproject commit 72018ca1b1a471226917e8246e8bbf9a374ccf97 diff --git a/meta-virtualization b/meta-virtualization index 1665d1e2a..6f3c1d8f9 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit 1665d1e2ad2244171dcdb580df4f54739d6444bd +Subproject commit 6f3c1d8f90947408a6587be222fec575a1ca5195 diff --git a/setup b/setup index b7453e61d..63ff0e0d7 100755 --- a/setup +++ b/setup @@ -1,36 +1,14 @@ #!/bin/bash +pushd . -# Check if the first argument is -v or --verbose -verbose=0 -if [[ $1 == "-v" ]] || [[ $1 == "--verbose" ]]; then - verbose=1 - # Shift the positional parameters to the left, effectively removing the first argument - shift -fi +source poky/oe-init-build-env ./build -# Define a function for verbose output -function verbose_output { - if [[ $verbose -eq 1 ]]; then - echo "$1" - fi -} +bitbake-layers add-layer ../meta-confidential-compute +bitbake-layers add-layer ../meta-openembedded/meta-oe +bitbake-layers add-layer ../meta-openembedded/meta-python +bitbake-layers add-layer ../meta-openembedded/meta-networking +bitbake-layers add-layer ../meta-openembedded/meta-filesystems +bitbake-layers add-layer ../meta-virtualization +bitbake-layers add-layer ../meta-dstack -# Set CURRENT_PATH to the current working directory -CURRENT_PATH=$(pwd) -verbose_output "Set CURRENT_PATH to $CURRENT_PATH" - -# Switch to the oe-init-build-env directory -pushd $CURRENT_PATH/poky/ -verbose_output "Switched to $(pwd)" - -# Source the oe-init-build-env script -source oe-init-build-env -verbose_output "Sourced the oe-init-build-env script" - -# Add the meta-evm, meta-confidential-compute, meta-secure-core/meta-tpm2, meta-openembedded/meta-python, meta-openembedded/meta-oe layers meta-rust-bin meta-clang to bblayers.conf -bitbake-layers add-layer ../../meta-confidential-compute -bitbake-layers add-layer ../../meta-dstack - -# Return to the original directory popd -verbose_output "Returned to $(pwd)" From 11e63d60730e16d20d9d67fe23a38a726179789d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 13 Oct 2024 10:57:04 +0800 Subject: [PATCH 005/435] Add some kernel config fragments --- build/emu | 15 +++++++++++++++ meta-dstack/conf/distro/dstack.conf | 3 ++- meta-dstack/conf/layer.conf | 2 +- .../base-files/base-files%.bbappend | 4 ++++ .../recipes-core/base-files/dstack-prebuilt.bb | 17 +++++++++++++++++ meta-dstack/recipes-core/base-files/files/motd | 1 + .../images/dstack-initramfs.bb | 5 ++++- .../recipes-core/linux/files/dstack-docker.cfg | 18 ++++++++++++++++++ .../recipes-core/linux/files/dstack-docker.scc | 3 +++ .../recipes-core/linux/files/dstack-tdx.cfg | 1 + .../recipes-core/linux/files/dstack-tdx.scc | 3 +++ .../recipes-core/linux/files/dstack.cfg | 2 ++ .../recipes-core/linux/files/dstack.scc | 3 +++ .../recipes-core/linux/linux-yocto%.bbappend | 16 ++++++++++++++++ 14 files changed, 90 insertions(+), 3 deletions(-) create mode 100755 build/emu create mode 100644 meta-dstack/recipes-core/base-files/base-files%.bbappend create mode 100644 meta-dstack/recipes-core/base-files/dstack-prebuilt.bb create mode 100644 meta-dstack/recipes-core/base-files/files/motd rename meta-dstack/{recipes => recipes-core}/images/dstack-initramfs.bb (94%) create mode 100644 meta-dstack/recipes-core/linux/files/dstack-docker.cfg create mode 100644 meta-dstack/recipes-core/linux/files/dstack-docker.scc create mode 100644 meta-dstack/recipes-core/linux/files/dstack-tdx.cfg create mode 100644 meta-dstack/recipes-core/linux/files/dstack-tdx.scc create mode 100644 meta-dstack/recipes-core/linux/files/dstack.cfg create mode 100644 meta-dstack/recipes-core/linux/files/dstack.scc create mode 100644 meta-dstack/recipes-core/linux/linux-yocto%.bbappend diff --git a/build/emu b/build/emu new file mode 100755 index 000000000..8b5080808 --- /dev/null +++ b/build/emu @@ -0,0 +1,15 @@ +#!/bin/sh +KERNEL=./tmp/deploy/images/tdx/bzImage +INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz +qemu-system-x86_64 \ + -kernel ${KERNEL} \ + -initrd ${INITRD} \ + -accel kvm \ + -m 2G -smp 4 \ + -cpu host \ + -machine q35,kernel_irqchip=split \ + -nographic \ + -nodefaults \ + -serial stdio \ + -append "console=ttyS0" + diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index afedd890a..56f3c5fff 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,6 +1,7 @@ require conf/distro/cvm.conf - DISTRO = "dstack" DISTRO_NAME = "DStack" +DISTRO_FEATURES:append = " virtualization seccomp" +DISTRO_VERSION = "1.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "dstack-initramfs" diff --git a/meta-dstack/conf/layer.conf b/meta-dstack/conf/layer.conf index 19f4ef0cd..15ad1890e 100644 --- a/meta-dstack/conf/layer.conf +++ b/meta-dstack/conf/layer.conf @@ -1,5 +1,5 @@ BBPATH .= ":${LAYERDIR}" -BBFILES += "${LAYERDIR}/recipes/*/*.bb ${LAYERDIR}/recipes/*/*.bbappend" +BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend" BBFILE_COLLECTIONS += "dstack" BBFILE_PATTERN_dstack := "^${LAYERDIR}/" diff --git a/meta-dstack/recipes-core/base-files/base-files%.bbappend b/meta-dstack/recipes-core/base-files/base-files%.bbappend new file mode 100644 index 000000000..f859de39f --- /dev/null +++ b/meta-dstack/recipes-core/base-files/base-files%.bbappend @@ -0,0 +1,4 @@ +do_install:append() { + # Remove the motd file from poky so we can use our own + rm -rf ${D}${sysconfdir}/motd +} diff --git a/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb b/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb new file mode 100644 index 000000000..4a7d71aa8 --- /dev/null +++ b/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb @@ -0,0 +1,17 @@ +SUMMARY = "Dstack base files" + +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" + +SRC_URI = "file://motd" + +inherit allarch + +S = "${WORKDIR}" + +do_install() { + install -d ${D}${sysconfdir}/ + install -d ${D}${sbindir}/ + install -m 0644 ${S}/motd ${D}${sysconfdir}/motd + install -m 0755 ${S}/mount-cgroups ${D}${sbindir}/mount-cgroups +} diff --git a/meta-dstack/recipes-core/base-files/files/motd b/meta-dstack/recipes-core/base-files/files/motd new file mode 100644 index 000000000..9cff826ce --- /dev/null +++ b/meta-dstack/recipes-core/base-files/files/motd @@ -0,0 +1 @@ +Welcome to Dstack! diff --git a/meta-dstack/recipes/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb similarity index 94% rename from meta-dstack/recipes/images/dstack-initramfs.bb rename to meta-dstack/recipes-core/images/dstack-initramfs.bb index 97e3bebaf..1e4299236 100644 --- a/meta-dstack/recipes/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -13,9 +13,12 @@ PACKAGE_INSTALL = "busybox-mdev \ netbase \ busybox-udhcpd \ ca-certificates \ + iptables \ sysvinit \ dropbear \ - docker-moby \ + docker \ + docker-compose \ + dstack-prebuilt \ ${VIRTUAL-RUNTIME_base-utils} \ ${ROOTFS_BOOTSTRAP_INSTALL}" diff --git a/meta-dstack/recipes-core/linux/files/dstack-docker.cfg b/meta-dstack/recipes-core/linux/files/dstack-docker.cfg new file mode 100644 index 000000000..7aec0d0fb --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack-docker.cfg @@ -0,0 +1,18 @@ +CONFIG_BRIDGE=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_BPF_SYSCALL=y +CONFIG_IP_VS=m +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_SECURITY_SELINUX=y +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_XFRM=y +CONFIG_XFRM_USER=y +CONFIG_XFRM_ALGO=y +CONFIG_INET_ESP=y +CONFIG_NETFILTER_XT_MATCH_BPF=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_IPVLAN=m +CONFIG_BTRFS_FS=m +CONFIG_BTRFS_FS_POSIX_ACL=y \ No newline at end of file diff --git a/meta-dstack/recipes-core/linux/files/dstack-docker.scc b/meta-dstack/recipes-core/linux/files/dstack-docker.scc new file mode 100644 index 000000000..40bf76dec --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack-docker.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "DStack Docker configuration" + +kconf non-hardware dstack-docker.cfg diff --git a/meta-dstack/recipes-core/linux/files/dstack-tdx.cfg b/meta-dstack/recipes-core/linux/files/dstack-tdx.cfg new file mode 100644 index 000000000..4dbf53d16 --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack-tdx.cfg @@ -0,0 +1 @@ +CONFIG_TDX_GUEST_DRIVER=n diff --git a/meta-dstack/recipes-core/linux/files/dstack-tdx.scc b/meta-dstack/recipes-core/linux/files/dstack-tdx.scc new file mode 100644 index 000000000..e7a9aba53 --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack-tdx.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "DStack TDX configuration" + +kconf non-hardware dstack-tdx.cfg diff --git a/meta-dstack/recipes-core/linux/files/dstack.cfg b/meta-dstack/recipes-core/linux/files/dstack.cfg new file mode 100644 index 000000000..ebf446f5b --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack.cfg @@ -0,0 +1,2 @@ +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y \ No newline at end of file diff --git a/meta-dstack/recipes-core/linux/files/dstack.scc b/meta-dstack/recipes-core/linux/files/dstack.scc new file mode 100644 index 000000000..4725b8084 --- /dev/null +++ b/meta-dstack/recipes-core/linux/files/dstack.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "DStack configuration" + +kconf non-hardware dstack.cfg diff --git a/meta-dstack/recipes-core/linux/linux-yocto%.bbappend b/meta-dstack/recipes-core/linux/linux-yocto%.bbappend new file mode 100644 index 000000000..3d78c2013 --- /dev/null +++ b/meta-dstack/recipes-core/linux/linux-yocto%.bbappend @@ -0,0 +1,16 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/cfg:" + +SRC_URI += "file://dstack-docker.cfg \ + file://dstack-docker.scc \ + file://dstack-tdx.cfg \ + file://dstack-tdx.scc \ + file://dstack.cfg \ + file://dstack.scc" + +KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ + features/overlayfs/overlayfs.scc \ + features/netfilter/netfilter.scc \ + dstack-docker.scc \ + dstack.scc" + +KERNEL_FEATURES:append:tdx = " dstack-tdx.scc" From 546b66e07d1230a3fab9dfdf6e04a9091060a947 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 14 Oct 2024 13:23:21 +0800 Subject: [PATCH 006/435] Config network and docker --- build/conf/local.conf | 2 ++ build/emu | 1 + meta-dstack/recipes-core/base-files/dstack-prebuilt.bb | 8 ++++---- meta-dstack/recipes-core/base-files/files/dockerd_env | 1 + meta-dstack/recipes-core/busybox/busybox%.bbappend | 2 ++ meta-dstack/recipes-core/busybox/files/fragment.cfg | 1 + meta-dstack/recipes-core/images/dstack-initramfs.bb | 6 +++--- meta-dstack/recipes-core/linux/linux-yocto%.bbappend | 2 +- 8 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 meta-dstack/recipes-core/base-files/files/dockerd_env create mode 100644 meta-dstack/recipes-core/busybox/busybox%.bbappend create mode 100644 meta-dstack/recipes-core/busybox/files/fragment.cfg diff --git a/build/conf/local.conf b/build/conf/local.conf index 204ff0796..f35516af8 100644 --- a/build/conf/local.conf +++ b/build/conf/local.conf @@ -286,3 +286,5 @@ PACKAGECONFIG:append:pn-qemu-system-native = " sdl" # track the version of this file when it was generated. This can safely be ignored if # this doesn't mean anything to you. CONF_VERSION = "2" + +SERIAL_CONSOLES = "115200;ttyS0" \ No newline at end of file diff --git a/build/emu b/build/emu index 8b5080808..d34709ebf 100755 --- a/build/emu +++ b/build/emu @@ -10,6 +10,7 @@ qemu-system-x86_64 \ -machine q35,kernel_irqchip=split \ -nographic \ -nodefaults \ + -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ -serial stdio \ -append "console=ttyS0" diff --git a/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb b/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb index 4a7d71aa8..f2270f993 100644 --- a/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb +++ b/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb @@ -3,15 +3,15 @@ SUMMARY = "Dstack base files" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" -SRC_URI = "file://motd" +SRC_URI = "file://motd \ + file://dockerd_env" inherit allarch S = "${WORKDIR}" do_install() { - install -d ${D}${sysconfdir}/ - install -d ${D}${sbindir}/ + install -d ${D}${sysconfdir}/sysconfig/ install -m 0644 ${S}/motd ${D}${sysconfdir}/motd - install -m 0755 ${S}/mount-cgroups ${D}${sbindir}/mount-cgroups + install -m 0755 ${S}/dockerd_env ${D}${sysconfdir}/sysconfig/dockerd } diff --git a/meta-dstack/recipes-core/base-files/files/dockerd_env b/meta-dstack/recipes-core/base-files/files/dockerd_env new file mode 100644 index 000000000..d5a17ad60 --- /dev/null +++ b/meta-dstack/recipes-core/base-files/files/dockerd_env @@ -0,0 +1 @@ +export DOCKER_RAMDISK=true \ No newline at end of file diff --git a/meta-dstack/recipes-core/busybox/busybox%.bbappend b/meta-dstack/recipes-core/busybox/busybox%.bbappend new file mode 100644 index 000000000..bef35ef5d --- /dev/null +++ b/meta-dstack/recipes-core/busybox/busybox%.bbappend @@ -0,0 +1,2 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" +SRC_URI += "file://fragment.cfg" diff --git a/meta-dstack/recipes-core/busybox/files/fragment.cfg b/meta-dstack/recipes-core/busybox/files/fragment.cfg new file mode 100644 index 000000000..b7dab01c0 --- /dev/null +++ b/meta-dstack/recipes-core/busybox/files/fragment.cfg @@ -0,0 +1 @@ +CONFIG_UDHCPD=n diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 1e4299236..8e55328df 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -11,7 +11,7 @@ PACKAGE_INSTALL = "busybox-mdev \ base-files \ base-passwd \ netbase \ - busybox-udhcpd \ + busybox-udhcpc \ ca-certificates \ iptables \ sysvinit \ @@ -25,9 +25,9 @@ PACKAGE_INSTALL = "busybox-mdev \ INITRAMFS_MAXSIZE = "1000000" # Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "debug-tweaks" +IMAGE_FEATURES = "debug-tweaks read-only-rootfs overlayfs-etc" -export IMAGE_BASENAME = "dstack-initramfs" +IMAGE_BASENAME = "dstack-initramfs" IMAGE_NAME_SUFFIX ?= "" IMAGE_LINGUAS = "" diff --git a/meta-dstack/recipes-core/linux/linux-yocto%.bbappend b/meta-dstack/recipes-core/linux/linux-yocto%.bbappend index 3d78c2013..fb0ef2cda 100644 --- a/meta-dstack/recipes-core/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-core/linux/linux-yocto%.bbappend @@ -1,4 +1,4 @@ -FILESEXTRAPATHS:prepend := "${THISDIR}/cfg:" +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" SRC_URI += "file://dstack-docker.cfg \ file://dstack-docker.scc \ From 9e2e485c426bd900dca98b800a666f7a667e7144 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 14 Oct 2024 16:02:30 +0800 Subject: [PATCH 007/435] Add CFS --- meta-dstack/recipes-core/images/dstack-initramfs.bb | 2 +- meta-dstack/recipes-core/linux/files/dstack-docker.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 8e55328df..2ff918afd 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -25,7 +25,7 @@ PACKAGE_INSTALL = "busybox-mdev \ INITRAMFS_MAXSIZE = "1000000" # Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "debug-tweaks read-only-rootfs overlayfs-etc" +IMAGE_FEATURES = "debug-tweaks read-only-rootfs" IMAGE_BASENAME = "dstack-initramfs" IMAGE_NAME_SUFFIX ?= "" diff --git a/meta-dstack/recipes-core/linux/files/dstack-docker.cfg b/meta-dstack/recipes-core/linux/files/dstack-docker.cfg index 7aec0d0fb..5cefef857 100644 --- a/meta-dstack/recipes-core/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-core/linux/files/dstack-docker.cfg @@ -15,4 +15,5 @@ CONFIG_NETFILTER_XT_MATCH_BPF=y CONFIG_CRYPTO_SEQIV=y CONFIG_IPVLAN=m CONFIG_BTRFS_FS=m -CONFIG_BTRFS_FS_POSIX_ACL=y \ No newline at end of file +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_CFS_BANDWIDTH=y From 716e09215edbd709117255ada576d44670c94216 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 14 Oct 2024 22:42:50 +0800 Subject: [PATCH 008/435] Add service tappd --- .gitmodules | 3 + build/conf/local.conf | 4 +- .../recipes-core/dstack-guest/dstack-guest.bb | 38 ++++++++ .../dstack-guest/files/tappd.init | 87 +++++++++++++++++++ .../recipes-core/images/dstack-initramfs.bb | 4 +- meta-rust-bin | 1 + setup | 1 + 7 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 meta-dstack/recipes-core/dstack-guest/dstack-guest.bb create mode 100644 meta-dstack/recipes-core/dstack-guest/files/tappd.init create mode 160000 meta-rust-bin diff --git a/.gitmodules b/.gitmodules index 0b7abaecf..c721e1d7b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "meta-openembedded"] path = meta-openembedded url = https://github.com/openembedded/meta-openembedded +[submodule "meta-rust-bin"] + path = meta-rust-bin + url = https://github.com/rust-embedded/meta-rust-bin diff --git a/build/conf/local.conf b/build/conf/local.conf index f35516af8..b236c288c 100644 --- a/build/conf/local.conf +++ b/build/conf/local.conf @@ -287,4 +287,6 @@ PACKAGECONFIG:append:pn-qemu-system-native = " sdl" # this doesn't mean anything to you. CONF_VERSION = "2" -SERIAL_CONSOLES = "115200;ttyS0" \ No newline at end of file +SERIAL_CONSOLES = "115200;ttyS0" +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.80.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.80.0" \ No newline at end of file diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb new file mode 100644 index 000000000..11316826f --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -0,0 +1,38 @@ +SUMMARY = "DStack guest binaries" +DESCRIPTION = "Guest binaries for DStack, a decentralized computing stack" +HOMEPAGE = "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/Phala-Network/dstack" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" +DEPENDS:append = " update-rc.d-native" + +SRC_URI = "git://github.com/Phala-Network/dstack;protocol=https;branch=master \ + file://tappd.init" +SRCREV = "9d5ad73d61d9c7bcacb5b2d822adbf4f62aa9631" + +S = "${WORKDIR}/git" + +inherit cargo_bin + +do_configure() { + cargo_bin_do_configure +} + +do_compile() { + cargo_bin_do_compile +} + +do_compile[network] = "1" + +do_install() { + install -d ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} + install -d ${D}${sysconfdir}/init.d + install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd + + # + # Create runlevel links + # + update-rc.d -r ${D} tappd start 90 2 3 4 5 . +} diff --git a/meta-dstack/recipes-core/dstack-guest/files/tappd.init b/meta-dstack/recipes-core/dstack-guest/files/tappd.init new file mode 100644 index 000000000..c27ee3cbb --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/tappd.init @@ -0,0 +1,87 @@ +#!/bin/sh +# +# SPDX-License-Identifier: GPL-2.0-only +# + +### BEGIN INIT INFO +# Provides: tappd +# Required-Start: $network $local_fs +# Required-Stop: $network $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start tappd daemon +# Description: Start the tappd daemon +### END INIT INFO + +# Source function library. +. /etc/init.d/functions + +prog="tappd" +exec="/usr/bin/$prog" +pidfile="/var/run/$prog.pid" +lockfile="/var/lock/subsys/$prog" +logfile="/var/log/$prog" + +[ -e /etc/default/$prog ] && . /etc/default/$prog + +start() { + [ -x $exec ] || exit 5 + + printf "Starting $prog:\t" + if ! [ -f $pidfile ]; then + $exec $DAEMON_OPTS & + echo $! > $pidfile + touch $lockfile + success + echo + else + failure + echo + printf "$pidfile still exists...\n" + exit 7 + fi +} + +stop() { + echo -n "Stopping $prog: " + killproc $prog + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +rh_status() { + status $prog +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + status) + rh_status + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 2 +esac + +exit $? diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 2ff918afd..0101c5d51 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -12,7 +12,6 @@ PACKAGE_INSTALL = "busybox-mdev \ base-passwd \ netbase \ busybox-udhcpc \ - ca-certificates \ iptables \ sysvinit \ dropbear \ @@ -20,7 +19,8 @@ PACKAGE_INSTALL = "busybox-mdev \ docker-compose \ dstack-prebuilt \ ${VIRTUAL-RUNTIME_base-utils} \ - ${ROOTFS_BOOTSTRAP_INSTALL}" + ${ROOTFS_BOOTSTRAP_INSTALL} \ + dstack-guest" INITRAMFS_MAXSIZE = "1000000" diff --git a/meta-rust-bin b/meta-rust-bin new file mode 160000 index 000000000..c05de0f89 --- /dev/null +++ b/meta-rust-bin @@ -0,0 +1 @@ +Subproject commit c05de0f8922ca8736af2c0eec5f570568fcd2c8e diff --git a/setup b/setup index 63ff0e0d7..d002b42a5 100755 --- a/setup +++ b/setup @@ -9,6 +9,7 @@ bitbake-layers add-layer ../meta-openembedded/meta-python bitbake-layers add-layer ../meta-openembedded/meta-networking bitbake-layers add-layer ../meta-openembedded/meta-filesystems bitbake-layers add-layer ../meta-virtualization +bitbake-layers add-layer ../meta-rust-bin bitbake-layers add-layer ../meta-dstack popd From 8f7fe12c86fa34341da4b8bc17039d1b6e7c2016 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 14 Oct 2024 23:23:17 +0800 Subject: [PATCH 009/435] Add mock shared files --- build/emu | 1 + build/mock/shared/certs/ca.cert | 11 +++++++++++ build/mock/shared/certs/tmp-ca.cert | 12 ++++++++++++ build/mock/shared/certs/tmp-ca.key | 5 +++++ build/mock/shared/config.json | 1 + build/mock/shared/docker-compose.yaml | 9 +++++++++ 6 files changed, 39 insertions(+) create mode 100644 build/mock/shared/certs/ca.cert create mode 100644 build/mock/shared/certs/tmp-ca.cert create mode 100644 build/mock/shared/certs/tmp-ca.key create mode 100644 build/mock/shared/config.json create mode 100644 build/mock/shared/docker-compose.yaml diff --git a/build/emu b/build/emu index d34709ebf..37f3fe1f0 100755 --- a/build/emu +++ b/build/emu @@ -12,5 +12,6 @@ qemu-system-x86_64 \ -nodefaults \ -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ -serial stdio \ + -virtfs local,path=./mock/shared,mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0 \ -append "console=ttyS0" diff --git a/build/mock/shared/certs/ca.cert b/build/mock/shared/certs/ca.cert new file mode 100644 index 000000000..0a6823669 --- /dev/null +++ b/build/mock/shared/certs/ca.cert @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBmDCCAT2gAwIBAgIUIH3aWOG0Cg7iYeAlvMFvt7oR/wYwCgYIKoZIzj0EAwIw +LzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEVMBMGA1UEAwwMUGhhbGEgS01TIENB +MCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAvMRYwFAYDVQQKDA1Q +aGFsYSBOZXR3b3JrMRUwEwYDVQQDDAxQaGFsYSBLTVMgQ0EwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAARAGfFiokX4htvaKc2nFntqzxNZLJduL7TgXrOKoZTAJkm7 +c5en/oGwBfSpqVVBjqeIBrbDHmyIBHXiewfMjsHBozUwMzAdBgNVHQ4EFgQURzKe +YZHAfYd6o4vAd//GfWIZ874wEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD +AgNJADBGAiEA9zuPQ7Ht9yobac8CSxJwN2hq9LL1CyEpt7YOpTOVe54CIQCITzn4 +9NX9hBMbmOJDcwa2PwIwub+SckwaIMbcmkCg9A== +-----END CERTIFICATE----- diff --git a/build/mock/shared/certs/tmp-ca.cert b/build/mock/shared/certs/tmp-ca.cert new file mode 100644 index 000000000..5180be5fa --- /dev/null +++ b/build/mock/shared/certs/tmp-ca.cert @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBrzCCAVWgAwIBAgIUU8HFA9rw6T0Dl/tlCJ6QRmuTAfowCgYIKoZIzj0EAwIw +OzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEhMB8GA1UEAwwYUGhhbGEgS01TIENs +aWVudCBUZW1wIENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjA7 +MRYwFAYDVQQKDA1QaGFsYSBOZXR3b3JrMSEwHwYDVQQDDBhQaGFsYSBLTVMgQ2xp +ZW50IFRlbXAgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATs4YKouJPDMeog +W59VRRgzO/kh8W6av18kLe9dxMqoy6R6Lmy0jm8kmLpUugZsKlF372fJX3F92HsQ +fI9JDssAozUwMzAdBgNVHQ4EFgQUlnyUeOVQWZFIPZdGnf0Ix94DxhMwEgYDVR0T +AQH/BAgwBgEB/wIBATAKBggqhkjOPQQDAgNIADBFAiBYF34VIHuevsuprscxJMGh +TbrbQuH5EQ69SSjn+Wpi3AIhALzw2Df/PBvLe+3+f2qhtaIogoKbRnvCmGAtPiIb +I5vR +-----END CERTIFICATE----- diff --git a/build/mock/shared/certs/tmp-ca.key b/build/mock/shared/certs/tmp-ca.key new file mode 100644 index 000000000..7cfe3c73d --- /dev/null +++ b/build/mock/shared/certs/tmp-ca.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgv2G1wga5HtcPGFKK +YqjMsWIKAHT69J0Rwbd/gtO/5lWhRANCAATs4YKouJPDMeogW59VRRgzO/kh8W6a +v18kLe9dxMqoy6R6Lmy0jm8kmLpUugZsKlF372fJX3F92HsQfI9JDssA +-----END PRIVATE KEY----- diff --git a/build/mock/shared/config.json b/build/mock/shared/config.json new file mode 100644 index 000000000..5704dfee2 --- /dev/null +++ b/build/mock/shared/config.json @@ -0,0 +1 @@ +{"rootfs_hash":"980e1b690743da60a33705d8f80425a193a37203b36c19792a5b4d94ddd0514d","kms_url":"https://kms.1022.kvin.wang:9043","tproxy_url":"https://tproxy.1022.kvin.wang:9010"} \ No newline at end of file diff --git a/build/mock/shared/docker-compose.yaml b/build/mock/shared/docker-compose.yaml new file mode 100644 index 000000000..2798ce386 --- /dev/null +++ b/build/mock/shared/docker-compose.yaml @@ -0,0 +1,9 @@ +version: '3' + +services: + nginx: + image: nginx:latest + ports: + - "8080:80" + restart: always + From db953d1d9a33dccb5b60c4e89330994f09083fec Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 14 Oct 2024 23:23:35 +0800 Subject: [PATCH 010/435] Enable 9p dirvers --- meta-dstack/recipes-core/linux/files/dstack.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/linux/files/dstack.cfg b/meta-dstack/recipes-core/linux/files/dstack.cfg index ebf446f5b..fc39d9150 100644 --- a/meta-dstack/recipes-core/linux/files/dstack.cfg +++ b/meta-dstack/recipes-core/linux/files/dstack.cfg @@ -1,2 +1,8 @@ CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y \ No newline at end of file +CONFIG_IKCONFIG_PROC=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_9P_FS=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_PCI=y +CONFIG_VIRTIO_PCI=y \ No newline at end of file From b8df57b1a05b7012c105eca12db6013a17565c5f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 15 Oct 2024 17:53:59 +0800 Subject: [PATCH 011/435] Add kernel module tdx-guest --- .gitmodules | 3 +++ dstack | 1 + .../recipes-core/dstack-guest/dstack-guest.bb | 13 +++++++------ .../recipes-core/images/dstack-initramfs.bb | 1 + .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 15 +++++++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) create mode 160000 dstack create mode 100644 meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb diff --git a/.gitmodules b/.gitmodules index c721e1d7b..9b9c09b5f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "meta-rust-bin"] path = meta-rust-bin url = https://github.com/rust-embedded/meta-rust-bin +[submodule "dstack"] + path = dstack + url = https://github.com/Phala-Network/dstack diff --git a/dstack b/dstack new file mode 160000 index 000000000..29f184c28 --- /dev/null +++ b/dstack @@ -0,0 +1 @@ +Subproject commit 29f184c2835ce0bc90ee5497d3b5a59724d9a2d8 diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 11316826f..300074530 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -1,15 +1,16 @@ -SUMMARY = "DStack guest binaries" -DESCRIPTION = "Guest binaries for DStack, a decentralized computing stack" -HOMEPAGE = "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/Phala-Network/dstack" +SUMMARY = "Guest binaries for DStack, a decentralized computing stack" +DESCRIPTION = "${SUMMARY}" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" + DEPENDS:append = " update-rc.d-native" -SRC_URI = "git://github.com/Phala-Network/dstack;protocol=https;branch=master \ +REPO_ROOT = "${THISDIR}/../../.." + +SRC_URI = "file://${REPO_ROOT}/dstack \ file://tappd.init" -SRCREV = "9d5ad73d61d9c7bcacb5b2d822adbf4f62aa9631" -S = "${WORKDIR}/git" +S = "${WORKDIR}/${REPO_ROOT}/dstack" inherit cargo_bin diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 0101c5d51..601ab335e 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -20,6 +20,7 @@ PACKAGE_INSTALL = "busybox-mdev \ dstack-prebuilt \ ${VIRTUAL-RUNTIME_base-utils} \ ${ROOTFS_BOOTSTRAP_INSTALL} \ + kernel-module-tdx-guest \ dstack-guest" INITRAMFS_MAXSIZE = "1000000" diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb new file mode 100644 index 000000000..7a13f6771 --- /dev/null +++ b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb @@ -0,0 +1,15 @@ +SUMMARY = "TDX guest kernel module for Intel Trust Domain Extensions" +DESCRIPTION = "${SUMMARY}" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" + +inherit module + +REPO_ROOT = "${THISDIR}/../../.." +SRC_DIR = "${REPO_ROOT}/dstack/mod-tdx-guest" + +SRC_URI = "file://${SRC_DIR}" + +S = "${WORKDIR}/${SRC_DIR}" + +RPROVIDES:${PN} += "kernel-module-tdx-guest" From 46590811dda907dd01da50dfa717aeb5a84868ee Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 15 Oct 2024 17:54:14 +0800 Subject: [PATCH 012/435] Add vda in emu --- build/emu | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/emu b/build/emu index 37f3fe1f0..4fbb4ef70 100755 --- a/build/emu +++ b/build/emu @@ -1,6 +1,9 @@ #!/bin/sh KERNEL=./tmp/deploy/images/tdx/bzImage INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz +if [ ! -f ./mock/vda.qcow2 ]; then + qemu-img create -f qcow2 ./mock/vda.qcow2 10G +fi qemu-system-x86_64 \ -kernel ${KERNEL} \ -initrd ${INITRD} \ @@ -13,5 +16,6 @@ qemu-system-x86_64 \ -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ -serial stdio \ -virtfs local,path=./mock/shared,mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0 \ + -drive file=./mock/vda.qcow2,if=virtio,format=qcow2 \ -append "console=ttyS0" From 52edb3f0554316028781f09db91f8a7a7d05d718 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 15 Oct 2024 13:23:52 +0000 Subject: [PATCH 013/435] Add fix_self_uid_map.sh --- scripts/fix_self_uid_map.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 scripts/fix_self_uid_map.sh diff --git a/scripts/fix_self_uid_map.sh b/scripts/fix_self_uid_map.sh new file mode 100755 index 000000000..be09997e5 --- /dev/null +++ b/scripts/fix_self_uid_map.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# BUG: https://bugs.launchpad.net/ubuntu/+source/apparmor/+bug/2056555 +sudo apparmor_parser -R /etc/apparmor.d/unprivileged_userns From d5bc5ef8bbedac37702b3804d399d90257f25cd6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Oct 2024 07:50:44 +0000 Subject: [PATCH 014/435] Fix run.sh --- build/run.sh | 56 +++++++++++++++++++ .../recipes-core/dstack-guest/dstack-guest.bb | 2 + 2 files changed, 58 insertions(+) create mode 100755 build/run.sh diff --git a/build/run.sh b/build/run.sh new file mode 100755 index 000000000..7e7a0b58e --- /dev/null +++ b/build/run.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +VMDIR=${1:-./mock} + +PROCESS_NAME=qemu + +KERNEL=./tmp/deploy/images/tdx/bzImage +INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz +VDA=${VMDIR}/vda.qcow2 +VDA_SIZE=10G +CMDLINE="console=ttyS0" +CONFIG_DIR=${VMDIR}/shared +TD=${TD:-1} +TDVF_FIRMWARE=/usr/share/ovmf/OVMF.fd +RO=${RO:-off} +CID=$(( ( RANDOM % 10000 ) + 3 )) + +ARGS="${ARGS} -kernel ${KERNEL}" +ARGS="${ARGS} -initrd ${INITRD}" + +if [ "${TD}" == "1" ]; then + MACHINE_ARGS=",confidential-guest-support=tdx,hpet=off" + PROCESS_NAME=td + TDX_ARGS="-device vhost-vsock-pci,guest-cid=${CID} -object tdx-guest,id=tdx" + BIOS="-bios ${TDVF_FIRMWARE}" +fi + +echo INITRD=${INITRD} +echo ARGS=${ARGS} +echo VDA=${VDA} +echo CMDLINE=${CMDLINE} +echo TD=${TD} +echo TDX_ARGS=${TDX_ARGS} +echo BIOS=${BIOS} + +sleep 2 + +if [ ! -f ${VDA} ]; then + qemu-img create -f qcow2 ${VDA} ${VDA_SIZE} +fi +qemu-system-x86_64 \ + -accel kvm \ + -m 8G -smp 16 \ + -name ${PROCESS_NAME},process=${PROCESS_NAME},debug-threads=on \ + -cpu host \ + -machine q35,kernel_irqchip=split${MACHINE_ARGS} \ + ${BIOS} \ + ${TDX_ARGS} \ + -nographic \ + -nodefaults \ + -chardev stdio,id=ser0,signal=on -serial chardev:ser0 \ + -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ + -drive file=${VDA},if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 \ + -virtfs local,path=${CONFIG_DIR},mount_tag=host-shared,readonly=${RO},security_model=mapped,id=virtfs0 \ + ${ARGS} \ + -append "${CMDLINE}" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 300074530..cceb2693f 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -32,6 +32,8 @@ do_install() { install -d ${D}${sysconfdir}/init.d install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd + install -d ${D}$/mnt/host-shared + # # Create runlevel links # From 37836a5f2d7c0820ec05407fac87f3eef874634c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Oct 2024 14:31:28 +0000 Subject: [PATCH 015/435] Switch to systemd --- build/emu | 3 +- build/run.sh | 2 +- meta-dstack/conf/distro/dstack.conf | 8 ++++- .../recipes-core/docker/docker-moby%.bbappend | 1 + .../recipes-core/dstack-guest/dstack-guest.bb | 26 +++++++++++----- .../dstack-guest/files/tappd.service | 12 ++++++++ .../recipes-core/images/dstack-initramfs.bb | 30 ++++++++++++++----- 7 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 meta-dstack/recipes-core/docker/docker-moby%.bbappend create mode 100644 meta-dstack/recipes-core/dstack-guest/files/tappd.service diff --git a/build/emu b/build/emu index 4fbb4ef70..ce552549e 100755 --- a/build/emu +++ b/build/emu @@ -17,5 +17,6 @@ qemu-system-x86_64 \ -serial stdio \ -virtfs local,path=./mock/shared,mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0 \ -drive file=./mock/vda.qcow2,if=virtio,format=qcow2 \ - -append "console=ttyS0" + -append "console=ttyS0 init=/init" + diff --git a/build/run.sh b/build/run.sh index 7e7a0b58e..66e620f97 100755 --- a/build/run.sh +++ b/build/run.sh @@ -8,7 +8,7 @@ KERNEL=./tmp/deploy/images/tdx/bzImage INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz VDA=${VMDIR}/vda.qcow2 VDA_SIZE=10G -CMDLINE="console=ttyS0" +CMDLINE="console=ttyS0 init=/init" CONFIG_DIR=${VMDIR}/shared TD=${TD:-1} TDVF_FIRMWARE=/usr/share/ovmf/OVMF.fd diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 56f3c5fff..4cc83a318 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,7 +1,13 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "DStack" -DISTRO_FEATURES:append = " virtualization seccomp" +DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" +DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" + DISTRO_VERSION = "1.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "dstack-initramfs" + +VIRTUAL-RUNTIME_init_manager = "systemd" +VIRTUAL-RUNTIME_initscripts = "systemd-compat-units" +VIRTUAL-RUNTIME_dev_manager = "systemd" diff --git a/meta-dstack/recipes-core/docker/docker-moby%.bbappend b/meta-dstack/recipes-core/docker/docker-moby%.bbappend new file mode 100644 index 000000000..0f9dc8692 --- /dev/null +++ b/meta-dstack/recipes-core/docker/docker-moby%.bbappend @@ -0,0 +1 @@ +SYSTEMD_SERVICE:${PN}:append = " docker.service" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index cceb2693f..acbe5f7e4 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -3,15 +3,24 @@ DESCRIPTION = "${SUMMARY}" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" -DEPENDS:append = " update-rc.d-native" +inherit systemd update-rc.d REPO_ROOT = "${THISDIR}/../../.." SRC_URI = "file://${REPO_ROOT}/dstack \ - file://tappd.init" + file://tappd.init \ + file://tappd.service" S = "${WORKDIR}/${REPO_ROOT}/dstack" +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" +SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','tappd.service','',d)}" +SYSTEMD_AUTO_ENABLE:${PN} = "enable" + +INITSCRIPT_PACKAGES += "${@bb.utils.contains('DISTRO_FEATURES','systemd','','${PN}',d)}" +INITSCRIPT_NAME:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','','tappd.init',d)}" +INITSCRIPT_PARAMS:${PN} = "defaults" + inherit cargo_bin do_configure() { @@ -29,13 +38,14 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} - install -d ${D}${sysconfdir}/init.d - install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd install -d ${D}$/mnt/host-shared - # - # Create runlevel links - # - update-rc.d -r ${D} tappd start 90 2 3 4 5 . + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/tappd.service ${D}${systemd_system_unitdir} + else + install -d ${D}${sysconfdir}/init.d + install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd.init + fi } diff --git a/meta-dstack/recipes-core/dstack-guest/files/tappd.service b/meta-dstack/recipes-core/dstack-guest/files/tappd.service new file mode 100644 index 000000000..4c7a7f856 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/tappd.service @@ -0,0 +1,12 @@ +[Unit] +Description=Tappd Service +After=network.target + +[Service] +ExecStart=/bin/tappd +Restart=always +User=root +Group=root + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 601ab335e..89acdf91c 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -5,23 +5,21 @@ first 'init' program more efficiently. core-image-tiny-initramfs doesn't \ actually generate an image but rather generates boot and rootfs artifacts \ that can subsequently be picked up by external image generation tools such as wic." -PACKAGE_INSTALL = "busybox-mdev \ - init-ifupdown \ - initscripts \ +PACKAGE_INSTALL = "\ + ${VIRTUAL-RUNTIME_base-utils} \ + ${ROOTFS_BOOTSTRAP_INSTALL} \ base-files \ base-passwd \ + systemd \ netbase \ - busybox-udhcpc \ iptables \ - sysvinit \ dropbear \ docker \ docker-compose \ dstack-prebuilt \ - ${VIRTUAL-RUNTIME_base-utils} \ - ${ROOTFS_BOOTSTRAP_INSTALL} \ kernel-module-tdx-guest \ - dstack-guest" + dstack-guest \ + curl" INITRAMFS_MAXSIZE = "1000000" @@ -46,3 +44,19 @@ IMAGE_ROOTFS_EXTRA_SPACE = "0" COMPATIBLE_HOST = "x86_64.*-linux" # QB_KERNEL_CMDLINE_APPEND += "debugshell=3 init=/bin/busybox sh init" + +# Remove sysvinit related files in a postprocess function +ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" + +remove_sysvinit_files() { + # Remove /etc/init.d directory and its contents + rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d + + # Remove /etc/rc*.d directories and their contents + for d in ${IMAGE_ROOTFS}${sysconfdir}/rc*.d; do + rm -rf $d + done + + # Remove other sysvinit specific files + rm -f ${IMAGE_ROOTFS}${sysconfdir}/inittab +} From 3b56715cd91a22f147917f2fde812f4944a1b3ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Oct 2024 15:14:23 +0000 Subject: [PATCH 016/435] Remove systemd-compat-units --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 4cc83a318..7f5c759bd 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -9,5 +9,5 @@ DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "dstack-initramfs" VIRTUAL-RUNTIME_init_manager = "systemd" -VIRTUAL-RUNTIME_initscripts = "systemd-compat-units" +VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" From 55de9fbda4b3ba3cbe5258b08f65e2a6da7c437c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Oct 2024 15:15:39 +0000 Subject: [PATCH 017/435] Disable LLMNR --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 9 +++++++-- meta-dstack/recipes-core/dstack-guest/files/llmnr.conf | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 meta-dstack/recipes-core/dstack-guest/files/llmnr.conf diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index acbe5f7e4..7ae6f1f1a 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -9,7 +9,9 @@ REPO_ROOT = "${THISDIR}/../../.." SRC_URI = "file://${REPO_ROOT}/dstack \ file://tappd.init \ - file://tappd.service" + file://tappd.service \ + file://llmnr.conf \ + " S = "${WORKDIR}/${REPO_ROOT}/dstack" @@ -42,8 +44,11 @@ do_install() { install -d ${D}$/mnt/host-shared if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then - install -d ${D}${systemd_system_unitdir} + install -d ${D}${systemd_system_unitdir} \ + ${D}${sysconfdir}/systemd/resolved.conf.d + install -m 0644 ${WORKDIR}/tappd.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d else install -d ${D}${sysconfdir}/init.d install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd.init diff --git a/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf b/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf new file mode 100644 index 000000000..4b0ccf845 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf @@ -0,0 +1,2 @@ +[Resolve] +LLMNR=no \ No newline at end of file From 32edeca978bca8ac611f24ff9d8e97627de3d7bb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 07:02:20 +0000 Subject: [PATCH 018/435] Split image into initramfs and rootfs --- .../recipes-core/images/dstack-initramfs.bb | 59 +++++-------------- .../recipes-core/images/dstack-rootfs.bb | 50 ++++++++++++++++ meta-dstack/recipes-core/images/files/init | 21 +++++++ 3 files changed, 87 insertions(+), 43 deletions(-) create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs.bb create mode 100644 meta-dstack/recipes-core/images/files/init diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 89acdf91c..928d3e001 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -1,62 +1,35 @@ -# Simple initramfs image artifact generation for tiny images. -DESCRIPTION = "Tiny image capable of booting a device. The kernel includes \ -the Minimal RAM-based Initial Root Filesystem (initramfs), which finds the \ -first 'init' program more efficiently. core-image-tiny-initramfs doesn't \ -actually generate an image but rather generates boot and rootfs artifacts \ -that can subsequently be picked up by external image generation tools such as wic." - -PACKAGE_INSTALL = "\ - ${VIRTUAL-RUNTIME_base-utils} \ - ${ROOTFS_BOOTSTRAP_INSTALL} \ - base-files \ - base-passwd \ - systemd \ - netbase \ - iptables \ - dropbear \ - docker \ - docker-compose \ - dstack-prebuilt \ - kernel-module-tdx-guest \ - dstack-guest \ - curl" - -INITRAMFS_MAXSIZE = "1000000" +PACKAGE_INSTALL = "${VIRTUAL-RUNTIME_base-utils} udev base-passwd ${ROOTFS_BOOTSTRAP_INSTALL} base-files" +PACKAGE_INSTALL += "kernel-module-tdx-guest" +PACKAGE_INSTALL += "dstack-guest curl jq" # Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "debug-tweaks read-only-rootfs" +IMAGE_FEATURES = "" + +# Don't allow the initramfs to contain a kernel +PACKAGE_EXCLUDE = "kernel-image-*" -IMAGE_BASENAME = "dstack-initramfs" IMAGE_NAME_SUFFIX ?= "" IMAGE_LINGUAS = "" +IMAGE_NAME = "dstack-initramfs" LICENSE = "MIT" -# don't actually generate an image, just the artifacts needed for one IMAGE_FSTYPES = "${INITRAMFS_FSTYPES}" - inherit core-image IMAGE_ROOTFS_SIZE = "8192" IMAGE_ROOTFS_EXTRA_SPACE = "0" -# Use the same restriction as initramfs-live-install -COMPATIBLE_HOST = "x86_64.*-linux" - -# QB_KERNEL_CMDLINE_APPEND += "debugshell=3 init=/bin/busybox sh init" +# Use the same restriction as initramfs-module-install +COMPATIBLE_HOST = '(x86_64.*|i.86.*|arm.*|aarch64.*|loongarch64.*)-(linux.*|freebsd.*)' # Remove sysvinit related files in a postprocess function -ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" +ROOTFS_POSTPROCESS_COMMAND += "postprocess_initramfs;" -remove_sysvinit_files() { - # Remove /etc/init.d directory and its contents - rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d - - # Remove /etc/rc*.d directories and their contents - for d in ${IMAGE_ROOTFS}${sysconfdir}/rc*.d; do - rm -rf $d - done +postprocess_initramfs() { + install -m 0755 ${THISDIR}/files/init ${IMAGE_ROOTFS}/init - # Remove other sysvinit specific files - rm -f ${IMAGE_ROOTFS}${sysconfdir}/inittab + rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d + rm -rf ${IMAGE_ROOTFS}${systemd_system_unitdir} + rm -rf ${IMAGE_ROOTFS}${bindir}/tappd } diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb new file mode 100644 index 000000000..070d81ce5 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -0,0 +1,50 @@ +PACKAGE_INSTALL = "\ + ${VIRTUAL-RUNTIME_base-utils} \ + ${ROOTFS_BOOTSTRAP_INSTALL} \ + base-files \ + base-passwd \ + systemd \ + netbase \ + iptables \ + dropbear \ + docker \ + docker-compose \ + dstack-prebuilt \ + kernel-module-tdx-guest \ + dstack-guest \ + curl jq" + +# Do not pollute the initrd image with rootfs features +IMAGE_FEATURES = "debug-tweaks read-only-rootfs" + +IMAGE_BASENAME = "dstack-rootfs" +IMAGE_NAME_SUFFIX ?= "" +IMAGE_LINGUAS = "" + +LICENSE = "MIT" + +IMAGE_FSTYPES = "${INITRAMFS_FSTYPES} iso" + +inherit core-image + +IMAGE_ROOTFS_SIZE = "8192" +IMAGE_ROOTFS_EXTRA_SPACE = "0" + +# Use the same restriction as initramfs-live-install +COMPATIBLE_HOST = "x86_64.*-linux" + +# Remove sysvinit related files in a postprocess function +ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" + +remove_sysvinit_files() { + # Remove /etc/init.d directory and its contents + rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d + + # Remove /etc/rc*.d directories and their contents + for d in ${IMAGE_ROOTFS}${sysconfdir}/rc*.d; do + rm -rf $d + done + + # Remove other sysvinit specific files + rm -f ${IMAGE_ROOTFS}${sysconfdir}/inittab +} diff --git a/meta-dstack/recipes-core/images/files/init b/meta-dstack/recipes-core/images/files/init new file mode 100644 index 000000000..26cc54cd6 --- /dev/null +++ b/meta-dstack/recipes-core/images/files/init @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +PATH=/sbin:/bin:/usr/sbin:/usr/bin + +mkdir -p /proc +mkdir -p /sys +mount -t proc proc /proc +mount -t sysfs sysfs /sys + +mkdir -p /dev +mount -t devtmpfs devtmpfs /dev + +mkdir -p /dev/pts +mount -t devpts devpts /dev/pts + +ifconfig lo up 127.0.0.1 +ifconfig eth0 up +udhcpc -i eth0 + +exec /bin/sh From 5084e8bc940e89ffdd3447fef15e107306ed0d9f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 09:15:38 +0000 Subject: [PATCH 019/435] Setup initramfs --- build/dist.sh | 14 ++ setup => build/setup-dev | 6 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 4 +- .../dstack-guest/files/tdx-attest.conf | 1 + .../recipes-core/images/dstack-initramfs.bb | 3 +- .../recipes-core/images/dstack-rootfs.bb | 3 +- .../images/{files => initrd-files}/init | 2 + .../recipes-core/images/initrd-files/kmfs | 128 ++++++++++++++++++ 9 files changed, 154 insertions(+), 9 deletions(-) create mode 100755 build/dist.sh rename setup => build/setup-dev (89%) create mode 100644 meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf rename meta-dstack/recipes-core/images/{files => initrd-files}/init (93%) create mode 100644 meta-dstack/recipes-core/images/initrd-files/kmfs diff --git a/build/dist.sh b/build/dist.sh new file mode 100755 index 000000000..dab0b33cd --- /dev/null +++ b/build/dist.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +INITRAMFS_IMAGE=tmp/deploy/images/tdx/dstack-initramfs.cpio.gz +ROOTFS_IMAGE=tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio + +echo "Copying initramfs..." +mkdir -p dist/ +cp $INITRAMFS_IMAGE dist/ + +echo "Copying rootfs..." +mkdir -p rootfs/ +cp $ROOTFS_IMAGE rootfs/rootfs.cpio +mkisofs -o dist/rootfs.iso --max-iso9660-filenames -input-charset utf-8 rootfs/ diff --git a/setup b/build/setup-dev similarity index 89% rename from setup rename to build/setup-dev index d002b42a5..ceb9a22a1 100755 --- a/setup +++ b/build/setup-dev @@ -1,7 +1,5 @@ #!/bin/bash -pushd . - -source poky/oe-init-build-env ./build +source ../poky/oe-init-build-env . bitbake-layers add-layer ../meta-confidential-compute bitbake-layers add-layer ../meta-openembedded/meta-oe @@ -11,5 +9,3 @@ bitbake-layers add-layer ../meta-openembedded/meta-filesystems bitbake-layers add-layer ../meta-virtualization bitbake-layers add-layer ../meta-rust-bin bitbake-layers add-layer ../meta-dstack - -popd diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 7f5c759bd..fc50a0211 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -6,7 +6,7 @@ DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "1.0" DISTROOVERRIDES = "poky:dstack" -INITRAMFS_IMAGE = "dstack-initramfs" +INITRAMFS_IMAGE = "" VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 7ae6f1f1a..77a387df5 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -11,6 +11,7 @@ SRC_URI = "file://${REPO_ROOT}/dstack \ file://tappd.init \ file://tappd.service \ file://llmnr.conf \ + file://tdx-attest.conf \ " S = "${WORKDIR}/${REPO_ROOT}/dstack" @@ -41,7 +42,8 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} - install -d ${D}$/mnt/host-shared + install -d ${D}${sysconfdir}/ + install -m 0644 ${WORKDIR}/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir} \ diff --git a/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf b/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf new file mode 100644 index 000000000..d7c6361ae --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf @@ -0,0 +1 @@ +port=4050 \ No newline at end of file diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 928d3e001..ca9c2a753 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -27,7 +27,8 @@ COMPATIBLE_HOST = '(x86_64.*|i.86.*|arm.*|aarch64.*|loongarch64.*)-(linux.*|free ROOTFS_POSTPROCESS_COMMAND += "postprocess_initramfs;" postprocess_initramfs() { - install -m 0755 ${THISDIR}/files/init ${IMAGE_ROOTFS}/init + install -m 0755 ${THISDIR}/initrd-files/init ${IMAGE_ROOTFS}/init + install -m 0755 ${THISDIR}/initrd-files/kmfs ${IMAGE_ROOTFS}/bin/kmfs rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d rm -rf ${IMAGE_ROOTFS}${systemd_system_unitdir} diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb index 070d81ce5..4f68ab393 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -20,10 +20,11 @@ IMAGE_FEATURES = "debug-tweaks read-only-rootfs" IMAGE_BASENAME = "dstack-rootfs" IMAGE_NAME_SUFFIX ?= "" IMAGE_LINGUAS = "" +INITRAMFS_MAXSIZE = "1000000" LICENSE = "MIT" -IMAGE_FSTYPES = "${INITRAMFS_FSTYPES} iso" +IMAGE_FSTYPES = "cpio" inherit core-image diff --git a/meta-dstack/recipes-core/images/files/init b/meta-dstack/recipes-core/images/initrd-files/init similarity index 93% rename from meta-dstack/recipes-core/images/files/init rename to meta-dstack/recipes-core/images/initrd-files/init index 26cc54cd6..ddf1a5755 100644 --- a/meta-dstack/recipes-core/images/files/init +++ b/meta-dstack/recipes-core/images/initrd-files/init @@ -18,4 +18,6 @@ ifconfig lo up 127.0.0.1 ifconfig eth0 up udhcpc -i eth0 +modprobe tdx-guest + exec /bin/sh diff --git a/meta-dstack/recipes-core/images/initrd-files/kmfs b/meta-dstack/recipes-core/images/initrd-files/kmfs new file mode 100644 index 000000000..b3df8a305 --- /dev/null +++ b/meta-dstack/recipes-core/images/initrd-files/kmfs @@ -0,0 +1,128 @@ +#!/bin/sh +INITDATA_DIR=/initdata +KMFS_WORK_DIR=/kmfs +CFG_DIR=${KMFS_WORK_DIR}/config +TMP_CA_CERT=${CFG_DIR}/certs/tmp-ca.cert +TMP_CA_KEY=${CFG_DIR}/certs/tmp-ca.key +CA_CERT=${CFG_DIR}/certs/ca.cert +CFG_FILE=${CFG_DIR}/config.json +APP_COMPOSE_FILE=${CFG_DIR}/docker-compose.yaml +HOST_SHARED=/mnt/host-shared + +kmfs_premount() { + local key_phrase rootfs_hash app_id kms_url ca_cert_hash + local actual_rootfs_hash + + # Mount host-shared + mkdir -p ${HOST_SHARED} || panic "FSINIT: Failed to create ${HOST_SHARED} directory" + mount -t 9p -o trans=virtio,version=9p2000.L host-shared ${HOST_SHARED} || panic "FSINIT: Failed to mount host-shared" + + mkdir -p ${CFG_DIR} || panic "FSINIT: Failed to create ${CFG_DIR} directory" + cp -r ${HOST_SHARED}/* ${CFG_DIR}/ || panic "FSINIT: Failed to copy config files to ${CFG_DIR}" + + # Calculate rootfs and app ids and extend them to RTMR + if [ ! -f "${CFG_FILE}" ]; then + panic "FSINIT: Config file ${CFG_FILE} does not exist" + fi + + if [ ! -f "${APP_COMPOSE_FILE}" ]; then + panic "FSINIT: Docker compose file ${APP_COMPOSE_FILE} does not exist" + fi + + rootfs_hash=$(jq -r .rootfs_hash ${CFG_FILE}) + app_id=$(iohash sha256 -o! -t- -i ${APP_COMPOSE_FILE}) + kms_url=$(jq -r .kms_url ${CFG_FILE}) + ca_cert_hash=$(iohash sha256 -o! -t- -i ${CA_CERT}) + + echo "FSINIT: Extending rootfs hash to RTMR, hash=${rootfs_hash}" + tdxctl extend -i 3 --digest ${rootfs_hash} --associated-data "rootfs-hash" || panic "FSINIT: Failed to extend rootfs to RTMR" + echo "FSINIT: Extending app id to RTMR, app_id=${app_id}" + tdxctl extend -i 3 --digest ${app_id} --associated-data "app-id" || panic "FSINIT: Failed to extend app id to RTMR" + echo "FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=${ca_cert_hash}" + tdxctl extend -i 3 --digest ${ca_cert_hash} --associated-data "ca-cert-hash" || panic "FSINIT: Failed to extend ca cert hash to RTMR" + + # Show the RTMR + tdxctl show + + # Generate temp cert for RA-TLS + echo "FSINIT: Generating RA-TLS cert" + mkdir -p ${KMFS_WORK_DIR}/certs + tdxctl gen-ra-cert \ + -c ${KMFS_WORK_DIR}/certs/cert.pem \ + -k ${KMFS_WORK_DIR}/certs/key.pem \ + --ca-cert ${TMP_CA_CERT} \ + --ca-key ${TMP_CA_KEY} || panic "FSINIT: Failed to generate RA-TLS cert" + + # Request app keys from KMS + echo "FSINIT: Requesting app keys from KMS (${kms_url})" + curl --cacert ${CA_CERT} \ + --cert ${KMFS_WORK_DIR}/certs/cert.pem \ + --key ${KMFS_WORK_DIR}/certs/key.pem \ + -o ${KMFS_WORK_DIR}/appkeys.json \ + ${kms_url}/prpc/KMS.GetAppKey \ + || panic "FSINIT: Failed to request app keys" + + key_phrase=$(jq -r .disk_crypt_key ${KMFS_WORK_DIR}/appkeys.json) + # Check if key_phrase is empty, null, or not set + if [ -z "$key_phrase" ] || [ "$key_phrase" = "null" ]; then + panic "FSINIT: Failed to get valid key phrase from KMS" + fi + + if [ -e /config/initialized ]; then + echo "[$(date +%H:%M:%S)] FSINIT: Mounting" + echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" + mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" + else + # Set up disk encryption and format the disk + + [ -z "${INITIMG}" ] && panic "FSINIT: initimg not set" + + echo "FSINIT: Mounting init image ${INITIMG}" + mkdir -p ${INITDATA_DIR} || panic "FSINIT: Failed to create ${INITDATA_DIR}" + + if [ -z "${INITIMGTYPE}" ]; then + case "${INITIMG}" in + /dev/cdrom*|/dev/sr*) + INITIMGTYPE="-t iso9660" + ;; + esac + fi + mount ${INITIMGTYPE} -o ro ${INITIMG} ${INITDATA_DIR} || panic "FSINIT: Failed to mount init image" + + echo "FSINIT: Initializing" + echo "[$(date +%H:%M:%S)] FSINIT: Formatting rootfs" + echo "$key_phrase" | cryptsetup luksFormat --type luks2 --cipher aes-xts-plain64 ${ROOTINTEGRITY} --pbkdf pbkdf2 -d- ${ROOT} || panic "FSINIT: Failed to format rootfs" + echo "[$(date +%H:%M:%S)] FSINIT: Formatting rootfs done" + echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" + mkfs.ext4 -L cloudimg-rootfs /dev/mapper/rootfs_crypt || panic "FSINIT: Failed to create ext4 filesystem" + mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" + + echo "[$(date +%H:%M:%S)] FSINIT: Extracting rootfs, this may take a while..." + extract_rootfs() { + iohash sha256 -t /tmp/rootfs.hash | cpio -i + } + if [ -e ${INITDATA_DIR}/rootfs.cpio ]; then + (cd /root && cat ${INITDATA_DIR}/rootfs.cpio | extract_rootfs) || panic "FSINIT: Failed to extract rootfs.cpio" + else + panic "FSINIT: No rootfs found" + fi + + actual_rootfs_hash=$(cat /tmp/rootfs.hash) + if [ -z "$actual_rootfs_hash" ]; then + panic "FSINIT: Rootfs hash is empty" + fi + if [ $actual_rootfs_hash != $rootfs_hash ]; then + panic "FSINIT: Rootfs hash mismatch" + fi + + cp -rT ${CFG_DIR} /root/tapp || panic "FSINIT: Failed to copy config to /root/tapp" + touch /config/initialized || echo "FSINIT: Failed to touch initialized" + fi + + cp ${KMFS_WORK_DIR}/appkeys.json /root/tapp/appkeys.json || panic "FSINIT: Failed to copy appkeys.json to /root/tapp" + cp ${CFG_DIR}/config.json /root/tapp/config.json || panic "FSINIT: Failed to copy config.json to /root/tapp" + echo "FSINIT: Copied config to /root/tapp" + umount /root || panic "FSINIT: Failed to unmount /root" + umount /config || panic "FSINIT: Failed to unmount /config" + ROOT=/dev/mapper/rootfs_crypt +} From fcbfceff6c88760c79bcfe80092f7011df413e8c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 09:36:02 +0000 Subject: [PATCH 020/435] Rearrange dir --- build/emu | 23 ++----------------- build/run.sh | 2 ++ .../images/dstack-initramfs-files.bb | 17 ++++++++++++++ .../init | 2 +- .../kmfs | 0 .../recipes-core/images/dstack-initramfs.bb | 18 ++++++++++----- .../linux/files/dstack-docker.cfg | 0 .../linux/files/dstack-docker.scc | 0 .../linux/files/dstack-tdx.cfg | 0 .../linux/files/dstack-tdx.scc | 0 .../linux/files/dstack.cfg | 0 .../linux/files/dstack.scc | 0 .../linux/linux-yocto%.bbappend | 0 13 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 meta-dstack/recipes-core/images/dstack-initramfs-files.bb rename meta-dstack/recipes-core/images/{initrd-files => dstack-initramfs-files}/init (91%) rename meta-dstack/recipes-core/images/{initrd-files => dstack-initramfs-files}/kmfs (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack-docker.cfg (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack-docker.scc (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack-tdx.cfg (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack-tdx.scc (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack.cfg (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/files/dstack.scc (100%) rename meta-dstack/{recipes-core => recipes-kernel}/linux/linux-yocto%.bbappend (100%) diff --git a/build/emu b/build/emu index ce552549e..5ef8b63f4 100755 --- a/build/emu +++ b/build/emu @@ -1,22 +1,3 @@ #!/bin/sh -KERNEL=./tmp/deploy/images/tdx/bzImage -INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz -if [ ! -f ./mock/vda.qcow2 ]; then - qemu-img create -f qcow2 ./mock/vda.qcow2 10G -fi -qemu-system-x86_64 \ - -kernel ${KERNEL} \ - -initrd ${INITRD} \ - -accel kvm \ - -m 2G -smp 4 \ - -cpu host \ - -machine q35,kernel_irqchip=split \ - -nographic \ - -nodefaults \ - -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ - -serial stdio \ - -virtfs local,path=./mock/shared,mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0 \ - -drive file=./mock/vda.qcow2,if=virtio,format=qcow2 \ - -append "console=ttyS0 init=/init" - - +THISDIR=$(dirname "$(readlink -f "$0")") +TD=0 $THISDIR/run.sh \ No newline at end of file diff --git a/build/run.sh b/build/run.sh index 66e620f97..526ffe3ca 100755 --- a/build/run.sh +++ b/build/run.sh @@ -14,6 +14,7 @@ TD=${TD:-1} TDVF_FIRMWARE=/usr/share/ovmf/OVMF.fd RO=${RO:-off} CID=$(( ( RANDOM % 10000 ) + 3 )) +CDROM=${ROOTFS_PATH:-${PWD}/dist/rootfs.iso} ARGS="${ARGS} -kernel ${KERNEL}" ARGS="${ARGS} -initrd ${INITRD}" @@ -52,5 +53,6 @@ qemu-system-x86_64 \ -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ -drive file=${VDA},if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 \ -virtfs local,path=${CONFIG_DIR},mount_tag=host-shared,readonly=${RO},security_model=mapped,id=virtfs0 \ + -cdrom ${CDROM} \ ${ARGS} \ -append "${CMDLINE}" diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb new file mode 100644 index 000000000..9e20a29f3 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb @@ -0,0 +1,17 @@ +SUMMARY = "Dstack initramfs files" + +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" + +SRC_URI = "file://init \ + file://kmfs" + +inherit allarch + +S = "${WORKDIR}" + +do_install() { + install -d ${D}/${bindir} + install -m 0755 ${S}/init ${D}/init + install -m 0755 ${S}/kmfs ${D}${bindir}/kmfs +} diff --git a/meta-dstack/recipes-core/images/initrd-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init similarity index 91% rename from meta-dstack/recipes-core/images/initrd-files/init rename to meta-dstack/recipes-core/images/dstack-initramfs-files/init index ddf1a5755..768c21d5c 100644 --- a/meta-dstack/recipes-core/images/initrd-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -18,6 +18,6 @@ ifconfig lo up 127.0.0.1 ifconfig eth0 up udhcpc -i eth0 -modprobe tdx-guest +# modprobe tdx-guest || true exec /bin/sh diff --git a/meta-dstack/recipes-core/images/initrd-files/kmfs b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs similarity index 100% rename from meta-dstack/recipes-core/images/initrd-files/kmfs rename to meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index ca9c2a753..0e3c72d53 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -1,6 +1,15 @@ -PACKAGE_INSTALL = "${VIRTUAL-RUNTIME_base-utils} udev base-passwd ${ROOTFS_BOOTSTRAP_INSTALL} base-files" -PACKAGE_INSTALL += "kernel-module-tdx-guest" -PACKAGE_INSTALL += "dstack-guest curl jq" +PACKAGE_INSTALL = "\ + ${VIRTUAL-RUNTIME_base-utils} \ + udev \ + base-passwd \ + ${ROOTFS_BOOTSTRAP_INSTALL} \ + base-files \ + kernel-module-tdx-guest \ + dstack-initramfs-files \ + dstack-guest \ + curl \ + jq \ +" # Do not pollute the initrd image with rootfs features IMAGE_FEATURES = "" @@ -27,9 +36,6 @@ COMPATIBLE_HOST = '(x86_64.*|i.86.*|arm.*|aarch64.*|loongarch64.*)-(linux.*|free ROOTFS_POSTPROCESS_COMMAND += "postprocess_initramfs;" postprocess_initramfs() { - install -m 0755 ${THISDIR}/initrd-files/init ${IMAGE_ROOTFS}/init - install -m 0755 ${THISDIR}/initrd-files/kmfs ${IMAGE_ROOTFS}/bin/kmfs - rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d rm -rf ${IMAGE_ROOTFS}${systemd_system_unitdir} rm -rf ${IMAGE_ROOTFS}${bindir}/tappd diff --git a/meta-dstack/recipes-core/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack-docker.cfg rename to meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg diff --git a/meta-dstack/recipes-core/linux/files/dstack-docker.scc b/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack-docker.scc rename to meta-dstack/recipes-kernel/linux/files/dstack-docker.scc diff --git a/meta-dstack/recipes-core/linux/files/dstack-tdx.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack-tdx.cfg rename to meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg diff --git a/meta-dstack/recipes-core/linux/files/dstack-tdx.scc b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack-tdx.scc rename to meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc diff --git a/meta-dstack/recipes-core/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack.cfg rename to meta-dstack/recipes-kernel/linux/files/dstack.cfg diff --git a/meta-dstack/recipes-core/linux/files/dstack.scc b/meta-dstack/recipes-kernel/linux/files/dstack.scc similarity index 100% rename from meta-dstack/recipes-core/linux/files/dstack.scc rename to meta-dstack/recipes-kernel/linux/files/dstack.scc diff --git a/meta-dstack/recipes-core/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend similarity index 100% rename from meta-dstack/recipes-core/linux/linux-yocto%.bbappend rename to meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend From 57ac29194fdf9d6adfbfcf35dc79e9030f9867c5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 10:13:39 +0000 Subject: [PATCH 021/435] Refactor build dir --- .gitignore | 1 + Makefile | 17 ++++++++++++----- build/emu | 3 --- build/dist.sh => dist.sh | 0 mock/config.json | 10 ++++++++++ {build/mock => mock}/shared/certs/ca.cert | 0 {build/mock => mock}/shared/certs/tmp-ca.cert | 0 {build/mock => mock}/shared/certs/tmp-ca.key | 0 {build/mock => mock}/shared/config.json | 0 {build/mock => mock}/shared/docker-compose.yaml | 0 build/run.sh => run_td.sh | 2 -- build/setup-dev => setup-dev | 4 +++- 12 files changed, 26 insertions(+), 11 deletions(-) delete mode 100755 build/emu rename build/dist.sh => dist.sh (100%) create mode 100644 mock/config.json rename {build/mock => mock}/shared/certs/ca.cert (100%) rename {build/mock => mock}/shared/certs/tmp-ca.cert (100%) rename {build/mock => mock}/shared/certs/tmp-ca.key (100%) rename {build/mock => mock}/shared/config.json (100%) rename {build/mock => mock}/shared/docker-compose.yaml (100%) rename build/run.sh => run_td.sh (99%) rename build/setup-dev => setup-dev (89%) diff --git a/.gitignore b/.gitignore index 796b96d1c..24629c1ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /build +*.qcow2 diff --git a/Makefile b/Makefile index d3bd987dd..761090150 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,14 @@ -.PHONY: build gen-measurements +.PHONY: all dist emu clean -build: - cd srcs/poky/ && bitbake dstack-initramfs +all: -gen-measurements: - echo "Not implemented" \ No newline at end of file +dist: + ./buildall.sh + +emu: + TD=0 ./run_td.sh +run: + ./run_td.sh + +clean: + git clean -xdff \ No newline at end of file diff --git a/build/emu b/build/emu deleted file mode 100755 index 5ef8b63f4..000000000 --- a/build/emu +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -THISDIR=$(dirname "$(readlink -f "$0")") -TD=0 $THISDIR/run.sh \ No newline at end of file diff --git a/build/dist.sh b/dist.sh similarity index 100% rename from build/dist.sh rename to dist.sh diff --git a/mock/config.json b/mock/config.json new file mode 100644 index 000000000..6d78b3a32 --- /dev/null +++ b/mock/config.json @@ -0,0 +1,10 @@ +{ + "id": "a2db9cc0-0459-41cf-b59a-33b8ab9d2bd9", + "name": "example", + "address": "3327603e03f5bd1f830812ca4a789277fc31f577", + "vcpu": 1, + "memory": 1024, + "disk_size": 20, + "image": "ubuntu-24.04", + "port_map": {} +} \ No newline at end of file diff --git a/build/mock/shared/certs/ca.cert b/mock/shared/certs/ca.cert similarity index 100% rename from build/mock/shared/certs/ca.cert rename to mock/shared/certs/ca.cert diff --git a/build/mock/shared/certs/tmp-ca.cert b/mock/shared/certs/tmp-ca.cert similarity index 100% rename from build/mock/shared/certs/tmp-ca.cert rename to mock/shared/certs/tmp-ca.cert diff --git a/build/mock/shared/certs/tmp-ca.key b/mock/shared/certs/tmp-ca.key similarity index 100% rename from build/mock/shared/certs/tmp-ca.key rename to mock/shared/certs/tmp-ca.key diff --git a/build/mock/shared/config.json b/mock/shared/config.json similarity index 100% rename from build/mock/shared/config.json rename to mock/shared/config.json diff --git a/build/mock/shared/docker-compose.yaml b/mock/shared/docker-compose.yaml similarity index 100% rename from build/mock/shared/docker-compose.yaml rename to mock/shared/docker-compose.yaml diff --git a/build/run.sh b/run_td.sh similarity index 99% rename from build/run.sh rename to run_td.sh index 526ffe3ca..30ae9ca16 100755 --- a/build/run.sh +++ b/run_td.sh @@ -34,8 +34,6 @@ echo TD=${TD} echo TDX_ARGS=${TDX_ARGS} echo BIOS=${BIOS} -sleep 2 - if [ ! -f ${VDA} ]; then qemu-img create -f qcow2 ${VDA} ${VDA_SIZE} fi diff --git a/build/setup-dev b/setup-dev similarity index 89% rename from build/setup-dev rename to setup-dev index ceb9a22a1..7f6a3dd83 100755 --- a/build/setup-dev +++ b/setup-dev @@ -1,5 +1,6 @@ #!/bin/bash -source ../poky/oe-init-build-env . +pushd . +source ../poky/oe-init-build-env build bitbake-layers add-layer ../meta-confidential-compute bitbake-layers add-layer ../meta-openembedded/meta-oe @@ -9,3 +10,4 @@ bitbake-layers add-layer ../meta-openembedded/meta-filesystems bitbake-layers add-layer ../meta-virtualization bitbake-layers add-layer ../meta-rust-bin bitbake-layers add-layer ../meta-dstack +popd \ No newline at end of file From 860814d2d2cc27d68654028e13ece7acbb51fbef Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 12:52:25 +0000 Subject: [PATCH 022/435] Add ovmf and refactor build script --- Makefile | 30 +++++++++++++++++-- setup-dev => dev-setup | 2 +- dist.sh | 29 +++++++++++++----- .../images/dstack-initramfs-files.bb | 3 ++ .../images/dstack-initramfs-files/init | 27 ++++++++++++++++- .../images/dstack-initramfs-files/kmfs | 11 ++++++- meta-dstack/recipes-core/ovmf/ovmf%.bbappend | 9 ++++++ run_td.sh | 12 ++++---- 8 files changed, 104 insertions(+), 19 deletions(-) rename setup-dev => dev-setup (91%) create mode 100644 meta-dstack/recipes-core/ovmf/ovmf%.bbappend diff --git a/Makefile b/Makefile index 761090150..8cc931e86 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,33 @@ +ifeq ($(BBPATH),) +$(error BBPATH is not set. Run `source dev-setup` first) +endif + +# INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz +# ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio +# KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage +# OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd + .PHONY: all dist emu clean -all: +BUILD_DIR ?= build +BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx +DIST_DIR ?= ${BUILD_DIR}/dist + +IMAGE_FILES = dstack-initramfs.cpio.gz \ + dstack-rootfs-tdx.cpio \ + bzImage \ + ovmf.fd + +ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}, ${IMAGE_FILES}) + + +all: dist + +dist: ${ABS_IMAGE_FILES} + DIST_DIR=${DIST_DIR} BUILD_DIR=${BUILD_DIR} ./dist.sh -dist: - ./buildall.sh +${ABS_IMAGE_FILES}: + bitbake dstack-initramfs dstack-rootfs ovmf emu: TD=0 ./run_td.sh diff --git a/setup-dev b/dev-setup similarity index 91% rename from setup-dev rename to dev-setup index 7f6a3dd83..834f85a76 100755 --- a/setup-dev +++ b/dev-setup @@ -1,6 +1,6 @@ #!/bin/bash pushd . -source ../poky/oe-init-build-env build +source poky/oe-init-build-env build bitbake-layers add-layer ../meta-confidential-compute bitbake-layers add-layer ../meta-openembedded/meta-oe diff --git a/dist.sh b/dist.sh index dab0b33cd..e8b1f2f9d 100755 --- a/dist.sh +++ b/dist.sh @@ -1,14 +1,27 @@ #!/bin/sh set -e -INITRAMFS_IMAGE=tmp/deploy/images/tdx/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio +BUILD_DIR=${BUILD_DIR:-build} +DIST_DIR=${DIST_DIR:-${BUILD_DIR}/dist} + +INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz +ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio +KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage +OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd echo "Copying initramfs..." -mkdir -p dist/ -cp $INITRAMFS_IMAGE dist/ +rm -rf ${DIST_DIR}/ +mkdir -p ${DIST_DIR}/ +cp $INITRAMFS_IMAGE ${DIST_DIR}/initramfs.cpio.gz + +echo "Copying kernel..." +cp $KERNEL_IMAGE ${DIST_DIR}/ + +echo "Copying OVMF firmware..." +cp $OVMF_FIRMWARE ${DIST_DIR}/ -echo "Copying rootfs..." -mkdir -p rootfs/ -cp $ROOTFS_IMAGE rootfs/rootfs.cpio -mkisofs -o dist/rootfs.iso --max-iso9660-filenames -input-charset utf-8 rootfs/ +echo "Making rootfs.iso..." +mkdir -p ${BUILD_DIR}/rootfs/ +cp $ROOTFS_IMAGE ${BUILD_DIR}/rootfs/rootfs.cpio +cp $ROOTFS_IMAGE ${DIST_DIR}/rootfs.cpio +mkisofs -o ${DIST_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${BUILD_DIR}/rootfs/ diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb index 9e20a29f3..5230be8f9 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb @@ -6,11 +6,14 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384 SRC_URI = "file://init \ file://kmfs" +FILES:${PN} = "*" + inherit allarch S = "${WORKDIR}" do_install() { + install -d ${D}/ install -d ${D}/${bindir} install -m 0755 ${S}/init ${D}/init install -m 0755 ${S}/kmfs ${D}${bindir}/kmfs diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index 768c21d5c..a97ea23d1 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -14,10 +14,35 @@ mount -t devtmpfs devtmpfs /dev mkdir -p /dev/pts mount -t devpts devpts /dev/pts +# Parse kernel cmdline into vars +for p in `cat /proc/cmdline`; do + if [ -n "$quoted" ]; then + value="$value $p" + if [ "`echo $p | sed -e 's/\"$//'`" != "$p" ]; then + eval "bootparam_${quoted}=${value}" + unset quoted + fi + continue + fi + + opt=`echo $p | cut -d'=' -f1` + opt=`echo $opt | sed -e 'y/.-/__/'` + if [ "`echo $p | cut -d'=' -f1`" = "$p" ]; then + eval "bootparam_${opt}=true" + else + value="`echo $p | cut -d'=' -f2-`" + if [ "`echo $value | sed -e 's/^\"//'`" != "$value" ]; then + quoted=${opt} + continue + fi + eval "bootparam_${opt}=\"${value}\"" + fi +done + ifconfig lo up 127.0.0.1 ifconfig eth0 up udhcpc -i eth0 -# modprobe tdx-guest || true +modprobe tdx-guest || true exec /bin/sh diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs index b3df8a305..afe6ee75e 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs @@ -1,4 +1,6 @@ #!/bin/sh +set -e + INITDATA_DIR=/initdata KMFS_WORK_DIR=/kmfs CFG_DIR=${KMFS_WORK_DIR}/config @@ -9,7 +11,12 @@ CFG_FILE=${CFG_DIR}/config.json APP_COMPOSE_FILE=${CFG_DIR}/docker-compose.yaml HOST_SHARED=/mnt/host-shared -kmfs_premount() { +panic() { + echo "$1" + exit 1 +} + +fde_prepare() { local key_phrase rootfs_hash app_id kms_url ca_cert_hash local actual_rootfs_hash @@ -126,3 +133,5 @@ kmfs_premount() { umount /config || panic "FSINIT: Failed to unmount /config" ROOT=/dev/mapper/rootfs_crypt } + +fde_prepare \ No newline at end of file diff --git a/meta-dstack/recipes-core/ovmf/ovmf%.bbappend b/meta-dstack/recipes-core/ovmf/ovmf%.bbappend new file mode 100644 index 000000000..b75d73f41 --- /dev/null +++ b/meta-dstack/recipes-core/ovmf/ovmf%.bbappend @@ -0,0 +1,9 @@ +do_deploy:class-target:append() { + for i in \ + ovmf \ + ovmf.code \ + ovmf.vars \ + ; do + cp ${WORKDIR}/ovmf/$i.fd ${DEPLOYDIR}/ + done +} \ No newline at end of file diff --git a/run_td.sh b/run_td.sh index 30ae9ca16..266b4cf71 100755 --- a/run_td.sh +++ b/run_td.sh @@ -4,17 +4,19 @@ VMDIR=${1:-./mock} PROCESS_NAME=qemu -KERNEL=./tmp/deploy/images/tdx/bzImage -INITRD=./tmp/deploy/images/tdx/dstack-initramfs-tdx.cpio.gz +KERNEL=./build/dist/bzImage +INITRD=./build/dist/initramfs.cpio.gz +CDROM=./build/dist/rootfs.iso +OVMF_FIRMWARE=./build/dist/ovmf.fd + VDA=${VMDIR}/vda.qcow2 VDA_SIZE=10G CMDLINE="console=ttyS0 init=/init" CONFIG_DIR=${VMDIR}/shared TD=${TD:-1} -TDVF_FIRMWARE=/usr/share/ovmf/OVMF.fd +TDVF_FIRMWARE=${OVMF_FIRMWARE} RO=${RO:-off} CID=$(( ( RANDOM % 10000 ) + 3 )) -CDROM=${ROOTFS_PATH:-${PWD}/dist/rootfs.iso} ARGS="${ARGS} -kernel ${KERNEL}" ARGS="${ARGS} -initrd ${INITRD}" @@ -40,7 +42,7 @@ fi qemu-system-x86_64 \ -accel kvm \ -m 8G -smp 16 \ - -name ${PROCESS_NAME},process=${PROCESS_NAME},debug-threads=on \ + -name ${PROCESS_NAME},process=${PROCESS_NAME} \ -cpu host \ -machine q35,kernel_irqchip=split${MACHINE_ARGS} \ ${BIOS} \ From de539448f36636da1bfa8923514d94a24b43e539 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 14:11:27 +0000 Subject: [PATCH 023/435] Fix image path error in Makefile --- Makefile | 8 +------- dstack | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8cc931e86..f42fe8aa6 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,6 @@ ifeq ($(BBPATH),) $(error BBPATH is not set. Run `source dev-setup` first) endif -# INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz -# ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio -# KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage -# OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd - .PHONY: all dist emu clean BUILD_DIR ?= build @@ -18,8 +13,7 @@ IMAGE_FILES = dstack-initramfs.cpio.gz \ bzImage \ ovmf.fd -ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}, ${IMAGE_FILES}) - +ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}/, ${IMAGE_FILES}) all: dist diff --git a/dstack b/dstack index 29f184c28..f804d47a7 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 29f184c2835ce0bc90ee5497d3b5a59724d9a2d8 +Subproject commit f804d47a75abceedd60417bda63fcec5204df8a4 From af2f03c9b9d23a74d94b0a782580feeb2cb124b0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 17 Oct 2024 17:32:11 +0000 Subject: [PATCH 024/435] It works --- Makefile | 13 +++-- .../recipes-core/dstack-guest/dstack-guest.bb | 10 +++- .../dstack-guest/files/app-compose.service | 14 ++++++ .../dstack-guest/files/tboot.service | 12 +++++ .../recipes-core/dstack-guest/files/tboot.sh | 49 +++++++++++++++++++ .../images/dstack-initramfs-files.bb | 7 ++- .../images/dstack-initramfs-files/boot-vars | 38 ++++++++++++++ .../images/dstack-initramfs-files/init | 41 ++++++---------- .../{kmfs => kmfs-setup} | 20 ++------ .../recipes-core/images/dstack-initramfs.bb | 2 + .../recipes-core/images/dstack-rootfs.bb | 3 +- .../recipes-kernel/linux/files/dstack.cfg | 4 +- mock/shared/certs/ca.cert | 12 ++--- mock/shared/certs/tmp-ca.cert | 14 +++--- mock/shared/certs/tmp-ca.key | 6 +-- mock/shared/config.json | 1 - run_td.sh | 20 ++++++-- 17 files changed, 196 insertions(+), 70 deletions(-) create mode 100644 meta-dstack/recipes-core/dstack-guest/files/app-compose.service create mode 100644 meta-dstack/recipes-core/dstack-guest/files/tboot.service create mode 100755 meta-dstack/recipes-core/dstack-guest/files/tboot.sh create mode 100644 meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars rename meta-dstack/recipes-core/images/dstack-initramfs-files/{kmfs => kmfs-setup} (90%) delete mode 100644 mock/shared/config.json diff --git a/Makefile b/Makefile index f42fe8aa6..cfc569ad6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(BBPATH),) $(error BBPATH is not set. Run `source dev-setup` first) endif -.PHONY: all dist emu clean +.PHONY: all dist emu clean images BUILD_DIR ?= build BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx @@ -17,16 +17,23 @@ ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}/, ${IMAGE_FILES}) all: dist -dist: ${ABS_IMAGE_FILES} +dist: $(ABS_IMAGE_FILES) DIST_DIR=${DIST_DIR} BUILD_DIR=${BUILD_DIR} ./dist.sh -${ABS_IMAGE_FILES}: +$(ABS_IMAGE_FILES): + make images + +images: bitbake dstack-initramfs dstack-rootfs ovmf emu: TD=0 ./run_td.sh + run: ./run_td.sh +test: + make images dist run + clean: git clean -xdff \ No newline at end of file diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 77a387df5..df82b62f9 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -12,12 +12,15 @@ SRC_URI = "file://${REPO_ROOT}/dstack \ file://tappd.service \ file://llmnr.conf \ file://tdx-attest.conf \ + file://tboot.sh \ + file://tboot.service \ + file://app-compose.service \ " S = "${WORKDIR}/${REPO_ROOT}/dstack" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" -SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','tappd.service','',d)}" +SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','tappd.service tboot.service app-compose.service','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" INITSCRIPT_PACKAGES += "${@bb.utils.contains('DISTRO_FEATURES','systemd','','${PN}',d)}" @@ -41,6 +44,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} + install -m 0755 ${WORKDIR}/tboot.sh ${D}${bindir} install -d ${D}${sysconfdir}/ install -m 0644 ${WORKDIR}/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf @@ -50,9 +54,13 @@ do_install() { ${D}${sysconfdir}/systemd/resolved.conf.d install -m 0644 ${WORKDIR}/tappd.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/tboot.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/app-compose.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d else install -d ${D}${sysconfdir}/init.d install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd.init + bberror "init scripts for sysvinit is not implemented yet" fi } diff --git a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service new file mode 100644 index 000000000..3bbda3d9b --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service @@ -0,0 +1,14 @@ +[Unit] +Description=App Compose Service +Requires=docker.service +After=docker.service tboot.service + +[Service] +Type=oneshot +RemainAfterExit=true +WorkingDirectory=/tapp +ExecStart=/usr/bin/env docker compose up -d +ExecStop=/usr/bin/env docker compose down + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.service b/meta-dstack/recipes-core/dstack-guest/files/tboot.service new file mode 100644 index 000000000..bc461e205 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.service @@ -0,0 +1,12 @@ +[Unit] +Description=Guest Boot Service +After=network.target +Before=app-compose.service tappd.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/env tboot.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh new file mode 100755 index 000000000..8ae48aae8 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh @@ -0,0 +1,49 @@ +#!/bin/sh +set -e + +CLIENT_PRIVATE_KEY=$(wg genkey) +CLIENT_PUBLIC_KEY=$(echo $CLIENT_PRIVATE_KEY | wg pubkey) + +TPROXY_URL=$(jq -r '.tproxy_url' /tapp/config.json) + +mkdir -p /etc/tappd +cp /tapp/certs/ca.cert /etc/tappd/ca.cert +jq -r '.app_key' /tapp/appkeys.json > /etc/tappd/app-ca.key +jq -r '.certificate_chain[]' /tapp/appkeys.json | awk 'NF {print $0 > "/etc/tappd/app-ca.cert"}' + +tdxctl gen-ra-cert \ + --ca-key /etc/tappd/app-ca.key \ + --ca-cert /etc/tappd/app-ca.cert \ + --cert-path /etc/tappd/tls.cert \ + --key-path /etc/tappd/tls.key + +cat /etc/tappd/app-ca.cert >> /etc/tappd/tls.cert + +curl ${TPROXY_URL}/prpc/Tproxy.RegisterCvm?json \ + --cacert /etc/tappd/ca.cert \ + --cert /etc/tappd/tls.cert \ + --key /etc/tappd/tls.key \ + -d"{\"client_public_key\":\"${CLIENT_PUBLIC_KEY}\"}" \ + -o /tmp/wginfo.json + +CLIENT_IP=$(jq -r '.client_ip' /tmp/wginfo.json) +SERVER_ENDPOINT=$(jq -r '.server_endpoint' /tmp/wginfo.json) +SERVER_PUBLIC_KEY=$(jq -r '.server_public_key' /tmp/wginfo.json) +SERVER_IP=$(jq -r '.server_ip' /tmp/wginfo.json) + +rm -f /tmp/wginfo.json + +mkdir -p /etc/wireguard +cat < /etc/wireguard/wg0.conf +[Interface] +PrivateKey = ${CLIENT_PRIVATE_KEY} +Address = ${CLIENT_IP}/24 + +[Peer] +PublicKey = ${SERVER_PUBLIC_KEY} +AllowedIPs = ${SERVER_IP}/24 +Endpoint = ${SERVER_ENDPOINT} +PersistentKeepalive = 25 +EOF + +wg-quick up wg0 \ No newline at end of file diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb index 5230be8f9..d335b8914 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files.bb @@ -4,7 +4,8 @@ LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" SRC_URI = "file://init \ - file://kmfs" + file://kmfs-setup \ + file://boot-vars" FILES:${PN} = "*" @@ -14,7 +15,9 @@ S = "${WORKDIR}" do_install() { install -d ${D}/ + install -d ${D}/scripts install -d ${D}/${bindir} install -m 0755 ${S}/init ${D}/init - install -m 0755 ${S}/kmfs ${D}${bindir}/kmfs + install -m 0755 ${S}/kmfs-setup ${D}/scripts/kmfs-setup + install -m 0755 ${S}/boot-vars ${D}/scripts/boot-vars } diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars new file mode 100644 index 000000000..30d33ded4 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars @@ -0,0 +1,38 @@ + +ROOT=/dev/vda +ROOTINTEGRITY= +INITIMG=/dev/sr0 +INITDATA_DIR=/initdata +KMFS_WORK_DIR=/kmfs +CFG_DIR=${KMFS_WORK_DIR}/config +TMP_CA_CERT=${CFG_DIR}/certs/tmp-ca.cert +TMP_CA_KEY=${CFG_DIR}/certs/tmp-ca.key +CA_CERT=${CFG_DIR}/certs/ca.cert +CFG_FILE=${CFG_DIR}/config.json +APP_COMPOSE_FILE=${CFG_DIR}/docker-compose.yaml +HOST_SHARED=/mnt/host-shared + +# Parse kernel cmdline into vars +for p in `cat /proc/cmdline`; do + if [ -n "$quoted" ]; then + value="$value $p" + if [ "`echo $p | sed -e 's/\"$//'`" != "$p" ]; then + eval "kargs_${quoted}=${value}" + unset quoted + fi + continue + fi + + opt=`echo $p | cut -d'=' -f1` + opt=`echo $opt | sed -e 'y/.-/__/'` + if [ "`echo $p | cut -d'=' -f1`" = "$p" ]; then + eval "kargs_${opt}=true" + else + value="`echo $p | cut -d'=' -f2-`" + if [ "`echo $value | sed -e 's/^\"//'`" != "$value" ]; then + quoted=${opt} + continue + fi + eval "kargs_${opt}=\"${value}\"" + fi +done diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index a97ea23d1..77fd626bb 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -5,8 +5,10 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin mkdir -p /proc mkdir -p /sys +mkdir -p /run mount -t proc proc /proc mount -t sysfs sysfs /sys +mount -t tmpfs tmpfs /run mkdir -p /dev mount -t devtmpfs devtmpfs /dev @@ -14,35 +16,22 @@ mount -t devtmpfs devtmpfs /dev mkdir -p /dev/pts mount -t devpts devpts /dev/pts -# Parse kernel cmdline into vars -for p in `cat /proc/cmdline`; do - if [ -n "$quoted" ]; then - value="$value $p" - if [ "`echo $p | sed -e 's/\"$//'`" != "$p" ]; then - eval "bootparam_${quoted}=${value}" - unset quoted - fi - continue - fi - - opt=`echo $p | cut -d'=' -f1` - opt=`echo $opt | sed -e 'y/.-/__/'` - if [ "`echo $p | cut -d'=' -f1`" = "$p" ]; then - eval "bootparam_${opt}=true" - else - value="`echo $p | cut -d'=' -f2-`" - if [ "`echo $value | sed -e 's/^\"//'`" != "$value" ]; then - quoted=${opt} - continue - fi - eval "bootparam_${opt}=\"${value}\"" - fi -done - ifconfig lo up 127.0.0.1 ifconfig eth0 up udhcpc -i eth0 modprobe tdx-guest || true -exec /bin/sh +source /scripts/boot-vars +source /scripts/kmfs-setup + +# mount move dirs +mkdir -p /root${HOST_SHARED} +mount --move ${HOST_SHARED} /root${HOST_SHARED} +mount --move /sys /root/sys +mount --move /proc /root/proc +mount --move /dev /root/dev +mount --move /run /root/run + +echo "Switching to new root" +exec switch_root /root /init diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup similarity index 90% rename from meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs rename to meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index afe6ee75e..b699e971c 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -1,16 +1,6 @@ #!/bin/sh set -e -INITDATA_DIR=/initdata -KMFS_WORK_DIR=/kmfs -CFG_DIR=${KMFS_WORK_DIR}/config -TMP_CA_CERT=${CFG_DIR}/certs/tmp-ca.cert -TMP_CA_KEY=${CFG_DIR}/certs/tmp-ca.key -CA_CERT=${CFG_DIR}/certs/ca.cert -CFG_FILE=${CFG_DIR}/config.json -APP_COMPOSE_FILE=${CFG_DIR}/docker-compose.yaml -HOST_SHARED=/mnt/host-shared - panic() { echo "$1" exit 1 @@ -75,7 +65,7 @@ fde_prepare() { panic "FSINIT: Failed to get valid key phrase from KMS" fi - if [ -e /config/initialized ]; then + if [ -e ${HOST_SHARED}/initialized ]; then echo "[$(date +%H:%M:%S)] FSINIT: Mounting" echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" @@ -119,19 +109,19 @@ fde_prepare() { panic "FSINIT: Rootfs hash is empty" fi if [ $actual_rootfs_hash != $rootfs_hash ]; then + echo "FSINIT: Rootfs hash mismatch" + echo "Expected: $rootfs_hash" + echo "Actual: $actual_rootfs_hash" panic "FSINIT: Rootfs hash mismatch" fi cp -rT ${CFG_DIR} /root/tapp || panic "FSINIT: Failed to copy config to /root/tapp" - touch /config/initialized || echo "FSINIT: Failed to touch initialized" + touch ${HOST_SHARED}/initialized || echo "FSINIT: Failed to touch initialized" fi cp ${KMFS_WORK_DIR}/appkeys.json /root/tapp/appkeys.json || panic "FSINIT: Failed to copy appkeys.json to /root/tapp" cp ${CFG_DIR}/config.json /root/tapp/config.json || panic "FSINIT: Failed to copy config.json to /root/tapp" echo "FSINIT: Copied config to /root/tapp" - umount /root || panic "FSINIT: Failed to unmount /root" - umount /config || panic "FSINIT: Failed to unmount /config" - ROOT=/dev/mapper/rootfs_crypt } fde_prepare \ No newline at end of file diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 0e3c72d53..97fbb94aa 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -4,6 +4,8 @@ PACKAGE_INSTALL = "\ base-passwd \ ${ROOTFS_BOOTSTRAP_INSTALL} \ base-files \ + cryptsetup \ + e2fsprogs-mke2fs \ kernel-module-tdx-guest \ dstack-initramfs-files \ dstack-guest \ diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb index 4f68ab393..d34cf6744 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -12,10 +12,11 @@ PACKAGE_INSTALL = "\ dstack-prebuilt \ kernel-module-tdx-guest \ dstack-guest \ + wireguard-tools \ curl jq" # Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "debug-tweaks read-only-rootfs" +IMAGE_FEATURES = "debug-tweaks" IMAGE_BASENAME = "dstack-rootfs" IMAGE_NAME_SUFFIX ?= "" diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index fc39d9150..9277e229c 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -5,4 +5,6 @@ CONFIG_NET_9P_VIRTIO=y CONFIG_9P_FS=y CONFIG_9P_FS_POSIX_ACL=y CONFIG_PCI=y -CONFIG_VIRTIO_PCI=y \ No newline at end of file +CONFIG_VIRTIO_PCI=y +CONFIG_ISO9660_FS=y +CONFIG_WIREGUARD=y \ No newline at end of file diff --git a/mock/shared/certs/ca.cert b/mock/shared/certs/ca.cert index 0a6823669..452fe5c69 100644 --- a/mock/shared/certs/ca.cert +++ b/mock/shared/certs/ca.cert @@ -1,11 +1,11 @@ -----BEGIN CERTIFICATE----- -MIIBmDCCAT2gAwIBAgIUIH3aWOG0Cg7iYeAlvMFvt7oR/wYwCgYIKoZIzj0EAwIw +MIIBmDCCAT2gAwIBAgIURtC9Iqqgh4/vxPWkd0UwPtbA5pcwCgYIKoZIzj0EAwIw LzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEVMBMGA1UEAwwMUGhhbGEgS01TIENB MCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAvMRYwFAYDVQQKDA1Q aGFsYSBOZXR3b3JrMRUwEwYDVQQDDAxQaGFsYSBLTVMgQ0EwWTATBgcqhkjOPQIB -BggqhkjOPQMBBwNCAARAGfFiokX4htvaKc2nFntqzxNZLJduL7TgXrOKoZTAJkm7 -c5en/oGwBfSpqVVBjqeIBrbDHmyIBHXiewfMjsHBozUwMzAdBgNVHQ4EFgQURzKe -YZHAfYd6o4vAd//GfWIZ874wEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD -AgNJADBGAiEA9zuPQ7Ht9yobac8CSxJwN2hq9LL1CyEpt7YOpTOVe54CIQCITzn4 -9NX9hBMbmOJDcwa2PwIwub+SckwaIMbcmkCg9A== +BggqhkjOPQMBBwNCAATXeN0OTqZFOzdDU8uq8eOXxYdxNoruXFiCe6fOSyQha9+S +zllhdJsAwEDKuvb+UcC6S2+Rq3pFzokoa+cC8f6cozUwMzAdBgNVHQ4EFgQUpDgt +ravZr8ybVH2mz9gIj+IHo+owEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD +AgNJADBGAiEA54mK5FOI5z8REY+fmN4Leo1imksQ6y5D9ekFaacOQkYCIQCKqGjb +rXgUSNTW/H6CJ24TA/OOVeOlLTqQ90cgPCrjvg== -----END CERTIFICATE----- diff --git a/mock/shared/certs/tmp-ca.cert b/mock/shared/certs/tmp-ca.cert index 5180be5fa..c98804d7f 100644 --- a/mock/shared/certs/tmp-ca.cert +++ b/mock/shared/certs/tmp-ca.cert @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- -MIIBrzCCAVWgAwIBAgIUU8HFA9rw6T0Dl/tlCJ6QRmuTAfowCgYIKoZIzj0EAwIw +MIIBrzCCAVWgAwIBAgIUDOcf3KNDCNLeosY9ssZzMLdF5aEwCgYIKoZIzj0EAwIw OzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEhMB8GA1UEAwwYUGhhbGEgS01TIENs aWVudCBUZW1wIENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjA7 MRYwFAYDVQQKDA1QaGFsYSBOZXR3b3JrMSEwHwYDVQQDDBhQaGFsYSBLTVMgQ2xp -ZW50IFRlbXAgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATs4YKouJPDMeog -W59VRRgzO/kh8W6av18kLe9dxMqoy6R6Lmy0jm8kmLpUugZsKlF372fJX3F92HsQ -fI9JDssAozUwMzAdBgNVHQ4EFgQUlnyUeOVQWZFIPZdGnf0Ix94DxhMwEgYDVR0T -AQH/BAgwBgEB/wIBATAKBggqhkjOPQQDAgNIADBFAiBYF34VIHuevsuprscxJMGh -TbrbQuH5EQ69SSjn+Wpi3AIhALzw2Df/PBvLe+3+f2qhtaIogoKbRnvCmGAtPiIb -I5vR +ZW50IFRlbXAgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARaNL6rTtrd6ue3 +w+qtR8Mbd8wS3UptSVyq9to5z074qBwQUyKbFbTXhlkhqJzWyqryqqde5u4cz+YQ +P8jE07vPozUwMzAdBgNVHQ4EFgQUss/28mUr2NdAVVcO+LnwZKVaqqEwEgYDVR0T +AQH/BAgwBgEB/wIBATAKBggqhkjOPQQDAgNIADBFAiEA8aSPOwQizznrAC6jEtHU +R8l/UT6NbQ7i+rVcL6qvme4CIBpheJAb+k0sfbmLH0w3z+gnG4FlTlpgwpcRFqI7 +oFED -----END CERTIFICATE----- diff --git a/mock/shared/certs/tmp-ca.key b/mock/shared/certs/tmp-ca.key index 7cfe3c73d..307455997 100644 --- a/mock/shared/certs/tmp-ca.key +++ b/mock/shared/certs/tmp-ca.key @@ -1,5 +1,5 @@ -----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgv2G1wga5HtcPGFKK -YqjMsWIKAHT69J0Rwbd/gtO/5lWhRANCAATs4YKouJPDMeogW59VRRgzO/kh8W6a -v18kLe9dxMqoy6R6Lmy0jm8kmLpUugZsKlF372fJX3F92HsQfI9JDssA +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgeDU11cytASj7Lqks +ASeAdcvf9XjgPWrtJ5ZSR2/WnHKhRANCAARaNL6rTtrd6ue3w+qtR8Mbd8wS3Upt +SVyq9to5z074qBwQUyKbFbTXhlkhqJzWyqryqqde5u4cz+YQP8jE07vP -----END PRIVATE KEY----- diff --git a/mock/shared/config.json b/mock/shared/config.json deleted file mode 100644 index 5704dfee2..000000000 --- a/mock/shared/config.json +++ /dev/null @@ -1 +0,0 @@ -{"rootfs_hash":"980e1b690743da60a33705d8f80425a193a37203b36c19792a5b4d94ddd0514d","kms_url":"https://kms.1022.kvin.wang:9043","tproxy_url":"https://tproxy.1022.kvin.wang:9010"} \ No newline at end of file diff --git a/run_td.sh b/run_td.sh index 266b4cf71..040f82d1f 100755 --- a/run_td.sh +++ b/run_td.sh @@ -4,10 +4,12 @@ VMDIR=${1:-./mock} PROCESS_NAME=qemu -KERNEL=./build/dist/bzImage -INITRD=./build/dist/initramfs.cpio.gz -CDROM=./build/dist/rootfs.iso -OVMF_FIRMWARE=./build/dist/ovmf.fd +DIST_DIR=./build/dist +KERNEL=${DIST_DIR}/bzImage +INITRD=${DIST_DIR}/initramfs.cpio.gz +CDROM=${DIST_DIR}/rootfs.iso +OVMF_FIRMWARE=${DIST_DIR}/ovmf.fd +ROOTFS_IMG=${DIST_DIR}/rootfs.cpio VDA=${VMDIR}/vda.qcow2 VDA_SIZE=10G @@ -36,6 +38,16 @@ echo TD=${TD} echo TDX_ARGS=${TDX_ARGS} echo BIOS=${BIOS} +ROOTFS_HASH=$(sha256sum "${DIST_DIR}/rootfs.cpio" | awk '{print $1}') +echo ROOTFS_HASH=${ROOTFS_HASH} +cat < ${VMDIR}/shared/config.json +{ + "rootfs_hash": "${ROOTFS_HASH}", + "kms_url": "https://kms.1022.kvin.wang:9043", + "tproxy_url": "https://tproxy.1022.kvin.wang:9010" +} +EOF + if [ ! -f ${VDA} ]; then qemu-img create -f qcow2 ${VDA} ${VDA_SIZE} fi From 45a371f45186e470684605ebfda00d50f69ec163 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 18 Oct 2024 02:03:53 +0000 Subject: [PATCH 025/435] Add README --- LICENSE | 21 +++++++++++++++++++++ README.md | 22 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..7eb781026 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Phala Network + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..c82ee23ee --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Yocto support for DStack Guest + +This project implements Yocto layer and the overall build scripts for DStack Base OS image. + +## Build + +```bash +git clone https://github.com/Phala-Network/meta-dstack.git --recursive +cd meta-dstack +source dev-setup + +mkdir dstack/build +cd dstack/build + +../build.sh +# Edit the config, and build again +../build.sh +``` + +## License + +This project is licensed under the MIT License. See the LICENSE file for more details. From 04fa612c6ff0360b11982812ab2200b694353f03 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 25 Oct 2024 07:09:53 +0000 Subject: [PATCH 026/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index f804d47a7..3c76a4fce 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f804d47a75abceedd60417bda63fcec5204df8a4 +Subproject commit 3c76a4fce183058808dd1646b6299909291f4155 From 64d29b492f01072066f09b00715a507bd0ae581e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 26 Oct 2024 10:42:39 +0000 Subject: [PATCH 027/435] Update dstack source --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 3c76a4fce..34f4fd046 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 3c76a4fce183058808dd1646b6299909291f4155 +Subproject commit 34f4fd0465bf0a20f79aab84a6f93511dce14fa4 From a5a8f669995009d4431ec6a73bb600a4b2d784b1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 27 Oct 2024 00:39:41 +0000 Subject: [PATCH 028/435] Update dstack source --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 34f4fd046..efa55edc3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 34f4fd0465bf0a20f79aab84a6f93511dce14fa4 +Subproject commit efa55edc3bfbf06aaabf378ac617daca809df8d9 From f7b7c267ec4512667302e565bdd4867282982948 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 27 Oct 2024 07:31:19 +0000 Subject: [PATCH 029/435] Seperate dev prod rootfs images --- Makefile | 3 ++- dist.sh | 5 +++++ dstack | 2 +- meta-dstack/recipes-core/images/dstack-rootfs-dev.bb | 5 +++++ meta-dstack/recipes-core/images/dstack-rootfs.bb | 3 +-- 5 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs-dev.bb diff --git a/Makefile b/Makefile index cfc569ad6..18e1a85db 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ DIST_DIR ?= ${BUILD_DIR}/dist IMAGE_FILES = dstack-initramfs.cpio.gz \ dstack-rootfs-tdx.cpio \ + dstack-rootfs-dev-tdx.cpio \ bzImage \ ovmf.fd @@ -24,7 +25,7 @@ $(ABS_IMAGE_FILES): make images images: - bitbake dstack-initramfs dstack-rootfs ovmf + bitbake dstack-initramfs dstack-rootfs dstack-rootfs-dev ovmf emu: TD=0 ./run_td.sh diff --git a/dist.sh b/dist.sh index e8b1f2f9d..ea558dc9e 100755 --- a/dist.sh +++ b/dist.sh @@ -6,6 +6,7 @@ DIST_DIR=${DIST_DIR:-${BUILD_DIR}/dist} INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio +ROOTFS_IMAGE_DEV=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-dev-tdx.cpio KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd @@ -22,6 +23,10 @@ cp $OVMF_FIRMWARE ${DIST_DIR}/ echo "Making rootfs.iso..." mkdir -p ${BUILD_DIR}/rootfs/ +mkdir -p ${BUILD_DIR}/rootfs-dev/ cp $ROOTFS_IMAGE ${BUILD_DIR}/rootfs/rootfs.cpio cp $ROOTFS_IMAGE ${DIST_DIR}/rootfs.cpio +cp $ROOTFS_IMAGE_DEV ${BUILD_DIR}/rootfs-dev/rootfs.cpio +cp $ROOTFS_IMAGE_DEV ${DIST_DIR}/rootfs-dev.cpio mkisofs -o ${DIST_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${BUILD_DIR}/rootfs/ +mkisofs -o ${DIST_DIR}/rootfs-dev.iso --max-iso9660-filenames -input-charset utf-8 ${BUILD_DIR}/rootfs-dev/ diff --git a/dstack b/dstack index efa55edc3..125cbae8c 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit efa55edc3bfbf06aaabf378ac617daca809df8d9 +Subproject commit 125cbae8c19ec68e6457cd9d59596673a1aa4b69 diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb b/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb new file mode 100644 index 000000000..00cc6ee8e --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb @@ -0,0 +1,5 @@ +require ${THISDIR}/dstack-rootfs.bb + +PACKAGE_INSTALL += "dropbear" +IMAGE_FEATURES = "debug-tweaks" +IMAGE_BASENAME = "dstack-rootfs-dev" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb index d34cf6744..a1c8b6c9f 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -6,7 +6,6 @@ PACKAGE_INSTALL = "\ systemd \ netbase \ iptables \ - dropbear \ docker \ docker-compose \ dstack-prebuilt \ @@ -16,7 +15,7 @@ PACKAGE_INSTALL = "\ curl jq" # Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "debug-tweaks" +IMAGE_FEATURES = "" IMAGE_BASENAME = "dstack-rootfs" IMAGE_NAME_SUFFIX ?= "" From 152d7cece8944835fa7a994f800d64e20b60a0fe Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 27 Oct 2024 07:59:17 +0000 Subject: [PATCH 030/435] Redirect app-compose log to CVM console --- dstack | 2 +- .../recipes-core/dstack-guest/files/app-compose.service | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 125cbae8c..4096069cf 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 125cbae8c19ec68e6457cd9d59596673a1aa4b69 +Subproject commit 4096069cf2be4d066b92aec8c607052b687f885c diff --git a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service index 3bbda3d9b..f16af5511 100644 --- a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service +++ b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service @@ -7,8 +7,9 @@ After=docker.service tboot.service Type=oneshot RemainAfterExit=true WorkingDirectory=/tapp -ExecStart=/usr/bin/env docker compose up -d +ExecStart=/usr/bin/env docker compose up ExecStop=/usr/bin/env docker compose down +StandardOutput=journal+console [Install] WantedBy=multi-user.target From e98746d3bd8ac8ba34336df4da9889892f69267f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 27 Oct 2024 08:02:11 +0000 Subject: [PATCH 031/435] Add make clean-dstack --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 18e1a85db..78307223d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(BBPATH),) $(error BBPATH is not set. Run `source dev-setup` first) endif -.PHONY: all dist emu clean images +.PHONY: all dist emu clean clean-dstack images BUILD_DIR ?= build BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx @@ -37,4 +37,7 @@ test: make images dist run clean: - git clean -xdff \ No newline at end of file + git clean -xdff + +clean-dstack: + bitbake -c cleansstate dstack-guest dstack-rootfs dstack-rootfs-dev From 405748634279f3cfc3290a19cefc1a6f3c9bb536 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 28 Oct 2024 17:06:03 +0000 Subject: [PATCH 032/435] Update dstack --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/files/tboot.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dstack b/dstack index 4096069cf..e5e74d0d8 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 4096069cf2be4d066b92aec8c607052b687f885c +Subproject commit e5e74d0d8068aaddbd932e13df83b1d9962e39cc diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh index 8ae48aae8..6f4050d0e 100755 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh @@ -26,10 +26,10 @@ curl ${TPROXY_URL}/prpc/Tproxy.RegisterCvm?json \ -d"{\"client_public_key\":\"${CLIENT_PUBLIC_KEY}\"}" \ -o /tmp/wginfo.json -CLIENT_IP=$(jq -r '.client_ip' /tmp/wginfo.json) -SERVER_ENDPOINT=$(jq -r '.server_endpoint' /tmp/wginfo.json) -SERVER_PUBLIC_KEY=$(jq -r '.server_public_key' /tmp/wginfo.json) -SERVER_IP=$(jq -r '.server_ip' /tmp/wginfo.json) +CLIENT_IP=$(jq -r '.wg.client_ip' /tmp/wginfo.json) +SERVER_ENDPOINT=$(jq -r '.wg.server_endpoint' /tmp/wginfo.json) +SERVER_PUBLIC_KEY=$(jq -r '.wg.server_public_key' /tmp/wginfo.json) +SERVER_IP=$(jq -r '.wg.server_ip' /tmp/wginfo.json) rm -f /tmp/wginfo.json From 9c076f01923f430e7293353018adb791753fa716 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 29 Oct 2024 10:13:54 +0000 Subject: [PATCH 033/435] Update dstack source --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index e5e74d0d8..9a4736058 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit e5e74d0d8068aaddbd932e13df83b1d9962e39cc +Subproject commit 9a4736058036efe4ed3bd4991fb15179b5258f6a From 30e59bbb2b6887a90218833b24f2392835763c18 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 30 Oct 2024 02:49:51 +0000 Subject: [PATCH 034/435] Add some logs to tboot.sh --- .../recipes-core/dstack-guest/files/app-compose.service | 1 + meta-dstack/recipes-core/dstack-guest/files/tboot.service | 2 ++ meta-dstack/recipes-core/dstack-guest/files/tboot.sh | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service index f16af5511..2051f3138 100644 --- a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service +++ b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service @@ -10,6 +10,7 @@ WorkingDirectory=/tapp ExecStart=/usr/bin/env docker compose up ExecStop=/usr/bin/env docker compose down StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.service b/meta-dstack/recipes-core/dstack-guest/files/tboot.service index bc461e205..f46d2f5e5 100644 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.service +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.service @@ -7,6 +7,8 @@ Before=app-compose.service tappd.service Type=oneshot ExecStart=/usr/bin/env tboot.sh RemainAfterExit=yes +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh index 6f4050d0e..4e237995f 100755 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh @@ -31,6 +31,11 @@ SERVER_ENDPOINT=$(jq -r '.wg.server_endpoint' /tmp/wginfo.json) SERVER_PUBLIC_KEY=$(jq -r '.wg.server_public_key' /tmp/wginfo.json) SERVER_IP=$(jq -r '.wg.server_ip' /tmp/wginfo.json) +echo "WG CLIENT_IP: ${CLIENT_IP}" +echo "WG SERVER_ENDPOINT: ${SERVER_ENDPOINT}" +echo "WG SERVER_PUBLIC_KEY: ${SERVER_PUBLIC_KEY}" +echo "WG SERVER_IP: ${SERVER_IP}" + rm -f /tmp/wginfo.json mkdir -p /etc/wireguard From 01eb9b42f9b984bd65eb176342919c142854412e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 30 Oct 2024 07:50:04 +0000 Subject: [PATCH 035/435] Add script calc_rtmr3.py --- scripts/calc_rtmr3.py | 61 +++++++++++++++++++++++++++++ scripts/samples/docker-compose.yaml | 9 +++++ scripts/samples/kms-ca.cert | 11 ++++++ 3 files changed, 81 insertions(+) create mode 100644 scripts/calc_rtmr3.py create mode 100644 scripts/samples/docker-compose.yaml create mode 100644 scripts/samples/kms-ca.cert diff --git a/scripts/calc_rtmr3.py b/scripts/calc_rtmr3.py new file mode 100644 index 000000000..f5d5d5f9a --- /dev/null +++ b/scripts/calc_rtmr3.py @@ -0,0 +1,61 @@ +""" +This script calculates the RTMR3 hash from the given docker-compose.yml file and KMS CA certificate. + +Log from a CVM: +``` +FSINIT: Extending rootfs hash to RTMR, hash=bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd21 +Extended RTMR 3: bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd2100000000000000000000000000000000 +FSINIT: Extending app id to RTMR, app_id=3327603e03f5bd1f830812ca4a789277fc31f577573ed149f47e0e2f3558e99e +Extended RTMR 3: 3327603e03f5bd1f830812ca4a789277fc31f577573ed149f47e0e2f3558e99e00000000000000000000000000000000 +FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=663a81d65c1c749fd4ae4634fa0452553396eb9e956c90b903d47d45ae10d719 +Extended RTMR 3: 663a81d65c1c749fd4ae4634fa0452553396eb9e956c90b903d47d45ae10d71900000000000000000000000000000000 +ParsedReport { + attributes: 0000001000000000, + xfam: e702060000000000, + mrtd: 7ba9e262ce6979087e34632603f354dd8f8a870f5947d116af8114db6c9d0d74c48bec4280e5b4f4a37025a10905bb29, + mrconfigid: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, + mrowner: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, + mrownerconfig: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, + rtmr0: 698a1e5764ff07840695fb46c809949cca352e6c9d26fc37dce872402adc071b3b069b0b217c1dcda68cf914253b6842, + rtmr1: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, + rtmr2: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, + rtmr3: 3c30787034cd9aabff0347bc8f08b9f24a0f6ae914bbca0f9aba681e857aa57a7a7cc5b0b67231779cdc345f107707c5, + servtd_hash: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, +} +``` +""" + +import hashlib + +INIT_MR= "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + +def rtmr_replay(history: list[str]): + """ + Replay the RTMR history to calculate the final RTMR value. + """ + if len(history) == 0: + return INIT_MR + mr = bytes.fromhex(INIT_MR) + for content in history: + # mr = sha384(concat(mr, content)) + # if content is shorter than 48 bytes, pad it with zeros + content = bytes.fromhex(content) + if len(content) < 48: + content = content.ljust(48, b'\0') + mr = hashlib.sha384(mr + content).digest() + return mr.hex() + + +def calc_rtmr3(rootfs_hash: str, app_id: str, ca_cert_hash: str): + """ + Calculate the RTMR3 hash from the given rootfs hash, app id and CA certificate hash. + """ + return rtmr_replay([rootfs_hash, app_id, ca_cert_hash]) + + +if __name__ == "__main__": + rootfs_hash = "bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd21" + app_id = hashlib.sha256(open("samples/docker-compose.yaml", "rb").read()).hexdigest() + ca_cert_hash = hashlib.sha256(open("samples/kms-ca.cert", "rb").read()).hexdigest() + rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) + print(rtmr3) diff --git a/scripts/samples/docker-compose.yaml b/scripts/samples/docker-compose.yaml new file mode 100644 index 000000000..2798ce386 --- /dev/null +++ b/scripts/samples/docker-compose.yaml @@ -0,0 +1,9 @@ +version: '3' + +services: + nginx: + image: nginx:latest + ports: + - "8080:80" + restart: always + diff --git a/scripts/samples/kms-ca.cert b/scripts/samples/kms-ca.cert new file mode 100644 index 000000000..452fe5c69 --- /dev/null +++ b/scripts/samples/kms-ca.cert @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBmDCCAT2gAwIBAgIURtC9Iqqgh4/vxPWkd0UwPtbA5pcwCgYIKoZIzj0EAwIw +LzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEVMBMGA1UEAwwMUGhhbGEgS01TIENB +MCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAvMRYwFAYDVQQKDA1Q +aGFsYSBOZXR3b3JrMRUwEwYDVQQDDAxQaGFsYSBLTVMgQ0EwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAATXeN0OTqZFOzdDU8uq8eOXxYdxNoruXFiCe6fOSyQha9+S +zllhdJsAwEDKuvb+UcC6S2+Rq3pFzokoa+cC8f6cozUwMzAdBgNVHQ4EFgQUpDgt +ravZr8ybVH2mz9gIj+IHo+owEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD +AgNJADBGAiEA54mK5FOI5z8REY+fmN4Leo1imksQ6y5D9ekFaacOQkYCIQCKqGjb +rXgUSNTW/H6CJ24TA/OOVeOlLTqQ90cgPCrjvg== +-----END CERTIFICATE----- From 34830c09b22c3cb467c052952474661efe3a5ac0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 30 Oct 2024 08:28:10 +0000 Subject: [PATCH 036/435] Update calc_rtmr3.py --- scripts/calc_rtmr3.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/scripts/calc_rtmr3.py b/scripts/calc_rtmr3.py index f5d5d5f9a..0f7a5bfa9 100644 --- a/scripts/calc_rtmr3.py +++ b/scripts/calc_rtmr3.py @@ -1,6 +1,10 @@ """ This script calculates the RTMR3 hash from the given docker-compose.yml file and KMS CA certificate. +Usage: + python calc_rtmr3.py by-file --compose --ca-cert + python calc_rtmr3.py by-vm --images-dir --vm-dir + Log from a CVM: ``` FSINIT: Extending rootfs hash to RTMR, hash=bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd21 @@ -26,6 +30,10 @@ """ import hashlib +import argparse +import json +from pathlib import Path + INIT_MR= "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" @@ -54,8 +62,29 @@ def calc_rtmr3(rootfs_hash: str, app_id: str, ca_cert_hash: str): if __name__ == "__main__": - rootfs_hash = "bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd21" - app_id = hashlib.sha256(open("samples/docker-compose.yaml", "rb").read()).hexdigest() - ca_cert_hash = hashlib.sha256(open("samples/kms-ca.cert", "rb").read()).hexdigest() - rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) - print(rtmr3) + parser = argparse.ArgumentParser(description="Calculate the RTMR3 hash from the given docker-compose.yml file and KMS CA certificate.") + parser.add_argument("mode", choices=["by-file", "by-vm"]) + parser.add_argument("--rootfs-cpio", help="The rootfs.cpio file to use.") + parser.add_argument("--compose", help="The docker-compose.yml file to use.") + parser.add_argument("--ca-cert", help="The KMS CA certificate to use.") + parser.add_argument("--images-dir", type=Path, help="The directory containing the VM images to use.") + parser.add_argument("--vm-dir", type=Path, help="The directory of a deployed VM.") + args = parser.parse_args() + + if args.mode == "by-file": + rootfs_hash = hashlib.sha256(open(args.rootfs_cpio, "rb").read()).hexdigest() + app_id = hashlib.sha256(open(args.compose, "rb").read()).hexdigest() + ca_cert_hash = hashlib.sha256(open(args.ca_cert, "rb").read()).hexdigest() + rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) + print(rtmr3) + elif args.mode == "by-vm": + vm_config = json.load(open(args.vm_dir / "config.json", "r")) + image_dir = args.images_dir / vm_config["image"] + image_metadata = json.load(open(image_dir / "metadata.json", "r")) + rootfs_hash = image_metadata["rootfs_hash"] + compose_file = args.vm_dir / "shared" / "docker-compose.yaml" + ca_cert_file = args.vm_dir / "shared" / "certs" / "ca.cert" + app_id = hashlib.sha256(open(compose_file, "rb").read()).hexdigest() + ca_cert_hash = hashlib.sha256(open(ca_cert_file, "rb").read()).hexdigest() + rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) + print(rtmr3) From d92649dcdf7b22e240d95892beb81c475bc0102a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 30 Oct 2024 13:11:22 +0000 Subject: [PATCH 037/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 9a4736058..191349f48 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 9a4736058036efe4ed3bd4991fb15179b5258f6a +Subproject commit 191349f488f955077a7b73ca7b4b14eb651dbdf7 From b71b95e1f1f2624e332f433096f9c46ec56f661d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 8 Nov 2024 04:24:18 +0000 Subject: [PATCH 038/435] Update repo URLs --- .gitmodules | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9b9c09b5f..d6326a8f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/yoctoproject/poky.git [submodule "meta-confidential-compute"] path = meta-confidential-compute - url = https://github.com/Phala-Network/meta-confidential-compute.git + url = https://github.com/Dstack-TEE/meta-confidential-compute.git [submodule "meta-virtualization"] path = meta-virtualization url = https://github.com/lgirdk/meta-virtualization @@ -15,4 +15,4 @@ url = https://github.com/rust-embedded/meta-rust-bin [submodule "dstack"] path = dstack - url = https://github.com/Phala-Network/dstack + url = https://github.com/Dstack-TEE/dstack diff --git a/README.md b/README.md index c82ee23ee..63d36b869 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This project implements Yocto layer and the overall build scripts for DStack Bas ## Build ```bash -git clone https://github.com/Phala-Network/meta-dstack.git --recursive +git clone https://github.com/Dstack-TEE/meta-dstack.git --recursive cd meta-dstack source dev-setup From 028691eff580c2bdbfb0ce4a802b0e4af2973e35 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 1 Nov 2024 01:40:40 +0000 Subject: [PATCH 039/435] kmfs: Add support for app upgrade --- dstack | 2 +- .../images/dstack-initramfs-files/kmfs-setup | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/dstack b/dstack index 191349f48..40d87271b 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 191349f488f955077a7b73ca7b4b14eb651dbdf7 +Subproject commit 40d87271b59aeb10da4c7ef0fa19d097aabe016e diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index b699e971c..5e1133d5a 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -27,14 +27,21 @@ fde_prepare() { fi rootfs_hash=$(jq -r .rootfs_hash ${CFG_FILE}) - app_id=$(iohash sha256 -o! -t- -i ${APP_COMPOSE_FILE}) kms_url=$(jq -r .kms_url ${CFG_FILE}) ca_cert_hash=$(iohash sha256 -o! -t- -i ${CA_CERT}) + upgraded_app_id=$(iohash sha256 -o! -t- -i ${APP_COMPOSE_FILE}) + if [ -f "${HOST_SHARED}/.app_id" ]; then + app_id=$(cat ${HOST_SHARED}/.app_id) + else + app_id=${upgraded_app_id} + fi echo "FSINIT: Extending rootfs hash to RTMR, hash=${rootfs_hash}" tdxctl extend -i 3 --digest ${rootfs_hash} --associated-data "rootfs-hash" || panic "FSINIT: Failed to extend rootfs to RTMR" echo "FSINIT: Extending app id to RTMR, app_id=${app_id}" tdxctl extend -i 3 --digest ${app_id} --associated-data "app-id" || panic "FSINIT: Failed to extend app id to RTMR" + echo "FSINIT: Extending upgraded app id to RTMR, upgraded_app_id=${upgraded_app_id}" + tdxctl extend -i 3 --digest ${upgraded_app_id} --associated-data "upgraded-app-id" || panic "FSINIT: Failed to extend upgraded app id to RTMR" echo "FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=${ca_cert_hash}" tdxctl extend -i 3 --digest ${ca_cert_hash} --associated-data "ca-cert-hash" || panic "FSINIT: Failed to extend ca cert hash to RTMR" @@ -65,10 +72,16 @@ fde_prepare() { panic "FSINIT: Failed to get valid key phrase from KMS" fi - if [ -e ${HOST_SHARED}/initialized ]; then + if [ -e ${HOST_SHARED}/.bootstraped ]; then echo "[$(date +%H:%M:%S)] FSINIT: Mounting" echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" + if [ ! -e /root/.bootstraped ]; then + # Check if the rootfs has been properly bootstrapped by looking for the .bootstraped file + # This file is created at the end of the initial setup process + # If it doesn't exist, the rootfs was not properly initialized + panic "FSINIT: Encrypted rootfs is not bootstrapped" + fi else # Set up disk encryption and format the disk @@ -116,7 +129,9 @@ fde_prepare() { fi cp -rT ${CFG_DIR} /root/tapp || panic "FSINIT: Failed to copy config to /root/tapp" - touch ${HOST_SHARED}/initialized || echo "FSINIT: Failed to touch initialized" + touch /root/.bootstraped || echo "FSINIT: Failed to touch bootstraped" + touch ${HOST_SHARED}/.bootstraped || echo "FSINIT: Failed to touch bootstraped" + echo ${app_id} > ${HOST_SHARED}/.app_id || echo "FSINIT: Failed to set app id" fi cp ${KMFS_WORK_DIR}/appkeys.json /root/tapp/appkeys.json || panic "FSINIT: Failed to copy appkeys.json to /root/tapp" From 103087e5099a5e69e93478749ca6e5f0a08208d6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 02:17:29 +0000 Subject: [PATCH 040/435] Add reccipe dstack-ovmf --- Makefile | 2 +- .../0001-Patch-for-dstack-yocto.patch | 297 ++++++++++++++++++ .../dstack-ovmf/dstack-ovmf_git.bb | 278 ++++++++++++++++ 3 files changed, 576 insertions(+), 1 deletion(-) create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb diff --git a/Makefile b/Makefile index 78307223d..11fd5f3b0 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ $(ABS_IMAGE_FILES): make images images: - bitbake dstack-initramfs dstack-rootfs dstack-rootfs-dev ovmf + bitbake dstack-initramfs dstack-rootfs dstack-rootfs-dev dstack-ovmf emu: TD=0 ./run_td.sh diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch new file mode 100644 index 000000000..678b9fc20 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch @@ -0,0 +1,297 @@ +From 262824639dd7726b74a5fbb8e2f2e8768ac95190 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Sun, 3 Nov 2024 13:04:05 +0000 +Subject: [PATCH] Patch for dstack yocto + +--- + BaseTools/Conf/tools_def.template | 18 +++++++------- + BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- + BaseTools/Source/C/Makefiles/header.makefile | 15 ++++++------ + .../Source/Python/AutoGen/BuildEngine.py | 3 +++ + BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- + .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- + OvmfPkg/build.sh | 2 +- + 7 files changed, 41 insertions(+), 34 deletions(-) + +diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template +index 0f110fbb4a..35205db748 100755 +--- a/BaseTools/Conf/tools_def.template ++++ b/BaseTools/Conf/tools_def.template +@@ -896,7 +896,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ + *_*_*_DTC_PATH = DEF(DTC_BIN) + + # All supported GCC archs except LOONGARCH64 support -mstack-protector-guard=global, so set that on everything except LOONGARCH64 +-DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector ++DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector ENV(GCC_PREFIX_MAP) + DEFINE GCC_IA32_X64_CC_FLAGS = -mstack-protector-guard=global + DEFINE GCC_ARM_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mlittle-endian -mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections -fdata-sections -fomit-frame-pointer -Wno-address -mthumb -fno-pic -fno-pie -mstack-protector-guard=global + DEFINE GCC_LOONGARCH64_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mabi=lp64d -fno-asynchronous-unwind-tables -Wno-address -fno-short-enums -fsigned-char -ffunction-sections -fdata-sections +@@ -918,8 +918,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere + DEFINE GCC_AARCH64_ASLDLINK_FLAGS = DEF(GCC_AARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) + DEFINE GCC_LOONGARCH64_ASLDLINK_FLAGS = DEF(GCC_LOONGARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) + DEFINE GCC_IA32_X64_DLINK_FLAGS = DEF(GCC_IA32_X64_DLINK_COMMON) --entry _$(IMAGE_ENTRY_POINT) --file-alignment 0x20 --section-alignment 0x20 -Map $(DEST_DIR_DEBUG)/$(BASE_NAME).map +-DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h +-DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ++DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) ++DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) + DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h + DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h + DEFINE GCC_ASLCC_FLAGS = -x c +@@ -1072,7 +1072,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( + *_GCC48_IA32_DLINK2_FLAGS = DEF(GCC48_IA32_DLINK2_FLAGS) + *_GCC48_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC48_IA32_OBJCOPY_FLAGS = +-*_GCC48_IA32_NASM_FLAGS = -f elf32 ++*_GCC48_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) + RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set-variable +@@ -1100,7 +1100,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set + *_GCC48_X64_DLINK2_FLAGS = DEF(GCC48_X64_DLINK2_FLAGS) + *_GCC48_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC48_X64_OBJCOPY_FLAGS = +-*_GCC48_X64_NASM_FLAGS = -f elf64 ++*_GCC48_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) + RELEASE_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) -Wno-unused-but-set-variable +@@ -1209,7 +1209,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s + *_GCC49_IA32_DLINK2_FLAGS = DEF(GCC49_IA32_DLINK2_FLAGS) + *_GCC49_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC49_IA32_OBJCOPY_FLAGS = +-*_GCC49_IA32_NASM_FLAGS = -f elf32 ++*_GCC49_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) + RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable +@@ -1237,7 +1237,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set + *_GCC49_X64_DLINK2_FLAGS = DEF(GCC49_X64_DLINK2_FLAGS) + *_GCC49_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC49_X64_OBJCOPY_FLAGS = +-*_GCC49_X64_NASM_FLAGS = -f elf64 ++*_GCC49_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) + RELEASE_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable +@@ -1496,7 +1496,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 + *_GCC5_IA32_DLINK2_FLAGS = DEF(GCC5_IA32_DLINK2_FLAGS) -no-pie + *_GCC5_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC5_IA32_OBJCOPY_FLAGS = +-*_GCC5_IA32_NASM_FLAGS = -f elf32 ++*_GCC5_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC5_IA32_CC_FLAGS = DEF(GCC5_IA32_CC_FLAGS) -flto + DEBUG_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl,-m,elf_i386,--oformat=elf32-i386 +@@ -1528,7 +1528,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, + *_GCC5_X64_DLINK2_FLAGS = DEF(GCC5_X64_DLINK2_FLAGS) + *_GCC5_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC5_X64_OBJCOPY_FLAGS = +-*_GCC5_X64_NASM_FLAGS = -f elf64 ++*_GCC5_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC5_X64_CC_FLAGS = DEF(GCC5_X64_CC_FLAGS) -flto -DUSING_LTO + DEBUG_GCC5_X64_DLINK_FLAGS = DEF(GCC5_X64_DLINK_FLAGS) -flto -Os +diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c +index 9d04fc612e..83fd6c9c05 100644 +--- a/BaseTools/Source/C/GenFw/Elf64Convert.c ++++ b/BaseTools/Source/C/GenFw/Elf64Convert.c +@@ -13,6 +13,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent + #ifndef __GNUC__ + #include + #include ++#else ++#define _GNU_SOURCE + #endif + #include + #include +@@ -988,7 +990,7 @@ ScanSections64 ( + } + mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + + sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + +- strlen(mInImageName) + 1; ++ strlen(basename(mInImageName)) + 1; + + // + // Add more space in the .debug data region for the DllCharacteristicsEx +@@ -2299,7 +2301,7 @@ WriteDebug64 ( + EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; + EFI_IMAGE_DEBUG_EX_DLLCHARACTERISTICS_ENTRY *DllEntry; + +- Len = strlen(mInImageName) + 1; ++ Len = strlen(basename(mInImageName)) + 1; + + NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); + DataDir = &NtHdr->Pe32Plus.OptionalHeader.DataDirectory[EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]; +@@ -2332,7 +2334,7 @@ WriteDebug64 ( + + Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); + Nb10->Signature = CODEVIEW_SIGNATURE_NB10; +- strcpy ((char *)(Nb10 + 1), mInImageName); ++ strcpy ((char *)(Nb10 + 1), basename(mInImageName)); + } + + STATIC +diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile +index d369908a09..22c670f316 100644 +--- a/BaseTools/Source/C/Makefiles/header.makefile ++++ b/BaseTools/Source/C/Makefiles/header.makefile +@@ -85,35 +85,34 @@ endif + + INCLUDE = $(TOOL_INCLUDE) -I $(MAKEROOT) -I $(MAKEROOT)/Include/Common -I $(MAKEROOT)/Include/ -I $(MAKEROOT)/Include/IndustryStandard -I $(MAKEROOT)/Common/ -I .. -I . $(ARCH_INCLUDE) + INCLUDE += -I $(EDK2_PATH)/MdePkg/Include +-CPPFLAGS = $(INCLUDE) ++CPPFLAGS += $(INCLUDE) + + # keep EXTRA_OPTFLAGS last + BUILD_OPTFLAGS = -O2 $(EXTRA_OPTFLAGS) + + ifeq ($(DARWIN),Darwin) + # assume clang or clang compatible flags on OS X +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ + -Wno-deprecated-declarations -Wno-self-assign -Wno-unused-result -nostdlib -g + else + ifneq ($(CLANG),) +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ + -fno-delete-null-pointer-checks -Wall -Werror \ + -Wno-deprecated-declarations -Wno-self-assign \ + -Wno-unused-result -nostdlib -g + else +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ + -fno-delete-null-pointer-checks -Wall -Werror \ + -Wno-deprecated-declarations -Wno-stringop-truncation -Wno-restrict \ + -Wno-unused-result -nostdlib -g + endif + endif + ifneq ($(CLANG),) +-LDFLAGS = +-CXXFLAGS = -Wno-deprecated-register -Wno-unused-result -std=c++14 ++CXXFLAGS += -Wno-deprecated-register -Wno-unused-result -std=c++14 + else +-LDFLAGS = +-CXXFLAGS = -Wno-unused-result ++CXXFLAGS += -Wno-unused-result + endif ++ + ifeq ($(HOST_ARCH), IA32) + # + # Snow Leopard is a 32-bit and 64-bit environment. uname -m returns i386, but gcc defaults +diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py +index 45b39d7878..3fed7d1736 100644 +--- a/BaseTools/Source/Python/AutoGen/BuildEngine.py ++++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py +@@ -70,6 +70,9 @@ class TargetDescBlock(object): + else: + return str(Other) == self.Target.Path + ++ def __lt__(self, other): ++ return str(self) < str(other) ++ + def AddInput(self, Input): + if Input not in self.Inputs: + self.Inputs.append(Input) +diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py +index 6d9c60b702..83ac267647 100755 +--- a/BaseTools/Source/Python/AutoGen/GenMake.py ++++ b/BaseTools/Source/Python/AutoGen/GenMake.py +@@ -576,7 +576,7 @@ cleanlib: + os.remove(RespFileList) + + # convert source files and binary files to build targets +- self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList] ++ self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList]) + if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0: + EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", + ExtraData="[%s]" % str(MyAgo)) +@@ -727,7 +727,7 @@ cleanlib: + OutputFile = '' + DepsFileList = [] + +- for Cmd in self.GenFfsList: ++ for Cmd in sorted(self.GenFfsList): + if Cmd[2]: + for CopyCmd in Cmd[2]: + Src, Dst = CopyCmd +@@ -760,7 +760,7 @@ cleanlib: + self.BuildTargetList.append('\t%s' % CmdString) + + self.ParseSecCmd(DepsFileList, Cmd[1]) +- for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList : ++ for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList): + self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile))) + self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd)) + self.FfsOutputFileList = [] +@@ -799,13 +799,13 @@ cleanlib: + + def CommandExceedLimit(self): + FlagDict = { +- 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False}, +- 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False}, +- 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False}, +- 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False}, +- 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False}, +- 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False}, +- 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False}, ++ 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : True}, ++ 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : True}, ++ 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : True}, ++ 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True}, ++ 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True}, ++ 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : True}, ++ 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True}, + } + + RespDict = {} +@@ -1008,9 +1008,9 @@ cleanlib: + if not self.ObjTargetDict.get(T.Target.SubDir): + self.ObjTargetDict[T.Target.SubDir] = set() + self.ObjTargetDict[T.Target.SubDir].add(NewFile) +- for Type in self._AutoGenObject.Targets: ++ for Type in sorted(self._AutoGenObject.Targets): + resp_file_number = 0 +- for T in self._AutoGenObject.Targets[Type]: ++ for T in sorted(self._AutoGenObject.Targets[Type]): + # Generate related macros if needed + if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: + self.FileListMacros[T.FileListMacro] = [] +diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +index 65a2176ca9..318b9611ce 100755 +--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py ++++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +@@ -1474,6 +1474,9 @@ class ModuleAutoGen(AutoGen): + for File in Files: + if File.lower().endswith('.pdb'): + AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) ++ ++ AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item']) ++ + HeaderComments = self.Module.HeaderComments + StartPos = 0 + for Index in range(len(HeaderComments)): +@@ -1749,7 +1752,7 @@ class ModuleAutoGen(AutoGen): + if os.path.exists (self.TimeStampPath): + os.remove (self.TimeStampPath) + +- SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) ++ SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False) + + # Ignore generating makefile when it is a binary module + if self.IsBinaryModule: +diff --git a/OvmfPkg/build.sh b/OvmfPkg/build.sh +index 279f0d099a..285f061bf4 100755 +--- a/OvmfPkg/build.sh ++++ b/OvmfPkg/build.sh +@@ -24,7 +24,7 @@ then + # this assumes svn pulls have the same root dir + # export EDK_TOOLS_PATH=`pwd`/../BaseTools + # This version is for the tools source in edk2 +- export EDK_TOOLS_PATH=`pwd`/BaseTools ++ export EDK_TOOLS_PATH=BBAKE_EDK_TOOLS_PATH/BaseTools + echo $EDK_TOOLS_PATH + source edksetup.sh BaseTools + else +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb new file mode 100644 index 000000000..ebb376793 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -0,0 +1,278 @@ +SUMMARY = "OVMF - UEFI firmware for Qemu and KVM" +DESCRIPTION = "OVMF is an EDK II based project to enable UEFI support for \ +Virtual Machines. OVMF contains sample UEFI firmware for QEMU and KVM" +HOMEPAGE = "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/tianocore/tianocore.github.io/wiki/OVMF" +LICENSE = "BSD-2-Clause-Patent" +LICENSE:class-target = "${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'BSD-2-Clause-Patent & OpenSSL', 'BSD-2-Clause-Patent', d)}" +LIC_FILES_CHKSUM = "file://OvmfPkg/License.txt;md5=06357ddc23f46577c2aeaeaf7b776d65" + +# Enabling Secure Boot adds a dependency on OpenSSL and implies +# compiling OVMF twice, so it is disabled by default. Distros +# may change that default. +PACKAGECONFIG ??= "" +PACKAGECONFIG += "${@bb.utils.contains('MACHINE_FEATURES', 'tpm', 'tpm', '', d)}" +PACKAGECONFIG += "${@bb.utils.contains('MACHINE_FEATURES', 'tpm2', 'tpm', '', d)}" +PACKAGECONFIG[secureboot] = ",,," +PACKAGECONFIG[tpm] = "-D TPM_ENABLE=TRUE,-D TPM_ENABLE=FALSE,," + +# GCC12 trips on it +#see https://src.fedoraproject.org/rpms/edk2/blob/rawhide/f/0032-Basetools-turn-off-gcc12-warning.patch +BUILD_CFLAGS += "-Wno-error=stringop-overflow" + +SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ + file://0001-Patch-for-dstack-yocto.patch \ + " + +PV = "edk2-3a3b12cbdae" +SRCREV = "3a3b12cbdae2e89b0e365eb01c378891d0d9037c" +UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" + +CVE_PRODUCT = "edk2" +CVE_VERSION = "${@d.getVar('PV').split('-')[1]}" + +CVE_STATUS[CVE-2014-8271] = "fixed-version: Fixed in svn_16280, which is an unusual versioning breaking version comparison." +CVE_STATUS[CVE-2014-4859] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2014-4860] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14553] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14559] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14562] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14563] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14575] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14586] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." +CVE_STATUS[CVE-2019-14587] = "fixed-version: The CPE in the NVD database doesn't reflect correctly the vulnerable versions." + +inherit deploy + +PARALLEL_MAKE = "" + +S = "${WORKDIR}/git" + +DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" + +EDK_TOOLS_DIR="edk2_basetools" + +# OVMF has trouble building with the default optimization of -O2. +BUILD_OPTIMIZATION = "" + +# OVMF supports IA only, although it could conceivably support ARM someday. +COMPATIBLE_HOST:class-target='(i.86|x86_64).*' + +# Additional build flags for OVMF with Secure Boot. +# Fedora also uses "-D SMM_REQUIRE -D EXCLUDE_SHELL_FROM_FD". +OVMF_SECURE_BOOT_EXTRA_FLAGS ??= "" +OVMF_SECURE_BOOT_FLAGS = "-DSECURE_BOOT_ENABLE=TRUE ${OVMF_SECURE_BOOT_EXTRA_FLAGS}" + +export PYTHON_COMMAND = "${HOSTTOOLS_DIR}/python3" + +do_patch[postfuncs] += "fix_basetools_location" +fix_basetools_location () { +} +fix_basetools_location:class-target() { + # Replaces the fake path inserted by 0002-ovmf-update-path-to-native-BaseTools.patch. + # Necessary for finding the actual BaseTools from ovmf-native. + sed -i -e 's#BBAKE_EDK_TOOLS_PATH#${STAGING_BINDIR_NATIVE}/${EDK_TOOLS_DIR}#' ${S}/OvmfPkg/build.sh +} + +do_patch[postfuncs] += "fix_iasl" +fix_iasl() { +} +fix_iasl:class-native() { + # iasl is not installed under /usr/bin when building with OE. + sed -i -e 's#/usr/bin/iasl#${STAGING_BINDIR_NATIVE}/iasl#' ${S}/BaseTools/Conf/tools_def.template +} + +# Inject CC and friends into the build. LINKER already is in GNUmakefile. +# Must be idempotent and thus remove old assignments that were inserted +# earlier. +do_patch[postfuncs] += "fix_toolchain" +fix_toolchain() { + sed -i \ + -e '/^\(CC\|CXX\|AS\|AR\|LD\|LINKER\) =/d' \ + -e '/^APPLICATION/a CC = ${CC}\nCXX = ${CXX}\nAS = ${AS}\nAR = ${AR}\nLD = ${LD}\nLINKER = $(CC)' \ + ${S}/BaseTools/Source/C/Makefiles/app.makefile + sed -i \ + -e '/^\(CC\|CXX\|AS\|AR\|LD\)/d' \ + -e '/^VFR_CPPFLAGS/a CC = ${CC}\nCXX = ${CXX}\nAS = ${AS}\nAR = ${AR}\nLD = ${LD}' \ + ${S}/BaseTools/Source/C/VfrCompile/GNUmakefile +} +fix_toolchain:append:class-native() { + # This tools_def.template is going to be used by the target ovmf and + # defines which compilers to use. For the GCC toolchain definitions, + # that will be ${HOST_PREFIX}gcc. However, "make" doesn't need that + # prefix. + # + # Injecting ENV(HOST_PREFIX) matches exporting that value as env + # variable in do_compile:class-target. + sed -i \ + -e 's#\(ENV\|DEF\)(GCC.*_PREFIX)#ENV(HOST_PREFIX)#' \ + -e 's#ENV(HOST_PREFIX)make#make#' \ + ${S}/BaseTools/Conf/tools_def.template + sed -i \ + -e '/^\(LFLAGS\|CFLAGS\) +=/d' \ + -e '/^LINKER/a LFLAGS += ${BUILD_LDFLAGS}\nCFLAGS += ${BUILD_CFLAGS}' \ + ${S}/BaseTools/Source/C/Makefiles/app.makefile \ + ${S}/BaseTools/Source/C/VfrCompile/GNUmakefile + # Linking with gold fails: + # internal error in do_layout, at ../../gold/object.cc:1821 + # make: *** [.../OUTPUT/Facs.acpi] Error 1 + # We intentionally hard-code the use of ld.bfd regardless of DISTRO_FEATURES + # to make ovmf-native reusable across distros. + sed -i \ + -e 's#^\(DEFINE GCC.*DLINK.*FLAGS *=\)#\1 -fuse-ld=bfd#' \ + -e 's#-flto#-fno-lto#g' \ + -e 's#-DUSING_LTO##g' \ + ${S}/BaseTools/Conf/tools_def.template +} + +# We disable lto above since the results are not reproducible and make it hard to compare +# binary build aretfacts to debug reproducibility problems. +# Surprisingly, if you disable lto, you see compiler warnings which are fatal. We therefore +# have to hack warnings overrides into GCC_PREFIX_MAP to allow it to build. + +# We want to pass ${DEBUG_PREFIX_MAP} to gcc commands and also pass in +# --debug-prefix-map to nasm (we carry a patch to nasm for this). The +# tools definitions are built by ovmf-native so we need to pass this in +# at target build time when we know the right values. +export NASM_PREFIX_MAP = "--debug-prefix-map=${WORKDIR}=${TARGET_DBGSRC_DIR}" +export GCC_PREFIX_MAP = "${DEBUG_PREFIX_MAP} -Wno-stringop-overflow -Wno-maybe-uninitialized" + +GCC_VER="$(${CC} -v 2>&1 | tail -n1 | awk '{print $3}')" + +fixup_target_tools() { + case ${1} in + 4.4.*) + FIXED_GCCVER=GCC44 + ;; + 4.5.*) + FIXED_GCCVER=GCC45 + ;; + 4.6.*) + FIXED_GCCVER=GCC46 + ;; + 4.7.*) + FIXED_GCCVER=GCC47 + ;; + 4.8.*) + FIXED_GCCVER=GCC48 + ;; + 4.9.*) + FIXED_GCCVER=GCC49 + ;; + *) + FIXED_GCCVER=GCC5 + ;; + esac + echo ${FIXED_GCCVER} +} + +do_compile:class-native() { + oe_runmake -C ${S}/BaseTools +} + +do_compile:class-target() { + export LFLAGS="${LDFLAGS}" + PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}" + OVMF_ARCH="X64" + if [ "${TARGET_ARCH}" != "x86_64" ] ; then + OVMF_ARCH="IA32" + fi + + # The build for the target uses BaseTools/Conf/tools_def.template + # from ovmf-native to find the compiler, which depends on + # exporting HOST_PREFIX. + export HOST_PREFIX="${HOST_PREFIX}" + + # BaseTools/Conf gets copied to Conf, but only if that does not + # exist yet. To ensure that an updated template gets used during + # incremental builds, we need to remove the copy before we start. + rm -f `ls ${S}/Conf/*.txt | grep -v ReadMe.txt` + + # ${WORKDIR}/ovmf is a well-known location where do_install and + # do_deploy will be able to find the files. + rm -rf ${WORKDIR}/ovmf + mkdir ${WORKDIR}/ovmf + OVMF_DIR_SUFFIX="X64" + if [ "${TARGET_ARCH}" != "x86_64" ] ; then + OVMF_DIR_SUFFIX="Ia32" # Note the different capitalization + fi + FIXED_GCCVER=$(fixup_target_tools ${GCC_VER}) + bbnote FIXED_GCCVER is ${FIXED_GCCVER} + build_dir="${S}/Build/Ovmf$OVMF_DIR_SUFFIX/RELEASE_${FIXED_GCCVER}" + + bbnote "Building without Secure Boot." + rm -rf ${S}/Build/Ovmf$OVMF_DIR_SUFFIX + ${S}/OvmfPkg/build.sh $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} + ln ${build_dir}/FV/OVMF.fd ${WORKDIR}/ovmf/ovmf.fd + ln ${build_dir}/FV/OVMF_CODE.fd ${WORKDIR}/ovmf/ovmf.code.fd + ln ${build_dir}/FV/OVMF_VARS.fd ${WORKDIR}/ovmf/ovmf.vars.fd + ln ${build_dir}/${OVMF_ARCH}/Shell.efi ${WORKDIR}/ovmf/ + + if ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'true', 'false', d)}; then + # Repeat build with the Secure Boot flags. + bbnote "Building with Secure Boot." + rm -rf ${S}/Build/Ovmf$OVMF_DIR_SUFFIX + ${S}/OvmfPkg/build.sh $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} ${OVMF_SECURE_BOOT_FLAGS} + ln ${build_dir}/FV/OVMF.fd ${WORKDIR}/ovmf/ovmf.secboot.fd + ln ${build_dir}/FV/OVMF_CODE.fd ${WORKDIR}/ovmf/ovmf.secboot.code.fd + ln ${build_dir}/${OVMF_ARCH}/EnrollDefaultKeys.efi ${WORKDIR}/ovmf/ + fi +} + +do_install:class-native() { + install -d ${D}/${bindir}/edk2_basetools + find ${S}/BaseTools -name \*.pyc -exec rm -rf \{\} \; + cp -r ${S}/BaseTools ${D}/${bindir}/${EDK_TOOLS_DIR} +} + +do_install:class-target() { + # Content for UEFI shell iso. We install the EFI shell as + # bootx64/ia32.efi because then it can be started even when the + # firmware itself does not contain it. + install -d ${D}/efi/boot + install ${WORKDIR}/ovmf/Shell.efi ${D}/efi/boot/boot${@ "ia32" if "${TARGET_ARCH}" != "x86_64" else "x64"}.efi + if ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'true', 'false', d)}; then + install ${WORKDIR}/ovmf/EnrollDefaultKeys.efi ${D} + fi +} + +# This always gets packaged because ovmf-shell-image depends on it. +# This allows testing that recipe in all configurations because it +# can always be part of a world build. +# +# However, EnrollDefaultKeys.efi is only included when Secure Boot is enabled. +PACKAGES =+ "ovmf-shell-efi" +FILES:ovmf-shell-efi = " \ + EnrollDefaultKeys.efi \ + efi/ \ +" + +DEPLOYDEP = "" +DEPLOYDEP:class-target = "qemu-system-native:do_populate_sysroot" +DEPLOYDEP:class-target += " ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'openssl-native:do_populate_sysroot', '', d)}" +do_deploy[depends] += "${DEPLOYDEP}" + +do_deploy() { +} +do_deploy:class-target() { + # For use with "runqemu ovmf". + for i in \ + ovmf \ + ovmf.code \ + ovmf.vars \ + ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'ovmf.secboot ovmf.secboot.code', '', d)} \ + ; do + cp ${WORKDIR}/ovmf/$i.fd ${DEPLOYDIR}/ + qemu-img convert -f raw -O qcow2 ${WORKDIR}/ovmf/$i.fd ${DEPLOYDIR}/$i.qcow2 + done + + if ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'true', 'false', d)}; then + # Create a test Platform Key and first Key Exchange Key to use with EnrollDefaultKeys + openssl req -new -x509 -newkey rsa:2048 -keyout ${DEPLOYDIR}/OvmfPkKek1.key \ + -out ${DEPLOYDIR}/OvmfPkKek1.crt -nodes -days 20 -subj "/CN=OVMFSecBootTest" + openssl x509 -in ${DEPLOYDIR}/OvmfPkKek1.crt -out ${DEPLOYDIR}/OvmfPkKek1.pem -outform PEM + fi +} +addtask do_deploy after do_compile before do_build + +BBCLASSEXTEND = "native" +TOOLCHAIN = "gcc" From 6217e67b09a1a16a607b8861e4e09b6df1697b5a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 02:17:39 +0000 Subject: [PATCH 041/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 40d87271b..2c512d764 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 40d87271b59aeb10da4c7ef0fa19d097aabe016e +Subproject commit 2c512d764bb049ee0b81a5acda59fea58edf4150 From 37225f737bd582a54aae7ed6e5bf2952b21e2ad6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 04:17:39 +0000 Subject: [PATCH 042/435] Fix ovmf build error --- .../0001-Patch-for-dstack-yocto.patch | 17 +++++++++++++++-- .../recipes-core/dstack-ovmf/dstack-ovmf_git.bb | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch index 678b9fc20..c2c749ffe 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch @@ -1,4 +1,4 @@ -From 262824639dd7726b74a5fbb8e2f2e8768ac95190 Mon Sep 17 00:00:00 2001 +From 4e0965b45c21d3263996a4e3218f7b542313010a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 3 Nov 2024 13:04:05 +0000 Subject: [PATCH] Patch for dstack yocto @@ -10,8 +10,9 @@ Subject: [PATCH] Patch for dstack yocto .../Source/Python/AutoGen/BuildEngine.py | 3 +++ BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- + OvmfPkg/Sec/SecMain.c | 1 + OvmfPkg/build.sh | 2 +- - 7 files changed, 41 insertions(+), 34 deletions(-) + 8 files changed, 42 insertions(+), 34 deletions(-) diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template index 0f110fbb4a..35205db748 100755 @@ -279,6 +280,18 @@ index 65a2176ca9..318b9611ce 100755 # Ignore generating makefile when it is a binary module if self.IsBinaryModule: +diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c +index d13a948509..2a44c4ed24 100644 +--- a/OvmfPkg/Sec/SecMain.c ++++ b/OvmfPkg/Sec/SecMain.c +@@ -885,6 +885,7 @@ SecCoreStartupWithStack ( + InitializeCpuExceptionHandlers (NULL); + } + ++ void ProcessLibraryConstructorList (void); + ProcessLibraryConstructorList (); + + if (!SevEsIsEnabled ()) { diff --git a/OvmfPkg/build.sh b/OvmfPkg/build.sh index 279f0d099a..285f061bf4 100755 --- a/OvmfPkg/build.sh diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index ebb376793..bb9ecdec7 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -23,7 +23,7 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0001-Patch-for-dstack-yocto.patch \ " -PV = "edk2-3a3b12cbdae" +PV = "edk2-3a3b12cb" SRCREV = "3a3b12cbdae2e89b0e365eb01c378891d0d9037c" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" From 512c13280cb1c235b170621e904353767ae0b856 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 08:24:09 +0000 Subject: [PATCH 043/435] Fix ovmf build error --- .../0001-Patch-for-dstack-yocto.patch | 310 ------------------ ...0001-Update-path-to-native-BaseTools.patch | 25 ++ ...ile-adjust-to-build-in-under-bitbake.patch | 59 ++++ .../dstack-ovmf/0003-Debug-prefix-map.patch | 90 +++++ .../dstack-ovmf/0004-Reproduciable.patch | 156 +++++++++ ...eclare-ProcessLibraryConstructorList.patch | 24 ++ .../dstack-ovmf/dstack-ovmf_git.bb | 6 +- 7 files changed, 359 insertions(+), 311 deletions(-) delete mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch deleted file mode 100644 index c2c749ffe..000000000 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Patch-for-dstack-yocto.patch +++ /dev/null @@ -1,310 +0,0 @@ -From 4e0965b45c21d3263996a4e3218f7b542313010a Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Sun, 3 Nov 2024 13:04:05 +0000 -Subject: [PATCH] Patch for dstack yocto - ---- - BaseTools/Conf/tools_def.template | 18 +++++++------- - BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- - BaseTools/Source/C/Makefiles/header.makefile | 15 ++++++------ - .../Source/Python/AutoGen/BuildEngine.py | 3 +++ - BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- - .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- - OvmfPkg/Sec/SecMain.c | 1 + - OvmfPkg/build.sh | 2 +- - 8 files changed, 42 insertions(+), 34 deletions(-) - -diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template -index 0f110fbb4a..35205db748 100755 ---- a/BaseTools/Conf/tools_def.template -+++ b/BaseTools/Conf/tools_def.template -@@ -896,7 +896,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ - *_*_*_DTC_PATH = DEF(DTC_BIN) - - # All supported GCC archs except LOONGARCH64 support -mstack-protector-guard=global, so set that on everything except LOONGARCH64 --DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector -+DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector ENV(GCC_PREFIX_MAP) - DEFINE GCC_IA32_X64_CC_FLAGS = -mstack-protector-guard=global - DEFINE GCC_ARM_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mlittle-endian -mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections -fdata-sections -fomit-frame-pointer -Wno-address -mthumb -fno-pic -fno-pie -mstack-protector-guard=global - DEFINE GCC_LOONGARCH64_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mabi=lp64d -fno-asynchronous-unwind-tables -Wno-address -fno-short-enums -fsigned-char -ffunction-sections -fdata-sections -@@ -918,8 +918,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere - DEFINE GCC_AARCH64_ASLDLINK_FLAGS = DEF(GCC_AARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) - DEFINE GCC_LOONGARCH64_ASLDLINK_FLAGS = DEF(GCC_LOONGARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) - DEFINE GCC_IA32_X64_DLINK_FLAGS = DEF(GCC_IA32_X64_DLINK_COMMON) --entry _$(IMAGE_ENTRY_POINT) --file-alignment 0x20 --section-alignment 0x20 -Map $(DEST_DIR_DEBUG)/$(BASE_NAME).map --DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h --DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h -+DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) -+DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) - DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h - DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h - DEFINE GCC_ASLCC_FLAGS = -x c -@@ -1072,7 +1072,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( - *_GCC48_IA32_DLINK2_FLAGS = DEF(GCC48_IA32_DLINK2_FLAGS) - *_GCC48_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) - *_GCC48_IA32_OBJCOPY_FLAGS = --*_GCC48_IA32_NASM_FLAGS = -f elf32 -+*_GCC48_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) - RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1100,7 +1100,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set - *_GCC48_X64_DLINK2_FLAGS = DEF(GCC48_X64_DLINK2_FLAGS) - *_GCC48_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) - *_GCC48_X64_OBJCOPY_FLAGS = --*_GCC48_X64_NASM_FLAGS = -f elf64 -+*_GCC48_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) - RELEASE_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1209,7 +1209,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s - *_GCC49_IA32_DLINK2_FLAGS = DEF(GCC49_IA32_DLINK2_FLAGS) - *_GCC49_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) - *_GCC49_IA32_OBJCOPY_FLAGS = --*_GCC49_IA32_NASM_FLAGS = -f elf32 -+*_GCC49_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) - RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1237,7 +1237,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set - *_GCC49_X64_DLINK2_FLAGS = DEF(GCC49_X64_DLINK2_FLAGS) - *_GCC49_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) - *_GCC49_X64_OBJCOPY_FLAGS = --*_GCC49_X64_NASM_FLAGS = -f elf64 -+*_GCC49_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) - RELEASE_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1496,7 +1496,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 - *_GCC5_IA32_DLINK2_FLAGS = DEF(GCC5_IA32_DLINK2_FLAGS) -no-pie - *_GCC5_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) - *_GCC5_IA32_OBJCOPY_FLAGS = --*_GCC5_IA32_NASM_FLAGS = -f elf32 -+*_GCC5_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC5_IA32_CC_FLAGS = DEF(GCC5_IA32_CC_FLAGS) -flto - DEBUG_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl,-m,elf_i386,--oformat=elf32-i386 -@@ -1528,7 +1528,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, - *_GCC5_X64_DLINK2_FLAGS = DEF(GCC5_X64_DLINK2_FLAGS) - *_GCC5_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) - *_GCC5_X64_OBJCOPY_FLAGS = --*_GCC5_X64_NASM_FLAGS = -f elf64 -+*_GCC5_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) - - DEBUG_GCC5_X64_CC_FLAGS = DEF(GCC5_X64_CC_FLAGS) -flto -DUSING_LTO - DEBUG_GCC5_X64_DLINK_FLAGS = DEF(GCC5_X64_DLINK_FLAGS) -flto -Os -diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c -index 9d04fc612e..83fd6c9c05 100644 ---- a/BaseTools/Source/C/GenFw/Elf64Convert.c -+++ b/BaseTools/Source/C/GenFw/Elf64Convert.c -@@ -13,6 +13,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent - #ifndef __GNUC__ - #include - #include -+#else -+#define _GNU_SOURCE - #endif - #include - #include -@@ -988,7 +990,7 @@ ScanSections64 ( - } - mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + - sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + -- strlen(mInImageName) + 1; -+ strlen(basename(mInImageName)) + 1; - - // - // Add more space in the .debug data region for the DllCharacteristicsEx -@@ -2299,7 +2301,7 @@ WriteDebug64 ( - EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; - EFI_IMAGE_DEBUG_EX_DLLCHARACTERISTICS_ENTRY *DllEntry; - -- Len = strlen(mInImageName) + 1; -+ Len = strlen(basename(mInImageName)) + 1; - - NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); - DataDir = &NtHdr->Pe32Plus.OptionalHeader.DataDirectory[EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]; -@@ -2332,7 +2334,7 @@ WriteDebug64 ( - - Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); - Nb10->Signature = CODEVIEW_SIGNATURE_NB10; -- strcpy ((char *)(Nb10 + 1), mInImageName); -+ strcpy ((char *)(Nb10 + 1), basename(mInImageName)); - } - - STATIC -diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile -index d369908a09..22c670f316 100644 ---- a/BaseTools/Source/C/Makefiles/header.makefile -+++ b/BaseTools/Source/C/Makefiles/header.makefile -@@ -85,35 +85,34 @@ endif - - INCLUDE = $(TOOL_INCLUDE) -I $(MAKEROOT) -I $(MAKEROOT)/Include/Common -I $(MAKEROOT)/Include/ -I $(MAKEROOT)/Include/IndustryStandard -I $(MAKEROOT)/Common/ -I .. -I . $(ARCH_INCLUDE) - INCLUDE += -I $(EDK2_PATH)/MdePkg/Include --CPPFLAGS = $(INCLUDE) -+CPPFLAGS += $(INCLUDE) - - # keep EXTRA_OPTFLAGS last - BUILD_OPTFLAGS = -O2 $(EXTRA_OPTFLAGS) - - ifeq ($(DARWIN),Darwin) - # assume clang or clang compatible flags on OS X --CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ -+CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ - -Wno-deprecated-declarations -Wno-self-assign -Wno-unused-result -nostdlib -g - else - ifneq ($(CLANG),) --CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ -+CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ - -fno-delete-null-pointer-checks -Wall -Werror \ - -Wno-deprecated-declarations -Wno-self-assign \ - -Wno-unused-result -nostdlib -g - else --CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ -+CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ - -fno-delete-null-pointer-checks -Wall -Werror \ - -Wno-deprecated-declarations -Wno-stringop-truncation -Wno-restrict \ - -Wno-unused-result -nostdlib -g - endif - endif - ifneq ($(CLANG),) --LDFLAGS = --CXXFLAGS = -Wno-deprecated-register -Wno-unused-result -std=c++14 -+CXXFLAGS += -Wno-deprecated-register -Wno-unused-result -std=c++14 - else --LDFLAGS = --CXXFLAGS = -Wno-unused-result -+CXXFLAGS += -Wno-unused-result - endif -+ - ifeq ($(HOST_ARCH), IA32) - # - # Snow Leopard is a 32-bit and 64-bit environment. uname -m returns i386, but gcc defaults -diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py -index 45b39d7878..3fed7d1736 100644 ---- a/BaseTools/Source/Python/AutoGen/BuildEngine.py -+++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py -@@ -70,6 +70,9 @@ class TargetDescBlock(object): - else: - return str(Other) == self.Target.Path - -+ def __lt__(self, other): -+ return str(self) < str(other) -+ - def AddInput(self, Input): - if Input not in self.Inputs: - self.Inputs.append(Input) -diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py -index 6d9c60b702..83ac267647 100755 ---- a/BaseTools/Source/Python/AutoGen/GenMake.py -+++ b/BaseTools/Source/Python/AutoGen/GenMake.py -@@ -576,7 +576,7 @@ cleanlib: - os.remove(RespFileList) - - # convert source files and binary files to build targets -- self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList] -+ self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList]) - if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0: - EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", - ExtraData="[%s]" % str(MyAgo)) -@@ -727,7 +727,7 @@ cleanlib: - OutputFile = '' - DepsFileList = [] - -- for Cmd in self.GenFfsList: -+ for Cmd in sorted(self.GenFfsList): - if Cmd[2]: - for CopyCmd in Cmd[2]: - Src, Dst = CopyCmd -@@ -760,7 +760,7 @@ cleanlib: - self.BuildTargetList.append('\t%s' % CmdString) - - self.ParseSecCmd(DepsFileList, Cmd[1]) -- for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList : -+ for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList): - self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile))) - self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd)) - self.FfsOutputFileList = [] -@@ -799,13 +799,13 @@ cleanlib: - - def CommandExceedLimit(self): - FlagDict = { -- 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False}, -- 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False}, -- 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False}, -- 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False}, -- 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False}, -- 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False}, -- 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False}, -+ 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : True}, -+ 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : True}, -+ 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : True}, -+ 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True}, -+ 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True}, -+ 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : True}, -+ 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True}, - } - - RespDict = {} -@@ -1008,9 +1008,9 @@ cleanlib: - if not self.ObjTargetDict.get(T.Target.SubDir): - self.ObjTargetDict[T.Target.SubDir] = set() - self.ObjTargetDict[T.Target.SubDir].add(NewFile) -- for Type in self._AutoGenObject.Targets: -+ for Type in sorted(self._AutoGenObject.Targets): - resp_file_number = 0 -- for T in self._AutoGenObject.Targets[Type]: -+ for T in sorted(self._AutoGenObject.Targets[Type]): - # Generate related macros if needed - if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: - self.FileListMacros[T.FileListMacro] = [] -diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py -index 65a2176ca9..318b9611ce 100755 ---- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py -+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py -@@ -1474,6 +1474,9 @@ class ModuleAutoGen(AutoGen): - for File in Files: - if File.lower().endswith('.pdb'): - AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) -+ -+ AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item']) -+ - HeaderComments = self.Module.HeaderComments - StartPos = 0 - for Index in range(len(HeaderComments)): -@@ -1749,7 +1752,7 @@ class ModuleAutoGen(AutoGen): - if os.path.exists (self.TimeStampPath): - os.remove (self.TimeStampPath) - -- SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) -+ SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False) - - # Ignore generating makefile when it is a binary module - if self.IsBinaryModule: -diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c -index d13a948509..2a44c4ed24 100644 ---- a/OvmfPkg/Sec/SecMain.c -+++ b/OvmfPkg/Sec/SecMain.c -@@ -885,6 +885,7 @@ SecCoreStartupWithStack ( - InitializeCpuExceptionHandlers (NULL); - } - -+ void ProcessLibraryConstructorList (void); - ProcessLibraryConstructorList (); - - if (!SevEsIsEnabled ()) { -diff --git a/OvmfPkg/build.sh b/OvmfPkg/build.sh -index 279f0d099a..285f061bf4 100755 ---- a/OvmfPkg/build.sh -+++ b/OvmfPkg/build.sh -@@ -24,7 +24,7 @@ then - # this assumes svn pulls have the same root dir - # export EDK_TOOLS_PATH=`pwd`/../BaseTools - # This version is for the tools source in edk2 -- export EDK_TOOLS_PATH=`pwd`/BaseTools -+ export EDK_TOOLS_PATH=BBAKE_EDK_TOOLS_PATH/BaseTools - echo $EDK_TOOLS_PATH - source edksetup.sh BaseTools - else --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch new file mode 100644 index 000000000..01f54b104 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch @@ -0,0 +1,25 @@ +From 77c79b5aeabe614ebaa993c6a0e6490798d92059 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 4 Nov 2024 04:27:21 +0000 +Subject: [PATCH 1/5] Update path to native BaseTools + +--- + OvmfPkg/build.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/OvmfPkg/build.sh b/OvmfPkg/build.sh +index 279f0d099a..285f061bf4 100755 +--- a/OvmfPkg/build.sh ++++ b/OvmfPkg/build.sh +@@ -24,7 +24,7 @@ then + # this assumes svn pulls have the same root dir + # export EDK_TOOLS_PATH=`pwd`/../BaseTools + # This version is for the tools source in edk2 +- export EDK_TOOLS_PATH=`pwd`/BaseTools ++ export EDK_TOOLS_PATH=BBAKE_EDK_TOOLS_PATH/BaseTools + echo $EDK_TOOLS_PATH + source edksetup.sh BaseTools + else +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch new file mode 100644 index 000000000..16db65bf4 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch @@ -0,0 +1,59 @@ +From a8f873986e7d817d6f0db2824c2348e5120a1094 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 4 Nov 2024 04:27:56 +0000 +Subject: [PATCH 2/5] BaseTools makefile adjust to build in under bitbake + +--- + BaseTools/Source/C/Makefiles/header.makefile | 15 +++++++-------- + 1 file changed, 7 insertions(+), 8 deletions(-) + +diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile +index d369908a09..22c670f316 100644 +--- a/BaseTools/Source/C/Makefiles/header.makefile ++++ b/BaseTools/Source/C/Makefiles/header.makefile +@@ -85,35 +85,34 @@ endif + + INCLUDE = $(TOOL_INCLUDE) -I $(MAKEROOT) -I $(MAKEROOT)/Include/Common -I $(MAKEROOT)/Include/ -I $(MAKEROOT)/Include/IndustryStandard -I $(MAKEROOT)/Common/ -I .. -I . $(ARCH_INCLUDE) + INCLUDE += -I $(EDK2_PATH)/MdePkg/Include +-CPPFLAGS = $(INCLUDE) ++CPPFLAGS += $(INCLUDE) + + # keep EXTRA_OPTFLAGS last + BUILD_OPTFLAGS = -O2 $(EXTRA_OPTFLAGS) + + ifeq ($(DARWIN),Darwin) + # assume clang or clang compatible flags on OS X +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -Wall -Werror \ + -Wno-deprecated-declarations -Wno-self-assign -Wno-unused-result -nostdlib -g + else + ifneq ($(CLANG),) +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ + -fno-delete-null-pointer-checks -Wall -Werror \ + -Wno-deprecated-declarations -Wno-self-assign \ + -Wno-unused-result -nostdlib -g + else +-CFLAGS = -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ ++CFLAGS += -MD -fshort-wchar -fno-strict-aliasing -fwrapv \ + -fno-delete-null-pointer-checks -Wall -Werror \ + -Wno-deprecated-declarations -Wno-stringop-truncation -Wno-restrict \ + -Wno-unused-result -nostdlib -g + endif + endif + ifneq ($(CLANG),) +-LDFLAGS = +-CXXFLAGS = -Wno-deprecated-register -Wno-unused-result -std=c++14 ++CXXFLAGS += -Wno-deprecated-register -Wno-unused-result -std=c++14 + else +-LDFLAGS = +-CXXFLAGS = -Wno-unused-result ++CXXFLAGS += -Wno-unused-result + endif ++ + ifeq ($(HOST_ARCH), IA32) + # + # Snow Leopard is a 32-bit and 64-bit environment. uname -m returns i386, but gcc defaults +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch new file mode 100644 index 000000000..a793d63f7 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch @@ -0,0 +1,90 @@ +From aa8d288279ef96ffe576a0e434c1d9f435fef1e7 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 4 Nov 2024 04:29:03 +0000 +Subject: [PATCH 3/5] Debug prefix map + +--- + BaseTools/Conf/tools_def.template | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template +index 0f110fbb4a..35205db748 100755 +--- a/BaseTools/Conf/tools_def.template ++++ b/BaseTools/Conf/tools_def.template +@@ -896,7 +896,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ + *_*_*_DTC_PATH = DEF(DTC_BIN) + + # All supported GCC archs except LOONGARCH64 support -mstack-protector-guard=global, so set that on everything except LOONGARCH64 +-DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector ++DEFINE GCC_ALL_CC_FLAGS = -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing -Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -fstack-protector ENV(GCC_PREFIX_MAP) + DEFINE GCC_IA32_X64_CC_FLAGS = -mstack-protector-guard=global + DEFINE GCC_ARM_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mlittle-endian -mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections -fdata-sections -fomit-frame-pointer -Wno-address -mthumb -fno-pic -fno-pie -mstack-protector-guard=global + DEFINE GCC_LOONGARCH64_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mabi=lp64d -fno-asynchronous-unwind-tables -Wno-address -fno-short-enums -fsigned-char -ffunction-sections -fdata-sections +@@ -918,8 +918,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere + DEFINE GCC_AARCH64_ASLDLINK_FLAGS = DEF(GCC_AARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) + DEFINE GCC_LOONGARCH64_ASLDLINK_FLAGS = DEF(GCC_LOONGARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) + DEFINE GCC_IA32_X64_DLINK_FLAGS = DEF(GCC_IA32_X64_DLINK_COMMON) --entry _$(IMAGE_ENTRY_POINT) --file-alignment 0x20 --section-alignment 0x20 -Map $(DEST_DIR_DEBUG)/$(BASE_NAME).map +-DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h +-DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ++DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) ++DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) + DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h + DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h + DEFINE GCC_ASLCC_FLAGS = -x c +@@ -1072,7 +1072,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( + *_GCC48_IA32_DLINK2_FLAGS = DEF(GCC48_IA32_DLINK2_FLAGS) + *_GCC48_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC48_IA32_OBJCOPY_FLAGS = +-*_GCC48_IA32_NASM_FLAGS = -f elf32 ++*_GCC48_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) + RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set-variable +@@ -1100,7 +1100,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set + *_GCC48_X64_DLINK2_FLAGS = DEF(GCC48_X64_DLINK2_FLAGS) + *_GCC48_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC48_X64_OBJCOPY_FLAGS = +-*_GCC48_X64_NASM_FLAGS = -f elf64 ++*_GCC48_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) + RELEASE_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) -Wno-unused-but-set-variable +@@ -1209,7 +1209,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s + *_GCC49_IA32_DLINK2_FLAGS = DEF(GCC49_IA32_DLINK2_FLAGS) + *_GCC49_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC49_IA32_OBJCOPY_FLAGS = +-*_GCC49_IA32_NASM_FLAGS = -f elf32 ++*_GCC49_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) + RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable +@@ -1237,7 +1237,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set + *_GCC49_X64_DLINK2_FLAGS = DEF(GCC49_X64_DLINK2_FLAGS) + *_GCC49_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC49_X64_OBJCOPY_FLAGS = +-*_GCC49_X64_NASM_FLAGS = -f elf64 ++*_GCC49_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) + RELEASE_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable +@@ -1496,7 +1496,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 + *_GCC5_IA32_DLINK2_FLAGS = DEF(GCC5_IA32_DLINK2_FLAGS) -no-pie + *_GCC5_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) + *_GCC5_IA32_OBJCOPY_FLAGS = +-*_GCC5_IA32_NASM_FLAGS = -f elf32 ++*_GCC5_IA32_NASM_FLAGS = -f elf32 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC5_IA32_CC_FLAGS = DEF(GCC5_IA32_CC_FLAGS) -flto + DEBUG_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl,-m,elf_i386,--oformat=elf32-i386 +@@ -1528,7 +1528,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, + *_GCC5_X64_DLINK2_FLAGS = DEF(GCC5_X64_DLINK2_FLAGS) + *_GCC5_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) + *_GCC5_X64_OBJCOPY_FLAGS = +-*_GCC5_X64_NASM_FLAGS = -f elf64 ++*_GCC5_X64_NASM_FLAGS = -f elf64 ENV(NASM_PREFIX_MAP) + + DEBUG_GCC5_X64_CC_FLAGS = DEF(GCC5_X64_CC_FLAGS) -flto -DUSING_LTO + DEBUG_GCC5_X64_DLINK_FLAGS = DEF(GCC5_X64_DLINK_FLAGS) -flto -Os +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch new file mode 100644 index 000000000..7b73762f8 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch @@ -0,0 +1,156 @@ +From 81fc61513cf51a9ef6947dcf4bc1388a50c13f42 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 4 Nov 2024 04:29:41 +0000 +Subject: [PATCH 4/5] Reproduciable + +--- + BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- + .../Source/Python/AutoGen/BuildEngine.py | 3 +++ + BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- + .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- + 4 files changed, 24 insertions(+), 16 deletions(-) + +diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c +index 9d04fc612e..83fd6c9c05 100644 +--- a/BaseTools/Source/C/GenFw/Elf64Convert.c ++++ b/BaseTools/Source/C/GenFw/Elf64Convert.c +@@ -13,6 +13,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent + #ifndef __GNUC__ + #include + #include ++#else ++#define _GNU_SOURCE + #endif + #include + #include +@@ -988,7 +990,7 @@ ScanSections64 ( + } + mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + + sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + +- strlen(mInImageName) + 1; ++ strlen(basename(mInImageName)) + 1; + + // + // Add more space in the .debug data region for the DllCharacteristicsEx +@@ -2299,7 +2301,7 @@ WriteDebug64 ( + EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; + EFI_IMAGE_DEBUG_EX_DLLCHARACTERISTICS_ENTRY *DllEntry; + +- Len = strlen(mInImageName) + 1; ++ Len = strlen(basename(mInImageName)) + 1; + + NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); + DataDir = &NtHdr->Pe32Plus.OptionalHeader.DataDirectory[EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]; +@@ -2332,7 +2334,7 @@ WriteDebug64 ( + + Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); + Nb10->Signature = CODEVIEW_SIGNATURE_NB10; +- strcpy ((char *)(Nb10 + 1), mInImageName); ++ strcpy ((char *)(Nb10 + 1), basename(mInImageName)); + } + + STATIC +diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py +index 45b39d7878..3fed7d1736 100644 +--- a/BaseTools/Source/Python/AutoGen/BuildEngine.py ++++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py +@@ -70,6 +70,9 @@ class TargetDescBlock(object): + else: + return str(Other) == self.Target.Path + ++ def __lt__(self, other): ++ return str(self) < str(other) ++ + def AddInput(self, Input): + if Input not in self.Inputs: + self.Inputs.append(Input) +diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py +index 6d9c60b702..83ac267647 100755 +--- a/BaseTools/Source/Python/AutoGen/GenMake.py ++++ b/BaseTools/Source/Python/AutoGen/GenMake.py +@@ -576,7 +576,7 @@ cleanlib: + os.remove(RespFileList) + + # convert source files and binary files to build targets +- self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList] ++ self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList]) + if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0: + EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", + ExtraData="[%s]" % str(MyAgo)) +@@ -727,7 +727,7 @@ cleanlib: + OutputFile = '' + DepsFileList = [] + +- for Cmd in self.GenFfsList: ++ for Cmd in sorted(self.GenFfsList): + if Cmd[2]: + for CopyCmd in Cmd[2]: + Src, Dst = CopyCmd +@@ -760,7 +760,7 @@ cleanlib: + self.BuildTargetList.append('\t%s' % CmdString) + + self.ParseSecCmd(DepsFileList, Cmd[1]) +- for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList : ++ for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList): + self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile))) + self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd)) + self.FfsOutputFileList = [] +@@ -799,13 +799,13 @@ cleanlib: + + def CommandExceedLimit(self): + FlagDict = { +- 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False}, +- 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False}, +- 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False}, +- 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False}, +- 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False}, +- 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False}, +- 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False}, ++ 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : True}, ++ 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : True}, ++ 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : True}, ++ 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True}, ++ 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True}, ++ 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : True}, ++ 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True}, + } + + RespDict = {} +@@ -1008,9 +1008,9 @@ cleanlib: + if not self.ObjTargetDict.get(T.Target.SubDir): + self.ObjTargetDict[T.Target.SubDir] = set() + self.ObjTargetDict[T.Target.SubDir].add(NewFile) +- for Type in self._AutoGenObject.Targets: ++ for Type in sorted(self._AutoGenObject.Targets): + resp_file_number = 0 +- for T in self._AutoGenObject.Targets[Type]: ++ for T in sorted(self._AutoGenObject.Targets[Type]): + # Generate related macros if needed + if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: + self.FileListMacros[T.FileListMacro] = [] +diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +index 65a2176ca9..318b9611ce 100755 +--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py ++++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +@@ -1474,6 +1474,9 @@ class ModuleAutoGen(AutoGen): + for File in Files: + if File.lower().endswith('.pdb'): + AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) ++ ++ AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item']) ++ + HeaderComments = self.Module.HeaderComments + StartPos = 0 + for Index in range(len(HeaderComments)): +@@ -1749,7 +1752,7 @@ class ModuleAutoGen(AutoGen): + if os.path.exists (self.TimeStampPath): + os.remove (self.TimeStampPath) + +- SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) ++ SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False) + + # Ignore generating makefile when it is a binary module + if self.IsBinaryModule: +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch new file mode 100644 index 000000000..523b169ec --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch @@ -0,0 +1,24 @@ +From 5c718d0c37c65e845b100060fbc08cae26412c35 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Mon, 4 Nov 2024 04:35:18 +0000 +Subject: [PATCH 5/5] Declare ProcessLibraryConstructorList + +--- + OvmfPkg/Sec/SecMain.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c +index d13a948509..2a44c4ed24 100644 +--- a/OvmfPkg/Sec/SecMain.c ++++ b/OvmfPkg/Sec/SecMain.c +@@ -885,6 +885,7 @@ SecCoreStartupWithStack ( + InitializeCpuExceptionHandlers (NULL); + } + ++ void ProcessLibraryConstructorList (void); + ProcessLibraryConstructorList (); + + if (!SevEsIsEnabled ()) { +-- +2.43.0 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index bb9ecdec7..b073af504 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -20,7 +20,11 @@ PACKAGECONFIG[tpm] = "-D TPM_ENABLE=TRUE,-D TPM_ENABLE=FALSE,," BUILD_CFLAGS += "-Wno-error=stringop-overflow" SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ - file://0001-Patch-for-dstack-yocto.patch \ + file://0001-Update-path-to-native-BaseTools.patch \ + file://0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch \ + file://0003-Debug-prefix-map.patch \ + file://0004-Reproduciable.patch \ + file://0005-Declare-ProcessLibraryConstructorList.patch \ " PV = "edk2-3a3b12cb" From 958ba7bc3bd5bb877c73122bf92fb88140ebc274 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 08:29:25 +0000 Subject: [PATCH 044/435] Switch to app-compose.json --- .../recipes-core/dstack-guest/files/tboot.sh | 86 ++++++++++++------- .../images/dstack-initramfs-files/boot-vars | 2 +- scripts/calc_rtmr3.py | 2 +- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh index 4e237995f..314af89e6 100755 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh @@ -1,45 +1,50 @@ #!/bin/sh set -e -CLIENT_PRIVATE_KEY=$(wg genkey) -CLIENT_PUBLIC_KEY=$(echo $CLIENT_PRIVATE_KEY | wg pubkey) +APP_COMPOSE_FILE=/tapp/app-compose.json -TPROXY_URL=$(jq -r '.tproxy_url' /tapp/config.json) +prepare_docker_compose() { + local runner=$(jq -r '.runner' $APP_COMPOSE_FILE) + if [ "$runner" = "docker-compose" ]; then + jq -r .docker_compose_file $APP_COMPOSE_FILE > /tapp/docker-compose.yaml + else + echo "Unsupported runner: $runner" + exit 1 + fi +} -mkdir -p /etc/tappd -cp /tapp/certs/ca.cert /etc/tappd/ca.cert -jq -r '.app_key' /tapp/appkeys.json > /etc/tappd/app-ca.key -jq -r '.certificate_chain[]' /tapp/appkeys.json | awk 'NF {print $0 > "/etc/tappd/app-ca.cert"}' +setup_tproxy_net() { + local FEATURES=$(jq -r '.features[]' $APP_COMPOSE_FILE) + if echo "$FEATURES" | grep -q "tproxy-net"; then + return + fi -tdxctl gen-ra-cert \ - --ca-key /etc/tappd/app-ca.key \ - --ca-cert /etc/tappd/app-ca.cert \ - --cert-path /etc/tappd/tls.cert \ - --key-path /etc/tappd/tls.key + local CLIENT_PRIVATE_KEY=$(wg genkey) + local CLIENT_PUBLIC_KEY=$(echo $CLIENT_PRIVATE_KEY | wg pubkey) -cat /etc/tappd/app-ca.cert >> /etc/tappd/tls.cert + local TPROXY_URL=$(jq -r '.tproxy_url' /tapp/config.json) -curl ${TPROXY_URL}/prpc/Tproxy.RegisterCvm?json \ - --cacert /etc/tappd/ca.cert \ - --cert /etc/tappd/tls.cert \ - --key /etc/tappd/tls.key \ - -d"{\"client_public_key\":\"${CLIENT_PUBLIC_KEY}\"}" \ - -o /tmp/wginfo.json + curl ${TPROXY_URL}/prpc/Tproxy.RegisterCvm?json \ + --cacert /etc/tappd/ca.cert \ + --cert /etc/tappd/tls.cert \ + --key /etc/tappd/tls.key \ + -d"{\"client_public_key\":\"${CLIENT_PUBLIC_KEY}\"}" \ + -o /tmp/wginfo.json -CLIENT_IP=$(jq -r '.wg.client_ip' /tmp/wginfo.json) -SERVER_ENDPOINT=$(jq -r '.wg.server_endpoint' /tmp/wginfo.json) -SERVER_PUBLIC_KEY=$(jq -r '.wg.server_public_key' /tmp/wginfo.json) -SERVER_IP=$(jq -r '.wg.server_ip' /tmp/wginfo.json) + local CLIENT_IP=$(jq -r '.wg.client_ip' /tmp/wginfo.json) + local SERVER_ENDPOINT=$(jq -r '.wg.server_endpoint' /tmp/wginfo.json) + local SERVER_PUBLIC_KEY=$(jq -r '.wg.server_public_key' /tmp/wginfo.json) + local SERVER_IP=$(jq -r '.wg.server_ip' /tmp/wginfo.json) -echo "WG CLIENT_IP: ${CLIENT_IP}" -echo "WG SERVER_ENDPOINT: ${SERVER_ENDPOINT}" -echo "WG SERVER_PUBLIC_KEY: ${SERVER_PUBLIC_KEY}" -echo "WG SERVER_IP: ${SERVER_IP}" + echo "WG CLIENT_IP: ${CLIENT_IP}" + echo "WG SERVER_ENDPOINT: ${SERVER_ENDPOINT}" + echo "WG SERVER_PUBLIC_KEY: ${SERVER_PUBLIC_KEY}" + echo "WG SERVER_IP: ${SERVER_IP}" -rm -f /tmp/wginfo.json + rm -f /tmp/wginfo.json -mkdir -p /etc/wireguard -cat < /etc/wireguard/wg0.conf + mkdir -p /etc/wireguard + cat < /etc/wireguard/wg0.conf [Interface] PrivateKey = ${CLIENT_PRIVATE_KEY} Address = ${CLIENT_IP}/24 @@ -50,5 +55,24 @@ AllowedIPs = ${SERVER_IP}/24 Endpoint = ${SERVER_ENDPOINT} PersistentKeepalive = 25 EOF + wg-quick up wg0 +} -wg-quick up wg0 \ No newline at end of file +prepare_certs() { + mkdir -p /etc/tappd + cp /tapp/certs/ca.cert /etc/tappd/ca.cert + jq -r '.app_key' /tapp/appkeys.json > /etc/tappd/app-ca.key + jq -r '.certificate_chain[]' /tapp/appkeys.json | awk 'NF {print $0 > "/etc/tappd/app-ca.cert"}' + + tdxctl gen-ra-cert \ + --ca-key /etc/tappd/app-ca.key \ + --ca-cert /etc/tappd/app-ca.cert \ + --cert-path /etc/tappd/tls.cert \ + --key-path /etc/tappd/tls.key + + cat /etc/tappd/app-ca.cert >> /etc/tappd/tls.cert +} + +prepare_certs && \ + setup_tproxy_net && \ + prepare_docker_compose diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars index 30d33ded4..5ccfe8725 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars @@ -9,7 +9,7 @@ TMP_CA_CERT=${CFG_DIR}/certs/tmp-ca.cert TMP_CA_KEY=${CFG_DIR}/certs/tmp-ca.key CA_CERT=${CFG_DIR}/certs/ca.cert CFG_FILE=${CFG_DIR}/config.json -APP_COMPOSE_FILE=${CFG_DIR}/docker-compose.yaml +APP_COMPOSE_FILE=${CFG_DIR}/app-compose.json HOST_SHARED=/mnt/host-shared # Parse kernel cmdline into vars diff --git a/scripts/calc_rtmr3.py b/scripts/calc_rtmr3.py index 0f7a5bfa9..6c54053c1 100644 --- a/scripts/calc_rtmr3.py +++ b/scripts/calc_rtmr3.py @@ -82,7 +82,7 @@ def calc_rtmr3(rootfs_hash: str, app_id: str, ca_cert_hash: str): image_dir = args.images_dir / vm_config["image"] image_metadata = json.load(open(image_dir / "metadata.json", "r")) rootfs_hash = image_metadata["rootfs_hash"] - compose_file = args.vm_dir / "shared" / "docker-compose.yaml" + compose_file = args.vm_dir / "shared" / "app-compose.json" ca_cert_file = args.vm_dir / "shared" / "certs" / "ca.cert" app_id = hashlib.sha256(open(compose_file, "rb").read()).hexdigest() ca_cert_hash = hashlib.sha256(open(ca_cert_file, "rb").read()).hexdigest() From 30f027fc5d00dae4b66b72c09f490ab47a96a0f4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 6 Nov 2024 05:40:20 +0000 Subject: [PATCH 045/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 2c512d764..79099b187 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 2c512d764bb049ee0b81a5acda59fea58edf4150 +Subproject commit 79099b1873389ac1541876f39e1b5b88cd6459bf From 9d6ef7caf91a8278d54997c582f8784388e84a03 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 6 Nov 2024 14:35:40 +0000 Subject: [PATCH 046/435] Fix tproxy setup in tboot.sh --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/files/tboot.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 79099b187..f3f584fcc 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 79099b1873389ac1541876f39e1b5b88cd6459bf +Subproject commit f3f584fccb9fa174cb42fa2a93486afcc28fe8a9 diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh index 314af89e6..df9f4e9e4 100755 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh +++ b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh @@ -15,7 +15,8 @@ prepare_docker_compose() { setup_tproxy_net() { local FEATURES=$(jq -r '.features[]' $APP_COMPOSE_FILE) - if echo "$FEATURES" | grep -q "tproxy-net"; then + if ! echo "$FEATURES" | grep -q "tproxy-net"; then + echo "tproxy is not enabled" return fi From d48a177d35d43adbac33ec760e66dc96672d1a1f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 8 Nov 2024 02:36:10 +0000 Subject: [PATCH 047/435] Add instance_id for vm instance --- dstack | 2 +- .../images/dstack-initramfs-files/kmfs-setup | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dstack b/dstack index f3f584fcc..7dc9c5bcd 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f3f584fccb9fa174cb42fa2a93486afcc28fe8a9 +Subproject commit 7dc9c5bcde6c9e3a94a501ae31967f2a433d754b diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 5e1133d5a..804930178 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -15,9 +15,8 @@ fde_prepare() { mount -t 9p -o trans=virtio,version=9p2000.L host-shared ${HOST_SHARED} || panic "FSINIT: Failed to mount host-shared" mkdir -p ${CFG_DIR} || panic "FSINIT: Failed to create ${CFG_DIR} directory" - cp -r ${HOST_SHARED}/* ${CFG_DIR}/ || panic "FSINIT: Failed to copy config files to ${CFG_DIR}" + cp -r ${HOST_SHARED}/* ${HOST_SHARED}/.* ${CFG_DIR}/ || panic "FSINIT: Failed to copy config files to ${CFG_DIR}" - # Calculate rootfs and app ids and extend them to RTMR if [ ! -f "${CFG_FILE}" ]; then panic "FSINIT: Config file ${CFG_FILE} does not exist" fi @@ -30,12 +29,15 @@ fde_prepare() { kms_url=$(jq -r .kms_url ${CFG_FILE}) ca_cert_hash=$(iohash sha256 -o! -t- -i ${CA_CERT}) upgraded_app_id=$(iohash sha256 -o! -t- -i ${APP_COMPOSE_FILE}) - if [ -f "${HOST_SHARED}/.app_id" ]; then - app_id=$(cat ${HOST_SHARED}/.app_id) + if [ -f "${CFG_DIR}/.bootstraped" ]; then + app_id=$(jq -r .app_id ${CFG_DIR}/.instance_info) + instance_id=$(jq -r .instance_id ${CFG_DIR}/.instance_info) else app_id=${upgraded_app_id} + instance_id=$(tdxctl rand -x -n 20) fi + # Calculate rootfs and app ids and extend them to RTMR echo "FSINIT: Extending rootfs hash to RTMR, hash=${rootfs_hash}" tdxctl extend -i 3 --digest ${rootfs_hash} --associated-data "rootfs-hash" || panic "FSINIT: Failed to extend rootfs to RTMR" echo "FSINIT: Extending app id to RTMR, app_id=${app_id}" @@ -44,6 +46,8 @@ fde_prepare() { tdxctl extend -i 3 --digest ${upgraded_app_id} --associated-data "upgraded-app-id" || panic "FSINIT: Failed to extend upgraded app id to RTMR" echo "FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=${ca_cert_hash}" tdxctl extend -i 3 --digest ${ca_cert_hash} --associated-data "ca-cert-hash" || panic "FSINIT: Failed to extend ca cert hash to RTMR" + echo "FSINIT: Extending instance id to RTMR, instance_id=${instance_id}" + tdxctl extend -i 3 --digest ${instance_id} --associated-data "instance-id" || panic "FSINIT: Failed to extend instance id to RTMR" # Show the RTMR tdxctl show @@ -72,7 +76,7 @@ fde_prepare() { panic "FSINIT: Failed to get valid key phrase from KMS" fi - if [ -e ${HOST_SHARED}/.bootstraped ]; then + if [ -e ${CFG_DIR}/.bootstraped ]; then echo "[$(date +%H:%M:%S)] FSINIT: Mounting" echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" @@ -129,9 +133,14 @@ fde_prepare() { fi cp -rT ${CFG_DIR} /root/tapp || panic "FSINIT: Failed to copy config to /root/tapp" - touch /root/.bootstraped || echo "FSINIT: Failed to touch bootstraped" - touch ${HOST_SHARED}/.bootstraped || echo "FSINIT: Failed to touch bootstraped" - echo ${app_id} > ${HOST_SHARED}/.app_id || echo "FSINIT: Failed to set app id" + cat <${HOST_SHARED}/.instance_info || panic "FSINIT: failed to write instance_info" + { + "app_id": "$app_id", + "instance_id": "$instance_id" + } +EOF + touch /root/.bootstraped || panic "FSINIT: Failed to touch bootstraped" + touch ${HOST_SHARED}/.bootstraped || panic "FSINIT: Failed to touch bootstraped" fi cp ${KMFS_WORK_DIR}/appkeys.json /root/tapp/appkeys.json || panic "FSINIT: Failed to copy appkeys.json to /root/tapp" From f03a75a45f0a7efd5211a7e76990f0ed7a9ba5ef Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 15 Nov 2024 03:32:01 +0000 Subject: [PATCH 048/435] Support for running CVM without KMS --- Makefile | 3 + dstack | 2 +- .../images/dstack-initramfs-files/boot-vars | 14 -- .../images/dstack-initramfs-files/init | 2 +- .../images/dstack-initramfs-files/kmfs-setup | 178 ++++-------------- 5 files changed, 38 insertions(+), 161 deletions(-) diff --git a/Makefile b/Makefile index 11fd5f3b0..f362e890d 100644 --- a/Makefile +++ b/Makefile @@ -41,3 +41,6 @@ clean: clean-dstack: bitbake -c cleansstate dstack-guest dstack-rootfs dstack-rootfs-dev + +clean-initrd: + bitbake -c cleansstate dstack-initramfs diff --git a/dstack b/dstack index 7dc9c5bcd..584c2a7f2 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 7dc9c5bcde6c9e3a94a501ae31967f2a433d754b +Subproject commit 584c2a7f27230cf3dae04c6fb0e3b89daf373946 diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars index 5ccfe8725..e55ed36a4 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars @@ -1,17 +1,3 @@ - -ROOT=/dev/vda -ROOTINTEGRITY= -INITIMG=/dev/sr0 -INITDATA_DIR=/initdata -KMFS_WORK_DIR=/kmfs -CFG_DIR=${KMFS_WORK_DIR}/config -TMP_CA_CERT=${CFG_DIR}/certs/tmp-ca.cert -TMP_CA_KEY=${CFG_DIR}/certs/tmp-ca.key -CA_CERT=${CFG_DIR}/certs/ca.cert -CFG_FILE=${CFG_DIR}/config.json -APP_COMPOSE_FILE=${CFG_DIR}/app-compose.json -HOST_SHARED=/mnt/host-shared - # Parse kernel cmdline into vars for p in `cat /proc/cmdline`; do if [ -n "$quoted" ]; then diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index 77fd626bb..3eabacd70 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -1,7 +1,7 @@ #!/bin/sh set -e -PATH=/sbin:/bin:/usr/sbin:/usr/bin +export PATH=/sbin:/bin:/usr/sbin:/usr/bin mkdir -p /proc mkdir -p /sys diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 804930178..2edede332 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -1,151 +1,39 @@ -#!/bin/sh -set -e -panic() { - echo "$1" - exit 1 -} - -fde_prepare() { - local key_phrase rootfs_hash app_id kms_url ca_cert_hash - local actual_rootfs_hash - - # Mount host-shared - mkdir -p ${HOST_SHARED} || panic "FSINIT: Failed to create ${HOST_SHARED} directory" - mount -t 9p -o trans=virtio,version=9p2000.L host-shared ${HOST_SHARED} || panic "FSINIT: Failed to mount host-shared" - - mkdir -p ${CFG_DIR} || panic "FSINIT: Failed to create ${CFG_DIR} directory" - cp -r ${HOST_SHARED}/* ${HOST_SHARED}/.* ${CFG_DIR}/ || panic "FSINIT: Failed to copy config files to ${CFG_DIR}" - - if [ ! -f "${CFG_FILE}" ]; then - panic "FSINIT: Config file ${CFG_FILE} does not exist" - fi - - if [ ! -f "${APP_COMPOSE_FILE}" ]; then - panic "FSINIT: Docker compose file ${APP_COMPOSE_FILE} does not exist" - fi - - rootfs_hash=$(jq -r .rootfs_hash ${CFG_FILE}) - kms_url=$(jq -r .kms_url ${CFG_FILE}) - ca_cert_hash=$(iohash sha256 -o! -t- -i ${CA_CERT}) - upgraded_app_id=$(iohash sha256 -o! -t- -i ${APP_COMPOSE_FILE}) - if [ -f "${CFG_DIR}/.bootstraped" ]; then - app_id=$(jq -r .app_id ${CFG_DIR}/.instance_info) - instance_id=$(jq -r .instance_id ${CFG_DIR}/.instance_info) - else - app_id=${upgraded_app_id} - instance_id=$(tdxctl rand -x -n 20) - fi - - # Calculate rootfs and app ids and extend them to RTMR - echo "FSINIT: Extending rootfs hash to RTMR, hash=${rootfs_hash}" - tdxctl extend -i 3 --digest ${rootfs_hash} --associated-data "rootfs-hash" || panic "FSINIT: Failed to extend rootfs to RTMR" - echo "FSINIT: Extending app id to RTMR, app_id=${app_id}" - tdxctl extend -i 3 --digest ${app_id} --associated-data "app-id" || panic "FSINIT: Failed to extend app id to RTMR" - echo "FSINIT: Extending upgraded app id to RTMR, upgraded_app_id=${upgraded_app_id}" - tdxctl extend -i 3 --digest ${upgraded_app_id} --associated-data "upgraded-app-id" || panic "FSINIT: Failed to extend upgraded app id to RTMR" - echo "FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=${ca_cert_hash}" - tdxctl extend -i 3 --digest ${ca_cert_hash} --associated-data "ca-cert-hash" || panic "FSINIT: Failed to extend ca cert hash to RTMR" - echo "FSINIT: Extending instance id to RTMR, instance_id=${instance_id}" - tdxctl extend -i 3 --digest ${instance_id} --associated-data "instance-id" || panic "FSINIT: Failed to extend instance id to RTMR" - - # Show the RTMR - tdxctl show +if [ x"${kargs_dstack_integrity}" = x"1" ]; then + ROOTINTEGRITY="--rootfs-integrity" +fi - # Generate temp cert for RA-TLS - echo "FSINIT: Generating RA-TLS cert" - mkdir -p ${KMFS_WORK_DIR}/certs - tdxctl gen-ra-cert \ - -c ${KMFS_WORK_DIR}/certs/cert.pem \ - -k ${KMFS_WORK_DIR}/certs/key.pem \ - --ca-cert ${TMP_CA_CERT} \ - --ca-key ${TMP_CA_KEY} || panic "FSINIT: Failed to generate RA-TLS cert" +ROOTFS_CD=/dev/sr0 +ROOTFS_CD_MNT=/mnt/rootfs-cdrom +ROOTFS_HD=/dev/vda +ROOTFS_DIR=/root - # Request app keys from KMS - echo "FSINIT: Requesting app keys from KMS (${kms_url})" - curl --cacert ${CA_CERT} \ - --cert ${KMFS_WORK_DIR}/certs/cert.pem \ - --key ${KMFS_WORK_DIR}/certs/key.pem \ - -o ${KMFS_WORK_DIR}/appkeys.json \ - ${kms_url}/prpc/KMS.GetAppKey \ - || panic "FSINIT: Failed to request app keys" +KMFS_WORK_DIR=/kmfs +SHARED_COPY=${KMFS_WORK_DIR}/config +HOST_SHARED=/mnt/host-shared - key_phrase=$(jq -r .disk_crypt_key ${KMFS_WORK_DIR}/appkeys.json) - # Check if key_phrase is empty, null, or not set - if [ -z "$key_phrase" ] || [ "$key_phrase" = "null" ]; then - panic "FSINIT: Failed to get valid key phrase from KMS" - fi - - if [ -e ${CFG_DIR}/.bootstraped ]; then - echo "[$(date +%H:%M:%S)] FSINIT: Mounting" - echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" - mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" - if [ ! -e /root/.bootstraped ]; then - # Check if the rootfs has been properly bootstrapped by looking for the .bootstraped file - # This file is created at the end of the initial setup process - # If it doesn't exist, the rootfs was not properly initialized - panic "FSINIT: Encrypted rootfs is not bootstrapped" - fi - else - # Set up disk encryption and format the disk - - [ -z "${INITIMG}" ] && panic "FSINIT: initimg not set" - - echo "FSINIT: Mounting init image ${INITIMG}" - mkdir -p ${INITDATA_DIR} || panic "FSINIT: Failed to create ${INITDATA_DIR}" - - if [ -z "${INITIMGTYPE}" ]; then - case "${INITIMG}" in - /dev/cdrom*|/dev/sr*) - INITIMGTYPE="-t iso9660" - ;; - esac - fi - mount ${INITIMGTYPE} -o ro ${INITIMG} ${INITDATA_DIR} || panic "FSINIT: Failed to mount init image" - - echo "FSINIT: Initializing" - echo "[$(date +%H:%M:%S)] FSINIT: Formatting rootfs" - echo "$key_phrase" | cryptsetup luksFormat --type luks2 --cipher aes-xts-plain64 ${ROOTINTEGRITY} --pbkdf pbkdf2 -d- ${ROOT} || panic "FSINIT: Failed to format rootfs" - echo "[$(date +%H:%M:%S)] FSINIT: Formatting rootfs done" - echo "$key_phrase" | cryptsetup luksOpen --type luks2 -d- ${ROOT} rootfs_crypt || panic "FSINIT: Failed to open encrypted rootfs" - mkfs.ext4 -L cloudimg-rootfs /dev/mapper/rootfs_crypt || panic "FSINIT: Failed to create ext4 filesystem" - mount /dev/mapper/rootfs_crypt /root || panic "FSINIT: Failed to mount rootfs" - - echo "[$(date +%H:%M:%S)] FSINIT: Extracting rootfs, this may take a while..." - extract_rootfs() { - iohash sha256 -t /tmp/rootfs.hash | cpio -i - } - if [ -e ${INITDATA_DIR}/rootfs.cpio ]; then - (cd /root && cat ${INITDATA_DIR}/rootfs.cpio | extract_rootfs) || panic "FSINIT: Failed to extract rootfs.cpio" - else - panic "FSINIT: No rootfs found" - fi - - actual_rootfs_hash=$(cat /tmp/rootfs.hash) - if [ -z "$actual_rootfs_hash" ]; then - panic "FSINIT: Rootfs hash is empty" - fi - if [ $actual_rootfs_hash != $rootfs_hash ]; then - echo "FSINIT: Rootfs hash mismatch" - echo "Expected: $rootfs_hash" - echo "Actual: $actual_rootfs_hash" - panic "FSINIT: Rootfs hash mismatch" - fi - - cp -rT ${CFG_DIR} /root/tapp || panic "FSINIT: Failed to copy config to /root/tapp" - cat <${HOST_SHARED}/.instance_info || panic "FSINIT: failed to write instance_info" - { - "app_id": "$app_id", - "instance_id": "$instance_id" - } -EOF - touch /root/.bootstraped || panic "FSINIT: Failed to touch bootstraped" - touch ${HOST_SHARED}/.bootstraped || panic "FSINIT: Failed to touch bootstraped" - fi - - cp ${KMFS_WORK_DIR}/appkeys.json /root/tapp/appkeys.json || panic "FSINIT: Failed to copy appkeys.json to /root/tapp" - cp ${CFG_DIR}/config.json /root/tapp/config.json || panic "FSINIT: Failed to copy config.json to /root/tapp" - echo "FSINIT: Copied config to /root/tapp" +panic() { + echo "$1" + exit 1 } -fde_prepare \ No newline at end of file +echo "Setting up FDE" +echo "FDE setup arguments:" +echo " HOST_SHARED=$HOST_SHARED" +echo " SHARED_COPY=$SHARED_COPY" +echo " KMFS_WORK_DIR=$KMFS_WORK_DIR" +echo " ROOTFS_DIR=$ROOTFS_DIR" +echo " ROOTFS_HD=$ROOTFS_HD" +echo " ROOTFS_CD=$ROOTFS_CD" +echo " ROOTFS_CD_MNT=$ROOTFS_CD_MNT" +echo " ROOTINTEGRITY=$ROOTINTEGRITY" + +tdxctl setup-fde \ + --host-shared $HOST_SHARED \ + --host-shared-copy $SHARED_COPY \ + --work-dir $KMFS_WORK_DIR \ + --rootfs-dir $ROOTFS_DIR \ + --root-hd $ROOTFS_HD \ + --root-cdrom $ROOTFS_CD \ + --root-cdrom-mnt $ROOTFS_CD_MNT \ + $ROOTINTEGRITY || panic "Failed to setup FDE" From dabf6981150fded2ad37d59fdb5ab8606fdec5c6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 18 Nov 2024 04:19:16 +0000 Subject: [PATCH 049/435] Refactor buildscripts --- .gitignore | 1 + Makefile | 15 +- README.md | 13 +- {build => bb-build}/conf/local.conf | 0 build.sh | 332 ++++++++++++++++++ dev-setup | 2 +- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 24 +- .../dstack-guest/files/app-compose.service | 16 - .../dstack-guest/files/llmnr.conf | 2 - .../dstack-guest/files/tappd.init | 87 ----- .../dstack-guest/files/tappd.service | 12 - .../dstack-guest/files/tboot.service | 14 - .../recipes-core/dstack-guest/files/tboot.sh | 79 ----- .../dstack-guest/files/tdx-attest.conf | 1 - run_td.sh | 70 ---- 17 files changed, 348 insertions(+), 324 deletions(-) rename {build => bb-build}/conf/local.conf (100%) create mode 100755 build.sh delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/app-compose.service delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/llmnr.conf delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/tappd.init delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/tappd.service delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/tboot.service delete mode 100755 meta-dstack/recipes-core/dstack-guest/files/tboot.sh delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf delete mode 100755 run_td.sh diff --git a/.gitignore b/.gitignore index 24629c1ff..175bd3f7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /build +/bb-build *.qcow2 diff --git a/Makefile b/Makefile index f362e890d..43ea75b4b 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ endif .PHONY: all dist emu clean clean-dstack images -BUILD_DIR ?= build +BUILD_DIR ?= bb-build BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx DIST_DIR ?= ${BUILD_DIR}/dist @@ -18,23 +18,14 @@ ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}/, ${IMAGE_FILES}) all: dist -dist: $(ABS_IMAGE_FILES) +dist: images DIST_DIR=${DIST_DIR} BUILD_DIR=${BUILD_DIR} ./dist.sh -$(ABS_IMAGE_FILES): - make images - images: bitbake dstack-initramfs dstack-rootfs dstack-rootfs-dev dstack-ovmf -emu: - TD=0 ./run_td.sh - -run: - ./run_td.sh - test: - make images dist run + make images dist clean: git clean -xdff diff --git a/README.md b/README.md index 63d36b869..79d4dbdb9 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,7 @@ This project implements Yocto layer and the overall build scripts for DStack Bas ## Build -```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git --recursive -cd meta-dstack -source dev-setup - -mkdir dstack/build -cd dstack/build - -../build.sh -# Edit the config, and build again -../build.sh -``` +See https://github.com/Dstack-TEE/dstack for more details. ## License diff --git a/build/conf/local.conf b/bb-build/conf/local.conf similarity index 100% rename from build/conf/local.conf rename to bb-build/conf/local.conf diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..50d525de2 --- /dev/null +++ b/build.sh @@ -0,0 +1,332 @@ +#!/bin/bash +SCRIPT_DIR=$(cd $(dirname $0); pwd) +ACTION=$1 + +META_DIR=$SCRIPT_DIR +DSTACK_DIR=$SCRIPT_DIR/dstack +CERTS_DIR=`pwd`/certs +IMAGES_DIR=`pwd`/images +RUN_DIR=`pwd`/run +CERBOT_WORKDIR=$RUN_DIR/certbot +KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry +KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ + +if [ -f ./build-config.sh ]; then + CONFIG_FILE=./build-config.sh +else + CONFIG_FILE=$SCRIPT_DIR/build-config.sh +fi + +cat < build-config.sh.tpl +# base domain of kms rpc and tproxy rpc +# 1022.kvin.wang resolves to 10.0.2.2 which is host ip at the +# cvm point of view +BASE_DOMAIN=1022.kvin.wang + +# kms and tproxy rpc listen port +TEEPOD_RPC_LISTEN_PORT=9080 +# CIDs allocated to VMs start from this number of type unsigned int32 +TEEPOD_CID_POOL_START=10000 +# CID pool size +TEEPOD_CID_POOL_SIZE=1000 +# Whether port mapping is enabled +TEEPOD_PORT_MAPPING_ENABLED=false + +KMS_RPC_LISTEN_PORT=9043 +TPROXY_RPC_LISTEN_PORT=9010 + +TPROXY_WG_INTERFACE=tproxy-$USER +TPROXY_WG_LISTEN_PORT=9182 +TPROXY_WG_IP=10.0.3.1 +TPROXY_WG_CLIENT_IP_RANGE=10.0.3.0/24 +TPROXY_SERVE_PORT=9443 + +BIND_PUBLIC_IP=0.0.0.0 + +TPROXY_PUBLIC_DOMAIN=app.kvin.wang +TPROXY_CERT=/etc/rproxy/certs/cert.pem +TPROXY_KEY=/etc/rproxy/certs/key.pem + +# for certbot +CF_API_TOKEN= +CF_ZONE_ID= +ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory +EOF + +check_config() { + local template_file=$1 + local config_file=$2 + + # extract all variables in template file + local variables=$(grep -oE '^\s*[A-Z_]+=' $template_file | sort) + + # check if each variable is set in config file + local var missing=0 + for var in $variables; do + if ! grep -qE "^\s*$var" $config_file; then + echo "Variable $var is not set in $config_file" + missing=1 + fi + done + if [ $missing -ne 0 ]; then + return 1 + fi + return 0 +} + +if [ -f $CONFIG_FILE ]; then + source $CONFIG_FILE + # check if any variable in build-config.sh.tpl is not set in build-config.sh. + # This might occur if the build-config.sh is generated from and old repo. + check_config build-config.sh.tpl $CONFIG_FILE + if [ $? -ne 0 ]; then + exit 1 + fi + rm -f build-config.sh.tpl +else + mv build-config.sh.tpl build-config.sh + echo "Config file $CONFIG_FILE created, please edit it to configure the build" + exit 1 +fi + + +if [ -z "$TPROXY_SERVE_PORT" ]; then + TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} +fi +TAPPD_PORT=8090 + +TPROXY_WG_KEY=$(wg genkey) +TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) + +# Step 1: build binaries +build_host() { + (cd $DSTACK_DIR && cargo build --release) + cp $DSTACK_DIR/target/release/{tproxy,kms,teepod,certbot,ct_monitor} . +} + +# Step 2: build guest images +make_image_dist() { + local img_name=$1 + local rootfs_name=$2 + local img_dist_dir=$IMAGES_DIR/$img_name + local rootfs_hash + + mkdir -p $img_dist_dir + rootfs_hash=$(sha256sum "$IMAGE_TMP_DIR/$rootfs_name.cpio" | awk '{print $1}') + cat < $img_dist_dir/metadata.json +{ + "bios": "ovmf.fd", + "kernel": "bzImage", + "cmdline": "console=ttyS0 init=/init dstack.integrity=0", + "initrd": "initramfs.cpio.gz", + "rootfs": "rootfs.iso", + "rootfs_hash": "$rootfs_hash" +} +EOF + + cp $IMAGE_TMP_DIR/ovmf.fd $img_dist_dir/ + cp $IMAGE_TMP_DIR/bzImage $img_dist_dir/ + cp $IMAGE_TMP_DIR/initramfs.cpio.gz $img_dist_dir/ + cp $IMAGE_TMP_DIR/$rootfs_name.iso $img_dist_dir/rootfs.iso +} + +build_guest() { + if [ -z "$BBPATH" ]; then + echo 'BBPATH is not set. Run `source dev-setup` in the meta-dstack/ directory' + pushd $SCRIPT_DIR/ + source dev-setup + popd + fi + DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) + IMAGE_NAME=dstack-$DSTACK_VERSION + IMAGE_TMP_DIR=`pwd`/tmp/images/$IMAGE_NAME + + echo "Building $IMAGE_NAME" + make -C $META_DIR dist DIST_DIR=$IMAGE_TMP_DIR + + make_image_dist $IMAGE_NAME rootfs + make_image_dist $IMAGE_NAME-dev rootfs-dev +} + +# Step 3: make certs +build_certs() { + make -C $DSTACK_DIR certs DOMAIN=$BASE_DOMAIN TO=$CERTS_DIR +} + +# Step 4: generate config files + +build_cfg() { + # kms + cat < kms.toml +log_level = "info" +address = "127.0.0.1" +port = $KMS_RPC_LISTEN_PORT + +[tls] +key = "$CERTS_DIR/kms-rpc.key" +certs = "$CERTS_DIR/kms-rpc.cert" + +[tls.mutual] +ca_certs = "$CERTS_DIR/tmp-ca.cert" +mandatory = false + +[core] +root_ca_cert = "$CERTS_DIR/root-ca.cert" +root_ca_key = "$CERTS_DIR/root-ca.key" +subject_postfix = ".phala" +upgrade_registry_dir = "$KMS_UPGRADE_REGISTRY_DIR" +cert_log_dir = "$KMS_CERT_LOG_DIR" + +[core.allowed_mr] +allow_all = true +mrtd = [] +rtmr0 = [] +rtmr1 = [] +rtmr2 = [] +EOF + + # tproxy + cat < tproxy.toml +log_level = "info" +address = "127.0.0.1" +port = $TPROXY_RPC_LISTEN_PORT + +[tls] +key = "$CERTS_DIR/tproxy-rpc.key" +certs = "$CERTS_DIR/tproxy-rpc.cert" + +[tls.mutual] +ca_certs = "$CERTS_DIR/root-ca.cert" +mandatory = false + +[core.certbot] +workdir = "$CERBOT_WORKDIR" + +[core.wg] +private_key = "$TPROXY_WG_KEY" +public_key = "$TPROXY_WG_PUBKEY" +ip = "$TPROXY_WG_IP" +listen_port = $TPROXY_WG_LISTEN_PORT +client_ip_range = "$TPROXY_WG_CLIENT_IP_RANGE" +config_path = "$RUN_DIR/wg.conf" +interface = "$TPROXY_WG_INTERFACE" +endpoint = "10.0.2.2:$TPROXY_WG_LISTEN_PORT" + +[core.proxy] +cert_chain = "$TPROXY_CERT" +cert_key = "$TPROXY_KEY" +base_domain = "$TPROXY_PUBLIC_DOMAIN" +listen_addr = "$BIND_PUBLIC_IP" +listen_port = $TPROXY_SERVE_PORT +tappd_port = $TAPPD_PORT +EOF + + # teepod + cat < teepod.toml +log_level = "info" +address = "127.0.0.1" +port = $TEEPOD_RPC_LISTEN_PORT +image_path = "$IMAGES_DIR" +run_path = "$RUN_DIR/vm" + +[cvm] +ca_cert = "$CERTS_DIR/root-ca.cert" +tmp_ca_cert = "$CERTS_DIR/tmp-ca.cert" +tmp_ca_key = "$CERTS_DIR/tmp-ca.key" +kms_url = "https://kms.$BASE_DOMAIN:$KMS_RPC_LISTEN_PORT" +tproxy_url = "https://tproxy.$BASE_DOMAIN:$TPROXY_RPC_LISTEN_PORT" +cid_start = $TEEPOD_CID_POOL_START +cid_pool_size = $TEEPOD_CID_POOL_SIZE +[cvm.port_mapping] +enabled = $TEEPOD_PORT_MAPPING_ENABLED +address = "127.0.0.1" +range = [ + { protocol = "tcp", from = 1, to = 20000 }, +] + +[gateway] +base_domain = "$TPROXY_PUBLIC_DOMAIN" +port = $TPROXY_SERVE_PORT +tappd_port = $TAPPD_PORT +EOF + + cat < certbot.toml +# Path to the working directory +workdir = "$CERBOT_WORKDIR" +# ACME server URL +acme_url = "$ACME_URL" +# Cloudflare API token +cf_api_token = "$CF_API_TOKEN" +# Cloudflare zone ID +cf_zone_id = "$CF_ZONE_ID" +# Auto set CAA record +auto_set_caa = true +# Domain to issue certificates for +domain = "*.$TPROXY_PUBLIC_DOMAIN" +# Renew interval in seconds +renew_interval = 3600 +# Number of days before expiration to trigger renewal +renew_days_before = 10 +# Renew timeout in seconds +renew_timeout = 120 +EOF + + cat < kms-allow-upgrade.sh +#!/bin/bash +if [ \$# -ne 2 ]; then + echo "Usage: \$0 " + exit 1 +fi +mkdir -p "$KMS_UPGRADE_REGISTRY_DIR/\$1" +touch "$KMS_UPGRADE_REGISTRY_DIR/\$1/\$2" +EOF + chmod +x kms-allow-upgrade.sh + mkdir -p $RUN_DIR + mkdir -p $CERBOT_WORKDIR/backup/preinstalled +} + +build_wg() { + # Step 6: setup wireguard interface + # Check if the WireGuard interface exists + if ! ip link show $TPROXY_WG_INTERFACE &> /dev/null; then + sudo ip link add $TPROXY_WG_INTERFACE type wireguard + sudo ip address add $TPROXY_WG_IP/24 dev $TPROXY_WG_INTERFACE + sudo ip link set $TPROXY_WG_INTERFACE up + echo "created and configured WireGuard interface $TPROXY_WG_INTERFACE" + else + echo "WireGuard interface $TPROXY_WG_INTERFACE already exists" + fi + # sudo ip route add $TPROXY_WG_CLIENT_IP_RANGE dev $TPROXY_WG_INTERFACE +} + + +case $ACTION in + host) + build_host + ;; + guest) + build_guest + ;; + cfg) + build_cfg + ;; + certs) + build_certs + ;; + wg) + build_wg + ;; + "") + # If no action specified, build everything + build_host + build_guest + build_certs + build_cfg + build_wg + ;; + *) + echo "Invalid action: $ACTION" + echo "Valid actions are: host, guest, cfg, certs, wg" + exit 1 + ;; +esac diff --git a/dev-setup b/dev-setup index 834f85a76..600831785 100755 --- a/dev-setup +++ b/dev-setup @@ -1,6 +1,6 @@ #!/bin/bash pushd . -source poky/oe-init-build-env build +source poky/oe-init-build-env bb-build bitbake-layers add-layer ../meta-confidential-compute bitbake-layers add-layer ../meta-openembedded/meta-oe diff --git a/dstack b/dstack index 584c2a7f2..beb451d6a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 584c2a7f27230cf3dae04c6fb0e3b89daf373946 +Subproject commit beb451d6a8cc27c05c6422a07214e279a1da3658 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index fc50a0211..b5be60737 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "1.0" +DISTRO_VERSION = "0.2.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index df82b62f9..bf9b0e361 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -7,15 +7,7 @@ inherit systemd update-rc.d REPO_ROOT = "${THISDIR}/../../.." -SRC_URI = "file://${REPO_ROOT}/dstack \ - file://tappd.init \ - file://tappd.service \ - file://llmnr.conf \ - file://tdx-attest.conf \ - file://tboot.sh \ - file://tboot.service \ - file://app-compose.service \ - " +SRC_URI = "file://${REPO_ROOT}/dstack" S = "${WORKDIR}/${REPO_ROOT}/dstack" @@ -44,23 +36,23 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} - install -m 0755 ${WORKDIR}/tboot.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -d ${D}${sysconfdir}/ - install -m 0644 ${WORKDIR}/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf + install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir} \ ${D}${sysconfdir}/systemd/resolved.conf.d - install -m 0644 ${WORKDIR}/tappd.service ${D}${systemd_system_unitdir} - install -m 0644 ${WORKDIR}/tboot.service ${D}${systemd_system_unitdir} - install -m 0644 ${WORKDIR}/app-compose.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tappd.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tboot.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} - install -m 0644 ${WORKDIR}/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d + install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d else install -d ${D}${sysconfdir}/init.d - install -m 0755 ${WORKDIR}/tappd.init ${D}${sysconfdir}/init.d/tappd.init + install -m 0755 ${S}/basefiles/tappd.init ${D}${sysconfdir}/init.d/tappd.init bberror "init scripts for sysvinit is not implemented yet" fi } diff --git a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service b/meta-dstack/recipes-core/dstack-guest/files/app-compose.service deleted file mode 100644 index 2051f3138..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/app-compose.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=App Compose Service -Requires=docker.service -After=docker.service tboot.service - -[Service] -Type=oneshot -RemainAfterExit=true -WorkingDirectory=/tapp -ExecStart=/usr/bin/env docker compose up -ExecStop=/usr/bin/env docker compose down -StandardOutput=journal+console -StandardError=journal+console - -[Install] -WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf b/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf deleted file mode 100644 index 4b0ccf845..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/llmnr.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Resolve] -LLMNR=no \ No newline at end of file diff --git a/meta-dstack/recipes-core/dstack-guest/files/tappd.init b/meta-dstack/recipes-core/dstack-guest/files/tappd.init deleted file mode 100644 index c27ee3cbb..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/tappd.init +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh -# -# SPDX-License-Identifier: GPL-2.0-only -# - -### BEGIN INIT INFO -# Provides: tappd -# Required-Start: $network $local_fs -# Required-Stop: $network $local_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start tappd daemon -# Description: Start the tappd daemon -### END INIT INFO - -# Source function library. -. /etc/init.d/functions - -prog="tappd" -exec="/usr/bin/$prog" -pidfile="/var/run/$prog.pid" -lockfile="/var/lock/subsys/$prog" -logfile="/var/log/$prog" - -[ -e /etc/default/$prog ] && . /etc/default/$prog - -start() { - [ -x $exec ] || exit 5 - - printf "Starting $prog:\t" - if ! [ -f $pidfile ]; then - $exec $DAEMON_OPTS & - echo $! > $pidfile - touch $lockfile - success - echo - else - failure - echo - printf "$pidfile still exists...\n" - exit 7 - fi -} - -stop() { - echo -n "Stopping $prog: " - killproc $prog - retval=$? - echo - [ $retval -eq 0 ] && rm -f $lockfile - return $retval -} - -restart() { - stop - start -} - -rh_status() { - status $prog -} - -rh_status_q() { - rh_status >/dev/null 2>&1 -} - -case "$1" in - start) - rh_status_q && exit 0 - $1 - ;; - stop) - rh_status_q || exit 0 - $1 - ;; - restart) - $1 - ;; - status) - rh_status - ;; - *) - echo "Usage: $0 {start|stop|status|restart}" - exit 2 -esac - -exit $? diff --git a/meta-dstack/recipes-core/dstack-guest/files/tappd.service b/meta-dstack/recipes-core/dstack-guest/files/tappd.service deleted file mode 100644 index 4c7a7f856..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/tappd.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Tappd Service -After=network.target - -[Service] -ExecStart=/bin/tappd -Restart=always -User=root -Group=root - -[Install] -WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.service b/meta-dstack/recipes-core/dstack-guest/files/tboot.service deleted file mode 100644 index f46d2f5e5..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Guest Boot Service -After=network.target -Before=app-compose.service tappd.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/env tboot.sh -RemainAfterExit=yes -StandardOutput=journal+console -StandardError=journal+console - -[Install] -WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh b/meta-dstack/recipes-core/dstack-guest/files/tboot.sh deleted file mode 100755 index df9f4e9e4..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/tboot.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh -set -e - -APP_COMPOSE_FILE=/tapp/app-compose.json - -prepare_docker_compose() { - local runner=$(jq -r '.runner' $APP_COMPOSE_FILE) - if [ "$runner" = "docker-compose" ]; then - jq -r .docker_compose_file $APP_COMPOSE_FILE > /tapp/docker-compose.yaml - else - echo "Unsupported runner: $runner" - exit 1 - fi -} - -setup_tproxy_net() { - local FEATURES=$(jq -r '.features[]' $APP_COMPOSE_FILE) - if ! echo "$FEATURES" | grep -q "tproxy-net"; then - echo "tproxy is not enabled" - return - fi - - local CLIENT_PRIVATE_KEY=$(wg genkey) - local CLIENT_PUBLIC_KEY=$(echo $CLIENT_PRIVATE_KEY | wg pubkey) - - local TPROXY_URL=$(jq -r '.tproxy_url' /tapp/config.json) - - curl ${TPROXY_URL}/prpc/Tproxy.RegisterCvm?json \ - --cacert /etc/tappd/ca.cert \ - --cert /etc/tappd/tls.cert \ - --key /etc/tappd/tls.key \ - -d"{\"client_public_key\":\"${CLIENT_PUBLIC_KEY}\"}" \ - -o /tmp/wginfo.json - - local CLIENT_IP=$(jq -r '.wg.client_ip' /tmp/wginfo.json) - local SERVER_ENDPOINT=$(jq -r '.wg.server_endpoint' /tmp/wginfo.json) - local SERVER_PUBLIC_KEY=$(jq -r '.wg.server_public_key' /tmp/wginfo.json) - local SERVER_IP=$(jq -r '.wg.server_ip' /tmp/wginfo.json) - - echo "WG CLIENT_IP: ${CLIENT_IP}" - echo "WG SERVER_ENDPOINT: ${SERVER_ENDPOINT}" - echo "WG SERVER_PUBLIC_KEY: ${SERVER_PUBLIC_KEY}" - echo "WG SERVER_IP: ${SERVER_IP}" - - rm -f /tmp/wginfo.json - - mkdir -p /etc/wireguard - cat < /etc/wireguard/wg0.conf -[Interface] -PrivateKey = ${CLIENT_PRIVATE_KEY} -Address = ${CLIENT_IP}/24 - -[Peer] -PublicKey = ${SERVER_PUBLIC_KEY} -AllowedIPs = ${SERVER_IP}/24 -Endpoint = ${SERVER_ENDPOINT} -PersistentKeepalive = 25 -EOF - wg-quick up wg0 -} - -prepare_certs() { - mkdir -p /etc/tappd - cp /tapp/certs/ca.cert /etc/tappd/ca.cert - jq -r '.app_key' /tapp/appkeys.json > /etc/tappd/app-ca.key - jq -r '.certificate_chain[]' /tapp/appkeys.json | awk 'NF {print $0 > "/etc/tappd/app-ca.cert"}' - - tdxctl gen-ra-cert \ - --ca-key /etc/tappd/app-ca.key \ - --ca-cert /etc/tappd/app-ca.cert \ - --cert-path /etc/tappd/tls.cert \ - --key-path /etc/tappd/tls.key - - cat /etc/tappd/app-ca.cert >> /etc/tappd/tls.cert -} - -prepare_certs && \ - setup_tproxy_net && \ - prepare_docker_compose diff --git a/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf b/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf deleted file mode 100644 index d7c6361ae..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/tdx-attest.conf +++ /dev/null @@ -1 +0,0 @@ -port=4050 \ No newline at end of file diff --git a/run_td.sh b/run_td.sh deleted file mode 100755 index 040f82d1f..000000000 --- a/run_td.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -VMDIR=${1:-./mock} - -PROCESS_NAME=qemu - -DIST_DIR=./build/dist -KERNEL=${DIST_DIR}/bzImage -INITRD=${DIST_DIR}/initramfs.cpio.gz -CDROM=${DIST_DIR}/rootfs.iso -OVMF_FIRMWARE=${DIST_DIR}/ovmf.fd -ROOTFS_IMG=${DIST_DIR}/rootfs.cpio - -VDA=${VMDIR}/vda.qcow2 -VDA_SIZE=10G -CMDLINE="console=ttyS0 init=/init" -CONFIG_DIR=${VMDIR}/shared -TD=${TD:-1} -TDVF_FIRMWARE=${OVMF_FIRMWARE} -RO=${RO:-off} -CID=$(( ( RANDOM % 10000 ) + 3 )) - -ARGS="${ARGS} -kernel ${KERNEL}" -ARGS="${ARGS} -initrd ${INITRD}" - -if [ "${TD}" == "1" ]; then - MACHINE_ARGS=",confidential-guest-support=tdx,hpet=off" - PROCESS_NAME=td - TDX_ARGS="-device vhost-vsock-pci,guest-cid=${CID} -object tdx-guest,id=tdx" - BIOS="-bios ${TDVF_FIRMWARE}" -fi - -echo INITRD=${INITRD} -echo ARGS=${ARGS} -echo VDA=${VDA} -echo CMDLINE=${CMDLINE} -echo TD=${TD} -echo TDX_ARGS=${TDX_ARGS} -echo BIOS=${BIOS} - -ROOTFS_HASH=$(sha256sum "${DIST_DIR}/rootfs.cpio" | awk '{print $1}') -echo ROOTFS_HASH=${ROOTFS_HASH} -cat < ${VMDIR}/shared/config.json -{ - "rootfs_hash": "${ROOTFS_HASH}", - "kms_url": "https://kms.1022.kvin.wang:9043", - "tproxy_url": "https://tproxy.1022.kvin.wang:9010" -} -EOF - -if [ ! -f ${VDA} ]; then - qemu-img create -f qcow2 ${VDA} ${VDA_SIZE} -fi -qemu-system-x86_64 \ - -accel kvm \ - -m 8G -smp 16 \ - -name ${PROCESS_NAME},process=${PROCESS_NAME} \ - -cpu host \ - -machine q35,kernel_irqchip=split${MACHINE_ARGS} \ - ${BIOS} \ - ${TDX_ARGS} \ - -nographic \ - -nodefaults \ - -chardev stdio,id=ser0,signal=on -serial chardev:ser0 \ - -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ - -drive file=${VDA},if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 \ - -virtfs local,path=${CONFIG_DIR},mount_tag=host-shared,readonly=${RO},security_model=mapped,id=virtfs0 \ - -cdrom ${CDROM} \ - ${ARGS} \ - -append "${CMDLINE}" From 2781958f55324fd1a395b689fe049386f8c03042 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 18 Nov 2024 08:09:11 +0000 Subject: [PATCH 050/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index beb451d6a..06ab627e8 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit beb451d6a8cc27c05c6422a07214e279a1da3658 +Subproject commit 06ab627e81b8a8930e4bc506e89b79e70a69a1b8 From 831905321acae448ff941ffdad6759d30c384efe Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 19 Nov 2024 02:21:30 +0000 Subject: [PATCH 051/435] Remove unused files --- mock/config.json | 10 ---- mock/shared/certs/ca.cert | 11 ---- mock/shared/certs/tmp-ca.cert | 12 ----- mock/shared/certs/tmp-ca.key | 5 -- mock/shared/docker-compose.yaml | 9 ---- scripts/calc_rtmr3.py | 90 --------------------------------- 6 files changed, 137 deletions(-) delete mode 100644 mock/config.json delete mode 100644 mock/shared/certs/ca.cert delete mode 100644 mock/shared/certs/tmp-ca.cert delete mode 100644 mock/shared/certs/tmp-ca.key delete mode 100644 mock/shared/docker-compose.yaml delete mode 100644 scripts/calc_rtmr3.py diff --git a/mock/config.json b/mock/config.json deleted file mode 100644 index 6d78b3a32..000000000 --- a/mock/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "id": "a2db9cc0-0459-41cf-b59a-33b8ab9d2bd9", - "name": "example", - "address": "3327603e03f5bd1f830812ca4a789277fc31f577", - "vcpu": 1, - "memory": 1024, - "disk_size": 20, - "image": "ubuntu-24.04", - "port_map": {} -} \ No newline at end of file diff --git a/mock/shared/certs/ca.cert b/mock/shared/certs/ca.cert deleted file mode 100644 index 452fe5c69..000000000 --- a/mock/shared/certs/ca.cert +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBmDCCAT2gAwIBAgIURtC9Iqqgh4/vxPWkd0UwPtbA5pcwCgYIKoZIzj0EAwIw -LzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEVMBMGA1UEAwwMUGhhbGEgS01TIENB -MCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAvMRYwFAYDVQQKDA1Q -aGFsYSBOZXR3b3JrMRUwEwYDVQQDDAxQaGFsYSBLTVMgQ0EwWTATBgcqhkjOPQIB -BggqhkjOPQMBBwNCAATXeN0OTqZFOzdDU8uq8eOXxYdxNoruXFiCe6fOSyQha9+S -zllhdJsAwEDKuvb+UcC6S2+Rq3pFzokoa+cC8f6cozUwMzAdBgNVHQ4EFgQUpDgt -ravZr8ybVH2mz9gIj+IHo+owEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD -AgNJADBGAiEA54mK5FOI5z8REY+fmN4Leo1imksQ6y5D9ekFaacOQkYCIQCKqGjb -rXgUSNTW/H6CJ24TA/OOVeOlLTqQ90cgPCrjvg== ------END CERTIFICATE----- diff --git a/mock/shared/certs/tmp-ca.cert b/mock/shared/certs/tmp-ca.cert deleted file mode 100644 index c98804d7f..000000000 --- a/mock/shared/certs/tmp-ca.cert +++ /dev/null @@ -1,12 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBrzCCAVWgAwIBAgIUDOcf3KNDCNLeosY9ssZzMLdF5aEwCgYIKoZIzj0EAwIw -OzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEhMB8GA1UEAwwYUGhhbGEgS01TIENs -aWVudCBUZW1wIENBMCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjA7 -MRYwFAYDVQQKDA1QaGFsYSBOZXR3b3JrMSEwHwYDVQQDDBhQaGFsYSBLTVMgQ2xp -ZW50IFRlbXAgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARaNL6rTtrd6ue3 -w+qtR8Mbd8wS3UptSVyq9to5z074qBwQUyKbFbTXhlkhqJzWyqryqqde5u4cz+YQ -P8jE07vPozUwMzAdBgNVHQ4EFgQUss/28mUr2NdAVVcO+LnwZKVaqqEwEgYDVR0T -AQH/BAgwBgEB/wIBATAKBggqhkjOPQQDAgNIADBFAiEA8aSPOwQizznrAC6jEtHU -R8l/UT6NbQ7i+rVcL6qvme4CIBpheJAb+k0sfbmLH0w3z+gnG4FlTlpgwpcRFqI7 -oFED ------END CERTIFICATE----- diff --git a/mock/shared/certs/tmp-ca.key b/mock/shared/certs/tmp-ca.key deleted file mode 100644 index 307455997..000000000 --- a/mock/shared/certs/tmp-ca.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgeDU11cytASj7Lqks -ASeAdcvf9XjgPWrtJ5ZSR2/WnHKhRANCAARaNL6rTtrd6ue3w+qtR8Mbd8wS3Upt -SVyq9to5z074qBwQUyKbFbTXhlkhqJzWyqryqqde5u4cz+YQP8jE07vP ------END PRIVATE KEY----- diff --git a/mock/shared/docker-compose.yaml b/mock/shared/docker-compose.yaml deleted file mode 100644 index 2798ce386..000000000 --- a/mock/shared/docker-compose.yaml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3' - -services: - nginx: - image: nginx:latest - ports: - - "8080:80" - restart: always - diff --git a/scripts/calc_rtmr3.py b/scripts/calc_rtmr3.py deleted file mode 100644 index 6c54053c1..000000000 --- a/scripts/calc_rtmr3.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -This script calculates the RTMR3 hash from the given docker-compose.yml file and KMS CA certificate. - -Usage: - python calc_rtmr3.py by-file --compose --ca-cert - python calc_rtmr3.py by-vm --images-dir --vm-dir - -Log from a CVM: -``` -FSINIT: Extending rootfs hash to RTMR, hash=bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd21 -Extended RTMR 3: bf06bf167df2d81dd54095e8a540e802dc634a31a96e1a448a20201a63d0bd2100000000000000000000000000000000 -FSINIT: Extending app id to RTMR, app_id=3327603e03f5bd1f830812ca4a789277fc31f577573ed149f47e0e2f3558e99e -Extended RTMR 3: 3327603e03f5bd1f830812ca4a789277fc31f577573ed149f47e0e2f3558e99e00000000000000000000000000000000 -FSINIT: Extending ca cert hash to RTMR, ca_cert_hash=663a81d65c1c749fd4ae4634fa0452553396eb9e956c90b903d47d45ae10d719 -Extended RTMR 3: 663a81d65c1c749fd4ae4634fa0452553396eb9e956c90b903d47d45ae10d71900000000000000000000000000000000 -ParsedReport { - attributes: 0000001000000000, - xfam: e702060000000000, - mrtd: 7ba9e262ce6979087e34632603f354dd8f8a870f5947d116af8114db6c9d0d74c48bec4280e5b4f4a37025a10905bb29, - mrconfigid: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, - mrowner: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, - mrownerconfig: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, - rtmr0: 698a1e5764ff07840695fb46c809949cca352e6c9d26fc37dce872402adc071b3b069b0b217c1dcda68cf914253b6842, - rtmr1: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, - rtmr2: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, - rtmr3: 3c30787034cd9aabff0347bc8f08b9f24a0f6ae914bbca0f9aba681e857aa57a7a7cc5b0b67231779cdc345f107707c5, - servtd_hash: 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, -} -``` -""" - -import hashlib -import argparse -import json -from pathlib import Path - - -INIT_MR= "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - -def rtmr_replay(history: list[str]): - """ - Replay the RTMR history to calculate the final RTMR value. - """ - if len(history) == 0: - return INIT_MR - mr = bytes.fromhex(INIT_MR) - for content in history: - # mr = sha384(concat(mr, content)) - # if content is shorter than 48 bytes, pad it with zeros - content = bytes.fromhex(content) - if len(content) < 48: - content = content.ljust(48, b'\0') - mr = hashlib.sha384(mr + content).digest() - return mr.hex() - - -def calc_rtmr3(rootfs_hash: str, app_id: str, ca_cert_hash: str): - """ - Calculate the RTMR3 hash from the given rootfs hash, app id and CA certificate hash. - """ - return rtmr_replay([rootfs_hash, app_id, ca_cert_hash]) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Calculate the RTMR3 hash from the given docker-compose.yml file and KMS CA certificate.") - parser.add_argument("mode", choices=["by-file", "by-vm"]) - parser.add_argument("--rootfs-cpio", help="The rootfs.cpio file to use.") - parser.add_argument("--compose", help="The docker-compose.yml file to use.") - parser.add_argument("--ca-cert", help="The KMS CA certificate to use.") - parser.add_argument("--images-dir", type=Path, help="The directory containing the VM images to use.") - parser.add_argument("--vm-dir", type=Path, help="The directory of a deployed VM.") - args = parser.parse_args() - - if args.mode == "by-file": - rootfs_hash = hashlib.sha256(open(args.rootfs_cpio, "rb").read()).hexdigest() - app_id = hashlib.sha256(open(args.compose, "rb").read()).hexdigest() - ca_cert_hash = hashlib.sha256(open(args.ca_cert, "rb").read()).hexdigest() - rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) - print(rtmr3) - elif args.mode == "by-vm": - vm_config = json.load(open(args.vm_dir / "config.json", "r")) - image_dir = args.images_dir / vm_config["image"] - image_metadata = json.load(open(image_dir / "metadata.json", "r")) - rootfs_hash = image_metadata["rootfs_hash"] - compose_file = args.vm_dir / "shared" / "app-compose.json" - ca_cert_file = args.vm_dir / "shared" / "certs" / "ca.cert" - app_id = hashlib.sha256(open(compose_file, "rb").read()).hexdigest() - ca_cert_hash = hashlib.sha256(open(ca_cert_file, "rb").read()).hexdigest() - rtmr3 = calc_rtmr3(rootfs_hash, app_id, ca_cert_hash) - print(rtmr3) From 165b6bf7d999db6cae3b3af36ff9a975deb1e7e4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 21 Nov 2024 08:29:24 +0000 Subject: [PATCH 052/435] Update dstack --- build.sh | 10 ++++++++-- dstack | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 50d525de2..84970d7b7 100755 --- a/build.sh +++ b/build.sh @@ -84,7 +84,7 @@ if [ -f $CONFIG_FILE ]; then fi rm -f build-config.sh.tpl else - mv build-config.sh.tpl build-config.sh + mv build-config.sh.tpl $CONFIG_FILE echo "Config file $CONFIG_FILE created, please edit it to configure the build" exit 1 fi @@ -100,6 +100,7 @@ TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) # Step 1: build binaries build_host() { + echo "Building binaries" (cd $DSTACK_DIR && cargo build --release) cp $DSTACK_DIR/target/release/{tproxy,kms,teepod,certbot,ct_monitor} . } @@ -117,7 +118,7 @@ make_image_dist() { { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.integrity=0", + "cmdline": "console=ttyS0 init=/init dstack.integrity=0 panic=1", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$rootfs_hash" @@ -131,6 +132,7 @@ EOF } build_guest() { + echo "Building guest images" if [ -z "$BBPATH" ]; then echo 'BBPATH is not set. Run `source dev-setup` in the meta-dstack/ directory' pushd $SCRIPT_DIR/ @@ -150,12 +152,14 @@ build_guest() { # Step 3: make certs build_certs() { + echo "Building certs" make -C $DSTACK_DIR certs DOMAIN=$BASE_DOMAIN TO=$CERTS_DIR } # Step 4: generate config files build_cfg() { + echo "Building config files" # kms cat < kms.toml log_level = "info" @@ -228,6 +232,7 @@ address = "127.0.0.1" port = $TEEPOD_RPC_LISTEN_PORT image_path = "$IMAGES_DIR" run_path = "$RUN_DIR/vm" +kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] ca_cert = "$CERTS_DIR/root-ca.cert" @@ -286,6 +291,7 @@ EOF } build_wg() { + echo "Setting up wireguard interface" # Step 6: setup wireguard interface # Check if the WireGuard interface exists if ! ip link show $TPROXY_WG_INTERFACE &> /dev/null; then diff --git a/dstack b/dstack index 06ab627e8..1e4c4fe7f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 06ab627e81b8a8930e4bc506e89b79e70a69a1b8 +Subproject commit 1e4c4fe7f64a086e4ff128da2d021e7d41ac1fa2 From 382f87e1d0326237f70d9553903854bbd122a605 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 22 Nov 2024 01:09:17 +0000 Subject: [PATCH 053/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 1e4c4fe7f..f537ae7bc 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 1e4c4fe7f64a086e4ff128da2d021e7d41ac1fa2 +Subproject commit f537ae7bc58a94c6a66115ae8ea59520c0c37a44 From 84e8f06b3fe8cda939844000280a2f79453a4945 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 23 Nov 2024 04:07:47 +0000 Subject: [PATCH 054/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index f537ae7bc..ac3b1851b 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f537ae7bc58a94c6a66115ae8ea59520c0c37a44 +Subproject commit ac3b1851b42f1f6c429fcf1eb71ea53ba3d373ea From 0a760ba5a050e90ccefda0536bf2d788e9541625 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 23 Nov 2024 04:08:22 +0000 Subject: [PATCH 055/435] Update linux kernel to linux-yocto-dev (v6.9) --- meta-dstack/conf/distro/dstack.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index b5be60737..4843e7c21 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -8,6 +8,9 @@ DISTRO_VERSION = "0.2.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" +PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" +PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" + VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" From 260e6a910a5e557ac3d9b9aed867d43478f970b2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 25 Nov 2024 03:11:41 +0000 Subject: [PATCH 056/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index ac3b1851b..5315686ba 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit ac3b1851b42f1f6c429fcf1eb71ea53ba3d373ea +Subproject commit 5315686ba9716de3b1b4984c4a40a7c0108cad00 From fcd51648dda831457661f0af71200e222ed2093d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 25 Nov 2024 08:54:42 +0000 Subject: [PATCH 057/435] Add kernel configs for docker+kernel6.9 --- .../linux/linux-yocto%.bbappend | 4 + scripts/docker-check-config.sh | 398 ++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100755 scripts/docker-check-config.sh diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index fb0ef2cda..3681df6d9 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -14,3 +14,7 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ dstack.scc" KERNEL_FEATURES:append:tdx = " dstack-tdx.scc" + +do_deploy:append() { + install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config +} diff --git a/scripts/docker-check-config.sh b/scripts/docker-check-config.sh new file mode 100755 index 000000000..0302de9b4 --- /dev/null +++ b/scripts/docker-check-config.sh @@ -0,0 +1,398 @@ +#!/usr/bin/env sh +set -e + +SCRIPT_DIR=$(dirname "$0") +EXITCODE=0 + +# bits of this were adapted from lxc-checkconfig +# see also https://github.com/lxc/lxc/blob/lxc-1.0.2/src/lxc/lxc-checkconfig.in + +possibleConfigs=" + /proc/config.gz + /boot/config-$(uname -r) + /usr/src/linux-$(uname -r)/.config + /usr/src/linux/.config +" + +if [ $# -gt 0 ]; then + CONFIG="$1" +else + : "${CONFIG:=$SCRIPT_DIR/../bb-build/tmp/deploy/images/tdx/kernel-config}" +fi + +if ! command -v zgrep > /dev/null 2>&1; then + zgrep() { + zcat "$2" | grep "$1" + } +fi + +useColor=true +if [ "$NO_COLOR" = "1" ] || [ ! -t 1 ]; then + useColor=false +fi +kernelVersion="$(uname -r)" +kernelMajor="${kernelVersion%%.*}" +kernelMinor="${kernelVersion#$kernelMajor.}" +kernelMinor="${kernelMinor%%.*}" + +is_set() { + zgrep "CONFIG_$1=[y|m]" "$CONFIG" > /dev/null +} +is_set_in_kernel() { + zgrep "CONFIG_$1=y" "$CONFIG" > /dev/null +} +is_set_as_module() { + zgrep "CONFIG_$1=m" "$CONFIG" > /dev/null +} + +color() { + # if stdout is not a terminal, then don't do color codes. + if [ "$useColor" = "false" ]; then + return 0 + fi + codes= + if [ "$1" = 'bold' ]; then + codes='1' + shift + fi + if [ "$#" -gt 0 ]; then + code= + case "$1" in + # see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors + black) code=30 ;; + red) code=31 ;; + green) code=32 ;; + yellow) code=33 ;; + blue) code=34 ;; + magenta) code=35 ;; + cyan) code=36 ;; + white) code=37 ;; + esac + if [ "$code" ]; then + codes="${codes:+$codes;}$code" + fi + fi + printf '\033[%sm' "$codes" +} +wrap_color() { + text="$1" + shift + color "$@" + printf '%s' "$text" + color reset + echo +} + +wrap_good() { + echo "$(wrap_color "$1" white): $(wrap_color "$2" green)" +} +wrap_bad() { + echo "$(wrap_color "$1" bold): $(wrap_color "$2" bold red)" +} +wrap_warning() { + wrap_color >&2 "$*" red +} + +check_flag() { + if is_set_in_kernel "$1"; then + wrap_good "CONFIG_$1" 'enabled' + elif is_set_as_module "$1"; then + wrap_good "CONFIG_$1" 'enabled (as module)' + else + wrap_bad "CONFIG_$1" 'missing' + EXITCODE=1 + fi +} + +check_flags() { + for flag in "$@"; do + printf -- '- ' + check_flag "$flag" + done +} + +check_command() { + if command -v "$1" > /dev/null 2>&1; then + wrap_good "$1 command" 'available' + else + wrap_bad "$1 command" 'missing' + EXITCODE=1 + fi +} + +check_device() { + if [ -c "$1" ]; then + wrap_good "$1" 'present' + else + wrap_bad "$1" 'missing' + EXITCODE=1 + fi +} + +if [ ! -e "$CONFIG" ]; then + wrap_warning "warning: $CONFIG does not exist, searching other paths for kernel config ..." + for tryConfig in $possibleConfigs; do + if [ -e "$tryConfig" ]; then + CONFIG="$tryConfig" + break + fi + done + if [ ! -e "$CONFIG" ]; then + wrap_warning "error: cannot find kernel config" + wrap_warning " try running this script again, specifying the kernel config:" + wrap_warning " CONFIG=/path/to/kernel/.config $0 or $0 /path/to/kernel/.config" + exit 1 + fi +fi + +wrap_color "info: reading kernel config from $CONFIG ..." white +echo + +echo 'Generally Necessary:' + +printf -- '- ' +if [ "$(stat -f -c %t /sys/fs/cgroup 2> /dev/null)" = '63677270' ]; then + wrap_good 'cgroup hierarchy' 'cgroupv2' + cgroupv2ControllerFile='/sys/fs/cgroup/cgroup.controllers' + if [ -f "$cgroupv2ControllerFile" ]; then + echo ' Controllers:' + for controller in cpu cpuset io memory pids; do + if grep -qE '(^| )'"$controller"'($| )' "$cgroupv2ControllerFile"; then + echo " - $(wrap_good "$controller" 'available')" + else + echo " - $(wrap_bad "$controller" 'missing')" + fi + done + else + wrap_bad "$cgroupv2ControllerFile" 'nonexistent??' + fi + # TODO find an efficient way to check if cgroup.freeze exists in subdir +else + cgroupSubsystemDir="$(sed -rne '/^[^ ]+ ([^ ]+) cgroup ([^ ]*,)?(cpu|cpuacct|cpuset|devices|freezer|memory)[, ].*$/ { s//\1/p; q }' /proc/mounts)" + cgroupDir="$(dirname "$cgroupSubsystemDir")" + if [ -d "$cgroupDir/cpu" ] || [ -d "$cgroupDir/cpuacct" ] || [ -d "$cgroupDir/cpuset" ] || [ -d "$cgroupDir/devices" ] || [ -d "$cgroupDir/freezer" ] || [ -d "$cgroupDir/memory" ]; then + echo "$(wrap_good 'cgroup hierarchy' 'properly mounted') [$cgroupDir]" + else + if [ "$cgroupSubsystemDir" ]; then + echo "$(wrap_bad 'cgroup hierarchy' 'single mountpoint!') [$cgroupSubsystemDir]" + else + wrap_bad 'cgroup hierarchy' 'nonexistent??' + fi + EXITCODE=1 + echo " $(wrap_color '(see https://github.com/tianon/cgroupfs-mount)' yellow)" + fi +fi + +if [ "$(cat /sys/module/apparmor/parameters/enabled 2> /dev/null)" = 'Y' ]; then + printf -- '- ' + if command -v apparmor_parser > /dev/null 2>&1; then + wrap_good 'apparmor' 'enabled and tools installed' + else + wrap_bad 'apparmor' 'enabled, but apparmor_parser missing' + printf ' ' + if command -v apt-get > /dev/null 2>&1; then + wrap_color '(use "apt-get install apparmor" to fix this)' + elif command -v yum > /dev/null 2>&1; then + wrap_color '(your best bet is "yum install apparmor-parser")' + else + wrap_color '(look for an "apparmor" package for your distribution)' + fi + EXITCODE=1 + fi +fi + +check_flags \ + NAMESPACES NET_NS PID_NS IPC_NS UTS_NS \ + CGROUPS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG \ + KEYS \ + VETH BRIDGE BRIDGE_NETFILTER \ + IP_NF_FILTER IP_NF_MANGLE IP_NF_TARGET_MASQUERADE \ + NETFILTER_XT_MATCH_ADDRTYPE \ + NETFILTER_XT_MATCH_CONNTRACK \ + NETFILTER_XT_MATCH_IPVS \ + NETFILTER_XT_MARK \ + IP_NF_NAT NF_NAT \ + POSIX_MQUEUE +# (POSIX_MQUEUE is required for bind-mounting /dev/mqueue into containers) + +if [ "$kernelMajor" -lt 4 ] || ([ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -lt 8 ]); then + check_flags DEVPTS_MULTIPLE_INSTANCES +fi + +if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 1 ]; then + check_flags NF_NAT_IPV4 +fi + +if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 2 ]; then + check_flags NF_NAT_NEEDED +fi +# check availability of BPF_CGROUP_DEVICE support +if [ "$kernelMajor" -ge 5 ] || ([ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -ge 15 ]); then + check_flags CGROUP_BPF +fi + +echo + +echo 'Optional Features:' +{ + check_flags USER_NS +} +{ + check_flags SECCOMP + check_flags SECCOMP_FILTER +} +{ + check_flags CGROUP_PIDS +} +{ + check_flags MEMCG_SWAP + # Kernel v5.8+ removes MEMCG_SWAP_ENABLED. + if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 8 ]; then + CODE=${EXITCODE} + check_flags MEMCG_SWAP_ENABLED + # FIXME this check is cgroupv1-specific + if [ -e /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes ]; then + echo " $(wrap_color '(cgroup swap accounting is currently enabled)' bold black)" + EXITCODE=${CODE} + elif is_set MEMCG_SWAP && ! is_set MEMCG_SWAP_ENABLED; then + echo " $(wrap_color '(cgroup swap accounting is currently not enabled, you can enable it by setting boot option "swapaccount=1")' bold black)" + fi + else + # Kernel v5.8+ enables swap accounting by default. + echo " $(wrap_color '(cgroup swap accounting is currently enabled)' bold black)" + fi +} +{ + if is_set LEGACY_VSYSCALL_NATIVE; then + printf -- '- ' + wrap_bad "CONFIG_LEGACY_VSYSCALL_NATIVE" 'enabled' + echo " $(wrap_color '(dangerous, provides an ASLR-bypassing target with usable ROP gadgets.)' bold black)" + elif is_set LEGACY_VSYSCALL_EMULATE; then + printf -- '- ' + wrap_good "CONFIG_LEGACY_VSYSCALL_EMULATE" 'enabled' + elif is_set LEGACY_VSYSCALL_NONE; then + printf -- '- ' + wrap_bad "CONFIG_LEGACY_VSYSCALL_NONE" 'enabled' + echo " $(wrap_color '(containers using eglibc <= 2.13 will not work. Switch to' bold black)" + echo " $(wrap_color ' "CONFIG_VSYSCALL_[NATIVE|EMULATE]" or use "vsyscall=[native|emulate]"' bold black)" + echo " $(wrap_color ' on kernel command line. Note that this will disable ASLR for the,' bold black)" + echo " $(wrap_color ' VDSO which may assist in exploiting security vulnerabilities.)' bold black)" + # else Older kernels (prior to 3dc33bd30f3e, released in v4.40-rc1) do + # not have these LEGACY_VSYSCALL options and are effectively + # LEGACY_VSYSCALL_EMULATE. Even older kernels are presumably + # effectively LEGACY_VSYSCALL_NATIVE. + fi +} + +if [ "$kernelMajor" -lt 4 ] || ([ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -le 5 ]); then + check_flags MEMCG_KMEM +fi + +if [ "$kernelMajor" -lt 3 ] || ([ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 18 ]); then + check_flags RESOURCE_COUNTERS +fi + +if [ "$kernelMajor" -lt 3 ] || ([ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 13 ]); then + netprio=NETPRIO_CGROUP +else + netprio=CGROUP_NET_PRIO +fi + +if [ "$kernelMajor" -lt 5 ]; then + check_flags IOSCHED_CFQ CFQ_GROUP_IOSCHED +fi + +check_flags \ + BLK_CGROUP BLK_DEV_THROTTLING \ + CGROUP_PERF \ + CGROUP_HUGETLB \ + NET_CLS_CGROUP $netprio \ + CFS_BANDWIDTH FAIR_GROUP_SCHED \ + IP_NF_TARGET_REDIRECT \ + IP_VS \ + IP_VS_NFCT \ + IP_VS_PROTO_TCP \ + IP_VS_PROTO_UDP \ + IP_VS_RR \ + SECURITY_SELINUX \ + SECURITY_APPARMOR + +if ! is_set EXT4_USE_FOR_EXT2; then + check_flags EXT3_FS EXT3_FS_XATTR EXT3_FS_POSIX_ACL EXT3_FS_SECURITY + if ! is_set EXT3_FS || ! is_set EXT3_FS_XATTR || ! is_set EXT3_FS_POSIX_ACL || ! is_set EXT3_FS_SECURITY; then + echo " $(wrap_color '(enable these ext3 configs if you are using ext3 as backing filesystem)' bold black)" + fi +fi + +check_flags EXT4_FS EXT4_FS_POSIX_ACL EXT4_FS_SECURITY +if ! is_set EXT4_FS || ! is_set EXT4_FS_POSIX_ACL || ! is_set EXT4_FS_SECURITY; then + if is_set EXT4_USE_FOR_EXT2; then + echo " $(wrap_color 'enable these ext4 configs if you are using ext3 or ext4 as backing filesystem' bold black)" + else + echo " $(wrap_color 'enable these ext4 configs if you are using ext4 as backing filesystem' bold black)" + fi +fi + +echo '- Network Drivers:' +echo " - \"$(wrap_color 'overlay' blue)\":" +check_flags VXLAN BRIDGE_VLAN_FILTERING | sed 's/^/ /' +echo ' Optional (for encrypted networks):' +check_flags CRYPTO CRYPTO_AEAD CRYPTO_GCM CRYPTO_SEQIV CRYPTO_GHASH \ + XFRM XFRM_USER XFRM_ALGO INET_ESP NETFILTER_XT_MATCH_BPF | sed 's/^/ /' +if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 3 ]; then + check_flags INET_XFRM_MODE_TRANSPORT | sed 's/^/ /' +fi +echo " - \"$(wrap_color 'ipvlan' blue)\":" +check_flags IPVLAN | sed 's/^/ /' +echo " - \"$(wrap_color 'macvlan' blue)\":" +check_flags MACVLAN DUMMY | sed 's/^/ /' +echo " - \"$(wrap_color 'ftp,tftp client in container' blue)\":" +check_flags NF_NAT_FTP NF_CONNTRACK_FTP NF_NAT_TFTP NF_CONNTRACK_TFTP | sed 's/^/ /' + +# only fail if no storage drivers available +CODE=${EXITCODE} +EXITCODE=0 +STORAGE=1 + +echo '- Storage Drivers:' +echo " - \"$(wrap_color 'btrfs' blue)\":" +check_flags BTRFS_FS | sed 's/^/ /' +check_flags BTRFS_FS_POSIX_ACL | sed 's/^/ /' +[ "$EXITCODE" = 0 ] && STORAGE=0 +EXITCODE=0 + +echo " - \"$(wrap_color 'overlay' blue)\":" +check_flags OVERLAY_FS | sed 's/^/ /' +[ "$EXITCODE" = 0 ] && STORAGE=0 +EXITCODE=0 + +echo " - \"$(wrap_color 'zfs' blue)\":" +printf ' - ' +check_device /dev/zfs +printf ' - ' +check_command zfs +printf ' - ' +check_command zpool +[ "$EXITCODE" = 0 ] && STORAGE=0 +EXITCODE=0 + +EXITCODE=$CODE +[ "$STORAGE" = 1 ] && EXITCODE=1 + +echo + +check_limit_over() { + if [ "$(cat "$1")" -le "$2" ]; then + wrap_bad "- $1" "$(cat "$1")" + wrap_color " This should be set to at least $2, for example set: sysctl -w kernel/keys/root_maxkeys=1000000" bold black + EXITCODE=1 + else + wrap_good "- $1" "$(cat "$1")" + fi +} + +echo 'Limits:' +check_limit_over /proc/sys/kernel/keys/root_maxkeys 10000 +echo + +exit $EXITCODE From c36f11919cd07103f2e0618dec39ad99db810399 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 25 Nov 2024 08:33:34 +0000 Subject: [PATCH 058/435] Add kernel configs for docker --- meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index 5cefef857..4bd4509e3 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -17,3 +17,8 @@ CONFIG_IPVLAN=m CONFIG_BTRFS_FS=m CONFIG_BTRFS_FS_POSIX_ACL=y CONFIG_CFS_BANDWIDTH=y +CONFIG_BRIDGE_NF_EBTABLES=y +CONFIG_NETFILTER_FAMILY_BRIDGE=y +CONFIG_VETH=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_VXLAN=y From 86fed2ec5132ac81729625c91dd9937b76c160c7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 29 Nov 2024 09:18:20 +0000 Subject: [PATCH 059/435] Support for opt-out FDE --- build.sh | 7 ++++--- dstack | 2 +- .../recipes-core/images/dstack-initramfs-files/kmfs-setup | 8 +++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 84970d7b7..54506e1ad 100755 --- a/build.sh +++ b/build.sh @@ -109,6 +109,7 @@ build_host() { make_image_dist() { local img_name=$1 local rootfs_name=$2 + local encfs=$3 local img_dist_dir=$IMAGES_DIR/$img_name local rootfs_hash @@ -118,7 +119,7 @@ make_image_dist() { { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.integrity=0 panic=1", + "cmdline": "console=ttyS0 init=/init dstack.fde=${encfs} panic=1 systemd.unified_cgroup_hierarchy=0", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$rootfs_hash" @@ -146,8 +147,8 @@ build_guest() { echo "Building $IMAGE_NAME" make -C $META_DIR dist DIST_DIR=$IMAGE_TMP_DIR - make_image_dist $IMAGE_NAME rootfs - make_image_dist $IMAGE_NAME-dev rootfs-dev + make_image_dist $IMAGE_NAME rootfs 1 + make_image_dist $IMAGE_NAME-dev rootfs-dev 0 } # Step 3: make certs diff --git a/dstack b/dstack index 5315686ba..57db2c6af 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 5315686ba9716de3b1b4984c4a40a7c0108cad00 +Subproject commit 57db2c6afae2028fa157bd4ae4d0c3e874ac9927 diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 2edede332..1530ba27e 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -1,4 +1,9 @@ +ROOTFS_ENCRYPTION="--rootfs-encryption=true" +if [ x"${kargs_dstack_fde}" = x"0" ]; then + echo "FDE is disabled" + ROOTFS_ENCRYPTION="--rootfs-encryption=false" +fi if [ x"${kargs_dstack_integrity}" = x"1" ]; then ROOTINTEGRITY="--rootfs-integrity" fi @@ -27,6 +32,7 @@ echo " ROOTFS_HD=$ROOTFS_HD" echo " ROOTFS_CD=$ROOTFS_CD" echo " ROOTFS_CD_MNT=$ROOTFS_CD_MNT" echo " ROOTINTEGRITY=$ROOTINTEGRITY" +echo " ROOTFS_ENCRYPTION=$ROOTFS_ENCRYPTION" tdxctl setup-fde \ --host-shared $HOST_SHARED \ @@ -36,4 +42,4 @@ tdxctl setup-fde \ --root-hd $ROOTFS_HD \ --root-cdrom $ROOTFS_CD \ --root-cdrom-mnt $ROOTFS_CD_MNT \ - $ROOTINTEGRITY || panic "Failed to setup FDE" + $ROOTFS_ENCRYPTION $ROOTINTEGRITY || panic "Failed to setup FDE" From 4b4d4550395fd7d09b95f8674ff4bcbe2cba4c1f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 29 Nov 2024 09:25:17 +0000 Subject: [PATCH 060/435] chore comment update --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 54506e1ad..ab7e82f60 100755 --- a/build.sh +++ b/build.sh @@ -23,13 +23,12 @@ cat < build-config.sh.tpl # cvm point of view BASE_DOMAIN=1022.kvin.wang -# kms and tproxy rpc listen port TEEPOD_RPC_LISTEN_PORT=9080 # CIDs allocated to VMs start from this number of type unsigned int32 TEEPOD_CID_POOL_START=10000 # CID pool size TEEPOD_CID_POOL_SIZE=1000 -# Whether port mapping is enabled +# Whether port mapping from host to CVM is allowed TEEPOD_PORT_MAPPING_ENABLED=false KMS_RPC_LISTEN_PORT=9043 From 199b4faf9de73bd56f4832865fc2f136321eaefd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 06:55:13 +0000 Subject: [PATCH 061/435] Add dstack cli and better images bb files --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 6 +- .../recipes-core/images/dstack-initramfs.bb | 4 +- .../images/dstack-rootfs-base.inc | 75 +++ .../recipes-core/images/dstack-rootfs-dev.bb | 6 +- .../recipes-core/images/dstack-rootfs-dev.inc | 4 + .../images/dstack-rootfs-prod.inc | 2 + .../recipes-core/images/dstack-rootfs.bb | 52 +- .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 7 +- scripts/bin/dstack.py | 445 ++++++++++++++++++ scripts/samples/kms-ca.cert | 11 - 12 files changed, 540 insertions(+), 76 deletions(-) create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs-base.inc create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs-dev.inc create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs-prod.inc create mode 100755 scripts/bin/dstack.py delete mode 100644 scripts/samples/kms-ca.cert diff --git a/dstack b/dstack index 57db2c6af..212405bf5 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 57db2c6afae2028fa157bd4ae4d0c3e874ac9927 +Subproject commit 212405bf5dd84f1a5d6ae0b056ca41224b037c6c diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 4843e7c21..b3a2d9b18 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.2.0" +DISTRO_VERSION = "0.3.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index bf9b0e361..44b455edc 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -7,9 +7,11 @@ inherit systemd update-rc.d REPO_ROOT = "${THISDIR}/../../.." -SRC_URI = "file://${REPO_ROOT}/dstack" +SRC_DIR = "${@oe.utils.conditional('DSTACK_SRC_URI', '', '${REPO_ROOT}/dstack', 'git', d)}" +SRC_URI = "${@oe.utils.conditional('DSTACK_SRC_URI', '', 'file://${REPO_ROOT}/dstack', '${DSTACK_SRC_URI}', d)}" +SRCREV = "${DSTACK_SRC_REV}" -S = "${WORKDIR}/${REPO_ROOT}/dstack" +S = "${WORKDIR}/${SRC_DIR}" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','tappd.service tboot.service app-compose.service','',d)}" diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 97fbb94aa..11d898f1d 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -1,12 +1,12 @@ PACKAGE_INSTALL = "\ - ${VIRTUAL-RUNTIME_base-utils} \ + busybox \ udev \ base-passwd \ ${ROOTFS_BOOTSTRAP_INSTALL} \ base-files \ cryptsetup \ e2fsprogs-mke2fs \ - kernel-module-tdx-guest \ + tdx-guest-ko \ dstack-initramfs-files \ dstack-guest \ curl \ diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc new file mode 100644 index 000000000..04af9d9b8 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -0,0 +1,75 @@ +IMAGE_BASENAME = "${PN}" + +IMAGE_INSTALL = "\ + ${VIRTUAL-RUNTIME_base-utils} \ + ${ROOTFS_BOOTSTRAP_INSTALL} \ + base-files \ + base-passwd \ + systemd \ + netbase \ + iptables \ + docker \ + docker-compose \ + dstack-prebuilt \ + tdx-guest-ko \ + dstack-guest \ + wireguard-tools \ + curl \ + jq \ +" + +# Do not pollute the initrd image with rootfs features +IMAGE_FEATURES = "" + +IMAGE_NAME_SUFFIX ?= "" +IMAGE_LINGUAS = "" +INITRAMFS_MAXSIZE = "1000000" + +LICENSE = "MIT" + +IMAGE_FSTYPES = "cpio" + +inherit core-image + +IMAGE_ROOTFS_SIZE = "8192" +IMAGE_ROOTFS_EXTRA_SPACE = "0" + +# Use the same restriction as initramfs-live-install +COMPATIBLE_HOST = "x86_64.*-linux" + +# Remove sysvinit related files in a postprocess function +ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" +ROOTFS_POSTPROCESS_COMMAND += "symlink_lib64;" +IMAGE_FEATURES[validitems] += "nologin" +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_getty_services", "",d)}' + +remove_sysvinit_files() { + # Remove /etc/init.d directory and its contents + rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d + + # Remove /etc/rc*.d directories and their contents + for d in ${IMAGE_ROOTFS}${sysconfdir}/rc*.d; do + rm -rf $d + done + + # Remove other sysvinit specific files + rm -f ${IMAGE_ROOTFS}${sysconfdir}/inittab +} + +symlink_lib64() { + ln -s lib ${IMAGE_ROOTFS}/lib64 +} + +disable_getty_services() { + for srv in getty getty-pre; do + rm -f ${IMAGE_ROOTFS}/etc/systemd/system/${srv}.target + rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}.target + done + for srv in autovt container-getty console-getty getty-generator serial-getty getty; do + rm -f ${IMAGE_ROOTFS}/etc/systemd/system/${srv}.service + rm -f ${IMAGE_ROOTFS}/etc/systemd/system/${srv}@.service + rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}.service + rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}@.service + done +} + diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb b/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb index 00cc6ee8e..59662e6d5 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb @@ -1,5 +1 @@ -require ${THISDIR}/dstack-rootfs.bb - -PACKAGE_INSTALL += "dropbear" -IMAGE_FEATURES = "debug-tweaks" -IMAGE_BASENAME = "dstack-rootfs-dev" +include dstack-rootfs-dev.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc new file mode 100644 index 000000000..da6cbd9b3 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -0,0 +1,4 @@ +include dstack-rootfs-base.inc + +PACKAGE_INSTALL += "dropbear strace" +IMAGE_FEATURES = "debug-tweaks" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc b/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc new file mode 100644 index 000000000..e3f805d0c --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc @@ -0,0 +1,2 @@ +include dstack-rootfs-base.inc +IMAGE_FEATURES += "nologin" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb index a1c8b6c9f..7dc317dfa 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -1,51 +1 @@ -PACKAGE_INSTALL = "\ - ${VIRTUAL-RUNTIME_base-utils} \ - ${ROOTFS_BOOTSTRAP_INSTALL} \ - base-files \ - base-passwd \ - systemd \ - netbase \ - iptables \ - docker \ - docker-compose \ - dstack-prebuilt \ - kernel-module-tdx-guest \ - dstack-guest \ - wireguard-tools \ - curl jq" - -# Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "" - -IMAGE_BASENAME = "dstack-rootfs" -IMAGE_NAME_SUFFIX ?= "" -IMAGE_LINGUAS = "" -INITRAMFS_MAXSIZE = "1000000" - -LICENSE = "MIT" - -IMAGE_FSTYPES = "cpio" - -inherit core-image - -IMAGE_ROOTFS_SIZE = "8192" -IMAGE_ROOTFS_EXTRA_SPACE = "0" - -# Use the same restriction as initramfs-live-install -COMPATIBLE_HOST = "x86_64.*-linux" - -# Remove sysvinit related files in a postprocess function -ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" - -remove_sysvinit_files() { - # Remove /etc/init.d directory and its contents - rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d - - # Remove /etc/rc*.d directories and their contents - for d in ${IMAGE_ROOTFS}${sysconfdir}/rc*.d; do - rm -rf $d - done - - # Remove other sysvinit specific files - rm -f ${IMAGE_ROOTFS}${sysconfdir}/inittab -} +include dstack-rootfs-prod.inc diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb index 7a13f6771..f7b7bf520 100644 --- a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb +++ b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb @@ -6,10 +6,11 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384 inherit module REPO_ROOT = "${THISDIR}/../../.." -SRC_DIR = "${REPO_ROOT}/dstack/mod-tdx-guest" -SRC_URI = "file://${SRC_DIR}" +SRC_DIR = "${@oe.utils.conditional('DSTACK_SRC_URI', '', '${REPO_ROOT}/dstack/mod-tdx-guest', 'git/mod-tdx-guest', d)}" +SRC_URI = "${@oe.utils.conditional('DSTACK_SRC_URI', '', 'file://${REPO_ROOT}/dstack', '${DSTACK_SRC_URI}', d)}" +SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" -RPROVIDES:${PN} += "kernel-module-tdx-guest" +RPROVIDES:${PN} += "tdx-guest-ko" diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py new file mode 100755 index 000000000..3bc4ce0ed --- /dev/null +++ b/scripts/bin/dstack.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 + +import argparse +import json +import logging +import os +import random +import string +import subprocess +import uuid +import configparser +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import List, Dict, Optional +from functools import reduce + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +def generate_config_paths(): + paths = [ + "/etc/dstack/client.conf", + os.path.expanduser("~/.config/dstack/client.conf"), + ] + current_dir = os.getcwd() + while current_dir != "/": + paths.append(os.path.join(current_dir, ".dstack", "client.conf")) + current_dir = os.path.dirname(current_dir) + return paths + + +@dataclass +class PortMap: + """Configuration for port mapping.""" + address: str + protocol: str + from_port: int + to_port: int + + def to_dict(self) -> Dict: + return { + "address": self.address, + "protocol": self.protocol, + "from": self.from_port, + "to": self.to_port + } + +@dataclass +class VMConfig: + """Configuration for VM instance.""" + id: str + name: str + vcpu: int + gpu: List[str] + memory: int + disk_size: int + image: str + port_map: List[PortMap] + created_at_ms: int + + def to_dict(self) -> Dict: + return { + "id": self.id, + "name": self.name, + "vcpu": self.vcpu, + "gpu": self.gpu, + "memory": self.memory, + "disk_size": self.disk_size, + "image": self.image, + "port_map": [p.to_dict() for p in self.port_map], + "created_at_ms": self.created_at_ms + } + +def merge2(a, b): + if isinstance(a, dict) and isinstance(b, dict): + c = a.copy() + for k, v in b.items(): + c[k] = merge2(a.get(k), v) + return c + if b is None: + return a + return b + + +def test_merge2(): + assert merge2({"a": 1}, {"b": 2}) == {"a": 1, "b": 2} + assert merge2({"a": 1}, {"a": 2}) == {"a": 2} + assert merge2({"a": {"b": 1}}, {"a": {"c": 2}}) == {"a": {"b": 1, "c": 2}} + + +def merge_dicts(*dicts): + return reduce(merge2, dicts, {}) + + +def test_merge_dicts(): + assert merge_dicts({"a": 1}, {"b": 2}) == {"a": 1, "b": 2} + assert merge_dicts({"a": 1}, {"a": 2}) == {"a": 2} + assert merge_dicts({"a": {"b": 1}}, {"a": {"c": 2}}) == {"a": {"b": 1, "c": 2}} + assert merge_dicts({"a": {"b": 1}}, {"a": {"b": 2}}) == {"a": {"b": 2}} + assert merge_dicts({"a": {"b": 1}}, {"a": {"b": 2}, "c": 3}) == {"a": {"b": 2}, "c": 3} + assert merge_dicts({"a": 1}, {"b": 2}, {"c": 3}) == {"a": 1, "b": 2, "c": 3} + assert merge_dicts({"a": 1}, {"a": 2}, {"c": 3}) == {"a": 2, "c": 3} + + +def ini_to_dict(filename): + config = configparser.ConfigParser() + config.read(filename) + + result = {} + for section in config.sections(): + result[section] = {} + for key, value in config.items(section): + result[section][key] = value + return result + + +def load_configs_merged(config_paths): + config = {} + for config_path in config_paths: + if os.path.exists(config_path): + logger.info(f"Loading configuration from {config_path}") + config = merge_dicts(config, ini_to_dict(config_path)) + return config + + +@dataclass +class DStackConfig: + """Configuration for DStack client.""" + docker_registry: Optional[str] = None + image_path: str = './images' + default_image_name: str = '' + qemu_path: str = 'qemu-system-x86_64' + + @classmethod + def load(cls) -> 'DStackConfig': + """Load configuration from file.""" + cfgs = load_configs_merged(generate_config_paths()) + def cfg_get(section, key, fallback): + if section in cfgs and key in cfgs[section]: + return cfgs[section][key] + return fallback + me = cls() + me.docker_registry = cfg_get('docker', 'registry', cls.docker_registry) + me.image_path = os.path.abspath(cfg_get('image', 'path', cls.image_path)) + me.default_image_name = cfg_get('image', 'default', cls.default_image_name) + me.qemu_path = cfg_get('qemu', 'path', cls.qemu_path) + return me + + +class DStackManager: + def __init__(self): + self.run_path = os.path.abspath(os.getenv('RUN_PATH', './vms')) + self.config = DStackConfig.load() + + def get_default_image_path(self) -> str: + """Get the full default image path.""" + return os.path.join(self.config.image_path, self.config.default_image_name) + + def _generate_instance_id(self) -> str: + """Generate a random instance ID.""" + return str(uuid.uuid4()) + + def _read_compose_file(self, compose_file: str) -> str: + """Read and validate compose file.""" + if not os.path.isfile(compose_file): + raise FileNotFoundError(f"Compose file not found: {compose_file}") + with open(compose_file, 'r') as f: + return f.read() + + def _read_image_metadata(self, image_path: str) -> str: + """Read and validate image metadata.""" + metadata_path = os.path.join(image_path, 'metadata.json') + if not os.path.isfile(metadata_path): + raise FileNotFoundError(f"Image metadata not found at {metadata_path}") + + try: + with open(metadata_path, 'r') as f: + metadata = json.load(f) + rootfs_hash = metadata.get('rootfs_hash') + if not rootfs_hash: + raise ValueError("Rootfs hash not found in image info") + return rootfs_hash + except json.JSONDecodeError: + raise ValueError(f"Invalid JSON in metadata file: {metadata_path}") + + def _create_directories(self, work_dir: str) -> tuple[str, str]: + """Create necessary directories.""" + if os.path.exists(work_dir): + raise FileExistsError(f"The instance already exists at {work_dir}") + + shared_dir = os.path.join(work_dir, 'shared') + certs_dir = os.path.join(shared_dir, 'certs') + os.makedirs(shared_dir, exist_ok=True) + os.makedirs(certs_dir, exist_ok=True) + return shared_dir, certs_dir + + def _convert_memory_to_mb(self, memory: str) -> int: + """Convert memory string to MB.""" + if memory.upper().endswith('T'): + return int(memory[:-1]) * 1024 * 1024 + if memory.upper().endswith('G'): + return int(memory[:-1]) * 1024 + if memory.upper().endswith('M'): + return int(memory[:-1]) + return int(memory) + + def _parse_port_mapping(self, port_str: str) -> PortMap: + """Parse port mapping string in format 'protocol[:address]:from:to'.""" + try: + parts = port_str.split(':') + if len(parts) == 3: + proto, from_port, to_port = parts + address = "127.0.0.1" # default to localhost + elif len(parts) == 4: + proto, address, from_port, to_port = parts + else: + raise ValueError("Invalid port mapping format. Use 'protocol[:address]:from:to'") + + return PortMap( + address=address, + protocol=proto.lower(), + from_port=int(from_port), + to_port=int(to_port) + ) + except ValueError as e: + raise ValueError(f"Invalid port mapping '{port_str}': {str(e)}") + + def setup_instance(self, args: argparse.Namespace) -> None: + """Set up a new instance with the provided configuration.""" + try: + # Generate instance ID if work_dir not provided + instance_id = os.path.basename(args.dir) if args.dir else self._generate_instance_id() + work_dir = args.dir or os.path.join(self.run_path, instance_id) + + # Create directories + shared_dir, certs_dir = self._create_directories(work_dir) + + # Read compose file + compose_content = self._read_compose_file(args.compose_file) + + # Create app-compose.json + app_compose = { + "manifest_version": 1, + "name": "example", + "version": "1.0.0", + "features": [], + "runner": "docker-compose", + "docker_compose_file": compose_content + } + with open(os.path.join(shared_dir, 'app-compose.json'), 'w') as f: + json.dump(app_compose, f, indent=4) + + # Read image metadata and create config.json + image_path = args.image or self.get_default_image_path() + rootfs_hash = self._read_image_metadata(image_path) + with open(os.path.join(shared_dir, 'config.json'), 'w') as f: + json.dump({"rootfs_hash": rootfs_hash, "docker_registry": self.config.docker_registry}, f, indent=4) + + # Create VM manifest + memory = self._convert_memory_to_mb(str(args.memory)) + disk_size = self._convert_memory_to_mb(str(args.disk)) // 1024 + port_map = [] + if args.port: + for port_str in args.port: + port_map.append(self._parse_port_mapping(port_str)) + + vm_config = VMConfig( + id=instance_id, + name="example", + vcpu=args.vcpus, + gpu=args.gpu or [], + memory=memory, + disk_size=disk_size, + image=os.path.basename(image_path), + port_map=port_map, + created_at_ms=int(datetime.now().timestamp() * 1000) + ) + + with open(os.path.join(work_dir, 'vm-manifest.json'), 'w') as f: + json.dump(vm_config.to_dict(), f, indent=4) + + logger.info(f"Work directory prepared successfully at: {work_dir}") + + except Exception as e: + logger.error(f"Failed to setup instance: {str(e)}") + raise + + def run_instance(self, vm_dir: str, memory: Optional[str] = None, vcpus: Optional[int] = None) -> None: + """Run a VM instance from the specified directory. + + Args: + vm_dir: Directory containing the VM configuration + memory: Optional memory size override (e.g., '2G', '512M') + vcpus: Optional number of virtual CPUs override + """ + manifest_path = os.path.join(vm_dir, 'vm-manifest.json') + if not os.path.exists(manifest_path): + raise ValueError(f"VM manifest not found in {vm_dir}") + + with open(manifest_path, 'r') as f: + manifest = json.load(f) + + # Get image path and metadata + image_path = os.path.join(self.config.image_path, manifest['image']) + img_metadata_path = os.path.join(image_path, 'metadata.json') + + if not os.path.exists(img_metadata_path): + raise ValueError(f"Image metadata not found at {img_metadata_path}") + + with open(img_metadata_path, 'r') as f: + img_metadata = json.load(f) + + # Prepare QEMU arguments + mem = memory if memory else f"{manifest['memory']}M" + vcpu_count = vcpus if vcpus is not None else manifest['vcpu'] + disk_size = manifest['disk_size'] + gpus = manifest.get('gpu', []) + + vda = os.path.join(vm_dir, 'hda.img') + config_dir = os.path.join(vm_dir, 'shared') + + # Create disk if it doesn't exist + if not os.path.exists(vda): + subprocess.run(['qemu-img', 'create', '-f', 'qcow2', vda, f"{disk_size}G"], check=True) + + cid = random.randint(1, 10000) + 3 + + # Prepare QEMU command + cmd = [ + self.config.qemu_path, + '-accel', 'kvm', + '-m', mem, + '-smp', str(vcpu_count), + '-cpu', 'host', + '-machine', 'q35,kernel_irqchip=split,confidential-guest-support=tdx,hpet=off', + '-object', 'tdx-guest,id=tdx', + '-nographic', + '-nodefaults', + '-chardev', 'stdio,id=ser0,signal=on', + '-serial', 'chardev:ser0', + '-kernel', os.path.join(image_path, img_metadata['kernel']), + '-initrd', os.path.join(image_path, img_metadata['initrd']), + '-bios', os.path.join(image_path, img_metadata['bios']), + '-cdrom', os.path.join(image_path, img_metadata['rootfs']), + '-drive', f'file={vda},if=none,id=virtio-disk0', + '-device', 'virtio-blk-pci,drive=virtio-disk0', + '-virtfs', f'local,path={config_dir},mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0', + '-device', f'vhost-vsock-pci,guest-cid={cid}', + ] + + # Add network configuration + port_args = [] + for port_map in manifest.get('port_map', []): + protocol = port_map.get('protocol', 'tcp') + bind_address = port_map.get('address', '127.0.0.1') + host_port = port_map['from'] + vm_port = port_map['to'] + port_args.append(f"hostfwd={protocol}:{bind_address}:{host_port}-:{vm_port}") + cmd.extend([ + '-device', 'virtio-net-pci,netdev=nic0_td', + '-netdev', f"user,id=nic0_td{','+','.join(port_args) if len(port_args) > 1 else ''}" + ]) + + if gpus: + cmd.extend([ + '-device', f'pcie-root-port,id=pci.1,bus=pcie.0', + '-fw_cfg', 'name=opt/ovmf/X-PciMmio64,string=262144', + ]) + # Use sudo when GPU is involved + cmd = ['sudo'] + cmd + for i, gpu_id in enumerate(gpus): + cmd.extend([ + '-object', f'iommufd,id=iommufd{i}', + '-device', f'vfio-pci,host={gpu_id},bus=pci.1,iommufd=iommufd{i}', + ]) + # Add kernel command line + cmd.extend(['-append', img_metadata['cmdline']]) + + print(" ".join(cmd)) + # Run the command + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to start VM: {e}") + + +def list_available_gpus() -> None: + """List available NVIDIA GPUs.""" + try: + result = subprocess.run(['lspci'], capture_output=True, text=True) + gpu_lines = [line for line in result.stdout.split('\n') if 'NVIDIA' in line] + if gpu_lines: + print("\nAvailable GPU IDs:") + print("ID Description") + for line in gpu_lines: + print(line) + print() + except subprocess.SubprocessError: + logger.warning("Could not list GPU devices") + + +def main(): + parser = argparse.ArgumentParser(description='DStack VM Management Tool') + subparsers = parser.add_subparsers(dest='command', help='Commands') + + # Setup command + setup_parser = subparsers.add_parser('new', help='Setup a new instance') + setup_parser.add_argument('compose_file', type=str, help='Docker compose file') + setup_parser.add_argument('-o', '--dir', type=str, help='Work directory') + setup_parser.add_argument('-i', '--image', type=str, help='VM image path') + setup_parser.add_argument('-c', '--vcpus', type=int, default=1, help='Number of vCPUs') + setup_parser.add_argument('-m', '--memory', type=str, default='1G', help='Memory size (e.g., 1G, 512M)') + setup_parser.add_argument('-d', '--disk', type=str, default='20G', help='Disk size (e.g., 20G)') + setup_parser.add_argument('-g', '--gpu', type=str, action='append', help='GPU device') + setup_parser.add_argument('-p', '--port', action='append', type=str, help='Port mapping in format: protocol[:address]:from:to') + setup_parser.add_argument('--no-fde', action='store_true', help='Disable Full Disk Encryption') + + # Start command + start_parser = subparsers.add_parser('run', help='Start an instance') + start_parser.add_argument('dir', type=str, help='Work directory') + start_parser.add_argument('-m', '--memory', type=str, help='Memory size (e.g. 2G, 512M)') + start_parser.add_argument('-c', '--vcpus', type=int, help='Number of virtual CPUs') + + # List Gpus command + subparsers.add_parser('lsgpu', help='List available GPUs') + + args = parser.parse_args() + + if args.command == 'new': + manager = DStackManager() + manager.setup_instance(args) + elif args.command == 'run': + manager = DStackManager() + manager.run_instance(args.dir, memory=args.memory, vcpus=args.vcpus) + elif args.command == 'lsgpu': + list_available_gpus() + else: + parser.print_help() + +if __name__ == '__main__': + main() diff --git a/scripts/samples/kms-ca.cert b/scripts/samples/kms-ca.cert deleted file mode 100644 index 452fe5c69..000000000 --- a/scripts/samples/kms-ca.cert +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBmDCCAT2gAwIBAgIURtC9Iqqgh4/vxPWkd0UwPtbA5pcwCgYIKoZIzj0EAwIw -LzEWMBQGA1UECgwNUGhhbGEgTmV0d29yazEVMBMGA1UEAwwMUGhhbGEgS01TIENB -MCAXDTc1MDEwMTAwMDAwMFoYDzQwOTYwMTAxMDAwMDAwWjAvMRYwFAYDVQQKDA1Q -aGFsYSBOZXR3b3JrMRUwEwYDVQQDDAxQaGFsYSBLTVMgQ0EwWTATBgcqhkjOPQIB -BggqhkjOPQMBBwNCAATXeN0OTqZFOzdDU8uq8eOXxYdxNoruXFiCe6fOSyQha9+S -zllhdJsAwEDKuvb+UcC6S2+Rq3pFzokoa+cC8f6cozUwMzAdBgNVHQ4EFgQUpDgt -ravZr8ybVH2mz9gIj+IHo+owEgYDVR0TAQH/BAgwBgEB/wIBAzAKBggqhkjOPQQD -AgNJADBGAiEA54mK5FOI5z8REY+fmN4Leo1imksQ6y5D9ekFaacOQkYCIQCKqGjb -rXgUSNTW/H6CJ24TA/OOVeOlLTqQ90cgPCrjvg== ------END CERTIFICATE----- From fbe4556e794ab02a1af841da1b6ed191f8e8bce4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 10:00:02 +0000 Subject: [PATCH 062/435] Better dev-setup --- dev-setup | 44 +++++++++++++++++++++++++++++++++----------- scripts/bin/dstack | 1 + 2 files changed, 34 insertions(+), 11 deletions(-) create mode 120000 scripts/bin/dstack diff --git a/dev-setup b/dev-setup index 600831785..cc813f3b4 100755 --- a/dev-setup +++ b/dev-setup @@ -1,13 +1,35 @@ #!/bin/bash +if [ -n "$BASH_SOURCE" ]; then + THIS_SCRIPT=$BASH_SOURCE +elif [ -n "$ZSH_NAME" ]; then + THIS_SCRIPT=$0 +else + THIS_SCRIPT="$(pwd)/dev-setup" + if [ ! -e "$THIS_SCRIPT" ]; then + echo "Error: $THIS_SCRIPT doesn't exist!" >&2 + echo "Please run this script in dev-setup's directory." >&2 + exit 1 + fi +fi +THIS_SCRIPT=$(realpath "$THIS_SCRIPT") +THIS_DIR=$(dirname "$THIS_SCRIPT") + pushd . -source poky/oe-init-build-env bb-build - -bitbake-layers add-layer ../meta-confidential-compute -bitbake-layers add-layer ../meta-openembedded/meta-oe -bitbake-layers add-layer ../meta-openembedded/meta-python -bitbake-layers add-layer ../meta-openembedded/meta-networking -bitbake-layers add-layer ../meta-openembedded/meta-filesystems -bitbake-layers add-layer ../meta-virtualization -bitbake-layers add-layer ../meta-rust-bin -bitbake-layers add-layer ../meta-dstack -popd \ No newline at end of file +source $THIS_DIR/poky/oe-init-build-env $THIS_DIR/bb-build + +bitbake-layers add-layer \ + $THIS_DIR/meta-confidential-compute \ + $THIS_DIR/meta-openembedded/meta-oe \ + $THIS_DIR/meta-openembedded/meta-python \ + $THIS_DIR/meta-openembedded/meta-networking \ + $THIS_DIR/meta-openembedded/meta-filesystems \ + $THIS_DIR/meta-virtualization \ + $THIS_DIR/meta-rust-bin \ + $THIS_DIR/meta-dstack \ + +popd + +# Add scripts/bin to PATH if not already present +if [[ ":$PATH:" != *":$THIS_DIR/scripts/bin:"* ]]; then + export PATH=$PATH:$THIS_DIR/scripts/bin +fi diff --git a/scripts/bin/dstack b/scripts/bin/dstack new file mode 120000 index 000000000..87fdc0b30 --- /dev/null +++ b/scripts/bin/dstack @@ -0,0 +1 @@ +dstack.py \ No newline at end of file From 42654833b257423a35d7de00cd58221efa62a0d0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 13:11:01 +0000 Subject: [PATCH 063/435] Refactor Makefile --- Makefile | 17 ++--- build.sh | 42 +---------- dist.sh | 32 -------- .../recipes-core/dstack-guest/dstack-guest.bb | 4 +- ...ack-rootfs-dev.bb => dstack-dev-rootfs.bb} | 0 .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 4 +- mkimage.sh | 75 +++++++++++++++++++ 7 files changed, 87 insertions(+), 87 deletions(-) delete mode 100755 dist.sh rename meta-dstack/recipes-core/images/{dstack-rootfs-dev.bb => dstack-dev-rootfs.bb} (100%) create mode 100755 mkimage.sh diff --git a/Makefile b/Makefile index 43ea75b4b..7c20ca1f4 100644 --- a/Makefile +++ b/Makefile @@ -7,22 +7,19 @@ endif BUILD_DIR ?= bb-build BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx DIST_DIR ?= ${BUILD_DIR}/dist +export BUILD_DIR +export DIST_DIR -IMAGE_FILES = dstack-initramfs.cpio.gz \ - dstack-rootfs-tdx.cpio \ - dstack-rootfs-dev-tdx.cpio \ - bzImage \ - ovmf.fd - -ABS_IMAGE_FILES = $(addprefix ${BUILD_IMAGES_DIR}/, ${IMAGE_FILES}) +DIST_NAMES ?= dstack dstack-dev +ROOTFS_IMAGE_NAMES = $(addsuffix -rootfs,${DIST_NAMES}) all: dist dist: images - DIST_DIR=${DIST_DIR} BUILD_DIR=${BUILD_DIR} ./dist.sh + $(foreach dist_name,${DIST_NAMES},./mkimage.sh --dist-name $(dist_name);) images: - bitbake dstack-initramfs dstack-rootfs dstack-rootfs-dev dstack-ovmf + bitbake dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) test: make images dist @@ -31,7 +28,7 @@ clean: git clean -xdff clean-dstack: - bitbake -c cleansstate dstack-guest dstack-rootfs dstack-rootfs-dev + bitbake -c cleansstate dstack-guest $(ROOTFS_IMAGE_NAMES) clean-initrd: bitbake -c cleansstate dstack-initramfs diff --git a/build.sh b/build.sh index ab7e82f60..a12b95e78 100755 --- a/build.sh +++ b/build.sh @@ -105,49 +105,9 @@ build_host() { } # Step 2: build guest images -make_image_dist() { - local img_name=$1 - local rootfs_name=$2 - local encfs=$3 - local img_dist_dir=$IMAGES_DIR/$img_name - local rootfs_hash - - mkdir -p $img_dist_dir - rootfs_hash=$(sha256sum "$IMAGE_TMP_DIR/$rootfs_name.cpio" | awk '{print $1}') - cat < $img_dist_dir/metadata.json -{ - "bios": "ovmf.fd", - "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.fde=${encfs} panic=1 systemd.unified_cgroup_hierarchy=0", - "initrd": "initramfs.cpio.gz", - "rootfs": "rootfs.iso", - "rootfs_hash": "$rootfs_hash" -} -EOF - - cp $IMAGE_TMP_DIR/ovmf.fd $img_dist_dir/ - cp $IMAGE_TMP_DIR/bzImage $img_dist_dir/ - cp $IMAGE_TMP_DIR/initramfs.cpio.gz $img_dist_dir/ - cp $IMAGE_TMP_DIR/$rootfs_name.iso $img_dist_dir/rootfs.iso -} - build_guest() { echo "Building guest images" - if [ -z "$BBPATH" ]; then - echo 'BBPATH is not set. Run `source dev-setup` in the meta-dstack/ directory' - pushd $SCRIPT_DIR/ - source dev-setup - popd - fi - DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) - IMAGE_NAME=dstack-$DSTACK_VERSION - IMAGE_TMP_DIR=`pwd`/tmp/images/$IMAGE_NAME - - echo "Building $IMAGE_NAME" - make -C $META_DIR dist DIST_DIR=$IMAGE_TMP_DIR - - make_image_dist $IMAGE_NAME rootfs 1 - make_image_dist $IMAGE_NAME-dev rootfs-dev 0 + make -C $META_DIR dist DIST_DIR=$IMAGES_DIR } # Step 3: make certs diff --git a/dist.sh b/dist.sh deleted file mode 100755 index ea558dc9e..000000000 --- a/dist.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -set -e - -BUILD_DIR=${BUILD_DIR:-build} -DIST_DIR=${DIST_DIR:-${BUILD_DIR}/dist} - -INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-tdx.cpio -ROOTFS_IMAGE_DEV=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-rootfs-dev-tdx.cpio -KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage -OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd - -echo "Copying initramfs..." -rm -rf ${DIST_DIR}/ -mkdir -p ${DIST_DIR}/ -cp $INITRAMFS_IMAGE ${DIST_DIR}/initramfs.cpio.gz - -echo "Copying kernel..." -cp $KERNEL_IMAGE ${DIST_DIR}/ - -echo "Copying OVMF firmware..." -cp $OVMF_FIRMWARE ${DIST_DIR}/ - -echo "Making rootfs.iso..." -mkdir -p ${BUILD_DIR}/rootfs/ -mkdir -p ${BUILD_DIR}/rootfs-dev/ -cp $ROOTFS_IMAGE ${BUILD_DIR}/rootfs/rootfs.cpio -cp $ROOTFS_IMAGE ${DIST_DIR}/rootfs.cpio -cp $ROOTFS_IMAGE_DEV ${BUILD_DIR}/rootfs-dev/rootfs.cpio -cp $ROOTFS_IMAGE_DEV ${DIST_DIR}/rootfs-dev.cpio -mkisofs -o ${DIST_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${BUILD_DIR}/rootfs/ -mkisofs -o ${DIST_DIR}/rootfs-dev.iso --max-iso9660-filenames -input-charset utf-8 ${BUILD_DIR}/rootfs-dev/ diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 44b455edc..3581f5a8f 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -7,8 +7,8 @@ inherit systemd update-rc.d REPO_ROOT = "${THISDIR}/../../.." -SRC_DIR = "${@oe.utils.conditional('DSTACK_SRC_URI', '', '${REPO_ROOT}/dstack', 'git', d)}" -SRC_URI = "${@oe.utils.conditional('DSTACK_SRC_URI', '', 'file://${REPO_ROOT}/dstack', '${DSTACK_SRC_URI}', d)}" +SRC_DIR = '${REPO_ROOT}/dstack' +SRC_URI = 'file://${REPO_ROOT}/dstack' SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.bb b/meta-dstack/recipes-core/images/dstack-dev-rootfs.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs-dev.bb rename to meta-dstack/recipes-core/images/dstack-dev-rootfs.bb diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb index f7b7bf520..87a72993f 100644 --- a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb +++ b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb @@ -7,8 +7,8 @@ inherit module REPO_ROOT = "${THISDIR}/../../.." -SRC_DIR = "${@oe.utils.conditional('DSTACK_SRC_URI', '', '${REPO_ROOT}/dstack/mod-tdx-guest', 'git/mod-tdx-guest', d)}" -SRC_URI = "${@oe.utils.conditional('DSTACK_SRC_URI', '', 'file://${REPO_ROOT}/dstack', '${DSTACK_SRC_URI}', d)}" +SRC_DIR = '${REPO_ROOT}/dstack/mod-tdx-guest' +SRC_URI = 'file://${REPO_ROOT}/dstack' SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" diff --git a/mkimage.sh b/mkimage.sh new file mode 100755 index 000000000..3f970c710 --- /dev/null +++ b/mkimage.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -e + +# Parse command line arguments +while [ $# -gt 0 ]; do + case "$1" in + --dist-name) + DIST_NAME="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 --image-name NAME --dist-name NAME [--dev]" + exit 1 + ;; + esac +done + +# Validate required arguments +if [ -z "$DIST_NAME" ]; then + echo "Error: --dist-name is required" + exit 1 +fi + + +if [[ "$DIST_NAME" == *-dev ]]; then + ENCFS=0 +else + ENCFS=1 +fi + +BUILD_DIR=${BUILD_DIR:-build} +DIST_DIR=${DIST_DIR:-${BUILD_DIR}/dist} + +ROOTFS_IMAGE_NAME=${DIST_NAME}-rootfs +WORK_DIR=${BUILD_DIR}/${ROOTFS_IMAGE_NAME}.tmp +INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz +ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/${ROOTFS_IMAGE_NAME}-tdx.cpio +KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage +OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd +ROOTFS_HASH=$(sha256sum "$ROOTFS_IMAGE" | awk '{print $1}') +DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) +OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} + +mkdir -p ${WORK_DIR} + +verbose() { + echo "$@" + $@ +} + +Q=verbose + +$Q rm -rf ${OUTPUT_DIR}/ +$Q mkdir -p ${OUTPUT_DIR}/ +$Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz +$Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ +$Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ + +$Q mkdir -p ${WORK_DIR}/rootfs/ +$Q cp $ROOTFS_IMAGE ${WORK_DIR}/rootfs/rootfs.cpio +$Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.cpio +$Q mkisofs -o ${OUTPUT_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${WORK_DIR}/rootfs/ + +echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" +cat < ${OUTPUT_DIR}/metadata.json +{ + "bios": "ovmf.fd", + "kernel": "bzImage", + "cmdline": "console=ttyS0 init=/init dstack.fde=${ENCFS} panic=1 systemd.unified_cgroup_hierarchy=0", + "initrd": "initramfs.cpio.gz", + "rootfs": "rootfs.iso", + "rootfs_hash": "$ROOTFS_HASH" +} +EOF From b9be0c2a3cf6777fc89c18b4a91700afe06cef70 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 13:24:21 +0000 Subject: [PATCH 064/435] Flexible Makefile --- Makefile | 10 +++------- mk.d/.gitignore | 0 2 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 mk.d/.gitignore diff --git a/Makefile b/Makefile index 7c20ca1f4..3e8366f28 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(BBPATH),) $(error BBPATH is not set. Run `source dev-setup` first) endif -.PHONY: all dist emu clean clean-dstack images +.PHONY: all dist clean-dstack clean-initrd images BUILD_DIR ?= bb-build BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx @@ -15,18 +15,14 @@ ROOTFS_IMAGE_NAMES = $(addsuffix -rootfs,${DIST_NAMES}) all: dist +-include $(wildcard mk.d/*.mk) + dist: images $(foreach dist_name,${DIST_NAMES},./mkimage.sh --dist-name $(dist_name);) images: bitbake dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) -test: - make images dist - -clean: - git clean -xdff - clean-dstack: bitbake -c cleansstate dstack-guest $(ROOTFS_IMAGE_NAMES) diff --git a/mk.d/.gitignore b/mk.d/.gitignore new file mode 100644 index 000000000..e69de29bb From e940004c835b1119b5cbc7c54a99e0c25a0609f5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 13:47:14 +0000 Subject: [PATCH 065/435] Extensible dev-setup --- dev-setup | 18 ++++++++++++------ setup.d/.gitignore | 0 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 setup.d/.gitignore diff --git a/dev-setup b/dev-setup index cc813f3b4..8ac29caef 100755 --- a/dev-setup +++ b/dev-setup @@ -1,4 +1,6 @@ #!/bin/bash +shopt -s nullglob + if [ -n "$BASH_SOURCE" ]; then THIS_SCRIPT=$BASH_SOURCE elif [ -n "$ZSH_NAME" ]; then @@ -14,21 +16,25 @@ fi THIS_SCRIPT=$(realpath "$THIS_SCRIPT") THIS_DIR=$(dirname "$THIS_SCRIPT") -pushd . -source $THIS_DIR/poky/oe-init-build-env $THIS_DIR/bb-build - -bitbake-layers add-layer \ - $THIS_DIR/meta-confidential-compute \ +LAYERS="$THIS_DIR/meta-confidential-compute \ $THIS_DIR/meta-openembedded/meta-oe \ $THIS_DIR/meta-openembedded/meta-python \ $THIS_DIR/meta-openembedded/meta-networking \ $THIS_DIR/meta-openembedded/meta-filesystems \ $THIS_DIR/meta-virtualization \ $THIS_DIR/meta-rust-bin \ - $THIS_DIR/meta-dstack \ + $THIS_DIR/meta-dstack" +pushd . +source $THIS_DIR/poky/oe-init-build-env $THIS_DIR/bb-build popd +for script in $THIS_DIR/setup.d/*.sh; do + source "$script" +done + +bitbake-layers add-layer $LAYERS + # Add scripts/bin to PATH if not already present if [[ ":$PATH:" != *":$THIS_DIR/scripts/bin:"* ]]; then export PATH=$PATH:$THIS_DIR/scripts/bin diff --git a/setup.d/.gitignore b/setup.d/.gitignore new file mode 100644 index 000000000..e69de29bb From b2c36555d0027f2c0dcb23f1e00eb1d14a8baae0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 14:15:09 +0000 Subject: [PATCH 066/435] chore: devsetup --- build.sh | 24 ++++++++++++------------ dev-setup | 14 +++++++++++--- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/build.sh b/build.sh index a12b95e78..315b01364 100755 --- a/build.sh +++ b/build.sh @@ -82,20 +82,18 @@ if [ -f $CONFIG_FILE ]; then exit 1 fi rm -f build-config.sh.tpl -else - mv build-config.sh.tpl $CONFIG_FILE - echo "Config file $CONFIG_FILE created, please edit it to configure the build" - exit 1 -fi - -if [ -z "$TPROXY_SERVE_PORT" ]; then - TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} + if [ -z "$TPROXY_SERVE_PORT" ]; then + TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} + fi + TAPPD_PORT=8090 +else + if [ x"$ACTION" != x"guest" ]; then + mv build-config.sh.tpl $CONFIG_FILE + echo "Config file $CONFIG_FILE created, please edit it to configure the build" + exit 1 + fi fi -TAPPD_PORT=8090 - -TPROXY_WG_KEY=$(wg genkey) -TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) # Step 1: build binaries build_host() { @@ -120,6 +118,8 @@ build_certs() { build_cfg() { echo "Building config files" + TPROXY_WG_KEY=$(wg genkey) + TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) # kms cat < kms.toml log_level = "info" diff --git a/dev-setup b/dev-setup index 8ac29caef..5201369eb 100755 --- a/dev-setup +++ b/dev-setup @@ -25,14 +25,22 @@ LAYERS="$THIS_DIR/meta-confidential-compute \ $THIS_DIR/meta-rust-bin \ $THIS_DIR/meta-dstack" -pushd . -source $THIS_DIR/poky/oe-init-build-env $THIS_DIR/bb-build -popd +if [ -z "$1" ]; then + BUILD_DIR=$THIS_DIR/bb-build +else + BUILD_DIR=$(realpath "$1") +fi + +OE_INIT=$THIS_DIR/poky/oe-init-build-env for script in $THIS_DIR/setup.d/*.sh; do source "$script" done +pushd . +source $OE_INIT $BUILD_DIR +popd + bitbake-layers add-layer $LAYERS # Add scripts/bin to PATH if not already present From c21142b77a6966608cc205199177336ba85d098e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 Nov 2024 14:36:41 +0000 Subject: [PATCH 067/435] Add git rev to image metadata.json --- mkimage.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 3f970c710..e7df50fc1 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -62,6 +62,7 @@ $Q cp $ROOTFS_IMAGE ${WORK_DIR}/rootfs/rootfs.cpio $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.cpio $Q mkisofs -o ${OUTPUT_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${WORK_DIR}/rootfs/ +GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" cat < ${OUTPUT_DIR}/metadata.json { @@ -70,6 +71,7 @@ cat < ${OUTPUT_DIR}/metadata.json "cmdline": "console=ttyS0 init=/init dstack.fde=${ENCFS} panic=1 systemd.unified_cgroup_hierarchy=0", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", - "rootfs_hash": "$ROOTFS_HASH" + "rootfs_hash": "$ROOTFS_HASH", + "git_revision": "$GIT_REVISION" } EOF From a045f77fbd0b00400b4e653bee74d5ba3a06cefa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 1 Dec 2024 01:46:57 +0000 Subject: [PATCH 068/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 212405bf5..82bebce1e 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 212405bf5dd84f1a5d6ae0b056ca41224b037c6c +Subproject commit 82bebce1efcd5ab556e869b69bea4a8ed652928c From 9f0654d080f1caab9f0123f3e4964f8b11718c89 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Dec 2024 09:38:50 +0000 Subject: [PATCH 069/435] build.sh: Auto source dev-setup --- build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sh b/build.sh index 315b01364..f7079460c 100755 --- a/build.sh +++ b/build.sh @@ -105,6 +105,9 @@ build_host() { # Step 2: build guest images build_guest() { echo "Building guest images" + if [ -z "$BBPATH" ]; then + source $SCRIPT_DIR/dev-setup.sh + fi make -C $META_DIR dist DIST_DIR=$IMAGES_DIR } From e30ea53ed6c222b382f3bfb350a2f7ee4e30ee90 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Dec 2024 10:44:26 +0000 Subject: [PATCH 070/435] Add binary supervisor --- build.sh | 2 +- dstack | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index f7079460c..c7391fcda 100755 --- a/build.sh +++ b/build.sh @@ -99,7 +99,7 @@ fi build_host() { echo "Building binaries" (cd $DSTACK_DIR && cargo build --release) - cp $DSTACK_DIR/target/release/{tproxy,kms,teepod,certbot,ct_monitor} . + cp $DSTACK_DIR/target/release/{tproxy,kms,teepod,certbot,ct_monitor,supervisor} . } # Step 2: build guest images diff --git a/dstack b/dstack index 82bebce1e..bc2a3a5e3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 82bebce1efcd5ab556e869b69bea4a8ed652928c +Subproject commit bc2a3a5e3909b90c15dc24868ddd7bd207fa25ad From d1a3372a3c6970e221de9ec9c316dc02698118f5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Dec 2024 16:14:17 +0000 Subject: [PATCH 071/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index bc2a3a5e3..5b0eb437f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit bc2a3a5e3909b90c15dc24868ddd7bd207fa25ad +Subproject commit 5b0eb437ff69a469ee6e313a385dc9d6772a904f From c18005806eebf41a3bb47b7c8f70526ab3043bba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Dec 2024 17:31:01 +0000 Subject: [PATCH 072/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 5b0eb437f..47fe66676 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 5b0eb437ff69a469ee6e313a385dc9d6772a904f +Subproject commit 47fe666760b6ff8846906570bf8febb7dce8bb15 From 02a5131f63b7b50d9828a7298f194b8f08789ca5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 5 Dec 2024 13:04:09 +0000 Subject: [PATCH 073/435] v0.3.1 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 47fe66676..76d5cc62e 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 47fe666760b6ff8846906570bf8febb7dce8bb15 +Subproject commit 76d5cc62ecfd11a1e5ec2499626e2fb92e6c5c36 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index b3a2d9b18..1ff2a2e59 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.3.0" +DISTRO_VERSION = "0.3.1" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From 31943cefd16388a817297d13be3120ae2293c177 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 6 Dec 2024 10:56:21 +0000 Subject: [PATCH 074/435] correct the filename --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index c7391fcda..10a3d835a 100755 --- a/build.sh +++ b/build.sh @@ -106,7 +106,7 @@ build_host() { build_guest() { echo "Building guest images" if [ -z "$BBPATH" ]; then - source $SCRIPT_DIR/dev-setup.sh + source $SCRIPT_DIR/dev-setup fi make -C $META_DIR dist DIST_DIR=$IMAGES_DIR } From 3e4bffb1069cd214f96081484f18d971ece52ace Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Dec 2024 02:43:40 +0000 Subject: [PATCH 075/435] Support for reproducible build --- .gitmodules | 2 +- Makefile | 7 +- build.sh | 91 +++++----- dev-setup | 4 + .../images/dstack-rootfs-base.inc | 2 +- meta-virtualization | 2 +- mkimage.sh | 59 ++++++- repro-build/.gitignore | 4 + repro-build/Dockerfile.repro | 30 ++++ repro-build/check.sh | 164 ++++++++++++++++++ repro-build/repro-build.sh | 84 +++++++++ 11 files changed, 387 insertions(+), 62 deletions(-) create mode 100644 repro-build/.gitignore create mode 100644 repro-build/Dockerfile.repro create mode 100755 repro-build/check.sh create mode 100755 repro-build/repro-build.sh diff --git a/.gitmodules b/.gitmodules index d6326a8f2..89b5efdc4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ url = https://github.com/Dstack-TEE/meta-confidential-compute.git [submodule "meta-virtualization"] path = meta-virtualization - url = https://github.com/lgirdk/meta-virtualization + url = https://github.com/Dstack-TEE/meta-virtualization.git [submodule "meta-openembedded"] path = meta-openembedded url = https://github.com/openembedded/meta-openembedded diff --git a/Makefile b/Makefile index 3e8366f28..30d448c0f 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,9 @@ endif .PHONY: all dist clean-dstack clean-initrd images -BUILD_DIR ?= bb-build -BUILD_IMAGES_DIR ?= ${BUILD_DIR}/tmp/deploy/images/tdx -DIST_DIR ?= ${BUILD_DIR}/dist -export BUILD_DIR +BB_BUILD_DIR ?= bb-build +DIST_DIR ?= ${BB_BUILD_DIR}/dist +export BB_BUILD_DIR export DIST_DIR DIST_NAMES ?= dstack dstack-dev diff --git a/build.sh b/build.sh index 10a3d835a..9f60dfeae 100755 --- a/build.sh +++ b/build.sh @@ -11,13 +11,31 @@ CERBOT_WORKDIR=$RUN_DIR/certbot KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ -if [ -f ./build-config.sh ]; then - CONFIG_FILE=./build-config.sh -else - CONFIG_FILE=$SCRIPT_DIR/build-config.sh -fi +CONFIG_FILE=./build-config.sh + +check_config() { + local template_file=$1 + local config_file=$2 + + # extract all variables in template file + local variables=$(grep -oE '^\s*[A-Z_]+=' $template_file | sort) + + # check if each variable is set in config file + local var missing=0 + for var in $variables; do + if ! grep -qE "^\s*$var" $config_file; then + echo "Variable $var is not set in $config_file" + missing=1 + fi + done + if [ $missing -ne 0 ]; then + return 1 + fi + return 0 +} -cat < build-config.sh.tpl +require_config() { + cat < build-config.sh.tpl # base domain of kms rpc and tproxy rpc # 1022.kvin.wang resolves to 10.0.2.2 which is host ip at the # cvm point of view @@ -51,49 +69,26 @@ CF_API_TOKEN= CF_ZONE_ID= ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory EOF - -check_config() { - local template_file=$1 - local config_file=$2 - - # extract all variables in template file - local variables=$(grep -oE '^\s*[A-Z_]+=' $template_file | sort) - - # check if each variable is set in config file - local var missing=0 - for var in $variables; do - if ! grep -qE "^\s*$var" $config_file; then - echo "Variable $var is not set in $config_file" - missing=1 + if [ -f $CONFIG_FILE ]; then + source $CONFIG_FILE + # check if any variable in build-config.sh.tpl is not set in build-config.sh. + # This might occur if the build-config.sh is generated from and old repo. + check_config build-config.sh.tpl $CONFIG_FILE + if [ $? -ne 0 ]; then + exit 1 fi - done - if [ $missing -ne 0 ]; then - return 1 - fi - return 0 -} - -if [ -f $CONFIG_FILE ]; then - source $CONFIG_FILE - # check if any variable in build-config.sh.tpl is not set in build-config.sh. - # This might occur if the build-config.sh is generated from and old repo. - check_config build-config.sh.tpl $CONFIG_FILE - if [ $? -ne 0 ]; then - exit 1 - fi - rm -f build-config.sh.tpl + rm -f build-config.sh.tpl - if [ -z "$TPROXY_SERVE_PORT" ]; then - TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} - fi - TAPPD_PORT=8090 -else - if [ x"$ACTION" != x"guest" ]; then + if [ -z "$TPROXY_SERVE_PORT" ]; then + TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} + fi + TAPPD_PORT=8090 + else mv build-config.sh.tpl $CONFIG_FILE echo "Config file $CONFIG_FILE created, please edit it to configure the build" exit 1 fi -fi +} # Step 1: build binaries build_host() { @@ -106,9 +101,9 @@ build_host() { build_guest() { echo "Building guest images" if [ -z "$BBPATH" ]; then - source $SCRIPT_DIR/dev-setup + source $SCRIPT_DIR/dev-setup $1 fi - make -C $META_DIR dist DIST_DIR=$IMAGES_DIR + make -C $META_DIR dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} } # Step 3: make certs @@ -274,19 +269,23 @@ case $ACTION in build_host ;; guest) - build_guest + build_guest $2 ;; cfg) + require_config build_cfg ;; certs) + require_config build_certs ;; wg) + require_config build_wg ;; "") # If no action specified, build everything + require_config build_host build_guest build_certs diff --git a/dev-setup b/dev-setup index 5201369eb..b45f54057 100755 --- a/dev-setup +++ b/dev-setup @@ -30,6 +30,10 @@ if [ -z "$1" ]; then else BUILD_DIR=$(realpath "$1") fi +if [ ! -f "$BUILD_DIR/conf/local.conf" ]; then + mkdir -p "$BUILD_DIR/conf" + cp -f "$THIS_DIR/bb-build/conf/local.conf" "$BUILD_DIR/conf/local.conf" +fi OE_INIT=$THIS_DIR/poky/oe-init-build-env diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 04af9d9b8..38dc15ea3 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -8,7 +8,7 @@ IMAGE_INSTALL = "\ systemd \ netbase \ iptables \ - docker \ + docker-moby \ docker-compose \ dstack-prebuilt \ tdx-guest-ko \ diff --git a/meta-virtualization b/meta-virtualization index 6f3c1d8f9..bce9962c5 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit 6f3c1d8f90947408a6587be222fec575a1ca5195 +Subproject commit bce9962c515089d62082953b795c0e80083c96e2 diff --git a/mkimage.sh b/mkimage.sh index e7df50fc1..b91280376 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -10,7 +10,7 @@ while [ $# -gt 0 ]; do ;; *) echo "Unknown option: $1" - echo "Usage: $0 --image-name NAME --dist-name NAME [--dev]" + echo "Usage: $0 --dist-name NAME" exit 1 ;; esac @@ -29,15 +29,15 @@ else ENCFS=1 fi -BUILD_DIR=${BUILD_DIR:-build} -DIST_DIR=${DIST_DIR:-${BUILD_DIR}/dist} +BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) +DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) ROOTFS_IMAGE_NAME=${DIST_NAME}-rootfs -WORK_DIR=${BUILD_DIR}/${ROOTFS_IMAGE_NAME}.tmp -INITRAMFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/${ROOTFS_IMAGE_NAME}-tdx.cpio -KERNEL_IMAGE=${BUILD_DIR}/tmp/deploy/images/tdx/bzImage -OVMF_FIRMWARE=${BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd +WORK_DIR=${BB_BUILD_DIR}/${ROOTFS_IMAGE_NAME}.tmp +INITRAMFS_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz +ROOTFS_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/${ROOTFS_IMAGE_NAME}-tdx.cpio +KERNEL_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/bzImage +OVMF_FIRMWARE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd ROOTFS_HASH=$(sha256sum "$ROOTFS_IMAGE" | awk '{print $1}') DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} @@ -51,6 +51,32 @@ verbose() { Q=verbose +makeiso() { + export SOURCE_DATE_EPOCH="$(date -d20010101 -u +%s)" + folder="$1" + output_filename="$2" + file_mode=0444 + + list="$(mktemp)" + (cd "$folder"; for f in *; do printf "%s\n" "$f=$PWD/$f"; done) | LC_ALL=C sort >"$list" + + xorriso \ + -preparer_id xorriso \ + -volume_date 'all_file_dates' "=$SOURCE_DATE_EPOCH" \ + -as mkisofs \ + -iso-level 3 \ + -graft-points \ + -full-iso9660-filenames \ + -joliet \ + -file-mode $file_mode \ + -uid 0 \ + -gid 0 \ + -path-list "$list" \ + -output "$output_filename" + + rm -f "$list" +} + $Q rm -rf ${OUTPUT_DIR}/ $Q mkdir -p ${OUTPUT_DIR}/ $Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz @@ -60,7 +86,7 @@ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ $Q mkdir -p ${WORK_DIR}/rootfs/ $Q cp $ROOTFS_IMAGE ${WORK_DIR}/rootfs/rootfs.cpio $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.cpio -$Q mkisofs -o ${OUTPUT_DIR}/rootfs.iso --max-iso9660-filenames -input-charset utf-8 ${WORK_DIR}/rootfs/ +$Q makeiso ${WORK_DIR}/rootfs/ ${OUTPUT_DIR}/rootfs.iso GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" @@ -75,3 +101,18 @@ cat < ${OUTPUT_DIR}/metadata.json "git_revision": "$GIT_REVISION" } EOF + +echo "Generating md5sum.txt and sha256sum.txt to ${OUTPUT_DIR}/" +pushd ${OUTPUT_DIR}/ +find . -type f -not -name md5sum.txt -not -name sha256sum.txt -exec md5sum {} + | sort -k 2 > md5sum.txt +find . -type f -not -name md5sum.txt -not -name sha256sum.txt -exec sha256sum {} + | sort -k 2 > sha256sum.txt +popd + +if [ x$DSTACK_TAR_RELEASE = x1 ]; then + echo "Archiving the output directory to ${OUTPUT_DIR}.tar.gz" + if [ x$DSTACK_TAR_EXCLUDE_ROOTFS_CPIO = x1 ]; then + TAR_ARGS=--exclude=rootfs.cpio + fi + (cd ${OUTPUT_DIR} && tar -czf ${OUTPUT_DIR}.tar.gz ${TAR_ARGS} .) + echo +fi diff --git a/repro-build/.gitignore b/repro-build/.gitignore new file mode 100644 index 000000000..4c119cb1f --- /dev/null +++ b/repro-build/.gitignore @@ -0,0 +1,4 @@ +/build-a +/build-b +/dist +*.tar.gz diff --git a/repro-build/Dockerfile.repro b/repro-build/Dockerfile.repro new file mode 100644 index 000000000..3af41ba4d --- /dev/null +++ b/repro-build/Dockerfile.repro @@ -0,0 +1,30 @@ +FROM ubuntu:22.04@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97 + +# Prevent timezone prompt by setting noninteractive frontend and configuring tzdata +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + TZDATA=Etc/UTC \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 + +# Set timezone +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Install required packages +RUN apt update && apt install -y \ + build-essential \ + chrpath \ + diffstat \ + lz4 \ + wireguard-tools \ + python3 \ + locales \ + git \ + file \ + gawk \ + wget \ + libclang-dev \ + xorriso + +# Generate locale for bitbake +RUN locale-gen en_US.UTF-8 diff --git a/repro-build/check.sh b/repro-build/check.sh new file mode 100755 index 000000000..38327ba01 --- /dev/null +++ b/repro-build/check.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +THIS_DIR=$(cd $(dirname $0); pwd) + +# Create a unique temporary directory and clean it up on exit +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +ACTION=$1 + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +ROOTFS_PATH=tmp/work/tdx-poky-linux/dstack-rootfs/1.0/rootfs +BUILD_DIR_A=${1:-${THIS_DIR}/build-a} +BUILD_DIR_B=${2:-${THIS_DIR}/build-b} +BB_DIR_A=${BB_DIR_A:-${BUILD_DIR_A}/bb-build} +BB_DIR_B=${BB_DIR_B:-${BUILD_DIR_B}/bb-build} +ROOTFS_A=${BB_DIR_A}/${ROOTFS_PATH} +ROOTFS_B=${BB_DIR_B}/${ROOTFS_PATH} + +check_files() { + local path_a="$1" + local path_b="$2" + local rel_path="$3" + + if [ ! -e "$path_a" ]; then + if [ -e "$path_b" ]; then + echo -e "${RED}File missing in A: $path_b" + return 1 + fi + return 0 + fi + if [ ! -e "$path_b" ]; then + if [ -e "$path_a" ]; then + echo -e "${RED}File missing in B: $path_a" + return 1 + fi + return 0 + fi + + if [ -d "$path_a" ]; then + if [ ! -d "$path_b" ]; then + echo -e "${RED}Path type mismatch: $rel_path is directory in A but not in B${NC}" + return 1 + fi + + local differences=0 + while IFS= read -r -d '' file; do + local rel_file="${file#$path_a/}" + check_files "$path_a/$rel_file" "$path_b/$rel_file" "$rel_path/$rel_file" + differences=$((differences + $?)) + done < <(find "$path_a" -maxdepth 1 -mindepth 1 -print0) + + return $differences + else + if [ ! -f "$path_b" ] && [ ! -L "$path_b" ]; then + echo -e "${RED}Path type mismatch: $rel_path is file in A but not in B${NC}" + ls -l $path_a + ls -l $path_b + return 1 + fi + + # Skip symlinks + if [ -L "$path_a" ] || [ -L "$path_b" ]; then + local link_a=$(readlink "$path_a") + local link_b=$(readlink "$path_b") + if [ "$link_a" != "$link_b" ]; then + echo -e "${RED}Symlink mismatch for $rel_path:${NC}" + echo -e "${RED}A: $link_a${NC}" + echo -e "${RED}B: $link_b${NC}" + return 1 + fi + return 0 + fi + + # Compare regular files + local hash_a=$(md5sum "$path_a" | cut -d' ' -f1) + local hash_b=$(md5sum "$path_b" | cut -d' ' -f1) + + if [ "$hash_a" != "$hash_b" ]; then + echo -e "${RED}Hash mismatch for $rel_path:${NC}" + echo -e "${RED}A: $hash_a${NC}" + echo -e "${RED}B: $hash_b${NC}" + analyze "$path_a" "$path_b" + return 1 + else + echo -e "${GREEN}Match for $rel_path${NC}" + return 0 + fi + fi +} + +analyze() { + local BIN_A=$1 + local BIN_B=$2 + + echo -e "\n${GREEN}Analyzing $BIN_A...${NC}" + ls -l $BIN_A + file $BIN_A + readelf -n $BIN_A + readelf -p .comment $BIN_A 2>/dev/null || true + + echo -e "\n${GREEN}Analyzing $BIN_B...${NC}" + ls -l $BIN_B + file $BIN_B + readelf -n $BIN_B + readelf -p .comment $BIN_B 2>/dev/null || true + + echo -e "\n${GREEN}Binary diff analysis:${NC}" + # Create hex dumps and compare + objdump -s $BIN_A > "$TEMP_DIR/bin_a.hex" + objdump -s $BIN_B > "$TEMP_DIR/bin_b.hex" + + echo "Differences:" + diff -u "$TEMP_DIR/bin_a.hex" "$TEMP_DIR/bin_b.hex" | grep -A 5 '^[^+-]' | head -n 100 +} + +check_images() { + echo -e "${YELLOW}Checking image files...${NC}" + find $BUILD_DIR_A/images -type f | while read file_a; do + rel_path=$(echo ${file_a} | sed "s#${BUILD_DIR_A}/images/##g") + file_b=$BUILD_DIR_B/images/$rel_path + if [ ! -f "$file_b" ]; then + echo -e "${RED}$rel_path is not found in $BUILD_DIR_B/images/${NC}" + continue + fi + hash_a=$(md5sum $file_a | cut -d' ' -f 1) + hash_b=$(md5sum $file_b | cut -d' ' -f 1) + if [ "$hash_a" != "$hash_b" ]; then + echo -e "${RED}Hash mismatch for $rel_path:${NC}" + echo -e "${RED}$hash_a $file_a${NC}" + echo -e "${RED}$hash_b $file_b${NC}" + return 1 + else + echo -e "${GREEN}Match for $rel_path${NC}" + fi + done +} + +check() { + echo -e "${YELLOW}Checking reproducibility...${NC}" + + if check_images; then + return 0 + fi + + echo -e "${YELLOW}Checking rootfs...${NC}: $ROOTFS_A -> $ROOTFS_B" + local differences=0 + check_files "$ROOTFS_A" "$ROOTFS_B" "" + differences=$? + + if [ $differences -eq 0 ]; then + echo -e "\n${GREEN}All files are identical!${NC}" + return 0 + else + echo -e "\n${RED}Found $differences differences${NC}" + return 1 + fi +} + +check diff --git a/repro-build/repro-build.sh b/repro-build/repro-build.sh new file mode 100755 index 000000000..676bcb2ef --- /dev/null +++ b/repro-build/repro-build.sh @@ -0,0 +1,84 @@ +#! /bin/bash +set -e + +usage() { + echo "Usage: $0 [-n]" + echo " -n: Don't check reproducibility" +} + +NO_CHECK=0 +while getopts ":n" opt; do + case $opt in + n) + NO_CHECK=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + esac +done + + +BUILDER_NAME=dstack-build +THIS_DIR=$(cd $(dirname $0); pwd) +REPO_ROOT=$(dirname $THIS_DIR) +GIT_DIR=$REPO_ROOT + +HOST_BUILD_DIR_A=${THIS_DIR}/build-a +HOST_BUILD_DIR_B=${THIS_DIR}/build-b + +# guest dirs +GUEST_BUILD_DIR=/dstack-build +GUEST_SRC_DIR=/meta-dstack + +cd $THIS_DIR + +mkdir -p .dummy +(cd .dummy && docker build --platform linux/amd64 -t $BUILDER_NAME -f ../Dockerfile.repro .) +rm -rf .dummy + +build_to() { + mkdir -p $1 + BUILD_CMD="${2} ${GUEST_SRC_DIR}/build.sh guest ./bb-build" + docker run --platform linux/amd64 --rm \ + --user $(id -u):$(id -g) \ + -v $REPO_ROOT:$GUEST_SRC_DIR \ + -v $1:$GUEST_BUILD_DIR \ + -w $GUEST_BUILD_DIR \ + $BUILDER_NAME bash -e -c "$BUILD_CMD" +} + +build_to $HOST_BUILD_DIR_A DSTACK_TAR_RELEASE=1 + +DIST_DIR=${THIS_DIR}/dist +mkdir -p $DIST_DIR +mv $HOST_BUILD_DIR_A/images/*.tar.gz $DIST_DIR/ +if [ $NO_CHECK -eq 0 ]; then + build_to $HOST_BUILD_DIR_B + ${THIS_DIR}/check.sh $HOST_BUILD_DIR_A $HOST_BUILD_DIR_B +fi + +if [[ -n $(git -C $GIT_DIR status --porcelain) ]]; then + echo "The working tree is not clean, skip generating reproducible build command" + exit 0 +fi + +echo "Reproducible build commands:" +echo "===========================" +cat < Date: Mon, 9 Dec 2024 02:53:47 +0000 Subject: [PATCH 076/435] Add Reproduible build in README.md --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 79d4dbdb9..65790d7dc 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,20 @@ This project implements Yocto layer and the overall build scripts for DStack Bas See https://github.com/Dstack-TEE/dstack for more details. +## Reproducible Build The Guest Image + +### Pre-requisites + +- X86_64 Linux system with Docker installed + +### Build commands + +```bash +git clone https://github.com/Dstack-TEE/meta-dstack.git +cd meta-dstack/repro-build/ +./repro-build.sh +``` + ## License This project is licensed under the MIT License. See the LICENSE file for more details. From 1a41bd391f26e91df7ec6ba76be5f66364a0e574 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Dec 2024 04:30:30 +0000 Subject: [PATCH 077/435] Add qemu-guest-agent --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 04af9d9b8..b08fd3a82 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -16,6 +16,7 @@ IMAGE_INSTALL = "\ wireguard-tools \ curl \ jq \ + qemu-guest-agent \ " # Do not pollute the initrd image with rootfs features From 47bc555ce42baefd82389b3e646d5be3f10ed1c1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Dec 2024 13:08:04 +0000 Subject: [PATCH 078/435] v0.3.2 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 76d5cc62e..bdfffbdc9 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 76d5cc62ecfd11a1e5ec2499626e2fb92e6c5c36 +Subproject commit bdfffbdc9c0251a5c7fb3f4256b77bf7a79c326f diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 1ff2a2e59..5122b095f 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.3.1" +DISTRO_VERSION = "0.3.2" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From 0622b3aacf6129f9f488b1527bb2326273d4093d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Dec 2024 10:18:33 +0000 Subject: [PATCH 079/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index bdfffbdc9..4e7d982bb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit bdfffbdc9c0251a5c7fb3f4256b77bf7a79c326f +Subproject commit 4e7d982bbd1814ffc56c5e106236d316d5867ed7 From 809bd5a27ad5114ce09ddcb5c432a90e800edefe Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 03:19:28 +0000 Subject: [PATCH 080/435] Move rust build target to build dir --- build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 9f60dfeae..986f3a0bf 100755 --- a/build.sh +++ b/build.sh @@ -7,6 +7,7 @@ DSTACK_DIR=$SCRIPT_DIR/dstack CERTS_DIR=`pwd`/certs IMAGES_DIR=`pwd`/images RUN_DIR=`pwd`/run +RUST_BUILD_DIR=`pwd`/rust-target CERBOT_WORKDIR=$RUN_DIR/certbot KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ @@ -93,8 +94,8 @@ EOF # Step 1: build binaries build_host() { echo "Building binaries" - (cd $DSTACK_DIR && cargo build --release) - cp $DSTACK_DIR/target/release/{tproxy,kms,teepod,certbot,ct_monitor,supervisor} . + (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) + cp ${RUST_BUILD_DIR}/release/{tproxy,kms,teepod,certbot,ct_monitor,supervisor} . } # Step 2: build guest images From 4b97c0c6e1dc73a6d8a2db3842e25cb0c0528f3c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 03:20:43 +0000 Subject: [PATCH 081/435] Adapt image 0.3.3 --- build.sh | 5 +++++ meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 1 + .../images/dstack-initramfs-files/init | 15 +++++++++------ .../images/dstack-initramfs-files/kmfs-setup | 2 +- mkimage.sh | 4 +++- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/build.sh b/build.sh index 986f3a0bf..928c376a0 100755 --- a/build.sh +++ b/build.sh @@ -49,6 +49,8 @@ TEEPOD_CID_POOL_START=10000 TEEPOD_CID_POOL_SIZE=1000 # Whether port mapping from host to CVM is allowed TEEPOD_PORT_MAPPING_ENABLED=false +# Host API configuration, type of uint32 +TEEPOD_VSOCK_LISTEN_PORT=9080 KMS_RPC_LISTEN_PORT=9043 TPROXY_RPC_LISTEN_PORT=9010 @@ -212,6 +214,9 @@ range = [ base_domain = "$TPROXY_PUBLIC_DOMAIN" port = $TPROXY_SERVE_PORT tappd_port = $TAPPD_PORT + +[host_api] +port = $TEEPOD_VSOCK_LISTEN_PORT EOF cat < certbot.toml diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 5122b095f..879f2c8c9 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.3.2" +DISTRO_VERSION = "0.3.3" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 3581f5a8f..9f0ddec5e 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -39,6 +39,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -d ${D}${sysconfdir}/ install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index 3eabacd70..37aa6c75c 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -6,9 +6,11 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bin mkdir -p /proc mkdir -p /sys mkdir -p /run +mkdir -p /tapp mount -t proc proc /proc mount -t sysfs sysfs /sys mount -t tmpfs tmpfs /run +mount -t tmpfs tmpfs /tapp mkdir -p /dev mount -t devtmpfs devtmpfs /dev @@ -26,12 +28,13 @@ source /scripts/boot-vars source /scripts/kmfs-setup # mount move dirs -mkdir -p /root${HOST_SHARED} -mount --move ${HOST_SHARED} /root${HOST_SHARED} -mount --move /sys /root/sys -mount --move /proc /root/proc -mount --move /dev /root/dev -mount --move /run /root/run +mount_move_all() { + for dir in $@; do + mkdir -p /root/$dir + mount --move /$dir /root/$dir + done +} +mount_move_all sys proc dev run tapp echo "Switching to new root" exec switch_root /root /init diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 1530ba27e..9190c3758 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -14,7 +14,7 @@ ROOTFS_HD=/dev/vda ROOTFS_DIR=/root KMFS_WORK_DIR=/kmfs -SHARED_COPY=${KMFS_WORK_DIR}/config +SHARED_COPY=/tapp HOST_SHARED=/mnt/host-shared panic() { diff --git a/mkimage.sh b/mkimage.sh index b91280376..d33a6da65 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -98,7 +98,9 @@ cat < ${OUTPUT_DIR}/metadata.json "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$ROOTFS_HASH", - "git_revision": "$GIT_REVISION" + "version": "$DSTACK_VERSION", + "git_revision": "$GIT_REVISION", + "shared_ro": true } EOF From 9af0a4f40f718fedd55059ad1e765d3afac30d0d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 03:26:27 +0000 Subject: [PATCH 082/435] Add docker daemon config file --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index 4e7d982bb..d6fedbd95 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 4e7d982bbd1814ffc56c5e106236d316d5867ed7 +Subproject commit d6fedbd9523fec873b58c5fcd34e6ceca34453ed diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 9f0ddec5e..94d929996 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -35,11 +35,13 @@ do_compile[network] = "1" do_install() { install -d ${D}${bindir} + install -d ${D}${sysconfdir}/docker install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -d ${D}${sysconfdir}/ install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf From df193ce4dc6a32d0c2e13c456a51c2e91c9ad1d1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 05:22:45 +0000 Subject: [PATCH 083/435] Add journald config --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 ++ meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index d6fedbd95..6e25422a7 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d6fedbd9523fec873b58c5fcd34e6ceca34453ed +Subproject commit 6e25422a789122558d52374bd5a6c0246edcd798 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 879f2c8c9..b8393d724 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -7,6 +7,8 @@ DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.3.3" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" +VOLATILE_LOG_DIR = "no" +VOLATILE_TMP_DIR = "yes" PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 94d929996..ceec9a10a 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -36,12 +36,14 @@ do_compile[network] = "1" do_install() { install -d ${D}${bindir} install -d ${D}${sysconfdir}/docker + install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${S}/basefiles/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json + install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf From 1dceeb268ba9c98e14b5e1186042be04c2c5b371 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 19 Dec 2024 08:11:38 +0000 Subject: [PATCH 084/435] Add wg-checker.sh and debug tools --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 6 ++++-- meta-dstack/recipes-core/images/dstack-initramfs.bb | 2 ++ meta-dstack/recipes-core/images/dstack-rootfs-dev.inc | 2 +- mkimage.sh | 5 ++++- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dstack b/dstack index 6e25422a7..dccc7a656 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6e25422a789122558d52374bd5a6c0246edcd798 +Subproject commit dccc7a656d9be4b46897b8851a43e09068f470f0 diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index ceec9a10a..168fe36b9 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -13,8 +13,9 @@ SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" +DSTACK_SERVICES = "tappd.service tboot.service app-compose.service wg-checker.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" -SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','tappd.service tboot.service app-compose.service','',d)}" +SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" INITSCRIPT_PACKAGES += "${@bb.utils.contains('DISTRO_FEATURES','systemd','','${PN}',d)}" @@ -42,6 +43,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf @@ -55,7 +57,7 @@ do_install() { install -m 0644 ${S}/basefiles/tappd.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/tboot.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} - + install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d else install -d ${D}${sysconfdir}/init.d diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 11d898f1d..04a138bc1 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -6,6 +6,8 @@ PACKAGE_INSTALL = "\ base-files \ cryptsetup \ e2fsprogs-mke2fs \ + e2fsprogs-resize2fs \ + e2fsprogs-e2fsck \ tdx-guest-ko \ dstack-initramfs-files \ dstack-guest \ diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index da6cbd9b3..37fb8861e 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc -PACKAGE_INSTALL += "dropbear strace" +IMAGE_INSTALL += "packagegroup-core-ssh-dropbear strace tcpdump gdb gdbserver" IMAGE_FEATURES = "debug-tweaks" diff --git a/mkimage.sh b/mkimage.sh index d33a6da65..2fe09863c 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -25,8 +25,10 @@ fi if [[ "$DIST_NAME" == *-dev ]]; then ENCFS=0 + IS_DEV=true else ENCFS=1 + IS_DEV=false fi BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) @@ -100,7 +102,8 @@ cat < ${OUTPUT_DIR}/metadata.json "rootfs_hash": "$ROOTFS_HASH", "version": "$DSTACK_VERSION", "git_revision": "$GIT_REVISION", - "shared_ro": true + "shared_ro": true, + "is_dev": ${IS_DEV} } EOF From 9c2671a81509e597976b4f073aa681f8b7c5a48c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 19 Dec 2024 09:01:37 +0000 Subject: [PATCH 085/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index dccc7a656..e48569810 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit dccc7a656d9be4b46897b8851a43e09068f470f0 +Subproject commit e48569810c3b46e7127989122fd34f6642f1af70 From 9190dd44ee01c9e10eb1c9060e958754a3f58af5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 19 Dec 2024 14:27:34 +0000 Subject: [PATCH 086/435] Also encrypt dev instance --- dstack | 2 +- mkimage.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dstack b/dstack index e48569810..d8e121b82 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit e48569810c3b46e7127989122fd34f6642f1af70 +Subproject commit d8e121b82413e452bea4ee7eb0c855cc3bdaa402 diff --git a/mkimage.sh b/mkimage.sh index 2fe09863c..78b889c6a 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -24,10 +24,8 @@ fi if [[ "$DIST_NAME" == *-dev ]]; then - ENCFS=0 IS_DEV=true else - ENCFS=1 IS_DEV=false fi @@ -96,7 +94,7 @@ cat < ${OUTPUT_DIR}/metadata.json { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.fde=${ENCFS} panic=1 systemd.unified_cgroup_hierarchy=0", + "cmdline": "console=ttyS0 init=/init dstack.fde=1 panic=1 systemd.unified_cgroup_hierarchy=0", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$ROOTFS_HASH", From 02facddd9f8059038ec441bc880a4ef85fa640fa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 26 Dec 2024 01:52:59 +0000 Subject: [PATCH 087/435] Add TEEPOD_PCCS_URL in build config --- build.sh | 2 ++ dstack | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 928c376a0..c177d00f3 100755 --- a/build.sh +++ b/build.sh @@ -51,6 +51,7 @@ TEEPOD_CID_POOL_SIZE=1000 TEEPOD_PORT_MAPPING_ENABLED=false # Host API configuration, type of uint32 TEEPOD_VSOCK_LISTEN_PORT=9080 +TEEPOD_PCCS_URL=https://10.0.2.2:8081/sgx/certification/v4/ KMS_RPC_LISTEN_PORT=9043 TPROXY_RPC_LISTEN_PORT=9010 @@ -201,6 +202,7 @@ tmp_ca_cert = "$CERTS_DIR/tmp-ca.cert" tmp_ca_key = "$CERTS_DIR/tmp-ca.key" kms_url = "https://kms.$BASE_DOMAIN:$KMS_RPC_LISTEN_PORT" tproxy_url = "https://tproxy.$BASE_DOMAIN:$TPROXY_RPC_LISTEN_PORT" +pccs_url = "$TEEPOD_PCCS_URL" cid_start = $TEEPOD_CID_POOL_START cid_pool_size = $TEEPOD_CID_POOL_SIZE [cvm.port_mapping] diff --git a/dstack b/dstack index d8e121b82..9b0fed0f3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d8e121b82413e452bea4ee7eb0c855cc3bdaa402 +Subproject commit 9b0fed0f3eb9002c882bc6d13e5e70ab74df6ada From 2eff4ce7e4a0c72745e463a1b9e4a39b7890ce2e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 26 Dec 2024 02:02:35 +0000 Subject: [PATCH 088/435] Move docker-daemon.json from dstack to here --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 5 +++-- .../recipes-core/dstack-guest/files/docker-daemon.json | 7 +++++++ 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json diff --git a/dstack b/dstack index 9b0fed0f3..f37d55044 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 9b0fed0f3eb9002c882bc6d13e5e70ab74df6ada +Subproject commit f37d55044e73dfdcd01e5f8840b93ea986de1399 diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 168fe36b9..1cfed3d73 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -8,7 +8,8 @@ inherit systemd update-rc.d REPO_ROOT = "${THISDIR}/../../.." SRC_DIR = '${REPO_ROOT}/dstack' -SRC_URI = 'file://${REPO_ROOT}/dstack' +SRC_URI = 'file://${REPO_ROOT}/dstack \ + file://docker-daemon.json' SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" @@ -44,7 +45,7 @@ do_install() { install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} - install -m 0755 ${S}/basefiles/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json + install -m 0755 ${WORKDIR}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json new file mode 100644 index 000000000..4d6c550d7 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json @@ -0,0 +1,7 @@ +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "100m", + "max-file": "10" + } +} From f77d9ec4e4dd16d35e78be631a9332577ae13038 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 26 Dec 2024 04:48:36 +0000 Subject: [PATCH 089/435] Update dstack.py --- scripts/bin/dstack.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 3bc4ce0ed..3f13043e3 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -249,7 +249,8 @@ def setup_instance(self, args: argparse.Namespace) -> None: "version": "1.0.0", "features": [], "runner": "docker-compose", - "docker_compose_file": compose_content + "docker_compose_file": compose_content, + "local_key_provider_enabled": args.local_key_provider } with open(os.path.join(shared_dir, 'app-compose.json'), 'w') as f: json.dump(app_compose, f, indent=4) @@ -258,7 +259,14 @@ def setup_instance(self, args: argparse.Namespace) -> None: image_path = args.image or self.get_default_image_path() rootfs_hash = self._read_image_metadata(image_path) with open(os.path.join(shared_dir, 'config.json'), 'w') as f: - json.dump({"rootfs_hash": rootfs_hash, "docker_registry": self.config.docker_registry}, f, indent=4) + config = { + "rootfs_hash": rootfs_hash, + "docker_registry": self.config.docker_registry, + "pccs_url": "https://api.trustedservices.intel.com/sgx/certification/v4", + "host_api_url": f"vsock://2:{args.host_vsock_port}/api", + "host_vsock_port": args.host_vsock_port, + } + json.dump(config, f, indent=4) # Create VM manifest memory = self._convert_memory_to_mb(str(args.memory)) @@ -418,6 +426,8 @@ def main(): setup_parser.add_argument('-g', '--gpu', type=str, action='append', help='GPU device') setup_parser.add_argument('-p', '--port', action='append', type=str, help='Port mapping in format: protocol[:address]:from:to') setup_parser.add_argument('--no-fde', action='store_true', help='Disable Full Disk Encryption') + setup_parser.add_argument('--local-key-provider', action='store_true', help='Enable local key provider') + setup_parser.add_argument('--host-vsock-port', type=int, default=3443, help='Host vsock port') # Start command start_parser = subparsers.add_parser('run', help='Start an instance') From 2847fe43e4a66ff7e40085d673c7163c5919ae12 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 28 Dec 2024 01:51:53 +0000 Subject: [PATCH 090/435] Update dstack.py to support local key provider --- .gitignore | 1 + scripts/bin/dstack.py | 34 +++++++++--- scripts/bin/host_api.py | 114 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 scripts/bin/host_api.py diff --git a/.gitignore b/.gitignore index 175bd3f7d..fa8f9fd9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /build /bb-build *.qcow2 +__pycache__/ diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 3f13043e3..9540ced60 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -9,6 +9,8 @@ import subprocess import uuid import configparser +import host_api +import threading from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -21,6 +23,7 @@ ) logger = logging.getLogger(__name__) + def generate_config_paths(): paths = [ "/etc/dstack/client.conf", @@ -49,6 +52,7 @@ def to_dict(self) -> Dict: "to": self.to_port } + @dataclass class VMConfig: """Configuration for VM instance.""" @@ -75,6 +79,7 @@ def to_dict(self) -> Dict: "created_at_ms": self.created_at_ms } + def merge2(a, b): if isinstance(a, dict) and isinstance(b, dict): c = a.copy() @@ -109,7 +114,7 @@ def test_merge_dicts(): def ini_to_dict(filename): config = configparser.ConfigParser() config.read(filename) - + result = {} for section in config.sections(): result[section] = {} @@ -176,7 +181,7 @@ def _read_image_metadata(self, image_path: str) -> str: metadata_path = os.path.join(image_path, 'metadata.json') if not os.path.isfile(metadata_path): raise FileNotFoundError(f"Image metadata not found at {metadata_path}") - + try: with open(metadata_path, 'r') as f: metadata = json.load(f) @@ -263,8 +268,6 @@ def setup_instance(self, args: argparse.Namespace) -> None: "rootfs_hash": rootfs_hash, "docker_registry": self.config.docker_registry, "pccs_url": "https://api.trustedservices.intel.com/sgx/certification/v4", - "host_api_url": f"vsock://2:{args.host_vsock_port}/api", - "host_vsock_port": args.host_vsock_port, } json.dump(config, f, indent=4) @@ -283,7 +286,7 @@ def setup_instance(self, args: argparse.Namespace) -> None: gpu=args.gpu or [], memory=memory, disk_size=disk_size, - image=os.path.basename(image_path), + image=os.path.basename(image_path.rstrip('/')), port_map=port_map, created_at_ms=int(datetime.now().timestamp() * 1000) ) @@ -297,7 +300,7 @@ def setup_instance(self, args: argparse.Namespace) -> None: logger.error(f"Failed to setup instance: {str(e)}") raise - def run_instance(self, vm_dir: str, memory: Optional[str] = None, vcpus: Optional[int] = None) -> None: + def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None, vcpus: Optional[int] = None) -> None: """Run a VM instance from the specified directory. Args: @@ -315,6 +318,15 @@ def run_instance(self, vm_dir: str, memory: Optional[str] = None, vcpus: Optiona # Get image path and metadata image_path = os.path.join(self.config.image_path, manifest['image']) img_metadata_path = os.path.join(image_path, 'metadata.json') + + # Update config.json with host API URL and port + shared_dir = os.path.join(vm_dir, 'shared') + config_file = os.path.join(shared_dir, 'config.json') + config = json.load(open(config_file, 'r')) + config['host_api_url'] = f"http://10.0.2.2:{host_port}/api" + config['host_vsock_port'] = host_port + with open(config_file, 'w') as f: + json.dump(config, f, indent=4) if not os.path.exists(img_metadata_path): raise ValueError(f"Image metadata not found at {img_metadata_path}") @@ -427,13 +439,13 @@ def main(): setup_parser.add_argument('-p', '--port', action='append', type=str, help='Port mapping in format: protocol[:address]:from:to') setup_parser.add_argument('--no-fde', action='store_true', help='Disable Full Disk Encryption') setup_parser.add_argument('--local-key-provider', action='store_true', help='Enable local key provider') - setup_parser.add_argument('--host-vsock-port', type=int, default=3443, help='Host vsock port') # Start command start_parser = subparsers.add_parser('run', help='Start an instance') start_parser.add_argument('dir', type=str, help='Work directory') start_parser.add_argument('-m', '--memory', type=str, help='Memory size (e.g. 2G, 512M)') start_parser.add_argument('-c', '--vcpus', type=int, help='Number of virtual CPUs') + start_parser.add_argument('--kp-port', type=int, default=3443, help='The key provider listening port') # List Gpus command subparsers.add_parser('lsgpu', help='List available GPUs') @@ -445,7 +457,12 @@ def main(): manager.setup_instance(args) elif args.command == 'run': manager = DStackManager() - manager.run_instance(args.dir, memory=args.memory, vcpus=args.vcpus) + config = host_api.ServerConfig(vm_dir=args.dir, kp_address="127.0.0.1", kp_port=args.kp_port) + api, host_port = host_api.create_http_server(config) + print(f"Starting HTTP server on localhost:{host_port}") + thread = threading.Thread(target=api.serve_forever, daemon=True) + thread.start() + manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus) elif args.command == 'lsgpu': list_available_gpus() else: @@ -453,3 +470,4 @@ def main(): if __name__ == '__main__': main() + diff --git a/scripts/bin/host_api.py b/scripts/bin/host_api.py new file mode 100644 index 000000000..9c5df9a5c --- /dev/null +++ b/scripts/bin/host_api.py @@ -0,0 +1,114 @@ +import os +import json +import struct +import socket +import urllib +from dataclasses import dataclass +from http.server import HTTPServer, BaseHTTPRequestHandler + + +@dataclass +class ServerConfig: + kp_address: str + kp_port: int + vm_dir: str + + +class QuoteError(Exception): + pass + + +@dataclass +class QuoteResponse: + encrypted_key: bytes + provider_quote: bytes + + @classmethod + def from_json(cls, data: dict) -> 'QuoteResponse': + return cls( + encrypted_key=bytes(data['encrypted_key']), + provider_quote=bytes(data['provider_quote']) + ) + + +def get_key(quote: bytes, address: str, port: int) -> QuoteResponse: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((address, port)) + + try: + payload = {"quote": list(quote)} + serialized = json.dumps(payload).encode() + length = len(serialized) + + sock.sendall(struct.pack('>I', length)) + sock.sendall(serialized) + + response_length_bytes = sock.recv(4) + response_length = struct.unpack('>I', response_length_bytes)[0] + + response_data = b'' + while len(response_data) < response_length: + chunk = sock.recv(min(4096, response_length - len(response_data))) + if not chunk: + raise ConnectionError("Connection closed prematurely") + response_data += chunk + + response_json = json.loads(response_data) + return QuoteResponse.from_json(response_json) + + finally: + sock.close() + + +class QuoteHandler(BaseHTTPRequestHandler): + def __init__(self, config: ServerConfig, *args, **kwargs): + self.config = config + super().__init__(*args, **kwargs) + + def do_POST(self): + parsed_path = urllib.parse.urlparse(self.path) + + content_length = int(self.headers['Content-Length']) + if content_length > 1024 * 128: + self.respond(400, json.dumps({'error': 'Request body too large'}).encode()) + return + + body = self.rfile.read(content_length) + + match parsed_path.path: + case "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/api/GetSealingKey": + quote = json.loads(body) + response = get_key(bytes.fromhex(quote['quote']), self.config.kp_address, self.config.kp_port) + response_data = { + 'encrypted_key': response.encrypted_key.hex(), + 'provider_quote': response.provider_quote.hex() + } + response_bytes = json.dumps(response_data).encode() + case "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/api/Notify": + info = json.loads(body) + if info['event'] == 'instance.info': + info_path = os.path.join(self.config.vm_dir, 'shared', '.instance_info') + open(info_path, 'w').write(info['payload']) + response_bytes = b'null' + case _: + self.respond(404, b'null') + return + + self.respond(200, response_bytes) + + def respond(self, status: int, data: bytes): + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', len(data)) + self.end_headers() + self.wfile.write(data) + + +def create_http_server(config: ServerConfig): + def handler(*args): + QuoteHandler(config, *args) + + server = HTTPServer(('localhost', 0), handler) + chosen_port = server.server_port + return server, chosen_port + From 56d171172d55b7d4aeebab26b3472a633fa60ecb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 31 Dec 2024 12:03:18 +0000 Subject: [PATCH 091/435] Put rootfs_hash to kernel cmdline --- .../recipes-core/images/dstack-initramfs-files/kmfs-setup | 1 + mkimage.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 9190c3758..343459c6f 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -42,4 +42,5 @@ tdxctl setup-fde \ --root-hd $ROOTFS_HD \ --root-cdrom $ROOTFS_CD \ --root-cdrom-mnt $ROOTFS_CD_MNT \ + --rootfs-hash $kargs_dstack_rootfs_hash \ $ROOTFS_ENCRYPTION $ROOTINTEGRITY || panic "Failed to setup FDE" diff --git a/mkimage.sh b/mkimage.sh index 78b889c6a..d1025b8a7 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -94,7 +94,7 @@ cat < ${OUTPUT_DIR}/metadata.json { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.fde=1 panic=1 systemd.unified_cgroup_hierarchy=0", + "cmdline": "console=ttyS0 init=/init dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH panic=1 systemd.unified_cgroup_hierarchy=0", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$ROOTFS_HASH", From 7ecdf1b68db266466171df12b99f35be2b144ca1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 2 Jan 2025 01:54:22 +0000 Subject: [PATCH 092/435] Add lsproc and lsvms --- scripts/bin/lsproc.py | 90 ++++++++++++++++++++++++++++ scripts/bin/lsvms.py | 136 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100755 scripts/bin/lsproc.py create mode 100755 scripts/bin/lsvms.py diff --git a/scripts/bin/lsproc.py b/scripts/bin/lsproc.py new file mode 100755 index 000000000..d9665cb99 --- /dev/null +++ b/scripts/bin/lsproc.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import os +from pathlib import Path + +def get_process_pids(process_name: str) -> list: + """Get PIDs for the specified process name""" + try: + result = subprocess.run(['pgrep', f'^{process_name}$'], + capture_output=True, text=True) + if result.returncode != 0: + print(f"No process named '{process_name}' found") + sys.exit(1) + return result.stdout.strip().split() + except subprocess.CalledProcessError: + print("Error running pgrep command") + sys.exit(1) + +def get_working_dir(pid: str) -> str: + """Get working directory for a PID""" + try: + result = subprocess.run(['sudo', 'pwdx', pid], + capture_output=True, text=True) + return result.stdout.split(':', 1)[1].strip() + except subprocess.CalledProcessError: + return "N/A" + +def get_socket_count(pid: str) -> int: + """Get socket count for a PID""" + try: + result = subprocess.run(['sudo', 'ls', '-l', f'/proc/{pid}/fd/'], + capture_output=True, text=True) + return result.stdout.count('socket') + except subprocess.CalledProcessError: + return 0 + +def get_nofile_limit(pid: str) -> str: + """Get max open files limit for a PID""" + try: + with open(f'/proc/{pid}/limits', 'r') as f: + for line in f: + if "Max open files" in line: + return line.split()[3] + except (PermissionError, FileNotFoundError): + return "N/A" + return "N/A" + +def get_memory_usage(pid: str) -> str: + """Get memory usage in human readable format""" + try: + result = subprocess.run(['sudo', 'ps', '-p', pid, '-o', 'rss='], + capture_output=True, text=True) + mem_kb = int(result.stdout.strip()) + + if mem_kb >= 1048576: # 1GB = 1048576KB + return f"{mem_kb/1048576:.2f}GB" + elif mem_kb >= 1024: # 1MB = 1024KB + return f"{mem_kb/1024:.2f}MB" + else: + return f"{mem_kb}KB" + except subprocess.CalledProcessError: + return "N/A" + +def main(): + if len(sys.argv) != 2: + print("Usage: script.py ") + sys.exit(1) + + process_name = sys.argv[1] + pids = get_process_pids(process_name) + + # Print header + print(f"{'PID':<10} {'FDs':<10} {'NOFILE':<12} {'MEMORY':<15} {'PATH'}") + print(f"{'---':<10} {'---':<10} {'------':<12} {'-------':<15} {'----'}") + + # Process each PID + for pid in pids: + if not Path(f"/proc/{pid}").exists(): + continue + + work_dir = get_working_dir(pid) + socket_count = get_socket_count(pid) + nofile = get_nofile_limit(pid) + memory = get_memory_usage(pid) + + print(f"{pid:<10} {socket_count:<10} {nofile:<12} {memory:<15} {work_dir}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/bin/lsvms.py b/scripts/bin/lsvms.py new file mode 100755 index 000000000..54839bd5d --- /dev/null +++ b/scripts/bin/lsvms.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +import os +import json +import sys +import argparse +from pathlib import Path + +def read_json_file(file_path: str, key: str) -> str: + """Read a value from a JSON file safely""" + try: + with open(file_path) as f: + return json.load(f).get(key, "N/A") + except (FileNotFoundError, json.JSONDecodeError, KeyError): + return "N/A" + +def get_instance_ip(tproxy_state_file: str, instance_id: str) -> str: + """Get instance IP from tproxy state file""" + try: + with open(tproxy_state_file) as f: + data = json.load(f) + return data.get('instances', {}).get(instance_id, {}).get('ip', 'N/A') + except (FileNotFoundError, json.JSONDecodeError): + return "N/A" + +def parse_simple_toml(file_path: Path) -> dict: + """Simple TOML parser for our specific needs""" + config = {'gateway': {}} + try: + with open(file_path) as f: + current_section = None + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + + if line.startswith('[') and line.endswith(']'): + current_section = line[1:-1] + if current_section not in config: + config[current_section] = {} + continue + + if '=' in line: + key, value = [x.strip() for x in line.split('=', 1)] + value = value.strip('"').strip("'") + try: + value = int(value) + except ValueError: + pass + if current_section: + config[current_section][key] = value + else: + config[key] = value + + return config + except (FileNotFoundError, IOError): + return {'gateway': {}} + +def get_dashboard_url(config: dict, app_id: str) -> str: + """Generate dashboard URL from config and app_id""" + try: + gateway = config.get('gateway', {}) + base_domain = gateway.get('base_domain') + tappd_port = gateway.get('tappd_port') + gateway_port = gateway.get('port') + + if base_domain and tappd_port: + url = f"https://{app_id}-{tappd_port}.{base_domain}" + if gateway_port: + url += f":{gateway_port}" + return url + "/" + except (KeyError, TypeError): + pass + return "N/A" + +def print_vm_info(vm_data: dict, verbose: bool = False): + """Pretty print VM information""" + if verbose: + print("=" * 100) + for key, value in vm_data.items(): + # Right-align keys in a 15-character space, followed by a colon and value + print(f"{key:>15}: {value}") + print() + else: + print(f"{vm_data['name']:<25} {vm_data['ip']:<15} {vm_data['image']}") + +def main(): + parser = argparse.ArgumentParser(description='List VMs information') + parser.add_argument('stack_dir', help='Stack directory path') + parser.add_argument('-v', '--verbose', action='store_true', help='Show detailed information') + args = parser.parse_args() + + stack_dir = Path(args.stack_dir) + vm_base_dir = stack_dir / "run" / "vm" + tproxy_state_file = stack_dir / "tproxy-state.json" + + if not vm_base_dir.exists(): + print(f"Error: Directory not found: {vm_base_dir}") + sys.exit(1) + + config = {} + if args.verbose: + config = parse_simple_toml(stack_dir / "teepod.toml") + + if not args.verbose: + # Print simple header + print(f"{'Name':<25} {'IP':<15} {'Image'}") + print("-" * 60) + + # Process each VM directory + for vm_dir in vm_base_dir.glob("*"): + if not vm_dir.is_dir(): + continue + + vm_data = { + 'name': read_json_file(str(vm_dir / "vm-manifest.json"), "name"), + 'image': read_json_file(str(vm_dir / "vm-manifest.json"), "image"), + } + + instance_info_file = vm_dir / "shared" / ".instance_info" + if instance_info_file.exists(): + instance_id = read_json_file(str(instance_info_file), "instance_id") + app_id = read_json_file(str(instance_info_file), "app_id") + vm_data.update({ + 'instance_id': instance_id, + 'app_id': app_id, + 'ip': get_instance_ip(str(tproxy_state_file), instance_id), + 'vm_dir': str(vm_dir), + }) + + if args.verbose: + vm_data['dashboard_url'] = get_dashboard_url(config, app_id) + + print_vm_info(vm_data, args.verbose) + +if __name__ == "__main__": + main() From 7f719ec93efa9344a7888ff705ec4a6c5b768d4f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 2 Jan 2025 03:59:22 +0000 Subject: [PATCH 093/435] Add pnpacpi=off to kernel args --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index d1025b8a7..c9c3de120 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -94,7 +94,7 @@ cat < ${OUTPUT_DIR}/metadata.json { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH panic=1 systemd.unified_cgroup_hierarchy=0", + "cmdline": "console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 pnpacpi=off dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$ROOTFS_HASH", From d6d452d466f3705bee9d39866caa2bf96f9abb1a Mon Sep 17 00:00:00 2001 From: Jianwei Zhu Date: Fri, 3 Jan 2025 16:17:40 -0800 Subject: [PATCH 094/435] bump version to v0.3.4 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index f37d55044..a5edadf26 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f37d55044e73dfdcd01e5f8840b93ea986de1399 +Subproject commit a5edadf269e6e823fc8b3efe97ba9a44fe57703a diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index b8393d724..0f11d1a53 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.3.3" +DISTRO_VERSION = "0.3.4" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 9db845779957acb50911b9423731528afb9da128 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 11 Jan 2025 07:58:23 +0000 Subject: [PATCH 095/435] Bump version to 0.4.0 --- build.sh | 7 ++++--- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index c177d00f3..bbaabd5bc 100755 --- a/build.sh +++ b/build.sh @@ -125,14 +125,16 @@ build_cfg() { # kms cat < kms.toml log_level = "info" + +[rpc] address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT -[tls] +[rpc.tls] key = "$CERTS_DIR/kms-rpc.key" certs = "$CERTS_DIR/kms-rpc.cert" -[tls.mutual] +[rpc.tls.mutual] ca_certs = "$CERTS_DIR/tmp-ca.cert" mandatory = false @@ -140,7 +142,6 @@ mandatory = false root_ca_cert = "$CERTS_DIR/root-ca.cert" root_ca_key = "$CERTS_DIR/root-ca.key" subject_postfix = ".phala" -upgrade_registry_dir = "$KMS_UPGRADE_REGISTRY_DIR" cert_log_dir = "$KMS_CERT_LOG_DIR" [core.allowed_mr] diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index b8393d724..d6023aa09 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.3.3" +DISTRO_VERSION = "0.4.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 332127f60b0cca29e555da47dc0d464f09c0ab45 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 13 Jan 2025 11:25:33 +0000 Subject: [PATCH 096/435] Sync dstack kms dev --- .gitignore | 1 + build.sh | 36 +++++++++---------- dstack | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 2 ++ .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 2 +- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index fa8f9fd9f..afab4c0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /bb-build *.qcow2 __pycache__/ +/.target diff --git a/build.sh b/build.sh index bbaabd5bc..25e4cfe6b 100755 --- a/build.sh +++ b/build.sh @@ -110,12 +110,6 @@ build_guest() { make -C $META_DIR dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} } -# Step 3: make certs -build_certs() { - echo "Building certs" - make -C $DSTACK_DIR certs DOMAIN=$BASE_DOMAIN TO=$CERTS_DIR -} - # Step 4: generate config files build_cfg() { @@ -141,15 +135,19 @@ mandatory = false [core] root_ca_cert = "$CERTS_DIR/root-ca.cert" root_ca_key = "$CERTS_DIR/root-ca.key" -subject_postfix = ".phala" -cert_log_dir = "$KMS_CERT_LOG_DIR" - -[core.allowed_mr] -allow_all = true -mrtd = [] -rtmr0 = [] -rtmr1 = [] -rtmr2 = [] +tmp_ca_cert = "$CERTS_DIR/tmp-ca.cert" +tmp_ca_key = "$CERTS_DIR/tmp-ca.key" +rpc_cert = "$CERTS_DIR/rpc.cert" +rpc_key = "$CERTS_DIR/rpc.key" +k256_key = "$CERTS_DIR/root-k256.key" + +[core.auth_api] +type = "dev" + +[core.onboard] +quote_enabled = false +address = "127.0.0.1" +port = $KMS_RPC_LISTEN_PORT EOF # tproxy @@ -166,6 +164,9 @@ certs = "$CERTS_DIR/tproxy-rpc.cert" ca_certs = "$CERTS_DIR/root-ca.cert" mandatory = false +[core] +kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" + [core.certbot] workdir = "$CERBOT_WORKDIR" @@ -198,9 +199,6 @@ run_path = "$RUN_DIR/vm" kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] -ca_cert = "$CERTS_DIR/root-ca.cert" -tmp_ca_cert = "$CERTS_DIR/tmp-ca.cert" -tmp_ca_key = "$CERTS_DIR/tmp-ca.key" kms_url = "https://kms.$BASE_DOMAIN:$KMS_RPC_LISTEN_PORT" tproxy_url = "https://tproxy.$BASE_DOMAIN:$TPROXY_RPC_LISTEN_PORT" pccs_url = "$TEEPOD_PCCS_URL" @@ -286,7 +284,6 @@ case $ACTION in ;; certs) require_config - build_certs ;; wg) require_config @@ -297,7 +294,6 @@ case $ACTION in require_config build_host build_guest - build_certs build_cfg build_wg ;; diff --git a/dstack b/dstack index f37d55044..55e7e1adb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f37d55044e73dfdcd01e5f8840b93ea986de1399 +Subproject commit 55e7e1adb6b0721e4482bb0dd8701285c83db1bf diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 1cfed3d73..c964795ca 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -14,6 +14,8 @@ SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" +RDEPENDS:${PN} += "bash" + DSTACK_SERVICES = "tappd.service tboot.service app-compose.service wg-checker.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb index 87a72993f..7f943d78c 100644 --- a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb +++ b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb @@ -8,7 +8,7 @@ inherit module REPO_ROOT = "${THISDIR}/../../.." SRC_DIR = '${REPO_ROOT}/dstack/mod-tdx-guest' -SRC_URI = 'file://${REPO_ROOT}/dstack' +SRC_URI = 'file://${REPO_ROOT}/dstack/mod-tdx-guest' SRCREV = "${DSTACK_SRC_REV}" S = "${WORKDIR}/${SRC_DIR}" From cd42f0b063be2988a5014e65501f47b401b42908 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 14 Jan 2025 05:53:06 +0000 Subject: [PATCH 097/435] Fix configs in build.sh --- build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 25e4cfe6b..e44767803 100755 --- a/build.sh +++ b/build.sh @@ -51,7 +51,6 @@ TEEPOD_CID_POOL_SIZE=1000 TEEPOD_PORT_MAPPING_ENABLED=false # Host API configuration, type of uint32 TEEPOD_VSOCK_LISTEN_PORT=9080 -TEEPOD_PCCS_URL=https://10.0.2.2:8081/sgx/certification/v4/ KMS_RPC_LISTEN_PORT=9043 TPROXY_RPC_LISTEN_PORT=9010 @@ -125,8 +124,8 @@ address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT [rpc.tls] -key = "$CERTS_DIR/kms-rpc.key" -certs = "$CERTS_DIR/kms-rpc.cert" +key = "$CERTS_DIR/rpc.key" +certs = "$CERTS_DIR/rpc.cert" [rpc.tls.mutual] ca_certs = "$CERTS_DIR/tmp-ca.cert" @@ -148,6 +147,7 @@ type = "dev" quote_enabled = false address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT +auto_bootstrap_domain = "kms.$BASE_DOMAIN" EOF # tproxy @@ -161,11 +161,12 @@ key = "$CERTS_DIR/tproxy-rpc.key" certs = "$CERTS_DIR/tproxy-rpc.cert" [tls.mutual] -ca_certs = "$CERTS_DIR/root-ca.cert" +ca_certs = "$CERTS_DIR/tproxy-ca.cert" mandatory = false [core] kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" +gen_certs_for = "tproxy.$BASE_DOMAIN" [core.certbot] workdir = "$CERBOT_WORKDIR" @@ -201,7 +202,6 @@ kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] kms_url = "https://kms.$BASE_DOMAIN:$KMS_RPC_LISTEN_PORT" tproxy_url = "https://tproxy.$BASE_DOMAIN:$TPROXY_RPC_LISTEN_PORT" -pccs_url = "$TEEPOD_PCCS_URL" cid_start = $TEEPOD_CID_POOL_START cid_pool_size = $TEEPOD_CID_POOL_SIZE [cvm.port_mapping] From 5a7a22e30603a12e5d2f787d96a6291066e28b10 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jan 2025 09:29:49 +0000 Subject: [PATCH 098/435] Update Rust to 1.84 --- bb-build/conf/local.conf | 4 ++-- meta-rust-bin | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index b236c288c..2aa792eee 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -288,5 +288,5 @@ PACKAGECONFIG:append:pn-qemu-system-native = " sdl" CONF_VERSION = "2" SERIAL_CONSOLES = "115200;ttyS0" -PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.80.0" -PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.80.0" \ No newline at end of file +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.84.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.84.0" diff --git a/meta-rust-bin b/meta-rust-bin index c05de0f89..540161165 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit c05de0f8922ca8736af2c0eec5f570568fcd2c8e +Subproject commit 5401611657e42a546f23df6a380d96d64ca78f9b From d50180244cbee1f5abfb6a3a36e6c7aafa9809ce Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jan 2025 09:30:06 +0000 Subject: [PATCH 099/435] Update dstack --- build.sh | 23 +++++++++-------------- dstack | 2 +- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/build.sh b/build.sh index e44767803..80c825c79 100755 --- a/build.sh +++ b/build.sh @@ -40,7 +40,8 @@ require_config() { # base domain of kms rpc and tproxy rpc # 1022.kvin.wang resolves to 10.0.2.2 which is host ip at the # cvm point of view -BASE_DOMAIN=1022.kvin.wang +KMS_DOMAIN=kms.1022.kvin.wang +TPROXY_DOMAIN=tproxy.1022.kvin.wang TEEPOD_RPC_LISTEN_PORT=9080 # CIDs allocated to VMs start from this number of type unsigned int32 @@ -125,20 +126,14 @@ port = $KMS_RPC_LISTEN_PORT [rpc.tls] key = "$CERTS_DIR/rpc.key" -certs = "$CERTS_DIR/rpc.cert" +certs = "$CERTS_DIR/rpc.crt" [rpc.tls.mutual] -ca_certs = "$CERTS_DIR/tmp-ca.cert" +ca_certs = "$CERTS_DIR/tmp-ca.crt" mandatory = false [core] -root_ca_cert = "$CERTS_DIR/root-ca.cert" -root_ca_key = "$CERTS_DIR/root-ca.key" -tmp_ca_cert = "$CERTS_DIR/tmp-ca.cert" -tmp_ca_key = "$CERTS_DIR/tmp-ca.key" -rpc_cert = "$CERTS_DIR/rpc.cert" -rpc_key = "$CERTS_DIR/rpc.key" -k256_key = "$CERTS_DIR/root-k256.key" +cert_dir = "$CERTS_DIR" [core.auth_api] type = "dev" @@ -147,7 +142,7 @@ type = "dev" quote_enabled = false address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT -auto_bootstrap_domain = "kms.$BASE_DOMAIN" +auto_bootstrap_domain = "$KMS_DOMAIN" EOF # tproxy @@ -166,7 +161,7 @@ mandatory = false [core] kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" -gen_certs_for = "tproxy.$BASE_DOMAIN" +tls_domain = "$TPROXY_DOMAIN" [core.certbot] workdir = "$CERBOT_WORKDIR" @@ -200,8 +195,8 @@ run_path = "$RUN_DIR/vm" kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] -kms_url = "https://kms.$BASE_DOMAIN:$KMS_RPC_LISTEN_PORT" -tproxy_url = "https://tproxy.$BASE_DOMAIN:$TPROXY_RPC_LISTEN_PORT" +kms_url = "https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT" +tproxy_url = "https://$TPROXY_DOMAIN:$TPROXY_RPC_LISTEN_PORT" cid_start = $TEEPOD_CID_POOL_START cid_pool_size = $TEEPOD_CID_POOL_SIZE [cvm.port_mapping] diff --git a/dstack b/dstack index 55e7e1adb..4d56b0818 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 55e7e1adb6b0721e4482bb0dd8701285c83db1bf +Subproject commit 4d56b0818b2fecb36d1b68e57e89c38008731674 From 15189bcb5397083b5c650a438243ce3f29e705f4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jan 2025 09:30:36 +0000 Subject: [PATCH 100/435] init.sh Sleep 1 after error --- .../recipes-core/images/dstack-initramfs-files/kmfs-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 343459c6f..859b3a76b 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -19,6 +19,7 @@ HOST_SHARED=/mnt/host-shared panic() { echo "$1" + sleep 1 exit 1 } From d33564431c754ca21ac504f2921a5840cd476749 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 18 Feb 2025 07:25:27 +0000 Subject: [PATCH 101/435] Remove wg-checker service --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index c964795ca..dbb9aa0b4 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -16,7 +16,7 @@ S = "${WORKDIR}/${SRC_DIR}" RDEPENDS:${PN} += "bash" -DSTACK_SERVICES = "tappd.service tboot.service app-compose.service wg-checker.service" +DSTACK_SERVICES = "tappd.service tboot.service app-compose.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -46,7 +46,6 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} - install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${WORKDIR}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf @@ -60,7 +59,6 @@ do_install() { install -m 0644 ${S}/basefiles/tappd.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/tboot.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d else install -d ${D}${sysconfdir}/init.d From cb1b09680059ea75c6107ed392065c3e0a0a7f2d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Feb 2025 07:08:10 +0000 Subject: [PATCH 102/435] Build config for 0.4.0 --- build.sh | 20 ++++++++++++++------ dstack | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/build.sh b/build.sh index 80c825c79..2b3e41b5c 100755 --- a/build.sh +++ b/build.sh @@ -58,8 +58,8 @@ TPROXY_RPC_LISTEN_PORT=9010 TPROXY_WG_INTERFACE=tproxy-$USER TPROXY_WG_LISTEN_PORT=9182 -TPROXY_WG_IP=10.0.3.1 -TPROXY_WG_CLIENT_IP_RANGE=10.0.3.0/24 +TPROXY_WG_IP=10.0.3.1/24 +TPROXY_WG_RESERVED_NET=10.0.3.1/32 TPROXY_SERVE_PORT=9443 BIND_PUBLIC_IP=0.0.0.0 @@ -161,7 +161,14 @@ mandatory = false [core] kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" -tls_domain = "$TPROXY_DOMAIN" +rpc_domain = "$TPROXY_DOMAIN" +run_as_tapp = false + +[core.sync] +enabled = true +interval = "30s" +my_url = "https://localhost:$TPROXY_RPC_LISTEN_PORT" +bootnode = "https://localhost:$TPROXY_RPC_LISTEN_PORT" [core.certbot] workdir = "$CERBOT_WORKDIR" @@ -169,9 +176,10 @@ workdir = "$CERBOT_WORKDIR" [core.wg] private_key = "$TPROXY_WG_KEY" public_key = "$TPROXY_WG_PUBKEY" -ip = "$TPROXY_WG_IP" listen_port = $TPROXY_WG_LISTEN_PORT -client_ip_range = "$TPROXY_WG_CLIENT_IP_RANGE" +ip = "$TPROXY_WG_IP" +reserved_net = "$TPROXY_WG_RESERVED_NET" +client_ip_range = "$TPROXY_WG_IP" config_path = "$RUN_DIR/wg.conf" interface = "$TPROXY_WG_INTERFACE" endpoint = "10.0.2.2:$TPROXY_WG_LISTEN_PORT" @@ -256,7 +264,7 @@ build_wg() { # Check if the WireGuard interface exists if ! ip link show $TPROXY_WG_INTERFACE &> /dev/null; then sudo ip link add $TPROXY_WG_INTERFACE type wireguard - sudo ip address add $TPROXY_WG_IP/24 dev $TPROXY_WG_INTERFACE + sudo ip address add $TPROXY_WG_IP dev $TPROXY_WG_INTERFACE sudo ip link set $TPROXY_WG_INTERFACE up echo "created and configured WireGuard interface $TPROXY_WG_INTERFACE" else diff --git a/dstack b/dstack index 4d56b0818..34398266e 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 4d56b0818b2fecb36d1b68e57e89c38008731674 +Subproject commit 34398266eb7968ec33b684c7f5c33055c0e770ee From 417fd23b9c4ca08b1e641a5c787fc5eda371077c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Feb 2025 07:08:29 +0000 Subject: [PATCH 103/435] Add ascii art --- meta-dstack/recipes-core/base-files/files/motd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta-dstack/recipes-core/base-files/files/motd b/meta-dstack/recipes-core/base-files/files/motd index 9cff826ce..047731e70 100644 --- a/meta-dstack/recipes-core/base-files/files/motd +++ b/meta-dstack/recipes-core/base-files/files/motd @@ -1 +1,8 @@ +██████╗ ███████╗████████╗ █████╗ ██████╗██╗ ██╗ ████████╗███████╗███████╗ ██████╗ ███████╗ +██╔══██╗██╔════╝╚══██╔══╝██╔══██╗██╔════╝██║ ██╔╝ ╚══██╔══╝██╔════╝██╔════╝ ██╔═══██╗██╔════╝ +██║ ██║███████╗ ██║ ███████║██║ █████╔╝ ██║ █████╗ █████╗ ██║ ██║███████╗ +██║ ██║╚════██║ ██║ ██╔══██║██║ ██╔═██╗ ██║ ██╔══╝ ██╔══╝ ██║ ██║╚════██║ +██████╔╝███████║ ██║ ██║ ██║╚██████╗██║ ██╗ ██║ ███████╗███████╗ ╚██████╔╝███████║ +╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝ ╚═════╝ ╚══════╝ + Welcome to Dstack! From 604821d5e7bfe5c9a056bb65073371c28f5fdb90 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 27 Feb 2025 10:25:58 +0000 Subject: [PATCH 104/435] Update dstack submodule --- build.sh | 4 ++-- dstack | 2 +- .../recipes-core/images/dstack-initramfs-files/kmfs-setup | 3 --- scripts/bin/dstack.py | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 2b3e41b5c..0cf97e914 100755 --- a/build.sh +++ b/build.sh @@ -203,8 +203,8 @@ run_path = "$RUN_DIR/vm" kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] -kms_url = "https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT" -tproxy_url = "https://$TPROXY_DOMAIN:$TPROXY_RPC_LISTEN_PORT" +kms_urls = ["https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT"] +tproxy_urls = ["https://$TPROXY_DOMAIN:$TPROXY_RPC_LISTEN_PORT"] cid_start = $TEEPOD_CID_POOL_START cid_pool_size = $TEEPOD_CID_POOL_SIZE [cvm.port_mapping] diff --git a/dstack b/dstack index 34398266e..e26b74daa 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 34398266eb7968ec33b684c7f5c33055c0e770ee +Subproject commit e26b74daacbc36aeede494491ae226b28a679a38 diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup index 859b3a76b..c874d1bbd 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup @@ -14,7 +14,6 @@ ROOTFS_HD=/dev/vda ROOTFS_DIR=/root KMFS_WORK_DIR=/kmfs -SHARED_COPY=/tapp HOST_SHARED=/mnt/host-shared panic() { @@ -26,7 +25,6 @@ panic() { echo "Setting up FDE" echo "FDE setup arguments:" echo " HOST_SHARED=$HOST_SHARED" -echo " SHARED_COPY=$SHARED_COPY" echo " KMFS_WORK_DIR=$KMFS_WORK_DIR" echo " ROOTFS_DIR=$ROOTFS_DIR" echo " ROOTFS_HD=$ROOTFS_HD" @@ -37,7 +35,6 @@ echo " ROOTFS_ENCRYPTION=$ROOTFS_ENCRYPTION" tdxctl setup-fde \ --host-shared $HOST_SHARED \ - --host-shared-copy $SHARED_COPY \ --work-dir $KMFS_WORK_DIR \ --rootfs-dir $ROOTFS_DIR \ --root-hd $ROOTFS_HD \ diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 9540ced60..face3c89e 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -263,7 +263,7 @@ def setup_instance(self, args: argparse.Namespace) -> None: # Read image metadata and create config.json image_path = args.image or self.get_default_image_path() rootfs_hash = self._read_image_metadata(image_path) - with open(os.path.join(shared_dir, 'config.json'), 'w') as f: + with open(os.path.join(shared_dir, '.sys-config.json'), 'w') as f: config = { "rootfs_hash": rootfs_hash, "docker_registry": self.config.docker_registry, @@ -321,7 +321,7 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None # Update config.json with host API URL and port shared_dir = os.path.join(vm_dir, 'shared') - config_file = os.path.join(shared_dir, 'config.json') + config_file = os.path.join(shared_dir, '.sys-config.json') config = json.load(open(config_file, 'r')) config['host_api_url'] = f"http://10.0.2.2:{host_port}/api" config['host_vsock_port'] = host_port From e09a2ecf88923e6a418d27997bb9e8d150e3649f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 28 Feb 2025 02:56:37 +0000 Subject: [PATCH 105/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index e26b74daa..6229f33bb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit e26b74daacbc36aeede494491ae226b28a679a38 +Subproject commit 6229f33bba215da7038932fadb066ed0217ebee9 From 61fbf7a18e96fd28e64cae542ed71180a5737a39 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 28 Feb 2025 09:35:24 +0000 Subject: [PATCH 106/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 6229f33bb..4b8fd558f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6229f33bba215da7038932fadb066ed0217ebee9 +Subproject commit 4b8fd558fce0db74557f34c1d22d577c6132d234 From 770788231109fc83a617ec1f2fa900f6c26717c9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 4 Mar 2025 03:17:41 +0000 Subject: [PATCH 107/435] Add --userns=host to docker in build script --- repro-build/repro-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/repro-build/repro-build.sh b/repro-build/repro-build.sh index 676bcb2ef..18a93b868 100755 --- a/repro-build/repro-build.sh +++ b/repro-build/repro-build.sh @@ -43,6 +43,7 @@ build_to() { mkdir -p $1 BUILD_CMD="${2} ${GUEST_SRC_DIR}/build.sh guest ./bb-build" docker run --platform linux/amd64 --rm \ + --userns=host \ --user $(id -u):$(id -g) \ -v $REPO_ROOT:$GUEST_SRC_DIR \ -v $1:$GUEST_BUILD_DIR \ From 414bed4aa9854de0462085f095aca6a80b2f0774 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 5 Mar 2025 06:24:41 +0000 Subject: [PATCH 108/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 4b8fd558f..edb4765ab 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 4b8fd558fce0db74557f34c1d22d577c6132d234 +Subproject commit edb4765abad88b33029eb928d06f63bdb5c864f2 From 9a0b797da61bfb378096fe6fb33956b32016751d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 5 Mar 2025 08:48:24 +0000 Subject: [PATCH 109/435] Update build.sh --- build.sh | 210 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 116 insertions(+), 94 deletions(-) diff --git a/build.sh b/build.sh index 0cf97e914..01224379d 100755 --- a/build.sh +++ b/build.sh @@ -1,13 +1,16 @@ #!/bin/bash -SCRIPT_DIR=$(cd $(dirname $0); pwd) +SCRIPT_DIR=$( + cd $(dirname $0) + pwd +) ACTION=$1 META_DIR=$SCRIPT_DIR DSTACK_DIR=$SCRIPT_DIR/dstack -CERTS_DIR=`pwd`/certs -IMAGES_DIR=`pwd`/images -RUN_DIR=`pwd`/run -RUST_BUILD_DIR=`pwd`/rust-target +CERTS_DIR=$(pwd)/certs +IMAGES_DIR=$(pwd)/images +RUN_DIR=$(pwd)/run +RUST_BUILD_DIR=$(pwd)/rust-target CERBOT_WORKDIR=$RUN_DIR/certbot KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ @@ -36,37 +39,38 @@ check_config() { } require_config() { - cat < build-config.sh.tpl -# base domain of kms rpc and tproxy rpc -# 1022.kvin.wang resolves to 10.0.2.2 which is host ip at the -# cvm point of view + cat <<'EOF' >build-config.sh.tpl +# DNS domain of kms rpc and tproxy rpc +# *.1022.kvin.wang resolves to 10.0.2.2 which is the IP of the host system +# from CVMs point of view KMS_DOMAIN=kms.1022.kvin.wang TPROXY_DOMAIN=tproxy.1022.kvin.wang -TEEPOD_RPC_LISTEN_PORT=9080 # CIDs allocated to VMs start from this number of type unsigned int32 -TEEPOD_CID_POOL_START=10000 +TEEPOD_CID_POOL_START=20000 # CID pool size TEEPOD_CID_POOL_SIZE=1000 + +# Base port for RPC services +BASE_PORT=13000 + +TEEPOD_RPC_LISTEN_PORT=$BASE_PORT # Whether port mapping from host to CVM is allowed TEEPOD_PORT_MAPPING_ENABLED=false # Host API configuration, type of uint32 -TEEPOD_VSOCK_LISTEN_PORT=9080 +TEEPOD_VSOCK_LISTEN_PORT=$BASE_PORT -KMS_RPC_LISTEN_PORT=9043 -TPROXY_RPC_LISTEN_PORT=9010 +KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) +TPROXY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) TPROXY_WG_INTERFACE=tproxy-$USER -TPROXY_WG_LISTEN_PORT=9182 -TPROXY_WG_IP=10.0.3.1/24 -TPROXY_WG_RESERVED_NET=10.0.3.1/32 -TPROXY_SERVE_PORT=9443 +TPROXY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) +TPROXY_WG_IP=10.3.3.1 +TPROXY_SERVE_PORT=$(($BASE_PORT + 4)) BIND_PUBLIC_IP=0.0.0.0 TPROXY_PUBLIC_DOMAIN=app.kvin.wang -TPROXY_CERT=/etc/rproxy/certs/cert.pem -TPROXY_KEY=/etc/rproxy/certs/key.pem # for certbot CF_API_TOKEN= @@ -117,7 +121,7 @@ build_cfg() { TPROXY_WG_KEY=$(wg genkey) TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) # kms - cat < kms.toml + cat <kms.toml log_level = "info" [rpc] @@ -146,7 +150,7 @@ auto_bootstrap_domain = "$KMS_DOMAIN" EOF # tproxy - cat < tproxy.toml + cat <tproxy.toml log_level = "info" address = "127.0.0.1" port = $TPROXY_RPC_LISTEN_PORT @@ -165,28 +169,43 @@ rpc_domain = "$TPROXY_DOMAIN" run_as_tapp = false [core.sync] -enabled = true -interval = "30s" -my_url = "https://localhost:$TPROXY_RPC_LISTEN_PORT" -bootnode = "https://localhost:$TPROXY_RPC_LISTEN_PORT" +enabled = false [core.certbot] +enabled = true +# Path to the working directory workdir = "$CERBOT_WORKDIR" +# ACME server URL +acme_url = "$ACME_URL" +# Cloudflare API token +cf_api_token = "$CF_API_TOKEN" +# Cloudflare zone ID +cf_zone_id = "$CF_ZONE_ID" +# Auto set CAA record +auto_set_caa = true +# Domain to issue certificates for +domain = "*.$TPROXY_PUBLIC_DOMAIN" +# Check renewal interval +renew_interval = "30m" +# Number of days before expiration to trigger renewal +renew_days_before = "10d" +# Renew timeout +renew_timeout = "10m" [core.wg] private_key = "$TPROXY_WG_KEY" public_key = "$TPROXY_WG_PUBKEY" listen_port = $TPROXY_WG_LISTEN_PORT -ip = "$TPROXY_WG_IP" -reserved_net = "$TPROXY_WG_RESERVED_NET" -client_ip_range = "$TPROXY_WG_IP" +ip = "$TPROXY_WG_IP/24" +reserved_net = "$TPROXY_WG_IP/31" +client_ip_range = "$TPROXY_WG_IP/24" config_path = "$RUN_DIR/wg.conf" interface = "$TPROXY_WG_INTERFACE" endpoint = "10.0.2.2:$TPROXY_WG_LISTEN_PORT" [core.proxy] -cert_chain = "$TPROXY_CERT" -cert_key = "$TPROXY_KEY" +cert_chain = "$CERBOT_WORKDIR/live/cert.pem" +cert_key = "$CERBOT_WORKDIR/live/key.pem" base_domain = "$TPROXY_PUBLIC_DOMAIN" listen_addr = "$BIND_PUBLIC_IP" listen_port = $TPROXY_SERVE_PORT @@ -194,7 +213,7 @@ tappd_port = $TAPPD_PORT EOF # teepod - cat < teepod.toml + cat <teepod.toml log_level = "info" address = "127.0.0.1" port = $TEEPOD_RPC_LISTEN_PORT @@ -212,6 +231,7 @@ enabled = $TEEPOD_PORT_MAPPING_ENABLED address = "127.0.0.1" range = [ { protocol = "tcp", from = 1, to = 20000 }, + { protocol = "udp", from = 1, to = 20000 }, ] [gateway] @@ -223,37 +243,6 @@ tappd_port = $TAPPD_PORT port = $TEEPOD_VSOCK_LISTEN_PORT EOF - cat < certbot.toml -# Path to the working directory -workdir = "$CERBOT_WORKDIR" -# ACME server URL -acme_url = "$ACME_URL" -# Cloudflare API token -cf_api_token = "$CF_API_TOKEN" -# Cloudflare zone ID -cf_zone_id = "$CF_ZONE_ID" -# Auto set CAA record -auto_set_caa = true -# Domain to issue certificates for -domain = "*.$TPROXY_PUBLIC_DOMAIN" -# Renew interval in seconds -renew_interval = 3600 -# Number of days before expiration to trigger renewal -renew_days_before = 10 -# Renew timeout in seconds -renew_timeout = 120 -EOF - - cat < kms-allow-upgrade.sh -#!/bin/bash -if [ \$# -ne 2 ]; then - echo "Usage: \$0 " - exit 1 -fi -mkdir -p "$KMS_UPGRADE_REGISTRY_DIR/\$1" -touch "$KMS_UPGRADE_REGISTRY_DIR/\$1/\$2" -EOF - chmod +x kms-allow-upgrade.sh mkdir -p $RUN_DIR mkdir -p $CERBOT_WORKDIR/backup/preinstalled } @@ -262,7 +251,7 @@ build_wg() { echo "Setting up wireguard interface" # Step 6: setup wireguard interface # Check if the WireGuard interface exists - if ! ip link show $TPROXY_WG_INTERFACE &> /dev/null; then + if ! ip link show $TPROXY_WG_INTERFACE &>/dev/null; then sudo ip link add $TPROXY_WG_INTERFACE type wireguard sudo ip address add $TPROXY_WG_IP dev $TPROXY_WG_INTERFACE sudo ip link set $TPROXY_WG_INTERFACE up @@ -273,36 +262,69 @@ build_wg() { # sudo ip route add $TPROXY_WG_CLIENT_IP_RANGE dev $TPROXY_WG_INTERFACE } +download_image() { + local VERSION="" + local IS_DEV="" + + # Parse arguments to support both formats + if [[ "$1" == "-dev" ]]; then + IS_DEV=1 + VERSION=$2 + else + VERSION=$1 + fi + + echo "Downloading image $VERSION${IS_DEV:+ (dev)}" + + TAG=v$VERSION + if [ x"$IS_DEV" = x"1" ]; then + BASENAME=dstack-dev-$VERSION + else + BASENAME=dstack-$VERSION + fi + URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/$TAG/$BASENAME.tar.gz + if [ -d $IMAGES_DIR/$BASENAME ]; then + echo "Image already exists" + else + mkdir -p $IMAGES_DIR/$BASENAME + curl -L $URL -o $IMAGES_DIR/$BASENAME.tar.gz + tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME + rm -f $IMAGES_DIR/$BASENAME.tar.gz + fi +} case $ACTION in - host) - build_host - ;; - guest) - build_guest $2 - ;; - cfg) - require_config - build_cfg - ;; - certs) - require_config - ;; - wg) - require_config - build_wg - ;; - "") - # If no action specified, build everything - require_config - build_host - build_guest - build_cfg - build_wg - ;; - *) - echo "Invalid action: $ACTION" - echo "Valid actions are: host, guest, cfg, certs, wg" - exit 1 - ;; +host) + build_host + ;; +guest) + build_guest $2 + ;; +cfg) + require_config + build_cfg + ;; +certs) + require_config + ;; +wg) + require_config + build_wg + ;; +dl) + download_image $2 $3 + ;; +"") + # If no action specified, build everything + require_config + build_host + build_guest + build_cfg + build_wg + ;; +*) + echo "Invalid action: $ACTION" + echo "Valid actions are: host, guest, cfg, certs, wg, dl" + exit 1 + ;; esac From 894034233e5c8b2820fb8b80794ab0c68cf172a6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 6 Mar 2025 09:06:14 +0000 Subject: [PATCH 110/435] Update submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index edb4765ab..86b7d93b5 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit edb4765abad88b33029eb928d06f63bdb5c864f2 +Subproject commit 86b7d93b5e89e12a0e4cc1a687813b8d4c0f41e3 From 6f8ff172db39cf79056e954a336e1b80ea1ffaeb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 6 Mar 2025 13:40:26 +0000 Subject: [PATCH 111/435] dstack.py: Add flag InUse --- scripts/bin/dstack.py | 59 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index face3c89e..4bf706b8c 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -411,16 +411,59 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None def list_available_gpus() -> None: """List available NVIDIA GPUs.""" try: - result = subprocess.run(['lspci'], capture_output=True, text=True) - gpu_lines = [line for line in result.stdout.split('\n') if 'NVIDIA' in line] - if gpu_lines: + # Run lspci with verbose output to get detailed information + result = subprocess.run(['lspci', '-vvk'], capture_output=True, text=True) + output_lines = result.stdout.split('\n') + + # Find all GPU entries and their details + gpu_blocks = [] + current_block = [] + in_gpu_block = False + + for line in output_lines: + if 'NVIDIA' in line and '3D controller' in line: + # Start of a new GPU block + if current_block: + gpu_blocks.append(current_block) + current_block = [line] + in_gpu_block = True + elif in_gpu_block: + if line.strip() == '' or (line[0] != '\t' and line[0] != ' ' and len(current_block) > 1): + # End of the current block + gpu_blocks.append(current_block) + current_block = [] + in_gpu_block = False + else: + # Continue adding lines to the current block + current_block.append(line) + + # Add the last block if it exists + if current_block: + gpu_blocks.append(current_block) + + if gpu_blocks: print("\nAvailable GPU IDs:") - print("ID Description") - for line in gpu_lines: - print(line) + print("ID In Use Description") + print("-------------------------------------") + + for block in gpu_blocks: + # Extract device ID from the first line + device_id = block[0].split()[0] + description = block[0].split(':', 2)[2].strip() + + # Check if GPU is in use by examining Control line and Latency + in_use = False + for line in block: + if 'Control:' in line and 'I/O+' in line and 'BusMaster+' in line: + in_use = True + elif 'Latency:' in line: + in_use = True + + status = "Yes" if in_use else "No" + print(f"{device_id} {status:8} {description}") print() - except subprocess.SubprocessError: - logger.warning("Could not list GPU devices") + except subprocess.SubprocessError as e: + logger.warning(f"Could not list GPU devices: {str(e)}") def main(): From 488b85047ed76f02ef485e4a81cbe86c724e7732 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 11 Mar 2025 07:24:38 +0000 Subject: [PATCH 112/435] dstack-cli: Add pin-numa and hugepages --- scripts/bin/dstack.py | 49 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 4bf706b8c..6cd4149c0 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -63,6 +63,7 @@ class VMConfig: memory: int disk_size: int image: str + image_path: str port_map: List[PortMap] created_at_ms: int @@ -75,6 +76,7 @@ def to_dict(self) -> Dict: "memory": self.memory, "disk_size": self.disk_size, "image": self.image, + "image_path": self.image_path, "port_map": [p.to_dict() for p in self.port_map], "created_at_ms": self.created_at_ms } @@ -136,7 +138,6 @@ def load_configs_merged(config_paths): class DStackConfig: """Configuration for DStack client.""" docker_registry: Optional[str] = None - image_path: str = './images' default_image_name: str = '' qemu_path: str = 'qemu-system-x86_64' @@ -150,7 +151,6 @@ def cfg_get(section, key, fallback): return fallback me = cls() me.docker_registry = cfg_get('docker', 'registry', cls.docker_registry) - me.image_path = os.path.abspath(cfg_get('image', 'path', cls.image_path)) me.default_image_name = cfg_get('image', 'default', cls.default_image_name) me.qemu_path = cfg_get('qemu', 'path', cls.qemu_path) return me @@ -161,10 +161,6 @@ def __init__(self): self.run_path = os.path.abspath(os.getenv('RUN_PATH', './vms')) self.config = DStackConfig.load() - def get_default_image_path(self) -> str: - """Get the full default image path.""" - return os.path.join(self.config.image_path, self.config.default_image_name) - def _generate_instance_id(self) -> str: """Generate a random instance ID.""" return str(uuid.uuid4()) @@ -196,7 +192,7 @@ def _create_directories(self, work_dir: str) -> tuple[str, str]: """Create necessary directories.""" if os.path.exists(work_dir): raise FileExistsError(f"The instance already exists at {work_dir}") - + shared_dir = os.path.join(work_dir, 'shared') certs_dir = os.path.join(shared_dir, 'certs') os.makedirs(shared_dir, exist_ok=True) @@ -240,13 +236,13 @@ def setup_instance(self, args: argparse.Namespace) -> None: # Generate instance ID if work_dir not provided instance_id = os.path.basename(args.dir) if args.dir else self._generate_instance_id() work_dir = args.dir or os.path.join(self.run_path, instance_id) - + # Create directories shared_dir, certs_dir = self._create_directories(work_dir) - + # Read compose file compose_content = self._read_compose_file(args.compose_file) - + # Create app-compose.json app_compose = { "manifest_version": 1, @@ -259,10 +255,8 @@ def setup_instance(self, args: argparse.Namespace) -> None: } with open(os.path.join(shared_dir, 'app-compose.json'), 'w') as f: json.dump(app_compose, f, indent=4) - # Read image metadata and create config.json - image_path = args.image or self.get_default_image_path() - rootfs_hash = self._read_image_metadata(image_path) + rootfs_hash = self._read_image_metadata(args.image) with open(os.path.join(shared_dir, '.sys-config.json'), 'w') as f: config = { "rootfs_hash": rootfs_hash, @@ -286,11 +280,12 @@ def setup_instance(self, args: argparse.Namespace) -> None: gpu=args.gpu or [], memory=memory, disk_size=disk_size, - image=os.path.basename(image_path.rstrip('/')), + image_path=args.image, + image=os.path.basename(args.image.rstrip('/')), port_map=port_map, created_at_ms=int(datetime.now().timestamp() * 1000) ) - + with open(os.path.join(work_dir, 'vm-manifest.json'), 'w') as f: json.dump(vm_config.to_dict(), f, indent=4) @@ -300,7 +295,7 @@ def setup_instance(self, args: argparse.Namespace) -> None: logger.error(f"Failed to setup instance: {str(e)}") raise - def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None, vcpus: Optional[int] = None) -> None: + def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None, vcpus: Optional[int] = None, imgdir: Optional[str] = None, pin_numa: bool = False, hugepage: bool = False) -> None: """Run a VM instance from the specified directory. Args: @@ -316,7 +311,7 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None manifest = json.load(f) # Get image path and metadata - image_path = os.path.join(self.config.image_path, manifest['image']) + image_path = manifest.get('image_path') or os.path.join(imgdir, manifest['image']) img_metadata_path = os.path.join(image_path, 'metadata.json') # Update config.json with host API URL and port @@ -327,7 +322,7 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None config['host_vsock_port'] = host_port with open(config_file, 'w') as f: json.dump(config, f, indent=4) - + if not os.path.exists(img_metadata_path): raise ValueError(f"Image metadata not found at {img_metadata_path}") @@ -342,7 +337,7 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None vda = os.path.join(vm_dir, 'hda.img') config_dir = os.path.join(vm_dir, 'shared') - + # Create disk if it doesn't exist if not os.path.exists(vda): subprocess.run(['qemu-img', 'create', '-f', 'qcow2', vda, f"{disk_size}G"], check=True) @@ -400,6 +395,16 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None # Add kernel command line cmd.extend(['-append', img_metadata['cmdline']]) + if pin_numa and len(gpus) == 1: + numa_node = subprocess.run(['cat', f'/sys/bus/pci/devices/0000:{gpus[0]}/numa_node'], capture_output=True, text=True).stdout.strip() + cpus = subprocess.run(['cat', f'/sys/devices/system/node/node{numa_node}/cpulist'], capture_output=True, text=True).stdout.strip() + cmd = ['taskset', '-c', cpus] + cmd + if hugepage: + cmd.extend([ + '-numa', f'node,nodeid=0,cpus=0-{vcpu_count-1},memdev=mem0', + '-object', f'memory-backend-file,id=mem0,size={mem},mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={numa_node},policy=bind', + ]) + print(" ".join(cmd)) # Run the command try: @@ -480,7 +485,6 @@ def main(): setup_parser.add_argument('-d', '--disk', type=str, default='20G', help='Disk size (e.g., 20G)') setup_parser.add_argument('-g', '--gpu', type=str, action='append', help='GPU device') setup_parser.add_argument('-p', '--port', action='append', type=str, help='Port mapping in format: protocol[:address]:from:to') - setup_parser.add_argument('--no-fde', action='store_true', help='Disable Full Disk Encryption') setup_parser.add_argument('--local-key-provider', action='store_true', help='Enable local key provider') # Start command @@ -488,7 +492,10 @@ def main(): start_parser.add_argument('dir', type=str, help='Work directory') start_parser.add_argument('-m', '--memory', type=str, help='Memory size (e.g. 2G, 512M)') start_parser.add_argument('-c', '--vcpus', type=int, help='Number of virtual CPUs') + start_parser.add_argument('--imgdir', type=str, help='The image directory') start_parser.add_argument('--kp-port', type=int, default=3443, help='The key provider listening port') + start_parser.add_argument('--pin-numa', type=bool, default=False, help='Pin the guest to given NUMA') + start_parser.add_argument('--hugepage', type=bool, default=False, help='Enable pre-allocate on hugepage. Must also --pin-numa') # List Gpus command subparsers.add_parser('lsgpu', help='List available GPUs') @@ -505,7 +512,7 @@ def main(): print(f"Starting HTTP server on localhost:{host_port}") thread = threading.Thread(target=api.serve_forever, daemon=True) thread.start() - manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus) + manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus, imgdir=args.imgdir, pin_numa=args.pin_numa, hugepage=args.hugepage) elif args.command == 'lsgpu': list_available_gpus() else: From ce05163e309cdb2f545275422fe1d57dbfc4538d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 11 Mar 2025 07:29:57 +0000 Subject: [PATCH 113/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 86b7d93b5..77b606ba6 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 86b7d93b5e89e12a0e4cc1a687813b8d4c0f41e3 +Subproject commit 77b606ba67b048ece3aef684090dc0b63a97aa07 From 65649e221e25990276895546ccfb00a77aab793d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 11 Mar 2025 08:22:19 +0000 Subject: [PATCH 114/435] dstack cli: Compatibility for older images --- scripts/bin/dstack.py | 56 +++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 6cd4149c0..d2f428339 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -134,6 +134,17 @@ def load_configs_merged(config_paths): return config +def update_guest_config(config_file: str, data: Dict): + if not os.path.exists(config_file): + config = {} + else: + with open(config_file, 'r') as f: + config = json.load(f) + config.update(data) + with open(config_file, 'w') as f: + json.dump(config, f, indent=4) + + @dataclass class DStackConfig: """Configuration for DStack client.""" @@ -257,13 +268,13 @@ def setup_instance(self, args: argparse.Namespace) -> None: json.dump(app_compose, f, indent=4) # Read image metadata and create config.json rootfs_hash = self._read_image_metadata(args.image) - with open(os.path.join(shared_dir, '.sys-config.json'), 'w') as f: - config = { + + for filename in ['config.json', '.sys-config.json']: + update_guest_config(os.path.join(shared_dir, filename), { "rootfs_hash": rootfs_hash, "docker_registry": self.config.docker_registry, "pccs_url": "https://api.trustedservices.intel.com/sgx/certification/v4", - } - json.dump(config, f, indent=4) + }) # Create VM manifest memory = self._convert_memory_to_mb(str(args.memory)) @@ -316,12 +327,12 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None # Update config.json with host API URL and port shared_dir = os.path.join(vm_dir, 'shared') - config_file = os.path.join(shared_dir, '.sys-config.json') - config = json.load(open(config_file, 'r')) - config['host_api_url'] = f"http://10.0.2.2:{host_port}/api" - config['host_vsock_port'] = host_port - with open(config_file, 'w') as f: - json.dump(config, f, indent=4) + for filename in ['config.json', '.sys-config.json']: + config_file = os.path.join(shared_dir, filename) + update_guest_config(config_file, { + "host_api_url": f"http://10.0.2.2:{host_port}/api", + "host_vsock_port": host_port + }) if not os.path.exists(img_metadata_path): raise ValueError(f"Image metadata not found at {img_metadata_path}") @@ -417,14 +428,15 @@ def list_available_gpus() -> None: """List available NVIDIA GPUs.""" try: # Run lspci with verbose output to get detailed information - result = subprocess.run(['lspci', '-vvk'], capture_output=True, text=True) + result = subprocess.run( + ['lspci', '-vvk'], capture_output=True, text=True) output_lines = result.stdout.split('\n') - + # Find all GPU entries and their details gpu_blocks = [] current_block = [] in_gpu_block = False - + for line in output_lines: if 'NVIDIA' in line and '3D controller' in line: # Start of a new GPU block @@ -441,21 +453,21 @@ def list_available_gpus() -> None: else: # Continue adding lines to the current block current_block.append(line) - + # Add the last block if it exists if current_block: gpu_blocks.append(current_block) - + if gpu_blocks: print("\nAvailable GPU IDs:") print("ID In Use Description") print("-------------------------------------") - + for block in gpu_blocks: # Extract device ID from the first line device_id = block[0].split()[0] description = block[0].split(':', 2)[2].strip() - + # Check if GPU is in use by examining Control line and Latency in_use = False for line in block: @@ -463,7 +475,7 @@ def list_available_gpus() -> None: in_use = True elif 'Latency:' in line: in_use = True - + status = "Yes" if in_use else "No" print(f"{device_id} {status:8} {description}") print() @@ -507,17 +519,19 @@ def main(): manager.setup_instance(args) elif args.command == 'run': manager = DStackManager() - config = host_api.ServerConfig(vm_dir=args.dir, kp_address="127.0.0.1", kp_port=args.kp_port) + config = host_api.ServerConfig( + vm_dir=args.dir, kp_address="127.0.0.1", kp_port=args.kp_port) api, host_port = host_api.create_http_server(config) print(f"Starting HTTP server on localhost:{host_port}") thread = threading.Thread(target=api.serve_forever, daemon=True) thread.start() - manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus, imgdir=args.imgdir, pin_numa=args.pin_numa, hugepage=args.hugepage) + manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus, + imgdir=args.imgdir, pin_numa=args.pin_numa, hugepage=args.hugepage) elif args.command == 'lsgpu': list_available_gpus() else: parser.print_help() + if __name__ == '__main__': main() - From 26d369b08950870128a422db02e560aa594c07fd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 14 Mar 2025 10:42:06 +0000 Subject: [PATCH 115/435] Update builder image to ubuntu 24.04 --- mkimage.sh | 4 +++- repro-build/Dockerfile.repro | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index c9c3de120..d4c4ecd98 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -41,6 +41,8 @@ OVMF_FIRMWARE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd ROOTFS_HASH=$(sha256sum "$ROOTFS_IMAGE" | awk '{print $1}') DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} +OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) +DSTACK_TAR_EXCLUDE_ROOTFS_CPIO=${DSTACK_TAR_EXCLUDE_ROOTFS_CPIO:-1} mkdir -p ${WORK_DIR} @@ -116,6 +118,6 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then if [ x$DSTACK_TAR_EXCLUDE_ROOTFS_CPIO = x1 ]; then TAR_ARGS=--exclude=rootfs.cpio fi - (cd ${OUTPUT_DIR} && tar -czf ${OUTPUT_DIR}.tar.gz ${TAR_ARGS} .) + (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${OUTPUT_DIR}.tar.gz ${TAR_ARGS} $(basename $OUTPUT_DIR)) echo fi diff --git a/repro-build/Dockerfile.repro b/repro-build/Dockerfile.repro index 3af41ba4d..00c3084d1 100644 --- a/repro-build/Dockerfile.repro +++ b/repro-build/Dockerfile.repro @@ -1,4 +1,4 @@ -FROM ubuntu:22.04@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97 +FROM ubuntu:24.04@sha256:3afff29dffbc200d202546dc6c4f614edc3b109691e7ab4aa23d02b42ba86790 # Prevent timezone prompt by setting noninteractive frontend and configuring tzdata ENV DEBIAN_FRONTEND=noninteractive \ @@ -24,7 +24,9 @@ RUN apt update && apt install -y \ gawk \ wget \ libclang-dev \ - xorriso + xorriso \ + cpio \ + zstd # Generate locale for bitbake RUN locale-gen en_US.UTF-8 From 2bf4bb967f9d725bf81b66290661f28f7a9b5c3c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 13 Mar 2025 15:20:10 +0000 Subject: [PATCH 116/435] v0.4.1 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 77b606ba6..1db737018 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 77b606ba67b048ece3aef684090dc0b63a97aa07 +Subproject commit 1db737018cff2d17d4b463204c18ebd6e9d4eb33 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index d6023aa09..658aa26cc 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.4.0" +DISTRO_VERSION = "0.4.1" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From afd2f1ef254e068d1a78ba4c70620ac0bca65de0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Mar 2025 01:35:01 +0000 Subject: [PATCH 117/435] build.sh: Adapt new tar structure --- build.sh | 10 ++++++++-- mkimage.sh | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 01224379d..9c7455443 100755 --- a/build.sh +++ b/build.sh @@ -286,10 +286,16 @@ download_image() { if [ -d $IMAGES_DIR/$BASENAME ]; then echo "Image already exists" else - mkdir -p $IMAGES_DIR/$BASENAME + mkdir -p $IMAGES_DIR/$BASENAME.tmp curl -L $URL -o $IMAGES_DIR/$BASENAME.tar.gz - tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME + tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME.tmp rm -f $IMAGES_DIR/$BASENAME.tar.gz + if [ -d $IMAGES_DIR/$BASENAME.tmp/$BASENAME ]; then + mv $IMAGES_DIR/$BASENAME.tmp/$BASENAME $IMAGES_DIR/$BASENAME + rm -rf $IMAGES_DIR/$BASENAME.tmp + else + mv $IMAGES_DIR/$BASENAME.tmp $IMAGES_DIR/$BASENAME + fi fi } diff --git a/mkimage.sh b/mkimage.sh index d4c4ecd98..7518e7279 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -41,7 +41,6 @@ OVMF_FIRMWARE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd ROOTFS_HASH=$(sha256sum "$ROOTFS_IMAGE" | awk '{print $1}') DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} -OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) DSTACK_TAR_EXCLUDE_ROOTFS_CPIO=${DSTACK_TAR_EXCLUDE_ROOTFS_CPIO:-1} mkdir -p ${WORK_DIR} @@ -114,6 +113,7 @@ find . -type f -not -name md5sum.txt -not -name sha256sum.txt -exec sha256sum {} popd if [ x$DSTACK_TAR_RELEASE = x1 ]; then + OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) echo "Archiving the output directory to ${OUTPUT_DIR}.tar.gz" if [ x$DSTACK_TAR_EXCLUDE_ROOTFS_CPIO = x1 ]; then TAR_ARGS=--exclude=rootfs.cpio From 318e9a6ff47ec8b6f8e542447b9c77761242293a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Mar 2025 02:52:43 +0000 Subject: [PATCH 118/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 1db737018..b84a314ae 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 1db737018cff2d17d4b463204c18ebd6e9d4eb33 +Subproject commit b84a314ae322eca33165fce5a046e4dcf6b7820e From 89ea7e6ffde95a73c41e93f6aee647df9d005a5a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 17 Mar 2025 14:09:55 +0000 Subject: [PATCH 119/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index b84a314ae..9648b9306 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit b84a314ae322eca33165fce5a046e4dcf6b7820e +Subproject commit 9648b930633af5925e9392c1ae6995ff774ba3fc From 4c9939f0566eddf8d413dd055d6cd3f1d3facdd0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 19 Mar 2025 07:12:39 +0000 Subject: [PATCH 120/435] Update dstack and build.sh --- build.sh | 14 ++++++++++---- dstack | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 9c7455443..5e987eaad 100755 --- a/build.sh +++ b/build.sh @@ -15,6 +15,9 @@ CERBOT_WORKDIR=$RUN_DIR/certbot KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ +TPROXY_CERT=${TPROXY_CERT:-$CERTS_DIR/live/cert.pem} +TPROXY_KEY=${TPROXY_KEY:-$CERTS_DIR/live/key.pem} + CONFIG_FILE=./build-config.sh check_config() { @@ -67,12 +70,15 @@ TPROXY_WG_INTERFACE=tproxy-$USER TPROXY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) TPROXY_WG_IP=10.3.3.1 TPROXY_SERVE_PORT=$(($BASE_PORT + 4)) +TPROXY_CERT= +TPROXY_KEY= BIND_PUBLIC_IP=0.0.0.0 TPROXY_PUBLIC_DOMAIN=app.kvin.wang # for certbot +CERTBOT_ENABLED=false CF_API_TOKEN= CF_ZONE_ID= ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory @@ -172,7 +178,7 @@ run_as_tapp = false enabled = false [core.certbot] -enabled = true +enabled = $CERTBOT_ENABLED # Path to the working directory workdir = "$CERBOT_WORKDIR" # ACME server URL @@ -197,15 +203,15 @@ private_key = "$TPROXY_WG_KEY" public_key = "$TPROXY_WG_PUBKEY" listen_port = $TPROXY_WG_LISTEN_PORT ip = "$TPROXY_WG_IP/24" -reserved_net = "$TPROXY_WG_IP/31" +reserved_net = ["$TPROXY_WG_IP/31"] client_ip_range = "$TPROXY_WG_IP/24" config_path = "$RUN_DIR/wg.conf" interface = "$TPROXY_WG_INTERFACE" endpoint = "10.0.2.2:$TPROXY_WG_LISTEN_PORT" [core.proxy] -cert_chain = "$CERBOT_WORKDIR/live/cert.pem" -cert_key = "$CERBOT_WORKDIR/live/key.pem" +cert_chain = "$TPROXY_CERT" +cert_key = "$TPROXY_KEY" base_domain = "$TPROXY_PUBLIC_DOMAIN" listen_addr = "$BIND_PUBLIC_IP" listen_port = $TPROXY_SERVE_PORT diff --git a/dstack b/dstack index 9648b9306..126b23e13 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 9648b930633af5925e9392c1ae6995ff774ba3fc +Subproject commit 126b23e130af0f29342c96393e78801fc51d1817 From 1f9d1341bb97b5098b7253ebd7c8ed12690ca13b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 19 Mar 2025 10:11:14 +0000 Subject: [PATCH 121/435] Use openssh instead of dropbear --- meta-dstack/recipes-core/images/dstack-rootfs-dev.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index 37fb8861e..698941925 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc -IMAGE_INSTALL += "packagegroup-core-ssh-dropbear strace tcpdump gdb gdbserver" +IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver" IMAGE_FEATURES = "debug-tweaks" From 2b57023cc9d13caa98c9611d0f6dd13b4fff54cb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 19 Mar 2025 10:30:15 +0000 Subject: [PATCH 122/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 126b23e13..0098af07d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 126b23e130af0f29342c96393e78801fc51d1817 +Subproject commit 0098af07d29eabb862af58be9b3cf39c471fd7e9 From bef2dfa850f4116ae4ece96d8c0948965c5874b3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 24 Mar 2025 01:52:10 +0000 Subject: [PATCH 123/435] Enable Secure System Time --- .../recipes-core/chrony/chrony%.bbappend | 3 + .../recipes-core/chrony/files/chrony.conf | 59 +++++++++++++++++++ .../images/dstack-initramfs-files/init | 1 + .../recipes-core/images/dstack-initramfs.bb | 1 + .../images/dstack-rootfs-base.inc | 1 + mkimage.sh | 7 ++- 6 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 meta-dstack/recipes-core/chrony/chrony%.bbappend create mode 100644 meta-dstack/recipes-core/chrony/files/chrony.conf diff --git a/meta-dstack/recipes-core/chrony/chrony%.bbappend b/meta-dstack/recipes-core/chrony/chrony%.bbappend new file mode 100644 index 000000000..ba3d2eafe --- /dev/null +++ b/meta-dstack/recipes-core/chrony/chrony%.bbappend @@ -0,0 +1,3 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +DEPENDS += "gnutls" diff --git a/meta-dstack/recipes-core/chrony/files/chrony.conf b/meta-dstack/recipes-core/chrony/files/chrony.conf new file mode 100644 index 000000000..d6c1b0f2b --- /dev/null +++ b/meta-dstack/recipes-core/chrony/files/chrony.conf @@ -0,0 +1,59 @@ +# Load config files matching the /etc/chrony/conf.d/*.conf pattern. +confdir /etc/chrony/conf.d + +# Use public NTP servers from the pool.ntp.org project. +# Please consider joining the pool project if possible by running your own +# server(s). +# If you are a vendor distributing a product using chrony, you *MUST* +# read and comply with http://www.pool.ntp.org/vendors.html +# pool 0.openembedded.pool.ntp.org iburst + +# Use a local timeserver in preference to the pool, if it's reachable. +#server 192.168.22.22 iburst minpoll 2 prefer +server time.cloudflare.com iburst nts +server nts.teambelgium.net iburst nts +server a.st1.ntp.br iburst nts +server time.bolha.one iburst nts +server ptbtime1.ptb.de iburst nts +server ntp2.glypnod.com iburst nts +server ntp1.glypnod.com iburst nts +server virginia.time.system76.com iburst nts + +# Sync to pulse-per-second from an onboard GPS. +#refclock PPS /dev/pps0 poll 0 prefer +# You'll want to enable CONFIG_PPS and CONFIG_PPS_CLIENT_GPIO in your kernel, +# and an entry something like this in your device tree: +# pps { +# compatible = "pps-gpio"; +# gpios = <&ps7_gpio_0 56 0>; +# }; + +# Load source files matching the /etc/chrony/sources.d/*.sources pattern. +# These can be reloaded using 'chronyc reload sources'. +sourcedir /etc/chrony/sources.d + +# In first three updates step the system clock instead of slew +# if the adjustment is larger than 1 second. +makestep 1.0 3 + +# Record the rate at which the system clock gains/loses time, +# improving accuracy after reboot +driftfile /var/lib/chrony/drift + +# Enable kernel synchronization of the hardware real-time clock (RTC). +# rtcsync + +# Allow NTP client access from local network. +#allow 192.168/16 + +# Serve time even if not synchronized to any NTP server. +#local stratum 10 + +# Specify file containing keys for NTP authentication. +#keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# Select which information is logged. +#log measurements statistics tracking diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index 37aa6c75c..25907f54d 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -21,6 +21,7 @@ mount -t devpts devpts /dev/pts ifconfig lo up 127.0.0.1 ifconfig eth0 up udhcpc -i eth0 +chronyd -q modprobe tdx-guest || true diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 04a138bc1..8aae04a6a 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -13,6 +13,7 @@ PACKAGE_INSTALL = "\ dstack-guest \ curl \ jq \ + chrony \ " # Do not pollute the initrd image with rootfs features diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 025985a08..e61eeed3c 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -16,6 +16,7 @@ IMAGE_INSTALL = "\ wireguard-tools \ curl \ jq \ + chrony \ qemu-guest-agent \ " diff --git a/mkimage.sh b/mkimage.sh index 7518e7279..03a0fe2e8 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -91,11 +91,16 @@ $Q makeiso ${WORK_DIR}/rootfs/ ${OUTPUT_DIR}/rootfs.iso GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" + +KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 pnpacpi=off" +KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" +KARG2="dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH" + cat < ${OUTPUT_DIR}/metadata.json { "bios": "ovmf.fd", "kernel": "bzImage", - "cmdline": "console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 pnpacpi=off dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH", + "cmdline": "$KARG0 $KARG1 $KARG2", "initrd": "initramfs.cpio.gz", "rootfs": "rootfs.iso", "rootfs_hash": "$ROOTFS_HASH", From 101996606aee303099ca3882675c1161d03ccb15 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 1 Apr 2025 01:36:51 +0000 Subject: [PATCH 124/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 0098af07d..52fe14c80 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 0098af07d29eabb862af58be9b3cf39c471fd7e9 +Subproject commit 52fe14c80d04d749d1161fca75245d63f520702f From a98a660994d17c9b0accc22d38754686718300f7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 2 Apr 2025 03:42:15 +0000 Subject: [PATCH 125/435] Update dstack & rename tappd to dstack-guest-agent --- build.sh | 6 +++--- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 16 ++++------------ .../recipes-core/images/dstack-initramfs.bb | 2 +- scripts/bin/lsvms.py | 6 +++--- 6 files changed, 13 insertions(+), 21 deletions(-) diff --git a/build.sh b/build.sh index 5e987eaad..7fa61512f 100755 --- a/build.sh +++ b/build.sh @@ -96,7 +96,7 @@ EOF if [ -z "$TPROXY_SERVE_PORT" ]; then TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} fi - TAPPD_PORT=8090 + AGENT_PORT=8090 else mv build-config.sh.tpl $CONFIG_FILE echo "Config file $CONFIG_FILE created, please edit it to configure the build" @@ -215,7 +215,7 @@ cert_key = "$TPROXY_KEY" base_domain = "$TPROXY_PUBLIC_DOMAIN" listen_addr = "$BIND_PUBLIC_IP" listen_port = $TPROXY_SERVE_PORT -tappd_port = $TAPPD_PORT +agent_port = $AGENT_PORT EOF # teepod @@ -243,7 +243,7 @@ range = [ [gateway] base_domain = "$TPROXY_PUBLIC_DOMAIN" port = $TPROXY_SERVE_PORT -tappd_port = $TAPPD_PORT +agent_port = $AGENT_PORT [host_api] port = $TEEPOD_VSOCK_LISTEN_PORT diff --git a/dstack b/dstack index 52fe14c80..d034574a3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 52fe14c80d04d749d1161fca75245d63f520702f +Subproject commit d034574a363f6cd85f18a357e93ec341abb65a23 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 658aa26cc..35c83a03a 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.4.1" +DISTRO_VERSION = "0.4.2" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index dbb9aa0b4..ff37a4085 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -3,7 +3,7 @@ DESCRIPTION = "${SUMMARY}" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" -inherit systemd update-rc.d +inherit systemd REPO_ROOT = "${THISDIR}/../../.." @@ -16,15 +16,11 @@ S = "${WORKDIR}/${SRC_DIR}" RDEPENDS:${PN} += "bash" -DSTACK_SERVICES = "tappd.service tboot.service app-compose.service" +DSTACK_SERVICES = "dstack-guest-agent.service tboot.service app-compose.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" -INITSCRIPT_PACKAGES += "${@bb.utils.contains('DISTRO_FEATURES','systemd','','${PN}',d)}" -INITSCRIPT_NAME:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','','tappd.init',d)}" -INITSCRIPT_PARAMS:${PN} = "defaults" - inherit cargo_bin do_configure() { @@ -43,7 +39,7 @@ do_install() { install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} - install -m 0755 ${CARGO_BINDIR}/tappd ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${WORKDIR}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json @@ -56,13 +52,9 @@ do_install() { install -d ${D}${systemd_system_unitdir} \ ${D}${sysconfdir}/systemd/resolved.conf.d - install -m 0644 ${S}/basefiles/tappd.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-guest-agent.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/tboot.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d - else - install -d ${D}${sysconfdir}/init.d - install -m 0755 ${S}/basefiles/tappd.init ${D}${sysconfdir}/init.d/tappd.init - bberror "init scripts for sysvinit is not implemented yet" fi } diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 8aae04a6a..94e7f0e99 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -43,5 +43,5 @@ ROOTFS_POSTPROCESS_COMMAND += "postprocess_initramfs;" postprocess_initramfs() { rm -rf ${IMAGE_ROOTFS}${sysconfdir}/init.d rm -rf ${IMAGE_ROOTFS}${systemd_system_unitdir} - rm -rf ${IMAGE_ROOTFS}${bindir}/tappd + rm -rf ${IMAGE_ROOTFS}${bindir}/dstack-guest-agent } diff --git a/scripts/bin/lsvms.py b/scripts/bin/lsvms.py index 54839bd5d..dbf2dbd9c 100755 --- a/scripts/bin/lsvms.py +++ b/scripts/bin/lsvms.py @@ -60,11 +60,11 @@ def get_dashboard_url(config: dict, app_id: str) -> str: try: gateway = config.get('gateway', {}) base_domain = gateway.get('base_domain') - tappd_port = gateway.get('tappd_port') + agent_port = gateway.get('agent_port') gateway_port = gateway.get('port') - if base_domain and tappd_port: - url = f"https://{app_id}-{tappd_port}.{base_domain}" + if base_domain and agent_port: + url = f"https://{app_id}-{agent_port}.{base_domain}" if gateway_port: url += f":{gateway_port}" return url + "/" From 3575cf6eae64b255853a9bb4868ec2efacefb64f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 2 Apr 2025 10:29:04 +0000 Subject: [PATCH 126/435] Rename tproxy to dstack-gateway --- build.sh | 105 +++++++++++++-------------------- dstack | 2 +- scripts/bin/lsvms.py | 136 ------------------------------------------- 3 files changed, 42 insertions(+), 201 deletions(-) delete mode 100755 scripts/bin/lsvms.py diff --git a/build.sh b/build.sh index 7fa61512f..9bd4bdf23 100755 --- a/build.sh +++ b/build.sh @@ -15,8 +15,8 @@ CERBOT_WORKDIR=$RUN_DIR/certbot KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ -TPROXY_CERT=${TPROXY_CERT:-$CERTS_DIR/live/cert.pem} -TPROXY_KEY=${TPROXY_KEY:-$CERTS_DIR/live/key.pem} +GATEWAY_CERT=${GATEWAY_CERT:-$CERTS_DIR/live/cert.pem} +GATEWAY_KEY=${GATEWAY_KEY:-$CERTS_DIR/live/key.pem} CONFIG_FILE=./build-config.sh @@ -43,11 +43,11 @@ check_config() { require_config() { cat <<'EOF' >build-config.sh.tpl -# DNS domain of kms rpc and tproxy rpc +# DNS domain of kms rpc and dstack-gateway rpc # *.1022.kvin.wang resolves to 10.0.2.2 which is the IP of the host system # from CVMs point of view KMS_DOMAIN=kms.1022.kvin.wang -TPROXY_DOMAIN=tproxy.1022.kvin.wang +GATEWAY_DOMAIN=gateway.1022.kvin.wang # CIDs allocated to VMs start from this number of type unsigned int32 TEEPOD_CID_POOL_START=20000 @@ -64,18 +64,18 @@ TEEPOD_PORT_MAPPING_ENABLED=false TEEPOD_VSOCK_LISTEN_PORT=$BASE_PORT KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) -TPROXY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) +GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) -TPROXY_WG_INTERFACE=tproxy-$USER -TPROXY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) -TPROXY_WG_IP=10.3.3.1 -TPROXY_SERVE_PORT=$(($BASE_PORT + 4)) -TPROXY_CERT= -TPROXY_KEY= +GATEWAY_WG_INTERFACE=dgw-$USER +GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) +GATEWAY_WG_IP=10.3.3.1 +GATEWAY_SERVE_PORT=$(($BASE_PORT + 4)) +GATEWAY_CERT= +GATEWAY_KEY= BIND_PUBLIC_IP=0.0.0.0 -TPROXY_PUBLIC_DOMAIN=app.kvin.wang +GATEWAY_PUBLIC_DOMAIN=app.kvin.wang # for certbot CERTBOT_ENABLED=false @@ -93,8 +93,8 @@ EOF fi rm -f build-config.sh.tpl - if [ -z "$TPROXY_SERVE_PORT" ]; then - TPROXY_SERVE_PORT=${TPROXY_LISTEN_PORT1} + if [ -z "$GATEWAY_SERVE_PORT" ]; then + GATEWAY_SERVE_PORT=${GATEWAY_LISTEN_PORT1} fi AGENT_PORT=8090 else @@ -108,7 +108,7 @@ EOF build_host() { echo "Building binaries" (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) - cp ${RUST_BUILD_DIR}/release/{tproxy,kms,teepod,certbot,ct_monitor,supervisor} . + cp ${RUST_BUILD_DIR}/release/{dstack-gateway,kms,teepod,supervisor} . } # Step 2: build guest images @@ -124,8 +124,8 @@ build_guest() { build_cfg() { echo "Building config files" - TPROXY_WG_KEY=$(wg genkey) - TPROXY_WG_PUBKEY=$(echo $TPROXY_WG_KEY | wg pubkey) + GATEWAY_WG_KEY=$(wg genkey) + GATEWAY_WG_PUBKEY=$(echo $GATEWAY_WG_KEY | wg pubkey) # kms cat <kms.toml log_level = "info" @@ -155,23 +155,23 @@ port = $KMS_RPC_LISTEN_PORT auto_bootstrap_domain = "$KMS_DOMAIN" EOF - # tproxy - cat <tproxy.toml + # dstack-gateway + cat <gateway.toml log_level = "info" address = "127.0.0.1" -port = $TPROXY_RPC_LISTEN_PORT +port = $GATEWAY_RPC_LISTEN_PORT [tls] -key = "$CERTS_DIR/tproxy-rpc.key" -certs = "$CERTS_DIR/tproxy-rpc.cert" +key = "$CERTS_DIR/gateway-rpc.key" +certs = "$CERTS_DIR/gateway-rpc.cert" [tls.mutual] -ca_certs = "$CERTS_DIR/tproxy-ca.cert" +ca_certs = "$CERTS_DIR/gateway-ca.cert" mandatory = false [core] kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" -rpc_domain = "$TPROXY_DOMAIN" +rpc_domain = "$GATEWAY_DOMAIN" run_as_tapp = false [core.sync] @@ -190,7 +190,7 @@ cf_zone_id = "$CF_ZONE_ID" # Auto set CAA record auto_set_caa = true # Domain to issue certificates for -domain = "*.$TPROXY_PUBLIC_DOMAIN" +domain = "*.$GATEWAY_PUBLIC_DOMAIN" # Check renewal interval renew_interval = "30m" # Number of days before expiration to trigger renewal @@ -199,22 +199,22 @@ renew_days_before = "10d" renew_timeout = "10m" [core.wg] -private_key = "$TPROXY_WG_KEY" -public_key = "$TPROXY_WG_PUBKEY" -listen_port = $TPROXY_WG_LISTEN_PORT -ip = "$TPROXY_WG_IP/24" -reserved_net = ["$TPROXY_WG_IP/31"] -client_ip_range = "$TPROXY_WG_IP/24" +private_key = "$GATEWAY_WG_KEY" +public_key = "$GATEWAY_WG_PUBKEY" +listen_port = $GATEWAY_WG_LISTEN_PORT +ip = "$GATEWAY_WG_IP/24" +reserved_net = ["$GATEWAY_WG_IP/31"] +client_ip_range = "$GATEWAY_WG_IP/24" config_path = "$RUN_DIR/wg.conf" -interface = "$TPROXY_WG_INTERFACE" -endpoint = "10.0.2.2:$TPROXY_WG_LISTEN_PORT" +interface = "$GATEWAY_WG_INTERFACE" +endpoint = "10.0.2.2:$GATEWAY_WG_LISTEN_PORT" [core.proxy] -cert_chain = "$TPROXY_CERT" -cert_key = "$TPROXY_KEY" -base_domain = "$TPROXY_PUBLIC_DOMAIN" +cert_chain = "$GATEWAY_CERT" +cert_key = "$GATEWAY_KEY" +base_domain = "$GATEWAY_PUBLIC_DOMAIN" listen_addr = "$BIND_PUBLIC_IP" -listen_port = $TPROXY_SERVE_PORT +listen_port = $GATEWAY_SERVE_PORT agent_port = $AGENT_PORT EOF @@ -229,7 +229,7 @@ kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] kms_urls = ["https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT"] -tproxy_urls = ["https://$TPROXY_DOMAIN:$TPROXY_RPC_LISTEN_PORT"] +gateway_urls = ["https://$GATEWAY_DOMAIN:$GATEWAY_RPC_LISTEN_PORT"] cid_start = $TEEPOD_CID_POOL_START cid_pool_size = $TEEPOD_CID_POOL_SIZE [cvm.port_mapping] @@ -241,8 +241,8 @@ range = [ ] [gateway] -base_domain = "$TPROXY_PUBLIC_DOMAIN" -port = $TPROXY_SERVE_PORT +base_domain = "$GATEWAY_PUBLIC_DOMAIN" +port = $GATEWAY_SERVE_PORT agent_port = $AGENT_PORT [host_api] @@ -253,21 +253,6 @@ EOF mkdir -p $CERBOT_WORKDIR/backup/preinstalled } -build_wg() { - echo "Setting up wireguard interface" - # Step 6: setup wireguard interface - # Check if the WireGuard interface exists - if ! ip link show $TPROXY_WG_INTERFACE &>/dev/null; then - sudo ip link add $TPROXY_WG_INTERFACE type wireguard - sudo ip address add $TPROXY_WG_IP dev $TPROXY_WG_INTERFACE - sudo ip link set $TPROXY_WG_INTERFACE up - echo "created and configured WireGuard interface $TPROXY_WG_INTERFACE" - else - echo "WireGuard interface $TPROXY_WG_INTERFACE already exists" - fi - # sudo ip route add $TPROXY_WG_CLIENT_IP_RANGE dev $TPROXY_WG_INTERFACE -} - download_image() { local VERSION="" local IS_DEV="" @@ -316,13 +301,6 @@ cfg) require_config build_cfg ;; -certs) - require_config - ;; -wg) - require_config - build_wg - ;; dl) download_image $2 $3 ;; @@ -332,11 +310,10 @@ dl) build_host build_guest build_cfg - build_wg ;; *) echo "Invalid action: $ACTION" - echo "Valid actions are: host, guest, cfg, certs, wg, dl" + echo "Valid actions are: host, guest, cfg, dl" exit 1 ;; esac diff --git a/dstack b/dstack index d034574a3..d9763e776 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d034574a363f6cd85f18a357e93ec341abb65a23 +Subproject commit d9763e776e655853a40580e05080307627194b3c diff --git a/scripts/bin/lsvms.py b/scripts/bin/lsvms.py deleted file mode 100755 index dbf2dbd9c..000000000 --- a/scripts/bin/lsvms.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -import os -import json -import sys -import argparse -from pathlib import Path - -def read_json_file(file_path: str, key: str) -> str: - """Read a value from a JSON file safely""" - try: - with open(file_path) as f: - return json.load(f).get(key, "N/A") - except (FileNotFoundError, json.JSONDecodeError, KeyError): - return "N/A" - -def get_instance_ip(tproxy_state_file: str, instance_id: str) -> str: - """Get instance IP from tproxy state file""" - try: - with open(tproxy_state_file) as f: - data = json.load(f) - return data.get('instances', {}).get(instance_id, {}).get('ip', 'N/A') - except (FileNotFoundError, json.JSONDecodeError): - return "N/A" - -def parse_simple_toml(file_path: Path) -> dict: - """Simple TOML parser for our specific needs""" - config = {'gateway': {}} - try: - with open(file_path) as f: - current_section = None - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - - if line.startswith('[') and line.endswith(']'): - current_section = line[1:-1] - if current_section not in config: - config[current_section] = {} - continue - - if '=' in line: - key, value = [x.strip() for x in line.split('=', 1)] - value = value.strip('"').strip("'") - try: - value = int(value) - except ValueError: - pass - if current_section: - config[current_section][key] = value - else: - config[key] = value - - return config - except (FileNotFoundError, IOError): - return {'gateway': {}} - -def get_dashboard_url(config: dict, app_id: str) -> str: - """Generate dashboard URL from config and app_id""" - try: - gateway = config.get('gateway', {}) - base_domain = gateway.get('base_domain') - agent_port = gateway.get('agent_port') - gateway_port = gateway.get('port') - - if base_domain and agent_port: - url = f"https://{app_id}-{agent_port}.{base_domain}" - if gateway_port: - url += f":{gateway_port}" - return url + "/" - except (KeyError, TypeError): - pass - return "N/A" - -def print_vm_info(vm_data: dict, verbose: bool = False): - """Pretty print VM information""" - if verbose: - print("=" * 100) - for key, value in vm_data.items(): - # Right-align keys in a 15-character space, followed by a colon and value - print(f"{key:>15}: {value}") - print() - else: - print(f"{vm_data['name']:<25} {vm_data['ip']:<15} {vm_data['image']}") - -def main(): - parser = argparse.ArgumentParser(description='List VMs information') - parser.add_argument('stack_dir', help='Stack directory path') - parser.add_argument('-v', '--verbose', action='store_true', help='Show detailed information') - args = parser.parse_args() - - stack_dir = Path(args.stack_dir) - vm_base_dir = stack_dir / "run" / "vm" - tproxy_state_file = stack_dir / "tproxy-state.json" - - if not vm_base_dir.exists(): - print(f"Error: Directory not found: {vm_base_dir}") - sys.exit(1) - - config = {} - if args.verbose: - config = parse_simple_toml(stack_dir / "teepod.toml") - - if not args.verbose: - # Print simple header - print(f"{'Name':<25} {'IP':<15} {'Image'}") - print("-" * 60) - - # Process each VM directory - for vm_dir in vm_base_dir.glob("*"): - if not vm_dir.is_dir(): - continue - - vm_data = { - 'name': read_json_file(str(vm_dir / "vm-manifest.json"), "name"), - 'image': read_json_file(str(vm_dir / "vm-manifest.json"), "image"), - } - - instance_info_file = vm_dir / "shared" / ".instance_info" - if instance_info_file.exists(): - instance_id = read_json_file(str(instance_info_file), "instance_id") - app_id = read_json_file(str(instance_info_file), "app_id") - vm_data.update({ - 'instance_id': instance_id, - 'app_id': app_id, - 'ip': get_instance_ip(str(tproxy_state_file), instance_id), - 'vm_dir': str(vm_dir), - }) - - if args.verbose: - vm_data['dashboard_url'] = get_dashboard_url(config, app_id) - - print_vm_info(vm_data, args.verbose) - -if __name__ == "__main__": - main() From af96577a6321f2b4083aba47b1a71a3d1c195b5b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 8 Apr 2025 03:15:53 +0000 Subject: [PATCH 127/435] Update dstack, API renaming --- build.sh | 26 +++++++++++++------------- dstack | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index 9bd4bdf23..a7e98a361 100755 --- a/build.sh +++ b/build.sh @@ -50,18 +50,18 @@ KMS_DOMAIN=kms.1022.kvin.wang GATEWAY_DOMAIN=gateway.1022.kvin.wang # CIDs allocated to VMs start from this number of type unsigned int32 -TEEPOD_CID_POOL_START=20000 +VMM_CID_POOL_START=20000 # CID pool size -TEEPOD_CID_POOL_SIZE=1000 +VMM_CID_POOL_SIZE=1000 # Base port for RPC services BASE_PORT=13000 -TEEPOD_RPC_LISTEN_PORT=$BASE_PORT +VMM_RPC_LISTEN_PORT=$BASE_PORT # Whether port mapping from host to CVM is allowed -TEEPOD_PORT_MAPPING_ENABLED=false +VMM_PORT_MAPPING_ENABLED=false # Host API configuration, type of uint32 -TEEPOD_VSOCK_LISTEN_PORT=$BASE_PORT +VMM_VSOCK_LISTEN_PORT=$BASE_PORT KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) @@ -108,7 +108,7 @@ EOF build_host() { echo "Building binaries" (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) - cp ${RUST_BUILD_DIR}/release/{dstack-gateway,kms,teepod,supervisor} . + cp ${RUST_BUILD_DIR}/release/{dstack-gateway,dstack-kms,dstack-vmm,supervisor} . } # Step 2: build guest images @@ -218,11 +218,11 @@ listen_port = $GATEWAY_SERVE_PORT agent_port = $AGENT_PORT EOF - # teepod - cat <teepod.toml + # dstack-vmm config + cat <vmm.toml log_level = "info" address = "127.0.0.1" -port = $TEEPOD_RPC_LISTEN_PORT +port = $VMM_RPC_LISTEN_PORT image_path = "$IMAGES_DIR" run_path = "$RUN_DIR/vm" kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" @@ -230,10 +230,10 @@ kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" [cvm] kms_urls = ["https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT"] gateway_urls = ["https://$GATEWAY_DOMAIN:$GATEWAY_RPC_LISTEN_PORT"] -cid_start = $TEEPOD_CID_POOL_START -cid_pool_size = $TEEPOD_CID_POOL_SIZE +cid_start = $VMM_CID_POOL_START +cid_pool_size = $VMM_CID_POOL_SIZE [cvm.port_mapping] -enabled = $TEEPOD_PORT_MAPPING_ENABLED +enabled = $VMM_PORT_MAPPING_ENABLED address = "127.0.0.1" range = [ { protocol = "tcp", from = 1, to = 20000 }, @@ -246,7 +246,7 @@ port = $GATEWAY_SERVE_PORT agent_port = $AGENT_PORT [host_api] -port = $TEEPOD_VSOCK_LISTEN_PORT +port = $VMM_VSOCK_LISTEN_PORT EOF mkdir -p $RUN_DIR diff --git a/dstack b/dstack index d9763e776..8d120e1fc 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d9763e776e655853a40580e05080307627194b3c +Subproject commit 8d120e1fcf5b2b0ab926e8de6aafb1c730251a69 From de827064fca9760455bd1fd28a1001f06000ffe0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 9 Apr 2025 08:06:55 +0000 Subject: [PATCH 128/435] Always use if name eth0 --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 03a0fe2e8..c7d676f8f 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -92,7 +92,7 @@ $Q makeiso ${WORK_DIR}/rootfs/ ${OUTPUT_DIR}/rootfs.iso GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" -KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 pnpacpi=off" +KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 net.ifnames=0 biosdevname=0" KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" KARG2="dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH" From ce9b514bb91cf865035e73fe1aba0c866758effa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 9 Apr 2025 12:01:07 +0000 Subject: [PATCH 129/435] Update dstack: Replace "tapp" with more descriptive terms --- build.sh | 2 +- dstack | 2 +- .../recipes-core/images/dstack-initramfs-files/init | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index a7e98a361..5443fca18 100755 --- a/build.sh +++ b/build.sh @@ -172,7 +172,7 @@ mandatory = false [core] kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" rpc_domain = "$GATEWAY_DOMAIN" -run_as_tapp = false +run_in_dstack = false [core.sync] enabled = false diff --git a/dstack b/dstack index 8d120e1fc..bfd3b25bf 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 8d120e1fcf5b2b0ab926e8de6aafb1c730251a69 +Subproject commit bfd3b25bf7871e5d184d390380ad95fdd6451854 diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init index 25907f54d..c2c7566f0 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ b/meta-dstack/recipes-core/images/dstack-initramfs-files/init @@ -6,11 +6,11 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bin mkdir -p /proc mkdir -p /sys mkdir -p /run -mkdir -p /tapp +mkdir -p /dstack mount -t proc proc /proc mount -t sysfs sysfs /sys mount -t tmpfs tmpfs /run -mount -t tmpfs tmpfs /tapp +mount -t tmpfs tmpfs /dstack mkdir -p /dev mount -t devtmpfs devtmpfs /dev @@ -35,7 +35,9 @@ mount_move_all() { mount --move /$dir /root/$dir done } -mount_move_all sys proc dev run tapp +mount_move_all sys proc dev run dstack + +ln -sf dstack /root/tapp || true echo "Switching to new root" exec switch_root /root /init From 9897dbb1f2178425a3bebef7a2ff0f3f590ba1ad Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Apr 2025 03:48:56 +0000 Subject: [PATCH 130/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index bfd3b25bf..9a80aefeb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit bfd3b25bf7871e5d184d390380ad95fdd6451854 +Subproject commit 9a80aefeb167a2514a71d240fb7cb08bf4e0acb2 From 798b4a1e741821ffcc34fdc5c24e31db563e2b93 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 18 Apr 2025 00:24:01 +0000 Subject: [PATCH 131/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 9a80aefeb..08cd5bb86 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 9a80aefeb167a2514a71d240fb7cb08bf4e0acb2 +Subproject commit 08cd5bb869950007f0500bb10d48ac5aa3f56e9a From 4804a3c266c4b55064d7495f5295804a64061175 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 28 Apr 2025 03:31:49 +0000 Subject: [PATCH 132/435] bump version to 0.5.0 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 35c83a03a..8fc596a02 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.4.2" +DISTRO_VERSION = "0.5.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 4826810f0366d8a355f4cda3273859888adad9e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 26 Apr 2025 05:50:28 +0000 Subject: [PATCH 133/435] Add submodule meta-security --- .gitmodules | 3 +++ meta-security | 1 + 2 files changed, 4 insertions(+) create mode 160000 meta-security diff --git a/.gitmodules b/.gitmodules index 89b5efdc4..224c63cb6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "dstack"] path = dstack url = https://github.com/Dstack-TEE/dstack +[submodule "meta-security"] + path = meta-security + url = https://git.yoctoproject.org/meta-security diff --git a/meta-security b/meta-security new file mode 160000 index 000000000..bc865c527 --- /dev/null +++ b/meta-security @@ -0,0 +1 @@ +Subproject commit bc865c5276c2ab4031229916e8d7c20148dfbac3 From fb6df5ed89a02193560f94e4516f6985df3c36c5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 28 Apr 2025 03:30:02 +0000 Subject: [PATCH 134/435] Enable dm-verity on rootfs --- Makefile | 5 +- dev-setup | 1 + dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 8 +-- .../images/dstack-initramfs-files/boot-vars | 24 -------- .../images/dstack-initramfs-files/init | 43 ------------- .../images/dstack-initramfs-files/kmfs-setup | 44 -------------- .../recipes-core/images/dstack-initramfs.bb | 18 +----- ...nitramfs-files.bb => dstack-initscript.bb} | 6 +- .../images/dstack-initscript/init | 52 ++++++++++++++++ .../images/dstack-rootfs-base.inc | 31 +++++++++- .../linux/linux-yocto%.bbappend | 2 + mkimage.sh | 60 +++++-------------- 14 files changed, 110 insertions(+), 188 deletions(-) delete mode 100644 meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars delete mode 100644 meta-dstack/recipes-core/images/dstack-initramfs-files/init delete mode 100644 meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup rename meta-dstack/recipes-core/images/{dstack-initramfs-files.bb => dstack-initscript.bb} (64%) create mode 100644 meta-dstack/recipes-core/images/dstack-initscript/init diff --git a/Makefile b/Makefile index 30d448c0f..d7d0625a2 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,10 @@ dist: images $(foreach dist_name,${DIST_NAMES},./mkimage.sh --dist-name $(dist_name);) images: - bitbake dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) + bitbake virtual/kernel dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) + +clean: + bitbake -c cleansstate virtual/kernel dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) clean-dstack: bitbake -c cleansstate dstack-guest $(ROOTFS_IMAGE_NAMES) diff --git a/dev-setup b/dev-setup index b45f54057..376ee34e6 100755 --- a/dev-setup +++ b/dev-setup @@ -23,6 +23,7 @@ LAYERS="$THIS_DIR/meta-confidential-compute \ $THIS_DIR/meta-openembedded/meta-filesystems \ $THIS_DIR/meta-virtualization \ $THIS_DIR/meta-rust-bin \ + $THIS_DIR/meta-security \ $THIS_DIR/meta-dstack" if [ -z "$1" ]; then diff --git a/dstack b/dstack index 08cd5bb86..21247621f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 08cd5bb869950007f0500bb10d48ac5aa3f56e9a +Subproject commit 21247621fe2ada54d338196565a320c93b2ca3b6 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 8fc596a02..25f629e85 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,7 +1,7 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "DStack" -DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge" +DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.5.0" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index ff37a4085..5239262ca 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -16,7 +16,7 @@ S = "${WORKDIR}/${SRC_DIR}" RDEPENDS:${PN} += "bash" -DSTACK_SERVICES = "dstack-guest-agent.service tboot.service app-compose.service" +DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -38,9 +38,9 @@ do_install() { install -d ${D}${sysconfdir}/docker install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} - install -m 0755 ${CARGO_BINDIR}/tdxctl ${D}${bindir} + install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} - install -m 0755 ${S}/basefiles/tboot.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${WORKDIR}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf @@ -53,7 +53,7 @@ do_install() { ${D}${sysconfdir}/systemd/resolved.conf.d install -m 0644 ${S}/basefiles/dstack-guest-agent.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/tboot.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d fi diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars b/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars deleted file mode 100644 index e55ed36a4..000000000 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/boot-vars +++ /dev/null @@ -1,24 +0,0 @@ -# Parse kernel cmdline into vars -for p in `cat /proc/cmdline`; do - if [ -n "$quoted" ]; then - value="$value $p" - if [ "`echo $p | sed -e 's/\"$//'`" != "$p" ]; then - eval "kargs_${quoted}=${value}" - unset quoted - fi - continue - fi - - opt=`echo $p | cut -d'=' -f1` - opt=`echo $opt | sed -e 'y/.-/__/'` - if [ "`echo $p | cut -d'=' -f1`" = "$p" ]; then - eval "kargs_${opt}=true" - else - value="`echo $p | cut -d'=' -f2-`" - if [ "`echo $value | sed -e 's/^\"//'`" != "$value" ]; then - quoted=${opt} - continue - fi - eval "kargs_${opt}=\"${value}\"" - fi -done diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/init b/meta-dstack/recipes-core/images/dstack-initramfs-files/init deleted file mode 100644 index c2c7566f0..000000000 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/init +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -set -e - -export PATH=/sbin:/bin:/usr/sbin:/usr/bin - -mkdir -p /proc -mkdir -p /sys -mkdir -p /run -mkdir -p /dstack -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t tmpfs tmpfs /run -mount -t tmpfs tmpfs /dstack - -mkdir -p /dev -mount -t devtmpfs devtmpfs /dev - -mkdir -p /dev/pts -mount -t devpts devpts /dev/pts - -ifconfig lo up 127.0.0.1 -ifconfig eth0 up -udhcpc -i eth0 -chronyd -q - -modprobe tdx-guest || true - -source /scripts/boot-vars -source /scripts/kmfs-setup - -# mount move dirs -mount_move_all() { - for dir in $@; do - mkdir -p /root/$dir - mount --move /$dir /root/$dir - done -} -mount_move_all sys proc dev run dstack - -ln -sf dstack /root/tapp || true - -echo "Switching to new root" -exec switch_root /root /init diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup b/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup deleted file mode 100644 index c874d1bbd..000000000 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files/kmfs-setup +++ /dev/null @@ -1,44 +0,0 @@ - -ROOTFS_ENCRYPTION="--rootfs-encryption=true" -if [ x"${kargs_dstack_fde}" = x"0" ]; then - echo "FDE is disabled" - ROOTFS_ENCRYPTION="--rootfs-encryption=false" -fi -if [ x"${kargs_dstack_integrity}" = x"1" ]; then - ROOTINTEGRITY="--rootfs-integrity" -fi - -ROOTFS_CD=/dev/sr0 -ROOTFS_CD_MNT=/mnt/rootfs-cdrom -ROOTFS_HD=/dev/vda -ROOTFS_DIR=/root - -KMFS_WORK_DIR=/kmfs -HOST_SHARED=/mnt/host-shared - -panic() { - echo "$1" - sleep 1 - exit 1 -} - -echo "Setting up FDE" -echo "FDE setup arguments:" -echo " HOST_SHARED=$HOST_SHARED" -echo " KMFS_WORK_DIR=$KMFS_WORK_DIR" -echo " ROOTFS_DIR=$ROOTFS_DIR" -echo " ROOTFS_HD=$ROOTFS_HD" -echo " ROOTFS_CD=$ROOTFS_CD" -echo " ROOTFS_CD_MNT=$ROOTFS_CD_MNT" -echo " ROOTINTEGRITY=$ROOTINTEGRITY" -echo " ROOTFS_ENCRYPTION=$ROOTFS_ENCRYPTION" - -tdxctl setup-fde \ - --host-shared $HOST_SHARED \ - --work-dir $KMFS_WORK_DIR \ - --rootfs-dir $ROOTFS_DIR \ - --root-hd $ROOTFS_HD \ - --root-cdrom $ROOTFS_CD \ - --root-cdrom-mnt $ROOTFS_CD_MNT \ - --rootfs-hash $kargs_dstack_rootfs_hash \ - $ROOTFS_ENCRYPTION $ROOTINTEGRITY || panic "Failed to setup FDE" diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/meta-dstack/recipes-core/images/dstack-initramfs.bb index 94e7f0e99..a6a8e1d7c 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs.bb +++ b/meta-dstack/recipes-core/images/dstack-initramfs.bb @@ -1,20 +1,4 @@ -PACKAGE_INSTALL = "\ - busybox \ - udev \ - base-passwd \ - ${ROOTFS_BOOTSTRAP_INSTALL} \ - base-files \ - cryptsetup \ - e2fsprogs-mke2fs \ - e2fsprogs-resize2fs \ - e2fsprogs-e2fsck \ - tdx-guest-ko \ - dstack-initramfs-files \ - dstack-guest \ - curl \ - jq \ - chrony \ -" +PACKAGE_INSTALL = "busybox cryptsetup dstack-initscript" # Do not pollute the initrd image with rootfs features IMAGE_FEATURES = "" diff --git a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb b/meta-dstack/recipes-core/images/dstack-initscript.bb similarity index 64% rename from meta-dstack/recipes-core/images/dstack-initramfs-files.bb rename to meta-dstack/recipes-core/images/dstack-initscript.bb index d335b8914..ee32b1aa6 100644 --- a/meta-dstack/recipes-core/images/dstack-initramfs-files.bb +++ b/meta-dstack/recipes-core/images/dstack-initscript.bb @@ -3,9 +3,7 @@ SUMMARY = "Dstack initramfs files" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" -SRC_URI = "file://init \ - file://kmfs-setup \ - file://boot-vars" +SRC_URI = "file://init" FILES:${PN} = "*" @@ -18,6 +16,4 @@ do_install() { install -d ${D}/scripts install -d ${D}/${bindir} install -m 0755 ${S}/init ${D}/init - install -m 0755 ${S}/kmfs-setup ${D}/scripts/kmfs-setup - install -m 0755 ${S}/boot-vars ${D}/scripts/boot-vars } diff --git a/meta-dstack/recipes-core/images/dstack-initscript/init b/meta-dstack/recipes-core/images/dstack-initscript/init new file mode 100644 index 000000000..8ba86235b --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-initscript/init @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +export PATH=/sbin:/bin:/usr/sbin:/usr/bin + +mkdir -p /proc +mkdir -p /sys +mkdir -p /run +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t tmpfs tmpfs /run + +mkdir -p /dev +mount -t devtmpfs devtmpfs /dev + +mkdir -p /dev/pts + +# Parse kernel parameters +for param in $(cat /proc/cmdline); do + case "$param" in + "dstack.rootfs_hash="*) + ROOT_HASH="${param#*=}" + ;; + "dstack.rootfs_size="*) + DATA_SIZE="${param#*=}" + ;; + *) + ;; + esac +done +ROOT_DEV=/dev/vda +ROOT_DIR=/root + +echo "Setting up verity device:" +echo " Root device: ${ROOT_DEV}" +echo " Root hash: ${ROOT_HASH}" +echo " Data size: ${DATA_SIZE}" + +veritysetup open ${ROOT_DEV} rootfs ${ROOT_DEV} "${ROOT_HASH}" --hash-offset="${DATA_SIZE}" + +echo "Mounting rootfs..." +mount -t ext4 -o ro /dev/mapper/rootfs ${ROOT_DIR} + +mount_move_all() { + for dir in $@; do + mount --move /$dir ${ROOT_DIR}/$dir + done +} +mount_move_all sys proc dev run + +echo "Switching to new root..." +exec switch_root ${ROOT_DIR} /sbin/init diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index e61eeed3c..f714c87ed 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -1,3 +1,5 @@ +inherit core-image + IMAGE_BASENAME = "${PN}" IMAGE_INSTALL = "\ @@ -14,9 +16,14 @@ IMAGE_INSTALL = "\ tdx-guest-ko \ dstack-guest \ wireguard-tools \ + e2fsprogs-mke2fs \ + e2fsprogs-resize2fs \ + e2fsprogs-e2fsck \ + cryptsetup \ curl \ jq \ chrony \ + chronyc \ qemu-guest-agent \ " @@ -29,9 +36,17 @@ INITRAMFS_MAXSIZE = "1000000" LICENSE = "MIT" -IMAGE_FSTYPES = "cpio" +IMAGE_CLASSES += "dm-verity-img" -inherit core-image +DM_VERITY_IMAGE = "${PN}" +DM_VERITY_IMAGE_TYPE = "ext4" +DM_VERITY_SEPARATE_HASH = "0" + +IMAGE_FSTYPES = "cpio ext4" +CONVERSIONTYPES += "verity" +IMAGE_TYPES += "ext4.verity" + +EXTRA_IMAGE_FEATURES = "read-only-rootfs" IMAGE_ROOTFS_SIZE = "8192" IMAGE_ROOTFS_EXTRA_SPACE = "0" @@ -44,6 +59,8 @@ ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" ROOTFS_POSTPROCESS_COMMAND += "symlink_lib64;" IMAGE_FEATURES[validitems] += "nologin" ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_getty_services", "",d)}' +ROOTFS_POSTPROCESS_COMMAND += "mkdirs;" + remove_sysvinit_files() { # Remove /etc/init.d directory and its contents @@ -75,3 +92,13 @@ disable_getty_services() { done } +mkdirs() { + mkdir -p ${IMAGE_ROOTFS}/dev + mkdir -p ${IMAGE_ROOTFS}/proc + mkdir -p ${IMAGE_ROOTFS}/sys + mkdir -p ${IMAGE_ROOTFS}/run + mkdir -p ${IMAGE_ROOTFS}/dstack + mkdir -p ${IMAGE_ROOTFS}/etc/wireguard + mkdir -p ${IMAGE_ROOTFS}/var/lib/docker + ln -sf dstack ${IMAGE_ROOTFS}/tapp +} diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 3681df6d9..fa352c6bc 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -13,6 +13,8 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ dstack-docker.scc \ dstack.scc" +KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", " features/device-mapper/dm-verity.scc", "" ,d)}" + KERNEL_FEATURES:append:tdx = " dstack-tdx.scc" do_deploy:append() { diff --git a/mkimage.sh b/mkimage.sh index c7d676f8f..dcea94260 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -32,18 +32,20 @@ fi BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) +IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/tdx ROOTFS_IMAGE_NAME=${DIST_NAME}-rootfs -WORK_DIR=${BB_BUILD_DIR}/${ROOTFS_IMAGE_NAME}.tmp -INITRAMFS_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/${ROOTFS_IMAGE_NAME}-tdx.cpio -KERNEL_IMAGE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/bzImage -OVMF_FIRMWARE=${BB_BUILD_DIR}/tmp/deploy/images/tdx/ovmf.fd -ROOTFS_HASH=$(sha256sum "$ROOTFS_IMAGE" | awk '{print $1}') + +INITRAMFS_IMAGE=${IMG_DIR}/dstack-initramfs.cpio.gz +ROOTFS_IMAGE=${IMG_DIR}/${ROOTFS_IMAGE_NAME}-tdx.ext4.verity +KERNEL_IMAGE=${IMG_DIR}/bzImage +OVMF_FIRMWARE=${IMG_DIR}/ovmf.fd +# Always use the work-shared directory which has the correct verity env +VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAME}.ext4.verity.env +echo "Loading verity env from ${VERITY_ENV_FILE}" +source ${VERITY_ENV_FILE} + DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} -DSTACK_TAR_EXCLUDE_ROOTFS_CPIO=${DSTACK_TAR_EXCLUDE_ROOTFS_CPIO:-1} - -mkdir -p ${WORK_DIR} verbose() { echo "$@" @@ -52,49 +54,19 @@ verbose() { Q=verbose -makeiso() { - export SOURCE_DATE_EPOCH="$(date -d20010101 -u +%s)" - folder="$1" - output_filename="$2" - file_mode=0444 - - list="$(mktemp)" - (cd "$folder"; for f in *; do printf "%s\n" "$f=$PWD/$f"; done) | LC_ALL=C sort >"$list" - - xorriso \ - -preparer_id xorriso \ - -volume_date 'all_file_dates' "=$SOURCE_DATE_EPOCH" \ - -as mkisofs \ - -iso-level 3 \ - -graft-points \ - -full-iso9660-filenames \ - -joliet \ - -file-mode $file_mode \ - -uid 0 \ - -gid 0 \ - -path-list "$list" \ - -output "$output_filename" - - rm -f "$list" -} - $Q rm -rf ${OUTPUT_DIR}/ $Q mkdir -p ${OUTPUT_DIR}/ $Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ - -$Q mkdir -p ${WORK_DIR}/rootfs/ -$Q cp $ROOTFS_IMAGE ${WORK_DIR}/rootfs/rootfs.cpio -$Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.cpio -$Q makeiso ${WORK_DIR}/rootfs/ ${OUTPUT_DIR}/rootfs.iso +$Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 net.ifnames=0 biosdevname=0" KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" -KARG2="dstack.fde=1 dstack.rootfs_hash=$ROOTFS_HASH" +KARG2="dstack.rootfs_hash=$ROOT_HASH dstack.rootfs_size=$DATA_SIZE" cat < ${OUTPUT_DIR}/metadata.json { @@ -102,8 +74,7 @@ cat < ${OUTPUT_DIR}/metadata.json "kernel": "bzImage", "cmdline": "$KARG0 $KARG1 $KARG2", "initrd": "initramfs.cpio.gz", - "rootfs": "rootfs.iso", - "rootfs_hash": "$ROOTFS_HASH", + "rootfs": "rootfs.img.verity", "version": "$DSTACK_VERSION", "git_revision": "$GIT_REVISION", "shared_ro": true, @@ -120,9 +91,6 @@ popd if [ x$DSTACK_TAR_RELEASE = x1 ]; then OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) echo "Archiving the output directory to ${OUTPUT_DIR}.tar.gz" - if [ x$DSTACK_TAR_EXCLUDE_ROOTFS_CPIO = x1 ]; then - TAR_ARGS=--exclude=rootfs.cpio - fi (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${OUTPUT_DIR}.tar.gz ${TAR_ARGS} $(basename $OUTPUT_DIR)) echo fi From 3da6243895dbb23bca016da8ab68611feb00f1a2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 9 May 2025 15:12:33 +0000 Subject: [PATCH 135/435] Add ZFS support --- ...Define-strndupa-if-it-does-not-exist.patch | 34 ++++++++ ...eadmmap-Replace-uint_t-with-uint32_t.patch | 46 +++++++++++ ...8a4630af60496c9d33db1d06a7d7d8983422.patch | 56 ++++++++++++++ .../dstack-zfs/dstack-zfs_2.2.5.bb | 77 +++++++++++++++++++ .../images/dstack-rootfs-base.inc | 1 + 5 files changed, 214 insertions(+) create mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch create mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch create mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch create mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch new file mode 100644 index 000000000..80955b3ca --- /dev/null +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch @@ -0,0 +1,34 @@ +From cc0cd6f71f6ef96fca2d7b730a3f0f6722fec696 Mon Sep 17 00:00:00 2001 +From: Khem Raj +Date: Sat, 7 May 2022 12:15:22 -0700 +Subject: [PATCH] Define strndupa if it does not exist + +musl e.g. does not supply strndupa, unlike glibc + +Upstream-Status: Pending +Signed-off-by: Khem Raj + +--- + etc/systemd/system-generators/zfs-mount-generator.c | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/etc/systemd/system-generators/zfs-mount-generator.c b/etc/systemd/system-generators/zfs-mount-generator.c +index f4c6c26..255bee4 100644 +--- a/etc/systemd/system-generators/zfs-mount-generator.c ++++ b/etc/systemd/system-generators/zfs-mount-generator.c +@@ -193,6 +193,15 @@ fopenat(int dirfd, const char *pathname, int flags, + return (fdopen(fd, stream_mode)); + } + ++#ifndef strndupa ++#define strndupa(s, n) \ ++ (__extension__ ({const char *__in = (s); \ ++ size_t __len = strnlen (__in, (n)) + 1; \ ++ char *__out = (char *) alloca (__len); \ ++ __out[__len-1] = '\0'; \ ++ (char *) memcpy (__out, __in, __len-1);})) ++#endif ++ + static int + line_worker(char *line, const char *cachefile) + { diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch new file mode 100644 index 000000000..f1cfab4da --- /dev/null +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch @@ -0,0 +1,46 @@ +From 1f9a5cb860b3509791e59a8cae9d5f265e832ed0 Mon Sep 17 00:00:00 2001 +From: Khem Raj +Date: Sun, 28 May 2023 16:33:15 -0700 +Subject: [PATCH] fs-tests/cmd/readmmap: Replace uint_t with uint32_t + +Makes it portable across glibc and musl + +Upstream-Status: Pending +Signed-off-by: Khem Raj +--- + tests/zfs-tests/cmd/readmmap.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/tests/zfs-tests/cmd/readmmap.c b/tests/zfs-tests/cmd/readmmap.c +index 704ffd55c8..a2590e0e8d 100644 +--- a/tests/zfs-tests/cmd/readmmap.c ++++ b/tests/zfs-tests/cmd/readmmap.c +@@ -38,6 +38,7 @@ + * 0 : no errors + * -------------------------------------------------------------- + */ ++#include + #include + #include + #include +@@ -55,7 +56,7 @@ main(int argc, char **argv) + char *buf = NULL; + char *map = NULL; + int fd = -1, bytes, retval = 0; +- uint_t seed; ++ uint32_t seed; + + if (argc < 2 || optind == argc) { + (void) fprintf(stderr, +@@ -92,7 +93,7 @@ main(int argc, char **argv) + retval = 1; + goto end; + } +- seed = (uint_t)time(NULL); ++ seed = (uint32_t)time(NULL); + srandom(seed); + + idx = random() % size; +-- +2.40.1 + diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch new file mode 100644 index 000000000..f5504b389 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch @@ -0,0 +1,56 @@ +From aaf28a4630af60496c9d33db1d06a7d7d8983422 Mon Sep 17 00:00:00 2001 +From: Sebastian Gottschall +Date: Tue, 23 May 2023 13:50:24 +0600 +Subject: [PATCH] fixes broken aarch64 inline assembly for gcc 13.1 + +fixes the following error + +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:549:24: error: the register specified for 'w7' is not general enough to be used as a register variable + 549 | register unsigned char w7 asm("v7") __attribute__((vector_size(16))); + | ^~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:193:9: note: in expansion of macro 'GEN_X_DEFINE_6_7' + 193 | GEN_X_DEFINE_6_7() \ + | ^~~~~~~~~~~~~~~~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' + 1360 | REC_PQR_DEFINE(); + | ^~~~~~~~~~~~~~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:551:24: error: the register specified for 'w8' is not general enough to be used as a register variable + 551 | register unsigned char w8 asm("v8") __attribute__((vector_size(16))); \ + | ^~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:194:9: note: in expansion of macro 'GEN_X_DEFINE_8_9' + 194 | GEN_X_DEFINE_8_9() \ + | ^~~~~~~~~~~~~~~~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' + 1360 | REC_PQR_DEFINE(); + | ^~~~~~~~~~~~~~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:552:24: error: the register specified for 'w9' is not general enough to be used as a register variable + 552 | register unsigned char w9 asm("v9") __attribute__((vector_size(16))); + | ^~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:194:9: note: in expansion of macro 'GEN_X_DEFINE_8_9' + 194 | GEN_X_DEFINE_8_9() \ + | ^~~~~~~~~~~~~~~~ +/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' + 1360 | REC_PQR_DEFINE(); + +Upstream-Status: Pending [https://github.com/BrainSlayer/zfs/commit/aaf28a4630af60496c9d33db1d06a7d7d8983422] +Signed-off-by: Sebastian Gottschall +Signed-off-by: Khem Raj +--- + module/Kbuild.in | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +--- a/module/Kbuild.in ++++ b/module/Kbuild.in +@@ -57,9 +57,9 @@ asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_ + ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) + + ifeq ($(CONFIG_ARM64),y) +-CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o += -mgeneral-regs-only +-CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o += -mgeneral-regs-only +-CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o += -mgeneral-regs-only ++CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o = -mgeneral-regs-only ++CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o = -mgeneral-regs-only ++CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o = -mgeneral-regs-only + endif + + # Suppress unused-value warnings in sparc64 architecture headers diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb new file mode 100644 index 000000000..1d4e512c7 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb @@ -0,0 +1,77 @@ +SUMMARY = "OpenZFS on Linux and FreeBSD" +DESCRIPTION = "OpenZFS on Linux and FreeBSD" +LICENSE = "CDDL-1.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=7087caaf1dc8a2856585619f4a787faa" +HOMEPAGE ="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/openzfs/zfs" + +SRCREV = "33174af15112ed5c53299da2d28e763b0163f428" +SRC_URI = "git://github.com/openzfs/zfs;protocol=https;branch=zfs-2.2-release \ + file://0001-Define-strndupa-if-it-does-not-exist.patch \ + file://aaf28a4630af60496c9d33db1d06a7d7d8983422.patch \ + file://0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch \ +" + +S = "${WORKDIR}/git" + +# Using both 'module' and 'autotools' classes seems a bit odd, they both +# define a do_compile function. +# That's why we opt for module-base, also this prevents module splitting. +inherit module-base pkgconfig autotools bash-completion + +DEPENDS = "virtual/kernel zlib util-linux libtirpc openssl curl" + +PACKAGECONFIG ?= "${@bb.utils.filter('DISTRO_FEATURES', 'systemd sysvinit', d)}" + +PACKAGECONFIG[pam] = "--enable-pam --with-pamconfigsdir=${datadir}/pam-configs --with-pammoduledir=${libdir}/security, --disable-pam" +PACKAGECONFIG[systemd] = "--enable-systemd,--disable-systemd," +PACKAGECONFIG[sysvinit] = "--enable-sysvinit,--disable-sysvinit," + +EXTRA_OECONF:append = " \ + --disable-pyzfs \ + --with-linux=${STAGING_KERNEL_DIR} --with-linux-obj=${STAGING_KERNEL_BUILDDIR} \ + --with-mounthelperdir=${base_sbin} \ + --with-udevdir=${base_libdir}/udev \ + --with-systemdunitdir=${systemd_system_unitdir} \ + --with-systemdgeneratordir=${nonarch_base_libdir}/systemd/system-generators \ + --with-systemdpresetdir=${nonarch_base_libdir}/systemd/system-preset \ + --with-systemdmodulesloaddir=${sysconfdir}/module-load.d \ + --without-dracutdir \ +" + +EXTRA_OEMAKE:append = " \ + INSTALL_MOD_PATH=${D}${root_prefix} \ +" + +do_install:append() { + # /usr/share/zfs contains the zfs-tests folder which we do not need: + rm -rf ${D}${datadir}/zfs + + rm -rf ${D}${datadir}/initramfs-tools +} + +FILES:${PN} += "\ + ${nonarch_base_libdir}/modules \ + ${systemd_system_unitdir} \ + ${nonarch_base_libdir}/systemd/system-generators \ + ${nonarch_base_libdir}/systemd/system-preset \ + ${sysconfdir}/modules-load.d/${BPN}.conf \ + ${sysconfdir}/default/${BPN} \ + ${sysconfdir}/sudoers.d/${BPN} \ + ${sysconfdir}/${BPN} \ + ${base_libdir}/udev \ + ${sbindir} \ + ${bindir} \ + ${libexecdir}/${BPN} \ + ${libdir} \ +" + +FILES:${PN}-dev += "\ + ${prefix}/src/zfs-${PV} \ + ${prefix}/src/spl-${PV} \ +" +# Not yet ported to rv32 +COMPATIBLE_HOST:riscv32 = "null" +# conflicting definition of ABS macro from asm/asm.h from kernel +COMPATIBLE_HOST:mips = "null" +# FTBFS on aarch64 with 6.2+ kernel see https://github.com/openzfs/zfs/issues/14555 +COMPATIBLE_HOST:aarch64 = "null" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index f714c87ed..074b29c05 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -25,6 +25,7 @@ IMAGE_INSTALL = "\ chrony \ chronyc \ qemu-guest-agent \ + dstack-zfs \ " # Do not pollute the initrd image with rootfs features From c925458e5341356eda378671551961149827f55a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 9 May 2025 15:12:48 +0000 Subject: [PATCH 136/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 21247621f..43a8c0f67 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 21247621fe2ada54d338196565a320c93b2ca3b6 +Subproject commit 43a8c0f67e6b869c348857f8837858816dcebf3f From 2b6c2caafd625d812f6df12254f9a538e869eb95 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 13 May 2025 08:36:31 +0000 Subject: [PATCH 137/435] Fix IMAGE_FEATURES --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 3 --- meta-dstack/recipes-core/images/dstack-rootfs-dev.inc | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 074b29c05..8a870f748 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -28,9 +28,6 @@ IMAGE_INSTALL = "\ dstack-zfs \ " -# Do not pollute the initrd image with rootfs features -IMAGE_FEATURES = "" - IMAGE_NAME_SUFFIX ?= "" IMAGE_LINGUAS = "" INITRAMFS_MAXSIZE = "1000000" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index 698941925..6fe567f57 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver" -IMAGE_FEATURES = "debug-tweaks" +EXTRA_IMAGE_FEATURES += "debug-tweaks" From 6daddd7c763ec2abc2b524e531d9df2d0962f840 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 May 2025 00:35:34 +0000 Subject: [PATCH 138/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 43a8c0f67..2c0514117 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 43a8c0f67e6b869c348857f8837858816dcebf3f +Subproject commit 2c0514117a2f45d98fa784dab998f341f57fc006 From 08db30d0237a8392a6790d5c20ac2708278e4f44 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 13 May 2025 08:50:22 +0000 Subject: [PATCH 139/435] Update dstack.py --- scripts/bin/dstack.py | 684 +++++++++++++++++++++++++++++++----------- 1 file changed, 501 insertions(+), 183 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index d2f428339..bd2a008c3 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -5,6 +5,7 @@ import logging import os import random +import re import string import subprocess import uuid @@ -36,52 +37,6 @@ def generate_config_paths(): return paths -@dataclass -class PortMap: - """Configuration for port mapping.""" - address: str - protocol: str - from_port: int - to_port: int - - def to_dict(self) -> Dict: - return { - "address": self.address, - "protocol": self.protocol, - "from": self.from_port, - "to": self.to_port - } - - -@dataclass -class VMConfig: - """Configuration for VM instance.""" - id: str - name: str - vcpu: int - gpu: List[str] - memory: int - disk_size: int - image: str - image_path: str - port_map: List[PortMap] - created_at_ms: int - - def to_dict(self) -> Dict: - return { - "id": self.id, - "name": self.name, - "vcpu": self.vcpu, - "gpu": self.gpu, - "memory": self.memory, - "disk_size": self.disk_size, - "image": self.image, - "image_path": self.image_path, - "port_map": [p.to_dict() for p in self.port_map], - "created_at_ms": self.created_at_ms - } - - def merge2(a, b): if isinstance(a, dict) and isinstance(b, dict): c = a.copy() @@ -106,13 +61,38 @@ def merge_dicts(*dicts): def test_merge_dicts(): assert merge_dicts({"a": 1}, {"b": 2}) == {"a": 1, "b": 2} assert merge_dicts({"a": 1}, {"a": 2}) == {"a": 2} - assert merge_dicts({"a": {"b": 1}}, {"a": {"c": 2}}) == {"a": {"b": 1, "c": 2}} + assert merge_dicts({"a": {"b": 1}}, {"a": {"c": 2}}) == { + "a": {"b": 1, "c": 2}} assert merge_dicts({"a": {"b": 1}}, {"a": {"b": 2}}) == {"a": {"b": 2}} - assert merge_dicts({"a": {"b": 1}}, {"a": {"b": 2}, "c": 3}) == {"a": {"b": 2}, "c": 3} - assert merge_dicts({"a": 1}, {"b": 2}, {"c": 3}) == {"a": 1, "b": 2, "c": 3} + assert merge_dicts({"a": {"b": 1}}, {"a": {"b": 2}, "c": 3}) == { + "a": {"b": 2}, "c": 3} + assert merge_dicts({"a": 1}, {"b": 2}, {"c": 3}) == { + "a": 1, "b": 2, "c": 3} assert merge_dicts({"a": 1}, {"a": 2}, {"c": 3}) == {"a": 2, "c": 3} +def round_up(value, multiple): + """ + Round up a value to the nearest multiple of another value. + If the value is already a multiple, it remains unchanged. + + Args: + value (int): The value to round up + multiple (int): The multiple to round up to + + Returns: + int: The rounded up value + """ + if multiple <= 1: + return value + + remainder = value % multiple + if remainder == 0: + return value + + return value + (multiple - remainder) + + def ini_to_dict(filename): config = configparser.ConfigParser() config.read(filename) @@ -145,6 +125,16 @@ def update_guest_config(config_file: str, data: Dict): json.dump(config, f, indent=4) +def gen_vm_config(vm_dir, host_port): + shared_dir = os.path.join(vm_dir, 'shared') + for filename in ['config.json', '.sys-config.json']: + config_file = os.path.join(shared_dir, filename) + update_guest_config(config_file, { + "host_api_url": f"http://10.0.2.2:{host_port}/api", + "host_vsock_port": host_port + }) + + @dataclass class DStackConfig: """Configuration for DStack client.""" @@ -156,13 +146,15 @@ class DStackConfig: def load(cls) -> 'DStackConfig': """Load configuration from file.""" cfgs = load_configs_merged(generate_config_paths()) + def cfg_get(section, key, fallback): if section in cfgs and key in cfgs[section]: return cfgs[section][key] return fallback me = cls() me.docker_registry = cfg_get('docker', 'registry', cls.docker_registry) - me.default_image_name = cfg_get('image', 'default', cls.default_image_name) + me.default_image_name = cfg_get( + 'image', 'default', cls.default_image_name) me.qemu_path = cfg_get('qemu', 'path', cls.qemu_path) return me @@ -183,22 +175,6 @@ def _read_compose_file(self, compose_file: str) -> str: with open(compose_file, 'r') as f: return f.read() - def _read_image_metadata(self, image_path: str) -> str: - """Read and validate image metadata.""" - metadata_path = os.path.join(image_path, 'metadata.json') - if not os.path.isfile(metadata_path): - raise FileNotFoundError(f"Image metadata not found at {metadata_path}") - - try: - with open(metadata_path, 'r') as f: - metadata = json.load(f) - rootfs_hash = metadata.get('rootfs_hash') - if not rootfs_hash: - raise ValueError("Rootfs hash not found in image info") - return rootfs_hash - except json.JSONDecodeError: - raise ValueError(f"Invalid JSON in metadata file: {metadata_path}") - def _create_directories(self, work_dir: str) -> tuple[str, str]: """Create necessary directories.""" if os.path.exists(work_dir): @@ -220,7 +196,7 @@ def _convert_memory_to_mb(self, memory: str) -> int: return int(memory[:-1]) return int(memory) - def _parse_port_mapping(self, port_str: str) -> PortMap: + def _parse_port_mapping(self, port_str: str) -> dict: """Parse port mapping string in format 'protocol[:address]:from:to'.""" try: parts = port_str.split(':') @@ -230,14 +206,15 @@ def _parse_port_mapping(self, port_str: str) -> PortMap: elif len(parts) == 4: proto, address, from_port, to_port = parts else: - raise ValueError("Invalid port mapping format. Use 'protocol[:address]:from:to'") - - return PortMap( - address=address, - protocol=proto.lower(), - from_port=int(from_port), - to_port=int(to_port) - ) + raise ValueError( + "Invalid port mapping format. Use 'protocol[:address]:from:to'") + + return { + "address": address, + "protocol": proto.lower(), + "from": int(from_port), + "to": int(to_port) + } except ValueError as e: raise ValueError(f"Invalid port mapping '{port_str}': {str(e)}") @@ -245,7 +222,8 @@ def setup_instance(self, args: argparse.Namespace) -> None: """Set up a new instance with the provided configuration.""" try: # Generate instance ID if work_dir not provided - instance_id = os.path.basename(args.dir) if args.dir else self._generate_instance_id() + instance_id = os.path.basename( + args.dir) if args.dir else self._generate_instance_id() work_dir = args.dir or os.path.join(self.run_path, instance_id) # Create directories @@ -262,18 +240,16 @@ def setup_instance(self, args: argparse.Namespace) -> None: "features": [], "runner": "docker-compose", "docker_compose_file": compose_content, - "local_key_provider_enabled": args.local_key_provider + "local_key_provider_enabled": args.local_key_provider, + "secure_time": False, } with open(os.path.join(shared_dir, 'app-compose.json'), 'w') as f: json.dump(app_compose, f, indent=4) # Read image metadata and create config.json - rootfs_hash = self._read_image_metadata(args.image) - for filename in ['config.json', '.sys-config.json']: - update_guest_config(os.path.join(shared_dir, filename), { - "rootfs_hash": rootfs_hash, + if self.config.docker_registry: + update_guest_config(os.path.join(shared_dir, '.sys-config.json'), { "docker_registry": self.config.docker_registry, - "pccs_url": "https://api.trustedservices.intel.com/sgx/certification/v4", }) # Create VM manifest @@ -284,36 +260,112 @@ def setup_instance(self, args: argparse.Namespace) -> None: for port_str in args.port: port_map.append(self._parse_port_mapping(port_str)) - vm_config = VMConfig( - id=instance_id, - name="example", - vcpu=args.vcpus, - gpu=args.gpu or [], - memory=memory, - disk_size=disk_size, - image_path=args.image, - image=os.path.basename(args.image.rstrip('/')), - port_map=port_map, - created_at_ms=int(datetime.now().timestamp() * 1000) - ) - + if args.gpu == ['all']: + gpus = { + "attach_mode": "all", + } + else: + gpus = { + "attach_mode": "listed", + "gpus": [ + {"slot": gpu} for gpu in args.gpu + ] + } + gpus = self.resolve_gpus(gpus) + vm_config = { + "id": instance_id, + "name": "", + "vcpu": args.vcpus, + "gpus": gpus, + "memory": memory, + "disk_size": disk_size, + "image_path": args.image, + "image": os.path.basename(args.image.rstrip('/')), + "port_map": port_map, + "pin_numa": args.pin_numa, + "hugepages": args.hugepages, + "created_at_ms": int(datetime.now().timestamp() * 1000) + } with open(os.path.join(work_dir, 'vm-manifest.json'), 'w') as f: - json.dump(vm_config.to_dict(), f, indent=4) - + json.dump(vm_config, f, indent=4) logger.info(f"Work directory prepared successfully at: {work_dir}") except Exception as e: logger.error(f"Failed to setup instance: {str(e)}") raise - def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None, vcpus: Optional[int] = None, imgdir: Optional[str] = None, pin_numa: bool = False, hugepage: bool = False) -> None: + @staticmethod + def collect_all_gpus() -> dict: + """Collect available NVIDIA GPUs and NVSwitches.""" + try: + # Find all NVIDIA GPUs (3D controllers) + gpu_cmd = subprocess.run( + ['lspci', '-d', '10de:', '-nn'], + capture_output=True, text=True, check=True + ) + gpu_output = gpu_cmd.stdout.strip() + + gpus = [] + bridges = [] + + # Process each line of output + for line in gpu_output.split('\n'): + if not line.strip(): + continue + + slot = line.split()[0] # Bus:Device.Function + + # Extract device ID from the line + match = re.search(r'\[10de:([0-9A-Fa-f]+)\]', line) + if not match: + continue + # Check if it's a GPU (3D controller) or NVSwitch (Bridge) + if '3D controller' in line: + gpus.append({ + "slot": slot, + }) + elif 'Bridge' in line: + bridges.append({ + "slot": slot, + }) + + logger.info( + f"Found {len(gpus)} GPU(s) and {len(bridges)} NVSwitch(es)") + + return { + "attach_mode": "all", + "gpus": gpus, + "bridges": bridges + } + except subprocess.SubprocessError as e: + logger.warning(f"Failed to collect GPU information: {str(e)}") + return { + "attach_mode": "all", + "gpus": [], + "bridges": [] + } + + @staticmethod + def resolve_gpus(gpus: dict) -> dict: + """Resolve GPU slots.""" + match gpus['attach_mode']: + case 'listed': + return gpus + case 'all': + return DStackManager.collect_all_gpus() + case _: + raise ValueError( + f"Invalid GPU attach mode: {gpus['attach_mode']}") + + def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None, dry_run: bool = False) -> None: """Run a VM instance from the specified directory. Args: vm_dir: Directory containing the VM configuration - memory: Optional memory size override (e.g., '2G', '512M') - vcpus: Optional number of virtual CPUs override + dry_run: Whether to run in dry run mode """ + gen_vm_config(vm_dir, host_port) + manifest_path = os.path.join(vm_dir, 'vm-manifest.json') if not os.path.exists(manifest_path): raise ValueError(f"VM manifest not found in {vm_dir}") @@ -321,45 +373,130 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None with open(manifest_path, 'r') as f: manifest = json.load(f) + if dry_run: + print("Manifest:") + print(json.dumps(manifest, indent=4)) + # Get image path and metadata - image_path = manifest.get('image_path') or os.path.join(imgdir, manifest['image']) + image_path = manifest.get('image_path') or os.path.join( + imgdir, manifest['image']) img_metadata_path = os.path.join(image_path, 'metadata.json') - # Update config.json with host API URL and port - shared_dir = os.path.join(vm_dir, 'shared') - for filename in ['config.json', '.sys-config.json']: - config_file = os.path.join(shared_dir, filename) - update_guest_config(config_file, { - "host_api_url": f"http://10.0.2.2:{host_port}/api", - "host_vsock_port": host_port - }) - if not os.path.exists(img_metadata_path): - raise ValueError(f"Image metadata not found at {img_metadata_path}") + raise ValueError( + f"Image metadata not found at {img_metadata_path}") with open(img_metadata_path, 'r') as f: img_metadata = json.load(f) - # Prepare QEMU arguments - mem = memory if memory else f"{manifest['memory']}M" - vcpu_count = vcpus if vcpus is not None else manifest['vcpu'] + mem_gb = manifest['memory'] // 1024 + vcpu_count = manifest['vcpu'] disk_size = manifest['disk_size'] - gpus = manifest.get('gpu', []) vda = os.path.join(vm_dir, 'hda.img') config_dir = os.path.join(vm_dir, 'shared') # Create disk if it doesn't exist if not os.path.exists(vda): - subprocess.run(['qemu-img', 'create', '-f', 'qcow2', vda, f"{disk_size}G"], check=True) + subprocess.run(['qemu-img', 'create', '-f', 'qcow2', + vda, f"{disk_size}G"], check=True) cid = random.randint(1, 10000) + 3 # Prepare QEMU command - cmd = [ + cmd_args = [] + rootfs_image = os.path.join(image_path, img_metadata['rootfs']) + if rootfs_image.endswith('.img.verity'): + cmd_args.extend([ + '-drive', f'file={rootfs_image},if=none,id=virtio-disk0,format=raw', + '-device', 'virtio-blk-pci,drive=virtio-disk0', + ]) + elif rootfs_image.endswith('.img'): + cmd_args.extend(['-cdrom', rootfs_image]) + else: + raise ValueError( + f"Unsupported rootfs image format: {rootfs_image}") + cmd_args.extend(['-drive', f'file={vda},if=none,id=virtio-disk1']) + cmd_args.extend(['-device', 'virtio-blk-pci,drive=virtio-disk1']) + + # Add network configuration + port_args = [] + for port_map in manifest.get('port_map', []): + protocol = port_map.get('protocol', 'tcp') + bind_address = port_map.get('address', '127.0.0.1') + host_port = port_map['from'] + vm_port = port_map['to'] + port_args.append( + f"hostfwd={protocol}:{bind_address}:{host_port}-:{vm_port}") + cmd_args.extend([ + '-device', 'virtio-net-pci,netdev=nic0_td', + '-netdev', f"user,id=nic0_td{','+','.join(port_args) if len(port_args) > 1 else ''}" + ]) + + # Handle GPUs + gpus_cfg = manifest.get('gpus', {}) + gpus = gpus_cfg.get('gpus', []) + bridges = gpus_cfg.get('bridges', []) + dev_num = 1 + hugepages = manifest.get('hugepages', False) + if hugepages: + numa_nodes = {} + if gpus: + for dev in gpus: + node = numa_node_of_device(dev['slot']) + if node not in numa_nodes: + numa_nodes[node] = 0 + numa_nodes[node] += 1 + else: + numa_nodes[0] = 0 + n_numa = len(numa_nodes) + # Round up cpu cores and memory to multiple times of numa nodes + vcpu_count = round_up(vcpu_count, n_numa) + mem_gb = round_up(mem_gb, n_numa) + vcpu_per_node = vcpu_count // n_numa + mem_per_node = mem_gb // n_numa + + bus_nr = 5 + for ind, (node, count) in enumerate(numa_nodes.items()): + cmd_args.extend([ + '-numa', f'node,nodeid={ind},cpus={ind * vcpu_per_node}-{(ind + 1) * vcpu_per_node - 1},memdev=mem{ind}', + '-object', f'memory-backend-file,id=mem{ind},size={mem_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind', + '-device', f'pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={0xa + ind},numa_node={ind},bus_nr={bus_nr}' + ]) + bus_nr += count + 1 + if gpus: + cmd_args.extend(['-object', 'iommufd,id=iommufd0']) + if not hugepages: + for dev in gpus: + slot = dev['slot'] + cmd_args.extend([ + '-device', f'pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}', + '-device', f'vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0', + ]) + dev_num += 1 + else: + for dev in gpus: + slot = dev['slot'] + node = numa_node_of_device(slot) + cmd_args.extend([ + '-device', f'pcie-root-port,id=pci.{dev_num},bus=pcie.node{node},chassis={dev_num}', + '-device', f'vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0', + ]) + dev_num += 1 + for bridge in bridges: + slot = bridge['slot'] + cmd_args.extend([ + '-device', f'pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}', + '-device', f'vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0', + ]) + dev_num += 1 + # Add kernel command line + cmd_args.extend(['-append', img_metadata['cmdline']]) + + base_args = [ self.config.qemu_path, '-accel', 'kvm', - '-m', mem, + '-m', f'{mem_gb}G', '-smp', str(vcpu_count), '-cpu', 'host', '-machine', 'q35,kernel_irqchip=split,confidential-guest-support=tdx,hpet=off', @@ -371,52 +508,22 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None '-kernel', os.path.join(image_path, img_metadata['kernel']), '-initrd', os.path.join(image_path, img_metadata['initrd']), '-bios', os.path.join(image_path, img_metadata['bios']), - '-cdrom', os.path.join(image_path, img_metadata['rootfs']), - '-drive', f'file={vda},if=none,id=virtio-disk0', - '-device', 'virtio-blk-pci,drive=virtio-disk0', '-virtfs', f'local,path={config_dir},mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0', - '-device', f'vhost-vsock-pci,guest-cid={cid}', + '-device', f'vhost-vsock-pci,guest-cid={cid}', ] - # Add network configuration - port_args = [] - for port_map in manifest.get('port_map', []): - protocol = port_map.get('protocol', 'tcp') - bind_address = port_map.get('address', '127.0.0.1') - host_port = port_map['from'] - vm_port = port_map['to'] - port_args.append(f"hostfwd={protocol}:{bind_address}:{host_port}-:{vm_port}") - cmd.extend([ - '-device', 'virtio-net-pci,netdev=nic0_td', - '-netdev', f"user,id=nic0_td{','+','.join(port_args) if len(port_args) > 1 else ''}" - ]) - - if gpus: - cmd.extend([ - '-device', f'pcie-root-port,id=pci.1,bus=pcie.0', - '-fw_cfg', 'name=opt/ovmf/X-PciMmio64,string=262144', - ]) - # Use sudo when GPU is involved - cmd = ['sudo'] + cmd - for i, gpu_id in enumerate(gpus): - cmd.extend([ - '-object', f'iommufd,id=iommufd{i}', - '-device', f'vfio-pci,host={gpu_id},bus=pci.1,iommufd=iommufd{i}', - ]) - # Add kernel command line - cmd.extend(['-append', img_metadata['cmdline']]) - - if pin_numa and len(gpus) == 1: - numa_node = subprocess.run(['cat', f'/sys/bus/pci/devices/0000:{gpus[0]}/numa_node'], capture_output=True, text=True).stdout.strip() - cpus = subprocess.run(['cat', f'/sys/devices/system/node/node{numa_node}/cpulist'], capture_output=True, text=True).stdout.strip() - cmd = ['taskset', '-c', cpus] + cmd - if hugepage: - cmd.extend([ - '-numa', f'node,nodeid=0,cpus=0-{vcpu_count-1},memdev=mem0', - '-object', f'memory-backend-file,id=mem0,size={mem},mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={numa_node},policy=bind', - ]) - - print(" ".join(cmd)) + pin_numa = manifest.get('pin_numa', False) + if pin_numa: + if gpus: + numa_node = numa_node_of_device(gpus[0]['slot']) + else: + numa_node = 0 + cpus = open(f'/sys/devices/system/node/node{numa_node}/cpulist').read().strip() + base_args = ['taskset', '-c', cpus] + base_args + cmd = base_args + cmd_args + print(" \n".join(cmd)) + if dry_run: + return # Run the command try: subprocess.run(cmd, check=True) @@ -424,6 +531,27 @@ def run_instance(self, vm_dir: str, host_port: int, memory: Optional[str] = None raise RuntimeError(f"Failed to start VM: {e}") +def numa_node_of_device(pci_slot): + """ + Get the NUMA node associated with a PCI device. + + Args: + pci_slot (str): PCI slot in format "domain:bus:device.function" (e.g., "0000:ab:00.0") + + Returns: + int: NUMA node number, or -1 if device doesn't exist or has no NUMA affinity + """ + # Normalize the PCI slot format if needed + if not pci_slot.startswith("0000:"): + pci_slot = f"0000:{pci_slot}" + # Try to read NUMA node from sysfs + numa_path = f"/sys/bus/pci/devices/{pci_slot}/numa_node" + + with open(numa_path, 'r') as f: + numa_node = int(f.read().strip()) + return numa_node + + def list_available_gpus() -> None: """List available NVIDIA GPUs.""" try: @@ -460,7 +588,7 @@ def list_available_gpus() -> None: if gpu_blocks: print("\nAvailable GPU IDs:") - print("ID In Use Description") + print("ID Numa Node In Use Description") print("-------------------------------------") for block in gpu_blocks: @@ -477,41 +605,230 @@ def list_available_gpus() -> None: in_use = True status = "Yes" if in_use else "No" - print(f"{device_id} {status:8} {description}") + node = numa_node_of_device(device_id) + print(f"{device_id} {node} {status:8} {description}") print() except subprocess.SubprocessError as e: logger.warning(f"Could not list GPU devices: {str(e)}") +def start_server(dir: str, kp_port: int): + config = host_api.ServerConfig( + vm_dir=dir, kp_address="127.0.0.1", kp_port=kp_port) + api, host_port = host_api.create_http_server(config) + print(f"Starting HTTP server on localhost:{host_port}") + thread = threading.Thread(target=api.serve_forever, daemon=True) + thread.host_port = host_port + thread.start() + return thread + + +def tag_vfio(): + """ + Tag NVIDIA GPUs and NVSwitches for VFIO passthrough. + Detects NVIDIA devices and configures them for VFIO passthrough. + """ + logging.info("==> Detecting NVIDIA GPUs and NVSwitches") + + try: + # Use a more structured approach to detect devices + devices = detect_nvidia_devices() + + if not devices['gpus'] and not devices['switches']: + logging.error("No NVIDIA GPUs or NVSwitches found. Exiting.") + return + + ngpu = len(devices['gpus']) + nsw = len(devices['switches']) + + logging.info(f"Found {ngpu} GPU(s)") + logging.info(f"Found {nsw} NVSwitch(es)") + + # Load VFIO modules + logging.info("==> Loading VFIO modules") + load_vfio_modules() + + # Tag devices for VFIO passthrough + logging.info("==> Tagging devices for VFIO passthrough") + + # Deduplicate device IDs + unique_dev_ids = {} + for device_type, device_list in devices.items(): + device_name = "GPU" if device_type == "gpus" else "NVSwitch" + for device in device_list: + dev_id = device['dev_id'] + if dev_id not in unique_dev_ids: + unique_dev_ids[dev_id] = device_name + # If the device ID is already in the dict but with a different type, + # we'll keep the original type for simplicity + + # Process unique device IDs + for dev_id, device_name in unique_dev_ids.items(): + tag_device_for_vfio(dev_id, device_name) + + logging.info("VFIO passthrough setup complete") + + except Exception as e: + logging.error(f"Failed to enable VFIO passthrough: {e}") + + +def detect_nvidia_devices(): + """ + Detect NVIDIA GPUs and NVSwitches in the system. + + Returns: + dict: Dictionary with 'gpus' and 'switches' keys, each containing a list of + dictionaries with 'dev_id' for each device. + """ + devices = { + 'gpus': [], + 'switches': [] + } + + try: + # Run lspci once and parse the output + lspci_output = subprocess.check_output( + "lspci -d 10de: -nn", + shell=True, text=True + ).strip().split('\n') + + for line in lspci_output: + if not line: + continue + + # Extract Device ID + dev_id_match = re.search(r'\[10de:([0-9A-Fa-f]+)\]', line) + if not dev_id_match: + continue + + dev_id = dev_id_match.group(1) + + # Categorize device + if '3D controller' in line: + devices['gpus'].append({'dev_id': dev_id}) + elif 'Bridge' in line: + devices['switches'].append({'dev_id': dev_id}) + + except subprocess.CalledProcessError as e: + logging.error(f"Failed to detect NVIDIA devices: {e}") + + return devices + + +def load_vfio_modules(): + """ + Load the VFIO kernel modules required for device passthrough. + Assumes the script is run with appropriate permissions. + + Raises: + RuntimeError: If modules cannot be loaded + """ + try: + subprocess.run(["modprobe", "vfio"], check=True) + subprocess.run(["modprobe", "vfio_pci"], check=True) + except subprocess.CalledProcessError as e: + logging.error(f"Failed to load VFIO modules: {e}") + raise RuntimeError("Failed to load VFIO modules") from e + + +def tag_device_for_vfio(dev_id, device_type): + """ + Tag a PCI device for VFIO passthrough. + + Args: + dev_id (str): Device ID + device_type (str): Type of device (GPU or NVSwitch) + """ + logging.info(f"Tagging {device_type} (DevID=10de:{dev_id})") + + new_id_path = "/sys/bus/pci/drivers/vfio-pci/new_id" + remove_id_path = "/sys/bus/pci/drivers/vfio-pci/remove_id" + device_id_value = f"10de {dev_id}" + + try: + # Directly write to the sysfs file + write_to_sysfs(new_id_path, device_id_value) + except Exception: + try: + # Remove the ID first in case it's already there + try: + write_to_sysfs(remove_id_path, device_id_value) + except Exception: + # Ignore errors when removing + pass + + # Try adding again + write_to_sysfs(new_id_path, device_id_value) + except Exception as e: + logging.error(f"Failed to tag {device_type}: {e}") + + +def write_to_sysfs(path, value): + """ + Write a value directly to a sysfs file. + Assumes the script is run with appropriate permissions. + + Args: + path (str): Path to the sysfs file + value (str): Value to write + + Raises: + IOError: If the write fails + """ + with open(path, 'w') as f: + f.write(value) + + def main(): parser = argparse.ArgumentParser(description='DStack VM Management Tool') subparsers = parser.add_subparsers(dest='command', help='Commands') # Setup command setup_parser = subparsers.add_parser('new', help='Setup a new instance') - setup_parser.add_argument('compose_file', type=str, help='Docker compose file') + setup_parser.add_argument('compose_file', type=str, + help='Docker compose file') setup_parser.add_argument('-o', '--dir', type=str, help='Work directory') setup_parser.add_argument('-i', '--image', type=str, help='VM image path') - setup_parser.add_argument('-c', '--vcpus', type=int, default=1, help='Number of vCPUs') - setup_parser.add_argument('-m', '--memory', type=str, default='1G', help='Memory size (e.g., 1G, 512M)') - setup_parser.add_argument('-d', '--disk', type=str, default='20G', help='Disk size (e.g., 20G)') - setup_parser.add_argument('-g', '--gpu', type=str, action='append', help='GPU device') - setup_parser.add_argument('-p', '--port', action='append', type=str, help='Port mapping in format: protocol[:address]:from:to') - setup_parser.add_argument('--local-key-provider', action='store_true', help='Enable local key provider') + setup_parser.add_argument( + '-c', '--vcpus', type=int, default=1, help='Number of vCPUs') + setup_parser.add_argument( + '-m', '--memory', type=str, default='2G', help='Memory size (e.g., 1G, 512M)') + setup_parser.add_argument('-d', '--disk', type=str, + default='20G', help='Disk size (e.g., 20G)') + setup_parser.add_argument('-g', '--gpu', type=str, + action='append', help='GPU device') + setup_parser.add_argument('-p', '--port', action='append', type=str, + help='Port mapping in format: protocol[:address]:from:to') + setup_parser.add_argument('--local-key-provider', '--lkp', + action='store_true', help='Enable local key provider') + setup_parser.add_argument( + '--pin-numa', action='store_true', help='Pin vCPUs to NUMA node') + setup_parser.add_argument( + '--hugepages', action='store_true', help='Enable hugepages') # Start command start_parser = subparsers.add_parser('run', help='Start an instance') start_parser.add_argument('dir', type=str, help='Work directory') - start_parser.add_argument('-m', '--memory', type=str, help='Memory size (e.g. 2G, 512M)') - start_parser.add_argument('-c', '--vcpus', type=int, help='Number of virtual CPUs') start_parser.add_argument('--imgdir', type=str, help='The image directory') - start_parser.add_argument('--kp-port', type=int, default=3443, help='The key provider listening port') - start_parser.add_argument('--pin-numa', type=bool, default=False, help='Pin the guest to given NUMA') - start_parser.add_argument('--hugepage', type=bool, default=False, help='Enable pre-allocate on hugepage. Must also --pin-numa') + start_parser.add_argument( + '--kp-port', type=int, default=3443, help='The key provider listening port') + start_parser.add_argument( + '--dry-run', action='store_true', help='Run in dry run mode') # List Gpus command subparsers.add_parser('lsgpu', help='List available GPUs') + # Tag VFIO command + subparsers.add_parser( + 'tag-vfio', help='Tag NVIDIA GPUs and NVSwitches for VFIO passthrough') + + # Run the host server only + serve_parser = subparsers.add_parser( + 'serve', help='Run the host server only') + serve_parser.add_argument('dir', type=str, help='Work directory') + serve_parser.add_argument( + '--kp-port', type=int, default=3443, help='The key provider listening port') + args = parser.parse_args() if args.command == 'new': @@ -519,16 +836,17 @@ def main(): manager.setup_instance(args) elif args.command == 'run': manager = DStackManager() - config = host_api.ServerConfig( - vm_dir=args.dir, kp_address="127.0.0.1", kp_port=args.kp_port) - api, host_port = host_api.create_http_server(config) - print(f"Starting HTTP server on localhost:{host_port}") - thread = threading.Thread(target=api.serve_forever, daemon=True) - thread.start() - manager.run_instance(args.dir, host_port, memory=args.memory, vcpus=args.vcpus, - imgdir=args.imgdir, pin_numa=args.pin_numa, hugepage=args.hugepage) + thread = start_server(args.dir, args.kp_port) + manager.run_instance(args.dir, thread.host_port, + imgdir=args.imgdir, dry_run=args.dry_run) elif args.command == 'lsgpu': list_available_gpus() + elif args.command == 'tag-vfio': + tag_vfio() + elif args.command == 'serve': + thread = start_server(args.dir, args.kp_port) + gen_vm_config(args.dir, thread.host_port) + thread.join() else: parser.print_help() From 1d596d95b4330397d5a51a4a87f1d9cf800b9fac Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 May 2025 04:29:42 +0000 Subject: [PATCH 140/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 2c0514117..6950758eb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 2c0514117a2f45d98fa784dab998f341f57fc006 +Subproject commit 6950758eb70663449617862e456f4038b281bf78 From 31de21bf6715d42cec4de3d3a7ecec10a1faf96c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 May 2025 06:07:52 +0000 Subject: [PATCH 141/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 6950758eb..30b093b1c 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6950758eb70663449617862e456f4038b281bf78 +Subproject commit 30b093b1c50d10f5abc9e37b97a9c4f017d13b97 From 146ad6f38e36cc6fe2e577c57259b4e0be024012 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 May 2025 07:49:01 +0000 Subject: [PATCH 142/435] Update rust toolchain to 1.86 --- bb-build/conf/local.conf | 4 ++-- dstack | 2 +- meta-rust-bin | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index 2aa792eee..181960482 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -288,5 +288,5 @@ PACKAGECONFIG:append:pn-qemu-system-native = " sdl" CONF_VERSION = "2" SERIAL_CONSOLES = "115200;ttyS0" -PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.84.0" -PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.84.0" +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" diff --git a/dstack b/dstack index 30b093b1c..59f66b9c3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 30b093b1c50d10f5abc9e37b97a9c4f017d13b97 +Subproject commit 59f66b9c390bf2d33e04b14fcf82baaafdfd2ac4 diff --git a/meta-rust-bin b/meta-rust-bin index 540161165..53974894f 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit 5401611657e42a546f23df6a380d96d64ca78f9b +Subproject commit 53974894fe1bf0fba26d0f6d5ded32b3c3f4a43e From c8bd81dc0c3e01896577cf83a2b147b0f3afb8d4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 May 2025 11:49:28 +0000 Subject: [PATCH 143/435] Only compile dstack-guest-agent and dstack-util in dstack --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 59f66b9c3..82871f592 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 59f66b9c390bf2d33e04b14fcf82baaafdfd2ac4 +Subproject commit 82871f592a88b920cee8672f2c2e6b3e6ceb9057 diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 5239262ca..9b04106ad 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -20,6 +20,7 @@ DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" +EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util" inherit cargo_bin @@ -37,7 +38,6 @@ do_install() { install -d ${D}${bindir} install -d ${D}${sysconfdir}/docker install -d ${D}${sysconfdir}/systemd/journald.conf.d - install -m 0755 ${CARGO_BINDIR}/iohash ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} From 89c3ff55555a160801b26969bb75e0ba5d9af531 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 16 May 2025 00:47:14 +0000 Subject: [PATCH 144/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 82871f592..ce6e88d04 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 82871f592a88b920cee8672f2c2e6b3e6ceb9057 +Subproject commit ce6e88d0401408949787f7beb0c5427e7456780a From deda70eccc811dfb6757f6832caf97cae9283d34 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 May 2025 12:41:12 +0000 Subject: [PATCH 145/435] Cleanup unused packages --- .../base-files/base-files%.bbappend | 5 +---- .../recipes-core/base-files/dstack-prebuilt.bb | 17 ----------------- .../recipes-core/base-files/files/dockerd_env | 1 - .../recipes-core/images/dstack-rootfs-base.inc | 4 ---- 4 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 meta-dstack/recipes-core/base-files/dstack-prebuilt.bb delete mode 100644 meta-dstack/recipes-core/base-files/files/dockerd_env diff --git a/meta-dstack/recipes-core/base-files/base-files%.bbappend b/meta-dstack/recipes-core/base-files/base-files%.bbappend index f859de39f..8802adbb4 100644 --- a/meta-dstack/recipes-core/base-files/base-files%.bbappend +++ b/meta-dstack/recipes-core/base-files/base-files%.bbappend @@ -1,4 +1 @@ -do_install:append() { - # Remove the motd file from poky so we can use our own - rm -rf ${D}${sysconfdir}/motd -} +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" diff --git a/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb b/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb deleted file mode 100644 index f2270f993..000000000 --- a/meta-dstack/recipes-core/base-files/dstack-prebuilt.bb +++ /dev/null @@ -1,17 +0,0 @@ -SUMMARY = "Dstack base files" - -LICENSE = "MIT" -LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" - -SRC_URI = "file://motd \ - file://dockerd_env" - -inherit allarch - -S = "${WORKDIR}" - -do_install() { - install -d ${D}${sysconfdir}/sysconfig/ - install -m 0644 ${S}/motd ${D}${sysconfdir}/motd - install -m 0755 ${S}/dockerd_env ${D}${sysconfdir}/sysconfig/dockerd -} diff --git a/meta-dstack/recipes-core/base-files/files/dockerd_env b/meta-dstack/recipes-core/base-files/files/dockerd_env deleted file mode 100644 index d5a17ad60..000000000 --- a/meta-dstack/recipes-core/base-files/files/dockerd_env +++ /dev/null @@ -1 +0,0 @@ -export DOCKER_RAMDISK=true \ No newline at end of file diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 8a870f748..0b2065b5f 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -12,13 +12,9 @@ IMAGE_INSTALL = "\ iptables \ docker-moby \ docker-compose \ - dstack-prebuilt \ tdx-guest-ko \ dstack-guest \ wireguard-tools \ - e2fsprogs-mke2fs \ - e2fsprogs-resize2fs \ - e2fsprogs-e2fsck \ cryptsetup \ curl \ jq \ From 54514d10240e9b1fdac5acb5d0fc3efc92904da7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 May 2025 07:30:58 +0000 Subject: [PATCH 146/435] Exclude target/ dir when unpacking dstack-guest source --- .../recipes-core/dstack-guest/dstack-guest.bb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 9b04106ad..7f8c811b3 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -8,14 +8,13 @@ inherit systemd REPO_ROOT = "${THISDIR}/../../.." SRC_DIR = '${REPO_ROOT}/dstack' -SRC_URI = 'file://${REPO_ROOT}/dstack \ - file://docker-daemon.json' -SRCREV = "${DSTACK_SRC_REV}" -S = "${WORKDIR}/${SRC_DIR}" +S = "${WORKDIR}/dstack" RDEPENDS:${PN} += "bash" +DEPENDS += "rsync-native" + DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" @@ -24,6 +23,12 @@ EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util" inherit cargo_bin +do_unpack() { + mkdir -p ${S} + rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ + cp ${THISDIR}/files/docker-daemon.json ${S}/ +} + do_configure() { cargo_bin_do_configure } @@ -42,7 +47,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} - install -m 0755 ${WORKDIR}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json + install -m 0755 ${S}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ From dc49e357bc4e7061512eab92b4148a00baddd811 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 May 2025 07:32:32 +0000 Subject: [PATCH 147/435] Calculate image digest at build time --- mkimage.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index dcea94260..1e769f5b7 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -46,6 +46,8 @@ source ${VERITY_ENV_FILE} DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} +IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} +IMAGE_TAR_NO_ROOTFS=${IMAGE_TAR_NO_ROOTFS:-"${DIST_DIR}/${DIST_NAME}-no-rootfs-${DSTACK_VERSION}.tar.gz"} verbose() { echo "$@" @@ -82,15 +84,19 @@ cat < ${OUTPUT_DIR}/metadata.json } EOF -echo "Generating md5sum.txt and sha256sum.txt to ${OUTPUT_DIR}/" +echo "Generating image digest to ${OUTPUT_DIR}/" pushd ${OUTPUT_DIR}/ -find . -type f -not -name md5sum.txt -not -name sha256sum.txt -exec md5sum {} + | sort -k 2 > md5sum.txt -find . -type f -not -name md5sum.txt -not -name sha256sum.txt -exec sha256sum {} + | sort -k 2 > sha256sum.txt +sha256sum ovmf.fd bzImage initramfs.cpio.gz metadata.json > sha256sum.txt +sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd if [ x$DSTACK_TAR_RELEASE = x1 ]; then OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) - echo "Archiving the output directory to ${OUTPUT_DIR}.tar.gz" - (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${OUTPUT_DIR}.tar.gz ${TAR_ARGS} $(basename $OUTPUT_DIR)) + echo "Archiving the output directory to ${IMAGE_TAR}" + (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${IMAGE_TAR} $(basename $OUTPUT_DIR)) + + echo "Creating archive without rootfs files to ${IMAGE_TAR_NO_ROOTFS}" + echo tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_NO_ROOTFS} --exclude="rootfs.*" + tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_NO_ROOTFS} --exclude="rootfs.*" . echo fi From a141fc162b9ddeb073dcce0bbd483aabbecdb413 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 03:57:54 +0000 Subject: [PATCH 148/435] Support for mr-image --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 6 ++++++ mkimage.sh | 9 ++++++--- scripts/bin/dstack.py | 14 ++++++++++++-- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/dstack b/dstack index ce6e88d04..c474cdb88 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit ce6e88d0401408949787f7beb0c5427e7456780a +Subproject commit c474cdb8827100e26065e2eb3b5c1076d83a59f2 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 25f629e85..fb42738f9 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.0" +DISTRO_VERSION = "0.5.1" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 7f8c811b3..7761dd34b 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -29,6 +29,12 @@ do_unpack() { cp ${THISDIR}/files/docker-daemon.json ${S}/ } +# Force the configure task to run every time to detect source changes +do_unpack[nostamp] = "1" + +# Add source directory to configure task dependencies +do_unpack[vardeps] += "SRC_DIR" + do_configure() { cargo_bin_do_configure } diff --git a/mkimage.sh b/mkimage.sh index 1e769f5b7..de3139c7f 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -47,7 +47,6 @@ source ${VERITY_ENV_FILE} DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} -IMAGE_TAR_NO_ROOTFS=${IMAGE_TAR_NO_ROOTFS:-"${DIST_DIR}/${DIST_NAME}-no-rootfs-${DSTACK_VERSION}.tar.gz"} verbose() { echo "$@" @@ -91,12 +90,16 @@ sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd if [ x$DSTACK_TAR_RELEASE = x1 ]; then + IMAGE_TAR_MR=${DIST_DIR}/mr_$(cat ${OUTPUT_DIR}/digest.txt | tr -d '\n').tar.gz + IMAGE_TAR_NO_ROOTFS=${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-mr.tar.gz OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) + rm -rf ${IMAGE_TAR} ${IMAGE_TAR_MR} ${IMAGE_TAR_NO_ROOTFS} echo "Archiving the output directory to ${IMAGE_TAR}" (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${IMAGE_TAR} $(basename $OUTPUT_DIR)) - echo "Creating archive without rootfs files to ${IMAGE_TAR_NO_ROOTFS}" + echo "Creating archive without rootfs files to ${IMAGE_TAR_NO_ROOTFS} -> ${IMAGE_TAR_MR}" echo tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_NO_ROOTFS} --exclude="rootfs.*" - tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_NO_ROOTFS} --exclude="rootfs.*" . + tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_MR} --exclude="rootfs.*" . + ln -sf $(basename ${IMAGE_TAR_MR}) ${IMAGE_TAR_NO_ROOTFS} echo fi diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index bd2a008c3..e3d755c4b 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -125,7 +125,7 @@ def update_guest_config(config_file: str, data: Dict): json.dump(config, f, indent=4) -def gen_vm_config(vm_dir, host_port): +def gen_vm_config(vm_dir, host_port, manifest=None, mr_image=None): shared_dir = os.path.join(vm_dir, 'shared') for filename in ['config.json', '.sys-config.json']: config_file = os.path.join(shared_dir, filename) @@ -133,6 +133,14 @@ def gen_vm_config(vm_dir, host_port): "host_api_url": f"http://10.0.2.2:{host_port}/api", "host_vsock_port": host_port }) + if manifest: + update_guest_config(config_file, { + "vm_config": { + "mr_image": mr_image, + "cpu_count": manifest['cpu_count'], + "memory_size": manifest['memory_size'] * 1024 * 1024 + } + }) @dataclass @@ -364,7 +372,6 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None vm_dir: Directory containing the VM configuration dry_run: Whether to run in dry run mode """ - gen_vm_config(vm_dir, host_port) manifest_path = os.path.join(vm_dir, 'vm-manifest.json') if not os.path.exists(manifest_path): @@ -389,6 +396,9 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None with open(img_metadata_path, 'r') as f: img_metadata = json.load(f) + mr_image = open(os.path.join(image_path, 'digest.txt'), 'rb').read() + gen_vm_config(vm_dir, host_port, manifest, mr_image) + mem_gb = manifest['memory'] // 1024 vcpu_count = manifest['vcpu'] disk_size = manifest['disk_size'] From cf51a88607117cec1658d7db5da9291d00422abf Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 04:14:08 +0000 Subject: [PATCH 149/435] Update dstack --- dstack | 2 +- mkimage.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index c474cdb88..5261e8bfb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit c474cdb8827100e26065e2eb3b5c1076d83a59f2 +Subproject commit 5261e8bfbc4a8e58222ddda016284d560352e63f diff --git a/mkimage.sh b/mkimage.sh index de3139c7f..ed5976b08 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} + # Parse command line arguments while [ $# -gt 0 ]; do case "$1" in From 62f585a0ad75dc31e78474426ad7e10691b2094a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 08:56:42 +0000 Subject: [PATCH 150/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 5261e8bfb..d6f22540c 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 5261e8bfbc4a8e58222ddda016284d560352e63f +Subproject commit d6f22540c47f98140279f774f874cf83909878a5 From ea06992f74ff9fced48e43e5dc1ed75794023712 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 09:01:09 +0000 Subject: [PATCH 151/435] Rename mr_image to os_image_hash --- scripts/bin/dstack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index e3d755c4b..298ac5b38 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -125,7 +125,7 @@ def update_guest_config(config_file: str, data: Dict): json.dump(config, f, indent=4) -def gen_vm_config(vm_dir, host_port, manifest=None, mr_image=None): +def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None): shared_dir = os.path.join(vm_dir, 'shared') for filename in ['config.json', '.sys-config.json']: config_file = os.path.join(shared_dir, filename) @@ -136,7 +136,7 @@ def gen_vm_config(vm_dir, host_port, manifest=None, mr_image=None): if manifest: update_guest_config(config_file, { "vm_config": { - "mr_image": mr_image, + "os_image_hash": os_image_hash, "cpu_count": manifest['cpu_count'], "memory_size": manifest['memory_size'] * 1024 * 1024 } @@ -396,8 +396,8 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None with open(img_metadata_path, 'r') as f: img_metadata = json.load(f) - mr_image = open(os.path.join(image_path, 'digest.txt'), 'rb').read() - gen_vm_config(vm_dir, host_port, manifest, mr_image) + os_image_hash = open(os.path.join(image_path, 'digest.txt'), 'rb').read() + gen_vm_config(vm_dir, host_port, manifest, os_image_hash) mem_gb = manifest['memory'] // 1024 vcpu_count = manifest['vcpu'] From 005a7f1b6d01726939140dac912c9203bd8476ca Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 09:18:54 +0000 Subject: [PATCH 152/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index d6f22540c..b8d54af12 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d6f22540c47f98140279f774f874cf83909878a5 +Subproject commit b8d54af127d1ac69563982872ae33a2489a6fb87 From b3d70b5bad5091cc52405f2b611ea35f3c25fe38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 10:00:45 +0000 Subject: [PATCH 153/435] Add tun.ko --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 0b2065b5f..ebfa3a603 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -22,6 +22,7 @@ IMAGE_INSTALL = "\ chronyc \ qemu-guest-agent \ dstack-zfs \ + kernel-module-tun \ " IMAGE_NAME_SUFFIX ?= "" From eed680073760808e6f2d3c415a6e3e04a0ed462d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 May 2025 11:16:03 +0000 Subject: [PATCH 154/435] Fix rsync not found --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 7761dd34b..1f571de99 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -15,6 +15,9 @@ RDEPENDS:${PN} += "bash" DEPENDS += "rsync-native" +# Ensure rsync-native is built before unpack runs +do_unpack[depends] += "rsync-native:do_populate_sysroot" + DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" From c46abbdaab1fa1c7ce9a3d6d355d5c4f832b6661 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 22 May 2025 02:37:54 +0000 Subject: [PATCH 155/435] Fix reproducible issues for rootfs --- .gitmodules | 2 +- meta-dstack/recipes-core/images/dstack-initscript/init | 2 +- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 7 ++++--- meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend | 1 + meta-security | 2 +- mkimage.sh | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index 224c63cb6..1a6548412 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,4 +18,4 @@ url = https://github.com/Dstack-TEE/dstack [submodule "meta-security"] path = meta-security - url = https://git.yoctoproject.org/meta-security + url = https://github.com/Dstack-TEE/meta-security.git diff --git a/meta-dstack/recipes-core/images/dstack-initscript/init b/meta-dstack/recipes-core/images/dstack-initscript/init index 8ba86235b..fba1182a4 100644 --- a/meta-dstack/recipes-core/images/dstack-initscript/init +++ b/meta-dstack/recipes-core/images/dstack-initscript/init @@ -39,7 +39,7 @@ echo " Data size: ${DATA_SIZE}" veritysetup open ${ROOT_DEV} rootfs ${ROOT_DEV} "${ROOT_HASH}" --hash-offset="${DATA_SIZE}" echo "Mounting rootfs..." -mount -t ext4 -o ro /dev/mapper/rootfs ${ROOT_DIR} +mount -t squashfs /dev/mapper/rootfs ${ROOT_DIR} mount_move_all() { for dir in $@; do diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index ebfa3a603..0ac9387a0 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -34,12 +34,13 @@ LICENSE = "MIT" IMAGE_CLASSES += "dm-verity-img" DM_VERITY_IMAGE = "${PN}" -DM_VERITY_IMAGE_TYPE = "ext4" +DM_VERITY_IMAGE_TYPE = "squashfs" DM_VERITY_SEPARATE_HASH = "0" +DM_VERITY_REPRODUCIBLE = "1" -IMAGE_FSTYPES = "cpio ext4" +IMAGE_FSTYPES = "cpio squashfs" CONVERSIONTYPES += "verity" -IMAGE_TYPES += "ext4.verity" +IMAGE_TYPES += "squashfs.verity" EXTRA_IMAGE_FEATURES = "read-only-rootfs" diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index fa352c6bc..5d9603aee 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -10,6 +10,7 @@ SRC_URI += "file://dstack-docker.cfg \ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ features/netfilter/netfilter.scc \ + cfg/fs/squashfs.scc \ dstack-docker.scc \ dstack.scc" diff --git a/meta-security b/meta-security index bc865c527..bc63d9574 160000 --- a/meta-security +++ b/meta-security @@ -1 +1 @@ -Subproject commit bc865c5276c2ab4031229916e8d7c20148dfbac3 +Subproject commit bc63d95746ef4f0ac6820165e5041d83647e8c9c diff --git a/mkimage.sh b/mkimage.sh index ed5976b08..415f6f298 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -38,11 +38,11 @@ IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/tdx ROOTFS_IMAGE_NAME=${DIST_NAME}-rootfs INITRAMFS_IMAGE=${IMG_DIR}/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=${IMG_DIR}/${ROOTFS_IMAGE_NAME}-tdx.ext4.verity +ROOTFS_IMAGE=${IMG_DIR}/${ROOTFS_IMAGE_NAME}-tdx.squashfs.verity KERNEL_IMAGE=${IMG_DIR}/bzImage OVMF_FIRMWARE=${IMG_DIR}/ovmf.fd # Always use the work-shared directory which has the correct verity env -VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAME}.ext4.verity.env +VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAME}.squashfs.verity.env echo "Loading verity env from ${VERITY_ENV_FILE}" source ${VERITY_ENV_FILE} From 20ad2f1150c7761c93499c0b1dffda7cd9352857 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 22 May 2025 02:49:19 +0000 Subject: [PATCH 156/435] Move some args from local.conf to distro conf --- bb-build/conf/local.conf | 8 ++------ meta-dstack/conf/distro/dstack.conf | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index 181960482..b87c10a2a 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -146,7 +146,7 @@ DISTRO ?= "dstack" # meta/classes-recipe/image.bbclass and # meta/classes-recipe/core-image.bbclass for more details. # We default to enabling the debugging tweaks. -EXTRA_IMAGE_FEATURES ?= "debug-tweaks" +# EXTRA_IMAGE_FEATURES ?= "debug-tweaks" # # Additional image features @@ -249,7 +249,7 @@ BB_DISKMON_DIRS ??= "\ # # By default native qemu will build with a builtin VNC server where graphical output can be # seen. The line below enables the SDL UI frontend too. -PACKAGECONFIG:append:pn-qemu-system-native = " sdl" +# PACKAGECONFIG:append:pn-qemu-system-native = " sdl" # By default libsdl2-native will be built, if you want to use your host's libSDL instead of # the minimal libsdl built by libsdl2-native then uncomment the ASSUME_PROVIDED line below. #ASSUME_PROVIDED += "libsdl2-native" @@ -286,7 +286,3 @@ PACKAGECONFIG:append:pn-qemu-system-native = " sdl" # track the version of this file when it was generated. This can safely be ignored if # this doesn't mean anything to you. CONF_VERSION = "2" - -SERIAL_CONSOLES = "115200;ttyS0" -PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" -PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index fb42738f9..b8d7f9e34 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -16,3 +16,7 @@ PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" + +SERIAL_CONSOLES = "115200;ttyS0" +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" From 9dcb89fbb847a05f67bf9af889eff67b8aa2e441 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 22 May 2025 02:52:00 +0000 Subject: [PATCH 157/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index b8d54af12..5ee967f15 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit b8d54af127d1ac69563982872ae33a2489a6fb87 +Subproject commit 5ee967f15afd6d7061af65f0b76c6af8eca744d1 From e75e57db37d772af331a26fb89f2413d2def0253 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 28 May 2025 02:20:50 +0000 Subject: [PATCH 158/435] Update dstack and build.sh --- build.sh | 37 +++++++++++++++++++++++++++---------- dstack | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index 5443fca18..705508665 100755 --- a/build.sh +++ b/build.sh @@ -42,7 +42,14 @@ check_config() { } require_config() { - cat <<'EOF' >build-config.sh.tpl + +# Base port for RPC services +BASE_PORT=$(($RANDOM % 1000 * 10 + 10000)) +CID_POOL_START=$(($RANDOM % 1000 * 1000 + 20000)) +SUBNET_INDEX=$(($RANDOM % 240 + 10)) + + + cat <build-config.sh.tpl # DNS domain of kms rpc and dstack-gateway rpc # *.1022.kvin.wang resolves to 10.0.2.2 which is the IP of the host system # from CVMs point of view @@ -50,16 +57,13 @@ KMS_DOMAIN=kms.1022.kvin.wang GATEWAY_DOMAIN=gateway.1022.kvin.wang # CIDs allocated to VMs start from this number of type unsigned int32 -VMM_CID_POOL_START=20000 +VMM_CID_POOL_START=$CID_POOL_START # CID pool size VMM_CID_POOL_SIZE=1000 -# Base port for RPC services -BASE_PORT=13000 - VMM_RPC_LISTEN_PORT=$BASE_PORT # Whether port mapping from host to CVM is allowed -VMM_PORT_MAPPING_ENABLED=false +VMM_PORT_MAPPING_ENABLED=true # Host API configuration, type of uint32 VMM_VSOCK_LISTEN_PORT=$BASE_PORT @@ -68,7 +72,7 @@ GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) GATEWAY_WG_INTERFACE=dgw-$USER GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) -GATEWAY_WG_IP=10.3.3.1 +GATEWAY_WG_IP=10.$SUBNET_INDEX.3.1 GATEWAY_SERVE_PORT=$(($BASE_PORT + 4)) GATEWAY_CERT= GATEWAY_KEY= @@ -153,6 +157,9 @@ quote_enabled = false address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT auto_bootstrap_domain = "$KMS_DOMAIN" + +[core.image] +verify = false EOF # dstack-gateway @@ -304,8 +311,12 @@ cfg) dl) download_image $2 $3 ;; -"") - # If no action specified, build everything +hostcfg) + require_config + build_host + build_cfg + ;; +all) require_config build_host build_guest @@ -313,7 +324,13 @@ dl) ;; *) echo "Invalid action: $ACTION" - echo "Valid actions are: host, guest, cfg, dl" + echo "Valid actions are:" + echo " host - Build host binaries only" + echo " guest - Build guest images only" + echo " cfg - Generate configuration files only" + echo " dl - Download a specific image" + echo " hostcfg - Build host binaries and generate configuration files" + echo " all - Build everything (host, guest, and configuration)" exit 1 ;; esac diff --git a/dstack b/dstack index 5ee967f15..265c4ab65 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 5ee967f15afd6d7061af65f0b76c6af8eca744d1 +Subproject commit 265c4ab658f7eba2cb9a943a3b6bdd98da7a7dfe From 402dd23395e38b74cf07669ce5de760345b505da Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 28 May 2025 02:23:02 +0000 Subject: [PATCH 159/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 265c4ab65..a7345f89e 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 265c4ab658f7eba2cb9a943a3b6bdd98da7a7dfe +Subproject commit a7345f89e5e3d13abdfb75242a31cd98829a972d From 5b63aec337f19a541798970c7cf8d846171f0ca9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 May 2025 08:03:34 +0000 Subject: [PATCH 160/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index a7345f89e..be9d0476a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit a7345f89e5e3d13abdfb75242a31cd98829a972d +Subproject commit be9d0476a63e937eda4c13659547a25088393394 From c23e8d74fa63d96f9f308154ff2a91f2d100778a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Jun 2025 05:18:45 +0000 Subject: [PATCH 161/435] Fix motd --- .../recipes-core/base-files/base-files%.bbappend | 12 ++++++++++++ .../base-files/files/{motd => dstack-motd} | 0 2 files changed, 12 insertions(+) rename meta-dstack/recipes-core/base-files/files/{motd => dstack-motd} (100%) diff --git a/meta-dstack/recipes-core/base-files/base-files%.bbappend b/meta-dstack/recipes-core/base-files/base-files%.bbappend index 8802adbb4..66848558b 100644 --- a/meta-dstack/recipes-core/base-files/base-files%.bbappend +++ b/meta-dstack/recipes-core/base-files/base-files%.bbappend @@ -1 +1,13 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI += "file://dstack-motd" + +do_install:append() { + if [ -f ${WORKDIR}/dstack-motd ];then + bbnote "Installing custom dstack motd file" + install -m 0644 ${WORKDIR}/dstack-motd ${D}${sysconfdir}/motd + else + bbwarn "Custom dstack-motd file not found in ${WORKDIR}" + ls -la ${WORKDIR} + fi +} diff --git a/meta-dstack/recipes-core/base-files/files/motd b/meta-dstack/recipes-core/base-files/files/dstack-motd similarity index 100% rename from meta-dstack/recipes-core/base-files/files/motd rename to meta-dstack/recipes-core/base-files/files/dstack-motd From 9410d3612752ee9d35ca90cd5cf7097ac1780f47 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Jun 2025 05:19:51 +0000 Subject: [PATCH 162/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index be9d0476a..8aa345762 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit be9d0476a63e937eda4c13659547a25088393394 +Subproject commit 8aa3457621be2b4d47322052c19f57aae5273834 From 8956146c212b206985ca7dabf98acb26c5537fe4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Jun 2025 02:41:11 +0000 Subject: [PATCH 163/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 8aa345762..55a1d52a4 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 8aa3457621be2b4d47322052c19f57aae5273834 +Subproject commit 55a1d52a4d5ec78baf0d6dde27d86eebcca06eb7 From c61c8e290205c755bcd73d3ab710bcfd010247ed Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Jun 2025 13:36:59 +0000 Subject: [PATCH 164/435] Dstack v0.5.2 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 55a1d52a4..45ebd05a2 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 55a1d52a4d5ec78baf0d6dde27d86eebcca06eb7 +Subproject commit 45ebd05a25ad4ffacce3b8f003e4f5a8b609b2e2 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index b8d7f9e34..8ab3359b3 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.1" +DISTRO_VERSION = "0.5.2" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From fbdf2e76fb6bd9142277fdd84809de87d86548ef Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 13 Jun 2025 05:33:17 +0000 Subject: [PATCH 165/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 45ebd05a2..c7b073d92 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 45ebd05a25ad4ffacce3b8f003e4f5a8b609b2e2 +Subproject commit c7b073d928131a8b757572e57bc715a505d7895e From 9d8f5a1a30be97b02ce0612ff3d076192a2c7ff6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 13 Jun 2025 09:36:02 +0000 Subject: [PATCH 166/435] Remove qemu-guest-agent from packages --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 0ac9387a0..d4da957fa 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -20,7 +20,6 @@ IMAGE_INSTALL = "\ jq \ chrony \ chronyc \ - qemu-guest-agent \ dstack-zfs \ kernel-module-tun \ " From 379427eab3071b81c2a31b8a8a1fb48ffe6925cb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Jun 2025 01:59:35 +0000 Subject: [PATCH 167/435] User tiny kernel --- meta-dstack/conf/distro/dstack.conf | 1 + .../recipes-core/images/dstack-rootfs-base.inc | 3 +++ .../linux/files/dstack-docker.cfg | 2 ++ .../recipes-kernel/linux/files/dstack.cfg | 17 ++++++++++++++++- .../recipes-kernel/linux/linux-yocto%.bbappend | 3 +++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 8ab3359b3..8cb551526 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -12,6 +12,7 @@ VOLATILE_TMP_DIR = "yes" PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" +LINUX_KERNEL_TYPE = "tiny" VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index d4da957fa..0f76c46d1 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -22,6 +22,9 @@ IMAGE_INSTALL = "\ chronyc \ dstack-zfs \ kernel-module-tun \ + kernel-module-fuse \ + fuse3 \ + fuse3-utils \ " IMAGE_NAME_SUFFIX ?= "" diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index 4bd4509e3..b8fd02442 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -6,6 +6,7 @@ CONFIG_IP_VS=m CONFIG_SECCOMP=y CONFIG_SECCOMP_FILTER=y CONFIG_SECURITY_SELINUX=y +CONFIG_VLAN_8021Q=y CONFIG_BRIDGE_VLAN_FILTERING=y CONFIG_XFRM=y CONFIG_XFRM_USER=y @@ -22,3 +23,4 @@ CONFIG_NETFILTER_FAMILY_BRIDGE=y CONFIG_VETH=y CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y CONFIG_VXLAN=y +CONFIG_CGROUP_BPF=y diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index 9277e229c..a1c5bb332 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -1,3 +1,4 @@ +CONFIG_MODULES=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_NET_9P=y @@ -5,6 +6,20 @@ CONFIG_NET_9P_VIRTIO=y CONFIG_9P_FS=y CONFIG_9P_FS_POSIX_ACL=y CONFIG_PCI=y +CONFIG_TUN=m CONFIG_VIRTIO_PCI=y CONFIG_ISO9660_FS=y -CONFIG_WIREGUARD=y \ No newline at end of file +CONFIG_WIREGUARD=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TMPFS_XATTR=y +# +CONFIG_PM=n +CONFIG_SUSPEND=n +CONFIG_PM_SLEEP=n +CONFIG_HOTPLUG_CPU=n +CONFIG_HOTPLUG_PCI=n +CONFIG_BT=n +CONFIG_MMC=n +CONFIG_SCSI=n +CONFIG_INPUT=n +CONFIG_WLAN=n diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 5d9603aee..a2a9d5d1f 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -1,5 +1,7 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" +LINUX_VERSION_EXTENSION = "-dstack" + SRC_URI += "file://dstack-docker.cfg \ file://dstack-docker.scc \ file://dstack-tdx.cfg \ @@ -10,6 +12,7 @@ SRC_URI += "file://dstack-docker.cfg \ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ features/netfilter/netfilter.scc \ + features/fuse/fuse.scc \ cfg/fs/squashfs.scc \ dstack-docker.scc \ dstack.scc" From 385ff326d4ae76d178abd231b8620418257ee966 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Jun 2025 02:00:33 +0000 Subject: [PATCH 168/435] Remove login related binaries for prod image --- meta-dstack/conf/distro/dstack.conf | 2 ++ .../recipes-core/busybox/busybox%.bbappend | 1 + .../recipes-core/busybox/files/fragment.cfg | 1 + .../images/dstack-rootfs-base.inc | 36 +++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 8cb551526..0f5259ed3 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -21,3 +21,5 @@ VIRTUAL-RUNTIME_dev_manager = "systemd" SERIAL_CONSOLES = "115200;ttyS0" PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" + +BAD_RECOMMENDATIONS = "busybox-syslog systemd-extra-utils" diff --git a/meta-dstack/recipes-core/busybox/busybox%.bbappend b/meta-dstack/recipes-core/busybox/busybox%.bbappend index bef35ef5d..0dd2f3573 100644 --- a/meta-dstack/recipes-core/busybox/busybox%.bbappend +++ b/meta-dstack/recipes-core/busybox/busybox%.bbappend @@ -1,2 +1,3 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" SRC_URI += "file://fragment.cfg" +RRECOMMENDS:${PN} = "" diff --git a/meta-dstack/recipes-core/busybox/files/fragment.cfg b/meta-dstack/recipes-core/busybox/files/fragment.cfg index b7dab01c0..34eb67ab3 100644 --- a/meta-dstack/recipes-core/busybox/files/fragment.cfg +++ b/meta-dstack/recipes-core/busybox/files/fragment.cfg @@ -1 +1,2 @@ CONFIG_UDHCPD=n +CONFIG_UDHCPC=n diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 0f76c46d1..bb8944206 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -88,6 +88,42 @@ disable_getty_services() { rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}.service rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}@.service done + + # Login/console related + rm -rf ${IMAGE_ROOTFS}/usr/sbin/getty + rm -rf ${IMAGE_ROOTFS}/usr/sbin/agetty + rm -rf ${IMAGE_ROOTFS}/usr/bin/login + rm -rf ${IMAGE_ROOTFS}/usr/bin/loginctl + rm -rf ${IMAGE_ROOTFS}/etc/systemd/system/getty.target.wants + + # Console fonts and keymaps + rm -rf ${IMAGE_ROOTFS}/usr/share/consolefonts + rm -rf ${IMAGE_ROOTFS}/usr/share/keymaps + + # Systemd components we don't need + rm -rf ${IMAGE_ROOTFS}/usr/bin/systemd-tty-ask-password-agent + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/debug-shell.service + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/suspend.target + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/systemd-hibernate.service + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/systemd-suspend-then-hibernate.service + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/systemd-hibernate-resume + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system-generators/systemd-getty-generator + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/systemd-logind + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/systemd-logind.service + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/multi-user.target.wants/systemd-logind.service + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/logind.conf.d + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/system/graphical.target.wants + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/user/graphical-session.target + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/user/graphical-session-pre.target + rm -rf ${IMAGE_ROOTFS}/usr/lib/systemd/user/bluetooth.target + + # Polkit + rm -rf ${IMAGE_ROOTFS}/usr/share/polkit-1 + + # Factory defaults + rm -rf ${IMAGE_ROOTFS}/usr/share/factory + + > ${IMAGE_ROOTFS}/etc/securetty } mkdirs() { From c9b5b92098d9063ce4e0f6186f03b89b754d0f5f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Jun 2025 07:35:50 +0000 Subject: [PATCH 169/435] Build OVMF with Config-B --- ...eclare-ProcessLibraryConstructorList.patch | 25 ++++++++++++++----- .../dstack-ovmf/dstack-ovmf_git.bb | 14 ++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch index 523b169ec..12a1c8cfc 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch @@ -1,21 +1,34 @@ -From 5c718d0c37c65e845b100060fbc08cae26412c35 Mon Sep 17 00:00:00 2001 +From 596f2e3bd7292c8f22cfe01ec95b8b46ce1bc8a1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 4 Nov 2024 04:35:18 +0000 -Subject: [PATCH 5/5] Declare ProcessLibraryConstructorList +Subject: [PATCH] Declare ProcessLibraryConstructorList --- - OvmfPkg/Sec/SecMain.c | 1 + - 1 file changed, 1 insertion(+) + OvmfPkg/IntelTdx/Sec/SecMain.c | 1 + + OvmfPkg/Sec/SecMain.c | 1 + + 2 files changed, 2 insertions(+) +diff --git a/OvmfPkg/IntelTdx/Sec/SecMain.c b/OvmfPkg/IntelTdx/Sec/SecMain.c +index 7f2d28af95..fcc6b97c28 100644 +--- a/OvmfPkg/IntelTdx/Sec/SecMain.c ++++ b/OvmfPkg/IntelTdx/Sec/SecMain.c +@@ -163,6 +163,7 @@ SecCoreStartupWithStack ( + IdtDescriptor.Base = (UINTN)&IdtTableInStack.IdtTable; + IdtDescriptor.Limit = (UINT16)(sizeof (IdtTableInStack.IdtTable) - 1); + ++ void ProcessLibraryConstructorList (); + ProcessLibraryConstructorList (); + + // diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c -index d13a948509..2a44c4ed24 100644 +index d13a948509..58c48c967c 100644 --- a/OvmfPkg/Sec/SecMain.c +++ b/OvmfPkg/Sec/SecMain.c @@ -885,6 +885,7 @@ SecCoreStartupWithStack ( InitializeCpuExceptionHandlers (NULL); } -+ void ProcessLibraryConstructorList (void); ++ void ProcessLibraryConstructorList (); ProcessLibraryConstructorList (); if (!SevEsIsEnabled ()) { diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index b073af504..96fcda3c0 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -195,17 +195,13 @@ do_compile:class-target() { # do_deploy will be able to find the files. rm -rf ${WORKDIR}/ovmf mkdir ${WORKDIR}/ovmf - OVMF_DIR_SUFFIX="X64" - if [ "${TARGET_ARCH}" != "x86_64" ] ; then - OVMF_DIR_SUFFIX="Ia32" # Note the different capitalization - fi FIXED_GCCVER=$(fixup_target_tools ${GCC_VER}) bbnote FIXED_GCCVER is ${FIXED_GCCVER} - build_dir="${S}/Build/Ovmf$OVMF_DIR_SUFFIX/RELEASE_${FIXED_GCCVER}" + build_dir="${S}/Build/IntelTdx/RELEASE_${FIXED_GCCVER}" bbnote "Building without Secure Boot." - rm -rf ${S}/Build/Ovmf$OVMF_DIR_SUFFIX - ${S}/OvmfPkg/build.sh $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} + rm -rf ${S}/Build/IntelTdx + ${S}/OvmfPkg/build.sh -p ${S}/OvmfPkg/IntelTdx/IntelTdxX64.dsc $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} ln ${build_dir}/FV/OVMF.fd ${WORKDIR}/ovmf/ovmf.fd ln ${build_dir}/FV/OVMF_CODE.fd ${WORKDIR}/ovmf/ovmf.code.fd ln ${build_dir}/FV/OVMF_VARS.fd ${WORKDIR}/ovmf/ovmf.vars.fd @@ -214,8 +210,8 @@ do_compile:class-target() { if ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'true', 'false', d)}; then # Repeat build with the Secure Boot flags. bbnote "Building with Secure Boot." - rm -rf ${S}/Build/Ovmf$OVMF_DIR_SUFFIX - ${S}/OvmfPkg/build.sh $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} ${OVMF_SECURE_BOOT_FLAGS} + rm -rf ${S}/Build/IntelTdx + ${S}/OvmfPkg/build.sh -p ${S}/OvmfPkg/IntelTdx/IntelTdxX64.dsc $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} ${OVMF_SECURE_BOOT_FLAGS} ln ${build_dir}/FV/OVMF.fd ${WORKDIR}/ovmf/ovmf.secboot.fd ln ${build_dir}/FV/OVMF_CODE.fd ${WORKDIR}/ovmf/ovmf.secboot.code.fd ln ${build_dir}/${OVMF_ARCH}/EnrollDefaultKeys.efi ${WORKDIR}/ovmf/ From 0cf0f6a40324b9edbcf037fdd4f2a3a61e10727c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Jun 2025 03:02:18 +0000 Subject: [PATCH 170/435] Fix error in dstack.py --- scripts/bin/dstack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 298ac5b38..e9c878e30 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -135,11 +135,11 @@ def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None): }) if manifest: update_guest_config(config_file, { - "vm_config": { + "vm_config": json.dumps({ "os_image_hash": os_image_hash, - "cpu_count": manifest['cpu_count'], - "memory_size": manifest['memory_size'] * 1024 * 1024 - } + "cpu_count": manifest['vcpu'], + "memory_size": manifest['memory'] * 1024 * 1024 + }) }) From 69ac0bb0b195fbd85329faeafa0f539c990fc503 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 23 Jun 2025 06:02:40 +0000 Subject: [PATCH 171/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index c7b073d92..c06e524bd 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit c7b073d928131a8b757572e57bc715a505d7895e +Subproject commit c06e524bd460fd9c9add835b634d155d4b08d7e7 From 1dbed7dd6eede0a99a0ae895f7bf99146d134387 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 24 Jun 2025 07:42:08 +0000 Subject: [PATCH 172/435] Bump version to 0.5.3 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index c06e524bd..910c0ae30 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit c06e524bd460fd9c9add835b634d155d4b08d7e7 +Subproject commit 910c0ae303126819184dbeb161ae78711e691d91 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 0f5259ed3..0d2f4627e 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.2" +DISTRO_VERSION = "0.5.3" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 1932d5c2fe95f4198e10f281d935853705a98305 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 24 Jun 2025 17:45:44 +0800 Subject: [PATCH 173/435] Update dstack.py --- scripts/bin/dstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index e9c878e30..e00a44094 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -396,7 +396,7 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None with open(img_metadata_path, 'r') as f: img_metadata = json.load(f) - os_image_hash = open(os.path.join(image_path, 'digest.txt'), 'rb').read() + os_image_hash = open(os.path.join(image_path, 'digest.txt'), 'r').read().strip() gen_vm_config(vm_dir, host_port, manifest, os_image_hash) mem_gb = manifest['memory'] // 1024 From 7d25a069b9ee81521d969b434ae17d3626b34fd2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 24 Jun 2025 11:02:01 +0000 Subject: [PATCH 174/435] Alias passwd files to /etc/users --- dstack | 2 +- .../recipes-core/images/dstack-rootfs-base.inc | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dstack b/dstack index 910c0ae30..6d60b27be 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 910c0ae303126819184dbeb161ae78711e691d91 +Subproject commit 6d60b27be5ce93008f3ad39303c08a6a2a830bcd diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index bb8944206..c4743ed2a 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -56,7 +56,7 @@ COMPATIBLE_HOST = "x86_64.*-linux" ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" ROOTFS_POSTPROCESS_COMMAND += "symlink_lib64;" IMAGE_FEATURES[validitems] += "nologin" -ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_getty_services", "",d)}' +ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_login", "",d)}' ROOTFS_POSTPROCESS_COMMAND += "mkdirs;" @@ -77,7 +77,7 @@ symlink_lib64() { ln -s lib ${IMAGE_ROOTFS}/lib64 } -disable_getty_services() { +disable_login() { for srv in getty getty-pre; do rm -f ${IMAGE_ROOTFS}/etc/systemd/system/${srv}.target rm -f ${IMAGE_ROOTFS}/usr/lib/systemd/system/${srv}.target @@ -135,4 +135,11 @@ mkdirs() { mkdir -p ${IMAGE_ROOTFS}/etc/wireguard mkdir -p ${IMAGE_ROOTFS}/var/lib/docker ln -sf dstack ${IMAGE_ROOTFS}/tapp + + # Aliases passwd files to a subdirectory + mkdir -p ${IMAGE_ROOTFS}/etc/users + mv ${IMAGE_ROOTFS}/etc/passwd ${IMAGE_ROOTFS}/etc/users/ + mv ${IMAGE_ROOTFS}/etc/shadow ${IMAGE_ROOTFS}/etc/users/ + ln -s users/passwd ${IMAGE_ROOTFS}/etc/passwd + ln -s users/shadow ${IMAGE_ROOTFS}/etc/shadow } From 108ef273355c72b71d9ca802eddddcfe1306af05 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 8 Jul 2025 02:15:52 +0000 Subject: [PATCH 175/435] Update build-config.sh --- build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 705508665..d8e9c0d08 100755 --- a/build.sh +++ b/build.sh @@ -66,6 +66,8 @@ VMM_RPC_LISTEN_PORT=$BASE_PORT VMM_PORT_MAPPING_ENABLED=true # Host API configuration, type of uint32 VMM_VSOCK_LISTEN_PORT=$BASE_PORT +# Whether to enable GPU support +VMM_ENABLE_GPU=false KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) @@ -74,8 +76,8 @@ GATEWAY_WG_INTERFACE=dgw-$USER GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) GATEWAY_WG_IP=10.$SUBNET_INDEX.3.1 GATEWAY_SERVE_PORT=$(($BASE_PORT + 4)) -GATEWAY_CERT= -GATEWAY_KEY= +GATEWAY_CERT=$CERBOT_WORKDIR/live/cert.pem +GATEWAY_KEY=$CERBOT_WORKDIR/live/key.pem BIND_PUBLIC_IP=0.0.0.0 @@ -149,6 +151,9 @@ mandatory = false [core] cert_dir = "$CERTS_DIR" +[core.gpu] +enabled = $VMM_ENABLE_GPU + [core.auth_api] type = "dev" From a056d425367c92c76597295d46fdea7a770517f9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 8 Jul 2025 02:16:09 +0000 Subject: [PATCH 176/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 6d60b27be..f09ad6754 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6d60b27be5ce93008f3ad39303c08a6a2a830bcd +Subproject commit f09ad6754c32ec7ef06294c8bdb8cc93402c04d6 From 6f7a5bb9360783758b30c6eff6b4207a3c82db78 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 16 Jul 2025 09:21:44 +0000 Subject: [PATCH 177/435] Update stack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index f09ad6754..39800da36 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f09ad6754c32ec7ef06294c8bdb8cc93402c04d6 +Subproject commit 39800da36a300f5fdd6a9e908f75a899a4a4b8e3 From 3b1a588dcd4f21c7d3ea9c6a037af6a41b966b38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 30 Jul 2025 02:04:30 +0000 Subject: [PATCH 178/435] Support for up to 512 CPU cores --- meta-dstack/recipes-kernel/linux/files/dstack.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index a1c5bb332..b7352e7ae 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -12,6 +12,7 @@ CONFIG_ISO9660_FS=y CONFIG_WIREGUARD=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_TMPFS_XATTR=y +CONFIG_NR_CPUS=512 # CONFIG_PM=n CONFIG_SUSPEND=n From 717d5fc88e71fac35a68bbfc0a283c3aac83e319 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 31 Jul 2025 08:59:39 +0000 Subject: [PATCH 179/435] kernel: Enable BTF --- meta-dstack/recipes-kernel/linux/files/dstack.cfg | 8 ++++++++ meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend | 3 +++ 2 files changed, 11 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index b7352e7ae..85cdc1511 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -13,6 +13,14 @@ CONFIG_WIREGUARD=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_TMPFS_XATTR=y CONFIG_NR_CPUS=512 + +# BPF +CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_MODULES=y +CONFIG_BPF=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y + # CONFIG_PM=n CONFIG_SUSPEND=n diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index a2a9d5d1f..3dbf36b55 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -21,6 +21,9 @@ KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", KERNEL_FEATURES:append:tdx = " dstack-tdx.scc" +# Enable BTF +KERNEL_DEBUG = "True" + do_deploy:append() { install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config } From b9005254ac0f8892924120247fe2fd8b3096bf81 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 10 Jul 2025 08:49:04 +0000 Subject: [PATCH 180/435] Add perf in dev image --- meta-dstack/recipes-core/images/dstack-rootfs-dev.inc | 2 +- meta-dstack/recipes-kernel/linux/files/dstack.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index 6fe567f57..9ef676a84 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver" -EXTRA_IMAGE_FEATURES += "debug-tweaks" +EXTRA_IMAGE_FEATURES += "debug-tweaks tools-profile" diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index 85cdc1511..cf5038c0f 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -21,6 +21,7 @@ CONFIG_BPF=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y +CONFIG_KPROBES=y # CONFIG_PM=n CONFIG_SUSPEND=n From 27839fdb34848659f245b3ab25e4e3a9de30eded Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 6 Aug 2025 02:42:55 +0000 Subject: [PATCH 181/435] default gw prefix to "_tapp-address" --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index d8e9c0d08..2ad72d046 100755 --- a/build.sh +++ b/build.sh @@ -228,6 +228,7 @@ base_domain = "$GATEWAY_PUBLIC_DOMAIN" listen_addr = "$BIND_PUBLIC_IP" listen_port = $GATEWAY_SERVE_PORT agent_port = $AGENT_PORT +app_address_ns_prefix = "_tapp-address" EOF # dstack-vmm config From f6300f0112c1ea11167c9296fc23493848317a38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 14 Aug 2025 13:58:32 +0000 Subject: [PATCH 182/435] docker: Pin to core0 and enable pigz --- .../recipes-core/docker/docker-moby%.bbappend | 12 ++++++++++++ .../docker/files/docker.service.d_override.conf | 3 +++ .../recipes-core/images/dstack-rootfs-base.inc | 1 + 3 files changed, 16 insertions(+) create mode 100644 meta-dstack/recipes-core/docker/files/docker.service.d_override.conf diff --git a/meta-dstack/recipes-core/docker/docker-moby%.bbappend b/meta-dstack/recipes-core/docker/docker-moby%.bbappend index 0f9dc8692..871cfbdf6 100644 --- a/meta-dstack/recipes-core/docker/docker-moby%.bbappend +++ b/meta-dstack/recipes-core/docker/docker-moby%.bbappend @@ -1 +1,13 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + SYSTEMD_SERVICE:${PN}:append = " docker.service" + +SRC_URI += "file://docker.service.d_override.conf" +FILES:${PN} += "${systemd_system_unitdir}/docker.service.d/override.conf" + +do_install:append() { + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir}/docker.service.d + install -m 0644 ${WORKDIR}/docker.service.d_override.conf ${D}${systemd_system_unitdir}/docker.service.d/override.conf + fi +} diff --git a/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf b/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf new file mode 100644 index 000000000..fecb5127d --- /dev/null +++ b/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf @@ -0,0 +1,3 @@ +[Service] +# Pin Docker daemon to CPU 0 for performance isolation +CPUAffinity=0 diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index c4743ed2a..2e4a55c25 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -25,6 +25,7 @@ IMAGE_INSTALL = "\ kernel-module-fuse \ fuse3 \ fuse3-utils \ + pigz \ " IMAGE_NAME_SUFFIX ?= "" From 473b4641e77a659805846c84da5e44d62570b837 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 14 Aug 2025 14:11:38 +0000 Subject: [PATCH 183/435] docker: Fix runtime warning nf_netfilter not found --- meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 0d2f4627e..861e3ec34 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,7 +1,7 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "DStack" -DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity" +DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.5.3" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index c4743ed2a..a33f0bf3d 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -23,6 +23,7 @@ IMAGE_INSTALL = "\ dstack-zfs \ kernel-module-tun \ kernel-module-fuse \ + kernel-module-br-netfilter \ fuse3 \ fuse3-utils \ " diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index b8fd02442..64a2bef3a 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -24,3 +24,8 @@ CONFIG_VETH=y CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y CONFIG_VXLAN=y CONFIG_CGROUP_BPF=y + +# BLK IO throttling support +CONFIG_BLK_CGROUP=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_DEV_THROTTLING_LOW=y From 1a27982f82e42bb4c33b4dc284cdd7b854461695 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 25 Aug 2025 07:17:05 +0000 Subject: [PATCH 184/435] Fix only one port map doesn't work in dstack.py --- scripts/bin/dstack.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index e00a44094..9d6f3067d 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -396,7 +396,8 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None with open(img_metadata_path, 'r') as f: img_metadata = json.load(f) - os_image_hash = open(os.path.join(image_path, 'digest.txt'), 'r').read().strip() + os_image_hash = open(os.path.join( + image_path, 'digest.txt'), 'r').read().strip() gen_vm_config(vm_dir, host_port, manifest, os_image_hash) mem_gb = manifest['memory'] // 1024 @@ -440,13 +441,13 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None f"hostfwd={protocol}:{bind_address}:{host_port}-:{vm_port}") cmd_args.extend([ '-device', 'virtio-net-pci,netdev=nic0_td', - '-netdev', f"user,id=nic0_td{','+','.join(port_args) if len(port_args) > 1 else ''}" + '-netdev', f"user,id=nic0_td{','+','.join(port_args) if len(port_args) > 0 else ''}" ]) # Handle GPUs - gpus_cfg = manifest.get('gpus', {}) - gpus = gpus_cfg.get('gpus', []) - bridges = gpus_cfg.get('bridges', []) + gpus_cfg = manifest.get('gpus') or {} + gpus = gpus_cfg.get('gpus') or [] + bridges = gpus_cfg.get('bridges') or [] dev_num = 1 hugepages = manifest.get('hugepages', False) if hugepages: @@ -528,7 +529,8 @@ def run_instance(self, vm_dir: str, host_port: int, imgdir: Optional[str] = None numa_node = numa_node_of_device(gpus[0]['slot']) else: numa_node = 0 - cpus = open(f'/sys/devices/system/node/node{numa_node}/cpulist').read().strip() + cpus = open( + f'/sys/devices/system/node/node{numa_node}/cpulist').read().strip() base_args = ['taskset', '-c', cpus] + base_args cmd = base_args + cmd_args print(" \n".join(cmd)) From 9521d1756f7ba2f6566fb12b54cfaae02810526b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Sep 2025 01:47:58 +0000 Subject: [PATCH 185/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 39800da36..3e4e462ca 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 39800da36a300f5fdd6a9e908f75a899a4a4b8e3 +Subproject commit 3e4e462cac2a57c204698d2443d252d13e75cd29 From 7a8491d17a90a69dc0f03deff5af0902c8c0083b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Sep 2025 01:48:28 +0000 Subject: [PATCH 186/435] Bump version to 0.5.4 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 861e3ec34..47e154c0a 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.3" +DISTRO_VERSION = "0.5.4" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 0c4b61318798aaf846b7bfd3d727344882858880 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Sep 2025 08:20:15 +0000 Subject: [PATCH 187/435] Don't re-generate wg key in build.sh cfg --- build.sh | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index 2ad72d046..c755a2976 100755 --- a/build.sh +++ b/build.sh @@ -43,11 +43,10 @@ check_config() { require_config() { -# Base port for RPC services -BASE_PORT=$(($RANDOM % 1000 * 10 + 10000)) -CID_POOL_START=$(($RANDOM % 1000 * 1000 + 20000)) -SUBNET_INDEX=$(($RANDOM % 240 + 10)) - + # Base port for RPC services + BASE_PORT=$(($RANDOM % 1000 * 10 + 10000)) + CID_POOL_START=$(($RANDOM % 1000 * 1000 + 20000)) + SUBNET_INDEX=$(($RANDOM % 240 + 10)) cat <build-config.sh.tpl # DNS domain of kms rpc and dstack-gateway rpc @@ -130,7 +129,32 @@ build_guest() { build_cfg() { echo "Building config files" - GATEWAY_WG_KEY=$(wg genkey) + if [ -f "gateway.toml" ]; then + echo "Reading existing WireGuard key from gateway.toml" + GATEWAY_WG_KEY=$(awk ' + /^\s*private_key\s*=/ { + # Remove leading whitespace and "private_key =" + gsub(/^\s*private_key\s*=\s*/, "") + # Remove quotes (both single and double) + gsub(/^["'"'"']|["'"'"']$/, "") + # Remove trailing whitespace and comments + gsub(/\s*(#.*)?$/, "") + if (length($0) > 0) { + print $0 + exit + } + } + ' gateway.toml) + + if [ -z "$GATEWAY_WG_KEY" ]; then + echo "Error: Could not read WireGuard key from existing gateway.toml" + exit 1 + fi + else + echo "Generating new WireGuard key" + GATEWAY_WG_KEY=$(wg genkey) + fi + GATEWAY_WG_PUBKEY=$(echo $GATEWAY_WG_KEY | wg pubkey) # kms cat <kms.toml From f7c795b76faa693f218e1c255007e3a68c541d79 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Sep 2025 08:50:34 +0000 Subject: [PATCH 188/435] Update pahole to 1.29 --- meta-dstack/recipes-core/pahole/pahole_1.25.bbappend | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 meta-dstack/recipes-core/pahole/pahole_1.25.bbappend diff --git a/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend b/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend new file mode 100644 index 000000000..9ebd86051 --- /dev/null +++ b/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend @@ -0,0 +1,2 @@ +# Update pahole to 1.29 for BTF reproducible build +SRCREV = "b9cc7963051b2099795129450f9b70c81950d02f" From 58240f365638b36d0939fad116d3acab002ebfd4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 5 Sep 2025 11:21:27 +0000 Subject: [PATCH 189/435] Add kernel module xt_mark --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 3b7a9cc3e..cdf65b55f 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -24,6 +24,8 @@ IMAGE_INSTALL = "\ kernel-module-tun \ kernel-module-fuse \ kernel-module-br-netfilter \ + kernel-module-xt-mark \ + kernel-module-xt-connmark \ fuse3 \ fuse3-utils \ pigz \ From 492d17fb8544e86bc52ddee907c00d898e4ff94a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 8 Sep 2025 01:55:21 +0000 Subject: [PATCH 190/435] Fix warnings for vconsole and autofs4 --- .../systemd/systemd/blacklist-autofs4.conf | 3 +++ .../recipes-core/systemd/systemd_%.bbappend | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf create mode 100644 meta-dstack/recipes-core/systemd/systemd_%.bbappend diff --git a/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf b/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf new file mode 100644 index 000000000..0b88b129f --- /dev/null +++ b/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf @@ -0,0 +1,3 @@ +# Disable autofs4 module loading +blacklist autofs4 +install autofs4 /bin/true diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend new file mode 100644 index 000000000..8c49ea4ec --- /dev/null +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -0,0 +1,16 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" + +SRC_URI += "file://blacklist-autofs4.conf" + +do_install:append() { + # Disable systemd-vconsole-setup.service + rm -f ${D}${systemd_system_unitdir}/sysinit.target.wants/systemd-vconsole-setup.service + + # Install modprobe blacklist for autofs4 + install -d ${D}${sysconfdir}/modprobe.d + install -m 0644 ${WORKDIR}/blacklist-autofs4.conf ${D}${sysconfdir}/modprobe.d/ +} + +SYSTEMD_SERVICE:${PN}-vconsole-setup = "" + +FILES:${PN} += "${sysconfdir}/modprobe.d/blacklist-autofs4.conf" From 4cd65036edac6cf8bc14a8c8392d437c5ea30558 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 8 Sep 2025 02:06:25 +0000 Subject: [PATCH 191/435] Don't make mr images by default --- mkimage.sh | 5 -- scripts/bin/mk-image-mr.sh | 109 +++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 5 deletions(-) create mode 100755 scripts/bin/mk-image-mr.sh diff --git a/mkimage.sh b/mkimage.sh index 415f6f298..3d36ccafe 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -98,10 +98,5 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then rm -rf ${IMAGE_TAR} ${IMAGE_TAR_MR} ${IMAGE_TAR_NO_ROOTFS} echo "Archiving the output directory to ${IMAGE_TAR}" (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${IMAGE_TAR} $(basename $OUTPUT_DIR)) - - echo "Creating archive without rootfs files to ${IMAGE_TAR_NO_ROOTFS} -> ${IMAGE_TAR_MR}" - echo tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_NO_ROOTFS} --exclude="rootfs.*" - tar -C "${OUTPUT_DIR}" -czvf ${IMAGE_TAR_MR} --exclude="rootfs.*" . - ln -sf $(basename ${IMAGE_TAR_MR}) ${IMAGE_TAR_NO_ROOTFS} echo fi diff --git a/scripts/bin/mk-image-mr.sh b/scripts/bin/mk-image-mr.sh new file mode 100755 index 000000000..b84cd741f --- /dev/null +++ b/scripts/bin/mk-image-mr.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -euo pipefail + +# Function to display usage +usage() { + echo "Usage: $0 " + echo "Example: $0 https://github.com/nearai/private-ml-sdk/releases/download/v0.5.3.1/dstack-nvidia-0.5.3.1.tar.gz" + echo "Example: $0 /path/to/local/file.tar.gz" + exit 1 +} + +# Check if argument is provided +if [ $# -ne 1 ]; then + usage +fi + +INPUT="$1" +TEMP_DIR=$(mktemp -d) +EXTRACT_DIR="$TEMP_DIR/extracted" + +# Cleanup function +cleanup() { + echo "Cleaning up temporary directory: $TEMP_DIR" + rm -rf "$TEMP_DIR" +} +trap cleanup EXIT + +echo "Working directory: $TEMP_DIR" + +# Download or copy the file +if [[ "$INPUT" =~ ^https?:// ]]; then + echo "Downloading from URL: $INPUT" + ARCHIVE_FILE="$TEMP_DIR/archive.tar.gz" + if command -v curl >/dev/null 2>&1; then + curl -L -o "$ARCHIVE_FILE" "$INPUT" + elif command -v wget >/dev/null 2>&1; then + wget -O "$ARCHIVE_FILE" "$INPUT" + else + echo "Error: Neither curl nor wget is available for downloading" + exit 1 + fi +else + echo "Using local file: $INPUT" + if [ ! -f "$INPUT" ]; then + echo "Error: Local file does not exist: $INPUT" + exit 1 + fi + ARCHIVE_FILE="$INPUT" +fi + +# Create extraction directory +mkdir -p "$EXTRACT_DIR" + +# Extract the archive +echo "Extracting archive to: $EXTRACT_DIR" +tar -xzf "$ARCHIVE_FILE" -C "$EXTRACT_DIR" + +# Find and read the digest +DIGEST_FILE=$(find "$EXTRACT_DIR" -name "digest.txt" -type f | head -1) +if [ -z "$DIGEST_FILE" ]; then + echo "Error: digest.txt file not found in the extracted archive" + exit 1 +fi + +DIGEST=$(cat "$DIGEST_FILE" | tr -d '\n\r' | sed 's/[^a-zA-Z0-9]//g') +if [ -z "$DIGEST" ]; then + echo "Error: Could not read digest from $DIGEST_FILE" + exit 1 +fi + +echo "Found digest: $DIGEST" + +# Remove rootfs file(s) +echo "Removing rootfs files..." +find "$EXTRACT_DIR" -name "rootfs*" -type f -delete +REMOVED_COUNT=$(find "$EXTRACT_DIR" -name "rootfs*" -type f 2>/dev/null | wc -l) +if [ $REMOVED_COUNT -eq 0 ]; then + echo "Rootfs files removed successfully" +else + echo "Warning: Some rootfs files may still exist" +fi + +# Create flattened structure in a new directory +FLATTEN_DIR="$TEMP_DIR/flattened" +mkdir -p "$FLATTEN_DIR" + +echo "Flattening directory structure..." +# Find all files (not directories) and copy them to the flattened directory +find "$EXTRACT_DIR" -type f -exec cp {} "$FLATTEN_DIR/" \; + +# Count files for verification +FILE_COUNT=$(find "$FLATTEN_DIR" -type f | wc -l) +echo "Flattened $FILE_COUNT files" + +# Create the final archive with the digest-based name +OUTPUT_FILE="mr_${DIGEST}.tar.gz" +echo "Creating final archive: $OUTPUT_FILE" + +# Change to the flattened directory and create archive without directory structure +cd "$FLATTEN_DIR" +tar -czf "../$OUTPUT_FILE" * +cd - >/dev/null + +# Move the final file to the current working directory +mv "$TEMP_DIR/$OUTPUT_FILE" "./$OUTPUT_FILE" + +echo "Successfully created: $OUTPUT_FILE" +echo "Archive contains $FILE_COUNT files with flattened structure" From bfc9c916a5520d9236cca7bb24b4ab1d23b7c534 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 25 Sep 2025 01:53:41 +0000 Subject: [PATCH 192/435] Replace kvin.wang with dstack.org --- build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index c755a2976..054c59f34 100755 --- a/build.sh +++ b/build.sh @@ -50,10 +50,10 @@ require_config() { cat <build-config.sh.tpl # DNS domain of kms rpc and dstack-gateway rpc -# *.1022.kvin.wang resolves to 10.0.2.2 which is the IP of the host system +# *.1022.dstack.org resolves to 10.0.2.2 which is the IP of the host system # from CVMs point of view -KMS_DOMAIN=kms.1022.kvin.wang -GATEWAY_DOMAIN=gateway.1022.kvin.wang +KMS_DOMAIN=kms.1022.dstack.org +GATEWAY_DOMAIN=gateway.1022.dstack.org # CIDs allocated to VMs start from this number of type unsigned int32 VMM_CID_POOL_START=$CID_POOL_START @@ -80,7 +80,7 @@ GATEWAY_KEY=$CERBOT_WORKDIR/live/key.pem BIND_PUBLIC_IP=0.0.0.0 -GATEWAY_PUBLIC_DOMAIN=app.kvin.wang +GATEWAY_PUBLIC_DOMAIN= # for certbot CERTBOT_ENABLED=false From ceb69c8020804aca27b9225466a83ec0d50c2f18 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 25 Sep 2025 01:55:50 +0000 Subject: [PATCH 193/435] Enabled cgroups v2 --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 3d36ccafe..7bb70facc 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -67,7 +67,7 @@ $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity GIT_REVISION=$(git rev-parse HEAD) echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" -KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 net.ifnames=0 biosdevname=0" +KARG0="console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0" KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" KARG2="dstack.rootfs_hash=$ROOT_HASH dstack.rootfs_size=$DATA_SIZE" From 9139a726a1d826bb88b7f29d3e96a9388992ae76 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 25 Sep 2025 02:05:26 +0000 Subject: [PATCH 194/435] mkimg: Allow running in non-git dir --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 3d36ccafe..8f0335f03 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -64,7 +64,7 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity -GIT_REVISION=$(git rev-parse HEAD) +GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" KARG0="console=ttyS0 init=/init panic=1 systemd.unified_cgroup_hierarchy=0 net.ifnames=0 biosdevname=0" From 046b05f83369f373d616955280b6dafa4b499c8b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 25 Sep 2025 02:09:00 +0000 Subject: [PATCH 195/435] Sync dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 3e4e462ca..8fa306864 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 3e4e462cac2a57c204698d2443d252d13e75cd29 +Subproject commit 8fa30686499c529c0b0d3e883105261e8f1b4f06 From c6dc59f2ab234996af9ae3f96e3326112c313a69 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 26 Sep 2025 01:25:58 +0000 Subject: [PATCH 196/435] add package e2fsprogs --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index cdf65b55f..975552fe5 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -29,6 +29,7 @@ IMAGE_INSTALL = "\ fuse3 \ fuse3-utils \ pigz \ + e2fsprogs \ " IMAGE_NAME_SUFFIX ?= "" From 3fca266feef9036adabf20bb12af8f037acf9bc6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 26 Sep 2025 03:48:54 +0000 Subject: [PATCH 197/435] Add cmd resize2fs --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 975552fe5..ed5eccd43 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -30,6 +30,7 @@ IMAGE_INSTALL = "\ fuse3-utils \ pigz \ e2fsprogs \ + e2fsprogs-resize2fs \ " IMAGE_NAME_SUFFIX ?= "" From 394dd6f50e955bea510e4e1ce38e52d8f7f69d78 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 26 Sep 2025 10:10:34 +0000 Subject: [PATCH 198/435] Add back wg-checker service --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 1f571de99..687a4a369 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -18,7 +18,7 @@ DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" -DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service" +DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -55,6 +55,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${S}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf @@ -69,6 +70,7 @@ do_install() { install -m 0644 ${S}/basefiles/dstack-guest-agent.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d fi } From f4df0f2f75908776f24e690cc4473ece0481ea38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 26 Sep 2025 10:13:13 +0000 Subject: [PATCH 199/435] Add vim to dev image --- meta-dstack/recipes-core/images/dstack-rootfs-dev.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index 9ef676a84..1996e0756 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc -IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver" +IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver vim" EXTRA_IMAGE_FEATURES += "debug-tweaks tools-profile" From c924eeb8e842bcd1966c6e8c21eb16f9188c918a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 30 Sep 2025 14:21:09 +0000 Subject: [PATCH 200/435] Update stack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 8fa306864..f21e23e26 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 8fa30686499c529c0b0d3e883105261e8f1b4f06 +Subproject commit f21e23e26c54d29f2a6f1f73d7298ae295775d38 From 4539d1669d59632606900b95f8bf35fd862b6fba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 30 Sep 2025 14:22:59 +0000 Subject: [PATCH 201/435] v0.5.5 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 47e154c0a..efabbc7de 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.4" +DISTRO_VERSION = "0.5.5" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 36e39020213df75f90826a296949cbb0c5a9e3c5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 9 Oct 2025 04:27:04 +0000 Subject: [PATCH 202/435] Add dep conf dstack-guest-agent.conf for docker --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index f21e23e26..97397ec16 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f21e23e26c54d29f2a6f1f73d7298ae295775d38 +Subproject commit 97397ec16d2ecf9d7445d0e69e6248d87bec524c diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 687a4a369..956faf059 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -72,5 +72,9 @@ do_install() { install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d + install -d ${D}${sysconfdir}/systemd/system/docker.service.d + install -m 0644 ${S}/basefiles/docker.service.d/dstack-guest-agent.conf ${D}${sysconfdir}/systemd/system/docker.service.d/ fi } + +FILES:${PN} += "${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf" From 896b642e2649a113166dbf1d4ee99c1e0fb45725 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 10 Oct 2025 08:02:14 +0000 Subject: [PATCH 203/435] Enable XFS --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend | 1 + 2 files changed, 2 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index ed5eccd43..4f4acc1bc 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -29,6 +29,7 @@ IMAGE_INSTALL = "\ fuse3 \ fuse3-utils \ pigz \ + xfsprogs \ e2fsprogs \ e2fsprogs-resize2fs \ " diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 3dbf36b55..22f4e1377 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -13,6 +13,7 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ features/netfilter/netfilter.scc \ features/fuse/fuse.scc \ + features/xfs/xfs.scc \ cfg/fs/squashfs.scc \ dstack-docker.scc \ dstack.scc" From 9932ef470cd648c46f6c01156586a161a78f4308 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Oct 2025 03:27:28 +0000 Subject: [PATCH 204/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 97397ec16..7bf1843a8 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 97397ec16d2ecf9d7445d0e69e6248d87bec524c +Subproject commit 7bf1843a8ddf877fbaebb9898a7f27a19a49ab73 From e45c87679c601160f6ccb5a6f10c8be06938969e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 4 Dec 2025 01:36:51 +0000 Subject: [PATCH 205/435] Add images with nvidia GPU-TEE support --- Makefile | 2 +- meta-dstack/conf/distro/dstack.conf | 2 + .../recipes-core/dstack-guest/dstack-guest.bb | 7 +- .../files/docker-daemon-nvidia.json | 13 ++ .../images/dstack-nvidia-dev-rootfs.bb | 2 + .../images/dstack-nvidia-rootfs.bb | 3 + .../images/dstack-rootfs-nvidia.inc | 12 ++ meta-nvidia/README.md | 138 ++++++++++++++++++ meta-nvidia/artwork/loaded_modules.png | Bin 0 -> 415525 bytes meta-nvidia/conf/layer.conf | 13 ++ .../custom-licenses/NVIDIA-Proprietary | 46 ++++++ .../containerd-config_1.0.0.bb | 13 ++ .../containerd-config/files/config.toml | 18 +++ .../ldconfig-compatibility-symlink_1.0.0.bb | 22 +++ .../libnvidia-container.inc | 42 ++++++ .../libnvidia-container/0001-build-fix.patch | 109 ++++++++++++++ .../libnvidia-container/0002-secomp-fix.patch | 13 ++ ...-fix-remove-buildpath-for-package-qa.patch | 24 +++ .../libnvidia-container_1.00.bb | 65 +++++++++ .../libnvidia-container/libtirpc134_1.3.4.bb | 31 ++++ ...ix-cgo-LDFLAGS-for-go-1.21-and-later.patch | 27 ++++ .../files/config.toml | 26 ++++ .../nvidia-container-toolkit.inc | 18 +++ .../nvidia-container-toolkit_1.00.bb | 54 +++++++ .../nvidia/files/nvidia-persistenced.service | 14 ++ .../recipes-graphics/nvidia/files/nvidia.conf | 1 + .../nvidia/libnvidia-nscq_570.172.08.bb | 39 +++++ .../nvidia/nvidia-fabricmanager_570.172.08.bb | 86 +++++++++++ .../nvidia/nvidia-kernel-module.inc | 16 ++ .../recipes-graphics/nvidia/nvidia-libs.inc | 76 ++++++++++ .../nvidia/nvidia-modprobe-config_1.0.bb | 16 ++ .../nvidia/nvidia-persistenced_1.0.bb | 17 +++ .../nvidia/nvidia_570.172.08.bb | 24 +++ .../recipes-kernel/linux/files/nvidia.cfg | 3 + .../recipes-kernel/linux/files/nvidia.scc | 3 + .../linux/linux-yocto%.bbappend | 7 + repro-build/check.sh | 3 +- repro-build/repro-build.sh | 4 +- scripts/bin/dstack.py | 5 + scripts/bin/enable_vfio_passthrough.sh | 56 +++++++ scripts/bin/lsproc.py | 90 ------------ scripts/samples/Dockerfile | 42 ++++++ scripts/samples/docker-compose.yaml | 20 ++- setup.d/nvidia-layer.sh | 2 + 44 files changed, 1122 insertions(+), 102 deletions(-) create mode 100644 meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json create mode 100644 meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb create mode 100644 meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc create mode 100644 meta-nvidia/README.md create mode 100644 meta-nvidia/artwork/loaded_modules.png create mode 100644 meta-nvidia/conf/layer.conf create mode 100644 meta-nvidia/custom-licenses/NVIDIA-Proprietary create mode 100644 meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb create mode 100644 meta-nvidia/recipes-graphics/containerd-config/files/config.toml create mode 100644 meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb create mode 100644 meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch create mode 100644 meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml create mode 100644 meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc create mode 100644 meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service create mode 100644 meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf create mode 100644 meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb create mode 100644 meta-nvidia/recipes-kernel/linux/files/nvidia.cfg create mode 100644 meta-nvidia/recipes-kernel/linux/files/nvidia.scc create mode 100644 meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend create mode 100644 scripts/bin/enable_vfio_passthrough.sh delete mode 100755 scripts/bin/lsproc.py create mode 100644 scripts/samples/Dockerfile create mode 100644 setup.d/nvidia-layer.sh diff --git a/Makefile b/Makefile index d7d0625a2..2d0a05329 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ DIST_DIR ?= ${BB_BUILD_DIR}/dist export BB_BUILD_DIR export DIST_DIR -DIST_NAMES ?= dstack dstack-dev +DIST_NAMES ?= dstack dstack-dev dstack-nvidia dstack-nvidia-dev ROOTFS_IMAGE_NAMES = $(addsuffix -rootfs,${DIST_NAMES}) all: dist diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index efabbc7de..39042d489 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -14,6 +14,8 @@ PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" LINUX_KERNEL_TYPE = "tiny" +MACHINE_FEATURES += "numa" + VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 956faf059..93f6d6b27 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -29,7 +29,12 @@ inherit cargo_bin do_unpack() { mkdir -p ${S} rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ - cp ${THISDIR}/files/docker-daemon.json ${S}/ + + if ${@bb.utils.contains('IMAGE_INSTALL', 'nvidia-container-toolkit', 'true', 'false', d)}; then + cp ${THISDIR}/files/docker-daemon-nvidia.json ${S}/docker-daemon.json + else + cp ${THISDIR}/files/docker-daemon.json ${S}/docker-daemon.json + fi } # Force the configure task to run every time to detect source changes diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json new file mode 100644 index 000000000..7ed24bdfa --- /dev/null +++ b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json @@ -0,0 +1,13 @@ +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "100m", + "max-file": "10" + }, + "runtimes": { + "nvidia": { + "args": [], + "path": "nvidia-container-runtime" + } + } +} diff --git a/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb b/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb new file mode 100644 index 000000000..01afbf25c --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb @@ -0,0 +1,2 @@ +include dstack-rootfs-dev.inc +include dstack-rootfs-nvidia.inc diff --git a/meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb b/meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb new file mode 100644 index 000000000..a351ef920 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb @@ -0,0 +1,3 @@ +include dstack-rootfs-prod.inc +include dstack-rootfs-nvidia.inc + diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc new file mode 100644 index 000000000..3dc7883da --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -0,0 +1,12 @@ +NVIDIA_GROUP = "acpid \ + nvidia \ + nvidia-firmware \ + nvidia-persistenced \ + nvidia-fabricmanager \ + libnvidia-nscq \ + nvidia-container-toolkit \ + kernel-module-video \ + numactl \ +" +KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm video" +IMAGE_INSTALL:append = " ${NVIDIA_GROUP}" diff --git a/meta-nvidia/README.md b/meta-nvidia/README.md new file mode 100644 index 000000000..aaf3f25d2 --- /dev/null +++ b/meta-nvidia/README.md @@ -0,0 +1,138 @@ +# Overview + +The meta-nvidia layer provides support for NVIDIA graphics drivers and related components for Yocto Project-based distributions. This layer includes recipes for building the NVIDIA binary graphics driver, the GL Vendor-Neutral Dispatch library (libglvnd), and modifications to the mesa package to ensure compatibility with NVIDIA's proprietary drivers. + +![loaded-modules](artwork/loaded_modules.png) +# Contents + +- Configuration: The layer.conf file contains the necessary configurations for the layer, including BBPATH, BBFILES, and other essential settings. +- Custom Licenses: The custom-licenses directory contains custom license files that may be required by the recipes in this layer. +- Recipes: + - libglvnd: Provides the GL Vendor-Neutral Dispatch library. + - mesa: Contains modifications to the mesa package to ensure compatibility with NVIDIA's proprietary drivers. + - nvidia: Contains recipes for building NVIDIA's binary graphics driver and related components. + +# Key Features + +- libglvnd: The GL Vendor-Neutral Dispatch library allows multiple OpenGL implementations to coexist on the same system. +- NVIDIA Binary Graphics Driver: Provides support for NVIDIA GPUs, enabling hardware-accelerated graphics, CUDA support, and other NVIDIA-specific features. +- Mesa Modifications: Ensures that the open-source Mesa graphics library can coexist with NVIDIA's proprietary drivers. + +# Usage + +To use the meta-nvidia layer in your Yocto Project build: + +- Clone the meta-nvidia repository to your local machine. +- Add the path to the meta-nvidia layer to your bblayers.conf file. +- Include the desired recipes in your image or build them individually using bitbake. + +To use this layer, include it in your bblayers.conf and add the +following to your BSP, distro, or local config: + +```bash +DISTRO_FEATURES:append = " x11 opengl" +DISTRO_FEATURES:remove = " wayland" +IMAGE_INSTALL:append = " libxshmfence cmake" +IMAGE_INSTALL:append = " packagegroup-core-buildessential" +IMAGE_INSTALL:append = " acpid" +IMAGE_INSTALL:append = " nvidia" +PREFERRED_PROVIDER_virtual/libgl = "libglvnd" +PREFERRED_PROVIDER_virtual/libgles1 = "libglvnd" +PREFERRED_PROVIDER_virtual/libgles3 = "libglvnd" +PREFERRED_PROVIDER_virtual/egl = "libglvnd" +PREFERRED_PROVIDER_virtual/libgl-native = "mesa-native" +PREFERRED_PROVIDER_virtual/nativesdk-libgl = "nativesdk-mesa-gl" +PREFERRED_PROVIDER_virtual/mesa = "libglvnd" +KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm" +XSERVER = " \ + ${XSERVER_X86_BASE} \ + ${XSERVER_X86_EXT} \ + ${XSERVER_X86_MODESETTING} \ + nvidia" +``` + +# Testing nvidia-container-toolkit and GPU Workloads + +- For testing nvidia-container-toolkit inside the container execute the following commands: + +```bash +sudo ctr images pull docker.io/nvidia/cuda:12.0.0-base-ubuntu20.04 + +sudo ctr run --rm --gpus 0 --runtime io.containerd.runc.v1 --privileged docker.io/nvidia/cuda:12.0.0-base-ubuntu20.04 nvidia-smi nvidia-smi +``` + +- For testing nvidia-container-toolkit with k3s + +```bash +cat <#{{vQX4Ihh%`SlZitR=2b@Lr}2)%+B>$&CKC5J1aZKXEq*wc5Z$Sj?ZEeYU)&b znm7mupAlrlzpA@u9IXP~KWgCpK6@Uq^UMzw`!iq8*=UTcdP4YT)1kG8OXR{Nt3Bq_c;+i>fE|e|M#< zYH~JP^q-ejd98!`U)nXxJo;Z-np%XE?jB(Mk7Fhm_caCnZ@1*jqQ-w2^Z(sWGn0%P z?*DiT?BwDNg8oZa|L+5r~EI#d3wP!1Q^!pi`#=k;`(X2Knl6KF2r~Pu3IdEuJy2&pHC? zbvEVV_JLA~NX*ccrO3z#`EHx$qWFJoFsTgg#Ci&E&GKAGcoR zre6#;*?db6OIKA+3idf;T>%!WYGjzsiji0T1X% zl1%+v7v^Z4QE;~NAk>ztB=gr^40ql!l6{FdM}~qiI^!%~_zUYH!|A)#j{`@=M4NF} z)>GVUm4n?6zBMzYxR^#IYd-{vh5JjE(^V%qln5-$s`{(2yc%0lK>wSTI*@CJ_8gk~xNIoeKa^J4C zFh*%u?Yr|iS*;4HQ-_Ylz|><6x#ZzF>Leq}0?<4uWZZi7Yx&Nox7iurJYPkV?=TG2 zPW4uE+*4hvo>hI4+`_E1SgWBL6{yeCS!Sib*Ui4y6$Gy?CX@~v+!;swh(*kAw$-Lt zaLE{nwO4Qx8^A<)~YZ+m4IK zieY`epoLc4rjqQDxVqwI$4b^8P5Gl>SgRDqix>vW$1`W`nD*E;RKDh(VEqZFbVmB5kc3=}!#w^t;Rw#|OPA!B^8#JT*f zRV0gZc(r)(PiBd(wr=;V?Dts-e-EhJ(_^-yf>BGYT?ks)Y%hA^y+1ra+ zO->Ct6sISDrk8<^QUit4&1aPi}8$bo^q$+u+0 zmD}TTK@mp+Hi_+h`mRh5_b#u<18s8Xm-!>uMTzL69_jXM=i|Px<12e_F!ADYr>W@* z(-3=8G~kSk{PDnP)HX}(|D73{!K$jFTWtQnq$(y8I^=oxejGRR^vQyoWRr!Za@h5?rGTc&KAy8gh5%=-WX=wVK3KLs>z0iSzwhGj&*LviTgPKR(JCz#7p9)llwjuI$Yah@ zp3zpWx-S^1EFbRd5zQ))JPd!TitL9-&htm{E2GXkUoW2*^lS(`M#W#p44E0~|i1}r4HfY9*gy3p0 zU63`${6iZKeaR1_%M8H?O(EdLL+JQ|oU|!)oc741aECRiKHjNC@2>g^R(DKt3FL~)33sJ6L3gKF zxSgo!l=RF_X$s^-P(&s|dsI9A94kx%4UJ5lSq69&TwJIKIj@|J;=>cMMf>Sk*h>^O z^Q_S(0fOBeEEQYEZf{05BiwG-j+2Wm0KI;BD3cqBcm(a&9FKS}% zH@eBprrJQ$?RjY7aj{;8yBoa%?)R`B@2WCo$AyFys1VI0m)SU|-DMBGF=Kdj?`}U+ zCs5>p8Kwl2`GpoPWozeQ!YMcB-9ygfK9l^o_Y%H@{*i&`@^*tc#tO(Z44Xj@vKf5o zUi|_83~cg_Ucwl>vo}N|au5YG(G)Wpa;7czNiehde)ZvE!{HmC$<`nz3ie74U*-w^ zV8@o{`^|FI+~xfCM>9sOXziZGO*^$@&#ZvMZG9f{j6czEbJ^CB-hw;`O9b{zc9jpk zHV%?}#-l4+TfE#=x!kyN@^VpB?vEy_Rdu;&)2CC%ku?XVXOc$wupS%z5g?J=&0+ge z)Ycsh;YL_kiha>pL?-QJUluj-f{5AakFjlx=Obcs!yVl1_F! zyfO-Qs?_gfw9he2)_h&&m(-QZbpK1?P(y5aYO3{6cK6BPD263DNZHrD!St4rZyJc_Yssc4{mAq84=QxX}amHJd2}NX8FNXSb+`7z89o|#PO0=#`X8KiZ zZ%1~IfG;}^u??62@)YokXSVWwGI;{xz;OSX4TVc

(n3)$fD#6(TGzSol#9KKqB@ zsoatJvE;mKbxUh=q-90u-zaI8B=VaL$$d55ochE>_|w>T+TXBHkf-1*Eg3Qf2nZL&y@EDR7)Sg+6e%^L7ybu%BCzF1|vfIGG z>3UnrZ(`-||62e4NjzSdAQiS$6JxTcCZ|oRSY_|lB=9rAUAaWi6n~XohXaW{rq;Mi zIHRL9QLD+Tr7ob!m1p{k8-zP-98H$CCRvF?a-3MMxyhf;oQ|yHjK#8g|13gwNHkL* zcz!$*31JQJo_`K@%jIM!{Sn8pqEeLPE>7_@xE2#6WtB%O%R#9+uX+v|X5fN~isp~Gz*bYrdJvuGDZ&`8odMZT0fgG=yFB%UMge8FL>w}LpvVDN zJ-&}>Q%?$fIdb6s)MLfLt@+YN>=0IG-$gC&?s7}rL8#RC~v7q@~ z4{`Vxr*XQ?<4()y%6|AIC_@F)V}TE{wc?wd9;27B_{48)IOas&F9IEUa2|N#b%whS z7@a8x|15>?u5C5dD$c`;rrp;(4tbwD-2au=@hW{7tU^nKJ;y_ZkNi<{dY{J%ehu2l zbG%=1W>k{*@)I5nVBKC)RqD=g@Mp>%U+-v|b7F>uDN}X{otQr`+)P5mGz6=c30r311qzShFm zz|Z5$n@GNqBtBnmy|%aCw&zG&^J}A2(y8Z0-s9IN082}VvI%mlkP}1hyFCZ?@hhR= z&VDF#HB{^CGI5&8o$INnc?HV}dV}l$4)km*z48oUqTS$qBH8*Se(@}E9|&5mGz1kIT7?}!y_<`+UPztz@GRbc9+fdH#gl&VR(joxxDCI$8R> zt`{TBqGL|H+$y;hcH`!WLJcF>i`ttrr{gzF7&zr;E$q}%BkU!&2|i4y?B7<4EL$A%$;w! z26$0FQ?xfn*G&+VPp|8~@CtBbe(PIpc?9*ZqNKGq2C~r1bt9ObWcDs(IZGehyh$*R z%ylM#369^PGaqaJ$pAu42CLjd`YLqmc*dEtx+)bIA=BO-0su^KhJ7OqFJH^B%J{p? zEiH`C^*}IubuF6+-k5I`N_Hn7hd`K`9P?{ux%$?rPc`qI5Vu&2XGVzFy}(~}%tn_q zsG|$t#}J%8roz&Gcs(S#-CnT-uQf5?-2x7DMkSHOD>YXM9TIvW_Hga8H7Rs>!4fM2qtijftS;`v<7^JwDQs)-cZ`uBeJn z>6!=Y*6!oLJF)h6Eje|E>6yohe37VP7vw7|YtyaHI`nPe+laccES7VdugQkE6+5Pz z+kc>o#nXq2S(Py0)U?8x>PSyDJT?Z;ka6vG2@dU#m+VH6VOE3EwI37nQ)_xz>P0%p zQTWieG|_UgrRwplh$_}$G{ZVXt>HDEhW`?)&4IJvlldGZEb>UNm+A0FjhR5&o!*r0 zl-tR6tnFrGjc_Hu?xw0-#1~_r$MVu#`l*!g#vxTgbD>(pt51vwAJt*9fma=wHCS$+ zb8_sR*|H7u@#mJ7lFYJ9_xqpq!9A`m#yLx;O}8#x?yWltF38I0uer&30((VRud?{v z7w(p24LbWg(Vx#N^~0(V!U4Qu!X80h+3tz+S(YXVf6;Q*mmio7Gv=~$UUjR!03h~E zFlzF{??WZGeDkazYJ^_xILVv#kqWAq$sq%V0I|QbP)CI}war^sV~F_25Jar;jK5P; z2{NPA)nx`-OB(4!hdWVPAG9>R+nqjDHpo}$M}pFQb14U#+Zfrg+0RLY?>ER>xBt;o z!IBjt-h?N3v`hf87N(UnAWP3U;WNmRAsfSM1UanAtT?+&ov6nd(mvMACS;*sp7UA> z0okO`!Oq=^(3yMeyY$w#U77QZ0nZ94aKFw@Y`V{dK-Iy`zK|WAhmf1wy)~>|AG2Q= zIyCQGQt!C?;JMN4{wvwdmZSWs@fsPI62tyjuF_cHE5q##foSn(IZg`#)H|T#*TE#S z%FdSO+flw71v2<7Pv+)`#9`5M$DQ!Ml0gpKnmTJkG+_hMy-@<6l7rlrD7hg&j(7na z%pkLF(8BM!%vo%w0egj_gJQMv_zBj&6jYJW)DX!)taFEF%_Hv2HYnYT!nDs{A;M(s zmH#pc2X=)+z{qHk#8KC zf<3N$6@3oDLSURkjR|(8+P{NX3%-S~DFu(Q)1Mc_`hK6ddvfMDIxF#J354}~J28NL z(*r|BQMDcjCwAsE)!oGHs&l!UMoGWHJS9hx^Y3nAuEZq)6syIA7@jx9pyafqAgSKu zJ|zY7+dp1+Ck|^G3YkOm!CJ#??T^bG^~0ENaL!P9&c~;YiTQA22CLCr1+EIlN0~}C z3v+IH%x~LM^$A==oI%NTJ+#kBD@~{GNdJC}HDGDjz7Vg{9-TYx=LPd2w+ECEu0?Vq zc}+F?B9!IjeB@Qj(-LA1?KqD1JvJOr|_X2o+p$& zBurZAZgUS>QG?{56|Os9ELVhr1)i%tK|6PU9|f}?!BdM*rr@jx{wj20K#EAUB98lU zl-+A0CXlY2aCa3{^OtMC${0Yl>u5)*m(bN>qLYLKUSALlsRJC*AvO>qQLosn2B}b* zb>A?qB!Y%T_NF=vss;V}f$I5bs42`+HT%i9N8#<#Y4#d~m#txlkKJ!rX=$^#_Jy3O z*`#1b*N{RRf+ISBi2)(E;P6~3tWJ<3Xx)$!U#JZE2Ym^yC+m~zM10=uefOOWZ=Q`IAvivy+r{12nP0z`%*Mz>6mvAk z@t_gzM(EvcU(?(fHC-GAnPlU^`|Y_$#Wza>j+WaakWj~*?owrFwKV}RR3RlVbn-h=PI)0d$2B_aLQ0Fy!`aT}OAL^AD;!>P2? zDV@jB=17O1fkV1J3`AQzkFa#+M^}==bYaKcBiD=!C#R-=KUc)>RBKx>#Aus3m$E%c z3ALJ@2P%LfX1!IzEitH?kEI+AueYA&FI&a23zwo^aHaa_N8;Uq;_m9>M}Td!O+vvH z5>(+njXZm7vqevgl@?9dw*TxXP(x6Vk{_BMN&+^(>EBfXEZ>&YHo{B=R` zHYCPjawYS90A8ZCrs$4$)oY&VtZa{)nrvtA34ea7&;nb8oop1*UjwnNrBlB1-9h_1^*gA(LPQjc6D3n(#($NKRUA!vG9qw zj!)zW=jtfMjaXok4ZK!=NWU~&2wdJ@ts`)?LfJ4<++A8!BVKGwjDkE2)k2hF^^al> zWo~h**~=<ojp|)N%}RgbCs5xokQvd5ra^k&htIx(CkXr6EQy zpB2-6J+_#q2Zk)x+b)OxBsm_!qY5RBS^a8o$c?^RSDxZ_@g<2=X83ve(Q49e8~?=i zFbijhv{K3eXb+Hk6|nybJPr@g#-ph{T&*g{wNl22nhCvY_nvRF39s=;^I;Exp+b7@ z0a(RZP5zrCW1;)`+LJfV(Mc3oeVylAj&3J%@=s81RYc!aaqlQIdD8WWJX%^^G=XBl zp>jZ0T|Edw_NRO@CeqGW^vv?>uqG+vqX1}Se;Ta(rhQ+AluVUsFZFgDRi0QVIe=R) z_yInN$LRi-WG#CwqAYhdTa#P&a;JgH z?_pV)hx=LVu{G{+T}kAIAp}X02aIKC3{5){%$( zWGsYGKf%i4i|un7o}~9c^gC{3Hfv0gl{0!WZ|)S`EHta{UZmd6wALpNiSG*0jl_-- z+kb7_+F{<*ei`Vs+srpoI-54cLw`m5)IP!IizuJWciVmQzzbwMZ0e7P(fAw-5emUE zVI8vpLuLLoYqo-CGxj!K9!haiLIOpk0Vj_nnZx9GZMz`-dDyh`ifWnX(b4t%ng<_I zbU7X~Eox>W^O(Le3-Nob=o2Zb>c@!DWIJ|BzSngcnNlPdcr4h` zxNAV#cBk+(A~1o%O5=21&DXwiV>Z{bjM;<{7}XI&R4b*KtaCrY;en$0Nmuow9H~bZ zatJ#Ul3Q;?A686m0@ycOw)&7i4X-lfsxTap)xh*B8SHFemxe8x;5eO%cg6toz2pUn zxD)Y26OUz%cl=JbPP%xi%V_<#7NGmV%d+50oDClonT51<$P!DA+ZxS9Fv#v-EgIVFx`h3LL#m|{M5jy?QtJ-6^8i&-Ty7CIS_-|V~bu<8Nc(X_rw+2T7N_7a@EXY&}p#PEQdp^TNn^>1^3zD!x(9oDk*T3|GOVJHqt6>dq%V zwK)qq{YZZKOW|~I+@5s(r_3bJRJBIHGG1?swYgk+zM0dpyZTA?DxSoM*^O)x^M{zC z`s4V9HVtYOXJKO<#CM+%bf*a1Mm!RVNe#Xs5*fBcV~gLs?Jl)eM~I5-5V@EZ8M7bM z|0=gOntZb-a$t=}ylv#CDuW)q`~w{~l_~k2C*dfp zPahxHl@E3o+l2(e)iIR4Ezcl)-n>5ewtrsLLdP_G%BS4_x)(iO@%vceBSih|fL`=P zSP$5J#I+EncrS+4i+nxN6R!ARP|(MwM%vXVjzCP)a~Stqr@gt0 zt_d20mqf;_D!a^J`A@$P%Gr8%_66wc+bjg#S0O-1PY781o)5poby%}lk_tcmiLd@gOAgbhn<;1grGx<|B*2Gdu|N4JERoYRv2b+HF9DL z*Nfx^2^-=*++*aJH6 zF$dG@!qW2eZwwS;Wo9z?kYN;-{JpIrd9?}T=o=d|S58YINZHruUO8{G=JIXWDR$Ym z68?Tm*ivos>Bf5gqK9&BMb!#j#4X+yIUI|L-m)vMx~d>$awUTbK9Wd9IC{p@nKLY7 zZ6_QZPJ0(296^V>F)U9qB>Ocu?xh&zhjYg5tQljj*Id|p4uI53<` z;V4XS;rdZ8&jc(+>#t;}L9r zWVW`x?EHB2w+h7O<4C}a2m4NsYK9yh7E#RdbFoK=>xf$tv(%zgVD~jQ_?)BvSfIBl zlGXXHsagkHV}dMwERLnqNNvtPk#Gc~3G}--5Yl^c&Xu`$M6gW8`T@j0mzIH~tBcOI zXPZB6gA+rKTdpxCG%!*6oHMsC1;xyo^Q{X@fclOuy+fXHV9ElE+{Pj8n>8ZSrewW9 z>5Jn<0BByI)GnBRrNda@kom-Twb!76uPuy90BqV5ctB>5O+g3NcZAUlN}6v zAxum^GM~L-!_?-TxFy8cG>fkCt2w;J&7b%nK@de-v(ZIx%KhWk<(rG(5e8!T69)Cl z1M~9gi7lp7oFiz_^o7%@zCASk%A!^Hp6}kvtw!Pn}XE)SFDBaGvv+~ofVro-&66smchC;7*~piZTKDx5atsWK8dKinKP z74lOnS1x98W7MLQQ!75Tq?rU^I&oH2+@^=zsWte)5{#-;d#J6hTER1j(K6`HlFf(zzyqc zA4VfRi6GJ2%@T`-`*_Bw?S?WW2nopUi`t!(`s8qG#pmlZ(A;Lj4lp{bUHzC;W-bhg zi)yoRV>+74I-qKH?c)z{-;Zf(uIZ#fBTOyU;_w-$V-(*0Bd$d&^*HlTND{s4q8Uk4 zrre;*W|t;*P~&w4lGVU$`S}k=*~uAi?#=Q%O1o~lyl|R#oVcj;77CGtKD#P(ID8l6 zRX!-%vEcsym})&`V~Rm#sbv2*y=O>A}Ui$VSa zrOZk8{tHT##gEFSZ7p`%XyxMEAhU7WiLX|~#U%~olJwJBl@uF1nMR@D@uvSWgJH2qNjQ@a7Ce6go(Fbe~uCA@UunTl9rL9xwHA0F5;0Y z18SKtw^VT#)5!YyQTMpQrJKhD!Y2}v)}!+;I2c=GPXg_UibB| zFaJR;iE|_=U+Rt3D9TJ_POPH^BFi;6tp!51tvaV52l;cB2tC`)kxC_tOUnGMVfokn z#3B)V-6>T!DfsNo;Qf&`8_?BXZTq=P0obRWe+-S43FaD9uUrmvSpKfnZm~eU((0^6 zU4*`zxr2@`%W+@vdIApSaIc7QWuO-j_1|@H=p5^e=YoRW((9u>`J8 zzq7kq<)N+Mm*Hp5G-7_xN|D+$}@%KkNLrKWnxb z3*k|b#016)4~kTqb$3Uuj=sUuro=m<;vbglEY8G_$HHv6zGQfQ?S%|t(gBzfXFf8G zKg-fst6?*7CD!W$<_4rkhkj|Do{n@3No!AJHmpU*l zs!$)(^{feA1H`QYRsTkYSe<2P+PUP?aX0U}>&Mx;GF_YYBXEwnNb&&dSZdX~?HcU| z63Rr^lCLG5fX)@rw=Fwq60RBCXYYr3qrl=h%k#K7g8Rah?*F@96R4KR*g$=OKUui+ zO_}_cCf7qPiexHV2J@*l4^dHFhHz@w+*IhR5-w|Hjo#fTg^sF23p8P_vJWM%bgB2E zvq!lGG`iuFO_gRq(dpbfZ{9;?VH`rjuyo+XAXb3;^V0#N#_k#vr-YWNgIwS%Hnw;~MFSX?@pf-i_@B z44Hgx`k>C^XIvo#*%-aEQW8Jm*%0b(vEFr0yQlx&`KL$=`0F|^-Mlrvi8#_-8?Stn z2J^k^qP+HOMZtwSrk#LIrZ%VXsj`h*AeRxnqj3?9>q+_;krH>89%8+V;+W45x@;1r z%Mu15e5;%Fr&Jba`{&rBFu*%3@A8=f-^8>T7`NYsT9xBNHblF{okXCKbaQ6$%g^86 ztr`xbZvA>1x%D(5DD__;wgSq zm%^C+d)JlR&fh4!hGtR3K&;D}41?T-gbvi{D{!p?>PSaX(9L_ML6cQUG>J=Cz^U`3 znYKMd^D2698}bVm^8!>t-HQ|;Vs7WBTRwU1KG$#nMwymoZYaCD_2phKRBw`hrW<~w zrreHhv2QnJoVe&)hMKs1SpcmRvRtltdv%~u?6iZ>-ho~s`!xWYvd{Pn?Mt0Rpd6A9 z;M-O`cqyM=-V=-1)j&wwN*E$iW~+K ze)@H>Rz3@#+vjoouW$1%Dl^X-NC~oAH(0p)c5jZD^RFt=qo9n$KG4fNEuU$IG5wn8 z3uxBAX-i(Muy_NmX{yalSq?#!v-0;A2{Q@O#@>!7`TJiC~6e@g1M|s^)82Jw06>G z#l{X}j+<6HM&9pvDz@iUd`{2Qa$nyQ@CW~peAesHN+48T;4e(7j~sN(GBL=-p{Gae z!W|U;y5gN0fp^s5SXF9YXa^ed7Thj;&uM`#RJS(8P4jCf1k$3uxqMK8r{(T6Rl#d@Aeq3LYF|YwJXS*opNPb7rslTn1L{QD#r2{=!Dy z&f3alpV{tnkWP#g7zmDaDd=fGUwlhqj-c*}GEYj(A;RZt{IPRaVCH0V0%W37}f_^ed9lwK^{8sGzZ=t;!}R`9kG4;hNgQr6SlfN!xW?}OJh+N+X+ z@;=2QSapB6T6p_NvO3%)xxWX>fWoFtR*XSHZYuYB@GC#>e>l|Do-$_J3qiR>9?};O zU}gj8YMrJeP?<*{#rVrrAbf!-zC+kXpJPt^b$~Wl6 z515}6^r@}QJ|}%wu3BTxnxytp_YM=*_&XD_zu-95GS7{FB6ikOC(dz!%*N}eMf~XQ zzDP`7QJApdH>;XF(=^p6u){?9LOPMSB0bHC40=0q4JQGdE&nJ^X)M1L;Fxzi+7wqg zR+Ib~f@y00oI$9W{677cY8lq<3D!Xef{Xfz)Ro{SbK=dzdO0i zQ*Ch;_=bT%9c{aoX>F`mIuSfK%G6~l+z}BvebRDk+*EJjOTgw21UyIll}nv(LJSYr znuXVQ9vf!e8j*`U-SLu?wGNXe(R0B_#q6!8I+K2dWp@g1o40xGGYUl+YfN9(XGnbS z^{dgwQoLaT7{IZ@*+4A?#c_t)W?WO#Q^Pv76%m2+`FB_$iOw5{Zx$U_7|{uA^m!S&Q1d>(NT!pr(8YATWpIqGFdN=GK>5-Y0hQ39E7Hrq(m;ipn`M5My6TyGk+d z+k_uteUoC9RhXW3L%UzX#E_E@1Lane&5JC?bF#d1A0Ieo(C-Mnqx~mQFMg-~hl{*& zCO}#(cyvPYtoFo0>vFa>K^&2ZGT=~AeRjV!x~NR2U&GqNZAevlHY2WtE?BdDKI(Ox6RA;%S z3Z4#yo~OpZ;~;y@A`ToaX8pbjBZCcg=lzl1yc@cS_!jJB|KJ^al}85bZA6_#fb4fO zdJjFU%M06XlVwWxie|FGa-xKoV_@x-$aJ#eW?vk5w=N)sGPT8tnYoyF2JN-hMPQz; zl^Cl=*^c~|bQ2bbQtgCAG+z?Z-DB7Z)yll;b3AC&>|6-oJP>r2Lq)thP9tJ`%@gP0 z77$8FEgYQ^;I$z5EGLX^Cq6G=mwOXsV6j+6s)2;7T5b5n9LC)EcqSm{`;eSsiqY=L zwzI%!hsJn2Fw&>y3!MJ>04bY;Wb<#buuM9b=ZEsYqjk{8WUYgyZ4)Y~gc`S*)cVbCzLJro+3!_5Gdw_Sd*T2TiaP z%&q_AcD5Kmag@_r<93~$LByW+9*I`Ca^KZN!}syQyUHN#!SkxvS;9J7TF&@T>G@-e zoErS3u!5g8*pVs9@5d67mkKOOI?pWUhHc;^-tF{3ArF3oFCzdZ!?c46%Bh`D(y(@1mP*(sFuoJi-IZ|jVGmFe z*6r$gcLzl;JRliqFXYn5$iy!OzL4cR%1d`7G&lDyhg!=zF$Qx_Nxnj&cso0MruG>v zHiFycXR&hBA-;OOsc*5~TfG)bc)GMY=aahDkwSjaK(8zf!s8Pqrw(v!=t{P9<_^=B zW1u#Cr=9x})$k&d3_e$q{;0kDf2{)ANl4%pWlM19>e@}=7I{Fz4W&ES6Fhr2!q+F( z=EgKW2J$qRKbumm=+|d3biZ4`bR7MnKL{sMr=Ks0ly&;i&&!}uraEZAZQ;zLi>m-@E#N-k0DnZsT)SGfy-+JN1SD*I)>;x?;Fj5%kD$q_RD6fc0Mob z{P2UZ?ED}*?fl*XLcbTjLe7Ua-SL=Z>C}n&gVPUn3DIB5-Vws9KZpuUc@weU`(-1B zFI|r^42aMy&Jrtm$4Nn2obL{1z!UMI5Oa!qg8)&mcZ48`YOlxuAlX!_ceoBe!hqkw z4J?)H*3WjP*LY90qb6zkMT+Bd4+1U90T))cpKWdVk0qw=%7{GCUAkyVTocQAmv9pm zOADr`Bske8MG_@688_E$Ub&x-j!7?qnAa#Dw@S2dA)+GbCT2L&w0$+TyC&~EJQnV4 zAQgxz54V5&T?{$G8h@aL+dKDp!)XQJ9ph`p5QJjq4u>t@vfL{NJDA-umW7OXJN&Z} z*u7xlBz4pN7|5F*{YXL=;{SKPy0MKk`R;oD>yQK=@8_|V7rSI;r_}tlJD}ai$?aDo z+!4(A{iL%y1%c`o1%7$j&`E?h4vFB7Ri5ZB*6$&(6Y{Ow`SqV+#fU30C@Tj0U+O|F z#k9Pc&=@`?)w$mT_}`BCdlyiCD<>?uMlJ?0Tr4{4k8m|ly-4fLsh{GnnjNV^!xL?_ zl)11mC3O-qB4=QeKf;lt{V)PoCVmj^9G8fM3~tL`rXKO3h)FB#owVjQ&#N2to#A8* z)1x=gU!`l--j6=ksz7U-c}a-}n<$%B2*O>5LL)mMb%KAc^t4j*^7#@U**-ond;} z-N_HWtQ4WR7T@cM6S0hrUCEt-`3nk;EVV=5tqU)h=~1&;#^{poSB$)FKg}wkwYdd9 zJ(|mUYQGl&Za>e1qk3^2*6NXmjocX~d4Z?Osy))|;oO(QxWZIpKERA3>lh;xd%FBN zb}(ofuNKe#=Rrg1hQtT;vXG#U)(8cuHomDLAH+TwnU%Z`tfu@{-E(%h`4mmLM1tB| zeJQu1Y>tw;*q=sw`}%h04kgh~_6N5Nc)y_V6L4oqUQ$;vk`_he;c>O;m<4RQV(`A< z8!m(VzdGi$kLOie)Mk3kt$}E7-kp; z&0Iw7my7r1$;5SiAwOEhP~MxrRxV-v{`vTU@n+oHm9kum>SO$Zj^bfDQ54h(%q+Fa z)A3gWdKu0^Q?=E6<^A(t3w+w0mR5rIzkj5u{X~WfCIpozsdO_Gs&b@p55SpmlCg+4 zoU*zS4d+LnV^|m4D=dF2AN?qnoNCb?URPyP<*1OX@*j`X?oX`RUnq>WZ&oArfQQ<}GJG@4% z(tam)mb352{Cbf9*$VfvrV?S#@h{ z&CAWQT$EfRKKs!0@yx9}g|#X^E6f`$E6HTP%?ImIHK_jBA_=1{&h(!&Ah=_1KqNx` zv+HXna{rDvot>QZ)lMM=v{5&W(zQa{F^b$f?E5DsZyIf}zn=}^wrE1%&}l7F#wvC* zjnuSvFn{EPL}q&P{Z%|UjJvXb%`(dOW06xN^W~KT|9mfiC-K}yYK#LoCZie(Ue+S6 z{}i-8fz`gSd<}$ccV^wN0V5c2dNLo|rD=APLAP!Op4Ve%zT~rS9gLp4n#Ma;mZrK@ zZ#a}r?`f=Q`>4%(x7WtE^aTkg3S6Ha+ihe&V=M=n-{8K~mYgd>ZRIYR0QWCSl!5J@ zPD<0A76wBM%a!_20sA+FGKV?F=zg54*v?s>Wko+D1{p4ij#3-BCzeGo5fBU#>K6tC0?^@L z{s#p;+_<|O&0M4Mvc)s&@aRqP(&!^r$K6A%JNF=0!2B=1ewP=l(J{*k^xr1VNUnb#<=GGtV#e7I%0tN%_`w2;}4Zc+14|Z>x^r09I}_ z-~Xpod;i0#?~X99O(B!OoUt6?xh9TxdCHEWVBLFdsfAr}3U1;ru%-MLyVf$M7}P=H zYM@B;j}U1v<0rNy(Ia#d znjbk^m=b?#p!3$z%EuXDuP;DxhW?UtHbK_OrdtwP9@J0PZU6l8&j4;N1F>ITc%%WwU^yM)i-3rB{j4jWiT+hZ{gm&iO9~ z71?XNtHe-)^$8~Si^2ZYftgBjpTivQi)rQ38_B~&dupZHd9dUIsH2_6dRyg(QgXr9 z>`F%O7!Izb)hgyG4j(aa6h*`M#xnapWs*l8TKQc!C}QH-JCO{_$R0FtEyQ81{Q`!3 zQ=E21oIx{WIVc6~1&!pTOva%=H?Z$y#SLSumwXpBnVMVap_v9k1G-jCiyDTQm;}(~ zzQ|)6^>sEvo0DBvTU*5kKRYfWM1}OG&yf9Ax~Ipk6S~S?D&h3_m;2I}6Hds)?%oVY z!M!EzO1YmZKInb6m*t3B&Vbz?+P$bI^zr82JK>cxNv?J6q%zt@Mfgx>?=h&`-+3+G zOBE;Oaf!KNcD1@Aeimw2##3WMVsBSG5mkEeD2)vL-h~Lsc6Kr_zFZ5ScKaummtGO6 zj3ucT8da(uiv z<~bbnVb2ohY`;gzE7YhZ*^PW|^<>@k^tLITZ$g)wsSGy@W~bY|R6-tsn3J3-lH=oy zG3cv_SqW4w;3i>#TtLxmy0U(n7QeF_f5h!!XTJR+K?t(iO25Ag!XvM&{FL&6sg7Nl z!zbx`1V@PDTHnmY@n8r?IETv&lvo{$a%8zR2H%erWdyGwoi(#!g-uj#ty{AN)X{7Q zR}DPa|0cJ0nz&q1#dum0#&LMAhyo2CwdE1FN_6TcA3DfyoeJ9{#?A8NTvbOGe>q@# z%o~d_&pz#2s7xf<8g2=11V-8CYfyUuC6Bzjuzt&-oz3d$yQifEp;mLHjjVR|Yj3?@~RrQlLO{>g)g?iQY=Od@MM; zxX|p7<6BzDTvI^uHcb%Vqqqad&LgqNwl~GzvZIYQr#HwB$61;9E3xaj#PG%w^KY95 zgifHj&(n&Isgdv?cikPOHMWUsGHYbtx3y#-cy9_`XS_pO$dAM8yrxnhDw@K4wMAF}%S4VG*F2uNWM-^U21@L@N_{wRd2ACXe)e(8--n4{L4@?vm+yBljNFo$feAyw|_A z#^4KVtjRGa{Aer2lX7s+_UWEtUwD9>0q7y$H?0k6cm|{%p5c{DClihe<99Z<`Nler zEx7llwiUfTQGA!Q`6qR4#1IS}!nb-nX4BX!sK+cTp3)j?mh$n_(oDE29%_GV|MHwv zC|3I*e>8&%<`*V>m3Y`oGaf3O^3BwzOqarO&W1M=m z!nv;Dk1QT=W0IbM#KT1diVSw2dZyq>Wo%a;7CwEbYO25&_^I##5$e|@cmM^St>Ech z*gk7^cxfiPpfJ-AbFo+O^qFE1xvaDOno|5=7Dr0?plH~q2v!jSnm4O=?!=ApFYNCC zJh+|D&_3x5Z!MBM3hWcjdUA%V! zZF=}iooBczgKPcek{^E6XQ4j`&X>R5)ftAMUGz=yzDs>+0yimWsMPFrJpZ9F)l|wx z4!H??%3Ty4zl+AED~9#p(xIbnnm5&63chmPJ zh-p|#1Awh?-hSM1Wol?QQ_fnum@~N-8DZKVW?wN~lTs73Abvw-zwJfqI&aU$(8NjT zbI8M)1*%;%)u84HRVAG0^=u31nfhXMc(fVj%jn;8kZi0)y5) zRf>vG=WalY2wx(B*J)sNClxaXZ_pFBBj!gBSQRUY_ITWN?tSLr<{4`AXIiEim!vhM z(S|7lPtU+y9tWdiW{!W zB+T*~})jS7KZn2x!gdfC3(ksf!fY46}PX@l;m*>1MK zj;JtUCu)y~^UFO`f3N%6mPyZpnx1P)>0=0M?^w}$^u$0&2;2MMfjc9UUgFZ?ym3)1 zvod`|Cl7h?KW`HLo`PQ;PR8gcScK1JDLkUm*X)cm2~_j$_RMwF?MPpkD7$ri)%)5( zxfT+^1_f)w`63gzBP^#4k+n@TR3|G1mL|F+(K1d1mwu;s=rH|UABV;jye~3Y%Rz{3W8qy9SeQrOMZwBqYFPapJt7FTPXaV-l#H*Djg0&f|)W5G*kt zU)Y+gum&QO(&>wJ7%jFshI9S6KOi3?5}hTUL*`|DZt^@#gNje@IJ7iKd5OArm49rD zN?d6RD={MMbVKD!Qddip(17qtIjws#9j(e;Mpe~5_o-BZ9KB8pLbIt5)J`7DL1+<-1m(KD37Dbx}(>Y8WxnNEvY3t>`xRZ zWlw4I6cX0kxJ7d8t-8oZmMgy84)a+@FtWqkT{`0wNaO*_v=)pfgQi|c3Mx} z;;8VZP51I-yK+15+g5l`$V0)OW$FzxSqeJdoa}zkp-lo@p#*c5A|HNA*DaZ#HH)Dg zGTYoT@1;JU8o(3-jBXh0aoWD#Bua=nkfbJHlXdND^gM!{a$GwI6bjNY@V`!-Ht-&; zlyz&fqoCk|@W;U}V+Y|nT2 z2p{AG9-f5jfoqK9>?T=%tvZYAOvg;AuT{8%IOP0^;!8*+XKtE58n94Pv2EYDlUd;$ z=fr4d7}*ba%AQ2WAra%KUr}Nde!eNdHTtt!KHIA=Ghip`)mi>Dpf*q{K*0am>BpB^ zJ?CE-62Ab8vwvjC`fWQx+-?%X^jZY_rc>O|0|oAut8Ub#C`F<_t-c8CTFx#cwBJbr zAVNPbjkJ@FpZocMLvE*~R;{tL5Shm4a2bN>dI;`g-hoI$dk%xY!kaE`2cZWHX6I=EV+(B! z1mC=K`Zf6;yZZ@75YRRy8foO~wHUAL{&a!8@v4%rb_WTYIC6?xk#S{iS<9$3gmrx_F}xjFo+v#)=} zx1aLz8LZ@`v9@;91V+?fL!7#J&-|j)a@w$W9mqon0=~%)Y@l_H^o8epQY`3r*S2t) zPPJT#+&_WJZkI8;&wjGZA`66YjIm`&K+Dff9r_LXpPnj%Ba?u(Z8J#n@09v3%06X2 z*IN^HW(HT|bsc=+Kb?9ztG?+gNR+z7mfgr2Hv7J=JX>9Ot~LSb57zrkakSC=n#~^Z z^Cy`qRI64iPqQURy!}*J_MoGisl)p)Y!vMASY&Uo2W}C2KUi1-u`|y7cNJJN#`Fii*O7t9Ll5f z=DN8Qxb2lBtLE#w{Bj>49O<%abu+Z}Y@|Uta<2ny%mqqHUsx$vI28m6`}MWbETkpu z^tatuDvDNaibOiD{7b-mz&Ff>d>JRD3kVuC;+0vas+xYr-4r3EJ=0%l40U-kNB!)* zn$HU((@;;Gi@+C3IPTuDA#u0nb87+V^6y%8<58YDJ4<;2Zs0zT`0!IPwo_*TN0NV34W5DpYH_Qm&7)}gm0T; zW*n*el;Ndj3o)LtqY5yQdO_vl)k3P$)yUSpSKbyJJr`qn8wyL^hYKk_$l@lE>QDCt zGBzC@mRnDMbIGuY5bVA+mkH)k7vNn^vp-)G5syQNi!YcZHNIy!=Z~WAMpSxYL6ykA9UACJWdy$S%=b+(NBm zeVhpRGSJ)s-tgALta4gb9O{C-*4pgbzp%0n98eWNS z4W-M5y_@Z}62v!YD{YkP=X2t5&X^xeS3P^R(yF_$d=<8sc)E3S5UJ^mm_99Z4!IJZ zyt)W$;yF~e5nD z?SY~dvFy#omxqG-h-SjuR(fTiz>?bcBZsi;^1Vs~`1c^UjxSHAbcXV3M=$Id5+z`m z;@&ZUN@0G8w_tJXJtGGsc=f=+OK=D<_gSU3CCZkTCa>W;I<*|nGYj?H@WPX{T(0D1 zka1^{`R17$-5LI+|D1RuYSH{$RQ3fKT z;o?1HZ#QsWy|&Yb9{!36#1@eYi9)Tk`fg8ECD33%vQK^gi3vltngM0a56SVDIS&a5 ze#SgOgfxpz)wtXfB}^9^k`toab-YmZF4HjX*?j$RG?IG zn7nRAo?T3$lPjPo@;X;$T8YW+?e0%>LQcMAyd{OuWWC7Y73uWfuNjMJ>ovJdXNINr9xG5(w zY!ogQ8Ou|fQh6sb@T-_gr`qYB?BJ!77U;|qVegEn+|gbp#={OIGRPy-w9&})`4Sr# z^o988I^$};b>!XxVz+v0szvms&7+i?OgdpLB44kR*W&#bqNmZ$+hVsBzN;ecH1Z9B zWE=XXT#ZK@q9;bly&Z@GGu)Ah2vlU`UghXr?1XHspqh@?MejnM#C2rhPFDL+6be#v z@5huSVZ-S1)}HEAP!dcF=4e%`1qMyO6Z4I~mnKl=KJ{4Xvb!yH7QG<49pRLI3NTsQMUDgx$#w)B_#7sePR? zT^g(}@ZOFP8eYH5$;5MUs^0DwCb@E-ViP6nz>a{xw)| z@cE;N_Z=i-gB`GZw7k0wA?6m$O;lJ-2A}#8_}qyc&uIHIY7NcS3HTonu*x;3(JghR z1;u)q0!0OPmxg0J)%uOw z<_BvxG!Xg=Xa28;n7%-=#>?>6O7_=b)>OBqskLGMJj8$hU1G4C=lb-AA^!LF4ia8t zLZIMC{`Nml_n)&4(j5%h|A!|2KFmP+Kfn9;S%;N%dEx&)#-IPsIXK<_yx#w#!E!oF z{)YixELB!bYW}<5|2pwWBx?-)pN1G3G1dM*y2o5N(EafLGl?xMvPK-WhB>#{+twfj zLq6D4xvegBtv}NN72l2Cz@#1n&*3!`y4eo5GU?hPD+Q;^_&c0qlRrPs_0Wx0jf4}z zmGYkN#Y=J#)D#4q|Mrx*0-{;kMLx`wao3=|8k%lo4iI(lZO{pa+o?&Gq z@02@W?mT%9J&b;;1b!z~lm%e?qD~vzm$sDsRLOY+nNJt?_x$!A2$gD6F?~GP#_j@4 zBs=NNuNM9Fu3R>FdaG7a8>dyK9}6j$t&KYBI!|uwnEsLG|Fmw4H2_&7W#AcQS7np{ zJz%*+aa3Wk|K=XGuybNnk?nl`HgNZ+tB2fQlrBZjHe5tJV zJ=6b6oTRi&Zg>97y2BE~`0~8l^;y9%q=QeSkzhwwGxWcx^zYM-N76>ylqzAD5dl1R z-kHs?>{)*=eDE7^i0eo=sgz8>6DM53H83OnG8&$sErm&g@1*f#FAWZU+P|>W@aK}$ zqcun8xrJW#7dJx-OObzF`qTT0n%Z$^NzDU!sVQEFlarla5m`g%9Yz|rH6L@W{_9eEHAU2B5^lX!~NEZ~ycyvu@74}0Ri#<&} zPfNEAYeWe2k(`R0!=lfhk?XgUFlkt(O&If-qskXv)$fxd#6s16o+1T5RmrAS_{#c(zTIKJ}jJw?ke6? zsW+8-dyj?@3h46Zq|$-=u&{7X4MFKTS-fBjODtlV$2lw zZw2x9*X5GblQnTPTOOdLw>Uo&6SPOz2v@TK*?^V1{!Bk6N;nlwU#h*pj*X--Mjcud z&3S&tF+*e0jZ?-%Q>q#kI?-i3hx&6IozvN z5B!XHR#$Ve*RHH2T+^$UZ3Ard?z z6gDK=)Fp@)rIyW>JiO`Zd&x%IR7WjOenv4O|vu020cAbWVGXIp`E~_ihf?i`- zjif`@f;+d#y^+bor8~P2-#og#lC(;w@R@JtFmsl!b^(8F)8*@c(tPP;F)PkAtFwyz zaVv_W2^XdM^WPai7Jk}xy}Al3e9$|0hgUdi-)zA!9KD5Q`HkZvJTV?grj(#0whl2M zEZLs;EJ7M)1_JxURgsA599|$4s70YWdCh=?kGeiQtZqmt2&9q5{}Y8^1QCCRscHpu z1iJg%&ull>HtU^>fE9_=wL_whP@;1d3dBnKV5dDB~{Df#oa7j&u$Nffa160U(vjYHzxtF?w~?e z6`F47#r0@m%7YKSxgz}W!MjRcRWIL{1MCMnoLz9km?N#)>V&u5XdvkLC3D))xZTzA zIx|(^>4CG6;obNRq3ElO<^bB_K4SAao2Sf!*>Qggrq{J_L!f^OA@-mNSb$g zf*GPBiZ8xj`BA)b=dmxuL8Y?1TRjiWSD+M#%r6Y(MMV#$);WwjqxY`cI8VME(Xam1 zi3)ASr!DuIhEGvc!aetrEqIc|q%nqG4QVMp-a<%Sjp zT1DVX$F=^*_psluuzrlfsS>uOIV6!8RUrz%f|Q07pdZ^3V;o(%c1viCpHIiVdp4+) z;Cno83x^Lb9yT3ln7!4%ZU!Pt-FA@Z@O5$g+Z(1`n%Nkygu^t<&mOSqrm@pev}a43 zZD?~(#&!1V5=S8&Ulq(-InUf?puQl|EEh+so#Rx z;TX|%o}#Nbp26tC0-1g+q~5eY=pYppF(S!x^RsaWE*Gk z4DN=%k*{vZAmVE=mV#w=@D(m)=Pj3(W*F&cV6>8a-d{{~7!BNiC#s&VkSCN9w3NM+ zUHgw>3Le+r-nNM?a~6mDw!~5Ag`b92)#Y=NV%?#pHWY%c-saW|nN!YqWIt_bm(R}t zycS4Zf8&%s*|oH1Rc^Da8Y|hSR`@5H{6nH{?kB63KbK>3r$4locM;~}FjXZzYIdDA zJ0PA^$TPQ*|lNVFI?7QvNqd)<85KA95=%m%)kqsbO8v;ouW8&jN zBtKpf;9eTq=$k4Rd@Or_+3z$OQlgDeVebC`RMR3=wmE7CfLe^Wc!F@B{7}v;=XL)i? zpE$nF$Ag-cm!UtX&2KB(@})o9)gexDrme)0)$(TCB@ZkU+p? zjB2tO%Lhg0I5xNTSBJ#!`tSufMA5h5)DY8991&v6T`P8U#RtASero0FAa_*xPm6|98y{)f?r(YTW@ zkC)4leIzswtPuj0`;AjYQ!e50ORGOda|Fd2v>Ey>jWHFQ7CMlVb0@SdbDK7XFQ69A zbD7HxYmF%K#PNW^AQf5{3Gh2`x1*==jkYs!ryThoRoAS5*7+Oa0a{m0k?J6p!_8U! zTk5g6q|!e9?bNQCAw$dK+K1%VT{Xn3Azh5tJK6xy=oW`~^bXdzcief!-n;o^AF=TO zg7Dwdv)7wD{hO!sqDT4&-r#jK58(-W+^d+yl`)v(;p8{v$@HCaE?~_S9ST9xD>)1klsc$ASMj8`E1rke5)%l=OVi&5oDVW*H{ zH~wBDxuiqpJo0c#DAu5P?Tho6?{EIDh~0N= zX%cTvY%dlVfR)=R{L)^GmEJmfaIUGs;oA#Z4Z_#L)jp4wgPMnmG+A6~H1(code8MY zXHl+z%svGKz`|GZRr}syojbpBUiCxeXMBJ5)!&4<;&2Mr6SDOExJJw#Pr@ya=SK?- z1LM~&MXVHY(bXmKAgU?$vz4CwQYW{aZ^t2n3}IlapA0^YT(=Z*43{ryz3M>|Qh7&X z^5Qh-59qi7F~vZ?0dh0u(LMA5{+=nSUpSk7fj^e zeb4Q}NE0;wraF8_E*%%ev$jd3u&`L6Y5cTQDohUtq<8doscxok%NVI$%DYyW?r3Jq z6R3P?&gCxFyHF}Ngj@qq3xCImCP_^H}LtF^_3( z7#bHEH=hmR_}jV3SQ^huPPNkJJ1Eh|ScG@0`=wdn-3Z0()H^u-ijHetT;IWjriJhJ zf=V(q)*d=<(}gz*qNHRl)0oQ8S$WTnR$E_)izAdG=-cZ{9E`gd6#=$Jb+EjJ0k=hF z^lwJH^RMK)*YSSp3O}F!n6jUpW&rG&kR-nHuw3i$x>H@cA(4Q^B8}hHaF*>R)um+I zV|xMmpb8AQ<-xyqH0;`if5U^4qP}A_C2(_l?&W6F^+gm@X1XS)qR{K8_OiMN{Q&`J z4)hQ`UCg)wn!7_tobOly^jN}Ic8ruQ;-{2yY`sT$hRvi;Vc5jZ{QyOz}%k4)* zuQP_;B9AlotU9xkz$^awE0z8Q)vL`N808jqsB#(AD>=cpoVY&^ttV^1itfMJGg|39 z^H}UoW_hsXE{ZnO=YNi|<*4OSM~p4k$JlbUKbnjcExSq&G;zZibB%OY31?okY^U6i zI*{z|$bs_VQd43a2~gQ`v*voYS-}J$3o92odK*#vTH6pA8m(Vip}C_}tJTR%_C#p% z=)!yrb@q&zFVb&5y)8HX1z?RSo9W(eiE??k>|S;WI=+0*o_jxXD%%2fa00U$Rj#vV zs$c3lT@hPQG9Ony5{_~1q8{Cmop-uqF#*xOblOF7d!o}~mH z`azvDGb3I`tfR z+5vPjIFxIZgq#zr;Mj7dN_$6Q} zMN`s~ReBF~S~c23Y+TtEbTWna;Y=@AOaw`bxrM2W|2>=GlWDINYYDARIqps)m?*pq zQ&e3X<50%_F>Ju{Iw+gcOQnWvj6vN1ZzBGd<0b2PQFei$vduFK#;3=zBz~1na%x?3 zT>>v=sxvJk8?^}9W@>ZYjn^m+Bog~)Km5*mpWcY7bdp?*G9<7Y4`6?3uCSEcQkYr9F>cU6(y$*1ednuYK0wlPxc)nXK=y7Z{Q{bAoECsT?W z5Z&|Zy-V?yL*jLLIB4_$28px|dfr zk2doM+rxCKUe1)5(|l0l(vB&{=*|#$iwDF6{Fid=CjFyafo9f!3zhUAaNa%^^>=)* zgR03xE4?93(H3tiu_N4UX46-f;hFeNi{O_b9r&{VQKZCCAy`nDBrZ`SHxJg&{#5n*C_bBZtd7Hin~6 z8!pF4f#CJ2IITsBs5|odJ?RUxWmL(8yz1`dwz92p3WKB0jOGwehWO}VpIi%cE~MB7 z-qIP~chgLo=UJDRJKC%P9~4TwZ7KMPcl?Dn?w$N=5L*3FO4{ROIXOnud9|F4+M2jL zj(=xUqZeD*rQb7t-`CAH*~+dWrvsK4+jiZ+d5dtjvMSY1vPt1fSfi*Znv^zCA71N; zu%m&9}|#Z64Y;UML9E-$&XxgSqcw z6r9Q*0SO+nw=+yq+qX?Gx6`9aBR#{~oXGhQnP!TigJCFK+yk=c?=^gZ5evLOwi1Hq zb0nxvcEYusQ~P!Vz27_ju7Luqpx-d|^EU=luSMBp-wVq-C3~gK5H&t&~W# zhxrKas*P66CD^xT?2X1<=lmf^2JyRmOX&JMoMb=`ndf!l691A^$tjv9EoLE5@6KKS zi|gLchu|E@sC&6Dh@N~M&rHNQd(|c^vI;~3v9<;?-%?wrG`IhxeHu*j$S3@VrroOd zK4weN>tf<$)iWHKXd1qpvf>P03Mq}yT+cP~X!lRP7vfz4R0xUF9l3Tp>Q8qkU&iMZ zMZC9RvHPRJ-q-W*>WoLBU1w2~o#~E(Jgm8F5;&7>W_1^ZMjMS*obRo;EISu$C+l1o z8c(~(>#o8Py<4dGAWT#jqpcnes$U6v7L@Hx_AL4lE`Gz{bymX+mmIrjQoR3}d^d=; z{~}-Bzheo4V8?P>AHYLtAwNZN8s!4H%ei_Pp~Wc=cm7ZkTI;^ley1+zqTP3vnnAlL z``#cT8I6r^Ma$Fj1iuH&?mgnRbEtXpHgx7^r&xY|pH;c>0_Uv$z+?aF_1RKpn8S49 za-?b2N`mW-24I=!jg{`jD3+38s;vRNFw?U0*WE?(6I;!zlTSZiLhH76VrEB*Ur3-p z4D}}L#+h7k(1t9?(lSwXw?_*p4+#X7U(l`_y?~vYw25@3MopH7K}|rWYg~b&0Bx?( z{>rOQd{=angT4;RmB^2BmHM}sT6{tMLP@m z^R}YbTxqb9m13iA`qnz%pKmqNCEr}RFCAUk(LT$&BMtn6}FU}v{X6-HS)~<+x~pU*dGUF2U;9|IK(k~n^H zV)i$JIAAX2bZ25pd9kAy8=EdJ7gx^n{Q2`MQ{<;~U>>8DSP7dSGg}Ya7Y;90newAl zlkO<7+r}*QiB}$*5C@;SmjYNnewB*ycR85JPDp){FdxdO!te9M9!_EEPXUi?#qmJe z={AuCr?#)WPi~romZe2Bt6+_`;yd`|X|;!YyE;lRxz-q8wO4g7i!TLp6l9F{CpFS* zHwF1q;@{iO4d3UH(?8>cUDLE1oCz{cmJ$8hc_kxW7JJ)VnKcZR#KNk~Vdnd?vXbe` zz$S;P2qk*wZE?-4^r1TzvOToiva;jjcQmklyO4E#)=E3}k~I3rTfh!G zp;&Iz)^?=oc3`%_lb9PH617^+RWbt3Y=S6*ySXu@rK|Kv#l|XPpg@{2_LF3^ zbw9?+J9ZiwiNaNAUt+D&!@mK{Dd2;WyK96y{p?R!(4T=-Mi?NV_76uDOy3RI-74xXe_1%EVza7&U zaV;P&a>`dx!rOlP4h7hsnjwTD$lGX|Ek2KkkW7PHfk1Nr7OTlFN1}ekOv8s(!oOZO zlO9uvFDkz%M2K&PwQ!!Q&wH=m6jZdu#*wv6&~yR=#W~Y1S6}jT2ZRydcaAMPRL^L* zQA|W2KU*3dEC_smCi6^D=7z5J)(tlf$w=ynEo)OVa~tve^Jje4x5Z?54|_4SwfGZe zm5<|QpR#Z+Dm#z7O*}S()1KY!!PPmuWzFQP$Wgw3Ah{Qn~M9k;~l4 zDXD{Lw*cCkZ!ZE3>h~TWX@Jp*htao~uw@_|_5IuP&_j)hpWo~%ALa&BZPN`D2yIlK zZ<(RUdnLc9&>qtDN+#}~0an~0!-D|K`jg+e#Fb|HG4a1Snij*+{B7-Q4nb`W2l%1* zqP3Ib6tc4$+hGPI!J<3QLCe~DQqM3oLkWgubj)?=!)S?xz#RAEj^zju@Ji3y@7%)M zYg;_lwpOnk5j6NERzaUqU3ay^lqVcJ?>@^Z&9+4hAG?SwpQG}=bZm8?o6ddEAI2e^ z=8j%sNQ0Pz8%r!(7(7UJf$Ln!o`GW-M4A#%%x%z4%59*bimrfQzU5&6-0%M+r~_%k zu6ecU?_n+Kh&{l7A3iLD54pXo9^J2wVYClz47ogveq_hb(DSMk~1IVWZ3Qk_i_B*&ZEfm@y{WqdjG!&H<_{^?W zu5aS#n5SR;$b7!cMkLZn2r)=!-LV@sAi^fltC{QbEmOJlNP!gzxhS<)eUrEnxO)iW?4{hI==DP6HCRT`GL%skbjz3EeFK z<(-CkR0fpOt*mlEwt7VA5WUyVm6jD8^Pg+sD;HjWc)Hj$%e9kNmplRL?Osv)JCEqD zyN&BdnTMrMCoykWZPoa81qJq)csPO*z4ycC3=iO=+2iA(;$%19wcF34OJOo*s~Ok! zqD-}dp1{VUrMERiQB>%A&)^G{bXRV_u(*joi1Mngl!uM_{DxoLC_2{p`=HE0=<0(Z z5)Lta!yty4Tp{;uaaYVYUgCp%(hj4&4Y0thWn2WQPb=QGYJC+^?6o{GL7dueEA`x9 zVI1hv6+B0Ev2?ms`h{G;is4OikOJ`#qb1rrQ1*$TG3iXc3LHlQRn3%LTFd9Co_=iK zG7lkT@%_(RjyzL}iCC&yA5AU$Y)CvF`FMwy?)}nA$3kBvvlcJZpd!4OjiQnQ6E|-z z;JUA9HCf=X*8BB~QLvnWNRo8Q&?25)_jmxMTG23E#4mEdkH^uhE9lV=i9Fj)Vx3$D zgOUe581pDcc6&@{#p$`%=U<8})!ae8w5+LKllkcU_EFj_p}i7}s(@TX)q>Qo&`6o@ zg`Uf3;n4nhUH@%{sY+?Zt}^~$!1c{~A%{>bt$M_#&|~nGbwh#&T4^wVU8j=Iwv@Ht zbR)IgRtkcj>2>I(K@vNU>(6Yu>?hcjy#CkbvgkNY&KxR(7SLPAX+ z@8g2R3MMc*n(v~mk1N4Tg_al-gB85(lVrAfOvm(!A)7*i|A%VUr;$naUnN$Hcoi;3 zC+>-uRoc4?j_}a9`~L(pV%|rP@8MPxxeXZTs|&*UCQZkh&K{S9_p@d$^WZ`BrR7Vs zh0lg827x^kqQ+DC0_%@O5__z^V)il2nboSmb%-3`z{Yr(_?%2Z3S_q64A^d9-*(tc z!XFjwqEV<7h!nlCm><&PQ|jH(*qMo&xZRMh3`dy=o1ys5xQ)^_j~T=b|HLTX3x&*i zv)XzH>O}}taX8_^R44%g>5%azLGZfK$rf0~lSX77goj-4*oE&7yniY-2*pMDFFX^p z!zrgA=+k7{-!Sk>o`sw(bBgK<-o4{HSLQbbEQ z7bN;^Pg&-=Z!v%c6IX%=@8Yq^ZFIkYAX*7NPV+Ku$&{)$o;r7pgPx1=E8R~=oPt=n zKvyk1rX@wn#;OZ>PkJS>Fcccki4NM1Z{Bby8LU-kef`XZos*Z_C2t2aWvU~?l^TpH z;kmi2dY?@jAB2se5c_SvWMuKoWL(O-ZT^jO*fm)tWuRMg+|~!|48QdKgTG1imkuKP zcl4Z#-_Z9w8^T%ppXv47P8{h_CAa3?PZYP zXz52&Z2q+xL0nbAera~X|Jn}(nl0Z|g=oT)MBJ#8aor`fTc2s3v4>zaOoPxhbq$8^CZ@VLbC zA9PoXbwgS=lrp7hy_3HLM{69+ZyL+7Wkwe}RrS;AU$HKwG-4yOxucP-Jp=@afW z`}UB3asgU^pIih`_v4X>nG|J}^kW0@zjR;lKNd!3$ajcdOnFc}NC=Y>BCZz$XmVVF%Ipgq|K_p=q>8EXIhK|o=oh_O0UQV# zg{Ni|(&&vrIA4~6`l5WqmbEf=w27$%J8) z2$@HXaA;@BU0>@JC`v+)jBW)hnML<{?C5&r#k#?ws%@H+Idq^y(Avb+|$Ig&H&6tvwzF%l}RR-=S( ze|eVaw&P?+L5{zS^6fzRF?;1S2`{WO`YEPL*p+UD)C8zD5#58sTK(J|Ev_P$8MD*3 z?nFZ2hZ8@5!ER(HziBa+vJwMFv<)#3Z*VzCh*B+bJ<;P}4Y;FwK4+)pC@y33@pX0M zX|7e&X5bXBm;SX4Gw_5V=rBYW;|fJ}8qwu>)!pfp6n_uy_FX;c1H{)rDU6Ctey@{Z zQyH`WioLQ_%0gG^URUluk@Hv8umLkcy~h|{{>Cqa>nUAqH8#xWm)%Ld<;3cggJGZ# zX8soG0J?urrRggSqF+6yYdVr=`>IrzdZ_BQE|mS4#-OQRVIEQRJ?dcNC3$0mrOW@g z!2#rj@}aj%^zBYArP`m?`z1Xc^V0bQfxidq=pLfYQm=lFk4aXBo*tKsiX+Pnp7!Z> z%-BP(Qf`k#zE(NJl*ie30qK(aW|>kqn?COTFk%QSn7jIwr%q3}%l|olx|vSXfYfY( z0^-MYW==g(WmMR^1?Z)V5|_+MU9KxJd4Pk{s`+8`EM($%!1oSc1jDG&+6MXWMcbq6 z2@pp(q03Fb`uk*_W82hQ7t!%hif1N@^;hUC9q_oP0BZ7nTdO~J#_y1I4^KC9_RMV@ z8Dlw{v*v_puCzG0rCd6&^>0rB{nN(p`gbtiKmw9(`7bMS1K8#J%sT!ARaG=mjooq99%iU%o)v?)UCnN=*~+V1@jc4<;K2wQ$*=c!?h#*k5Ac zR{UY~6(%?Y?*6}T$0+~(eip+9${3G0&r8S;!v^yIVuKaqTSV#M)F7aw?%(#WZ}k{M&hpRH^)=FX@@r z&Q%c*Bu=iWFj#aMn>S;Kh~ri=L*FnSHZz;0W_LJPbqdCLq8si44o|s4YswSi+Tnwy zk}jwBME^>6w=EeZ@PR{$ZQ;9CU0lB-G6FnTmo+<>>tBY+O_wWPzuFYvr8vIV==&`a zoe$ym{dXq5pJ|eW(`X}jZG$Xu4QvhAGJ;2mUPhU9g=S^fUE3&7=XsQ`4-n0c?ts_o z=dtu3VVGX&<{1m;pTwd(XV}rOojB#`-BC;Xm3K`Hd&z<`2eE`DCS8@c7EXH@zp;vS z^|V8XlxJDz=n#s(eac9}a@*4|fGeC3{dp9ak(C^F810BzKw`Nal*H7l*>GKA`4`zyV|xtm}J7%L^@ z=-7QNydjF7aB!*nz6aleqg9K9^N8RD1n9oWk-(m&4paTSs5SV_P ztP@$yVtDw_HI|res3+iRveKGe)SVjsjW^7^q6*!hYusb=`r|_UhjDo#S{sIwW7WrY z(?6#zM1?l8;+$?YFuC5)eM(HJbj=`n(*KXTw+f1+555Hl8yMW(b#RA)#$|AKA7F58 z++kpFcXxMpX<%@7ch|<@kPZpf<$(ZF`PCzV)q0wrC$OV-IlSD+DX2no&bW)YbHlY@pvFK#fp%N%H zR!?l|c0YW!qZu0MF zT532bXCyd&%14R+J0W9EaR1-t;USYEzyF8zczLNq@5}$mLLB>Q{{Nk=nQbMAlciG+ z?*DXQXc`Um|Nrq@|GzekSA4O<5vnjN=MDK*o|PlT+U@?A{$$u?aw#KiijkW6WgifOOUwRIE33&m#|DQAQ7q=I`#3OJ8c(Ml~ba zV9LFNl$ob25;e9|WtP*-iSIKGUg~tAI$a{0n_>#i%LfJ11C^*ErE z_*fra*e6(aC~3r+=~Rx^*6Eu=7W9z9ZqtQez6B5r#|!X+Db#f*kxw`alT%gR)>mLnCO``tiW3A`m?1#`wZ z;LKO+Zk(yQ-_mz^upZ<=I*u?$&)(dpx|zNAlQ!78+PYs8KYtXH6$T*TRI^I)x2Dg7 zyha{re13WE1a@h#MCzyX{*^dIO?l!P4#!jZ@F^m?11qz8wyLi^?%3Ocsvwm59p?WW zQyLB$yk7R6dT}{(2l~AFeYbo756*MnZjFE=X-RAcO)7bJmLCpPi#ndsKur7-`#@p> zq1i>#>kZ#gWA;h54{#OVowwJ5-FYPO-fSR>B7HZd$ws)Y>$of7ES*>@>d$Z5e1Jc! z(VWz42W7DO8xeKh?k4-@yh)!H;J&ots{Nja`#iJ^&k~>hY(34FyFIsSqh+|LD&bCb zIrF(T)<|v6T*Dju1o&~>>QvxZODzWT#O(o+lndQ`N2V2QL$1~f^;{LUaLzVwa_WRw z)GK>D$)I=lgYOyh^OIPuLqpq#FxV#tg%v9Rm`FE1r*f8d2bB%t3tcAIaW)=;neqz# zuWRKB245qbrX(};v!f;PY>rq=j8aBY&-U$Dg}8+rlHeEh$EZT{+C32l*$nB>`Tq1M}g}`=P%lT5V)Hp-v_GhE5&FF3m4+N?ZNwkvWIR_#27+>^d6`x)8IgYoWPmF4#}V0zBooSxilOR&$LHJ4GWP>M%B>Rq$0lvMQyeN0_uJ;@S; z{7^kEIyIDp&`=YoN9f_h^71rMm4q>2FqI%+uu!2bApQOQZsq-iB#&Z*NE70ZUXVIZ|Rit6Bn2KT)OTgxk2Fs1+cDNJ&wylRVjiM@SDDpS)g?nt)82MrvV zY3a6CzM(Lksr?lMWGwhR_e`%pr|-*D3-F=P@|OKg;pu|E&ReqkMjQIJqOOoc7&}!g zxRdYNuG|@iVhKB&-wMnJ*H>ST8N)R3vXh4pCs^G;8|yseEKaJpS@lxygq6%<`RiQu zUNhq^^a^g0YwS+8KB}D89;VJT0?w%;_IIgplqM*@6H9lA^`rvLU(_C>bB<1wBma$w zhGo+JU=MqvOUYXb@c-{XkcHf{o3R&Fc1>Xk*_V%CAPDR#SJzgAsk3>(!^7VT)4DWr z`_mY9W*hcB79Y*j5rUt($l|7`^nNd{rFz}%%kLyouvIk<#NPxfIP!o6wCGU0M$`%{-%eC7T^Z}# z>J|A2SR5wo%UXHVP5z*+r7k8bA7rp{&!yx`9jOrcHS;Vyh>C$90?10QLd?LLi(;Zy<-gmw#iZCqWE(&g9hvK#u;Jka?PzO%{F@8SD|1or56NTW0=4^R7i8dG8z*^ zl}_*pwgAB`gl~qlI(o+!0UJENvLv6~KLz_UJK4+e3NWC&eCd$X-Evaqu(zZbqt)uj zy8>}IH;rSqSdjnq|50IHmru-{$=P7aB;YsT8VULR_D zckt!1!NYq%x|lSLbcU*;qP-iYo?aO+#a*(N9mecutpSV*rl;b6=Sz2Fh(_1eTdv{r zLuDLtCM9VYq>-YPz7|_-Iltg`w~a~E=dWt`rEANH&_PEwhJJDdGdjC8A zVZnq1nAolC*#zd{A|UAX8jKy2gg19p|M*73k+jj~?=J$8u(sLIH|`ty4r;~R?e7H0 z3267eKtZ8Ftlw2yucYbj*=Z3xm>NG`?s5(ELerk3uL`~;*d1ORXj^7!7G6^~ICDSq zX~ZX>tA(|y9NwYeDo-NSRiP6QNQx0L1F!+Os9xr*PedLX6HHFtLT9gwpHpHo*Vxj! zbq~0`laA>P3bpXC{EsAVP&RTW4_GTYl`nruwsGvZehia@tKU2%#rM%1GOc2E^qN0B zV{Uj-gkGS&aM>{d7JvSz>Xpf1}yN|Nb2-ifMB=O0J!!;{`g| z)`U;@ciDCRvtM$`Bwb%>NF&#PmFT(-z3WEG&wS z@ki4eor|dz;VSW7@ABE$GFU@cxCajn&TEgTNY1uM@J?~Y{_Wwx-Bk)2BRL{3so76v zyf~%ql8;&7jy^fzYc+Bxo5A-4(*z?8z&83uuHZrGY=i$+W{3YETqdsH8Z=cQyIZNf>exMPII?)#hZz&fc`xK_JE7U#_Nmv!T?r zB~kl!(uCsCxmXUz_aA~O;}xiS-&{NN-Dy}!Z9DXsBE)_%*+iGheE&AlCbxLGQd zF0ux0^lEPZ?K@FDhPGzbDBU+|v!Ldj3%0 zuL~Kjp5IB%CYcAFJ=pZKGjkcu7UVw>?xzX&P8g)zAA7y?c@=k06eyoyxyTWX+8mp8 zdDlmEStYfxMR(05ihFW9fK+WuRFoQ>MZLs@M#w!jrDR7v`fI2;Se!1VGI&Qy3zT`_ zCc@c|(Wh}f{LZigw9%{ARpIsOfmYpqIO+L<7P>OMVcARg7ee{FvY%>llt33ee4vu< zOJi+T17YZk#KO&PAGACP-$W_Kq9iM(jfcQ|2r&QL+2AR5poulQzOsa5c7w*HzgV{A zd?ZgT08R~}%)!aC31icC{jw$t6VjZ$CR1P;Nm9#9>v;wT1*`1@lq!ApaKYaEk3<%{ z8yw=-HbCKI#SwiZiX8LH`Tj{n`19o2W9n7@LU=ECDLN{d`IEm|ZDiKeKYcFDlrcV$ zaqfeKiJ5`C*!C}#7bKXd3lq~oX`4E5%3O75)HXyw(fN3iTnzVsdUe4YYcb1oGn4MA zl;#H@9r4Ol46ChJQnRON5yrsh4$ZNS0f;>FqZ`R%v;xBZayaIt!e7uiTTFF)O%#E; za+;3w;f`%NQpw>SS%LvLgD+MwD7Z*FL2tA-zXu8u=6_s6ch-lbQ@nK8_C)&-lb%gm5W4O6@gF5QTrzZyAzqzSa$m}*be=FaL23V}M;*MWsO0+sG z_8>>NAmupB2F+Dr{7n6#ASOrmP^GtZ)|H3K;45K!u)XyA2Do1+;G<`A5@gPtlpOYiR6S{{yTjeI`s=V&}W^6@P=pYNFocNy>ViaJceZ@hhh*w-dYE z{!%c;Q&mncw9Hk>=;bm+DVQlIe6?GHR$sHEinOYp=KSK8LO`O#{iT6g4O zy|b?qX|wj)O<{mTi6SSj*Kg3`dRpoJkWp2Bb>Iv_Jc%0Dt^2t8p#J8G`Iz`SBxKh4 zit?llMPXfQIj-zmlHv1?(Yc8{afh*yBcuRhI(pUDsW|(sAK&iB23w(($|Y#<7vO|C zJVSSN-uK7u&Sp%p1UPl3s~j0cI_g>oy#IFU?a0em$#ddtv_ z%!Z6QGc|^|fAk^onyVd;mkgsw6iC-Ck&U(=>41Z+vEs#Lmldkv9&+U>rOK!R#FkEY z7L+vlG)xD4hw}P6QS*o`kM~w8{1#YFYfHCKf-5gv6;Wc>_t{#$9fHhUYEY2GHKOt0E8vgaZZ^=@s|b8=DHXTj?{-2NyEO3+M3>6%JD zG+alQ;viHaSLR$_ zdY`xin)55X6t*D8>2ocg#~}7@3@pNpL({xBa+htb$!p_J0X1K;(@8sdK@=@pGcE%Z zt#RTExZ-Q>L46Pppzb+c_SAiKAFa{?MZSSJ4&rdcaamY_m!Xi%?xLmSwOkP-iedQ| zc9+&P4%7D!dpk03(?+a{;1BzE;qSAt-((E(LqTvyEg&moyC~hBey1-)&CRmAcQ~CX zfDcv?J-t$1kFO*dL=$~5IOjeWet9py#lb(GHWyIf`}0GLH2&l$A!xD}b29lQG-QQt z$1i2(lDF(i>ZiUF>YY60j?617h$MT3dT&ndgVhb?EqS32wAXivNaGfO~|W8YGoSVlZPKvTd!HXSSx203R<7Ip~G zB;_w=l-^cWF*|iOzscx(dazE@2Ex#WHVje8`gD;CR(S@oGz!`ncxBY6A+%;3J~O>_ zZCx06Y^y7#i|Rpw9*&2{e8)2_#Y7CA0u2N${vzbGwEe zd%08(B5Fd{l31vHYN(`fcTGbC3x^0i`~lBcVl%X<{(@B=wa;HDC%Kdrb%kCha*@sI z2e0y`jb1m0h2l!5>(57Glri}b=(u9VLC)wKtzHznb%hF4g^_*kTwe=(128tk7wi&7 z&P{dJm%lf4&?^dP{vQMOh9Xm1TS4#!nQb>zF#KY0GyV7HBLP&J*h=ZX?6F{^ z-XemhEnC0AWSNt@8M?7Vb5Tce3jI&%wD!J=l+lQ0pWV9DWGJ&~Dii~5-?gKfOVStY zgNIJp`-tPAbeVW~t?#3pb}JBSYR^%p6&FEwFFSG&Abv@j78qBwl&G3_0}E8N1G$2Tx-X8|Gl~>)D9hZ zNJsOXVso1C@{NB`M;Ca8qptmZ%@7kw_DPoUY^t}}j()A%jx&)OHX4N0Igtt^pB@0R zauJo5azm0n!=mK`1xJyh@?G6d)Y2S#@Npv~W}u_&pi9pD35FNyUY9Z0vL1D+bUZ?* zG9VYEv-E9=g25=ugXEw&V5{2|QEu-X>Pw|CS6k+v3H_?1dBbkCS;vEm^swg#nQ8K%AhsSgPD5mz*fN>-uLx1DekeB?`kn^a{Q|g+3Jra z2$?fkE>KMuiH?fIHy{3|qdD2O=WtRJG|ngBJNHYRJIyh!Pjfm5&O9V*CHqi7&|vF= zw+ocIvvkU>!^f?417MpFb*MbF$&z4j;j4f5%3WIiH{<}SJC%A-_VQK_yS|SKk-%!} zcwyT8`T6Y4vQlzdWyY?nQ{ES(s|2pGe3{Pt=}8N1jgJ zxJ^4ft2zn`k>3qLZulHXE_bPc= zaeZyDC68_znnb%@RD@A2-tAjvcs?asQCGzB)^X5VEQ5O#e5PaO$BzgRQy+7nj{aUY z#Yqr4r-$(9ZbgR<$#e&p<*I;=o`f%{~}4DSktlQER8vFy-dr{)s&u_HN#%{pnf3J3d)(_&u#VrpknH24xN@O?gqr zfH)So2RB^=gL`7hVV&D7LrW1hMB!p-Q(iUV2Q%w{l>q^#KY#o(4vXk*0#DMlu~e{y zDAZlQq0+(L3Gjho{P{Z!^!e{u2F(%C1I>jZSz?5KImkH&>=&%A>p$?HF`XRmduW|P zRgP3|u8ue&-XjqmV-DB|y50}BDXL_!=thuJ2AW3)r?~8L>HTtgoAvDms`Lit%;<2G zS!{@@hz+Q~KWH6}Rb9(Kk4S=5=|jehqu4Plvmf5m{4tkR3T)R;2b#Qd{Y?f0CsHtu zirYJB#(K)8_{UF`tOYM?-2CZd<3!EmRSa$|`;nuoT+M<46cM=J{fPT124yFx2sy3)+$4dx0nlqB-?&CvxAlD49v%I;N_DhE9b z`POe-mQyO`BjdeQ_=SmyHAmm{fF zMQd;Mk)4kj0sXwYfX@(|5#SyI9H;J1eRb+Z;AHwqlmJQA8?l<9*xo_)CgLIc}mJo!y_?-mgh3 z*}VKR2(0sH`ZrJIuYIS>dW$gGdNT^U^yo;(M9t0QCO-9i6$Gs%>RC#6)z{JGR8{uV zl_u%U_f=xU-f>*`RMyGE+yPM_j3Z#|_cbjtzx9d0=~?#K5aQGVywE+g)FNaHA_j zl5Zew!5y+v7YP;2WWPHBW&c1?;Gu{Aa3&dVDG=iCr?`d4oJ@lAw-6{H;2E>N3Evm_ z_20ANJZewj9>2A=>+At@cz?~;Ia;&FoE06yWny{-9HxgA#LPFoMn33*gnzG(_xSKO z0%pdl^H%7#UXIoK*R}QH#OL{uQRG~Dd2z{KN=h_Ayx=~g=^(}FY(G*pc^c3yG&=e? z$@iXOF1^91g3$6Srck-%Zo5@mEeDOH?U@*0u+Mg?B@9o3-(Y@r9o6ydE9i71sM$#2 zD(T9XjfbNPutVOb^`ZZYP5KQr; zkvBonc|21DnI)HXc+M7TiXeG7MfE(Hf!`PqDVR;OXj3X;@vKTuR|3(nw$%7X6zAp? ztAgxa*A&9V5UT+WDd{aLq z5|wTrnC8^^VgJa-Z7V2rcpxYAG$O`fC4Iw$^!>sH#*FsQ+ls6LT}rSkyLOWk`;pW) zxj8(hz2;@XQ^Mk?f*gbgQD{|Xe8pKun9$OvSB5WibeyH(u1%@M**iH6+Tq!W&()SDmdpc9 zv+*mEhf|?ir_h|gpH*U3$CD0xbGxjeLLMGY)5Jd@v02vPU!Twp{%WwVWguZAS z0$;|-;sY;D`scv$L19yAOEazqgwgY!HeR~a0MG^b@vy*SIONG^2a&1`tKcB!e9}gG zQgpMh0Xu0fPgfuj&h-b8*$wxhCTRF$?f3L4+jBafi)k@m9_N8v`PhHow>=-Z+R&^9 zr2a*-LI%4F^AIwp7Z2>o#(VbXzyo!Tb`6{T$5b)4j*)|kR-Nem^Biafc8kGE8W!YG zqHC0c-Jg$3sj|dMFgU{Gr@bBm_$&c$&G4R+8S-Zxi*B=r08=^32v>WZ0cQ6At4@La z9N>8MQCBcngL<82(#tU5hsFL~QLk_CPGzi`A)i1YKuwPlO!S=CYUeeYl<(?5g2LdT z(t7yoC&=J17w27MS-;~|iYA}p2ux&nHCuiCuL?jdu08r_@*I8p=E7trKpZBGdTGv< z#yw!WADCMEL@aLoc52yLO#~ie!c9tHq%IPuPEv*RuuTept^ZQwKF^&@$uWfCplp_ z8lkVZg`${$=-duwXEsRSjUHDlF3OW&vF@em94EOf#PYAnBzcc;wgU%vkh8kXQK&b zY$@pwwC-^^9KYOOg>U5Z{MUjWzG83qgV=`OZX`k`zX$Q;td#lJ)W+ULt8dJ&LnJtd z)zl+{tUAd838wwdVgWOSjA4u?Q!<(7j?rX6Nx!qbNg=AM9h>rYm- z>oioT0wc&9K_H3skIPE$RlP`%Hx}veTJS~YvP^Q8i~@J1uMRRMZmHVlw!mMjB_V$| zqS=ko^RSaCFhJ0b_E)?X-HWfL#4E&svpZpr@F@7@+ zR(A@o&hz*2!T&>}jI8>$;1k5j@1~+V_}!3$?a~nY)&dy_egtZ5-sR?%`yH)D910!} z-V1{q>;F<$l+6V+FM5yU&#&Jy|H8PxRQu4-0`?+@bC*wkvM6T1dt|dD#_h#GtEUG= z$fu3tU<1XQ!pTP?ZCHOX2ac&sV}AWSUv|e%x$lcGG^WL27a>@b!64)ztgtyCEmN);$eT zOIG*ep-qM_3&|h9EIanMf7p*<=qpw_snZ`&@2#c-ZFbai$`&n5n z`#-Fr>h}GlYMATErvbx7FMBD>)@U@9Z5yhO3Q85=F@FuwG!Eze_xnk_ha(#6vD20> zZJ10StuYU6U4!?ODi3OC0@qqA1pM2`0DWKnXo{m)Wn;~U9UX?~g(E1xt)G=RKc{8@ zp!T@awuOPnpKtBTKVi)6YHfNp`Bc-X9~C`s5wRuSCk_od$7Ux5g{#S+aBmM*z3RN# zN0X2Xi(;hHx+b}8iJJtOgixUiP6<*3c3###&Cnc12R>tU##%w$m3bn;>Z)}X+`d@b z#v6OXX-q-G%&9{t23wicH?rnmBnz^%CxR!L4vdPR;aWI}N7&cBt=<3C?ab_H4Im+L zHYhB{*1gE+6TRxaf3mMO%8ANXe+@vi@p$TM5!~uadOxsuZaF}x+xX2Q`%@H?`}I=i zjahv=f*%5{Ugg(r=o$l-=i^r9D}9?;e66#>W{xsfsHFSj)>5X}{(_3BpPUpRarIo* z$YAin#yFaegR6TYIJ#QB*sm8e6!x6T+!YF}5pR$bdfLh`)^~c4lx3lqE|J?Sl4_D< z&{^okIisT!U{6tBHp4vv*Sj7!1s~#>iF$NbrquUkaXf{!$%h#p72E=JS>HB>ajP~L zkBy@Ww%_|x)JK@u<7{H)#bbv4VY;qr9F81Fi56TcJP7B?VXs?AT z$;186uK9Z1`PNsxx?#04DAe%%sy_7&i+0o$+&_MuB~zg~7uq}gU>u*`a^}i>zn`AsD&#MXky)xN;zCZ&{qyTc8m`=H z-ml+Er~R5J^i5Bq=4A=g)j%%>GjTOOy!A;q9ijBz7kmPlIHGOPaGyQ`UouD2+hSl# zG*F}Q2v4Na$2jHvP{N6}8j_MYL$i1A;;iO`btcMR-AS_IQPed=---isg{HZqLKUw!zn7BKS=dl3ZA|us}@#>(# z=yj=kl}Y1MWx}{9)u;34F_81T$FhL<_v*n}V`19KTN{S@(JkH9dL=BQ7ZOno8~Kx& zw~z7e-}eKKQ|{khwrphSpkWg;mk|nWi;aerPZcKhY)M;TL@xAB%VJ~|fN%7@)4==h z*4f)!^T+lm+W$P3t&=cXuiJI1e`)QX%QgA`R9>U>i|&AvP~7QCh%3EMrmODDtr_HD zN}QtqT_6>N+v#b)%oRVx^8{9XiodeJGQJ3#or2xNPa2nKgw+mt1isi_1 z#z&F0QPK;jdb9dwsO|rB;(8!#`2S(=NsNqY`%})Hbt&_?_po5@R#RX)(nTnoQ(*gP5}=_tp&BEl-VB8rK+eUQ2#|0jr+9V6&UK(sW!jXi->HZ~D=FX<-E6wllsrhVC#DO+V|9+I{vmy{{9o&`ix&2HMkR*N7QcEZlW~`g@OAE^0_=!P z4lDDmqc{JxQGnPS!MJVQm!gA@*z+J7{R9wvt z?0bjqR8b48*tkw_ENswW8FKL`Mq`+FM;`AP+J@c-Z70PibZ7*wK6P9`tBWsRN!kL& zaIj-t0#oXT57zUYg4aP{aw?tVgmUr7>jQ?i%H6@b6QcA8QCsP>@u$-EC9kP(lUD-&id?xF1mP50j*Y{kJ@QDnNYNcJ< zIN7hl`)`)hKBk4cq2woYdTX5kpBHotyShh0h$8-g2Q1=?7~d%f_%CdejLOg$LqQ+9 zdAkRF@Z;u$u|y3Gx*IK#2!UZHq3t^F)NFv1@GcIp;_4Ea82D71pw`R#M0LiMP0IJk z+ydf)3FGmXk!Xi_jFW4q_9NP^j#Gh*y>Z)um;d6XzRa2TWwp+~7K8yNbEevhYMDT2 zJ;ud+=9o$e=tVYY*k$?KHjWGV_A_IgLd2-|M^Ci-FOsfH&D$-t1j!od*mDkyJFXcJ z(bP^SwwgKQu}wBS1V#*_s>_3$(!~E0n~l$xnljw~kg1RkHC}u0M&%jk$RNR4Sp&UZ zJ=GW==_|29N;^GIm1Dl9U%W>+i*yRro##8O(UT^&g4d&#_!Yay*BQ+M{1c>#455;?789?+2WPs}N%j92au5Il9^P?^ z9;9gFBM{?_G!g-}RGA^VWtV+xp$(jU+Q&V3N%ESq6`eX;@N{#QRya8!2^Q-KVU}MQ zaqbC6=donrti|$IJhB({d8L165GBJYvq`zT*jdHLMMyBoJb>df=qY6is0(r7-J!h# z-W+VaVrkyt9~&%$os-WJrTJdCvKeoWVAl(Y&Uj7o!7T?{TWx(>PG#;bvyc5`bDRyo z=Jd2jubo{pKqVW51(;E`yqe}X&c_q_F6T1b5y2q_rw6L^>d8>_5!L9f8Sq-pmg>Bz zZPq%9Dy1$ask@R==Nh*BrHTxcNTKaGSMsL<9bC>diwL%KF#%xYLDMJ`$=Q#q^Jf{p zm@mS8{i8S^U+i?%o+pIT2!gd(J$^8Ewq&qBNMDU@Z?zS0R7F?%gPCBti1KSRa@z>n zm%Zl-!o^li6X*x>RW!ik?Y^@48k`!?lpGq(toauN6_HKGx%5khT{Vn&X2z{)HoIkP ze-k)&bG7=TiDWJk%n2QoW;!xD2ZXbn!>N3%vmbS?y-q|mEqv(pZWM>_s=Z%(!|s2H z&pcS0E;|1+b1aoQ6i`F(GNgTM&X+N>6P4!+z$pJBqb(Tb&g2yp zeg(V+Oc@zf)|B_77mOk3{?K@YI|b9krV&L#EAa8z+}$gQ7bYP&MnO)7 z9dt*nn!-_pz9^Xu3n=L#hLNrb|81ixvE<5=chLdThR^z**FTm>wm<5G?#f#ZAwu z2%p8vLpD|*D*#}!%GdMJl|GqQ(s|ZFFbryru08MFWLV}4|5klpBuw=X3`joo!tFTW z8+g2sP-^e4z0C6g1h6b@kUCt<>)u~1RNytPLkmgFy#LE+HXD?p&Af$G&iWXk^qg%< z$ddnAWhe8y(2V&3MpxIdx@s>JL`Sz+Zn|^_r#zsQmmh}9ajup}altbtFFLqDHIc6} zF8qBloJ>>%3FhTk%%?d4!=-An22bmdhe)fBOOG}*#h8ax3b9gNP4LEMoNl+gkBCOZ zG&9Y8n3Aguj^K^h_(gRS{?BJf#4X3gMH>O)pumop-_!ylB< ztEW~QTbtWazGL0kWQok$e#WU;DfeaXoJ@a3GnN-`9b0nis^L_QD+Rf4@2lLx)<=;5 zk~V0Ef9&=8cVzGn2uC@GoA_84KU(#8W(0%#qZMgSHm($v3VooL8>^H(ILsz6S=-v6jpQk{)z3~4@1WTZNeJ=9 zAC7@}d=Zs91;+m+v(%l$1#>1;ojh!R$S$nWokW|Vv>#R2OIfSXHhdb)Rx$V`XkThY z+(pD>=^pmmsv6NQzZ+m1pv7?Lf^f6xNyE24A&=N5o+3eK0*_<<&9@kITF=9)@wyE! zo-$pa&b2lY?Y@ANxHH1oOv;%yhSKucdVbgxequ!Bn)5JD3$ERUXd*gBYjL17oe*M? ztQJJoh)X_KPoX8&2m$OaHO_>lhf{OrYh@p>7;_~+gZ>$@!u)I@{r$qDNv~-$PP2IU z=Cu)6;c1Nhg5cIN^On!x;fu7sh**>X5ACehhEy_$GDzWg;pAL7p;AAnIH~ewUGH8t zbvQ}~S^HRe1E{!br0I0Tlu3OM6?GYNmUrVcLC9x?>Tjva7JxtlxKiZnJh#coM<)# z1P3dKdG{F&Pe1lVW^(8EYIQ29&B$`5`5L}YS=I?g`;CmlVBrb`=w2-bvS9^8x_n=6 zvHXd{O|&>Um`|}}tKEnBUtHyV#+6*3BZawzB+HSl?e?MxP(CrC5t^3&(-@d1P?*?QyO zcIRLyFM59<{cxZQ0V~SrnFbesX^h~C48!qS4AmF7626X0dW}?k8^Da!EovJlvKW?P z09fU4wq|}Xf}}Q12Ii@xjDg6E@z=F z#PXaG>rxb#mr1MYQ?4`)u23?nhhZGCaYt6uoh)0KJ^Mr$k>OTOwqnPi{EKG|7Nc zw<{{?+GPsRG>hVJKtf0&{s8e`w$#gTI{VC)6e<-)8d9F=eSLAz%4p2K2Jv}Wn96Zu zXz8g18wwI4o8ROwZ~g%m5os2kDYn-CSqz1*L+txcLDAEkyn~nWCqpCX zzeHT#4YbO%>hLKwWY zBjGmL6LHMb(w$9vrd4jql`o8k7Bj@_5b}NYXSk6&eghZ7-}(s$YlFBN&xUq+g&QqpWTmCzIXza>nIK{%=BtY#AOK>8=l(7W#9ZZx1X9JD^?L?mxkYW2j1{QTNFQe$(ka zR-^qZIs9kMzFZokEy6-1nbog=Vs}A&;%s>k1gbRpSELyyL*B|*XVChoAG=GMlN8t= z8ezUL|E%?Wva?RPrONJ@#xziple}U;I`1lRS<&MMLxQ?~&-u>4@y%xZt~S#Cx8Ba}sqxd9 zu(b5VF|raVuX4Py+=92jF&xh(WKbdy&KS0kF`j*QD?g%aGOj2+Y9hZWcbt6x9IIqp~G-* z4CvpC0}T$tmOlp8Pm1zr{x);bM$wulu8>{GWjyQP{oA9ax`i%W+%>Q1#Y?@oq6TNJ zkQ}8S^k8K{sMX2-@mf6loL*nC=}>dx7?05UD#rz=_a>e`xhYCq6#cLO)I&7Q#V-t8 zJF6cLuL3Cjxit5V$}C8r3mZ=k zlS{jSnUN@L`t1hJ5+}Y*`#+|0!@$MPUwo}%4;-f5h=2N2apx)8pJ^|y^2i~u{E;fx zIQ#nbX*m)snRWNtj2l(#386J~!wZKHMAYb9pjui|3B~D6;!FaydYRtUXIfNN&V(6c z77fp@xV$LWD{ZV-yf&*4cIG^b5FCgeP)PQbm9A*%gN|nLcP~P>)!;Z;>2q~S^gYU1 zi6JE|Vi|m_sDSrX>`a)Ud+ozKBD$9Yt8Rb8`#L)XuZ0&SjyI2JslDsMq2(qfQ;F}$ z*Ja52t#j$NHlL78CoUw!H9k%H4W7t$0v`a4^W_5JnK-8tW*?t(toOwNcK#8rJY#c- zuMKWNi-TjP^8@T_OdO_?=C*{U`!rTRvM-RCVsq5xzNtW%x7IvpR>4GVZKnLZyB#QK z_edBjk4o)hrmgEU5RP2A6=Y+N7|g_p5HtN*v>p$f@UK6b>qRTQyoKW>;L?#C_SC#G zZU}IGh)W$q9D}b;#p4Ng5KdeL!S?|NH9`SwMhKZ{Xrm)N0q(o`f}x$2?ryKpXw5N} z93DpBs~L3qs|gH>+$DkRw@ItjgdWPdf6dW(Uaz1{tJ-arGWJFG4+pL^_k+(x;LZl> ztGQSRYYQnXc5%F#_8k7`-8pkVuw5h;4w%t@bv1qbu*A73A%hA3ZgXmE6&Gu`5Q?gW zM{OyjxVxi$IeGANrzdoO?1d@J>6=hr^C|R(H*Ma~xIb49^-Z9&qMCO`sjv5Vh%FEE z9+e#YH=5|0k7&W=>r-AXM$V__w;#>2*Wb9}=h#k((xXK~Kf_?b+u_6#;dM#>YSI7U zN1dYO0U5&inEol5L_Ms^=ZImObjgSUx#Dw z2b8DW6boLq4rIxoVRN&laSC2rfsrBK<%!<|7EaE&noSdQMDCuj;;4$HvR|pQX9tLX zisVnPNL(x_3l3%Z+S_BU7Y|BZvwU{H*gJ(i*$UHLQRa`c#SZzW=jQ88Yp`R7LMp3c zqhyZJ&VJ^d@1`UgjQhtepWQD!XsYRIVor0GgjTiNl9H*vQ!B`wt|XS*?#TOXFUbq- zdyjQ`4zWb4T3R#(;+H;E3tU&}RQ{0pmrKpnGa_ki^o5FD#cjFj-#eaMhc5 zZ4KJhZvDljcWrl|C~vdHZ;`tvVd~05pf^b&*O6eO-MCEGawP{+J!NcdQk}Lj;}Ji` z)>vZoMY#@ULr2n%T0Hq#%?QPO{YTqGlw!59WK-*SdR3F{<5x{XjUlQJm@ki+MN7e@ z51@wp@RZxJXtC94;)1<;Yr+q#8=tU|`gVrcmIASz2|}}r8JMweRZ{7HqNiImtnfK1 zq(iEOo;!8=GJlNp2rkrv7vK5{rM4DAhj9+BL+pfsGIEu}r_1a-9~WWsKWi_+-{T2n zcP532Qrni%qOI9cw05c!0tYx=4o-?LB(=UnR@qh|$4xnSWr!DJhj~(bjj+hOSPM=% z9rZOjy;D|m(d4MIO-WR6aQ^qbnGz{g+^m_%<1>*^P&Cby;sMWIUXkv-v1IR}Y%Dg4 zSe?^3b8KhfL^{-XjOj|gSw28!Gf5fV8i%+10y|mMzK!J5U@-_u{~qj!)V%;?C}P18 zM;^`5|J?+EV#`PUHHyIv@GRz9w24sk@3eKE@&~m@*8Keb(@1lEVbc5nrMunlMv2hD zHs4K7jmviqOczF{)(FBH&BxqD1Ec0B!c#eZVF$-iGu*elRHe}EC^K$VHn8e8JE2KP>wT3`LM^^b*=QKrauWHlKm#FC$SR4)4>@SE_FR) zL#7%ZUDkXUE-GidHE#c(BoP}p!-k?|Y38DX!jrAR;f$yLTG5II1iRDUfFa6 z7Z|FMLVx;_l7i&a(MwmGYN|=MkI&u@E>F2tDjwJNM3{Wzrzfn_B%HRqRFOXn0r41S zGxIP;PrT)-d*?)0pI`Xfj6Z06e2BG0EYNHG>u33{)}WcrY?$PZ2*-o+QLE=W4E!CE|UO$^8 zMOPJoi*10624au=MJ;LQnGf6Se3aspCS^@I*sA9(bv1vTJ)~T=B!Nq7VJrW zqQAJ2(_nF*vW{Be4UI<#eX;8L9fT)XD#bSn+RWPDv@-#f zDhJ74X-HEYclk6|dmQAxPxsw&wJ|;0Rlc^IDYSq*+Q)J4b30=nQP22%L^3MdFRRS7SjjVk(~vi- z%=b-8gWeUF{n%*=+$kDJyEk_!w$ zi8eDKR#AV|rvs=Y>n2acG4O`fv>>igqXjko&zw-@!^h=`s!L&vk%m=OG**VMH$yf* zT$g|;AwTWT{;7$Bmk!&oDdA>OdrjEGv+r;&03G#THM6%|`8kHg4n3*cf^lCnGq7bf zIfKP>H*d^`Y}Ct;NHy~7EU{F2(lJ_Jzh8b`vIz2-#ex_ zgfuVdz|8Hb|6b$!|LZj#pRi7qaN6@ywSBL5p{9^8q(SKZL^&2is{j^dI#&$#2%s_? za_?jWXR~jrpAoT?4V85X$$uyZAq@P#n0w2xwz{otv=myjP@uTAxLa^-i@UqKYjAC$ zxD|IO?(W4kSa6r%P#`!2hr_e?x8Jkh{p|DO{5og;NY=`=uFR~t<`@}s-1nIGNMP@- zQ36lN`ptsn*iXsu)-TqLpUOl^R2L(UgiBFWOUt?V0-2xqnw)t9b>|zn_~Pt|qKWVx zkdWAYv1wv`s0S3c^9!{@b`d@lJF5L?bddK9S>)ZZv~0%p1~&&Q2B}vcF$215l;ySX zSeyjCS^M6X&LY=)NF)Gnbhz9a5p6aX13+(sj3fnrx*7|nQyi_mmLeh=!P&Ay@<2?f z`;2#X3LC}bUE(19;GLdmk`u){^2X%Q4=0gmwy!;OO=-4LV^rOom$c3K>!V z!g8;8T(irdN-ezS^(~sB8k(FBrJ{Q5^Arl$zKp{+uCN}#daQ3hU}HFAX+A7>H)6-8 z!ti=nPM|#JNBhef%gq!13!Whyf6kRQujyt@%we_MHX^Z)%FF^YLQ;rDy^FhGA4Q4N zn}z6LgclmV|3e?yo*Cc$M7K)c@LznI#1K4-FIxNn1L$8GvI+ z?ReqcTyeD@S|tyUr}e@6ILm%V8zLxWT6u>*#xe1{*r<)dDkcmV8+|_W51bM9HL_b8COJOMw zkX+;fBXQ=dxHdJ_x)^P+Y{RLl(J#}5&?iHb268@|&des=JJAg*Xp7RNgp?awNRv_< za7^0jxOW?p(U)Ex8SzZ1cXdiAqlen`fHahi9pJk*YG<1J&4zE^y!SBCtoF6Ke49)b{`}V5 z;P3+F_ifp>Kq`6;JpyI@eB+tc^N_78g2>On zatg*jf}s}trvC4CSA&u#2%^>a`0vHUT?cQUifR1TBaLY5Cr3Pr>h@32Hy5MFYaO#6 zQTv#_WZjf)Obr1|=aEtEIg8bbLQ`dN#E-=N3ht>QaGWYVFN_3Q(RR+>GT)_CwDjFM zG@7)_^+i|YId5QnITF7;1X2XBa1w?P_=>piGvb9?(tJ9((T=|8CkaEO;{|lCZ$&8N zB&#dsxbfIiO@We@s`6e6E;iNu2FCdTQ8ZB=K}o%!NVb}!$KjlpXhj6ALXx+kCL!f= zrY&51c=fx+B@OE!S{|kVGLqXkopKWUC}W)3s|260jOLkq`Cf8}WQ1AqBx|2#=h6Z` zbK;+39Hx1{jGlQNv_s7=as_)rramk(+%zL{Lwl%QC@=;rV(o!q5trm~V8P&=34PBp z*T5PxDqhQFG&}kmBON_GU17Hms}>F!{-m|hnRK2h50lPDM+UmAmR@R6l`fTKuRr~4 zRN&ywp#4Q*HDZoVjpvs+b>sju=-e659)GcebM9N{sJf@9#JI%HZl|?{^MbO43cNx6Hn zCY58dpKQ0;$s#3uxV;s>^b+;!$6pfegr4{U>d9ZXD*@ZSAyo6O7gSV1^|swCZ{OqF z`h|;nbIrX+Q~kkNDponavoE1}^(Zw(^D%>gfi0d7mVgoNdRFA8E*KqtA^ z9dHJwKJ4Ohv$&>lFB!?}J>0Zz=N}CWeo>Dla8D>MRWHb;Xm9s){E_Lu%K2de*~5iE zb}cPwF-GnYjIuCCYWkH?WY37?j8oKx?&?*#ItWi2-wMD{L%a}cSkE>;r)R0Gh8gHga$SS|Cl-pGS8 z+e0Fe(|uxneV9PEo6(}fQ91&Hki#8Zc@K|IokgXRhooXOmggYndsqBSdMY6IWFL&O z#4^)L?ynMM7t8i|jK)p_92~MEY1cf#!)9C{)i`n_9~Sf4;H|Kl7-1u`!KI)lSH4jm zu|%xBJ{1I)e_9&#WupoqQGe}Ad)+_##KtXd^>)UzNS{ZdRzt#A_B%xM1k&}>yXAY4 zNdov1AV5ftji*GE=v!%a15=YX_ldG7tOyD$0^wfWTZ=y+@MTGkSxvx{@F4>t*2cCoBZiU-M=Mvt#ER7NbqV0Sc(T~=f%7yTm%bxhrwe2HWv-eUAn|uY3 zHE6rvNMJ)&y6)i4UvRzLd8qu)nBfus*|F`OIj<>m&nV*qK+)JTKb#h#o8!k;qLOv7 z2%~m7ok5Xn=t4+z4(tN;1+Uk6fjB;hU8jaodEztt>XUjS0dh-K1#2}{@JhH9nW<%V zNBZQ-cJp%x5S?G%xDA+4xrXsCG*a}t&4Us3HaRP4%p5RI3;phw%$vgJ^n{1@c|@+~ z)GLa!%a+tec20_F9Ua*D5c-^t@awElZD361yS2QdD{axPpbisJB4J4>wvH|)J5BZu z@hS9bFDEoDtZ(7NEN!6Tq+Q}5tFAm^wbGRP3vFg%{s^acDrH1^2aA=S9+UabhTYc8 zW(2V6)_Ic!Xsqfr?Hzajx;RI9>bR|$h<_alR|BHw`!XTvY;@9r&D4Cb&7$XsQsbhz z^Fb3>7f}46y9vG*#7!%j->PJD3E&vQJmOQ5?m8yy503>cWv1`P=*21X)&NQeP24lEUTAQ~ zOI(e|58P)=LA(DrnGh8p}j9gMjO{7lp{oa)^N9b|dZvuJ4_D_T=5A zgfo+_33TLekK&Uo!gdJX7-q#|@?UmCu_|cq9$Sl1UI@0i1N7x01ustCSDh5-Oyp!y-=VCVYXc2o^1y06yS^L}<^G$h(5 z+ks}+4C`yfR1G6(4(av{*}GTBS3vprN!|6X#%6&qRT5-x3~Y;)#gnh?lN46kf7rCs8#_uiX4+r+t%Q5LF(b9pMBQ#t37ZETba}~x=*8xh`ajT z9OZL1kiEjeXw!Etn9ABpwqyHfudsVdPBe0)0Rl731Eagq9Hi>L@;7=YTlAz~GM%Ol z#|U^y5QIEbCs#5jktGK_-mq?%pgR^xBcd|%3zp)E}y^Sf@?En>(^?a}UI`5RCCvcJp;5fp3(U$3iK zpVGbvDvkK=a(9lJ<$!lHA@`4>wN(qW%7Qumh=ie*AmKWep`>|m$fydj;7u2;CW8=n z7@OcodXvb#PZ|DX_f`#*`VD4-E#cjhe2$d586~pfX!14Qk{2MX3KqTpu%L82smIjsrJq zr3(>$B#pO3YGv&XPomZW$JI4>G<%fd%7?fRpg1=v=2ORkUV-p&bJbkHfZ0D5H|I@M zDI$({N7to)$5x5gEso2=#9Da()6o}nRaO5xC+k)MH8}p{quPp}bOGfQBU&8*t&^qz zhK(wpp%9jtvAr8D_nu(cjb4pXB7>~MN?XI7JAR%C5FOiZ8Jiokou~}?;R?{%WIg(} zy(Y!_q6d$P`8D<6FO!08c*b;h#}z2Vi{4F^hOA zReYP>(}>82c`(3awaxGv%gX!OX{mlrgXBvc%9HjL?F+Qgk%6;Z(aza@ z_F{Jd78F?qm$-H!7Y7uaX5O9sAr3QaqYha6A;~q)3ZK%kY}q^Jx?cXNj+fxjGXc7q zir8#0yJxbwZB7H;X{X&=-Yc&X+b)#b%PC~M>Xm$@OCOv^P3=n2f&U`xIw;VZ93&m zY*UKAGhhwu9|ZEAqa191wdIVF^K^Qh(z-8~hPeA7AF@`P;#x!||JjmazWP}4x6?|0 zX<*93G%exVJqx>IRj+g3LIpdkdOIm#YLqZk%nGWF_7Ve@BMDV_SWSAQC4y~wD}TWJ zv#&wo`0~3yC6^n1aDo_pXDOY$S6zgjIngse8DLM?ke>(`^cRxwm}Roa)-3qBuH3l` zmTJlSPxe1@r0lZ=DsZCJ&;-t4mNLU>h@?Vg5t+S z<{AN17%H!{E6bOSq0z;qkmPxN2P=VL7%V%T*IIpe{PqF7IOouEE9`wuLLzq6>GUAj zx=%^@ed5#11h$vPPiFe?;)f5I)xT&La5``mYYgEK#8lrG6l*HINBDq{PuJe`YU=e5 zLS@O-%r!Ep;*y`M(BM7S=sihxe(&q!On!HZY@fnk5N<@Lg|ppar``3#P+;8y@7kT|)+%Zpy)v zItvT(&`{Z{Aj|N7?8dQL-8(&Gee?AqznQBDB0B14%S`sJHT5)R8PgU{MiHAdLokA9 zZ`@?)Tr!%*?LO@LCl zP`iHy$nM@9vt{oCG96wALhaFGR88;G`ps>fGdq$0L!imyogzd#02Z@0kpE^42+gf) zB&drm5wzdS*2kDAbC*ctC>BNTI)p{P=`em@j%k4(pJn@A1suL;kZ!!xk(8^C2 z8T`^YVuUC^TN9A!zoPhAI38`NdBUJk0Gnn=6|zM~L}ifmmV5M`v^UmZ+vtudVLP-U ze5+H95Hx)_V(Li>ed3KJ7C+l zUiTb*fgtws}oO1U9s-Et<)yffiOH<(i_wu>sMhitNovnuY>0lk76 z!%xlAs6H?UiKjawwq;mWKU$?|yM5DCj;p7SB*oo2opeyr&~2`+l4kOU_wM`-uQhwT zoKGsg?4Q&KChcJRxeXHTrp+tK6I#Gh@-nIua%69O+d50>Mmi0;ejm|Dp3#Y2rT?BR zuyGC70YF-S%eP*Gk;K<~)1rxD2D8NbMRx;)uxkLz~{r^m!? z6Ug=#ayzT+Eg^XK)op}uCbYKps;NyegpR{SKdcxds1|_Zdo2MTNJWhmCXd5l=!#!T ze9@NF1xmgP*irlPkMu?LD@Fo5tSwvx(tla|E##*yCnv%N$f_}lqFBUuPZh-^jTt__eaTZQrt$$3{ zN@%3bqjh~nx$iXi>@$8m!;)~1&u8AA=hv*7Q&o21M9dF-ay2{lAf>-?+FjVn?c#MJ zF)`L1%rY4;cG_nH@b_IZsE5OfGT?(?%H&C9kH16#Q(PmXk*zAJdt!M3_SCx|BD zld4sq8`ePSSvEEtsInNaIJ@-BvCCNvh7cTvd~JPN52!z8=yJ`5pFd2a=}+Y4grace z7GjuLrp9=jB$^}Wyh9tu`r+D{U45)&=MdNYXaRu=W>xU|ecvg5)TcAru+)G4MR?M& z5c;_cE|{jjY_3RZR;Ed2xjHB;`H|rBu&^+z&6XgV?Piq2vV`aM?whtp@6iG^OQ^-- z)v5>N!QO|9=_X>F?EY|=jHpBP*oD8TexyJ=azkWCzDb2Q!e9}j|Ki>zh={%=>8V|F zGkB^a}Y0jRhI1e{liid8(A%<@XX1zv6QqeK~le)peS0nm4wx zS`18DP9IWY^mKZkw}%r|TA4b!g?hW*fq`_lFQ%PTXN<4@VAGT*apb09ZyRF*0*$jl z>z>0QM>HsD1sqXJ>N1_D@r~&XBg~}SZ~GlArQ{S|5lJLfY=oxk{%YkzPxC(lXaBQPW4NwY!?v1bZ5zZ@(5n2>It>?MRuzU1WB%oGGEq~{IcT7G2r>M!|KQKl4`+lR>e;e#C zLi%*6G`A{{MKY$3i2LVuQ+i&&?}+ARYf+=bz?Z{)_Tr^intI;V5U#y&s<2e?%Zw(^ z+jky!V~lu=V;iAY(RUl(GwauI;EQ3ay{3({+YK~0UjWW0} z`unc@pdPO*@Ml@YJUi^%-FcBd=)|mBbmj@O9ak%q8a|Vp^SgdSwlW)UF>o|ukoqvw z%jnaFhjB^RKf1;Zq~(j=uJ6(MVFrp^?Bi}f&iC(BxB;#@#ntUAOlOH+rmfwc=iGZb zsH1;*XYmp_41{gv{KBWQny(W4PN4E5wKmFg?P!1#U_i|l0b(saGNgX04+LmoAj#hI z=HmMIdh0#MXml>c?{yA4(}blL0k#X_e9j-;jFs#DAWo09w9BNuU7pJY=~A0u9LRt-mG1EsR$im6VZEyczs*QAN-i%X0DuTwTKu%4gBJ zNo-!&UwwGUP2jUvWE(c$1?J}FbI=}MlRnWWXqslnUc6aEYurg#ZBw_ej($x9^7jh@ zo!n=rA;wkOrl*RJ4lG6zM0=(%$4i5I*&E*$=Bq^E7o~T)ftC zArFnX{^KKXv4QsIna=5<>iwi}>P?TusD5?SA08dp5u03GjIAl;6tq8Yhh`(cz2Xdt zHmIOM&58<~7C#+eP`Lr_+sjjKv*NxQpUF<mfbzl}V zeTl>Z>5orx-dK~*55qsakBz&g{T=I(49eVT^;n#Ka-hrkX3>ylj0IEBX=UuG@V47i z=Q-^6w$|mL-rRkY>Zy(K+>4i-5*8x7;qXb(Ft86W_%5U{N9@`$=tVX!; zIg!^NKg%yCwTN?b=J!YQPLQ~J>hr_nk>W9a`J*AXatRI z%1Ev{9*C(H=%#-55j@K8>v!56{Df}1)}JLTh;O~q!@TBP||vOmP8VT7;?_2l=z$JK&8UV>UD&tF?GMw-MT;Y&K*8h0!t4X zAqP5R#1RkV!c;BOjWgEg!crjF>MFdlU2#khmAh5`NDO$^el3FBS;NL@lhC~ecp+wZ zFLtR+OIBxp znJS@x>a+$&rN`_U9Vvdeo6p=OVSqD=vYIPI$C0`JGNmOCT^hbK(+LyW>#5WFQl(PS zPpeYIHEW|@=yUt=;i0WMpBl^k0SU3u-e@_=#8Wq!$Nj-taQ_wQQ$mSW;Dj{}okj-& z+>@V&_mWhY@JLB-$U>(TYV7@1`revt zv#l_XL&iQ|Bz7~Dr!_L&&ksn0E(jrJA4A#XiyR27Yng64jy{wEIqe+67B5n#rmfvbWSj-n)ROmz+ckv$(<2U)pz)IjjMEXG?b&``W{=%?V>E zywBDL^~5@jYvYUk`!lT`!X+nTdz0nJH*xEkF)b!|u-yyK81SY&-qS6Gi!aPZ;BMiu z7aB-0rci>tgZQ5uaBJ+dji;^ww1b7$$?5XIBH z%LiUK-ATYEU?gUAD^+N)wF8lOCoiY5GJ(8M4%Pk4kEH`)OGvbDHm83J$RFqgOQ4dv*T>{syfBA*|E!93(-`>C(r885cttmE~#2j7K^hJ7H4wqw?=B%;xJ_Tp9A?LDTy!M){e7Jx_WPJg!`CYzzrXXUe39jMb-X zx>GZjZ>SvF8%skG^u$E_Vn4XK5?J-w-ls1vKbgB3asN{u{NKKoa*bKOg3tp&{g?O! z%1pCy9cAvWUr%VaK@G3Elm;f|P-ai%KXu;!?bSKT_JR9Y>DXW{VZbhZy4jUy)<*T0 zOUdTVt!6dVfb&6@KlI7sjZDXVkXTu+6dja0A~bZFRr_v!He2E^TlinC87{kZpw>{@ zqU~FKgGqDev?=-$N~_vSf6)cQfA8F$-0t3|Ffy_aWg%}4IH$kBgoTOeB%tQ@*R1_p zOE#l}zWra_{p-Y3!szAx{@$FlP+G(Pbvw{oBa>&!KM8e;J~eU%y%V;s4<` z6;%IM`H*JDm>&CYx`rhy$;gKxQ(dUl5*NVh%E&wZhU#=Ga|%R9DUYE-6|jh-GH>f zsHEy+CQ-R1_3{in77j3h?BUfC$4wSept z7!2>Bg%ceRvf5%NY}j2S%W2JR^Z9WXbH-urn^#ns(iDq{PfE?`;cPVK#Ra~s>x<+2 zkB1U_nEuxYi)AV>Wa$C2yi0FP2?W|wjNW~$2a*N5GzV3hgq$CQCT7&~7bMD?{HWiq zIT643eDRA>q;jw60xkE>F6Pt=G`4cO@)2@(Cmb%%q>PH=)j5=2YU1Zze zyTwFQdU@da#dQcOzo7+iq5!676CL;3=Luxv&F&@_TF&&@9Nn`^>5Sk8gm>BZDHs*- zp2jzXn?QP-19^e6Nt**rQcKKlZ(d=0L;cIMT7EX9=kcVQ=4jrQ&YXAw?9!74Mqc$c zv1baUeeOBC_jry;A!ca%jRF6Ba8*J;__-r|a#wUq_J8Kt|2ZHQHPXN{j(G}n5~Mh5 zdS%W&nT~ogFh{=mTDj?BwnE|TuzTgDLc2ecyosQLhs#?UTabMU9Y z_w4b03)veAKhQQG{T>xRE3x^gqGMG10h3<^=t-D-5;F1<8fR{H4anK*wTBHFXL>q+ z=E>4%<(7y6{Kw4sYuYCUOHH8Ck z#ak0#y9@tibH{=m`rw1Vwfr=2di-6J^|W{~O8y;9HBip}O!Y;f^OfvaWjZ)nBzLjH zi$aH+j9Q4I@9^6H9f82QHEh4!iTdB%-9HZJ(|a&lz22y_NWmY>yem+xCQGVEFh)Ny znJ>M9hD$RpEQ=8d{Sf6SRN5j;pxY0O{UCCAd}OQL5%mY?=}FDtmuRihDTSX%pGes? z5AS`>yD3jfWqqv=vt1nDr__+%*F39IjtMpAEK$ut@XO+g8_S@y-cxTZUmQu@wps1` z?$GbWVXYD~asK7TF?#y>$aHJ!idbN4_!CR);b@CkWV^Q#DYv}oy#%(_-5@?+a`mgX ze%$pr9oM=m9e)ZXIIZW-`UY=`>_A_WY{x$}TW6W?ls8bW&0!kAS0@#CWp3iz$m2m< zN)#7(VfUsTjDvdC8DanFECO}0+&Evip6|{nxuDg4OC4_95cF_?`Z;%E0)Cz?$bZ58 z7?nxkb$*KIUFTblsTonpg0}m;!>&ZTXUpe?Hr^^?Z=&ZU^>WyPamAH6h6VI%TO|I* zOqW3b4&cTmQrg~kZp!fK=Fl1j*`28W3g1K?b@_6aE1lwIE6*2f6Hba_Vy__x) za$N+xnDQV>ZsIZ#ma!ra0_|2E#4a9kD1Gl zwRQFmlFJY@pQA5)+ze6kIO9gf{BZ>{5~8)N|+^>aXCljfLln0d zEEtbYkb8_h6ze1;==}sz16y$SIZjU|f^fg@Uk7ukxS!5fEe4ssdms|-I^7?47Ac;` z`80o_qxFbDC zEf&l%theuBH#`Gv)4}g!pxWs*WZ77%haPx|3q4i9Yv6CzDMj&DS6R0|uS1&hMH`Pe ztKWu9+7D``xjrUsv_nk#W7{;08`OWNYWVpkrL zfG2X=yT~jy1V4mD#rwcp2Q}ApoKPG{&5GI0?CYWB=(9^SeBC@5Xu5pW7lAT7P4kvYm$h>5-c) zVzqN;{vz3%jVq`fug^M8X}9${2|9Ae2ElyvxK7@)QYNREG+qi_t;y)zEjJqP*0k-H zRw@wV=UX9^r*nYs?c{P-AQ(y2A>Ax{ciY}n^WCA0#?xC>d}3C12MV#XUGqu-Ag`;` zGQMbr6iHi=)sD=2G~xFVIp`t%PTNDTdrE%u(CWqllBY7hhS4sOt@$= zuPc&S#CRB4u|n3(=;REa{iPQ3Heqy`xz?~i_r%@f`AR+G1Tq7F-kOlgB#AkbcL%N$ zV%8roriMxj;p5+M{sE`2Wf>_pDJ;e&tClukxyv_a!qfz-v!l1O*PhIEw&|38MildW zEtZ<-vq{ka55x7b9nXTTey$uJ*Skff4v_q6cO zu1(VX{sBiWq1l>5F{UpF$@N(nb?H)*qS>PgebYln;IbJ(TfG;3VC|2%UE16M+J0+5 z_D`*dnk6&Mq)(|AR#WO+ug7-LMSpkb9JXBj%wU<*6@ z1JdF7yyuacxJ*nBcoM$1`H1SHs*U3h@x|rmQT=J>3)>c!y5x1QZ8;e9y_1?~Y{_>w zk7#_x^6#EOWx{Q;toj@C9((TTDZWC zFJkTZ54}{vb*jfBKaMrX+&G6VRGu@O2^_$$pGF~7lG$P;V?NG7OyP`$#`Uu@NOO^6LS5x z4V9Q>(qd4g2KU}?1I(d-0`g4Zhqz_(o)XmBV^Baag|bQ?+}QSG1GS{xGr3DR-oj~n46*o+ks;k|P)CfM88~Itl+#9M*nKA_ zq?cVjflGcgW0)-De8MC71aaPR_V1xaCBFtPhnyxj_cnDy5~Jt%kE~_%h7A0(yq=}L zK;F64IrUSTFJ**dd7?GE!FHu~7D3Je4_EscK8wGP)&!?73#IL2G&gBRk3Zc$HxJBy zCgeqNe*3VWpr3G}ISkJN&%bM3XU+V0=ZnTGZvn(z-|jLJ12ZS6nUlu&?-%7m#=HYv z!q(24(6u_`->*}UL$vPumjy2R80GofeXW)NZ&RxmAK3%T8mWz z=i+7#CKBjB5qy_=)f?b0i9>$;r&k4Fe%?#=oF~G065yju5AkPVf@jZ~@JmWuW875L z->v^@#U-Lm)k&d>`20 zVs1o+M@RH7=TxuvzXZBtiAEY&TEal&A^=t#=@tJZGh;}7u2M#lptcgDV7K(}!5c=w z46zozwZcoEW^{dm^p%O)#se=-q`}vm3_51<^?n`Sq(({z$>TJF4qQFjb z2m9GgtnpY6Pq%5BA93GP^-Q<=a<+KYo0(KeIrkUps7VQ0J}tveQrIs3$KqKy`e;llyx)S38~WS$Vb|xu`40OV(J3I|vl37MiV)~zdWQJEL&MNAjlWv&<)EcvNF)(D48R7K_SUl(Y|A)r+$ z|410MzYLXB9|eXi-mIDq2yuLqAaws_(;~j2nTwoRwl*5*-Z8po;IM7#yt`s*NC$*h zkn;~7NL6F6O0vK?CKmp|MF1*KZ4PwEO8;sq@1Llca@4L3o3_R0482bi7%r;2j<#c=q+c+(3P3@$f5 z_;SV1r$jZfS^2iV)g?zuToELz3+idnQWI`aE)j#5Hb16trFmxNO$Iq>>sir2$8zYUp(v|Dl2*yBsh1^DNW zx6Z#dBm>56P4302szlpa1AzPb0JAkQ-#E_uMNKSr#P1s+RR$3LQNJ6Vw~6HvW#2}h z$^Y;>yH{3;9Y|ncO^q5^WSri7W5|s6dbDIc+c2)(iTZ~r z8Y4Ytx&To9EMk0dbk}B}Il_eF0_|IOBw1m$h4I?shG#Z2y)l4gPQ$e}{?kmeJJJc5 z0A#XSW(p`G@<^~7ZZ{a)F6wuf$Wwa73pTZ1#0c7;2mNgYt-Dqm#zkG{xl5i4zzQaH zXRDTtr6fWLn=!|;bh3fxLDYNovp>Tsp)VkWKmKB_AQZf zR8xH@<)&enEfiEI%LZeOF+pn%RCVvOsXk_Sciu*=aVhK_VZ3$T%Vc_48m_;d)qfYR z!6;HQ>JDHE7ePFKkI_J~G4Z|rBF9AL);VE}2}v|Pe{Id|sI4&`2 ze?Yhal2PheA81#f8&6r1C*4H|YK`!AGh63AUPTf-QYqHAf%W_tzVF{>@Ili-Zp7&>6_Ko_DYk7)8Lxc_+AhMVzVK2RWxv_{d}-*B=bTW5b_j`LC+Y? zReN?~=Sr%v!E*c zbm>|ghphWbN*JZ9kTC1y1XfNY=u}YQ(>|sQzz+v?FZMDH$*U#5+lB966z}oyInU7Py$HT# zF6?Z1oW*AIy1WysA(@BRm?Zq^3mg|+w|TV?mdT%X-6{0bPEG0lEK%0ufCy%hP}=yT zzUU;+wX!|U{nK^e1+VAdypv>XyofnXRAwjR_UW$>?og4y1|fA z`Q7>(b>9zPHCwqjY*s_P5|SpgZLmQe`Lf`eeEWrHf<0_Ytni+U zJhqPPK9lOVCs$qHK69(6NT~z~VsaZ}bw0nhkfSww6&&k;4n$QB}(f# zGD3M~Tk{vQ-SlhRlYNEGCH2$dF}bGs^)jZ`YB=%1 zC>P7aWYWue(E0eNS)UmmAv-n?ko^rd3tAU!?X{5axlMG7PASbTWvu@w`FV`i{xghL z-ZDMfg=iwCwROI0&G;pCz(iX3%p`oVR8{i^yAhn2d6UwNI&Su%x3d{esYP%s!nnVE zJj4}TXA$P!bS=8g=WZpr+GRYXb8C}*=j(BQ|Ahmo{uT8Qf2W1Vi+H8`so+jkWBdF| zqCS+>hn=RaJ`lQ^86w};I~VGRL$kgPeF6u2;704_fXj_$L)>C$8QZ^J*9DibQQE2 z5tE;K_I$l2{ku+xhh&+xUR%u-z-7tXgi6`Yuwk(WmXhKWtMX!y*fJ>;43qBlR>CMG zcynITv^r8J?(pwnN2lbF6W)vsh1ty5K0uxt6Z#Nj5FUrU1|yernrADd+?2=89Xb)jdr+KquWZGdY|!0Ys> z!=3aJ4?oi-o->eXhrZ)V$s^q+?rZId$K#2MZa07*_93^q{|BAnhP|LvhslC)nzz)j zKRYjJG+H1m-~2wM$M$5W($23M+jkwGYq;P!F{FX~opxex?Xlxn%$3dOz+gQ2#N9Iy zVyRJJe4Tpn-O~@*EOU6?z_l6CHEG1cx>QWm8SxE=Bk%HVYj#hO$LoXYlzR>-lszpE9kB1 zhdNk+(d%FXLDHLj)U1|PzW!RdKbof%D+%c#9ldxuyQfQnW*LOLH3_YO?p))>GyK<6 zwdx-i2lNF;Cd~U-kIR#4f*ujdcI^gQIH9)CdutSianqTrx`g#Ywmi@6U@VQcA@Lek z`)8={am7(AB-u%zvpC+-n>`7aP^n0L{XCpZ-UaeJ za{aJeXpquC9X482uX@yfT7oh8k+N)YHALF|oym46?%ueP`4Zf&y#7!@kltz)*;w5T zmg>{+b*pOiwb-6E{7GCv)k>l5tE}XsF}A?|p+L7EMs4;A9`Q&KW97q3V_P zj1?s1^mOV*8|`D1p2ihcRTQRTB9{Vs9(esz#!_SS6$5u2@ZLy1E1Cw(@N9$3PMv9{Ci4Z)yGwBg_Q5TypRNO-Go$(axO3PonHe|!5$9Sx1KzibNnRLU`O`~~JgUW8`yhoLNVNILP z5QWzLDxQy{hC&uW6Na~6_A%i?%+pXGo{$MFz)lpjILNhSnnp>sIM}ukGhMZeKbJw< zr&vh3`w88@a6E+FG{|o$P>NBM_Jfq6D=eZ&;Rj!o%*zp%YyU7FGevjCAVmzLu zLWyUOI;|C^9A+Yr2bAnEr18WH~soBn(2dqMT-|8qM1b(@Ey-EW_RyckKoGP+K79k0)OWzCZ-YXrB=XACxz1InsW6mqqv~A@AO&H7?xs| zHn_J$zn!@II|QFDtdAsSa{_ksf_NkV8((+^Tb@bf@*e3(rE2fG*QVt+hw`9n_0iz; zT&a2kFgiS~P7xR9p<0fMq|<{hWBx$)MxVd;VuLvmqI|)J8=Lo-bk+|~CnmeTlW;6u zvNxH@^qqrK+dcM)a~PY%W26@IUTyo$@BgAe{ol~B6B$bDZlF>rox8wHzR83K%kF?YW&&z;oLSYpsPag{zy#okIsp%pl9Op@d9cvv! zf|qHn_e}C!+#!0`=yrjOhSW7@1^*XcZynXr-5H zh661#|MmFWw+jzFSRc|uKa{+F%nDiHD<)1J`GqeC`39!lN70G*VH>I}ZDD9*8zy8f3QFX&J%? zgRlBpD868I4K96XQyJtg=P{>67+WvnI9mzkb9j<7-JB`J@=eDu-Ov4rCZb8Wq53Se zdVkK znGPCdg7YaFUkTuj>7KUR955M-wlDvSk??}`(K0rq_5n9jX9q8Yeq`;fR&CWc%UD>f z*_PDBL~0?a(^uU}1Ba2i=Z|*l!YMG00DIwj&3SdbP{EKS(Cp@8hP{_Ew5G=A%8w+F zIZYT?lfw^QZCK*Pc1>0*XN5OoQsEScx)d>J3E>896N)@L`IerZ=tygR_#rsve}hxW zWU6M!Y1b=NWwenQEuIC0jw_9Z?8m}k17a7> zsZE|nSGotRtoO9!KC+=Ib=qV@_=^4tzjXpb#(YOYXTJ)t`Cwc~@T^K2 zT>rIFnQ%bTr(~{(oE*tWxUbQ__P~VU{?(%6nOMVA&_z)+Otq+nYUx``GkS-^COWaj zUyr}LQ$)!xbE2jUQ<=y@o*AKSiDQL)d0z`XI2SuOvXM?`KVGel&1c54{yCoT|5#-~ z{Y20$N9zSzY~a!tuXTbBWJH&QD~>~u$9!Ik_*y>h&Y`yzzb5GRbD`{FjT(k3BaRjy zd|LG=hu-Cud)f;}uKA`Y(y{D0{xSn4&E%j~@=d@tbb1ZdrI{!Wg`SCTW^O{J#l$6( zb0t@d=LoU^ZzSPHL%T)*Nx-10aB&5n?7iogme6~l-5#`Dp5Nnyg~X;*G|*+=j~aNV zokEZ=k9v;z|F0CLpDWrpZ~6T32;H}lmqrl%qfZIjtt)w&KcR^xHSZxlA>UYkbm@@^ zoZIG{t^@rG_$i4sR8GMgao3l_%qrw@$&bY?!KK4 zI@SCc{b~GBKzEz7q^dEeUHfJKzY`jI!gy`~>g<_T$Pnv?eHB2*ue>y`^gk_K?0b7% z+FgEFQu!v*5PF5r(f9SudqVfS#Xd-sm>nqX|0?uc0G20Dt`jc#K%&tohSdq;s zHax4gD8;;S-yhu4qPoHLr`U;pB<{APcXyhzIc;UdonEty%e=J|oIaCVMY}C_KA;;o zT4sLPopnIv!s>SRZ~a^6cm%VZnB5Q0olxyGb7QV3ots zR5ZG!5t+Nnp{sa^*ch8NzpvLtsR>YSr@0Yc{k~@SG8+ZoTmEkn9&jUjX`17Avkyp4 zLz(@>th0SDnx_KxCZUPAY7Ov?Z5%VXd7*VV65flFiIHE^Zevrcnx>xZEs=WDRAZHe zjcwfV>%nOE*Y3ir=!OX0l2ao7M-N5NR*+R@3-XJRcdF+8GP3VKj4V|J zIJUmU#+gW!jJ8oK`hg{>f&N#=t9JL}*Y*4qJPQnh749ZI}NRx4)_W^qrMDE6VsRea3dtt<`8xFGXL|g7M{` zjYbvBw+^pb97aN-S|@4sC^#7=+!|{q`)p>RCQ{E@PD|L`+Pm^{4ru556QKL9Crb#6 zH()BZ$!7ByVlEN`M0PCSnvXCnT%2~mkt~# z#s=nY+CTh{znp8*`&ETRLy`Wn*`u?8l3G(oq;f)&Os9)tHJ4K?DI^zq|5b~Xq157^ zN+z@qSfY#9=Ib_ClP%g-?N#)H7XJ_%t*YzpwQwJG^@E2EuCZ zTo*%}1jhZS9$25NdQ_4#2BO!me_rL~Ysd%pag5A#!TUE6uNwxNHc}Mnd{+wFE1L;uaF4)v6F^Z3` z=9M7|J+YxLIK1pp2?zV?nyW>yOY3t--YsRpc8ai~&rU=SuFlSD&T8Yvth*u}kp7Q+tNP0PE~Qv63=y+UZnAFuHUxK~8~26$TJGKb6DgHN*q|F) zOp#(PpsRVx__%~fIQ?jY^1fc*f^TK!MH9TEo;$eR4&vA)hhwLm3n{(%4=N z0S_oxR1Td4Efs4%_kkB$RPO-n#gaOukbY)bK}~$7?~R@Uuq9s+(c%Fj_A&|p(}SKD z_@gRsgByWi`^R+cBNNCwROcpFSMxeFFWC1WpDqW&i*lLJ6hq_j0F6XQo!+Hin$`2z zs_(#@OBX!pb0!E>M~FlDMikroY_YA7-nw3r5KuYE2y;D{&MT~B!K);&OHygD9$PfV zHAbqTlHKjtyOFXdSZLX!-SF%(xC>hLc0RoWgF452J2x`#?T~Wh9os`&_1Ta8)JbR4}rm$n) z5kn`ICD#0X?`QMT3$56_kD_lVzhVF9hxkQ`nXTCDaVdu}u`mmn)E=KZs}>x6iG7k4 zbaRWtWz-Z=#HnDm`RvYW!A4-2qYg1(+vJ_h>+$P`gAx4)VVyf-XCXjGjEy>KHdfaN z8kTonew>H(TL-LRhH=hd$&p#@$95JZ3`CQx>-x)KsSE9|d7|8_TKoxw_ejfi-LGO46OU-he6B&>GvV z!-L7J)n}%QoUt&MC&8~c?8@%scj1hAlvp*?`J~#;RYi*#k@m;c@DjRt)Pe^06TYv5 z+|{P|TdQ^N$;$ABZCz=4ORI^nrs|L^Kw@P#3Dsb&o<5&ygor(}%#){Nb0VPgOn$u& zOVzI*SW@5~XN&CSN}@ch4$rZ9bFw8adT*$aXE3$xRoX{EV`;ci)YxGDTz-3H^o!yQ za{QBPnm$Lp8#5qKe-Mt`%M-P(Nxo1s+V&pukf_>5cc8mADMg!Q(Kko(NB20MYzN7A z=mssjpl{`C6V%c5$Y{C02iMLsr#qCpzkWkX?re(ZXB!N0x__J8M;4}TN z?r;(MC}mil2}HyFN@IZGkOXAJtQ4(Sf|?-LXGz-F@CdRQFZUllRS{Y}4Pu1pp44*& zfD0ayry<5>@GOMhx7wlSAvACeYoTcQ>dV7*vVSnI{cho37(A9EMe%qd*=6>QmP)@f zeY4?Zjr8JsSZZE8R*;Mjr#wP_ZJ59s!7?~R^@w2`yzdK8W0MjQ7yy8 zGkJ$uc0^2JzkH!b#X>haa*0m`OVwmhM&=h1QP6z!BIb& ztaY$B)Nm9R>1kOtS~bK5@1L$Y2@Jnyw4s-je|h>wAgjuO57RaD;$l+MGEs+8PMmzLKLp>K$CnZ(A(A^O zy`k{;Z`X4-B_%zZafelxU)}#0tIWR(-z7b~ou`Jj4eX{j_s$4sL!q^M~j* z;fALzr#vO!efIk=va?pv^OapcJ^V!#&W1eRtQkZ-S1=1ner|Qjtn7Lo?!o2$?8qS2 zaJ_P{T#8dw7PzUGWKbuxqZev%dtzz;dP6wl*$|u!bn<^(#%F-$3pG1U3;n>8Y{+ywf6izKRZvEWtW+FPx%_y zB{>08F7?KKkgb6_M0f@s^Y_#};=G;kcD7(7;j}-~{u8CyWitAtOVos{p~fV)o-4~6~5cYn*xH=H4@ubmgD&!Er6~=A=;!)_ag&nAG(50d&Lqe zqU>mW%Um5}Fz~0eC(Zt4%=1?In>#3O(PR3>SqY~UTa%Q``GBjN4`13D*0vvfPv;Dr z*G!{WkKA-RMQWEIkEwKd1l!q*VjVb#e#@$JH?*RZ*h4fC$NQy-+Y%DGTemCcPX-|P=@@N~E$z5ZriyM8b1UM6(S zY$8mW3Z-MF4C9dGL}8?2-ee7M&k2pwI1+GA~8srRi<%d^IRRFRDPVWdDV z`$|m@fq&Bd%B0>wH!`A={N1Q~ShLU9sW$hTlUQNxyEER??R%FK6b~zQMJ^ZdiA|pC zJ1OSDljK=0W5d!{tEF*9(%}f87<&|V1V%P*RKWt8Z*Ur7ik^4D`TZkzlwdHlway7RJWvu^T}b^ z1=wy+TjTYNrvM9e|2+cH_$n`Nz?@68zr~}x7CFMUI;)}2c-R6kR2ren^sK=p$G_i} z;3{XN{>)ot)j-76VFXAaG}I68?+jq|FAf8jCty{n!bT;ZhU9xX9<&UOce1!>?Nj@2 z^FX}WEWw*OhNGqvpq$G-x)b`7y_caruTh0qd+s5j{23ZGTf4un&NY;yI4$R}JFVe@HT2Nv^> zSf#wY!$29csS+C!eH=c2OJqX+^xgl|;3wQiDUjcc?dtI@Ey;%|R^4~!6heWaDw=Ti zH3vzWr#f(-i~2H+7a+LWE>sMj8)8nm{eQl2fJToiuK21Nc$K;wq}nk84)~5NV8enn zRuLp$$NYY^=L+8)mlF~_%2Bjp91aAaB-}b~Sra_ozn)2YcS?Qc5Ehhh(nO>Ux%CPS&Yg#I3xR<2b3L;b(Yf7JCTgpBb9)B-vofwnMe9WEKB2^T+lRZgId{M| zeC}J2>D#lI)PaT97)p+o=dZ>1+7>RqxtiJX1hOe6*4dI+(RwWrwhNp;j3B7Pjl^X> zZGjBzg+mGlfV#wWzTQlZU7)oYc~i!sqs33qV|p9-Ujdwa3vk?D@hMM(<}B%p9b; z3$iYT8~GTU-3~=n-DUfGZ{KqZ5W*cTGdRoDb+ zP1afej#&%o2_r^tIJRV%P-LOE)-a)wb&`0B?S1~EW%vWP#+SHc_N7p5iq zvUWRkwRUlcQQGEy-l@zf@00KQcRXEyQpZLRDMGJvOMB1nBbN#y3yqvZ6?Ej27#ubU2Cs)5v0^r=hiyFUeG{^^kFNIOMs4 zj(SOgBKn;387C1(Jhyc!SN^@5dxuQ*mQIFFBPJ))JGkg(@5W>e$CHoQ?gr(d<*4br znFV!g7PeE3F6Y*t1`d5a%q9(AO9fa|T6^G|hs68<9qW0sWc1PEYASrgwrFXO=-@Ue zv?bItv7Jnt?Mz1d(O2gW&f%R#{?yh7#z&T}H?;s+EhG|O^R^fBI9h*h11AS-aGX{c z_C}~8MmE!DUMD~Q3xL+q*#eUzJ@IJ_;lM7~=!G6NPJ11rS8S!|jC}{Ccp!fTL&~Akvz8}6KBC_v$;yq+8_W#U#7I>COO6$qoPk3C>wclmtn8}{M-rSl( zY$GOh`g8Wlv_d?GsCr^(a?ZMD;Li0?AP|iovC6m`3|=%GQo6982)lHHqZMtWc<(F9 zTWIQgh)mn$In@Gt9ZaBN`3)21;MbdjzH2O#V{u+oKL#T|KKvZChpapt@4V=9`r_2v zlDgK-K|F05V7B7f(F7}~=->`g7rJ3rhD2BI^DGM$4 zrj5E>8Dn)qRHS%7gLM8TOR0z43UAWY)K) zYCI77j;d<>2V|yItOI5Hd-{rNb6rFVZZwUT#95cRKh8*@#oA5nO-0EAB>k~=0p|49 zAM>Ay@x07xwqM4Fg*s>dMzm*bw&@au!bG}T9S*wdkndBIDFl9rCdH~CRi#>uPu3c$ z9kDicApvE#+JPVzq^wPa5X1Y4Bk1bwB15hO?$s$Uil`}UWxxTOB&R!TxOo0zG^72a z5qXbOa0Z!_>>J8$mut;=(znubLyS?IQ3-XG?Bm2G;}r&r8tdV7^9={)EChs{R597q zW>QWKYU+_c@p!d^q3ku4z+w+TA%N@g(*|FO1P5^5#FieZ$xD?h~pe8@qNJ!B; zv_NW765evsuc{z1(om2| zJbR@{osrmJm~HlY@h}*5wuV%%t;ncR(jgB=Sf@MzspBkB*wvX3alUufGGVIOU>glS zrha2!^X=C~2zjj7WU`<3w_!Bl)2cmw3B@;H>fMuJ??^U7o$h^=2Fs1)OVyM-RfV1T z{wbc#samWtU*UunDfgee0$~7dte-;LhJMYx%P+y-oTV%-*5V8olM!>%j*GyLh(9%@0Plw8H66XKy6*BMiYQ|B9i5n>qRtx& z5)b-wMRu6scbs1PJ~M@0X58{!;0<8yoXfw0a4I0`%i0aN&6KM$u<9BR_IPs6cRuZQ zEu*Vd==a4}V%Og$Wb&ko6>#fsi;tfKMrRwz`x%aia zf6lz`NOAXOX1MB|Dodk<@{{v9P&7&DPQj~1?L8Ods67BkgwAPJF^YjeVJm@ z-`%{Uu5Wdt4>;6Ko@6{=Pu_Rcvr%Ziq;)9`;xgN{{n)y+m(|5)8X&0P{W@DCJ?ioc&3l<4cjn_$F=M zyRekPxZ0^^H52UTAm!-xn{oE3A+4iL!EwVGJ)(_p=ZLcq8t`;=ppScl2GY)%S!@@x zBj#`tg|Gh;M9P!+l)s?dppG!CMKU&V5^Cp1rlQsL+_A^1R3@U!2ep6ElYUjJ(hE^3 zTy}K%n9HcGFTj6%i4&%MJg*Er_`*^5`YE-#^2+nuFoyT@=*Uc+=Et~Y zRwfoR23470y%YYxp5GHDgz3aQ%qqQpmir%W3rG>Vw_fj`xG~1$T zjSIn0i?)ZYd>EnTb1SiC$cFmnwzNUupaKr<9 z7|6++&9xZ*$`Ok;H`IUC_8H-AaI;sb@XS=pn#s=2p}joE~#)^vWwR;$yrxiSFEX{t3d zyLW8~Q`MK-WxlrtHQy1;RpZyMqZW7Uw5>5*LpSVE$p!BxxoFDcc_XsWo_5T$BWHA= zavvy#*u>xx4|-9P9byNss|V~2M`y>Mu5N&ENJI6OpNTlbWJ@J%L!TiI5H+9$-iWxv zeF!UI@WFTd5wAH)Ar>M15Do8RGUl%E`R)`&1+kZw-iL`1yW14N)Tgb&?Xmp%t@ZB& zdU0eMsJ~y`r3u<^lozA~ehtWvP zU8(*T>QbA}f17q%35TZeCyT4Y^##ZlhPb?&`HootH!AlCts`CQOaOGTn)eME<=d`u zE}cI$)BhhYN$-%RkGm!P$D%zYi)|%m5+^KnY$f6TTXX#%#Xd_pU7xwTm$Kw65eJ4R zrDuB&Qe)USlDMqemi~rh_{R-p5YIW8Nm^;hpbG?O#PdFe)CZ>F|MrbT?A!ld6>o_` zt@FjTLTORg?;-1K<15IdDTQYZCo*z!1&?k{b$z<0rvRT3Ud-df zj50d*J#+9+7tYS{dVFC7y_whLuU;e`hTIhtP29m#LH_`BUI|3tPJl{fd(Y;KhlzFL8a&pG^@oqx}Y?|I8Q>X@! zyd{E@`*zIpo${yHh3zNU;-GKh-KJk43xUDMbaEJLK%TS1sxFk=ZkTAY_J~FTbaE>V z&Tf+wcX?HGMX^*vCGRyiurwLE(v?@m*clW$v3h9+2e9Vv8s$~~Y3p!t(K3Xd`7Wnw zg2C+_gk6{^5Mn)E{j^#huv0YuCFKc$x;NqbR;e|0FCCukG4rRE1C|Z!qZEDP?)iw4 z`Hi)5+$U8x{PPDWJkLai+Q&^94)Vb<0dl!4lOeoxu)p?9xe989iRH?_sXAS<){y;n*J4AXA)1!IP1~s?1^+VG!bNb;M{$wtmqTxU z+@rSI9FE7kuZc#QH!6uPa!-j7AcX#76k|Di#lbxM>QK@R*I{8?jdsZuiU39k*S@B9@IH|&Rf8fgt zy{>yW+hNwdrO0#VaI_p)q6F@rW{R~bLIlhDjVfF ze!c+A+VkEA>Sf&>lkAcOTa(+#sGfCv)Y<)#CUfv4=T8l=-0R%Y%JC~8j$d|7ZLZy? zo7HDn?*;cz4#@R6x*L^*|5z6PQ`C|7XS7_T_&y^oFgYXnv&y|Czc=zRNI)IY8~NcC zlHXp}@>NQ2c;Qhq$Gz3#aX0pSn%h@l$NQgV0m?Gt`R>3>X5E~D3d7&vipd~?MS-EA zInCIop2G^Q7bjJJtpV3a%3Z1R;-rcru*~)>s%s%ktgjXu^QDK8rmgi-bfU{t7C8X{ zM13(Tl~R`%tP=tcfAmCcZ+EIUF9l@&5gnTix7p@7yGF#IA|b05g!pG zA}JpwUl9*RmKD|6+@AIsHtXCvMYNc1Za>6M-uv2+4$DnPJH~Uw*CCbDT(DGjrv;ML zHP<(73#%<3iFvub#`H|IW>7|y-qUV7hr81qXfP0r>~Kk2GN-pgdGk4HOg~XmMf84- z=;OR4_0XS}*tYQ~QI(O`4zF~4egP5Lu}y~Yz+IcsXAmEk#d&N{%{jk4KGb?Z5qh-a zfC{L6`ta22&@O1i_h3hM75`PTcars%ny^}~)hQmZmSsBZ-kPIEIa%iY-)X%W0uWf~$7E38*&0fu`1KKf^43-!qidYs} z7zp#~dHU65BZ59SxVavJ;-OsN z|H$D_5x7vlj)$Rl$jA*L__R5zm83?xFd!K+s>v-?fqm{!=YY(w{_-wBT+gZ6mF7*6 zaFv4(Z5;VAIkm5O7nBR-ab-+fs4R1r#pLO|yDTh+o2Xe34Q2E}oQaSIJ1+i0%U}iS zY{7w~#K^qlBAL;6M6jubz@l!1XcgA%_Hv>W((M;*QM!w9XANY0zZeTu$B%@QcBO|p zRNU|wj0#m?tH`z@{@z{Q4z&qmKG*o~(*RivwF*PB0n$~IHx<$|({^U^dVJ1&1L*br zWxi?c9_DW>o!HZwma@B(C7yn{iZ$y z2)7}uE<_??Fk@7jrCj2MV>EjlUN}T5aiq>Ng4JRVKMs5VsP47wbQ@3P43k{Ee4<>-QV z2<%(lMZ~4+>#Qqxx0*~{YZx_FVRuo=Sy;otK>%_yRHw` z_ygYBNqzJ#K6%w!x%OP&<9or9+*=wpBVxHL6{#{moH@`glp~vMTmGDh3`m>}MI8sKI1H@-mri zn@Z8doS@WjqQ^HCXHNdeZN^mF)eYaY4ny;Fq!#C1?q~xUIjXE0V8pmQLA5&{9(@zuvp;^pK_`8;YjpTc$O86Zy8KwJ{r$3nWN@?g1 zk{-q-rv{CNslwp9?m7QpgNd7B`-Dkp6%hJ5QG<3BO)PDy(Z442qgE2NoPY4TT~Ulw zb!X^?%EvX^2rj8yn{mu;ZsaxmY+5B+r3u&7bSO5TWK*k>yUWKrA#RI(8O#EJcR^Jk zLz*o2>N$*+U&_FrZxq@WhZeyS?3|jeW_3nbG+0_P?TmhmQX|&csA^f0)j3RjHPHy}xDpb~nyK$=h_-)cqBWp!4*HmZc zKHJhiU3=o`I*9C2YAP=Bo3t_d07hH8>7sBqcqw^dIkJ`0KFE|){MZg2uPTg`R7%~H z4r=6#AL#oLdA#JzCPw0DyPC-tgKD(d->9DnIOZ!9doC|T%T&(PcN}8){=0=z@Lm(l zwoaw0@h;X*6k52HN}kN@O6<<{JQ9zfVNF;@C-2SihfB6jr--wXSm#ydM<>@M1C}NF zC%w4~tybP|Gd=t1)a7?8e3{O{d*;#lgToLS&6;>OpZ<(InwoHLzY&>7<~I&|DJ^&B(WNUzSUC z(A$(TmC?b|JQTo7Jpx>gXkvnxr{g;sJVA%1VQ!pGPoX!d9UoAEsMcw-Q93Olh5UPL zu01Jo`W;bb$ek8ivRGj;w2hy|TeJ;DsvOGr+qBrv@WP1$zD9gl+b%V4Si^}Xb5u1V>Tnv%jJMzLMv@q91>gotIt?l#NUJXNebr?DEj!6RTKZQd^8R{(eKfU9DGU4rvjSBCN8|P zkS3gOzA8BhRCghZbxjriVKV9rG}IYuecl8Z3a~3#?K>`dSUBP1Vd+Fsi0Aso?ay49 z@eTJE(gC0opZc=(E$=^T5^FIA+%3}1It)tuF=cx}*3!R`m9ROpY74YK)EOh+b~{$X zVU5*G4mOcL#BX{q`J3$T8+Lwb-IrfnxRjv1nh$2WY7NvUowu15 zIw|dY)OaayUcS!^2sZOHtT>h^FYF$W|AU?9vK%$(m*>;@X~fP;#V&A3l^H07h~|Q) zjcc_Ry=3)de|nZ?>ugJ*)@YsH+`e5T9w}loO!I`c{%|2&qq0op7yJmf8{~I+m(zjJ zN<-_XIc{~F;;yuz+V9jHpPZ`t5+W@i&RvYnrY)O4Hq~ARKG6F09{*<*co*PB-LEHkgR9 z2Jtu<2~@Y8U#<9+3>CpbdWCdDJ5K4!nK+8pxabDYjU4_!P-6*Fb8b^VzVJUlTO}h3 zI}#)uecL6H;CWLB7sWua_Y3t&%P*4Z%IR{{$DRT5+?<|_Qq%Wt^|=It2-ldAhx^^o z_JJV1nReG)zFW=u5`t+Xq-Ap=Z0{*m^0{>o?N&$c&G3M4Z`T+5M3%h-NNhE?{mayE zX31k>j)Kk?;&yqVCc)&u#P_6Z@|My;?NcOw@9HxuD>l90(RG)3mBU?Z9YaqjrmLs% zqEgw~(s(yG5~@Uj;4 zpuZd#s{9*P)tJ|e{m~gU0VxH*4&v}lYyN2~Z%v>e_KCH>W0KkR2$E5*Q75daZ^o7- zwUFyS9@_A-Dcx7rv0$`3-H|QWe4+(-pr;_y>7R~8nrv^Cnq6><7OM9;1Ki*08hq&l zgVjTA9}A2BZ&KyJwURRH1EuKnDkpL6w3kJ}R~wUz(&~%~)u=X}YBZ|qn^(@a*4Mc{ z5fv1c8V-z*T+Z|Kx!wt9CInpZoeI`FHjWtdHB`aQ*#b6IGJ5U zLK&4NdBgvV14-{Kc>R6azP&%8%zV-k3F{HxzbL8aW;*2{e_&j(F;)3d^^`2Z^XVe@ zqWQ+d=DefaV^^0y^?4SR1yAFn-PbCI*5Bf%+e>c^Rb3af4pWqr1&=-&fmK?&IwS3D zuAMGj-o;%YWeUo!>lgBUl_yatREf&5@0O!_Rol9b`gZuBvO3M;+rbc)>hecYv`4I1a2%b1Jeok zb@2GtV5yuZow*q~@Dfh6M$_m7jcfg_{+P}TzsS#80SEKtECvkP;E}jv@8=K6hJOSQ zQ}f&iTR9!&_31T9VF$1&!z(b9SEAPXlkTc+mNvbelX0#}2JN8kYa5i8^CQAyTt*E( zs>JHj_B2HWZ9I;3Vc!&WTb?;{ZW9p8L(bi*<4a0B8vTRM+#5F5f{#%6{GxdZ_zl#L z#@pRB(=|Y1G+#kom6WXxiq?CaGBYw;HQCbvo#;OEz!u-s4-3~PIT~apqukYwE2Zc{ z(K*J8{(y!1hb~eczaVi7idTFt#j<-t&1K1NSq|Yq2G0phT%Aa*F#~)f6rW<9r^23Y zY&xIHInRVNi1CwfvJ7?}^vM>^A0%V4!#^=8f7Mx`C=!!3B$cK*{w3cI^FgJ1GZq4I z1^wx=9%e?F@%^q_SED#Ml6%ZEu5q&U%K`t*QJ4v<7g$R2;ROV8E(BK8hGa^ zMsOp;ciq)@@}%_-27?}KOMHwnm?xLb18`{^9~QsMEaE%CS|ULdG%bF;5FSD zR(4OEEkgdwSFXc&))%}z_IT$ZiXecFN-w$0G*_g&6ICtC=a`!2UMMVmd*+@@c05)h ziOJL88Dr5}x$!%x324sAA?4^ifxc?U@gWj8-;MQwd--840vDejnlH6P%0;X}S~_1! zh!4C=x-;frT4U%n(nNLhor)AZ|A6Z?aM7>0Wu-d^FTu91QAx8>-tfa4bI*utk4A|x z<|Tg|LEh=yil%%TOmoBgP-Ql>k$$RPq!l}Do6JS%8`c7o#1Xb({Lpr9toF%?MJ%gR zQK{I0;B2CH*xI!1+8m%!jm2f{P(}7IS?VKGQXUTKV%_^}ui0Qnf6?o+>M{F;5hU~*bn!Ypg)^5PNzbOf&^@~G zrgNUO@1lADGJ9iT*L33PRG3^OLh`(A6#ql@cqP4amA<*jUV-yNK=`okjbjTvB8PU} zGpC`&Ni%n;{tk$|eCJzWV1c2kmxAq(pni3-_I zzhovRFxjnh2rRi+*CRK7U8$3DWq`ZgCW;%k;{mydxn!_-$`_SRwFai`$r?sT=eJ3#zJ6w)dI^3NB_C?3EuiTNQY{OrqIx!9$jrv6tRk z1PnfcUyYB5pJWV^j6F$OYWNmZ!5l9BH=h#0v_!`_na}gM075^5@s?j3&{0mg;$o+HUZA)G5vv#dc4oSdH(gBF5Lg)CK+_ALaJRgVMUBG7r4-7$(18Go08In zPN_J!fWumIdIvASYs4t4LB46Mg?AyX6Q`p#XODbw6d;67agFhO+9v5kFsBL}c-v?} z1Ii};MK`?3?kj(ZW;eZB(H{cs(MeRPCz~b5k|oTeAvK%HSH&yj>Es}8;`Zy==Q2rc zx#ihN(|p4LArs|(8luyu*c)vr24zK;@q`O<&iV3^L=A5{*O3qJAp=_nVT<2SU+uJEBlzb04`$ zOi6ahNa$)GOBGsk;%PlU`e~Qia(_b{(_8%9PFwZZ5%Syflq6zIV2BGAlwG0yqTNt9 zk*{`|a)TcC*beM6J+nh*&a^vs>Axr!Rh*S`M&Ax2YeTOg=tob8-c^LgzD#car?mdT z;a;g7#`Fql`SD1pNt$!$A-TKHl2&LFDMgWVXZ+O2zJK>-`bB$*Pfue7S#j{;m?sR% zf9t(YM|%y+6y!z)$ZhuI@@=TsVKy9U6jj6`D@X!Sl$F03cK*3Dq2u+vr@g?EIYwYq z*eO>nSLS5A_uSaE4P~H>*ChlyTCcqYe_I{gGB$r}@1PxbpUY%2I5cgmLGsL4c;U#F z-zVX8^h;ael<(8|&O8SjG@H|lE+Xb(d*;g?zjvI+oLZX16zU9 zh8;tIgu1Kq0cRo@!te-F~9kc+VK z&3>>T_)W}Lko(r(ixZfA7eMo1PiSEN@vl-Pj7s6+xR?K-cw$Bu5fHx4fbfoO=LbIf zy*chNBdT1NGcHy1H1G?W%Gf{nxHpuju{!f~=kF);vDnL>dHk>glg6scOGN*%NUPVO+FmV5CdBA5u z+p7r!e@!)V%2%maGJ}%_u{z-7gZF~=v zn-81@{-b0Q4HGSV0XIWAFn9=lO(BTr;{IbR*F}Gc>9h4z>4LzV5Agtw9mwrpZ}mnP z;p-WT$?WaWyMP~Jz;~;={NO)Lk^jq7|1z@(dmBNk2TOIvaOlx4M7}b)-m{qoBV(pt zI9I?-rfq9G_q*c*Ew?DElh>YGW3}D@B1-$;`HH`%4ZR4Wf%=hA)bc1|W|0mu^^S>?p zM%h&QXZe5qy!5ry^Z$Pz_b*V2;<;P*<@vPvQ$dg6LGS#pkB6oK;laK-FYhPW7yaTh{u+lZct(&=kKri zpxJCvJGu8We$%aUv!@`@H?tbEOWx!~=#=-iZeOBRxb1D)#X<;V%dpPc4fz=UKe7yu zrANC`tQ(X960=t}N|hQkGTtp)hb8gv$iUD#3!aue`aK6)ylbMx7%(IrIeog$1-)6L zO!_1QK9SWBM-w@?mw!TB4`XL@&`2StSyWEtOHKiH+ISrq;qRw8tuj>Czg%Z6VU;AX zhP%p<#gZa4@K+e;-d@0@Vvz;(s2nIbrkRbqKN_HVXFYALg&{1=x}1qly%J8fIfMvC zBHNixTMG7PEdV_$3ETq_3yN9s5nvY(R;hzYavma1920xc*8gMb8~-Yezpmq?$(lS9 zPOizeZQHhW!eraFZBBMgwp}ONPdA>={rmm{=f%0!#@cJIy{-}PNC4Qk#Mk7F{{P(n zV?sxO4&qbPYm(kpMZyCDZ8Fau(ayn&+UQI6w0Hv!-jkuagz{f+8-_i|Edwg9{?EgE+=OLE@|F_wpW9J`6IsfLkbcit)Zd&B`&k_P0*?M&YJA98MW%JD z2{DnO5}`}DACuAHb4+}=oT)!9>C$bDo+ir|fe>z=&J#pWqAdWsF%EZLmIN~^nPqhg zq*(8_q}{=dpG-Mv{Ud-Jr^0_ZSg_Z*l(z30jKjml4W_zgpxZZ(Ukoewh&>UfBK&ntTOH_jev-_aWj7N9*e!GiPI$w zeDa#eFXh0>bYXGMZ#jQyl3e}eZqJJy<{ebQqMo|+Zj@6r0{2ZpN^>kl5Zq!5@z_^Aah z2J=~_J76J}Zgd(xe$>E~6aJGGArpWOz|Q(fRJ3jF(H%Vxbo;dxT@lQ1Fg;*rCb=!1 ztk-f7|5xYTZk7XhG9O|0aA24;)%Fb%dun$OY19UXHh6Dt`1eKDhVV#vusGQx?e2&E z(y7bG(-GX*4@aO4E_zBg{#Y&>a=b}rrVboL`{VA;N0vvalFnE=&WQ6JgNTDeKO+{9 zF`>t8gS13f^Xt+~M$!@4-~(sw?(v>n6yUyeqmiyDW4=ID$XAj>>of#wv>e>V!_I{c z62rONY%uv$_OD-who`lk<|0)ZEA}{lKZZ+?5|?Z3`(;z5-p3bNh42_ou^HuYS7z#t z19nS`H1!(Qr52%Ih@)@LoDF@JJw!iz2OS3TDx#*J{62pF{thX1=~{W_lgInrB>nex z7aaGqx{T!VmP=oHTI!sZ`rC!sTO$acW3w4^4LCctC&H=P)yHXX3wM5|#w}I!3Uu7Yy+$V&=dcvsD;&^gn5XR4x_yq)H{j87O0&0t0 z(c(4p;@u&O^zbpsSC1#$ovcNEqdh~FA0;S=KY@au#Ftw3t$4n@y?i!0C$qkLm;2w4 zN#W=?1uAEs)pxg}ntacWeOo_DpSsy|v}P^%NY;VKU4GEd)}7urfxl0+wDK%b0F^I`zvtXD^m9EK^U+a%(|*sm zx>`fDP`6}<1e~`_ro>zSzU(tsiYUjPLzg|4iXJ0#;3^-`jJ96r)%+pL&8#?G9nJP~G`zi*%6<_L zQD)%|00Jq(FIqk(UN9Cgdu*q#bTgtGaX-SnIi3 zz=sDl19OE-g2=r|>tuh>o;LfIPe%XkYz?kw8p1G~k#?64NO-@c z^LZxvxV45Iu`T-b^;Pfc5)rCf!bzr#wxU##i8$Pkz)k6NPMw?EQ=N-70sIqPvg;?Bb4c6M|s^9=1*7L|~2s6+^Cx4@MTdyE*4B zy(89~qISh-H<4e!@qu-2cI3K?;aL$I9@jqu2SsJ5T?U4^FhTXd70g( zy!I>8WG27!G4xbU_rU$v*7HyWyJu37;=86YmCYzpb*sF|F~*lQ)m@F=>bGc51jgZO;j5fJ8qR#y_DG_=fCXmCLR__6ENCmpl zy$=aAKTnGu$W(kT-X+hux+YcQ%%*%Ta^btvxCbBW(VH${uE|?#c{3j;-O1?ycsYYkn&ws|N{&bsKq~0`y7Sq96xJ}hv@5MaZ z9-U5=mHZ34ImZWjN%0L-W-c@%Sko-k!GiCdah3bJMKy z#^VO}SO}89c1{aK!V$c*z`g{Jpp969iPSE zwczks1p>kJe+BkhET<6&-LiK#Rt?pt`HarkQApZHv#^lox;6IX5YzYGNrs*EIhh~t zCw!0So+VK2mViFDw~6@_U;VkTQm-Ksp)fE_K9&FjBhT%$o*31i?RUt7m3&Mh+WMdQ zlh>QC(|D4i}zpU1v$(kRe{)bGmM$?aE&uo*ugjJBpkYaw8Qfd*4v{qS}9vt}}FKv`(MMbe-CNvZt^)W@a z#3qfPvP;dcRPa;+2^4;Gpi72|D!*ta=<{e|J$5^i7jIb=K$(Z^ZJbF?%zZSVeC{Jh>jlG<&_}i)za|L!7=}$ z51}V(ErjC*E^s1sb_tkyzw)Z=%oZw6YCC}D&oy5AZFxe)Qa*jT-9jcXZ@r0RCLoX& zT~*4sLy8>%Tw3=|wR}5e823pK|NiMu^kF-gsM{SH5ODJ@Hsf`Y@#cE?Ou$)pbM~DG zE3nx;FnGA4n88gc@>~rGo!k6D6JhvNY@7qNiqRhE3z~SJCJ{A3iey_dN%0_ZQ zwy}aOCPJ(p=O4VUZrW-LRePZa&5X8(_b&oR1|hNmGlwOP~8uU`9M2`rhE#Q)d}Zsj9oV{acHxqNIt zFzv8z5L9!5LhV|lSHqY#+7@!ea8Wk!`+O@Qn$~zd^pr;9EuWR6MHqeaEte2)m~gt7 zDVBCs!ik%HqDb%7fYrl>w%R~ad30c1YU%*dbpT+JMyB}fi&f>GAX~Zjz zCttM+U13e?wWB){Me@iyO{E5XTqr%zYRL?LW5Du`PMzjmPUevUyuI579#zxGf1RA3 zu6ce6<3UjCj?RIQPb}WxGI(qC={kQ+VEm8w{PWRxDKY|oSt-2u+#4keZ?36@?-S_O z1%|MRA#2ScC^`Mm3j33Nk&J!7Unb+0J*}!#*nWIASW@WJ$&16e1!deu=KFAow6VxA zW~Y)8I4lLk8j7Y7U5e~cBe%@n#Hs!2-H8y+QgP$?aqr^6j2w)fHtyU5w45;WfZf(a zUMup6hCQVXq!%*PwPUnchtobTL(qjT-L`Ty;67uiP;e!lJzn}QD&mXr$y{St?Bvi8 z56|FXf;lyxntCQ8ndiG6Uy`v$7oeC)bsGwwzpOL zel)u}JEFj^(1tl#KDo2Y;qivWp~&ntG;ONaF&gL@0UOP7%u@uD(NVQswyr&s5}mMB zl3n5IBy{^blZbb1%85gxIh4__T);ILO6or79K@M3{;dc@W?;3Tuxnn`3q?#JeN2E1 zC0dy8k;cV*Gh_+xgxZPJ!Eo0cIA$R<0$*F&Da&%gb!G!W0L>hC_6)ocm<6QoMNIf@ z3lxm4I748(iZQqk1eGQ{LiB~!>%B_oz< z7GYFNVun4|IN3h_q7yB~+Ivlc{9L)VGK}v}ySL2N8sb!B|9tm$MNSp)l0!wYx1Gt< zM8d0m&d)Htld?ueL}sPiW>eBtBs2qCX0etCjj=7N@DO}ri*S-~_DP>VdtLU~ShiE# zRb_~!5&%!EITnFqBgZ&Zjwj?Jz1iiM%IcKqB&*QJ;-bmZqmC3yvm#u}d{>dbZmOPl zot3Ec7+ACzZ%eBhL909S-Mvg8@aXYJ1BW)zI~V;UG0gE~O_(FID#7&MXlpRw{cp56 zwXzm$g$A?Z*>1R+BI?E&{?HgqPfzUZj4@dqf|mceHq;T z{AbG}Mw^C6Px60HX<$DaB*{38Rozby(s@gARX%&AYC81pKk>FRlq9{_>@Q1nWp={i zcP`Pb_?eg?x4+sgOEVX{Z3(~8Grzvl9m5qALS9KBt!LN}07kA2@^R*ln z3@4`|fM{TU&A#!gQNy3!u*YO0@ z!tYa2XG2~Mwzw%!MX%N!@x-~iyD1wUk84&OKvnx1(cw5D_X<0TB<`eX6{&>zu=SM> z@*eL8^fh`MJb(17(@%s7Gqypb*xe-tPJNo!Sq%3U&kO^lts)s(Hyf#~2|K|A`vgcn zxucF6R|mc|zYesVe{Ak{>uZg6+VC*HI#zAb(&vBJUTjG`)%4UEzfT@%eH!;xpSf9U z-2~8`(BUyFyv-Gn%spZl!|;c>9piQKeGsg&Q598$-!xX9rdEJnO^|N8*QU<37pQmH ztl`E>=KGQUt!oU$w(B{pRxosx_(=9ttJwQ!=82ECtZ>=jE141eT0CVumKI=WG&BA6 zN2>=ozUA||pQR}YCCsgRA=3fJrIA%52>~b+hNabHw>pnHgd6Uok442C-f^nHKU_+2`BALjO zBir1x0D?mu^cvo~$u6@IT*&DUUJX?0up5(vv7ArG1NJoPAunPKbO5q@#h zbA*xHnS@kCK?1X<1hK! zVgiea(dHxgw@3rM{LD>p;gO^9z8)@s2(2#0kO@5>3feZC%u!AF#iH-CE94va@V{tg z`Xm;LGts>ZTFPGm08^qJF4`@iOn8L;YqR^g9a^ID`C_tT+o@`dpH&7@VCxxf``5oQ zoSqqP46TiKXQDX(KjbqX>}SoY4-OSXjmgs;)|YMw6cPd*MPf@NoVc#GY%as=0XZ{^=WX=x*^6GI1P3MY|i+sZkM^cWlO&bBM* z`WyR+yD^T(;+&yRzzuzt@Pc<`DQryE4Y&KN|uw;%Y0Yym|cG9iQ!AD$|9J$=^S%_`F`d3uji#LNvmA5`>z^ykZH>SsIb)>!3lBp+bi!UV3C)s}R zn44AWcAAl)O3%=ON1fqNy(M*24s-pCN&lQk8h10Zewwr$0#*{ayz806X%Q*UHjd*Y znV+yn)jyG6FcUDIiGmXGCw}rEWdtMAkYh?1Unh(4JUN#?k2z}+%bSbo^itwG2wU>r zk%P<-0+xaY(&BkW8Mwigl~Tc#E*-oZS#)PL<(@dm@P(DscTCCoB3W7=%T(dhnF8$* z7CW8(KcRcv|6rPh=}(!j={tQD#knl1t|g?-T53IiXLL4WE!DC>xurYXSeKrLpF(JT zb{lqGAH5kC`cTk4(3oK?ubAnIOUeP7me-AO{^RY96x37aw)H0mq23`|L{%J1RJg6~ zbva8)Q~vYUG(fbUcf$)?l^YmcKLy}M@Q0bj zYpLLP=&=1ksAxfbHt-$eV#(h?*tC9nk7E@mHg;NVb`uj80xeGBOT?C%Lr~s|L4tu1j^iuaGT7GO(@o_J00sRXth|X$U=8&j}*G(OMAtTXrH}ENv%5$pv2c zeYZMm?`@@HfYiT0&EoBBsjnx|C|Up}B(_&Na~Si)<)c5{DqKe1WV(3Wo1 zo(jBA`zA2ByiDnsxtE<{xlTQyD8PIDbFp0PwQCW_mr$JwHzQ;m(7{O+kuUa)(frb_ zxR7q#g~7Vv!A&-Pr@zn|9Ap`GV%n8oTfs3_UoVvgHEunCFW^ICeDM5W^~SuEZ43DP z$8ZNb54aJ(7E9%`#C!}VrpWM&Gx=kR)@ei&B^ka9-8E>IfP8fk`aWP*pcLYHd9z~H z+Y0NKI2A?2XZDbsdW1g6Snc?rPMi*Aqga_QN>I-6Il5Pv38f(4Eg+*)Pa)FmSAO?f8e!O8SS7X7_lnwaMXgX?p% zccsEXoNQdFK*?;CnL5Jh54iX4?Q&>Pg$;4cRwhcFlgn^dAexFdjYAmDA&)Xu@<6|o zf%tNQVnL{9W)4hv&;bh;(=!=|QITgY7Cg^XB`V^mgGAb+hDOjTqPDsR+tGytw5Vc& z`zwSQakSIN8&7`DKmwqBTceE1g|o3pf>tguz^}_!O!PfzB!Lxu`E%V+qQ3z0%Y~|y z+ohJ@zD72Jj-?6HlI2=^r8O&JW+g`VzAAzgYuMig?cMZG+MWW|@Un-s!>blMhW`TW zZ{yYSHkVyAk087S>}dxwmHf^KFbWDaE)=n93_4ro{|pf+{~aPY@Xt{2<2}Gcd~Saa z+hUL);#m-dNAQfJE(He$vL9Dd=W)9EpJ7qG{AUt1qWw{4MuFOOVdpAiJt$QqlG&%b z7er;H;YrqmV8Rl&^V-UgN9Qkz0(h@J-C_E&TEPHwnl%Kyrk@UyP$1W+byt%6u%v**^n@Htewur3Lx<2` zSYx~n$LNoBDh-(We0JtM7pcneex505CaL{wUCi5>=muHv zHr)>gq=NS6Je05A}1R>@gqKi_L-gvCT4V&r5dK^bTRRIR5 zjL{f_2kA2R`3iTn+)oj$q&s^e-AaVK=k&{uFA~Y1SMK}@E$OL?CoZPXi}7TOTrP$r z#q;j-nkOK$NrqG@dfIHZkc8BPYdN01RiE}z*CWDF5`3lv(g)^!Dh?;5;hX}Hd@la! zc7MJw&#igSrPUy@s;XQj4c)a=+}>AoOT|CHQcAl4^O);Wk*-X07M%r1YxQbr^L#>i z{rCupTCMgCyWKlRn}@VntDq-}Ms8xs}_^o$t(CqA$YgfEos+O|3r z=5S3TTCo&E|5g859J-yPd6=kXt}2eoOXyAr2gLP9v@DP0+5JZyfiq{Iy%#wEyZGUO z!19xQzEG-1a~X4coEiJk@wnJFh=N^#yb|flR!r@?xwO5hNdbw>~`ct|mUg@!0p9_eH|+n@;8tD+$WgCdW+6WyzJmTwBoX8}m5^*W$*w=}yhh?RSN{(&D+a#J^=>f1d6_%)TV` zYQIjt^xxeLp?D`~Key$3knNhM^(t(7i|7yo&coK8X@zt%`-08TDZ!Y-)>rSSb6Var zv!;``8!80;6u(drP|JfL^vf zvn7818}~?SB8oH^6iH(B7q-uR$pr@PuexeQ{Dc*MTw-e4nwGPp0gIhD;7EAh3+z6P z4yezQq0rlwY*Ll2t1~kNyz@TR3y%%JV^V0Lh+v|1Z2wMwIz0?Ir3!H~fs-%BKlW%e??6yc7pBs3o$ z=H0-J)&v!h|DC`rdN0UPW1XZoDi|GbD=K5N041}I3M9Nkz;;R-C_}w{^4v60B;&U) ze!%Ww^wM-Yg=xi+#R;1#D&4&IDYi`)&R~S+CqB4S1o*Z`v>s)P5NBhKo2rIHcMpH9 z5d`qQ`IC8>-3as~D?dICjOzN&&LGzKr3c=fPJgOa8w|GR1t!)t3baIax;Aa?Bz!Im zD8!Dix1?Sk>P|T|bd8{buFhD3(wH>&3g)MR8o!QT4pwYAlIoeH+x+7j_oPjDCQDxW zBcoscKM6lZR!Hx!l8>NUJJ8aiX(OJ4!{Oe=0Jx~Hn zV4!y`hg0YExAimC$Lb554aAM*x2ziaqV`0QYPOcuHl3WUZOEFVkfbj*on*PnrSxHtR2KXiZ zE%L}=c->OpU=k8&Q{z;a^Td`?^(VX2>fBX$&KSn3>j4N16hyM+|57I$-=iQgJUAWZ zjvPZ{ySb+oUZSSxcAeF{v!XeSUs~M7Z4SxL=K}`Mvf>hMgif%wZL%`_cOy(4$%Bq^ zeqQJ1{C9H(T&yKNv0O9c@ICnEpjyTIUX{#dNkBqozP)Th+d!DH=0D~z(hU8r#%hf0 zpI@trg%W-Njy!q=b*ONz(|PS9{#POGgDK*phrggq@qS=lSukcW>hqT2f4_MY(iy(UIcx`m=MDCD^Y*G;* zaP!&hcGOW1+Xdl=)y|1ETk2I7EyXJSYOC%G!lgUU*}DG`e%YY~b!SZ0FPHt=*g~9D z-4P<*n-+j2H{bP&Di zjw8!Z;o*FgcTe|}q++qwMJ1`TF60A*j!@d9U3<&QQ$5(HRL_|$;dl?(@ z>1kYNT`CSGQFs2;9_?0Ew05r-oOOeDV9JY2z15i_4?H?Id}nud^gHlcE85uWR4_L% zrlZ#)!4Tcv%LsR_Bv?UnDP2afBmpCXKV5RHvXpXp2c0$4ZqZlNiAZV+5n<#^K8dY+ zNp&qrxka7jYr$CSl@dZA%&rA)B6Ti|(KT4qwQ#GWZ}Lr*w?X6+L@B+Wf)PhpNrYad zHL+h3j=46|9M?tnEHPO%*i(0AfMg^PezZMtkSW*lOcQbxH*D97g5e%`XE3|C(y7_c zjDS_AjU60aSt8XPE6&vxC}L$KJxBZZ`^_O=1)Sb8N2H+u>&tctv~N9@ZfC7EfZ~&Wbvu7963CKTSplrUMoN=bUtaDkQ|@G z*LXC>1B`X4ZOu?hHShC6m*bY>gQlj|;HqESieq!J(%*3niLgSR3kKa^^$BjPA@nuT+xbI_=niv@~6XpA~#*s5Ayf~aAUBwkSnH*j>$ z>c9Xyi~8H<)o&IWzmMcK2g9t0HAv+Sf?>?jCDr6^{FrDo}gjx8Cg4n#O{m z2zr!C+L(cl1u=@#>ix1;wuZ-UI&N#Vs6`#B1NT2c@wDc1t14Cz2!xO z2BEBkXx3t1RPLPkC#w*5JdV|fO~+Yfi;mr}CFS{;DK584+DHAX*qm1W$~%4Ty4-mw z1#}^A>JGXojQ~Iz3M56bnN4CH$AJwELBoOdxU`%vIGXrkm%M)uN)AJU$va3|i(fOg zS5uR#A|*bWZKRoiv_Zn4;1!zngmkjpio?yzt$I))i*dVU+%EF_T!VohKwR9|5qTEaS%H^RAU;@iJ?wd=(ve&$(Rl_0J~!`A z66!VVGy>-||8}2RMZP~kbJMcQPtev7B*Da#=+%-ySmr0o81;6OO1;(g#wQ-{oQn*6LV6s_Z>73MVpS- z0t}?S+9a)CM7FLUos4G8lP;{Hl5KzMW3U~_Vib2o0CoZlWqR>plP7tl6x-|#1f*zg zaz}eDDUQ6N@5;m>KRI_*3Q%ockPV;Sa-PoLbQdquCjWgzUAr)Mky@lw+l$#rVe%&4 z&~`gS<>&Ih(9&I(222?-t?11WZ*L#R;;6;WN-QNkW3I0;de7)ILI*T%=*(~BaQ#R7C?hBm+cbR8U*vq@m$*8 zH`S!3K%&3YN?ySCy;lKj@{bN^S#SQUG#zv3V>_mnp(kj{ z!(=0BhSZ+;ZQf~NikS_`em~f@FwBU7O44Uf3nAPhxMEpf;0HoPFywZ1K(RZL0? z+*w_?CyG!^aI=2pF_?VD3KiA3JcRDh4*@7smqRF+^2Mb!d(6|wN=$Z43ix<@Q^Eh> zi2tX-qlHS8JWy&DDxx}#vx9)w|8Mr^<3+Mlt8+3@hx+NJ0Hw34sW0L`YQ)I2Q8j4} z$~U{_b!AA1hhCmz#Tu}RvO*GJkkkwvKD3u*AoQ&@0k6m;fca`Mthjy5|6Ej6VH{F| zU^)qN?z!*`_VC>$tFJO} zx}tq52_dNTsib@Afp0pWFOs@_p?Im@_xbS-t=RjOx2AJ8upD!}GIzCtKg_Ya_1G+r zDxRw)5>~M6(%~q148 z@G02zA9QpXy+;JP`4M`Mnw~8AO6N;B&pl-LT^Z4&%R4JqDx!9}6JqD}66r9^)3%DG zBpO_f0GI~qaU^kvVU4Fj%Rh2IDX3>I7BsQE9@1oVU->R~^kejH=G?oIP9 zY*6m5Lg8{9^BFxuW;91U-&wt0mkLh&SMG7Tx_glc%evUKw9P(pbjBhQkv!V&lFM|wbNrj9wFNU-LHMKs;S-I94lX@EJwhsKvgxCiscuu0+Z2VcA7zB zqobBmsvCh-BBx(a+#TE_w3(6^_kWMlcE83?aMfaw zw!73Oqx7z1j!Wi7Pe3mG=r}o{@Jy4QlMOL%NC|-v8c2($po-ljzqLHwJ1&|nm=q7| zTf$)@$u`?1c&vUe^6f5t?DUV^zfdw*5KnL$%;BwSt(AX3aJtmmvIFb5q}qFISB0ED z?PMxbw6(z_K~^`Y9Oc3B;K;UqdnKN=O4~UJkigMw$;Pmfyly|(&(&}qalDX~7BkI* zc|3K}^f_@nF-biGN1o!1fvU>40sat-QUv<;Qx7Y;v~?{RU(W zIA2=4-rKFa15%>25L1Sd?kd&`C;yZr4!a&A1r;V>q^zK6+q(_3&_B=3df3{X^h7tU zHROxh_iXV>17u+=Xj2U<0G!DxWM^0*ZO}sk>B4AbqPkxnnn#*alf^_?j!yU4N zD8JBdb=!n4s)-PsJz9FmYubVl4PJH0t?@atX^n@m`S|hrzR(a-S-Klq>*R}>#_u3K zMPbicGWZr-hJ+ld*6l+&-0F%jxvy1cK_NwN!)x>2GBLs)Xu!cg!DMThiFo;qNC3<2 znH&Yhf(!Rt58|K{6e&v#o3!ctG0#d(M1=NcpFUC#=_f`PVK~Hi6qA*(q{qmh47CQc z>35=_<`Sc}=DYI;N}SjwTdYzQMq3{wYu)sIF`$WKuAq!^JKd7}u* zm*C3YRhbyRP51$d;qfyCoF-QJu;b0LQYGqlr0;e5Nyz3o{KY4cZjA|J0vcf_9*lnc*H#nGS0&KwE<2d6lXVDI2> z^Xs}F*u(K;`~w|pXN@0JrbmQy$K{@(n^CaUNhS$GSGmB7@^&b47UY)89ZO!U z9r?VM2K+3U!6U7PJNP?`fWAcqtz_f3MW`)=_3Rn z_Czxd;_*0O`c+fA2&W{3uC3T#rQs~-buQ=Jyo!ewsW6!3ZVtffq){)S7mQM%FvfGm zSZtiAr@<}wT=kV(jc#OSV;HrZ^7JRKz(Ag|$!XVYh=FgV@pEt<^H>FUIgb3QNgKz1 zRlY+ae6vbSa~M!uT#3ziZB&AlTiKYD`CMc?OhH+W{c#&ZCPoI!jFiy&%wF*H7iDWgmp!d$!0?xd4?zaWX@olQ zqA$egbdS_8(K}qk=mRh-I)x0D>oG9YOY(fwS?!Z`mr7vNW0KNM)V3`>$)Gn|VoPYk zVZFk~L?%)Mqm>_fhSl&?7Jz25$X8ermwjYHHaj;Pp;2GK4Md2<=+B!hVL~s#I4ITL z;2^?Ap;+-GojYM}ry=G_MDI#I zX#$B@{d5t(U3&>v_V!?PY26O3snjFNQ&G9>jaTvgtH~nrY@jSd3*&T({PSsRIDXS) zt(|;uKP^FMa;;Wr6=du)@-i9KH)&K+*CYBX4kaPB?GRJj!G&plhn#hnZK%a3?&98) z(REBWUpej4$zePm9}7<|f5vR@L&~+D;?64TzmiOW`K}N2XLezToH&bjn)s+9A**Ex zf)P=88;=st;qiVqfPv{53DNw*&S3Iohxy_pOg$f3 zNAJ<-Y`yeNhs2Q&)ofuNK{cSB8&NY+h}%`FgWsu4&Ynak2~UPOq$Qh0W-b3ttCEN@ z?d9I=Us)wYowix9-15LoZ->N)+JL}hR#5^DkqQnusd7t<<7&90j=8Xh zCoFM)eYVWEnL15HvB41emtNNX>u&6zrOXbsRQdcqTgU{)-jX%!1#`ylU?dZlCXYF~ z2Jd;aYO-Q<7F$~Sw|lDYFJXFg|5NR_Eub>0IhqtO6S6o*K37%Gc#HGP^Plk;h9?2x z-+CtxzMtBCt8)WFY;J|5NbhYV(^N<&jO#AiFG# zxnvZ{UGR>1O^A;UaLWn^T7?L!A+MkReLFawB&m__Ly!|Y&qB>SW7;2~4E{kKFXS|& z%QvpR!8f-Qa~QOV_q$oQM&chS`YXkASzUEoPnbrITkgm7`4x(QeYHpr(Kq|rJ#D&u z(#)z?v*P@bJAEk(+YLi3K9w>4B$keaP4HK4H+^Uj0m#O=fE8x{S__sG&2ICTECdla zN+aGgF`Vd=5d>Y-x9j_OUMNnhVW*qxph%xvHo8My<;u`~TJFPZDR{w|)82ZFLG}h( z8Wf;t;G5F!2_w6HXo9(UG7Mtl)sU^R%qP4V zDE&L+J1MKF*%LJQ-^JM<;)q8~jQgs!btM57E-+LgUp^3r(MqdW9%ZG^6Y?_C2}8%B z&GyH*&$W`@wsK4@w&eBnG;(;J9_@cna0xumd@$5x_WC>DE_zw9`Y(mg_)~qx+03 zwARem_}WcuN5*=TVX4C>;^u5A3vA6-d;f~KYro7bR*kByb;gsbteQSmlXC@co1Jl& zW7Im7dsQ&Nz>i|xJUVXqBK6Si@{ExX!|LEPR!>{EH3j`fI(f}vq{1eeOBA&;gJd7D zg{+se)9^$VPL*9JxSz8fchKBulDMhd|Ft^tki_fNiz!ehVhLV1WDhD z771qQ^&5p>TVX+I_Xpq1;F&Zoq>*B}0kg*9jaIL7T73w6tZk-Cver>Wh*o6?`-ZX- z*o~FFtex_6L0afI_vQ4f(MEFa_FF--2l%M#xWDZ`LNZT(M>Mp3tH4aGK@T-JEwyzz zuSdPr^~pF(nTtI&Pp}^JW+wdB&%uTd4;vku1-x5j6zW0I0z$*7Mw7S6} zz$JWVN#fYRy;_OL4uxf(_e!ZNc(~b2DUq1GAQ>Fwqo%1RjLhFY`;u&wDWY9FSbq14 zA=7ri`t?YlkE-UT)4m^pYP^G zdih`tPp(t9<=$Ws-aeB-=N;eaYlK^7uHer8v9Vs|O-`<9vl!M$8cS5`+n^bh3iqs; zEeM+Wkdnwu|Lup`P;=<%r)3CX?PI`zx^|wmlTy(%Nlx-Ja<#fjP5`Dbbs%W8SUsZ;LQ6^sIYr^Gu}HD?czm`^?H{$T$-$smApogv>kC-L?Y zqzlWUy3lR2TJ})8Xe&y@=4fM@`r!ruFk@plTa zMMMYGxOw^_8e?h`m7Jw*ZhXTJ7mOZ89S;+kp@|llf*d|q>XJvhn+YrAI%(j&U2wGN zXP18h0VG+Gjh(tw=Vd-%CSa4CFW4Y(HL$_~;rNCLfglD2Hv!?eqNs+%RB=b}-P|Ce z9@D>Msct1b!8sR?78jKUi%Vim)Dq_T%ZxzqBc;YNS;IlX?|3TM!};-QAv#gr-V7{> z@x<*=8w{OmdtR~t^8@G7cceW4TZwQrJeI>PO`HdEq4TU_R9hV`<1AyraX=rm+EPi6 zxX%H63nCr{L^a_MDKLz^jnfQ}RhA%8CqLCyHL6u;PFU+PFO9@KgM-%VPt^~YiL&OhI^2Fo+lMOyh?H= z!w{E6x+l2NOP$s398>&G5w}u6cNkko?_dr~z6!>n1W%7fVTHl{BBvK-pXiHpS8wGs z-lO~cWW}4u=>Q$wB{8fK8G)jd1pTr7JO)nv6p}X+yhk_UY~fRIxj9cMrV@5=5(H6o z|JvWAEhbiGGoE#e)PC|7Sc3N4Z*$?1r0%ML@n3o=$eEGQf5f z;zd1I(m~|?Q%=AQg@$hty}x$WQ5z27Thid+c(^o|8yndkRluvKJR^X@ff#9$b8)@j zP-PYtkg{N0aO0N#KG-J4 z=3#BLn-~k#=N7mrHK%)KF3d4o*q6cu5TC7wb!>(tlDA2UO(!l2^5WJF!0#;7^QqWgTf$&^)B@Hd zm@i9q=!3#S_v&~rMrT2p%SX6D39`Y1(`}7=v7ZIQfy9p!q@wz~_MHC7!9@8s`O41!NJm8g(Qz=i8(myjA5gxf29qzH;9A8im<`FBDvgG#=6B(57 z4SxFvxS^^X_5m&(2bcTq9P&pLM`(<^YX7S0`mS6?mDkkZ=KkRBK)x-xF}DP+0rhpG z^bWDiX0N!VF6?~poD%fo{Ihfwzu0eZ##&ocTYv>q%DfmN!QtbP)mwK^EP_?Mv2NOs zlDacwO#Rv?E=Wwhbm5O!V&n(-IUuje7DIeO8h;%^6ISz{w>5%!@Y@{O4;(qYu}EGd zctq52F+m(~RzCL*%ZM zl#hBiKArN_Nx;M0BF-UgZ-zr9Ay#o%-9pp};_(XGRS@L5FzDOuK)xhLLMr*g%h@TSj@nT zM#MJ9Y2WB98T)nAeH!bLoBCgAeTmQKjL0geNDIV0Mm$Mh>H{idb8Pu+yOVCWx{M~o zyIhiZsMO^OXtY&Bq{24DBVP zpChL0$kA+Vo8A{*bBdotviHGKXh@7H)sSEAEev$ehXSqG>jO+$Wu4W@*=62>5=_?qV=q8#Ze`cco2;r0aBa^_id?o=r zr=zVhr#VnRS(Vfb<)wue6sRxGNfgFpi&ynw{nS~gLXNpgI1O9|L(c)vwoHD2lv*wG z@m}N6+cBcW9T6w0s)b2S^O=uN=Pr#BIyM%3%3Z41dTvdB$qIvhy_1|xQrF*}RP+Ae;2|*rf4@f{jqAtwb)KH{-tTp}V->&$hMvNdJ57J5LBPwM zF`0X-7*0*5CA){dE03v2sOpSmoxbMjUrwbmxzrGvyR;_{sl69OuDU zr4Fg^LmoTJ((j7JCre2aZK2;0{y);w3V*B7*S+;-AJFr+xeDyln-<~4@#pgTrBnGm z&!mt|XP#>xjy(Q;6O%ry;PTDew=NG^c|;0;-R$m6p-@tcD?t`A2Xa@hhM=uZ8CWA9 ze-3bXs8$g-rKUnra2N4gtc>4&W8QA4$5lFXr)0TT!g5>l+OS<;S90-3JVr}~^X5sV<(?wFu*DV?u zQcp`Be3ts!BZ@kwklwLJzZB8q9+;1>mG#qV#}DBPh4-4Q`=PEavMd9JXD(0Y{91DV z?(MI%#X3FsqSYw-_gsQ3T&o?vD?vN*|CSEPq62!Ta;eC*zUAaQSis#>KckT;{%&}_ zSbA?aMT|=3&4-b2p588PY@0(_zuyxldly0wDoq(@pL^R&ZhHxSalI9c0!=4ap=f)Q z8#MRI&H~+8#@~yD?_vy(r2W>0O~(85L(bXrioLJMHJDrF!_I`&iLUvd#iFg*3;Ig9 zUS1wNyXc*+`d8NeT(p^@9dWyk$RD-&57o}`s;43w`r#z|cXdg!uKN$rTKe%X?8R&` zqn>JT$9Z6OGEQM7>_={DBu~P1c-?f2X|^F!M|Ebm#xK?ianj2w9icwQrYDMWBr0IX zT|Uhc_v_ji@Nxf?SV?FNPZS@VLXI6Gfb0?IExJqV24cg2xl?(54mf-Y(pLnExu8kv zcDjqh$Ir~|K*BHRYNt;e+}EM#-s1;K+-uFL+Zc65CtbU< z9dh1SAgJ}D7v<(E^j+8ugZ-8Q9b8qa$leUx%QCz^eY~onA7~p}8>^ffV^PqAV}-K8 z*mSClS}fp9YR_&B5ltCW>N}U3Xnp>CcM}X_bQ()Xw@}PlA$J(i5_azw-U8K&7R;^- zu-~sdTNeHP^%1=)QI3%ZsT9LE5%Ez0mp*=n5F;@uimGT}ez)Q6tcUhp*E#>^)E0Ji z<4yQ=44{v{;!0kC1v*nH68D0VUhj{5?G9pl8ysT^8#yBy5p^JM)+y2q>V(2;rz7@J zVxKyKM~44uCYcZA`E=oEdgeytNJwXrDn}V?-;T!trk!TDmr>9q*@Wqv69#ydp2gbh zNR<%9r|8rXf)N(XI%8%ele#+`gmVj2-3Mm%6#&onz9vt1L{UlE}Z-Pv}+! zMt%Li+aLC{r)4pNnqiZ5o$H8YUcQ@F7TnHnxCGn%Hhwgd zBZf?}Eqqu6&E9){Vg-BBF?(>uox2!zrG3&bS>r%1^Be($un9sn*KN-;n*=M-tXFzg zs&l5rb9bOISt^E1Wrnsx*O0}mV^QF*{r!z&zRbM3aT9uh*fW^{J75KZtUF2n@eK#~ z;yKxtFGVbEDZzc(I~Q+%#<5Y8uAXF!x+-SieI(zy=Es6&eWsU}`d_VO2Q#s8`?viY z5f!XcP2|}E54v0|mLyL`a<}q7%{o2?`NrG{5CyGo29Pey<9vRjw!r_1#Cpn^=HCks zO4Zpu-;z%?v<1tOBw}sG-A*8|Tt1{C}a+PA0`J-lsmCCpf7<<9J zlk4>g#b8R)cUo|}yJ#lGmKpwA#xV*WCT9MI+9j3Upa-f|-BjYhe`5i*9#(fh>Am%O zD0Vzh*VlAbl?5MFMe%=hR1S=y*D#yNTD`o46wazvsU$gpI4IzKR%Jz=NDO`3v#)rL zg1${J5zyjUX2~V>mx)pb^$JX$CVn3isEsEruK!if@=IJr#BAcvgtiwq-j)JWm45y_ z*X7i!`grn1upm2!PO7&NDQyC(Uwp}22|qQnu`k!%VvBrk{dAO0159upE0NR zM9v&8$Goed8Xt`Nsa{ElR3uS;{~>qG7t=8Gjg|Bwe^k}Jf!7&7nS{OR;KiJHDb-)y zg@;CiaTVdJJPT^MI^)ZfC1ExJCWQ+GcRm3}-j;VlbiO-sAHGe-93-wSEFihpuC%@i9$)gwC%TOl&M^EOC=I(o}K(Y`Nr} zp(UnK&ONu$4Ais;$Wmw^T!BoC1ykRUVK(_O4qX@LB?VsRh{P&3tVVXKBrUNe=9YSO zr;zeoN#8|$^fNTth5D*BD%frfenfAbN7C&!7mMukniEawYRY1mZ zMmZZz6>%@F|Ls*;n|+yPBKQY_53koNdG*vX=L>N&T;^g3R&D&CwyD!WeE!msH0;Ol zxs*lb{g4*)^vn)}@xxh9``w+qL0SHdLc#2DIpHPFO5SP?#J<)q`KITzWef!}Y|ApX zle(Bgf*PTi#FQBmaJ==yoaH^AXR`N;P~lpR91qt7U)>jYSjm6?+$wat?WT=zh{~u7 zANW&*Gm}E!7H1mZ9KqN1Jbt>WcI(b`;(qfp$tnMz7*zdL&@Xg9i`zmS_vgyw^;yzq zp&6<9ZnzR7ccEvdA&g>ttc;NkK*}8;r6w|R#iT^Cz2nL-<%K^By+@eO%3jB3cq&h<`|HzP3j6#gUw{Sih zqMdy?J+T!c24z2oKU~tAu{bUGo{MKh@|=MUTkWYj2T{b`;lOkA20SK&N+q~?V_lwH z&;u8I%kJal6}(*Wgb%34%!!!gzT=e$4X_v z+TgYc#wv=NpDUJ>Ho*44#SaELM6|+hhGavu+bP4@YQF}B^+`>ac=M8!2lvuY{mhhOfvUGiGaDDa{y(k#*8YcWy$2&2_D0&I5#&S7tmG&Ay za=`L}_uuj;I(SG5G)Y{f>6?&AxufYv&-NMGtoPKhW=jfndMx&4XEl4YG%SRqi( zI-oS1=a;*LX7(B!{WGrZAj_@^xd-zVz+XWK)Gu3gc}a`rItQiV06{^~q*a0ot&jZE zpgP}G8-a==8SPN_9it1dEk6fEFZBJ#+vLNC8LN!Lgfi|pcXpqH+|^+XtMAPfGwA{% zY3ZbDy^2l+Bh^*-wN1P;eLFepR>~VDbH8(~>a6)1*5F}y)h6PecGW7UlMVeWK0+D0 z(zgp7FSAxFDf?)$<{G9yrdu9HKlzI!^kM5@4?imHPdPl%eQu6I9+?jDZ_C1QJTn3B ztFC&dMP_Rv1T8nvauq}yQ|gU0kA^8juGHTTQY~h*YXoKhP-q~w3pl8bhmqgD^>!iU z)1Gf&I$dLKq-7NLQ@q}Z&rS)Wsl+_Ghg4u=e zfOYsMyy-(q$C>R*{ zoHp+R4S9cW_Et1I`zpgZ{|tI(j)5TYqvLM<)5t;2@bmMO>lXkPdKT)$s8CSny@ZGy0P*+nlbUOw&5r`&((FHH@;~E~Xd^hT^|kxA8Fq z;yAV69Xsz`DC_b2QW0Fz@>P5BO2y_cj&IWrviWJQ z(`Im&VrVw}Y5D2sxnSI#+QrsArw?@hYE!l>gzTTIC0gQ1)?#guOdAz3w)S7VrrnX5_6sH$t@p zQJH$Hlhr%C%M;)(mi)mDer2-w&?e%Y?#QFpjO=0e8t({3-D-KksWBq|{j}rTnumf? z#A4Fdh+VMA@KL9V{uJrD(Ej-iQ$fw^lNS~0iIkeMYUw_~3)B~#_HkXh83(nC4pAwy zFW~mvXyHtis%S4-`zdF`^L*(ci>?RDCTt)sV_|hDN^?W(D*m+m)sj{!Zs2383HeVQ zp^RHArRNcTazRdO%8dXur?0i*agGH0jgZCcS5{X%&YV5gPEfnUxs$pAW7wA5?z^4z zWj}fp%L=elIc;)=TY=@PMTmv^nn!=lexc^WjnO)-xK}o9)YJ1@r_oC{Mvt2&JwL^- zkGXgHNwdVheqekGiw=;wn;<(XBMxArZ}fF_tY(f;yUl@={gTGu3wg*w+;*h<4e~7y zTQ#s8wi#vq;aAo;EXin|vEeN;?%0tVe|RbGxG@al;{I#}s_|{HuQ$jPy0f%nA`72C zU_bq8X@B08S1c_ri};)LELiAIB6sjnXqb%`ll@AZ*bN`JOto5!7zx&NO*qT7>ORN~^xz ziZqR;>M4j8setWqbm-voR;aPo``;V!Wyc=7N4>3EbOW^*VR+=q#D~Ol@#_z3gRURD zEU@y!{c;0t6#Bw-Iv#-v3!;rl?Cm9UWg-n%_lY}S-`3@Jv7LNykM-q_RNf-J7(C#C z*F{6sm7sV+Bd;~gA*S&L-0=T@Yg!LykUR-u3<|C?Wze-@gn66@aP3S~$rMVl*r!8054 zoNwm~E74Tky)81pZ`47(w?egTjoH520k5;#_u)mi&t)#qVB~xK#(DQnE_At`KqZss zq2XNGXkpiqfZLn@5&<>h`qYs9@%HPWLQ53Q7hE>Gkotr180IVgR8G~n>iglXcA;X| z(t?zxmz4JePxqHoaR-f_RGRme2KZGRyV@@RA8ghMfxG6Ul^bn##+IEQy^XviTYEOS zY~Iq@W5n2~gfGcuTe-&?x4YfI`Srcb%=wa7Y^KNa{MqtH_B&VP{-8k3n-slgtx<>p zC(?Vy*CE>tp8fH=Uj5(Imf#CK$BVsgnbidvAi%y4kX00X_Ve)P>YDt{#VpusZ*;#s zARi^1Txo!l#FYeWv5EOnaSeTImU&Yn_C9;ecUO7+<<};wd5hu0K^0(uI(=lw`6KU$ zG_6n{m1QmG1#+okqVSxxO81)llXj!;HP=LYbBf>w2N7-h>j8e=QRg_Cf-gtcq*fjC zvt+|wP!kzv@IiXj#aPtE#f?|z7aRuXt(Eo*4Pt|UT7*E2cc_FoHd0fiw_ZvfYcw=T zi%;#4MGFaCR7T;s%*m}mXk2-^fAm0{NT0{b2mTq^`+I}fRgqU*5lFx2dg~2yP?3nt z6N3h7Ut~aEC92gTcUEw<{0dwYd$K8~!iW0qY*|MB(7e5pJ0* z=r&H6D{7C;H{+XPn>G?PdrjGF2IIX`-%*prx4&2GagsA}T^W(SXZr8PtQea2kXMvT zh{I_23iv%9KXH+NM(g0}Wfcr`^6fD^8Ron5u*7>Ep2xbMFY_C9_18+;giiK!cY)Vd zr;C18*`@NL$G-B!jhD2DXw(2tn!M~7T%6w|hBO#L#^Ik zp4Y(bZhe1|cm-6ACKE&j?<~Hh5xk`D0FcXfyJ&AoSL^m!8-xa@y_yv?u-06Zu<-7 zb%*`YXzhXiW5?Z|S4HE8vk+gfH0RBymEt&X38?8}q&|2+E&D-b=zO5|>tRvxeV6bT z5JG77j$s!FZ#42pzeko?4>|e<`E^J9v{UebB$^9d>q~7cM%t)0pfESA4`t=*q*T&pk6vUwK zl(DjPge=7M@4aE;8ihzRFRxH=JEvc{kJMVi~@cjb@UZY0(LC{@qZ zUHwfE=78sMmDS^fD{rEHeIlf;{rLSR4sTd^;N?!(1fqRz&OW19X9@bW-tPSdBlaaR z5n*g#|2uX23>e+u)r6T=<;rG(iqe7+0=u}+c^WLyt1X`4TpqB;f_Uc}IYILR8nl<< z&rA!;-~4iBAcNvr@<5hD06~+}ASP$0eC_;u;$9x!o$GN%up5#((U+^{4c*o5f?n}j zm97S>@R#iIN&UK#>khrH7hQkUTfSEfTOYY#ALW0lGC;=bx^nkjo7uR6N#LRXy2s^- z)XR3FZPm^?lCb6F;LPsL6`aws%wk*Q5^IdvY$lhz)2#o= z?I;ZMz_L&NJNUZC??kI6R$nS{RPRky`%Tj-J?k;2$BeTRjUO`9blgD~q-Mq?V(ylcnu9yiMDBsY&1+G6&O3v_i>zZ}Hix zFNHte`1%ReSQI2UJyK2ic+{DmvgBb$zPys_4j_N_Xb{C}M`2a)r-ZNoZD1sw^Md`aY&bq$V$?e9YF zKl}(At4;6HRWQJhH)8grK*KC5^te?m$m#GjOe5$uVD%&%cQSWAE=`K z#V0UW_Z;@W4wlZDPIZ5Fciy@+#WD{`&u1;E%bYvF6|39{mnZ7HmAy z_x8l|=p(T6b9mFYz80R}oQVrN`X3ZOn$cTH0i)E!Yc%0XF|qAvcFIG-(;RpSP%IBJd!~A|2MHJi|YTL%zuwYnEvmd5SO>QxQ(Z{ zHm^{Er=9$RjFidxeSfy#NM{4#QYYk7$^L+QU!>B_BbT*yZU(p5`-_@{_)khFHNht< z)uGuwpag@W<>aeBD#z%;27t)@oNtsFq)#{ItGS=yC{$jogc+sm;M5#ue>^HsAfr@# z>uL(U4&lsNZ3-tYYjoOF)3Lr|!+WRuRo#ZWRBT*e4jBcdf5;*s$1^4E`DwAi0^299 z*lB81h_^O$w$QrRVzz`hvTOW{%NYBuqgLM_+7uN)w#_57ZniV|E;Y8u4dKt?6~UdG zU#@)#-qO5zn?ch@{~;;gXJb4l8URV8CA>|tg5ogmk?Nn1%{&IhSQ7mJt-w5a5A z=vw0>N$uSQ4X7_S`nE%bV)&47Dk0pgQxzP1a6ZU++JlZ7Q>(m_qm26E&X(&)Cl~1Q zPo8hllJt2Ud@ww0R4drSu7W^+$A;{AAw2rh9ER^hmerNmz(zgh^-Rgj9M^RV1k02} znT!gi;6*7hg6mlw4%uJ&^sj2~mBn9}xpHyvQ05T3`sMO{u(T=P#6joLB;L1qdu&dr z@3heO0w%uM=8Uf>)opbsM-#L~t5O?;&ZY{hre19awCi<6rT@vHXJtF8St6nzAo}Go zThT7~OfR4Tp}W@tY%g7{_X%G1egLKP)Yc2@4_(Yf>y}t>%NRbhG~GQ0y_1j;w(ZD? z`Z9^OlO}M<5X40D9>owgX+_TML~3_9|G4M)pgqH-IV{oQCfds@A-7=`baMe|f%@+a za!oFqRz&EzWTG~+GlSn^tXz~D*?Nx>mr zN6Xh=iCGG7a;r0D9cLmKa5xw{KhH<%T`q>V+xmTb^>M(d#9ehonu(X}jSnVT^h--D z-j-Hu*d*2ojbP09h`Ap#@CH@nX#Mi@IiUXNj*Ga=Tz#VW?F%pIybdZpP)NPv+YV}!$bEdU z#D`X!4u6)!I!;>1O6B_MdUs^N-BWY^G1EGfvIN=we>Cxb4WzY`loscLBaj9!d)Kt_ zKng_c$M-M8HS$nvy!T*Dsagc=rE{J$Vjv*f7Vj#eQDBwp5Dn2xf0O4lOPeRuISv{( zu(HG7Wo_$3m3`hB1(4Dc`)Xa!`1(2a;Gtc_3GW5BTb|lG-{v7w`k~-qj98vXrkJnV z|6Qi0gHfuXBMe;iX=R{sUwZpsnjgw@NQ*nR(m!<%#EuR#R1gq(d7DpSFBcIG7sh)d z+1}j&blmLdQe}BQ{1hm3OJ>`p+q=Car`_R72@@%oFaC3%rFUaqhyFAnV&mAs^#$9A z4LZHDmU;Po5&eFPNBOOBLa2|sp*E;;_$1?kFHWQ(Byp*@*OR8cHEN+qNc31%167X9 zJ|`f|A4s~HDq5PJLbXBS0R(1_HI$idRLN)h-S(R1vQ{iWl3GJ^g?0OWHZ57^iZ@AQ-w4eM_eYs%%?b@+67de!0hG4w-XK1{nCnpiHk9xi?hu<69l5e5pE?;~3 z@g>v7n2bEOkEm#u+;)TIz3P}d5x)gvV@X z0=pZdqIdCX6vKq}77_)TWpQOUXQM>_Sg zp+s^6UWt>KRc9^}?Dw9X6{=3lsP0O3Oz=6_~vowt%*MdbM2 z`DYFPG3dm%Km-_c2DWXio?dyY9p69+em#5o_1U_X82vRU+<6ET+R-d*b5i<;h-1>1 zUHv{el6o3C{}l@9<&Wir-i%ol;kj=Sk$I6qNJ&BmGL-83x+pz_mzTW)35jU&uGTNk zKXi1X8Hsd8F4aLE^4pf5c`s0lG|Kk5Z1fUkdpfl^u=ssESpg|k=0owj6lxg2z%2!< zx|Dh=?YN7Nj?qbV;^(U)VTHO8iY|Nc<@fVe(rRVWHnYg&-9zuBY_KlbhxlrN7;U_# zU{_gf*;lR0I4jXQo|_V7Sl2op0KKINU#%Z!rpD=({83?}T z7gUAgqe4`SeqszhQaBOvuZi2KbRWeWw{&oA0a>EQPP_ztA6PP*rdV%tj*sG1v^Yk> z)^a%qTvuAbq51+mC`N2?sr6RpBxo)Jnni;nkN? z%{`GIg>0l{NKJCrby-^akJR7zdHpOzLewZ-=lhnzTCtbQQC-$i*mHw=a1;!eW=)t5 z>077py_7w^2})xRVDoWc%0@ykzWhC)e&=eoT}qu4n5zG8hw1MT%QS+qX1)q5q8@43 zz=Pw{?Rh%avp6P8=Zf>L*Xb(R?~U>6-`jReV@BuTSJSCWIkczdvbAkX_}jh0@tp$# zGdRv{<_WaT&b!+-cf0JnQyeomjzwX2vb**M+Ks2rfoKO${m2eo`6vyH-bPt>9y2-J zzd|=4O9x8Y&3$unuZZLx0fQ6g9^Hs(&Rra}pcy8G2CQW_x>vtb5^r$?6CrSqJkjNo z_Dh#@`}2`W?4GvTXt1n4B5(sJcDTla%DMwx7g&+u?|bQbHJSorKiKU1e<+xEB@Zv_;jMyLY_anFi52!s{k=2hHk$AqacsIQVYU}`-s%|;kv>%r|h5dcVkx&HN zaZ8`?7>i(4Z+W_%4+tf>0+M6M?OSeQESn*%-uJmaf3YW`Uo8+$TrT^R@s+vsS3s~i2K>+;pOViSpJ~W3`_A}9q%U`QVGT4H?wDsg z%GS<=hb}x5FMd5D0lgg-FB)HzX=fx5@o4b*&XsE|-1cL?E()tDr-6{;U~LWiO;mt- zX^76I6M+!fE#=8da?sKU>APz9(2pl@)EW$PDlxIJeIu+!#8P-Q7gf7ZN5_qcJW^hu zP_4{LaFb~g*w#u@)u#lDKalmiO(Q?Ji8h>&*jMkS7+YW;?_<3k+bO>S9ub=_Z z0b8U=`g5 z(%zk$mZOn9V7XGX@s4@GeXToMf3O$9IsFTKU-08QZH?M>g2(s<_`oBO4v;5_dMH1I2Yj`->giy;(B*O8e+ ziaL4{NBxs0o1`^8<;Cl!Kl2Zgh{<+zJhWKfQ@DrC60nX_G6O*jTVDzL26fnXkbI*+ z@|?tK8NIvQ=!MYt2DQ8=-j(=)BynFvthkh=!Q?RxsoS{(gT>G(qMUCEoUfz*-HCw99lwVb1KeVbEUq;boP3B4`Q z!--*EG_CM_d>KbgV4=Lyh=yGbKQY%#f;4@PM5^*H{_(upyh-5fYk=hjp8bhrBjMHX$uldY$h|P^1#fGt>k$Pni7H8{CGeVPD4Q3ROk9AI`0@oKF6B%xK>rsRVB+>-`+PB=>u3T0~9zK?Mxlt04ArNn;fA7i%ia{ld)AeyHPtuqoXEH7fUp<+qi+I88G5~S zkKHuWE^W2zSb{4xm)TH7bgr#K@9H;Dh*|#^kMU#^x;JRha+HcHRx`NKu0+v$nv~4_ z@D5yJPHZ%1eBu~!Xgf=&_e396)F0(eX2Nd_b3N8ftlC)DH2SLt`9wZ02QD2BW@{=$ zm^Y;jSfdPW&tA7Nbu`9G)9}YPCPYRUNgX%s&Q=1TBEnIv>FgqcVhf)&P+u)pAI^xO zOFnOXBC)h;=FPfMI!f$0`P*z5c5?4Sylj<#psl8R)v|MDRG&t6JUb#=_-XTg1#!O7 z_X4pjyA8Hd96*nCpFu+ZIcvA$?A7_&i;jfT0!222x(O^C-!U6@@Wj95MuUgp=Bhs<3Nwda_Fsgqia6kfqABE$B)e9b zV6~YEDxp0`qp`ObG%pNs-fUq!7}@!}Wj6BU-4bcyH9ks<32pRw;6OfjV#Cl+RX?5i z(tGtjnh1({>=Hga7yz2h9J)HAT~Mj}5<|gd7>HigIk0A*9jG>=ICf6T>3%P+K|E}l zC9E3g&5OFTrRhMQ?H-lBUkA)d+pmL=ib2xlqPsQfZE{pgxZL(+8b{7I$z8jUk`)V@XNcfpAEN?vXhR7QgP zyCboc51`LBC4~3S3;AB~fbv6e>y zohv!<1QDF=R-Uif^$=F5GHPG9AW!1L8FZrscBO)x7z~$fx&+~BRc`=Hu(T@4tq~CrnYh?`}t@Vf=w|Zs0 z9;o(4im0u)W1MnL>H^XTRTVz7Xj@3Pf4CE{m}5W;u(wk3FKULz-IK(V2>yX36B#!A`%v|R>xJwD?WY+NML=Aclp1TRB(Bre+V8)f;D|4P4iDLlE+lg3;LE)GdW%h=m<-J|I0WnWv7LbvHC$mqO`Ec@h-8f zw0klufEFX0uBTfb(`&XE-2KSsK|9oW1E6_z<#>IesRXawr6!~iS06}LIry^(enkX` zyk-?XP-mYYX+CWnu6{n0v!xt8dmeme z%sR$ToV_t*_mZTqGA(bw7j(ptQ>-)}9&Ug-Uprcqr;jvWVobJali|J_E1-mOo{^KZ zdmu{5)_Fa=9vW+NwFS0em=4sWG;VXlt96EN9@})QbALqmSvR{ki1k z=X8W4tW_kFL(p-xR=bCcE>2uMJfR>DFY!0}v1)tC_tjzjA}>r|q#s3PaJ7>wU}l{1 zNHgbb!oyE|)8r|SFdWFJCwW0MJu6G+Cr!65DBAzfrqglBFpfQ2CoorL9^;mnCMHo9TXxfW zJeEJi7w_hT##%ayK5-```_IMQNF%W7&U+EmabQ33$3xEX9L|yL?w-Ma;uw4(?ogTC z)|wj_9*)$K3Hn19dy&$%>{hqiz?9)e;d6Z6x^P25F<}yci1T>;D+cIe%ISrMV!5Zz zJSiLFJ`QZ$RoDPU4ixJlW2W^xEPa^=b*B9Ao_)_M3g3-6RJvr_a*z$2xvlf?a90qq zK_~3H@NWR2IVEAvhoMFM`7oIF-p;TiwO2C3%K}9gDAX0HK=Gbs_aejE2S}XQ&Tq0K zJ%;XZzrAz(2nQs3N!cP-IQaPEW+ob7u%dkcq|YrE=cn4gnh0yWx)3aUzR0eA%yUV9H16rPTqR6&l`5p+>vBPam$^`t~ditm#oh2h=kikrBt zgK`}9$0);aZ0W=FnUl*trrhj@WUERk;Z$JN&mXgJCMsfKr@FZ^tCbQc!rcu0GeJ~< z3g9hGh*y+6i`v6woHrTHPBSyh(q*3N%jaX^hI!c@a^^cqQ4#J2;5)0;Ob4{9gCP z_{~c0L}zN~h9gbcBCIMPeUmSQu*)7Ask!d`?YPQO={W<={HH+Pm(ZiFd-H5&0x3S% z?$94$&vfDFyU|K=a||Ikxae{b)Ronx{)Ys+pN#8`m@&jU)jU;7BnM8MVs;LU@k&1m zLL6<$onZps(YN_Z5e&7htt$CbRE4_lJyE3KK#~=p2k{qIcA1=AfC;*G=1v7d(@? zSPv76u4{FOmdk;L!}sc-3X-+Q1@o)T@Y*_9>d6h~1!`*N$$fHQFobiAXKf?&@v4sRR;? z`huZ%-ww%cc7+?CJF-Lq(m-QNZ4q5b;q7#|&>)8055rVHK7vP(kRgX;{*2ngp&lYeO%ra-`Q{F`sv<-+ep(M zt^w^=;*1Phg2-(aTvb9{ac#NU3NG(&Xt}AmHA6L&1hb8GId47IyPox#Sl9lV`2p`P zPdMRYNCe6XM^$QXcbWcKL>W3ejoRkzvy=Y3ze~nlXq7$X-9d+uEotN4WA?5r4v&Y9 zu;+gzQs@(ac?=eMsKu%-EC5{w4Jx`^Ho1|wxJhmw#%sDtlFs^XC_^;gX%*vV>O+54 z8?kgCC$}v#!&wt*bL}|Y&TqIq-sOS#r%Z#ph5f8f&sVPi#{4KSK^0e`vMB-M@5T=`De5_k@b3qFq+id*ikf>_Bk7jNu{-cyrs zO&Awh8coQYnB34UU(Ud$7zPI{7Q~(@Z20!iJVTNM`0RI5uk?KGC@j-J&eZ80AN;gaXdL3gWb=#8pEp zXtiFx(8cQmrix5aGU^>96E$v-Ax--stouRiR|3QrRN%rtj! z<(kj<(s*%P*aAhW0^A7WJJW{%)~8fCBpmUl-wW58C^$^DpqEUsH(kLt8f?CmKUdU^ zq4!-o%_1U4n`1TjL^MWkQpiO6gaSozOXt5bgn|pc5KqzasYia7;^hB+65V3lGdhGF z{AmG91`D`YdTyO&Lni9`ld$lM|EX)X(ZON*r{Rk&{BTlrWK7?-2xZLl8RdBNy(zkD z0tkJ!iMWt!-XP{E$CRA1r&gqT8n5f0ri4VJLGDfOzMk1xA;h)19vjBVWV#z6MNcvw zP7UyK;_pT5fr!;n6QXLm$Q^*#wmn>O)&Dtv`-S zP@;vMU5Q3Q+ezB@tnV`8!mQDgAK#)d=U9^eR$&WWSOk6~1%{zw+aVg2NEfobyVbKP zUYo0YvK9u!?Q;$f0FHPJ9{l;n-!XraIl4Xfa+vffA7{PRA1riS>dkTeZGnKw*pPo@ zR7llZ`NMtlBQ5dQwGa3tAq%KOy`4d%D|H;|wSAQ|he#OCd0c&!w5EL7YsM&nzrNsq zGDe^DUUMV)0*_$Xaf=Sh;``rN32A;egCfRc<^cP#5+3uIpAtiCctM6N(o6lz>Lt5e zYbhEHQME&^@0)b7I2!A$1`WXOr5Y5Ecg_ya_8(_q8}pzXrV_Dra@zH5o!>r+BZhkr zgVs~#bSfBaX6LpRUGONneK!O>Z?v&h%!C}kPmngW!CKoY-f9h^@|umZ7n)~ib-|!Ozz4by@VT+rtKSJQHWIf zX82vG-Vcm^yU(On;wJH~9i<&fi;%l0M|1w50q6-iQ(1%SjuslZY%X8eAScIA87TzaTFsZkjkjcEdAyPv|*6kR9MV!!l$nde3dzwmiOQVb+dDZ zv#E%6-npFwR$yD%8kVu;IV)u8xZK9PVk7ZPq1;HVVCKd?OScjbk;$8ye2LHmwAw=i z2~@4?mNxusVcaTop^TxNSvMO=;IfqN4Ki#Pflo4eJK1SZ`&3sNNASXBr`j~hZlPu; zy5p_gR15SzKGREH?SDQY(y8NrYVV#;o6m-#pvQVkPSti)$!)1T#iH2>?bxkWPck>qO1t=Oy8$`sAxP6WCN zs8aM?XQMZ|2br~pCc4|)Y(L|qxi@@m5tA!5Lo*zr=Xq9@rq=QpT^!0GUfQrBD{w%A zMsgupb(}HCiNmYc-PTm+nry^!QmaYm{?jXuTtP@37(eiWUab&VB6oM!jeiVt#h?ot z38Ec#S*;=zf4T=WK%X?P_WKw@tb0*Rse-v73peb=vz_6Sf zf#Sp89ggD&gPaefjrt5%obDVtLfPNy0wIK^p3u|s+~SseM|SmK#^7m9CX0cR@|K>s z#wJ5CPdKIdCR>qK4IsOBVtW`hnGX; znR{GTW^Lb3co)e$N}%havH4GbR=y^>hcnIRW6Ltp7=qk^p>qTYOX&NS#~Z-gq?D#W8?7PN3GKf+d)vXqpspj*Y=1d!F;Rtn zNHf9bluw3J4PGCC$>rj{qQrdlTt#< zxGUIn<>tuuouhlPlyR?_K$XDd@9wkc&70d`jD4j$KKdNBi55YhK}y$Jt=DW>jV@ZF zW-lIM=HgJmGkv+$deAE_*^13L!5C(5DamtzIf~IkL_xKhJcWkQo&HNpw&Z>iE6JO& zGdxd?s=d%*|6?g%l0VhHOBL2*ngyC&dw`2wudZA~!TFW(Y+CEFn`6Q>D6*Aya_|am zV20HE$1Ii3CQBoHRq9PiSzmcE|nsKYX8ymgLxrR|= zT>;KYu`5V#<0())rg+<#ZwX^1)V9^Kxe~i5<2ftZWUBc0(Qz8>jeBs6k9#CB>htAM zWUoL#G-gok$o zH78`fG1Y$^Y=>!^1?7%eQLpza1Jd8kSdOaX>F!+0u)0G*XBem<8pYcEB?M+nC$n1S z;Ll7i=SIi{(+g&eZPAxF(Q$NH)xToPhE2^*e-F|PFX^r+{<1x78EwsABm@NOkv|J~ z$7$hv7!%8}My+Pa8MteQxv0Z^HyRGv(s1ayZ_?=AgKmiE`ZSQKRLh|t`~2c-^j&g0 zY3PcJ|6!aYthH}Nbw0@{UYVA1eCO(w?Aqq}vu^gHMEF4Xy6yII=`_p1Q5l+}>{@&1 zlRD0pl^~2m5>OnBsd~JUCE2Ol*!R<4?fXJdeGy^DJ+lH?CHAe1XHr_K31aGKeCyN0 zUI|?&TEjZj9jaU=^6gcSWW@a-rpNVHO6%0&0!-+zUmNPX^D)TCIa=O6krA%Iqr{T( z2tGBMAwwEpiS-+842n{$xGqw){(vdsE6e?cL3m z^X~+6yUWCjTSX3X?!crvu^MxT+BQdxF5{1~k1&AM>}>`*on~KEEL|S`-;Pr+O_s7U z4PebZa{fSWb5U@%3JJ#Fr_HgRXGPeWi`w%q*p+k0%NxiA^O|@}|(8!f> z?DR!ZdVEu0#VMh1Pw9MPaWqx1LB1ix)9pShjl0f+!fO7QjB>4q_Un(c2L?`Si44VCsPRHpXaV`)*5X_!#$)4d`nYCV#HTUzt5Ma9Ez~6Q#d@$;kOdCM2 z9+1jnzB#V_e1()arqvz2I@y!2d5($%Qd~u3vs{h^$#kDp;eJB|tXw@I8NQhTO$(1t zS@e9DwbAHaGJ&xNubG$&FbB?1$V+{cmQeSkmIvVI#yYg0NHCKxJRSH^RkN?V{{Dfe{@^N_5!`o>|W~OuM4#0oaNxV z3n*9hGQZ86joClRqZF+r6l#*aV~p@msZ1=&qO@Pe)EQt@^+01yG?5b+>=DCkDA0j} zsb$P#KDN}~h;fYT41}wAhVo~G@2mJrIv-Bgh;uH9sPvfrz7V&l`NtMn@YC8woJ`v~bpj0&Ge3S?Pm=v1lJs&*^;6op+r*LrN?)nNjpeNpQz!PNX!r ztoTBV_=;vq$@m^Q$!1tf$bp1bx*iF7=!sD^dG0QFQc%gwp^|4N6q3|i>#*?}ifvYc zZDAPILY=@l*IC7^hI0#Uc=rTkY_$0I;8je&V0@Jfj7LC6qu}-+%nRY98nE=+5Y!K> zyE32PfMp4Olj!zszs=f6>b?(o|Np2aaXpvogJHuNTTBtYYXk5$T07RrC|uEWhLa9z zc?K!5^WL3t@QS$4J4Q}XXI)ICZ)Q#hX0HJb&IYUY@tbqy|3~H4_^+oaq+s+~k84P3 zd64y>wVP6jny=-~IaPno8&d&a11^*!!h1`sye@(Ds7af5b@;e-;UYH#LdNa90O^Z4 z5_gL;H;$x-#6f}m{_X#80g_nbrmJ5HkqjlQR{Do=SytF;`~ykqzb?t5q4|sy-38n0)Il|FpL7al8(lWNT6G_Lsti&S;u@ zaK6Bkj1ZXrjXK?%n-fuFFa;g$!F~R<2Y=~ITOHuz*vtMx{is1FqUgVv-9yy@uJAGf zeN&UJOuuh0y#St6X49AN(NkP7(92E67Q`F!j-REq36=IZ)^Ess&j)Mgfv50U)oxoB z7gqSwvR!K`BR&@7d21{3=HEMEk1{E6t9{hS(eh+$iTh@mmjj!g05O@o^f}dR3EZJ% zKKLQX#GDR|>OV2rZzgN3haf)z4Thv8wGqXA z-NQVWQFfwS&8ls+*xpmS5L!s2uePVOQ?+#$j@4iEMQ*8E6LYJaDk@#LM~$0gu6A$+ zmYeNmkW}f9zE`}2Rrrg>%2gXR-CPxRGZK?5sy)>Gug7(7@GrTA(Wj1)&$j)FAORq{ za8r9n03U^#HwWV&LA#-Gy@A)&^IXe(At>-SpHwo-m1=N|#@ON%l%<1Pdec-6a%+?V ziKnQc(N@x|&>J9&J(-{FM$!jWo|ot?mh4OY<@^2CW?g#Zbh)2u7nhg8ud@-# zXi9j0aoZyuzB4jtj#i=7pQvNTXrN_83uPym{H;RHZuw*V)P{nLI&;D3?P9$FGQcIP zaiDH>)|T7w{7Vf`DV426CgJRxmo?hd<`%c()rYrrC;GQ1O1~3>&O-iQG$7VTyy+1M zempW3b0GfN62vVt$DjBF-IK>!3k?pR3Sx)5{1bAk3w{dJ`I z8}xU|cHdy=NRbL_Je(f-Q5Z0tzab>UilZ9xeM&T#>7NGX-j77FWxZeDkaK(Xm?>4t zpi!a4SU-(O8Wq$OqAU2ng$sHv#d*7iC0G)dvZIAcv~L9D0D}e5j{4o2a}@wp#PNDM zRnjb|f0!I+PdAf9@;w_TD)cy3xDm#EHiHzT@LX~fYj%EQO*7Mq+PNoEE2Ike&l58dN^M?P}|41?v&Q;emoAocy41! z8>1(QL~?rc!l^(h4()|;%^EsUh?;l6F9@7}yNJHrQWmb?M1GH)S&D_&3Z`J@rF5RO z&;QEKhAAo6Ck7-;t|2Sp6Drw5&-^`Cd|p1SS#rtQ!0 zqMTlu@^=@|l)XH1oVKhG`y%TvaX%ctF(s0pfr%SdhIy-hjO#w9xxK~p&%k?7C-C~` zPU0ypOoLI2f)U~(iNF^dKUx>0nw9>w*3j`vHp7#Wo|x&Etih^-En&J{Y_L3?ik-tjy*JJre&F~N3J=`&DpglQZNlw;e$WaDK?lz5@ zFa=haUR>I6i&`e??}q+!Wa9r%$Dcy*L`DZ)1FdzSiO0i!>J%XphKO{>d z2&}T(9Sm=-;l;L7mFP34&{}#dgudfWo{I`1L-z`!!r9Pth z_sRciXyX5GUw9c`^9~kQaDGMjGH%Vm_j~&N2MnH9LZ?J;euw(`VKskUKsM}Tz)@Bf zLBNVv3;!(Rar&L=n|Y3~--M9IYR0WFN5a*SZ8+qX_a3ep0PwsWX?1mw8^acQ_vOcj zzurW`C_I`cm-$nE(t7!5o9e#b23e%5-%BZ_FN^w zRzJLthtG_xn|(X2^)vC=F7A*%=s+=r2ey4fi7Lg4&aphkR%jF6Py!|u7x0OvDtMD$ zczD`#E?1!$&dbVUqm&cF^^xaOAesZ)*GMiS1GGJ%UJvJ@)ojF_)}amSidZb7u`*-` zSF1p>lq<$46c5(O?U*exfl5cSWTAHs26aOc1nw{M#6%C0F_T)w z#ucop6n$iHkfNQNSf9!YaPRctTaFT}GxL67`TXi0-et1sG8EcGdepsPd$Q-B)+>tQ zzqY|}VQyTjE^WxizR5PtDOm`}A!+R_3_0AgP0L4RBV#tJufFT9xbwajy?DCgIJvt$ zkpq9>+neC?lVsreW1}LBrN+T%2K~f(XLd>ig>UTfLzQXM9*!AT5t=nzd@xp{WC)k^ zRGzmdbadg3$%+!&OpI}J>(EL)IarR6?Zyeoc|WeCg-)Keb?fX4Tn}!HF5i^TM|)+1zoFJl zvqn6&upb-N=o}>_7_!)Yjmv2`bi&>-#DVKjMV_{Rhj78q(;f)rw^&FBM?dZzN=!1jRZsd%q0{!J_+R9&6)15} zt2}r74Ht435>v-zcODs+H$Wh0RxtAoB?K*fUD5oF1kYP-8r6JLGGChJ#od&a6xx0` z;zwJy+9OF-P|4ckJ$& zoZeK7H6(jesLp0wV6##+Sq>_!aQTEssM!J;@d<|><(ozMXcDB{sFd6|*93jqyxO)`ju-kh?OKvi$J6TjpQ8ZareOkP@hdi+vQK=dIxl-XVE3Vm>ZKpc? zpi`)*aWmt4>>bQm6Zo6<#}Bu4HE@ynVD)Zx!-T`s@>Nl0>K?xMK;*pJahc7EB(lH7 zi1`gy+^1G)M?0eM1IdV6FbKR2#cxv;o(GsiKMnUu%F9e z;emQRmt4Mx%Nk{sCf`X`^U4?e@{GPs2j#VN3dN zOD+nt;=LlL%Q3ErJ|c49c%%@`DIFnIdZQiiGPlzn-Fz3RL>+ZVlAf?8D&E*Y0YI@= zp|@=H(uSH;l@Q99W@Ah+xa<~=xl8dv{VjmIw zJXE(YXF;gdd1K%5Y+{@FZn5BZnq^hz6Zf43=rr{6a=e!?TMi)U{Bo?THAAPmAmz-E z$eQ`X!=o{@^7#k4)VDt+n%;0m!741?pYVZ?AXaLnT&tm%e!}$8-)aXew=r#QMcMII zqj}@(2G>2p5DxzDdamcfJl=s?~-(i{N;5jt>zSyujmuk}A$ zZ=71UP!cGnMg3PVTmGk)HJ;VBN^7W(|Fw98lG1AaDKU&u`}SD|{<(17A_l`ZrjnKF z-4dR(y0s66loQvy`L}NoOUfmi{&{QPyzXp-+dzT~5Qf}e|Zf?3S>4$;j<%n^IT4IC5aHvaW!{?GZz>s5AbNrQlemA zxa0>q#$Q)jMD;Xm^xVJ_uOr%YaC^VZv^1A-Fp95K(T2Wu5?wgRywdEEIqiuC>F5~xJu=UITf-t3dF)BFp?#=x%F6u)}|9lQ-m$Iyuw%6XMnz#rV+8o=yC0rtIMd{3ge z8HFS%e*FdBd|O|#k$uhZ`O{@%vP~6rlqapP>EtinNYolZLGhhVc-ZXy@M0>X9KG5z z)0uYRjs{G>^FdtBd0gVF}(=e>bJ7abfO06>2bg zNpJXgz*n?f%^GFZoENOOJ9uj(SHR#FyXI-rSjvaaF@v zt<@)Q+ye4#R%(vCk9?cKAOBiz=)S}lt09b2nf|$I=Gc2h?|@+UAZu(EfL0+JbH2Gv zwZAlhtpken!0#)JIJ!4M((|oLI9y~9SzMn5zA*y(rW4Z-8F}hS@S_UfX}FA~_gz(R zI&gy%kVc)g`N}{xyJ^hCAR6z*Ru%d_Yk{F;LSk9|C!)Njk#)m4c9dFDbg$cw&(BI2 zI;S@G!D?b78_G_$5&Za7f-go>N-Fm@_jz{<4U;F0{&#elz4>(g^XPY}sE#BZ39k(ZkH9vyBwa($f8$h=H?JZSH7B;I<6<1XB{h}My4MoMP) z1D_LFS;?N5SE0-Tz6&)@pK1bhgbw15&y(*mDV{WP~dYvFP z)wxs3XK*a4q} zm2Z~VOZ@9u9NG94Hilc#+KS{TpBArfftg6s`)acn#};6r@%H6#T&iEw=2F>B+Rzd~8?YwEX(k;34;Ty8#eJ|wP&&P)gA%jV6xz&6h+Joep z=?XeWu##Y zmD|Y~uT0w%ZWo%S0?Y?D(lxlE7%886^Sy}Z9^J1uwA)b9RPFxgui_T(`TdR?F@Jei zUBaAwP7}Y3#I}Wd2b&MT?ydPJ!81L`)B#Ls*;b))zm93v9r`03ey6T&^6n{;!9|IE6 zji}t!`F&g9i^D0ky~(eKLht+n4iw{D7zCs#;F9rYp8KU=%-Ug9IhM0 zY`K^8KEfjVMbN$#(p!I#q^^%3+1U4nOfRJ{9yA9JC>IT@i z4wKL!L?i_1&Us%ji6cR&xZ-Q{jJG>!;HMblll{^~JRD0f%F*k-v-_!e^F*8vh@^5S z#+O6S>8*reor7ukWjvF$c4RHbSXhuIluXuyGk!UyM|-^G%VloA{!ioUgQ^S`^Z7WQ zf$Pp`3(YUQoF{$$t+YGO1QRuCRIjZXvuThdqiI2jP0@~uGd}$e=lbV|^x`gWA`ir6 zl+epXj}5Oc(p2T3%HFRF^(L6 z4%gFEU4}aHy9+K?9_H+6l7G5IS;``1=UI~Z9=#vb>+Mf3v~BP_n4+cV3;E~*ctOo? zn8{mi=zmuu1CSERRlAr6X^C=|(P)E2>-%I7?B(yGt&a#9}N(xUrS% zqa&YjYm!;5?@r`g(q{`f)`4QEg<-9^n-3juJZ=?cJFxn+W;rld%b$3NY&{nkqG|^ zI;Tn}Wj0lY)#yeHtGf;3hIEO)4Uj`+U5@`xbQz@DH%fFj4kuDCV%&F>rEN#ae{1C8 zTz~a&6y;U}2S0=k(@eD?hFz+i>@RdFG#$%YMitWCf$kx%Q~|Vm@dC9%F%QvJM_c$R z52UYkyWJjIA8zTlX|ajdgX91O^mMqVM8;#GN%zkEt0>5Km&t&8&W9JaC{9FG!Gf<` zZ}>+Kw>U12;(uD#lXAQOQPW>8ktttN?!9Loj8ayq*zdP+n%kfJvd|&E23>Wl@(-Lf zf}P)In)H&JX&-AS2`ZLq4(1a&{EFSU@IqTW60=pT7J+9t3I*d1!wZ=-JOEzW<{Sqy zrHjCg0X;8vXvGLb$D3{XF~#g$p=idRA%TswPG+mD-xR|M&<%Oj4DA2 z_l74Xob|guC9aix1OY9_s@vP7v&AiU(wSUZUy!n7(KmJbx@BI1Qcy`YR@>J=O6)}8 zj&Me8M58yvtXyXPI*^DU#b}N!?_NxLnK0E<|K+rT%gF}t0!iwq(8==7ndhB_?t_#1 zSV+x;b9x@>4InY{t(2Y}w}XXLyI_J$#F*GgnP%S@U~Zga`hK>H${9-nZR0A;_*-wI zE+=A?XEH+uzI^K+S`Q<-kMj(UE?`|3hKICS0tSq<)9KpYmtJ&SZ1b4hUdPSDarM~K zvVpShIx+T^X%@V=_sxWfuA9IvRNEhTooa8t?s*q03W}p85L}7_ZN#ek_OVSdZ;r_p zgc4jt`PH*u5IXFW;x3{6Pa4AuZ32-$jxsMd=4afYQf<&uvdrQDnb8op^PT)7&)e^b z^nvue#Gf#|;|x%aHV07!J4n@}yRCC?^97yks5&Hq`| zTRnoLkp$W5!J%lUJ)>cQm&JKIz|EnhHK}2IL<(34q7hr`zhU|R-RpQCr(Wu#gAp?) zKB918ve)>B8jW8r@3l;IbLGo#rCKJy0PoV(C;E!$%+Wy<3X&J74QlO zt4dF~d4GvUm4egdq3pOSzEuChC`Cu+4|W)9Cr&)PXA`9t^Nb@oR`uU91e z{2|C;_}TjNPUJB+_Eo#@wc`2DfPFEghD*^?->=5(+}sz&7pXK~;gWGT?^)@duw>>_ zRckp^L$6i0eF{c;0|OpmWr=>K>R7xp8gAk1QGe!D<}Ft1TT`|fbi2jp%q1$ieL^2u zJ*OX!+W)qSPt3J&P+8^LIbHv%ESRs;Dk7;$zZbctIKDkv5DepAc$B@m2^Uj5Bk_9P zN)e)9RgVt$q^P($c8R)eAWDo8Vd=K9H5~xS(<*;845*1NjzPd*|}fxDN$_ zY9==uoV(nAS^b9#u*>$PpXKR|9@Wx}ljf5SgPGUS0$#Z|uF?Y`eM%^@8B6=k0-n&R zCY8nX(=(&_N6)()Ba%#yn@=3u@$cSbivm3!0A9D_d`jTbyaVA@i)_Y&0vZG?wWSyW zX;cE3dkNd$44D&B9WpadRVibcuxvaw@{l>pO@0Jt&aB4Gbg1-Cl8YOw4Z-MZ+U_wG zzscvGavQxS)nV_4vblZHm_872*ez!Dvd)3Ac}r{4>ryfP27yToXe|7}qV{cVGvQih z4vpIswTs=r*<1K9Gp?F>to~wT_5;(^;$@*rj9R2tJvX@VneH|1@p(BjSu8f9_@8)~ zDNpTIsukt7M7?Y=G?wZ4#C_Vg^vsV7H5}|9>obxG_$=*yYXry%pM}Pl@;=L#&{5h_ zT(W#7p9m8#TI+NTI^t5_R_tT`h)ui@q*0#RDvFnXMK2=zs@cYBa7H~0KKL&JV=!?R z14Cj%WczY`kxac{E%UK$$VSiWUAo4E%P!54kGtOWak$f@p_*;1s{p|23$GN>{EGF# z#Gg1S8vT~WjI!oxw(L#myn3JAM{0D%lJryyz(#!kanX|RYbDT4$LduX^^o`Yp8!|~ zrzSXJOLUvmJv&m;TL?l<TfikpKZuU2 zh`5Gi38!RRKr0=SKFcQ=AJj@xduQ|n9nv1#F>;j>-qXFw8lYS(PA;z(x&@dp6VD7e zk*wkr%$%-2S*o5gY~yr_=E{#PvwK_Jcq%kR-UU#{0hA`kupEO$3{OOV{VdH_3sAgr zbM;ANRBICkobo;}H7`ZpLGcc_Y0L+c)Gv1FWaAqo^N|=%_O)j6L@|l-^Z9nyfOL8% z;LbiNt2+Y+*Wv|`03#($D~XEs@P_vKNA3*GS`x;VwVrsd(qHz*433E*kSY}q>YrZ0 zLBb(SO~=_aS^fRxOe(!I-&dgYM$7pE98KL;bO1AHJmI)#mI82hiY}lDwh3U>2>!95 zKlIb*8x}KJ@f}8LiRR(4dqAk{u`lghzDqvo8x8852mYQj=n3`wZ|3k-0cdZ~Q=(M$ zZPzq=Z4}B%lfyL1dwnt63iUBvrGTG?^U0a$YU3ua%kyr2^53Om)JJl1RCyT1V~=yX zidyQyb)%N+jNZM5udn!Fz|>om_(5_nDl)4pdBd@&LG5g0#RVrdsc^P=x-6*7g5P$1 ziln{j+7yj11ZS`vJO&XlEII|As!3lKlJ;xhJFeFE0L=$gPf}kGgA81G?jf-mp!mG& z4h~Rbs@P_mvvsp&D{Rj{y!KwEHW;;-4hy~4sTKQyA35BFkcO>fwLkF;ob)$zov-~$ zEeK@=`iMTKM}3D8tl7vd^Q`#EA4OZUHte#Ifzskf#qu|K?Slg$?o^l35Sc_Xje~|= zkgBD~vn9_rGH6>_K6ZL@+t8yON1q15D6Z^TX^eZ3300E^d%jji2n@_O`61$`6}aSv z4XazR9dV@L%bX-i2R|>D!KuOo-#bUZD0fIPzo&JT?QNn2Z}Fb+;rNty^VP)3AG4j0 zvC=B@=}UFL%Wa4&s8I=D@2_;%_(g~vfe=c0m1@F+dmUS`+rZQj=h<_|{efji(%4zC z-1TQ9vN$&42SSRKu1?ZYBWl`!%MtO6GkNySF&pE)yuf|6D(cmeT99K98t=myq?9K$ zwI^F&%kv#(y9Znh;ib;?4E`;@QR-rz{^z4{#yjVN>%;k7na;5_hG%rF8W9W`gfgso z+ZHx6LZSnY_3aJeOQ7D_2`fK{{uoo@`@qkm;_CkLO88%Gi0Cq&aW0@X)d;{~R>jzt z-wjghG(6VUBhTV^G;21g3bTkE9rrpRgX)E{=jK9j9@O(K8=GGOgd(VSx?Vrq8IY%= zZAe;X6XF#`LKrE8%-Yqnf{5kwU_F8^s-VvI!EzY$w8WWYtA=ST?p%;?6wxYtT8OY z>G>>M-cnEHwMPxM57F1ods7vr(VSD^TAocVK#tK_46Y{Q>irnD(v9|y?V=>$a2&J5 z(2&i>H6-6G1gvf@WjEF482XKN#=ntQ$>xA()m7X<=xoSWX;eX-Ri2seI(QSg+|KK~ z?({_#8=dJ~X{wj6V-TShDl+um+K$urj~hmkV~rS2!P4%?JR}ctgx^gF5NgZ=_`Sl< zE9msWgUdY> z%k(yE*phQ&+Zq=mDisyFKK=XTYq`h6h87}5fRl5oynu6>7=0M#;q6c7+o{(i)m;Cv znbsCu4sbq2v!s{->aAzmRVwm-CJW5gN)nz6H$Xp~EMM8(x&6iFuUw=~4?eJsC$H;a!ChOiOKQDLD)0LGLdXZHdC_9m;i2^%UyvBJ`kp~F}rnNtM;AQq{k2PqfqG3 z&OQlZ!2|lm6r-FxV}G4}u7amhuKxz2U~`3WxNr0BXpva~oEYf~Uq_>m%4<3vsjK#& z98pukm<)Rd>ge#Zxjbbrzm4vaWQ?J$_Ye1S(Ky?R-&>LmqVu{T`>uHN6+CF7wsp8S zqk8@H(>c!0p3`~28$sLUVn&^kw1@cZ2y1mWa!9cc4gh?F-RLb-5fL-}=ShvUNR*=X z&JBZwuk;sI*3xSvkL3{!?ZlL%y}~sZL$-<3{Cxp8cDlHkVMA9@wkJh>u0amgIPVZ0 zFwui@x5X|D&;wH8oUSW}?dk(y|C7=6HU4lh4hIB{Srj;rBJ@y5+Pmj6C#Yalc&l+r{ z-54Kz}0`i;zYo8 z!+t7sjW2@)a=np8MT1jAUa@FC`B7w7g`SC;Tlo4jcE&8ti3yVwQBP zzCwRGe^W|^iPG#Iikd^``>X{W-58!FMPq(!&A(FTi45nFx3-+}wiou{!-#L(OfD0M zktUK77DtHOIC{`W(y~jKkXHLs7l`^WbKK9Exjv#p28-R}h_|>KPirg;RuMCF2jxG` z`Q>zkrPL|=J|XkF7ZKkeWm7m<+-ONZ+-sL>t!W07Yl>$kKQNGaDoaRJO62rUHnY^W z&?=Sa$RWywuXpXTm+I9;!F$R9?=?{*xfzQ&Y$YCZgx$>by@zx%M?vXxPsjLEH9_!5 z*`mZ7XOodprh$6RAW#*D|GG|p=^9$m2>4~~7R`<0Smf3}t&-5M^Ie|Zab$Y+i^L6Hf=|R|PI;6I)Y(mJh6`CQ6Q_zF zJm*g~J5I>nkf7{7?G$R4kegB&+#+#k;|_fnAX>PaIQ~UB!t3@~DspMs$=r455RNT2 z8rt2I*BDui!wY^)$?<;0Mi#tHrxy5yn{VSQ_kG(onXtpbBJJMM)n3PpZDMND-g)^! z!kmB=XSqG!mx@6AMxPUWv6gU~Xj`9N<`@>G^!&^ajZ>|EY%QjNT=ef)H0y)gY{!h0 zbb((KyI5+P=+;qr7PJp{L{x^I8 zME>#-zCI(x9~I-B*gv`$Boe5kuMC-dKM14bVG)S%3F{H*$@``FUKZ%w%RP;h{!cJR zQ?7dgDnrhD*Ph{i-JK%=$1bCy$5^o>@;&-c21uw`6?fK;sALUTr65nOlc4j*GFxe# zs>f90dwYI-EQlZwec%@uhlEgQW=_m=&{F?H4}3*df6ee8nvwRWauw~C6^UiR6}KcU z&WMLz?D1>WJ)eS^eqmLla&N>*>RQb>=5W4C4*P-*y^_XFX)@+7)Jm_jHfAJMkHQD9 zSGQcsOvLv-1mEq!5n(UCQB*+Pv|LiN4m@>?bIVO2LlVoahYq+RchSk)jCdXY{E+oC zN2r9p&IKe+%5k|`Xs00HlPs1wnXbzgRn)G_ZewB_k)h);6>J(wmGeylj%NtAJ0!K9 zvosz$8+^T*cNc-3E!E0_cTIp)k~rRMHIn;I>Q@gnFHck>>k0#IeyaLYO|=5QgE}c&b#c z{C+`$i~gOF5A|_5g`>}n_6=4;db$5_yH`hyj?dPPEpZ`I_MUz5$M5r1{!T>5=3*GO zI_ulzeAX{k%(QAD9}=?AXqc00k%e(@WOb}36o4IpZ+z_iTJfZw0rfd*dP;v%5(mx^ zN*dD-CyY8TN4t2Bk@ZD}A6SfnY35U1^hoqa?p~{m$`dWYBC+20pXRS_nZ?8~)6CNF z!ongMK>Yne;ReJeZCtF(m)=rSP?(>}l5W86lzp4$_@X(&4Wv{Wp&=&PTN)g60)HCr zq*yKmFeGi>)z00SmUFRM@nTJTOKteJ#|?RG$xo`E^VgC8uQ3Xb1lysF+P_C=*K=`? zwIw0(yO0o66%J!-npFV1m7mYOr>mcxQ!8ZVsGxMYzPLf{hSWsu!dOU{0n)@~!sY9K zIJj<+HXH{n!P2ruu3<^V)JZ8U&#fQ+K(0wDVXE#-n|N4O`7Emu@wWE$s+e=f;?*x0 zXQ|4=Mx=_8vS(#D8P|Y(v*QcHZbD^>Q?L?M?4*|yHFm6*m3%K^d8&{|(LxM_img}h z6+K5NHzv+U%@y~)#;H)Gn67u!64l|XV-{NbmMhE81gEZ#$Iv9|v6Q-%*%MKWjME@r zgnEVwR+B!s;w2itUjrH7^FAeC$NYD_jp5R)AMjejOxnwhA$3ODD5zAOeb}coDn-N# z0UXaBllh)HypAml?6G!)+oy6#4h+lT>NJ^UX&ZU)O!hvv8`AH|!nI(WYU%yFNHw&N zPWgAm#9Fy82otk}1L9Wvxk}hN9gabe7h-Y&2#c6qqA(7T9W)UAVvinEml;yd>xV_Qzq-f}PcR9@u>O8AiT zy>bmNu3hmP@CfZ1ine+~0A|JfmkwFMvV|z!jCRA*gBgJj^7i|aNiLvv(<#H-D%1Vi z7R+aQmo~hVD(}$|QrIvxGi3I=^mn=&W5Mb_54^?OeHy7fdDfaf2|0`^ z;=PDq=<0cfLvXbPRnhuVHPxt096P-SI>CHu^B-4YNsA{0yI;)#6wSHV9v?@3#v8nssA5&A8GbakkjV8*jPLOVWNGfff}-N0 zWfdz88cpg#%ZKpzFVr!!l)?^!npgA^H6pPrrMvhQM21^^$lHyM5byr5U332WD7wK| z3>M>7>bkvaQN0w}%)_#u&$4(CZ)e*{(Cehi?upQ;bVX*nP=0`0A{4sg4fMCi zX0_bse@z*_Zzl>78!}VKSOw0)7Y&y0@Sa!7-N%O;DfOMA8+!T34VDvOj+2<>?q`M3 zg17xQNpB|2{T;x~JiuIA*uV;7|TWEhL1+xUNZp*gCqGF|M@(x2B#Wzor z9iDrMQ%8HmeYcdE`vP&CQ&kwf$FIPF)yjuWGUnSbh7Ga7Z5HeOSP7a<2=Di#w@V); z8Yp*$se8c9qI;9VM~`L*!;lv#^HM~Yj1(sy{MO-}78BsA1nmMl^`rQ#rW_OPQy2;r z?qF(8#+`YY^B;fu$(LN!ot#rD5XEbY!FsN%Ei9iWA0jF{({{abMW1&{B+_z-G}T9G zOm5bnsdYm950QP$wytxP15r%I)henXo{Ud`+C0%^5Suyiw?I>$cvo4u9-*qv>gQ@N zXuEl)k39$E@*U$=er@UE{p+YE?(9O_8j`tJN$gf(Q*gZ*E-_9W6#1pJlLV@jeC=9U zT6YO1){8gRbi`=9kY()+g8!{~0=;HD5*zjHX z3gzwfVzGbNdR6nXK-kZM;S5MOj*2htEPlJTgUzK@W0aV9NWT*$>B(zsEZi#oRvy5v z_2nMp5JCd7KkVL(sBS2kZ!r6D0TK!P{o@D|^c4{el4N=gM02BXeK;>Y3x zxZYqSer);1pPr&(%i+qBQZBRKo4kV3?!UX#+Hu9oeTvSjDnT+QPuCpyoukBUd~BR5 z%|@E&)BWk*&Gh1U+WKXmZQ1wFodh-+EVjtDVo45J2Bg&qMSs}~EO6UtuHJT;r5+pO zO1$TWL9u?!S7pDZK3bgjt6xmN+8hP${aF4{swX?}utWU0ax zV~sA$+^*vHy@dW5SeZ7B+x8mJ%+SNYK@h*3u#9UgWkT zKv+{*UV$L1*LX*az?MAv;DLAW7QWr`V@^anOX4M5%{*G<8uC@uqvC&Z5!9WDRbsd3 zs<*S9t+^j@jSdbvKsXtXk1!b00>y80*1w#HNM;^L{=%^MD|WLcpibtGeNJ?Xp~DbV_4{#hG3s>P43U`WBzwi=zA~r@W#tLkLit{|;corb%_&Ti* zclkDIR_h1n)$Xd|AXExSrvZf)Y z1`}{>D6Wi-Jl#eG(c0^_2d=Qx_|R$q8 zJDD&5yF{V&XdKV_374xJa}2hIhWh$IvK&QMv0bEL0rNrO>M zGovOU?r>3_R*BVHE|PRKvSs_T6}&VwJCCd z23J}fSm~)Hi`1heUA6`DbL|qmR@E)=ZVl&`vxoB>CmI_!&7-MJ{UJrF>Ea1{T*IXI zSZRq_oV5|r_|Kl3R`A*DbJd?l)1~LnTQjVea82?>+N7z>C-2w=Xrt|5`IzcnVZ0{0 z!2!appBnLXkLrJ`3ATGItu%w<%o>tYvGTc;>P6`pj zJhnF^T|aZaMx*JpeadDtEDy=2EC0%&d{Y)*v=J9rR|QFZC0}{U61-7H=*V??6Y0_P zYIgy(QG~|n5k%xi`xww*_P{VpBV&=CC0Z&ZMM}-=d?=34n%7z@Ndbx2aQZ;iB@>mi zR|Kv`EfD$%tQH9dB##6n2Y4cv=u%B`TVw}x6WB>543IGzIrF#a~VQpm|<9CPnY zF2{h?J;#oXoUd@C6<#YpAEC=UGl$_7Pfu(cJK$50n!LZDAet|s-@FYXo-i0yrpErnQT)f_M84~YGh06Kqf6Neq0kB)3eIgP2jw?SKi{B9dll0 z$)YT-b}LdP6L}>Xam5tW%j_qgm*+y#F!rzro6z!nGje-lR4O=GD{XnzLl0z<6f(Y= z9hB83x0dQU(9idLFOfhqIGwAk!L7ELPEh4e0?E3o-tV$iWZ{S41N&v|i|!(3aH77% z^H~W`ZEBWHC7c{P47$v!6dZP=R!Owd*&Z80+S09(ixQg79%AM8G=yUUf|;$Ul-|ne z?<}?)WMW5dE~SgNxh2iF$v&qH?CI`AP~TG^_eOy-4nUC+gKAt2&ypakjy6~clSNuT z|7F0fLr#Oy(Cg=l#4lgOflggtD&BnT|Mq}Vlww9IUN^!-tQDl0^0x1mS*3kCboJ(~ z6k6JC)4(qr(Ys@m(|n8jqol3(sr9KPgK$U>yXaGw{%u$0iK%|hiPy-{5RKUh8i-c< zahe^l!5)z+Sn&Ou(obNiDX)) zJC(6X#oncI$w;20V7iG3S7Ro9C#bOMJb&7cXRj47mH&S+_tjrZ-bR&pG#; zyU*S&z7~Ubb}lTp@w49r#Rj{}Of^ltsto@*g}Oj;wx}YuNzxYoD_UI@$9G0~B`L^_ zMm0L7mOinQd-e(K*nr4LxYl#7EEZgdfk4yA8S?`&K9ZgVjRx{WZQJfdDYE@7&jXQ- zOXHL`JZeCjr6j4uN<9qSyDe(KQCP=sAMFYeV~sN7J&1sO;x(D-46E7gS{l*h3|?l| zB!1suQf4(U=~*o=(b4|{#z*N(vz4tt_1ih^rLGJr9?Q+^Tci;RQH6PJhoxgD8}pW7 zLKuGMa-v*)f=S_?=}LsMDn;kY`xA{qeFbw}y5wmwCVe2lXxgnDy#`=#=giF#XZ*q8 zh_iNykzs2DN8kyntvZ2Eza_|dm-E6w5mudbJQ1ci_I#jOZ%E4u4KaLvK zn`1ky4bcaRe!8?K7{gn<6CNC+&F$hv*EMJk@f9%nNUv|>ZhxkMo*%7(0d;c~6T<+- zcDK97i7>Hu=kSuH5s$2zO@JGb8}Y0j^6a107UF6dZ0zn|Ap3BsS5K>!$-!c50&57v zY|(hZnT9_To8UYSu=C4m%c4RcQ__z1*>L1tS*tjl*iTNMbI{Ei@ea^QIN#1eBZXmP zolu)po;egrDtziNY?eF+&69y$wB9b?GD+-ISt$x$U>zvtI;8~Jh>+jfesI1(eXd%# z(U2%^SADPY8-*SoS)n{{Y7v&%e&1+A$(2whwFEYa|tt~?)q%- z0Ifk=%C&@5o#vixNt6_U> ze$0V-MfY_#@aA;RgMS5TeszNm|AA$2A)R(-&st_O9@G?*9L9{MdKEG1IL8!ig6}xS5@&0P~ z+A8bU3}*D)h?4s`H_?@A!ITuE`Z9EO$&){l0}Aa4-zIBTe#!DgkNm{a`Z_CQXaW9= ze|$7x&88ilM`S-=7@ul@e?lN(OKQCpQPnRCC)3Uia=!({oy~Tbn zrdgk3UI4SqdFkciB%mrvB6T2GDxiTerIEfQeRB5IQc`h)<*Ha93I4@%_A)|kI zLBte&8b_ek&Zc$89bd!%t@_!Qnd^UVZf~SecO1OX%YVj+T@o4R>#z>;!R6t7bnb~>q-h9nEt;RpO1xZyLIY{-h6ti}xFM$%S!xbh+;NBdHzayOevgLA7 zzKAgTPo(5Q0|($ktTVaOGlv17RHrW#9nZAdQ~CyE2!0|H49cKx951%#5hjfeL>K8D zS7XxT2BC;+Ba`BnY&MU4-0jc5AYj@uf0{VBC`}&L*gv$=UPO4Ok<@mMeG*y^b{=zc z<#Sxo>EBwH&1F%!2k^^plIMy&A$SiOWCk_`?a%zqXk*E_?rJ&Ib&Z%b>-;T0wzj(+ z=2{{+WkvKQOKQvhD;{OyB%;7pdCh|$=)>?zaX@OOp$716w9ECzba}Fb34Oa2KUX+P z_(Qv-Rb2^w`*9%fRHubVcy0huZG3IoSy&YdWBX=4^l%9rR^EhS?y_}Ss+KIJCp4Lv zf$Tx^lSP1Z!V;RCsQH1s*ttc3d!yZGZ=sr)t;F>5s{R$oJDAJA^1NivY2^m#XDMTl zf~24~Pa0ylt?A%ACF@d%^vT}mmSd6K*+QHG_a^NLHT4z_k=~%(oG|=u2$q-8s13@?>8A(XTE(@{y$? zASk8w_$o$>&n-7CzfnO`@Z?K417+E|W0$IQyjuYf8RNw#ReP?9%!h)bh1!t?s_*IZ z6m_?*mdvAU^cW19uuG7xGIQF{3K=39EC}``S!!2{ovxH$f9V-*K zCYxizYtm$Wn^Y>^`Xvo>DAjB>H*$AtsrSH0X1#6@_v4I#86HBXpyIR^k~waUDE&!$aY>*12>9G<=C=RS*{ktRf4k7X=Zzv;LQgMR?v@== z^6l2#@HM<{{W&Z>?oBd;AIac(7P_7&U0RcK>IVWY@4FKYWyk0Fvg$kA&U^m2E#E$> z2~L-PR(7hNvXwiTx7kbW6(9^nl!*!2*(6|t$;76)?I=IDFNy|{?*ysYjhr%nPO;MF zkt+%o9XMRgKAuo1n5pX4uxGpT1 zV?knls`1&l)nlp4%vN=DH;W-qeR4}dvSplL^S&wJTi+{OSCRk}@%VM+bI`z4mVXc>Wk36f$+reYBe#l^x~f zk08X^6Z)y)s-&Osl4wimi-#+r%T|(JzKx5Vri?v^`=&$h1xt+K} z{Q<(m5~U;HG$k-_$ki7%jy}gZDm={z+n>NU(;_UZbly|$skM$PI10%~o~@=!i{_CV zp<}^rixV^m4g;zOaXEJR23{T;4-e^XL`a@g$*%l&JmIW*3iH!`*W2ph-;3luoP<#K zWynQDs`ITqBz2{=Uv){;P|mxOjpaFXU}E=2Dp8n8)YT>2E&#$H%{?s%5+S#9(j2$K za-xp`>qoWc0^mT;jz6d6?z^~R(Sg`( z>b&P&HEXz~x97TYt0sBck)sO4Jm+B8x|k4aJJW5#LY_cZLGM=_;ZPEy8NH0V*B5)|juIY(q+35q1hRmu%`$Zy$nd zk{51rcg#iMmv8wts)WL%&W~`5!6!rz;+BM9$%eimUh?EL?FKe#|D@WbnRWhin3szta(ZKeYsHVL#_JQ z->*mqRCcTiu4?GfhUKdJs2_L@U!N!}Ntzd*werqHo!H-jDY%|8()jW9ez38zCaeU6 znBIvI1U3ObWg3ONG1~APWp`O@jJfyEWFHiset_kx5Sb+sHk^$LhS8Z=zd?Y)wR$UX zINK{UAGCku^za>YCkkw2TrAdUIxlj^tGNER%L|E_8MzF$&k+v@22#J$p31c}bxpaD z!lANtJo?5ieeGvQS6ZG2DA-8)xh~$DhJQZydtmon=(8u{G8%&(w&^kNIl;*a5%e!f z#wpWjV;Sq$)A^zZ@g#3*m~;j0ld8;4BECM{3aMwVf33m9NSe-V3@2a><0bq~)!=zc zYKBR^qib&cl0Zi8^fC@N5IK0lb1K`r$p2=BF}x_6oJBOKDW1*Fx2`)#@jjULo2Sm}*@N(KP6K$+b-wsxt-@*1T3~RNtV$QlCnZuk z^A)WQxi6Wqrn?0~7T8t#*V1`E&h9vut)DQPsD>YS1?d>4isVY>rwerFFr=k*rn31n zZS^-M&WvAEG%0r-Xa=lwNe;O;VG8~stc~F#X&Wp}SQj;5swtDydWLs5-i_sdrmT5n zTrF0j+MgFtv|i}#FOnpJI@RAGPh{$e6vny!x_M)pDms!u){7F%(w=1+xub0QenggF zu&c~80qDg)&9iL{9lEer;Q+ixyv>gd)p4KA^<1t7&c<)uPUp=t0_8W z0k<+Y#xMWOmSSE7>9{V7U9hw?KX(+NBnrp$(xtK5!`U7Cm>%%n`TF~$8UXc|Zm`-$ zNae!gV(t2_bSk=A^l={s1~vY{5AqvMMW!Y#B4t(<(-48*;EULn$`BhUH?l;&_EWnt z2K>R``(yLxZpf~7+`LAN365C@@rRNB2*7XN4^Snqi~fW&ZVab-K|RsqbG`d--na74 zyM1+?Mf2je7M5`>b6~v@hIVFtnL4z;I?rx?X7E$1*jSGbXTOV%uKY^(&nLm}{pc@@ z`y=(0;p+Uu${eXWDj)sW{n;2v%jwsb+)!L-KldYL1(SPMypk*jYJs2!h zs;JCzxW*P1wMTE!cFWRm0N1WP*E201S?HpQ9bKMVAiLmmO{1szL3QC9D~`M^*k5*A zuE&~uoJcZfnee|oWoG8jDz~!+A&w)H1nDu;zoDo960Nw~Qa7e5vmi1WZ2jp~fr-xP zbwg=$UtCpH6_+DmXVEkKcfCKhN;1TFc|EZy`K|Sfu;`%{vlmJrS5k$(|I|Q`K+QGO zNL!OOft5P76*Jig)uiip5`QarCvKN6e%EhIy3`hK6HHKfdOi2R1*DU~1jheE{Z#^W zM>=_j_RJ3c_7sh_&;?^4Ai?CZwtsyp${xMp0@o$mvmxQX7<@!WMi&qxtP@jo;aF{@ z@Bo7ry=NvWY;60N(dEP#7Gsj2GZR7l?MG`lJ zcq_g3UA=CXhc`hR&c2@SU>rb>b_Q zwAQg#LUS&__x2E?mfyyh;V~1&A-f}s_eaT#^BkNjl+cvkTWJ;e`Fr7&CN_c)!k%D* zK5RK3UcRIN~qH+cB=oJpq9sCiNabYvajq z)hoTKV|l;Dvl>f-#`$8GnzVS5EGwEyMS{&5H~^f-)F**?AEDdQ}Oc+oYfSh2PgI3yIA8aqX~ z>0?{{#tDcm(osepnsjH(53d2H3@>8%y)M=6)#7#*u$XI)uVtJ95=X)wDu>%MR>amO zv2xE&f+l~K=4Afuh7S5?y5&|fG{peZ!#&EcT*bs{S)jW5Zosd%I4*+UrUl)8bcTH* z5AQB`wEwUv+(Svvy(^63a`|${>TlK}pmbyVEiLxT_h#S8rhbxyR-O6vs`WrI_d7jo zLJ8$SiO|)Lud1?iO5_zuWzxrPG#Of9HK~FBd!nV#$-bR2Tc=Qm>{{XlqQhLl0A;SI zbLD3~`wEqf-$L3^Z*_gcD{LiI6w3=sw9rF8qk8&;CfOw8Za-ufPdOKBC#m=l zT#S31;^o+y{bjJ%6}=@zJBr0u?9&|F<>f4;5x*1u*rGg|CFf<+{j&)4Uaccq_~Kyi zm9AV$cBE}eFiK^!r!teq*n`{hZFya8d*xXVd$mxfV9m@>{jk^O+Iyt3t~LTPnl)_s z*Z?c_AHfOejfUx z<93>ycJmZYx2wiB=GXnKP`^#GOM2`jE-f6~n$*crbZX{a)=U%KX#T29arRRoBH{aL zuPftacbm61`+hqT6v^*n%{lGp%Z3O)mlg8 z;pgx-le$XRwtH7{LelvQ-ttjO7N>uY_C0pS>)A>l_L3o7>pK_WGO=4S5~W6ZoJoTH z_QA#ZzuGJfD!NOJ%2kEO9rO0n8QXz6(CnlgH#^#rpGby= z>a9PJ>BPFig7wWzzcX|oJ|5Uj-$XhE56B8yrOx+hP}5fMWS!=@mo)DT*r%u&2iR}8 z>NTuQ=QV&nlKEiZ!pGiFeRf>NOAJtF-|8nw;5S9i3$mZ()WBMH2s@Jhi%$_A+$Ehk zEPf+RT`4#n^7&BAi0!aI!ijLm(I6~cj)z36pJQ^4d)vdmxx%jlqILSd)8etiEW|V4 zLcLZ&T6i$6J{Ft8I&cwiQLN6&a3;IC^1J|K@kqx zkgzuGLB@;s(6-)?a8K+PhBWLOjpQMa#N#C2G~>USA@(R2tZ7`Zq3!n%nHRyoOrzZZ zr#px8uKotZFS#2*Z&?sVa_C=9wQlg}lOCggte%*AoWl*dE8l$~aS~Ja&N_DQYy|4X zZO@*1;AhCMSiVQ~dZU{0$Phu@KM~p z!nBmUd3|x=FR|mdn|^D1DUH2lznT0KH3Qv3zE%lSYVv}t+dGn2B+=ekzD8WrS8=IX zlLng=k#M`rbnaYz7NEtAD!jeU?fB|Wt3B?jbI5ZQa7=WE;)s8;Y5s{gagQNou?FIY zYhdaYZsgU*i-*SkiP(Nvdeq*&Mg?pw`W0(=wc)QuSE_uMFLUr(4p91@T1azD-S@<*PFGTN$VQ(K5h-!*65VSGmaLuI;UChQK&vzBEA_3IiePoTBrv!xCS84Sw;`dz_ziE7-S zX>I_}PO9m9762ZcuXAUvQ8wdi=AQL8clw*D{zL8fKp!O+o#EskZkuio{jb3~ZhK8e z)r@AZ@GX*Cji;P}RTIOW18G+7a-&ki3I4B%M!;Ox zUIXRqbhrmu5@bYu|F;=DEZPrAVct@2U{WkR)6ZK#hr)z*|i16KH@I}!^_)kA-+m}EC82@Cg>AM@!ct#`lQIi;}QD-GR`OuW4 zD#y)mr5lGCqo5JIeM;Nw_lMtxgE(ua)79b&6PT{>o)#mXMO#64S(TV4TYn0^69|Cy zOOUT&9)Gp*cz&kY={EJxg*`wsB`=Oo_%^Y-PxF(y`qV-u6YYU}s#;#0@P~9MCk^tH zb0h>3tfL! zc^cxKIzvDQYcGO?j%eGuDsOxS46}Mz>c*?05*RrN}(itx3RTBgcnGj?m7h+lL7PCux~^KZ-F-)@6iiFz-sw}|c5%6l8^X1{Ra zVpVxA@m^5~%BVOX0W=W{d2Ga^rM6Jm6$7ZX6YYmU7eu+%B3y=(?nC&G_4NvbbGI^~ z=^nbbw`{svi&5GrL-nP|S!TJO+0C}bE5l0Rpf6=aofbs4Ka8jwuRG;`w*=v}4Vz9k zKBmamV5`?)4treZ>MY3Km=P?lG+MfxMVW@JXsZzh9fm^l%w=JJvMcdCaS znVH}Yule$HZPt40P1F>gV~j`q&B91&cDPbWe=B2z?{wh0>C}sLs8W5|8dGuPc}HDn zWn?xmcor9!m!7E;_HF&?(GLZxKfKflUp16!4QI@{{9E3w!k^`+S@rOlI`kve0dKl~ zR?@cud<7E|dhwUE>Qco0(pVMXNvWB#<^FzdtoV9L(~{-}#F#ww1wvxJ0-2nA_E(Rj z*>v5G*4K))rJ9{#z6Jb2p2d~ASbtePQkl{7&;D0!;eVD2U4!yKf{%Y$Hb`VsqW+cI z7&|S;@ISRv{xyncTK}J4u<`-q{ckM5|MTVle|(>vEo7;8Lhp%O;vCoE(R1jgp5Wz=*asn{8SfBxi*B&@ z94Xyn;IRXu2G+53-biSHR*CR?abEdOyXCZzjtmjE!=l&BWpS#N?v1IBeD{6_hpRmk(ym*pJ;mT>Qcph5Gype1y(5^td-qZIt{8|4DGcEe`?b)Q}MYW zCg6Nyu^2#-M%WP>&qpWN*SEugPxLHnupciI0`@}HQA$0qLL6WB116l>Q9o1}(AvB& z<8FyJ2zTR~8dA?efQEX9=#<$?-TvU(|KkwRd_xEpF!lT`8V6EdCr8 zPay8osxNR)ZVjvlNLJ9N7)K+C3EQdH`b5~D>k|#I-eE_=F@5ue#MXCQ8&|<$(tQaq zVGaFDgx(fGHjX!Kx1e0`ECN`U^j!`t_jli|ajSP^GJllFum@|1}lLWFK=Y;#yW2~ z22;MlT=@-62>FI|4cgsAW#h0;4vv7Y_DI@xDEGsv`v1#eoHT97MdBWn*eeYYOobL(0c$8&qdKE_AwI z+|r~l=cD>Go}N~o>FUtY!;QB@i&;{%K|mvuGPynY)rK&a2X^nsi7L{pCC+347463! z!1s8XoyN&@uU_7%+L!#ZEp9w??{*n?x+t6BSs@}zV4U+YXi7fK?%BWl*lk07)K@9^#;->GL8VJ}5Xw}PTd zr&H%NScGJB2sl0GEw6f!zgDR6c^EAFG~bmbUxZ;8JDLL+x)Xs z9Y7T@WAh!lOr0X{vnpF-K9#|sMqQ~tS+16T`TLsCjAQ_m6&LA5F8!7;vgEf#6wj03 zRq6Mor^WFWaqg2DQZWQOQsc)q#Hore$|2{vZf0OcL8Z&EmMXW$#+@4*z07!T&F46_ z-s!ab4qKq@X7|7C(j!uUKCpy+-3lZ#iC_3G&mDnIv*f%2)g5 zS4fNXlJ-xe;ZJe|!Uw2Q{AAK1-8nT?&-J?s=XGk~^Dg1AP41s(^46}%We=xskC^7! zha8;Tjv0`KNC((FQA^PWBRhT~a@`zNrQJBrXk&iZ##rF~WM$t@Jqo+&Bnv#}`tv5U z{c)7bWP7@~g?(D@WcJ`;p*BY=;wAqV?b>I9%W8fknJhF`Zz@-O#?Ua12KoWHbLD`*CC6>xn^4watcFAiX>TIWrW7qR!%OlB6lw4)=c^A_ zY&U*b1St!Hh_y}0y%g5+FI=v`PpAg3oBPm2-yT}dkoT>U6bg!=1*^)@u`?>ATg@Ex zGl?2cbt)u0bel>f8%j8BZ^LW0ujw?Ryl0drpT!yz;|)^#LbiI*hU6`2PK=&*jYz9n zSABbvunm8#X(SIG9}JrX=i-9(kyK~%7jp2hX$?)wEzYzZ0xpwk=5aQLwxb$sX%28k z24(4+1}2)jLMSV}Tm~$Kq_4@;)2+0WixfL;t;P)YTiqIy?P_N-&qjW}Y|Q;oDoOeVO%IYf6`Pk9CzGT+Q#)#fpyK1-UZ; zm#DvP7Gy;1V;+-dDqQAV6th#U+83R5q3n!ThFvj-A2puiuqq+n1q+~9Zis|5&c(;K z0(MK|@)G=Dhi2^U7Q(7#6LZ4ngIszWkk=}d1ZGd{C43DvS4_4YE>L5MqEz8;b;K>& zDKQyEp;%xSOc$25Wu_e2Dq4r=E0kxVva<|TVgnVcuC4~Rlf&0gNEeG1fwlX zs<+GNV|K?J)PiKnnYoP1#Ze zlEQ&aREz0K!N~Rq4=1sOcorI$NBi(mh3h}9iqQca zr5Rte8SrOI-VDhm|EXdkNw;!}u;p8UpxFc(wrr?c&){FiI&sP1o?dhHN`eRhv zBVegDmMt-K3FJNbf87JkSBdf>@u^gbGnkOxDRS{I10t;x8T{o4kx3=;ZAwOSnGg@1*+pm{p(M%yfqEYdyl zZMSgaZqnkxpqZ!s6+j&8tY~xeQFR}(agXH{Ek!;>ffQZ?SgLgqn&yvIE>0M>F+|1_hS*E$WH)FP z8=;k2O29WHkRgaUWOVhQiYT_+8Z01IYW!uP*nYa$=_P^F>CTFKjx`-hkrq+1%nIbC zRyS&q)U-M7RcZi1mX8k3?qm{Xwgz1Gto93Um(U-KZFT`ovealvfGc7mb{?w$#uiPucNmszEokUCd9HtHBy1Bgn2GFl-E z*&a%=FzQjNGumsKY48HTU*mC+Gdc}wD0dYiR>Pu6DfFHAd!w|?~Gl3Wm)}5RWY>1U#N4X}R*|WWR!F1~iDSjk`p|-3V;BU9_ zCbhE1hD%L#Bbk&EjUI`J(l(632?bo(Q>~D5T!_zQP_>6GH>JIj-Y)E13V z$kw}h*=>0(>8VMa_V5#-b>+f`zaOdTGb z@oopGz1{%jd1XNTNu{|wGiR;WV1BzKVk4mH3j~gBFr?Vi@u$sHsc~8`Abb?jnul$B zPmI=KCEeeQrRyc(`7^NQ=eP%<41jnei)%;sl%HH zMO-RbR z6u?^WE$PP7oe}+$sLT_qW-z+?BhOe5L3?5=^NX3t+?>dk#tc5rhE`#`C5Voi!d5DM zjne2!sF`o;_oy|Etwc3duB1zTk>vjQmNheXa-r|%V08!jdl+Zd?>*hTi=$!L4;;&p zbe#oSKEA|T^X%pq#=@s6S1WI95B0|B?tdD{3nc{*J?{|(;aSU$g3LhL&O?^*>hm7o zHQX3#N(}T$Z+EwZ34Xl@8S~dYdvaVnUxhWFeo5ASK85~CTX$5o>>@{3oZ#gW_tQWA z-((*eLbM!eY4-Z!C8770g%^w~B+WytqyvpH#Wg0-T_!OhZ@aAe(yE*7`&Al>mT~Qx zbQY7RYL+H&n6g+yOfYtk*snMw^x;yD!&gc-oB0wg>pVyJs*~`Td@+@l2et7JhT*D4 z#(M(M3sed=!a+O*)Wj1Raw!R`rZK#fqqR6QB6z1Bw@bg_#sSPgtZ?N< zkir{;r8k1^gvkC7xNnC+Z6*)c%xM)V!)I^?J9am-8KYUn=kf2H^AcBEIgIFVBwi*> z(sOzuN0b!n*$aMr?pgXrh?0HYt|3!`ezp*+qa=Ph0g0vxMhyz7u~LpryQie1P91=F z>ag~HKy$y5CzQnpp0KQMK0R$0Uaij9FXRRfhZ@-N+ zbByo1g86J5JihGgZ!DZ~jaxV?ZT#=8BXe#Zq{)Y+(-9d-3fy;5t*7^d&E);tZC9e5YT2LT&5>Q9eH6f-~(DGxhS z;)_GMQP$~7?;PN-vCRy+2EYJ|{IVC|H6(eTFaA$Ld-ENPx_Z#!tqTq>>uQ!*ti#vmQ^jBI0JS+4$?Y0KKcm8h@YC4aWbt_LxbQ z3u+aD&h} zI80|=OY4{*E;U<(M61c1hJ=@N^`_$W#L!p9DH{p-wq++<<(FcelfVzMlLGDlNi62a&9;tdQ{-;ZT@GWhEI>#ON z$;p*xnDmM+*NL2IXOx18{^U!re1}d=dhLpEZCw2V;jdatah4sKetSENgX?#8BbR-* z9oM~FP&>F+Z`I|h&uGac`*T)KGVZNKHYYpRJCJ5pVRGw&Nhah+%^q^DW;K}(5)!-! z%COWt6Rh*>LB<+_>Rq*T>y6~uw_GePuPjcJ>xz^%nd)T~haCJXvt2Ke#Xqzx>W&?~ z&M9r{?l)awe;UV1s5y6AKKeP#Zh77IvnkESW#(19^6htWx4q01zt6q>c=C;V??n^A z!Wt1XPCt3wOK{k{sJ+}C0Y zfoNeGr0?xvykY#T>5VKA`IeChqyxCC0E0&n;36Ef>;P>IMM2Q5ax?$d5&=FlW z30x4Px-u`lhVQ$z5v~%`wz>mCrT6)y%{`AFH{Z{z_l<;L_jc@% zZ|9&?qzr-TQQ*DOETub%Cuy0|3M5jR=)S#6Jz^lb%B4D&VL?}^*LwQizCuER2zo3+ z((t4|yGCi1*5JFja4@3}lE%vLX}m4{O@!@!FCybs{l>`E>hY6$`e`XT;kRLT<^J~m zy|(hzgdN{+))@Ocx)L;wOv!vEuV{@FJh{wgr!Nl2Q|6}!)=s{>fBHjS%c9reL*c!7 zE#?}ja=4B(cG|wvP-FAfKX$;M&tN;R1fpo9Q~yJdryv1W1|qWv`EJdZGU3Ym4#e<<#MvP1KiyO)9Z>`Nwg{E+YbtYsIQl6B?-d~V0}pe26a6RN6ACtmD#fWyvRnGco$8l`)W{7UT;iW z5661#=xl398ChV~94(#^a(A;}2$XA{$N1g>1q z`dxC(4|)#$1H1n;@w|o*_=4)hTde6CCw68?Fm&FB z3%YwWLTbI0kpc6zatgBzsKNPgZ`E?aL8@CNL0 z6`?J0-n0!rS1Ikt2{jq=L_V-U`udWW&D4%5Vng5wz*36gdMwr~^^K$)PZyo@+Db=L2Xdob)P#f?39*{-ha& z{0X8q{}iY8J$~puj8&(zeF$^v9HGkFkvNuXR(t)#{W;v*tPjarL^g89m6%~p1^NuRqu;zLd5YrS+xq|7nqm-rKO=C`X9osnEa zupkL#O~;B((Wg~6wsS^-$!l6ObV!os$;(dhr^C?hl&wZ zsUv8wF>{}rOLP9Qx@C!&@%2z0N16?1-^mgaI7srtR-9}yFjhGXZ$3LlSmfShWNkeo zF7-S_(ZBPgH2^!Kg@&4_R9;C^v(Lg7!ifEUI@=MKp+B`XquIa6E;oG$_G)3IHalps zToO{9%nFX<-Nyih6p`EuvEhHfx$kHZzFa0k1H91uP<9_jaGcOJ;)KTj#@Z2$?;7LU zGJf;Z&$X8=J5sSf8QE`>maoa9cfvQozHHetsi?1! zXHU^k=?e871()pT!sT!MfMC!MMYUi)_M5t9UpjIN#R#LwiYK&XFK}=;;zmMD88OHs znZH*itOpPD6Z2jfvp4ma2e37e)BMLct7n*ROu%zg`k!bm$1C^T;tEB3lR-dj>F{OU z6Q#^ZVUA8--$wIrDeP?4}h-@H^GoD1h58X0+r4O z;*o_Rl8x-o_NhXFjCq&B7mKBlF1T0r0wyoQ`;vsO!ukL1WFVWA^7ugjj0plT0h5=Qq=p7y_ z1FwrPxX)GjgQR4;nRwfH0tQAhL-zhz{2_xb zRg|ucpUK0>{%3U^2~!UA(R)dHhIYI9#5Sk@t+rm3A!VS&Z58@^*_U{2u?6+sADDPC z8R^4@PFJ>l;EEq{sJqF{q$iOU{AtuXr77&d(mQl-mKqh;j`S&@A$TPJBXvlel2>1o zFpXeQKH|CHg>d~dnWfBmLP z1$utl89k>TvIIH2k1x0WEc#sV-UsdoAE$&LwHHgJEi~dJJrW(>q^=OS8>6xL{tnu5 zP^6mam*@a`gjSU4O2mFNORma`_Tr4JmOpUHCFo2GB3K$cn1tl&bW`<||B+g}I2Pf? z;3(&Gg-!N_&#gKp%3^d32Be85YRi0}(H6LlJrRw8#t)la9TPQQ%Tg2~r+Di+tnb-R z(G!Sk29EaSCGsGh{>dsyZSTdmD5B%C*bjVF_m-K9v+i`s8S0<|6&C=zqCdt`W4;K! z0QYcIFYMjhHl1R7&mH#(1^|kUCVmvbgs*E$p|E*2@G zAP0!mDh98?yN~tfsGLHuV20>8n!AQQymj6 z`J(1_7aUb3y1{tR^C$d8d4wF5x6m4H_can*X+6)DDAUJlgl~JgztOWVPp92=LX7R2 zDA5nQK$pa6%%@&Y^O;1ikq{wIKYd~q8@WRLl}q_L=`c1rXz|{Zd=0+p)Za+-v^2ex zVM?v8)@?7XrF=DdDY0n~P}CEio$Od#qiC~PgdIJyTsk$0PU7ud`lReH)6II6zlLmR z*4v@BsaS3P&0OIiR|~Y&TRdP4q6F z-@jR(S*+QPiNF{gO)l9P;H%gr1ZGG>!&tCbn4V)|+em;)#?4K0cJYuZ+GfX{f*ybL zEpn<&ZySdQ@)XNy@NaDQmn|Ot0;A3r#kYHGW)V$vDaenRgcH0#Q{Yu{rCPR%DPq(1 zC?%DW=WK7LMKWp&s}Zz1<1N ziiVcE=vQa$KR+xouHEJZHSJ><_-*IsW|NuJJnE%mn5{U5sC zIxene*#dRh7^Eu-XOT>Gk+6i|+J_;R~N2`DdOuI{%FKTF3xSd8p=|ih?P&vmQ8X+Qwm$%<)j~KBBQUah zNGyC10-J(twl!{o6TSV?eZE8TWlpzB=Q(yw4U~SYD^yq~_*`}PTrRmiovXbjT<75k zztu~Y&v6X~z2rli*F>p5j=sUKl~X2w*c^$IMJ{qUnQ#pM$%x_KDo0;x)_G>{Hh(_S zI62(`*BpkJ_BzV5n134tL>R8kp@mz>C7m6C@NEFbGBI9Hb!7Ce?O_0+v;osYJl0dO z&oCnG-ZWyLiyg3igxj9dPYa~<^ya6lRPwro9W)q--(1T;H(>JO)o!5JBl%Cf%=&h%1 z6xV>Nh-GynR$BUE`3tu-bT@fpMlXDU?otcDLR<}8_1V+n1TZioeWySA0pU@BW=;9m zS22*c(;L@}L%DK;8z*@a1ct}w9GZUXOZ`*67`=r-}xqk61D1=B7cydw#f3j?&X!kcvIW}6+FtX54;cR`j? zPDU#v*i*3{?hoJLMwm~&rEAR;DV6eZKHTK!8l9XkWqBS#RGg#WftH3c?d;;kV|d0t z1RSZ7#$4M}F!PV`_FElB$gQVa9zVVSQ??3Ij$43J|MvQn@m0o=?Wz?f6~UZtONF9% zzw zSX<-WPijg6DBp}x!2{rg=b=B4>hE=2CEO*zTFwFrdt`o0>{HSDD3YB#Vv9vM_LQ&$ z^!dXEKaT*P6+T|PW;-I2{=P|@mE-*X0lNDBfK7qHj9Zzd(Kw#^# z@znGnck8b47v@};{-Eg?^!r+-ZB`gM8mQ%}xd1V{iRpdE&n(IBVr?*Nk9=J-#i&_qrPH$Dz5&Ke1;sE>lo10$5!exmy!tKM zc-rW`zErH%KGxFUUvJ!qu#YzTmz%hrcs_o#+1D1N`SLkjm>j265%iPu!1b`7Gk0_0 zehv+KXG}7jWc72**C-;y%%bjK0C7%tSXG^i;^&wtm~Kx?#sGEE=}jM8S-?!0R$S}k zli7HAP<~RqzKws!d!M8$gYtfs7l%U z^eK#TQ2=kU+VodqqGfM5gd9e-!f)7VrbOWyI{QUedX4+5JuYP5?el=x=S*b8c3zM znQDm`Pyx)AfD)DaHO<7Zb{N2><78<~n=*X& zAr-LbPuS<2RACuEW3T&9Ri~%y%Ta*wku?gpO-b)f-1tC<&R4+~_ZE9M$C+>Nyja7? z*pGcEf#UNRj_AK!obN;5l3Z7#Ne=bjar4o+%_}#7ug=)ZbkARCn_z^2J!s^S9!z8tf zQvuCKRtSzq_6nsyivTZw2!E90ww<{!&*@d8BZ=#fCx9pJYqJdU1O-39mm;QDA2wM1 z!KbXk!W4Jk{GnW`6S6c4?hPShLeyK{BZ{ZnZ6(c6Ntv{ciQr#O7E%J*UEp z5V}(|2SW7M-r&vI z_SI;DmjD#p)78E{2uv+GLUCMHgHw1BZ}BEeC5jwZ)4IblA?_|Z3_Z`?AT`Vr&t3DE z3|D%mUxn#zKMQIz9F){F;61K)i=c*-6z@_4F8C7_d43Txj(5O|GgJTJgS;ss!*BoX zfNh=2JiAXrAMLWXztIz`dymCljx<*~?m$}HGo7s%4;y&*rn`}#k>%u)pEKSI%s2i6bD2T5R+GFK@2JB!-q2>fA!!=MayJIP{o+Cj zVRRqZR2RTTsG=0{wW)+~dh>I%)V;;TXgMqU0?1iDTZZ$7n4dQpMG-zwU=K9(hAc~r z!2_{69d7~a3Bi%`14B9*URjPuJ}CXL$+4pzxt6Wnw*5EDuC@aau;-M0tDQjO<(TPe z5bb}r!?2*L(rNOtU0M(PDKQ=%`lVEfG4pJYSsRM4ie$I8W&MV$Q##am@+Ez}RybG5 z_(7g>^H_G2MBVB0O}uFZv}=3f>ZnSdoV%E+mZg8l)bbkgw1Fb|hm_b{4g14VnKP7qiBNCT(#o{0h zU}oVn8~bs-C8qHYXlkNb8QQ@|^JZ$AAX4Z92}4H#_MwhVOmkd4gWRUs`#|%6$s%h( zt0``SezFU{QI0F*{ut1`rxL;m;X>NpNqX^NV+146K#U__v!`s-GT-*+aY-#jqo8*O zE$SAqwArC!tK7rx8%T0#2$OrcvjdqF;?wgmNjAi6`^ftJ6~j#7wd%FG*ADt}+nJ#r z{*%x!2H*ye204t)jfmnmh5){NEZ6_*jR;bqy7cnxhP&KE=)6}D&tlKjosbwt`G8m! zfi}9x+b`Si%sB;KtytvS!ka)UaQ6u8%TWAdQ6o4ttnGQv!jK-JGw~)0fdME8prMij z8&Xw61m?BaPXl-0o}OhuS$pYFm#D2#_gNBf5JFR%whPj0etThgW6cdIl@;)4XivYr z29J6lG0~E1-X7dLhCnO{`5!UYdf=*y_)Fv(e_e{4qIdDvOp(92mO-BrAH%itI&B1+ z4TOXAc3%~Yj_T`hUcMlPZ(83~&TP1&Fnzp#d*}goP9HLO z+b%k-qWD;Z%|iR{cAJ@#&kfx&R@{Dueh;C#7!YcB;)@$YARND6<==M)(4;gLiT=(M zu~;_#gn9I%e@#Y1z?Lv(j??b6E48qYy^|T1dqdFC957<51uNIOeQ%-*pzKGduJUwq zr7pKK;;`G_Q-WE1#CH@wZ*JaFGgHo9hn!IR+JD>r{H*fG>fkg%z$9=@%ccRijB8IHR)QVg`?a_k*Dz zks?Yr8S{Cf^L~k$ZQtG#lM+b%4S{j2C4?to($}+*=gJt4vX?pE#&DbH2nJKRs;m}_Pg2GMw0;nqYi}_` zWwIKYEk<}GiVqW2XZB3E4^@jUqS6$NS&arg69?z}M-F|{qSO}ljw#oFtyBrXdm+yL z7S}LhK1m%xMk*RbOsZQn?My@pjr}vvziYP`)x))48E!SIamq8r)A?~q{ohh4#!9UI zx-vuGm2`eaZK1ZGpDUBqRNXdj9J?B3D}+an)n2CS&DFQpVGpy;h4G(c@Foe3Q0fgo zpwTCksEO=&boBEuB>2Cvn^tTvPEQ2azpyN_`J;a~RlB%m2iXmpLqOk&0Jlsn9hTvi zK&goKAVmttIRnmLLto#f(XVi(=eHR{$xX*Fd7H=gd|Qh$ajXpmhD+Q5bPEsB#}}GI zfwY_8cSb<6F-ZokD?e7j-9t3|qv;PjCaq4A!hnonlOh{9Y&<_pq+AE;H4h@VMtYSI z440V!9@mWIPp?}&IJ|z};t701^}!jJ;s}$E13JNFXY`rq+ zoS#4=#14usfqSb*Okxb~6m~zXe9e!TE#srRy&kY5d*J+U+MMOEBaZs)n0M}>+hv2O zN$)JC>ASajf-I(soVSu09H+lA?WVv@9JVx=P;Z3oG}-rNh9x+NyO>jw@Y~Z5uT4#2b>V;VGaAn8xd)AhdoEFZ7n!$zTR~p>fKYWru7w~&k)k%#tr;FCq;L| zg<%}Nf5$3kO6qKur8bP@hJ8UIn%fs(OuA;W*_|C1c=F|tfGfTc<`#OBz>c8HKKe8Q zxb^IWwDEH#AVb8MPK3b|?HdcQsPw3wJb1I@ii|Cds5Rmh*Pv3(HD6xe0k4hbRe;l} zV$NndZ~9eG;t#F6ZVlk5*YX>)rt^ zaI`9WbD3PLiNyPxT-7eJIq9#69agg95>OM?N3Bt+(U`&NvuB58et1prX!VwZ$vKkW z#nO2)S29%)4ip%ws%#e%epgvw3}N0J!b)&Tl*>ha2nr|AJD0>dqiR`jOsEej2T}(J zfeUZ!dAMrJir^#7HI{y5n@eaR7n|(|&qMWzv=+ZMz>HZ$?yLf?Lc<)ww;t0=Pyj%1aL9&tA0NZ1_T z@n$b3#aoPMx*j(5h*X>A760dDvSB`{rmcb?Sa2=zN?7M8;MlU}LN_>|3!N-@R z4uE)`zBGff)Hd)5eQD#kD)xyKBNC8OORL({%M6M{H6KLdG@qW-_tOm@XkL^Bu-ZRV zsxGvSd_1@IvHf{%X~PJh$iEmprzfO@zq zExK!>16rqW)c&HfN0*i_smYnQqdDyj_yC5CEDkoCDD!z208t`{uw64w`D8t@jr} zC40`grLnP18TDFOYLsMEd&(vShBTk!3Xv=ttRyS!W~W!4&X2z;aVjhyuwCqq;m>fg z%mt)bMpi_m-ohhTCT>Y(DdpKFMjTr%#5+S{EU`}TXI^tJt^6Pnuh0t{*89RXT`YQC zgMFM*lZswiq4{bLTL2JgVCO%%L)x0iO;}~?JszqB%)oT+A5+q{99+_DU@7VBlx$_q zFnD~}I}(7*7UL0LtflZ>s(!ULU*ZAyFe_n@K*}8Z$^7VI%NgX1b}c#c@Vr{E-yK^N zeXD#S1Bg}q%w%ga74>wxs!)2f-5Yp1QQ5%omH@IR`0_!$0(>rcz~EQOAuDz1?4uclW#gsL_6}#{ z{z*ngAbCJkBR8$xSB3H^|D}}pt&WgLUhP(M<^qaR4LqBB6{c(_P$_&xLhD3rDV1cu zXXFTKqy5U)|H0Sr`;k01_bpA0Um@z2NUPd~<6kgh=nNk{8cdJHOWdZQ*}kvw#(6xP zNyTp8_2*ZtJb)|^7cs3=*KKYr5-Oa`k1bGKIPJ}+m~w{p9r;A&;}ga1Zj^zT2n;sU zmUA&Tf`^8kV$&^*iO8g!vCySIT`K7H4uHj58##XhpmQ|u30ZK|n$CP$dAr@sd0Y7d z|7ZqA;N*v9+*CuWQgfH|)8GFB*2Dh+mYNX=A9ss&mWfYlc2Fg#j{@J(b?3>o?cCz! z8j*7^gXIZ}i=gEk&BAGmTCyRt9V_|axglxF%52e#CKZsGWZN#!_;xmm*gTP8apup0 zEe)%s%qJ7}$9vOikWppd4a3nR8fMbnStL}>XucQ{~{P@!&HTXfm034s{jhbgn=LV*9QBe81 zpRuaM`C(ttZVKMJt97}aYRqvy%Nt#ppJO$|luH|O*T$bVZ>fl+2l+f$L!elvxrsC1 za01^V^s~0@M0Q4p`#klYc&hOvbdCx<_V+K(kU4VeT+XC>3VOcs3BYU7m8hocxZb~+ zmkbj0-qO#U#UV<8)w(z2$pRX|93d;c6=!MTw-p=*3%H~>7Q^?^4snTVUnAldaTbvtpB~9G z3?p;*mzf^>M7?!(ZgBWP4O#9zntDGyj#0;G7fi`aAnz91RZlftKM>nk&9gqj*IU6M;(G`cCPB2%FD}hzp)KB7#1RtTHjxD=*eh8$d5tL^-iSF^fW6Q^;eMnsZdnGPJ01X z1Ky}mMx*LW&B9a{`>!vsPY~khzB(&_;V8w)m3!nracKq8_|}xXTbX62EO)lsoZ%TaDQix)3}ibnadc2qxSF#oW3K+w z^E-e`o)NRBy3Z9(-)*H<9nEj{s4A}&9Plr#W@N7xz%JD5fE5yjKjwP)lhq=NmgF^; zwM5cauf0jCh_DlbsFkipW~Ox!_Z9R)j{Ve;fx*gxn!CF9=n8EqXXB)`Zr&QYZI|gN z%I9S>iA(y>Uz*5C7qjhM-_kgyLiB6HRy1qRn6B~fS5nc;4z#&kL|(JqfHz9@DOkFn+_)86(H94O3q z%^{@tWB>cEjIDK zdsVZZm^HijeL$3=c^clg3aO4a6n!TMN*e);ITvWT1S|En>e==^3#BVEu1LyC+pjY{ zz4(gs9mMVqqmFgxql6qA;sfK{Zd=7H!*VwSbKKDGoc0~oY1Q6ZrS}iU%@1Zc=ka|i z*q;nxdQo{MYST=zT9X43T|}VtV}8d7BCz6-4gd*@Y1(}4^W_Hf_p}xJ-*>48Zl+eu zXF$e1ENLCLx}xJa z&x~2%fBTQuGPsUQVLN4Ou2IDJe`^7p{F06E=z};5 zFUP&>iuQj1o5m#t&k#s`ygAke<=QI%SjPmg5iPmzuph4pznt3)J%gBcM?(zLc-#ZG zuLu>33Y2}=k0Vw87*#qHRG;3e>JU7W&7i?wd2Y&?ROoc(6P0FZEem-Qu_asgkp8@T zZa1IGL4wzN;g6NKU(cj^5;7!ztY12N@}et5r~VX8x$0KCDY3MxXY@ssLBCj@mb1pY z>~88NP^!%yHd3nvDb>EakIYY`I48Q?gBU|Yl9hSvjeZOrX=0~X;%1#J^I!BDC; z5-T}h&>fNXSxVH{&d#Ivr|wnr(%#yggrww^J${no_3#q>H%HeTQ#WW^<6)duYwlrt zrRHWQ-YON^nI`QX;}zy`719KU7tMJqL#dH_v7|?;s{%`QyCH=A;lQy9g9RVD-iN0a zc8QuRn8~>53%~D;x}r|W`0N|Z!g2iN!ync&wk+j-!MWVg0k6ud5 zX;gungpAQ4KWR-(f8ny;)UO;jMcXGQ$o2A{6rUS=^3}+p5&kKFo#U%o7Z^KD)#D1L zhQp`_Peiu4u#N3bTsv*9TL9a3KL}rG8z60Pe|GX+>jz`}ED*nFrJOhc{a2#g-GgAW z3xZl{W1;K{Sn226wb50E9c~Yq%m;M}^Xnwfdv*2IW|}uR31MD(+o{Lx_qUDC0hwVE zr5gU!2q#(UyB2N+ln=`|^mAxTZe3LZS29+A7XIZ#XX=fB0GZkL$m-alEaaSwAeP=L z`wbhL#owa@D;Ge=1k}q3VPNYM{3;RAuQ2Jr4A@rc@rs6^E#qVKpWg=u5(XOVd>H%k zj}IM1d05p3R8peI6Dp(Vj2|xZcRVkwXbxab+%qfK*kdVIo~QgRAdg6yT|=qMqVM$t zICkndb$No|N9s^y{Ub<~q2@6w37dZNfbkgp>qf7Du?A<8E!)xs0A(UtrNO~9!52`6 z^O+(er`D#P3Fv|bZ+HuZ$^!1$v@*}fel@NkGCD=DFDHDbaZKswq)!w1J{fXP=|k-s~D0)p6-qHFra>PmG!wfU%%R;XRT%-7jX|P9&##S zzy@1y<#!c);T1h}Z%qa*g*J|(Meon@)$Rzf6?D!^r{cMs9YCW^w`6NO?qD9TCEL%d zuFP9YOJ`s10)ybZ8z#( ze=RR0<>8@9@NS&OO`;)QpmbaZZ~oD9Mp97T#bPwpp57AiH`aA+5Ie$wtH~GgTLUgU z(+SoD3gwL5_TQE6$G+oWVm@qj&ys-kxGmKJY}nDt=VPXQILDw!3Tuc;-*shB>8mhx zoD_7g-zK7$?$K)F%Fr7I+lHT)PX>h-kSPGISBYu4R!Yd7HA-+;3xSEmM+qNDeef&b z!&DDPahBhvbiSI`$Jvs~nK?`Im@=PzIw4IxAs6S7oZhmHtr$|}jp(F3jhSCL_-E$x z>E!6GYfnENKS^|?brPju$6Eww8*{Yg>Lu5vX69!qPtm|}$T_%3nKk??Y&Zr#tnX-Eq6hbyge zr4IEHah92{^X|sl;W`E!W2xC%9}=0KaSsL#1>Uoj>ba#CI16uNP-oE6x=mY+WhE^+ zD&tN?h%tL*s;~L(o;0Mu2dWaN!`y>+Nn}kg)O(6|3v;rvJP;Mizjj8d>k1FXo?`(| z&*?EuU(7i$^X0}+-6YaadD4(ATrCS8l z5b=EYrsKc|$w`A53XF8$!l)Nt9pM^S1X~%rLTip7iOmqxJuzK9h?o@D@_&e|S#3m!6G^aw9`paZVY>kUiks#{vW`14n>k5Y!a zKPe9?UERx8@C!T}BHAT#q@+22$bsZ^{*ZHxLF1EJql|_&Wr`}87Wzvq%!}{i-P`w9 zUc{-O{jl@x9^qPU`e#UW^b6VS3hKSly1!==E7&$m@|GCx)&lo|mouqP9LzE;S+R&|^xD#cJ|6!R~h@-Bj7W z{wYV0xN(t4I*8fgfLF^f)Tg2QojD2ro*tE?}|Zfl)Ilv z!fHtEE*=WB{9;YuY9pm_EPdxuZ1hK;`r?fI=9Fi!o~^nmT{s$0P<;{H(FEV8n|bji z^EBPgRXpK$q0i6NZMCLX1%~V&m+T27h}shX#g8B)g7(^EAl88aTm*6fd%@CN8XNox ze1P-5WGywh%GtDJGtuCFI{!RiFO2B;v)D&S-Em;lmV2Q^aGJ4R&$_H#ZqRhOez?U9 z{rb*MP)dx*&+pi#n8cJ%>-lUvLN24w7Udm%(=E#X9YTLgM0Dg~spP43aVO0Fab*z| zF|sa>AsIRk_`LiQqNn~RsAVIyLm}A{i7_>Iw!bG`WR}O|6WMyA{oQXWx&`%(3yw&L zLyl*i=%yOg{eRLrBFk@8B-T31C1n=-!2tdB#^*sZ+NNN1gYFd2y*I?|p`kLYaK^Sb zGAvT-11_aL@Soi`Ug~6Ie>ztL@@$-`H9qp%r2U9tt@D{nyMcfgrt6T<(Vbfv%OHi`@r z@;6>5Nb5FJ=Bs|i`x8~*)$Er@)~)8;Ab2=NW71MQ{ohmk|HYRdPz8_T;!MVP{wAD+ zxbMdO_VSD&cYm#dts>0N@(Vh_s+xb0+N$#5$&xwC&YNKNBk$IKPR#bGN+=z+^lQR4 zCYQ_K$~2myR*(%PN8b0AxT)oi%!jr;iA%i5o3OvwhIOT&@C7-I_)0vaHWOS>D)i$D>%?cP|Y{^5j)bdx=jhL=U z&0IVaWn_xg>Zmqn8r?OAE3f}TPl$HH z>(;*mW5=_#f%C+2{~D>2s5#iJHuqHWMPZ;^Y>|8eObPqj>P2ip+jnxJuo|+L`nUJL zzw{qFbrZ}?aFhMD*MIf;hFr{>a{m7y`QLAe*Zgoq|Nr~!&}WpFdG>!^`A>V}#Q*%z z|M|QSHW@$N|F-A9TZsXw{$r8<(|r!ufexyzgL1@?3=qG~741iwE>e5XjEPW^qv4%7 z3OQeK-=BM3_eoE&oHOhrK(8Q>&>#vj_VpR~pf4V?TB_0>&5Y3@rG5LPmmMPj`-l{w zD=|?)8}M!?oZ?_`a{AC9YgFd)et)E|2@axM`*H3B3qy+PghlVxYw$v|-43B{+qg~X zkcumBVb@lM|CFc1mXn~)R(z#JwRrC|DmLR~ouk~vB!!%1hwE_jK47Fr`dJ2AOj}?} zkBqH0?@ zYG%Pi`b=eU-tkq)fZf^}t<`{zG{(#okL${-vA^RPPuz>@qDnsw8Tt4ggka}1mamV=cWm6M;?Z94rE23WMgqblGKnN> z-(i+HPt-^7$W=c1XRc=U9*BlgVHULq(j_c8%ux`(;edmb9zFhmZ)K9QBun$()>p)p z1ZmCtZ?pev)`X=|3jYVH8tM@S>WFJl<_icaC_#(KH!x&AIsFf^KUj~SR@wu zT^7GBZw_@B6mCcR-CAAsTjborZoPT^DMWvz$0#*3x`Li4uXW+K=)86A30s+XA1XfQ zWnc%=PTi|7f699yfk*(lNL~Q#Ok#b$$;FcX#l#!(3*{>V_54qbf|Hp;iS8Dc_X(aH zPmEBZ_`UH^Tt;s~;mEbb-NFvEyYj-%CvVAzxYw8>-15jP-ev(q?lx$+`d5gYsIQGs za9F(pJ1Aj^3e}gDgfF$p@#T zzvFlwaOHc;>ZdUxn?amnKaJVnY$G!1OOa8@_EJ>Q*K2cSyZkm*SL;n}$)&z*c1W+5 zuiVx0)=#yh?eR(L6D9T>d>Y~aqnOTD+o88&h<(T`+0-q^kbPW}Yi#)1M3?0@n=^*T zE*bT4P@;6xD0t36p?TE7OJS^OiT8Lu zzcsx*-W`v6PCGjpW;aaTIM`bn>caMk61?QD@?M~JvF7{oXQwXJ|2sN#n(#$OPq-tq z07q3p=Z*WibKi4)IHo#{W^;*1Z}!-^K`Jq<%wr!DAx+FacF4MXueR^m0G` zq`q+a9gh4nQ5qUoA3gw`B-@MP< zv5?o(ju$HabV3Erz#rKle&lgle9o|NaUT zGV&E#MB6k?`Xw`wou{Q1?ehRH0PwOA<`_&?;@;SmLfH2u5-^PgVbVgil{FcpSDRM% z;DwWa`8_N9{LT#RPZP1Fet9@zwOp{;f)hT6AwpV{)` z`@87f)NDmcnPgvR*k6(8>>dDfkCv)}9p}nkGh+`Ru4!GrvBZ{+)JizJy#8Y#O~uO< zG?2RyaLO+sBIR=FP%hTvEoi0)`gBnL=BE59ngVZ%5yY{+ltrNn;G!Jh_qUXAtu>bA zsW$Iz>ZsUf5OdE~_%TjIt+4Nss0D*{soQo z1>PF(gqXx*nKGz7dt9Q#3sya@E0~Ud!&klt0xcGha9XHz&rKmZE;@fOWd7j(giV7f zZZ~7{vFYx9u6@`a&2~4Y830NIuewrS$a?!$82bTbhC56fa&3Dc93SSM|HOKUy5~fq zX00z&a9-;q)i4N-$>@p{uMH%WWi;;(5X_P?k=d!X4kh-sO19RzLUM|;-#QcCXCJj` z1%Y-aYm7l!rjxMtuKHK?{LXpbYTwbjiC9=s7P_1m_}r-ub&CISF&MkasdFN*h5TG{ zD3kZ;C)xiUy!qAOa<3-Rd zHjmhHVXrZ8u?(;Ml*((orOa;;h2na|9Eqh=?R|*d(69z*i35gKTnZKbk5S)G+xJQ@-6S?kG=aq*DPL3h#M}- zT9(Nb=|)rFeej(T`fiw(tjA3DTxsRm^t_zPwUT3gsYzSkIpP%X;=(qKak96p(H4Mu zx2Vha-3U4~0c_PUA=+rIxMkg~GQ6k}I{uBC(7H0VWgC@k`$)EoV;Lycljac_GIFTF zZ^eXP&e6^?EI{D)PC$+AYSYJ$Ni&Ma_!2vZ-Yyl@A-G7L7SC!ET5 zVhYp8)`u#3!5-&WxTn=ZeV^&-wdD1lEa^oFjG?kGS!OWN4Vv)D_qxkZP34QI>6ew| z{i#5_8C%b^-hyXhC+XlcYR8VrZ+kIDHcJd}4W%%1I|Q2k+6Var8N`%Il%js?F?o~Of?_nAJ;giOmgyh!U`z)@J9ygF zYy77fZGpR^rM63u8tk06JGli-j^3zpm8bYbMT`kjGCp!)tC;h zo6h~lEhL+9g>5+O{Id+`=@vp!F^q%I)g!cic)ztD%RP?sg>`+-iA?^PGf4__ntuIj zWDcGE!-1+ieFum82BEi(Kn(w)cIw$kTU7Q>!4~zK?eqD~=C(&2 zypkPFI5-6z`pI?QIbg?~xz-&-J2VNwqh6F`8nDS=!kjUXPV9H7NjviNSRw(J-G2H5 zY-h#eerZ+bG%pL6i` zmOk6UsYBM3ybi~AfRAR!hEyX&)2CB_Wu+$vtJ8-@r!{T42J5vUb$*pePZOBKkJcvz zEv+Zq*|LT!L>#;W%zfGG?%?0Q{NyXWozBL<=vsa6q0bB;GM<+bDPx}}lgv|D(uV}@ z^b4TeB}-uSw^q%?8<7j37f*q(Ice`nCU4Pyz^13>;xZ$Np+6=poc?L<_ryDq)JHz0 z5yZ|v`H?aU?tP2)#o}^l-^W~i$30#6VNdZ9mQi0zH;Kgy$r+1Okot>GMkFpba$pnG zoowg@KuMmOP@kle?d|5*-&YfgSF-urPt3%PT@iZ=$$Pw$`|5q8Sxjv$Hhel2{98@Y z3QURd!0Pf<$EUD2!uEkqV+P$GH~FT87C>K=Fg)~3pTHkJ0s~|HVO64mCZG0~-(%P; z^lbDf>a@pxNjZ@HlK#MPI+q$Hi(sdSy=-y2o4*#@GpfGDLtT<6XU9V@FH z$&*+0zY!OlMia)Fe!GGXV=^1;*L=ROv=S&*+^Pt6->cwrqos#7uJu|cs8@49Acuz; z9k^k(6duByL9;fR!mRe6%;`Yo?K<0*ZxpGEx~>2R7{g2v1ayQx8_|O9jZd}AD~H$8 z&u%6bEL{}_Egoqjph_e8JLAbVzc+`hzr4Lf&pBynZD#g(tqEcZNeucLJ_v28H${efio|+WXwbe@h^bewRXV zkH}ckb(IeNi0OJ&`}x8wPvXRM?&xbYtd$6RyTJ%LGIeZT0pQ3|Fi1nxkXVmFhY=5$!Fhe`C26 zQ~uOZ5~|`~Hudcfpiz->u4EKab*g!sh-!)XyCsVo;z+8M*0AbL;*K_aHTx?ga5{^Zqh4e8Q=3y6q z)WPk(fDN4#Z-ec#_RuNgTyXtyn_{y?uyo4UksgzDCaKj@w6sxwCZSZ5={p=1#3ab` zp5iUCH=o{o!$rX9w1BjYY3KoW9TriI{TKk z30HgcoDU%3gcFTcXyoJb0xk6Lk+|?JZ@YN02ccl8#UI<`QCK8!5IRqU2esq7%y^7t zrgr|LkJy`w;wm&9;9g1kl6TdeQjG#X2fEinH$mwWd76L`{><_9!RWhfe;XAg9G!#Z zk=illpZ$lAZ+Se1#irBd+;$>~^IknQ0|QEi-SRW>!g8|A=#IQErB*ZCSuaD6sU+-e zS)vsq(P%S9%vQS-@=Gtjk1vpW1=8B;Xh#{i$yb8mkd)C%3t_T#{*92SRy}nF6)OD5 z>1WWzgBNKnFPjAV-OOk2k5>elqa@_XJ~cJq2#qaOTLW$(p;SM*|ET- zck~g8zT>iucN5ILW6h1T{{-Lh-u=P~$wx3fDJZ}|0b4*jdZ)K=mc`}(h2c2RZ|&Ev zn~B}*$*jqXhDZcTF2s(qKat$W`<(ca6c!?1ctuuZVyB4J)x0-@K?ijLJm^_yH{`NN zl>fLO#?thdv?z_()J^B2khnBBoJ9N;PYEOFK2TyWiE0bQ9^q{l0zK&Zl}U9}@SvjQlOz&5LeQP3Ki;Wbo>7p!LHHwH%Bu za1aFPXjpU;a6~deMAb7Hmax4XS^W?YBm=vpwlLuIn~+@PO@$Gj^S1tUp!{>C32wi& z7J4*!_fo!~0%{Prf$EkKLTOm3z}-Ov&v&3+QU?jaTQK7M-iRmbl{+qxU8i*M$kL0G z#=OU~3N~szuO59SPM;#j6O|2WTS?-pAh74(Wqt0g)NJ0hI2kK@P$-4H>Fb!_8T0Rt zI=tQIwV_kwZE$_oE;(i7P8}sosC6kax(CDWbR*q&%{~_$9wyc93=yN1mE|;X<@IN! zJY5U|U%CEJd1{?3ZL~G44**OVe@ITBi%z)eE{+J9)zmiV8!rz|vON>X6gV8(4d zqbDge_2^y8WYrJNBaYAWffQZ*O9A*_LALJZgc;?5AvlEWE&a_5q>4nT6tNo+dz3l| z%>D!Em-t2YecR<3W}4I-9OA7S=EWENH4#`?4z1ZsB+UJ)v6&_du4(jq|CH6$HpuwX z?3a2Q;^LSw$}n6|0&^?+cD4Ol4Ic>Q!>}nY*A}M`jDM8P+#u^=1;I>dd|?eDc2mA8 zf5U#dsColUGQ4e!$hFcW_A>nN>6r)6pj}b~e z=Zp{)$_N!j9+hnENCHN0qr8wQ`OEnDbZTdkP>Uic)Zo1dE3exxl@R2G{ID6$IBy?% zfITyh1yt_HQ<0jUW4$mB7qQm1D=eGy*l-YgmMs?&bpJB7DO8#@HdH$woL@$TOAL3| zY*kNwl2K_EIMA7M8K`G1=QlnsG2Ol^W7bF<6QTQ+=;^ClySjT%gBLd@4hUpWgsP$x zsiI>JpHMNo zkqNN%8T`33JA$OHSIB+XeEK*X^R=ea5L=PZ3zLa$$d2mV>NeK2eG$x)J@#8dniBn9 zH0-IqMP^Bkx6R+#(}yC5YwvSy659efB4P@WLkNbJ6S{U(1I8I9dG5@)rx=3*T*6Jy z&Rq7?r;yBYNUr^_^mIWJMmYoM+LuLeiieeZvscs6p>a9+4N0Van%F!877!FE=tr^mZr4J1#5t&E;} zOa$9($(MQNO$f++vzoM-FK+(rnaA&OMGMzhe@vw8|4(ZLOxsp^b80m(Dn-0z0&u4_ zdcu}_*cbOz9Lzhqj;eps8zHeRLUtrwPHgX4kc=2f>dIrh8H`98u2?(Xm}^rBaEIr^ z(nfMi(>JawI9AfwB-u@$+I|#3c+iS2t zhi52aA14c&mIz(g?3ifwa@bvh=ZYiyz?kFjW-sRiu)y&h`MiP)6pO_6c_b?k zZ<#nBynBW_(;a?hmbWW+;7?BW*oAf4_yux%R6W$uCX+UYKXU3+>U(&Fv~G!X85pNo zn9hvyyB-)_D%Z})xOJ908uEvjmIWgVCVD1n>xB!S&BMRc$a@9Ql4%dhpFRN0gActo z*l6Um6xBik)>VSz6k7(uytJ>c#x>&77ls@1eG<kg-|Xh-XQca-|By@_ zPgpLhtlJ-E+}7r-EDTvqw)_=vb8ZHex7wW%{1BB{h^-)yIW|#tLddZUC~{DEJ!bcX zwu$+M)~iQ`Tc=zFhtJ5(A?cKO_p*6Uy!I>)_^%4*Q;j8$H5XYl!W?iR`Rv!v)Fdcy1{ zyVKphLET2At`c_NSgZ5uTvGnFo8po0`Ep9O$TL&#qq$ zy^@Y=eUAzi@v#r+m}N%P6g&cE@XjwuUuz6FrS7}#yMP!1N0p1C(X$XMG*p;*rtVUE zcS%9qh0^KqWJE*P{8sCWJ)LqC7QyU-BbZhe`3b)w0A+svU zxdFCy3^4Mq&_4>gdaAfQQw#s2YI%L+`|Wk%c!&@e-#FzpH(A||*dyR~_518qjVC?5 zn``H1RLfJ`oRk&G+b0&_&rP4qQ3%dg1Z!LB#o^|K<|^(^>?B$%;>IS}P;N&*q|JQK<)yyo=Kwa~fr=R5NR0nVBG<8cgK z*ZVpan48s5;nm9q8#=T1BW1?@s{}U(`6wv5ZOAZxFFebp#4ehxIxDz~%pa`!grJ9W zT8&gzo6(@`@z3dYvgZ92M*Az#-XM+Hd@n|Qm{Q*zG=iRhdk~xZPlFpOF{_qPvL%W; zR$Gi z2W}Z2Hd(y0ho`e-KTI~y>IF$)9o#CAk>A}+hL-`dX;xgLb%QDu3henQr2@Wd%p)3) z(3@Bktca2F8JE0Ma)8fu)^oe+&(BvDUY+Ma*@As<4@7Ec3r=}v3Beh7xIZqLc%?Tws!3yQCVW`(lnOJcStdRbx9#fE6I8n44~UB4zKNR=q3Zs6L}k% z+Cu5E%(+L3`x#XmOj3XT17&ad30#}S`u@JjGl&CKA4Lt+mDq)BS9;S#C-#o1iiVZ1NM%qf+zbtDL{{$uee6Vy z|A`;Q#X)0~W-b z<;5@Hq(97<%v{m=yX`_H!Lp`VpOl3mHf+z8a|_WcBwm@12OXXK9LlY+Rn@#Gu1s}Ym4%Y+73x{Sn9JI6;7@0(No z%Rs5O@fNLMEz+b5;?5v~qilI6o@et|M(ZKllWwcZc_s8djeZ9cWi@zvPuo2tF24Dq zE+pIzdxH{kYEr1U$AxMIIieKi`5Lf+B3Z4`%Dkdnn5xnurrDlxPIslH$2VENFdk+^ ziiAzAjIom1v!!i!Mu|RaKd%?QySlRywxRa|I4eD9uOAQfd(GzP`~f@XbKVSo-k>1f zMq7|UM67CY;oA@Dn5*M;2=#%%&5Jbwqvu`e5n_Dn#(*pVrww~<32n>9)T$=H*fn!Z z36!UKSN*S$=04Pc+gSt4v+fbAbr*sV@WW}Xi!UBH{<|&uKX25^zE)xz74(P}FX8+% zVi<`3#8*h0w#|d&5yX}RoCL| zoa`%*lrviii+G5M~*lg860t#{I|BEzF0knfqLn1Q4dfxY`7I8T zxu)1B?>oQT0fpaRfA-xV>?{1DDokgk1fls5XQjY$qD!9)EN45@J1Q|pdabrMnZ&T~ z&C{Aw1=u;akRbQ^Fx|w$Ea-yf7czFyh^> z9VY^mW3zHKc@28>j7zahwtKr~^~jP^;`7i0@$u?AF(ISVgA;O}GeZu2%qmN8H~^lUQ>ET=+9_GJ)Jpiu9&scEz4ZEiI44{|o8W+SMt=TM+g-mF+2j3>b(%%qk zM=Sh%xbt!k)0WYVAXR&cyY_%|(`>cREo2hQ#Ey_~l2w{}msf4d-2nT4+hXjhoLKQk)duu(S1=lCUE)biaiofwBI zi7j~7uW|13%3fnwd$3a9;r`BnJlcPQyL*X(W6x0{AA{?6`?Wljt63JImTYo z7(YBdK{rmuyl2is1II&QVUZ{>?YFsYsM~fWlaNJ-0hvvl>=p2+$l5tk1KH_V>L;iL z!!JZxi=xuvj!q{ewP*q#>+aEtWLUVIIA|JflA)H6W ztJT)Hy-y~Icku0zyDt?K$947%+aMA2+~6q4tbP^M1C|+iKTvu_^U3KncCbsVD^@E~ zXehPXW|%-mgq+N~rGCpyo#WM$XS)=16U%X^j|AdRIP#eEuvF+PNCpun#zw8rkp6 zxWkbOkcT#i{Q)}4AVrT?Ss$;kgp_M{=7);5msDX9+M0S^*joZC4VixH2pA}rlF zTs@162wW4&UJUEU=n|naPd=AN90DPQ7^~MlCOpkCB0rQwKZxu;|IIM|_Rq(UKT(xL zl*=Kl+pI9nr-nLnl0iJ=!`o3fN zKq$GSNOm?-*7}}#+YSv0L0WrZt;i z4)#4`?*#@*ALRc{N)xX~uu$E`-qm|kM(R%#X1eWu=hbzJdgVv ze3;B0O<+R8`bjo=eJ|LdEwa^VM$~gCZXsf zne2q~@=8N#k(li;6g)Qvqr9Zm($XTn!blz@m|ip6Nv_d4{vx>9<9x9H?#d{T9I;_u zXVMREUXzfYoPqK6okr!+wQa!EjBg8f>h1KmW#8=g$keL}4UUdH!o&nUK2RGfA!r|1 zr;4Nh;zrw9gWsmcu;|eQ;%t(A@X3u~U?$tX!FnMP6ENw{PEIX6Qfcag^Y2cD^75Zd z)$hbAA?|zaB5L2&wz=NwDowFpOj_xwJzJ zy`1aA`XAXg7`!ByG zFkTELz#;*AV|e4mjx9oU+Sz-6c>z$+#$mw9^?uan*W_>l0`_sGe4(oSvB^s7>XJ$C z@TjPiq1Vo8Cl`>-fXI+!u*dWe>V|xDNQ$J?=D_cO&jW|Kb#88(?!F;F)MKm9J_R{c z+V)ao;#x8q#LA9juI_P570`S7Ej#e*k#hvYw(u5TLnyYty72qZdEc10KJ$HP-s$=) zg{9to@&F?Yx2tgG&O{{K6k;r+@yVJ$bEO91H|-20$DR}12c!joo^!Ukx@X=@Gvwp3 zer~-Cu)&Cj_Ney}<<5K34tJw(7i$CEitJHW$OO&TqP*|rc>WTe@kI7b-Kv7dW}erk zY*kMGmosZub7<=Z_K77ToO!@{Vc4o`Y&yn`@#YqB$A6qzRiSJuFDfpn!aw$BOrzyn z)mb7ytrt1ZAZ@>el0%Y~eWk92+OwYwCd>C+seraQQF7Fox8p5=R!+#gc5#@yv)Te-)m6X*il(-lUIK+ab73BKi=**@s^rqs` zAMrVfx=5i&GsPc_nY-FQXd?d@OJD~4#?p-neC-j*AHg{?ll^%@^yo_wUco|@ipd?9 zB-ut)uJ(%^11gInTp?{YWyb6olTMtKGiZD{N=j33yCGX^X}s+5(w7?{^PojfA&;2*EwuqAl3Ge0|P=T6@?;{E4g@XsYD&WTA4E$a@>ahzqZ&SdisvC|-_KD*;4 zqjBL{Tx>|qF4(3l!^j$1f}=N`7EkYYreZQDGIPFT(WU7VsA6QH`1K3-L`auw{*vxq z)OopoOrWgG;X$UR25gS&p1Pa_n|ae$10UOl7OXDTV3a(d>xt9FYK>QZHu08-#v)k{ z^KUDS;3i*yHt`iMO~j54URr(JpPkd;|CAIYPp!V zJ>u1hg8q7LWAd60K_`6{B4j@AvS^p#rU&1OSRS%t-ldM^9~D|8dGjMmHrd%ev#zg9 zj^O&h|7!M)HQCL@=Zmyc2Hm1RQ&GLeyyp@x7_0Q9_*bf?+@IRGN zWD>}-NWdg}@(({aq3}oFm-C|-B`wEMt{Xn);d~hx_@iVr? z7fUVtYw<>Y%3Uhr#=S46NULU+T)I>M?11bit-*-Ysi&ZOns8-FPC5`ZpKA6XNxt z<-0~_>BFzkjTPMR!jI5fzw5q-`6^G~LI-V;qx42kb7UU=spmNDK`NmaPX3h zFC_x+{6?)Sux2CR&9>E?nYn;F6XUKjFgfCCn_#eTb#9C82(+d)Xjd0Da&Ck7@}$Bx z!pT|$5@(a>JI|w#Mw$6LjAhyyT0nAt3d$Pnkv~1^9o0pLXTwTJvdC*fVZ!vddFc&gy(IuG3^cqM+Wbh z*`%%3HLe;1=@c2E%#KXrOYz z5i-~jt7)Di@;c_9Zog@L591Gj6yGstiSuyD?>udG#B01U67p3ky7<0G`;c#SJV8l3 zLO>|9>`nT2a%uWA-O%vmVK%e$c=k8Sh}g81>E$i~R-bE#6O@!|q>Rtva<7f_>N2PP z=J^CJB(3^M>rMZ5(8EyX@QQr}BCotnhVvlu3Na$D3{#X!Z8U#$Q(BjU!QGA^-oSW~DeNtpNKyP-&e@O|E$uu`zQcRj1X~YPpwkAMn@u-X^=WTYRD2c4y?T4G@N58*AyRZ+P|x%> zIDBs-E3W?(uu{Rr5*f**Yq(|VV+Yu1cUWC&;`DeAZ*~aJ4?)XBG<+DbHGi|QZ)9iJ zRBY7U_~8jUb<3@f;n#yc6u4Hu?@!i$ol$AQuq~igv*T7RBfLBZ{j+!cI!|6O{T>41 zMIC<}nyF`{L@C{4*9IPfTd&24t5&LXLp7TygdVzougmp9$l2h1pKNp`>r{+-ar130AG19)=nY@cdzAW+ zTBTiCbWEJ5Wg8nDaUaT!{+My|k|Vv?c0=r3MZK>9=iGR3ma=%oW8p@8LgLJwsYy-80zCushvi?sjjH#E>ad@U8uaTs;$oreuhKeT@%1T;Dtn>dT> zn&~5U;W=&NZvlb#LpGs?b0lR@!2EwDNp3^Pqe;=f>f1z+{htHn`lV9XAcDRis7WQE znLbws{hOpoS9@5Sn7b}Nu%nj+d=PldF%7!;h@EjgdgjmL_K>MPvMfJSbMkO70I+O} zi`BYV@#HvotEKR=Lqgg16>;k(v)6k9QZXx)>5cJjBjJ-XzZE;a$dSqQ8uo*Vk$lJn zAnB$cza^ZI^NPn^ez6;&3&QIzkW`0Ddeq{rR;(LY&fjjnSohAR@Y#D_3(cXg{(HTw z_vP!~Xs$|rgoJ>02yX1Pr8HQz23k7UsQBG*l)!A(w^y4H4SM-G_U?T$?_OMFD$~Pg zx4*y{e+d?3qP+Kv1l{@zs*)s|JVfujS687XqlIynCfn$2RuX+N*q~^FR#PX(6@Q<} z9LQQSgLH2tzQMLB@a4X$t?tYfd6%>~?kNg+J=hTWI1_j?iilq2<1X{v}v{qneUhIlMEqpwDG{0ODa>i6=Z%5OP@?wV0lkIR0Agy`4C zU)yT~A8+hP-_jb6-4{-j&j>o21Mz^b#nYQ*x`=%JS6+V-uUx-t7PwHr&_STo+Gm|B zKBGsGvG=u4ANv%%)hyj;QyXS_6w_PN$X%3-4dM-r#~BaKo7UAsFGrdb?kaHVh$=#t zv@abtm2iQj)CRVfnJP=&RYxl^{G!cDLp(w2Geg^rKImA&Rnm7{sirrKklh*+fi)q9 zkQR1Q;jf+RfisY!W3QB8>Fz%0i*HK}6BSN)I_%GACbF&_?wOop+vS<5%ULHb74vds zKeL`?4a76DT-icn&74k5Ho4S6> zM?yZdE1*2DNoKV#7rk?8I+tH4cF7^LXMj67y}hl_@%ge7bp9K@kZ679KQ(7ZYRp%U z5Q-5H^P5(nYY_Qqk>K+!?Au0JW4R-I_wQ`eZwJxjhv@o1)cGMpP6>{=LZG-->$zbo zHN75fMSSJA)220t|KZT}v+}=aY*2=1>$|V0e_>(nicMO&RpE%#PaLK`nIVo_-c;B= z@fbPR7$uN%=cRg(HtKpc(*N1j#3PoikM>~Ohb;Qiv*Y@)Pwv*4#09l6gO0Chza|^F z&v@|Xm@Exn^+)5dK! z^T~I^1a;jNx_9#YXM!>(r^gEiu-ml3^=>{Iahsr5U z`HOX^vBU%y3^>7P-fPWLE8pPt_r&7+Eue%?ov|)Eky++HuuxH_8u3fWN+pUsh53AY zZ_3UQ2x#ZRdF$MRKr8pV8PDE7*rL}f*W1^-xJoG}rBgxp)%GpJ(;8mM~nahaQ^F zw#i9?WN20_za>dp$WieVjI89;pa5rYQ)*mX=Q1n7TV>V1%tw3j0dKTTTUd+;kkokTq4QO<_eP49dmFF9 z4*C3@@WuVi?K~0}jw$q>zH{t=l57fqL$-d|78fh5U(|K{R^d)y9pug-`8 z)cD037;vMzA@co%9TUVyXb|2+Oq8@ovopiI*_?)bHs7>?Z|c$@NC>!FN|sCIOrIi- zume>5s6omg4B~GHPS+=D>qrHc-->@rq?TWtJs~t&cx`%n$ea>B^mNXp%0Bk=FkFmw zWIE|kR7ba`7$OyFbi5FGsx$+ghM<4(fM)i-shwZj0v=OCrDmH)*4yeWcMw^_~|W8hOa5ZcPZu zdP@Yc_XJ+PE>eUwa204x5xBURPF7K^8?o4KuOffM()rkz#b1AE4OPIi7|U>?6tw`h z_jJ5s2BcYsAcMF7^t36q%%-f0K#$!E;$LBIpRs3jvq%X8@i|z|Jr5-j7N+#ZY*U9FLBW(+lXv^gD5y_|o8+7QI~BZHznH&U1O~Vi%hgcRK6? z3!9Ep9&+Ovvcs8SD+eJedpZb=i2zN|$AO9)2I4k1ovy>8gtaY`nCrN7!gFA=+yCty zOZ=rxkQJun*?8K06uGCVj5;K;+34qbxRi}*!hjbSbBh-tH(&N9>z?idLOo&a&3E08 zho9#YZNHCuOE`~{{WDHoEX53ahCuj{MwMY>YmVq&$m6Q(!kN-M?Q zw8|R24dSzYRUB)*IS(O+IfDOfR6XV|9R+dCRkC3eMb0(}>5 z?B>wFaAZ2h|NkE$gRohV($c^0S&l;bO*Q@uUZ=H-)bW`3`uG}^mZ~}S-zNU|Rd2G8 zqUZi^NkfcxLwq&n{~jX#&m;d0h$9%pA6ftJJ^cTgE=9Bf@b7uffBwn8KPj6?|Dy}? zzpkQ#{GVkwZnL0W!9Qxe>V3XqYt}|^%4R?O&6^04~Dag#HtTfXkBLTPrE3D_gF?rpT z0mVX2#4l9oTZt8cu~okgbcCFKX7hjKb>OP5k~I@9ZV*JxOc5C7_okMXf4ss)LDDM1oQm7~(F@7?oioQ1L`QK_28592#d5+x4|qzedx|dZ?bgB#opa2<$Pz+Z)>yr=T^D zsZe^Kc)Xm}vF~?^Dnz$g4*wBiY48%R&6PVwDsMJfNRQbecH+WQi_IK?=XbO|1Z93L zSFOy?$RK-{T+W(nD0?fooPz(j_}cok~p7^}^;J9H}G?t^ll;sD1Q z5??>{xpCV^^K87}l-9Um`Cn%9%|qg)v--*Y&AS+Vvyk|tM;v)sYKnbF3o>QseHra6D1>-(N4dRo_BQP3MEK`=&z<)YU|Ci#i0GvJ%24Qs4(WHoP9?r zw>zGkmfkdI39?zU&b{#L>%j?xfH%+2TT_+TaZCsAFrYUd7#_ zB`A;?6`M%lE?~@@Y97_z02HuK49nC*gFWf-wCz_06jf(leM(tfh!w6pS22%g;3vKa zwbCKRTq{Y}&(!@bbS#vW@A}KxaMp#{?55A5iPM;vXS4I~j>&&Q<{Ar`GIh(b{GBC7E6rD?sJOLBj(3ju!E9o#K2zcLXQRJ=A|$yd zOV%W!qQ~%aimM>wN1%4!xeWd9p7tW9dWa!S=g)+5-@>WHZA^yFno}a3@(d>~9h+<4 zJ`SVj>v4KM8s(+=1ld6pwFS>B{%6n?{hGYhR(i|#i_4A;jo>{m_pw6}YC}>Mkj+Sh zDvco{_f0eOf_okzyQ6F$!ClI)1#HhArY)Q1SrvyaqXV0B4fHXjmy3Ed!*H@grZGtena)nAolGU;B? zOO2!7a2U6c*-w00YU)pHzT>U)B99_lax+mgN(s3$udy*jW%%ra^AU~l5~C}Wp9U&( zoP^j(EiFFm9Yvr&*Sa;>4TzazCfis_PXilEkcRAw74hihj;v%e6Ula#koY|OBg*mb z__Am_>gkQme}7=?DBse#hG}f!ql~zOdd+O3Av>0bC!XfDXCa;q`j4nx z>XpjvpSci6qTn&M&Jde{DyL{?2i#=#~ zs9sBYj+We^l>F!z2@D%>q#u%prNRmP6iH$WSN66(xUaT9=v|>DvpK8T2~Ft1ijk-h z2btL{{U2Ho@{Sk4KJ>~Jq0m0SGu;WFv^q<&82$W_tCgh~vhT-B0fFfj-TF8#F}0E!0q0U5gJwPA>e?&2@zkaFdHfI+o;D|S@n=ki+xl#xclB?b zqO-MK94vH_@sjCzFyGv7QzES+l5oWG!$#B}&*)b(AUDF7ab{!(YEw8D5#LFEXR zL?%#BBnSHh`a?R?Z?M3^fD`|C8&E%e->U*Lr9KAntr!rwyawBRK{~Qvup>~gcinJi z3^V0hESdo;hbk2UuX+aLBk8wjPJt};kF&2L$a#mVH{ac(((3Z=*bz9h#82$;E5Jmp>}(04~fHzdlk6!(e^8wu0z(Q+yrbQII>_!?Xj?R1a7k7sHwYAd6{Xow}9*J8;x zCVar|h#BM~$3`f>i?j9Tr4QJX0tcEMwlH38=v_b=n3Z;X{lY|}aHUCT4ooJfPc)D% z#;;QJGi|YJ8nLED8}h4+)~hcmd+SD_sjhj8#7BMypS@;}|L_XxM{(TTU+(R6GC0>Z zYJ-RZ9x-lLF{F#VMoP+_?(DEt$A;U&jNG?u^}>pt_VbmeJ@?BtEer;$y~!x=PuqgE zmn|pkmYVT(^-_Bil&uTaOSj(>b9OjIFemnTus-!Be)P5=Yxl$wgHBv_Pu7Zd2-s( z*iInR)w$PYi2zV%w=Wl_UN*K7E*0L|tlj zGS^eE6rZoP+=r&)I7!T~7M$?O>%Fx_!?sSgxw+0#{T~3USMT_eTiJtkPw+Iy(^~d7 zJf(Aw;1=$V$EA^SPjBm44nHaWk?jlSz)#0Fu8PPLKU?6>@ykdge`9^&A3VguW?2&Qg2q>6zYfKmgJz(~uV}S`N2a?p zkzpp!S1yS9c^WZ-ac5=>T1P_NDvGcf3>vDU>0O#T_f-bGi)qVy%(@lH6&kf{>gUU% z>VjfDdDgt}K}{*3yxrH+`odp*mT3P|c>I^!h`YrdH?N&_B(C-dZ6~kjWq52X`X(Tu z7Xq2*h0i~%rMWHEPtF*w&Kz&PbVOAW5Z$}GAr?40#La7&70}MGyftm$asLfmfki|< z%OTxp9ZudxLLzn2nq-xcSt6PI9WV*!BIg#PLUT4o^=9yGCVZOUl*Byv@?dd;s+AkB zs88{SKiwxDkeoMB+hWNki$Yzhk1Nm-c(xqZCFk9OLE-ki;TavDqQwPCTk-+aRA)p+ z!lo`n(cwiCAkB1n_?jnzLYHwFi&TV{R~O!xSY^Sji2bsedKFp3uIzIUJ>=TV5NGks z6^h^J&sDTBj-rNWrQNyYCJjeJi&y7z(Mr4qi9do#x6@acUq@HJ>jDEP;~!hAcIH*N zfEkQGvaYh1JIdG`_P!>$Z}G3rN$iC|_ZImC`+t}W@4=0DJI+QP>F4lamswpnO#c>TBF)l8 zHiI#t2O_0sx~)i=A3tM|#tIZozfGx^rL_eew0b_%MaET|LI360oW4e>{q5#=0j*i@ zo##ylPSp@8LT-Y0ZN=gPj)v#RuVo4S^j8N;P08aq%H4DRl4rxIBrKn3raE((8|(kP zT`IVp$dR`#-H7jPsik3`@x;om<1v^eFf!lP9Npo2dv?K@b`(vvTS%C7eV%AuOkXVm zB=tpESqtsTHc#hA=yi;(E#>nVc|ZB`ke6z<`lVf}jyt9Q&t$x%NyZjW6#Vw!=DZPS zGbnHzb!@$jTWt#EeSMW3fL~#&nkUb0^R$B7;~;V+3wID*D8Hx9Xvc0?gOR2xd7>ncXp;->P(T+ z+oLsyWE$Z|xxZY2Z=;-q|KitS)9V1q^4 z&{|z&my=JXmaiU9BiJyWr;K_L^3%aqvHK~Zb&5rG)-yMMOqse&Kc1%JdIiKfp7k?H z^Qf*P`P%dbD)@!~Ut9Nem-cU=kT=dbMZ;c@+wuB1vWe<1kJ^{4Z=-q|r#Vhf&N)5#mhUf1zd!Uf ztr*4F>0;*G4Mw^Zhu6B@Bi1EABb|gFI@&fW%vz?*M~irL=heFeJ*mpnnha0Ho^z)M z9FS$6BL@iv)Dd^mUQz2v4<2Lk6$bKjau^ZG3H7GIB>D{WBo8u+ z>FoFi9yPlTK^D{vUYBf?^S9q4Oalg_W-rc(F3DO%L!>@9@n2(QfccPxn9MD(t4{~sDO=VL;OHGcW7<1g!h8Z9TskoWUqn=^B;%>+JMLvH24P1cD8LtGTL5DMjNqF^arC>)%Ws z;-9W9&*6UR(_a7cg#Aa~?M37A*E_D`k|DTJJXw>Mk7Rj~uhB}&5WnnM_+oFo-EpMn zr^_A^VO2{kV-0fR=Z*Uoqdqb&769|b*@~bZ{* z1fJmFG7v4{YWz^A`b}uC;G!}vs)VfJ%qBST4BmCYWaUHBmb?5hAYe~v5rO9x;0yr4 z_QpNlEJRw$V%QciM6OT{YSNn2hVt$In(3J%@r-NoAdjkd>EqPfIKjsF%DLn6xb#*& zt#Mi5;kUaYbR_WDu~T1tbfzKFH2o;_1GW%2cb58CK^sk){ecq+Bc`G4YVbx{#t~ez@@znftXr5-`OVt_>ZC3eiu?kb!*1MbI zbux>|A{$L#Tynk$uOF)y{uFr_ zmabQ6ub70>c6R;NmBaDH0#3iwZ6Tjph^Rjs`?AOu4}39VU@BJ=Y_Lt@?P`Ed?lTu? zF^pd5A}iUl6zSyWP0`H)8}9A%KR!Kb*D<#Fl;qaT@l%%-vh_gkSr7I^d(MXj1RHk8 z7$Z8SkTHMLK!y6wTr-?syGW>DqN$gYZ3Q~&#z8uR3piTmL-9)|K=^aa^!{^8^6&GN zMSz)0vLV;|P}-LcK&0@TB}gK-HCk})d%5CU4T?A6n_d`hur};jPoV|$y>kH5E=O?W zHdant(a&|F{?#ioW8;}J-YCVUnvxNtlIs6;3dw3>m}v}WAd`OuT9Z3$z<%#BvT&cs z{|2Y^PirEcgKNowW$^%`L==5mbJ0~)qL^A^cbX4nUlTmR`4@8$WJyUuu=~iGXZ)$m z{jH9$U|pN5D{J8Tgs{HdtL)E|vJP)dvNb4*-G1C;5c0ObL?VjiwGK6e4rY_16urCl z@1K<*Rk=nCCv}DyZ`)^O&wDkSeeG=Km(_ssiAqN9Ji04v8#K6H%6mlvg$M9VPVT51Gg#^p#te+y8Zos?PnHlF`z4*B2 zXbC7Z$hdg%KMayCJbsxHJoHBv>1E0W$+Dp*9+T(JclepdRKz#X$1!8f7>;p#$W>X+ zl3r#HeL@V0+uts)aC&=TaV!&yoWCg>1c#|z$xI&eB)I_PvSsl|Goq?|kU`B2X^ZlL zidah6xn<8`OttHmJX}G`BUc|=1-6q)A3LNv>`@z@{cf+PR!z9vG7pQ38!13uv*xHw zBI4!VQGM*EMHEJ|VTL;?wQi4lxtlS&oX1yP)?4R2i0>s0kiaLBn{K+K9TM>b8iiz@ zx;0h`KjVJsDw9F7=;k$OXp1$&c$O3n4C=!#DS7c?8>ig-qx-4avc4l(_KT2GbV1>? z!KJCnR7nQCs+2Q)Y%a&o$Z{3@*ZX~C0F}X3d&_Z~qqTz`hrfPfcdqgANw-ws7mnJM zupgJ>hSmQc)tOp7mNV+7E}QMp3JhN5*`ev2WU8&$i$2$24COgc1W~;`3m?au&$57# z8`(J&d^sH<0wXL#l~arh#j?%Kmsp29D=BJ>N>DeYF~*e=p>N-|KM@*nzU61xdVSL1 z7H1^2!HjV`URq&3);(CFJ=G5{(9Yc*%AAECOh+SAwzESf^I*v%7bBDygH{Pm58B?> zJ0>Qi&hIC&Mn1-WeJN$QI-3Hp>StviaE-$ph%kP!av&=9YM7&xId}=|`;G zxHF;2+)s~%6jy9HW%{=|h9RZ{JHqz;_idhv4pRgTtWTJV(}h zKj)mkU;o-OYwy*)cUM<+b=7qh2AP=+Uo~hsx*2!WUPk z)=pfkQ*8y)MbfzmE=cI(c-gF9c*QX@ri}6zCWNkbHC1C+uludHO!&;^{&e7X{>7V`^sufEVqT`>3r+*8$^X9gKyrN zdTQQnXCw9uh?#KgIehSDdeoQ1JnFf~7W%^NbGLeW%$^lS74Vnw$3AhoVOF1#)bQ61 zl@GJN>BAEy@I;!#O>n1~^8gCS3dwzC6%Eju$O^Eoxc?vCW8mlPAy9Q$EMup)#k^Y= z{O?BCaN@U7LITKWq|q!%P2XhR0f=+Rf zO`rRlDPKH+ZZ@|@{lI6rplYX5sa@gv^Gqb%RVNQSkWqbej)+e^saLQa9HBwk}=e$lw7p zTTnPWcHLl4r-090^SkKHg8RjsPu2Off4jcSAXn=jB5B zVrh~~O3>!c2c7XrtOA{t4Gsen=p{Rb<=IQ%gJT*srm}j5VB6W%MW0S`)~6{4OFB78 z%phCrki zd1@>AcKw(W6B29?@FgvMwC`VZmv}n7ov@_rt$u{N&pOI` zwqT7VwqEo^KB-SL{w4~0a%6x#E8h{#A{f!<9%tRqhPulQ77@G|gFDw{84nkO{=uKPL@}wPDWSTK5C#N3vhj-ijJ$k)Bv41w)u>t=Z4(6c zrBimhm58G+eJZ>>4~ zMYd;Ph?j!J@GFxY6jH^U0Fb%M_cJgLgCUof55Gz4Z3?I9K<#0aR!V2tT2HbrNbc)d z{@B{vwnlGMrR%o?%uuoN-&IR--Ohg7yRK^ct4k67y8dD_VwRuH1%_`n!2Vyl0$aTMI zk`Aznyt@F8$s=9$YTm2yC)Y(uC^4yIF|eV&cH`^v4w1>v7ba;*(=)haEnjmB;@y4v z>l)n)ENHa^E%`0^Iq};S8$Xi$HsUEWt` zjUiwZMp!;n_lt(rK@zQM`h_eNMn&~<{9i<;MF#wk{sctL=8cUojyQ=imd&axeyZTV z3#1>d!*vF8DaHu)@2vr2&QG4k>rV(~wVjKFY6KT;TB>ApOYiIPCT*Si%o6yVSKF1o z_FV(QTesM0@7ZEi!cEz)y&Td*ZuxSOc7_6kl1rw)bH|-@jb?fXo%LLj%l?gkUf4@i z4Fq`X_rvCi62l95RdTk!&^|npLEX!K*>{!oPK%X%_7xb`letUh#4EEPQ<@C=95Ddi z<0N4AB7}Ivam!(u3%e9}a+{y_m@zFr3)%azrQM%ci!Boy(hOZ&wczvzedWYTZ*pNM z8wr=fV~}f*Y2_fQZ6p*@+~MnYxqAnxG_$VI{;-{n0lhmdf_zH-U-rU-a zzqOg>mjAIqE(=b=Q+~m`{{}t#A(RZ?w77wQ87@eWi+)%k$&%UfYkJouu~@&rWueM- z_J4cOBATnmbxeOFwVLDn#EXOEzc<6?(`wB`C+hT&$>rx)f0ycr>Ss}nSEL{y_oDer1xHN%J-UTh0sty zXl{bAH`pSW6MoOh=g(}P5p8sF^rqqIi52?vL=-^Wn%gvMwl-_;kpGBYUVZxM*F%D@ zPy*^Tl0p(3EG6&D^0@pp?uy6YDFfY3ex>4LPp*7 zv=w)Nyj-}jYpnetexom*MMebf+a!uiQ_lCfpH*+bZkD(KS!H42S-(1#-=z9RZl-oK zqX`Rnj!$s5`&qr!)48~Dy+@5@@VvaU*%Y%vE5UymQc#$mWK~`EH~>s6#m3BYHu9X| zWA^2g+ad3cyUqVLSqVOdJ?1Y@3JblH){0uVS~gZ{3!?bFwg{&wgq}!oTf9Qx5G_$R zrJj14-_rsc6+m*5CF4<$EcMr|%+zu^z1zUKEnU<~Bt zek<+gEDuk!G4Z82j$}Z~So@W2I32m+yuX2cU#yWacwBWOOi9u2mi2Kxh0AV$HZd0C za6Z4#NaaRJK?a+H`*q~5cF6xq_8N*jCp!w)JVF$u@aezDDJd>xc{D}{um^#jFC0`5%p)XI~^Co!7Db^F;`0%rH zj>~fDo8RnC!XG9RNuvh^_u!u-Jz&VNy^EBdoG ztE@Ruqp_5rgyFFf5uIkE5_%L=QGt`Yq0_%}kv}tZBt9sIHGZs?>CT&ue}LUfy>UgH zx}>-6vY)LtlgcnfY%}x4C@x!WTC5=J$(b(3`Js(6q_tS^Sz}?EsDBXpo#zQvobqJ! z{}U(Cnq(z3mmIaABXgggBrEGQbBG53Tv*_VBP#9{eBaa=0yX%azSk4M{CHV;U1v3$ z5zL%EUM)e!05X5ImbZ5XJEeZs`_)zKwd*($_xTF}K?ajYDuP8}PjfW1JMN8+F%4}c zu+5*IN?YxU=HoaK8v5|eE9JkWe=v>0qt{Dmuxm>2_5~lA?`FsFwJYxa?i@D~wfRt^ zQ`wy1im~&x8mTPRgV7arluCc;-m3BZ&!g8+Rpqe&gJI&RCWCuk+dXL^x>r7%l2fiv zskU4>8Uon3%t4Q3?(f`B`2|>nBu9VW)gA0Xa@A%_NXaE%k^Kk@%=F?-J$$OKod0qH zTeBRiO!(75PX#g_I70?I(r0BgmAz`~(ySA&1jc zG>6gH7TP{!d)AXB_GCL`yx};?$S3)7Qa6{5eYpRD>0ennBmWm4!oY})V`(EOgMkf6 zs(%s!p7J+120StRC+orv)9T>s+pCC7YJ#dciQIw$<39^jBpUThva|JnsoL62F%*2% zzff)ec~6Ud7r4xWmCIrWl2eqrqRr8H5fu#1;5F~gBmFO`18N};dCa#3g`6}PZp@>e z)vSwkoQ#(M=Ga3mRgPH!*&iapLDyb*tKq)5)}SP(AOs6Tn0`Ii| zXUtFmXPX0Klz6{`Lx{p9&(3?qUI!J>xt*Pjpt*7P;}Ce=*8~4MA>qI0pH$Q`kbh-H zp6=RdPxh|DqW&w&!RtfBQWo=|!DGJqxcJomGs)?P?el=Ks{Z$>=2s`-B(A-O`SuE> z1&CDR9DO|mEpdaj%r8Ed`TrzFApQR8H+rQ^qH35I`#U_MM55M0^io~_quQ^blE!k(d_Cr$sIwN0ur^rewo(&~COU`)1y1E6DnlCw&3~v(U z-J~M(X2t!gvGd8u7LRhqxo*gRksI)FdhI|F)CMy(>ipMA!EnMjF`t1d1fu7Zm=YN3>frTv&W?xnOxpMNrT zL7D#UqZvxc0}Sd4|603!bi`+2UgZphrTpZAt&A6hSC}hA z2bJEgcAL&~dZW4nZ1_k~^^`vf2jRNa!N`s)7L=BehTd2FAO>K*f|U&vcmXZGs3 zeaP;0UOK$A=xe%unffEKC#M8@F>VV3f1z76!Yt;v=&VvnQN>t8pV12t7|~Q zTBaJqd!ZpG%bxpy>rxRDI)`0rcDygw!$AAVxL|_7lI-=h0~=(*gr~I5Hlsz`VC{%h zw>=g-=4#4W&W_1Qf6SoDc69Uzhh83xHxL`_sn4QH$}Fr*Ea}8fh+aO3Xp^ZC9?Kac zU^)3_ZWvJFsnU2c949>qD*zf(UA497Xs2){tUw9nX3AM~$k!Ks%8FL#kxQbHHrt_G zG36LdG?h5O~_4^D@2*VWl)%S@Zx)??c zI>)Wn##<+uxQlzh875;uC!_s_VLU&kbJJ!QlSEW-!Q`UNYJL1;P;-3ya#jvu6XqhG z!8hGLh+(KN@zaZ23onLRl}a}}L3H{p%YGYEHSq;RL0YSs$c6br1BIbFJE!$RbO=4q zbGjIfb~dUYuh|>SN6aot(4osfM{OLZo&}p*pHnW`k9EnQ*OC`D*{TwoQiWdJR)F)t zCw56BZelHY`zn)XtZRG^9WR`JHb)7px0Hjx?x2Z@C{Ja+5mP9_7h8np_m zDGLY7@N154k1y#h{Jjk9j)2MbQg)P`wK$SSoF^Xpo_o1Gb~`PZ4i<}@;yOv^3HAIy=Ov$I=y z(>SK}d96ClRL2MShDIG6PsdorVw4C*I(b^%FvEUdm+l8ra3ivH{nkS4o)wkT+D&fmFauEkLc|oXf@l-}!i* z^s6bR<=kEmO)8#iXdYjp_J~XldB{U@)gu!&MDM(`Q^&QG6_CWyeOl1(BlPys7#{-W z4HYKy{QHTCVuzD4n3FF4g@wa(t^Pgv=ttd4o5)%JMAO7gz_7m5qEhcl(tqay(2g0g zxkaSZwD>V%Zj&z5{YUHDU<#QZEEWnh{+4VHYSv;aWKLlDAX($~_-^M2y6gzO06|U*ETQacl+_rjN{P$G>lZ-R@tM8~O|JfwuVn4TBboP(caHsQ{y{=X%UG}-O*$*=o z8BCK9m`L8JDE8B7=X+&iT<3c(Y1RBfJHLFj>$*=4I4L^SZ9>p6STPZ2KCdZzl`}TJ z+r-!QuB;>r*dGHtqMk>tR@wiX45~K6njYggXA$#gI*b84SdTxg`(FBN_4pa2zub1* z=8FB+(-G6@yReQCvrs%$CctcWAZoJvG#p*rsdK4hEMd^?=z)X&?5f!{AnEaZZwX|G z_-plY!t=X`l^5>Jca{`HX{>#NrN3i3$A8HO{bf3x3tH=1waOLyzYr)rH%!#|Ir9P% zl~|u4sjEBH=QcYUu*hFLg7C-1%gR(@Ql|{VUMY8s?fX#D;bnYTFHUFIUG&$nj8B`= zy2?WiMJrwRmc%K`VB3VwMv+X}qFFp@&v`31!#Dv(L&zt;Z7GrA&#VvBLiE zc~3d2ZzjcBL*Ypfukg{)^9eyipFpQNm`f;lS*AeV-kR&p0sON&SIawP;p<0e`RI9_ z{X6dqxwcCO-V;=#{@SyvWVn-0@YxRU7zH%acs+J1%MP`1GfW9pnL8x6E4f`Q3o1a1 zKnF;}(sY(r_eH0F-b}LC+SOB73}PRiAMXhKQ!(K*^P{!m`)@@Z&hIQ`%~jY@S=|%y zj#Ye{PQ#EZi_R<8slyhqQ@2y05c$wG_H#O&&s~SNyF1M9$qZ=agxd>#aih4Wq|hm! z6ukWPyWo+}fQa5}rSb@d_xRO}8Vszc6y)+kH94tBFnolD>kKE`vkr1mTfF4ij)l=Wcm+&- z$xN(H24;aWU&THpV4;t~JDmznJ8c$^AFSdjeL)I$gA2Y-*~uJ`=#&m$&yHGI;VyW>`;MC$(E6(!V>0B%F0J<)?@| zGHsG zy0|Oki}ZVB(VfN^X(=uC`fMu|wU)#%CdI2*?X*IxU!h@r+qiKC&Cb=?D}b$u&Q(wm zvNxrHt8_Ae`~lfB1Wr05`OM{dWZAyG;_whdQ{|XikF9ev(h#j0^TrEt(34 z&;2M(+9!p3&VxKXdJ}&A?AZ$seI(xN=RU1fGWTmT6xfUy;iIS5SC%0V@h#U0wCsb| z`=bZ`iWCiRPLNO13nlOba~s>bd5$K!%1CmAvUlq(4)cROYNT&rBprHLh(dk;CcJ}QI*HAL zUXn{?DvG8|hib0>;IMbV zOBN`N`<-|U6Krb0_Ghxu*kz)3b`8;B8w0S&BvPQsJ+?O|j9hX>|NV#f--^!=NZA6| zXor%=E`qXK=*KUsE6eGk*UVP8CUJdF^A}<5^#oT=t#{c{<$HfeRy0pJ`x3J9iplW# zrK@g>$=uhsWqS_+Lg57>mD{z*6RCrZpk+$8?$_R@7p`q?dY1C?vMxSJFyrG9^dSWF;^cx7CE+>qA%+(v?Qo&zv}yb8N|FFok#wpwvv|Lc$pT`) z0etN!w}9-T>b2yfBWR03WFg)D==JScbIiUVjQ-%B#Axo# z{y9cGV5MgP_-y`|M3tdXy{^TR!Eewq-XLRXF^*OpH}|}+U05~f{Sv=ybImkxWn@o* z$}-6yNUAp*q2h`zXr7Nk@3M00fyNraEV0>-80Qvke9_R&OgcTa&0)RIFk>SRt^COp zdKZs<3&#tbzo|PQsnRBU&a~X+QTh)>2n=d zY;;PHxD6cXec~(2dBlvC-kb2-e-DLQqXAi>q5B1Di@pbhbZx`$7LG(|9~OfPR#}W5 z8{(<&16A8EFkN7pM4eLv1K~Q*Is(77xkSzGQN|gfr&Ru!Vsf9i^*`73Vp==XAqw~I z)z&xk8*wb&IoCp+9FBe2;$|~D2?itgRw(YBE4ZvEBfrfB&mpXJa5Ld&JAO%jueU=0 zYbl6b{pWT3;XC_TqlMZn~Z5&Ua zXOZa>32$RG29{5A7vifpO)W(RSox)lO^?YeVjO!gXzF}LR&?hh&A_{0x5Kw_ZL%u$ zn~1kpNkmEqKWD8DMH+K*K4+G!*8FpiLMid}MX*78$<&p(r`sQ6DwW;{Sp)k*Ra!qM z((vsYt;(MgUm@(2GvXWHRfM_ajcIUd9Jtw&%OeVHvS|50Jr$@49y|b*9#x zGS;9LhJ}XH?oAby08~^%JiYNlIR0hJWsN3UtTW_%!uoWIIr*{vOMlgKZdzh19?u(IzM>s;aTd0?^f z;s>O!+<)NoQ(`ia_orB; zuTx5VdWxvGt&-YKi!WN?X5(z;p!UA@{pat^*H6K8-B%0a6^0}&X7d=Y*{ww^ZoB45 z9c#pRqeqN4>e3!o1<2mMEm0_fSWnV}lcQH5OQKK?&Yo${8aSifx0L$kg|sseqw5r{;EC+9#7`nKz^l_*@CGFERez?M>EMJOklVum<8@mWY6! zql|*(J7Q~Ttj2wnF&8Y(*%;5>U4&q4N4V;!yH3Nj|7jv(DRWNUe4nzDdMY*aLY3rL z^KC4qfkiCXoYXd^(-U)-evPoXKQmw*Df04SnzqGu5qs~3=;FW;a*J3eCq}<$kHC<7 zKd3Lkg$;a|(rp>T(8`sBHS#WKG~~k{j$SJS6VUNSr;ZxvzCjOk*WS)zd%;MV>gez= z3%8LO;YGON_pZX}5q(QYnwO}BR~_j%ACjH7+BR$XCS>`Bqlxl}oQ9=rSP~xRIqPV$ z5w6uz)RK>s&TbUWna!;SEsj=wZ>_{qrwIGSA~z{-VnT1yBL4LrN;&)pGt17EV6$U6 zdz)t#Uer_%hzbD92^&e5-4ltJy?PCuN0j*0jebe_{D`m}s{>s{Pg~lBI^7$dq~XbJ z6!t^~4wt)s1(zOtS29io4jU!jryD_HU9!OUjn&=@t%80AA}blrq?60cK8#+B>_79F zI_x1eujE13Ccdc@qNegQJ{ZH4XM3nSQQna+0Byr*ADUBH)p!x-bRG{stF-;N_Rnvs zZjEVBoz^ww1y?pbFa7KF~crex5aJU;TBzJrM>;(#h}VA2}pjS?H28OK@59xq1=VtKA;MF*`Vn`i#n=_+=QSCEs53st^FJ*0xBXH zyEVSq)#qQn(mn!$z$6H+5kq&*JAq}~kN?zRs#8Q24+q79JKTTTO?h1xxN>zp*q(%G z2X_@6Ob_vSDpy8RDc_q8C+sUoXWmWODyP=w{PbuQkPZ)9ANearI@c1g+^Q5?OwM>N z0>LPkesohPR$;~!xiS3~ePVnVH*F(83jC}`WkCnqLFCj$Jw%s4H6i9dHs_!i);t;8 ze?e2qz4543Z-~ynD985nufY$laN5MwqFuuyKg<`!bDs%P)KHLj33zv?t8iWghEeeD!xzs z7NqgUc&|ui2J^F?tEQLjWd7&rnRxiyAOi_;@$ipS#f(+u>0e@0X|?H(h}~afM=s_J zIa9)a??nQzbrp60w4(U4*GQ)TGFIHjKDBgJ&Vb*xhKK%J!=(gQll*J`lWSF;I_~!g z<2=(Ea+5i%7ZhI^NSsUH5Zh;AeqUz)e1*I$YkYHl)pyXq^-pz!xc<-KkY>{^JxQYo zVj>AhStwdK=D)Y6gj23U(tj%ThX;D>T=J6Oolx{e;n^T*pOI*@?Xo>vw~j{q+h{^f z2P*z+IJ}*~?uUT2 zJ$%I(ALP4goiEerAKbpEX#I~dD$i;B>myQU6E!85D9z?v?en|5q~DFM^@FG=X3F*8 z6Uurl552r!U_+|r?KCDfG1QUCK;!=(!$U&*&+oSdjx5nYY3yP{RE^AAFvFHx`(sbR zVfAO%CR}skKRWtt}J)3MYS{G!vkDOxk zUt$Y~IXz`veTdMMj8mUe7()*RbSKKL+wNSU>rrR&OHMT>wD}tU`8={YTyWZYAY+cFy`-@1&77YF8pyC?i02~+H7TTu6hXii# z$Z*0=WgR+gszSqqthm88&^_c^QYT$Ie?-j@1oa7JV2rIa+N>QXruUbH}w?JP(N)f{tzs z$GSS8ka`uKeZO#@i;Gb!ucY{Y5{I}#Q#|RZlzgWkrD(o1j(VFeITk~JgAaVi<)rN7&|}hZx*vYzQH%*?Tpr0$sZ&X;y%`Ry#t8h;PP`}!xmbJzcV&8b=xi9NcgV&Nd-@HiK&d@5pR_tJw%;vthN(1EE#WTFT=e-ZX8*LNdXFI?q z2q=M1L7j%D&uHVC1<47B=7zX-qg9}etG2G6*BuAkrv<*$7Se zk4F8xO?hOswz{b!DLnbkkMj!1#27Z$D-|l}la+9p_-KliUE8WZ{jVi@?jOZ~eOcsc zbRe_f`0WL_Ls;FRTcB!25N!UwF~rM^Sa9S|HC(M`u^r=ZDW3d8_U^U{Bd==|&C94? znu7d?WGB`*36_7jA01Cb)!X-t4H107hrw9pBauG(0`<}gJtyR&d+P?phTRm6ZB0t0 ze=fma$?a7mw|krZbd?#340t~y+x9GRP*4Sj*DV~ESfvw6F8dUf4Fwc9s$W{dSD zOj5pG(UM@>eJ3>`e<2LE$CCTYmxcmP0I{MY=*1A^-&F)?^s2f{4-s=XbqPjSorA+e zK=>}M;!lHKjVC?Om*wJ=0K65+R!T~ztMJ7(!=(i;zIYCI%-?c47IhqMufW2KCkqW% zgL?3X0XjI`Xw2d=Aw`X^qi~xS`q%8@_UeK{hs}>u%7i6anWzJ=`Q+Jf8*ZWd?#TSd zx-0xu(U5+<)g^!NCU>_y&{=_mZvDO$7*Yn$^@qA?e zQ_w8VqV-Vq@9RDsd$zaI97&W$e)oBf55&s=V`!lzwOEYX1YKDnRQB9|eIa@%>ZMOP z-wNuF%s=y!Iv!KgU>6R@SQT-y=7kpDFmV=m`0Q#*PusJig$~VlxWJzYl>k07pFt_; zv`mH7fo2#91*~V6(RWinx}W``v>o2}Su}mIs&^GMXi5WHJA|w@Gwr3_L3rJKIQ(Uf z{me|OzQ|)NSuK5;m5iV7pe^cB#}UCo($3JkN1qIqsCB(UZLr~&FX!w#bhe-h#%PP1 zWh(YfiYg;UY$DiMy2_{0U$`f>zpDibrN$DxGEjRKvOC_5^e)5)`hZNV{s{lZyAhlT zT?Oh(O=$R3KCARU-iiK0FNo(8Itm?yhgllVam}oM4vx4BCisMzHr^z|{WYMS?fuyM zW@ko{WQlhm;gG2SFShy?_k821_bC?FVB;qd8-b(-YMb6pbH>U;^@xd%!HoHiBs_E~ z-qaLW_Gc2iTdh*X`UQ<*W%?8(KYN?&mm7+bj;3BOaY(S=4Zm9N2Pqk1&5jP6s;H98M>pRJ0R=o)sI=G=hGDC`@4I~*X$ zfua~b-KNNhQ!EcRAj%XQ34_g6l89lTwJ5XoQ?QF3}A1(EsjNbEQI^e<9 z$i+PlHMz@Xg0FU*{M)W(P@dLUE54ep^V`mx+q-um&jBER5$X{}MEQ zlRVeC{fiAjdt(rlxd@{O5mcb+d9=o~tB7tBt4%?Q_2WA0fLof_vJzyXHOt=9qd$DH zKjUZ2`SlWYTIB7IzhWdf2~R!vyNx3o(|SOQefHN)fQ>;{-~{UuD)q(cIJw}Jz?Jt)2|A!DoIf zpV42zRbcw2SC1>+VK3Ut*4@vLdini+71&3cF7`Q?z(0Vx%lo&pkmu7=ulkbHh0VN^^VQQdtAWn z`F=$tk0f93+&`pa>Y3S*PlO?T=<04>MNmFgetQ=fw$jxSMelOB_2S3AR&?zrT2hu^ zz@(|^?b@dM`fy`384bf?j46&Eb=>#Y9*amAZQku{Ys7C_@n3+Jp_S%sppZ!rqSgln zqT_72o%00Fql_!z&8+8WF+dyi-!^oE*Qg8t+4|0ODajS3M*ITAwI9v zSI8XDa=x*^QLb0HujljTujAnPXf{cG0%J>Ts*GW`c|g+DIB&Tn9+lmh@^enPI}#Yt zNru<^bqvGc0E#g&&=^ME=+*S3-yB~w%eVCj8_K!sN2WVtbZmf3`wE*J5aK$z40o3q zFkF}GW#{yG9whc4G|G%w1C!ic%(sxnZP?*8U)}3G0F%GScR6|sSGjp4y$aHNoQoPi z3GlFcc&sZq@{xJ2|3Jo`%^K3)UtU00TaKGa3#&q;8@4F@QzdTcn&Z2noM*pY$Ch6M zXJ_T;J<0{7AfXoenJoxA(kVP}=rp^O$*(E_J3sHC8Qp5`?qCDgSLlG-_0QHBpAhOj zP5A0$tH5#j4cCvYOq(ZI8z+)cp6;Y>Gku8{4p`Q}>Lp)iLxO>gq1UHd%fRccW|$SfeRx-A{4-Os@=x?}5bT z%`ovaJ=k{t4W*3})ZuFP3Y|c_2vr>uV0F3w6_m~hvss&M7GOrnB%!3UH#vi{1e;!j z;>@Ll?0Wa^6)mWq#Qnhev99T#cGEVJ&nNbMFMB&s>m!=Qlfw4$nXsMhR;;JDDi+&K zK)+U_gqLW|w@|*_KkuSB*{XtM=J_GbV4@BrC00RakMQ+l&?iB&0*4)Om!Hw~*1F_3 zU`8@~$uCso85fv&*;dXv<9Hbt^X*hAm&;;PBQ_k+ID%=4`(KvM!H#Of<4pdQh&j=c zDRpU%x~H&4Yo5a-88I4#pZCWnyTWb^(Z7R||PT zfteEvbgMd4OoT{XcW8Ng(F5Ya zmLgRifG2)N8G^UwpI%{el384HU1-E@`fA5guLS6`9-#uvMuHE3BmhYsGaKy~xgL-= zo;}HDVie%RD&J^i-E@FfyII_7=DB7Ezih?r8p1?`-IUneye(u~NsMTm?UY{XHInYg zlX|>L{?z6u)FbC**3JvVSBxT-(&0Wcc$y_;7KKH$6t7mJ$ChY2-BAtD|%u7@g$+2OQ*FAGe!iMjSxf;6vWH_Qn=0&muu?M?Wm z;TlQVBewhp2QgI^0WaFlCZGS-<)^Nk?AZ=$3%X;rVoRFbFiUt0W4UzH2p0%qVsz?W zK7OOL@D%>DduXs4Fn3y|klcMU*++E@JUR-`9BzzI+&brs!2PKA$}ZKLGn1ipowq&0 z&+joL)SyWE9IHUL$j-&efQ9>`#jWz;J^}*q*zR4wIl=cScv^&s=B~l;Wh$y2CW4{+ zfWW(%Gvv-Jn1cDf?h|*OU2Ltfupeynfe+46m7e30&h_)P?knD1?1+3S68K?jzsIm0jb>>x(L{Td0qElPDLBG?L>r7RakmN zev9EiMRUir`z`*WQ8tfAJj`e@D185Ita#Rl!pG7di-)CK%QiYB5qYv!bHkcNFSvco zojF*_=;n&Y^I3c?i@F1u%Ym8cXgd;@ytsH;%TSJv8N6r!n9D#UrAw>=WpWps@L^Hs z`td7|jetz9{bz3e$mmfBk0|+P5gME2GS90lZE#iNq286E<$u}CcQNu8$&jW zHXJ?fRB~--O1Z6NFUBxGA11fTj4`=$imkH96$|8NEDcQ96I2`D%fxvP>wgDZ8r`aI z?kKH(^VwM9NAhL&*Hi%QVbu0!SZ&dgB|O7E9B?L z63^zE2hc4;_&c|HjeKa?a`BO!quZ!FtTwA!vQ*`$a5AMH27-cX z;sH)Pn=Qv%2)3=2%(jPPIhSIi-c#$YA8#MwMT@xCh{8#Gl>I>}4P==^x*2*LVFV}7 z2~l*M7?A*4_KOm14?lHAWCgai1E)zJ*Hrj9ElU(p=Cc<$qvEnORFyW^i<$qr^iQPw zwV58@#GAJ0F@py+ly90GdVcy{u}tTc#{dSmo^8oCURpjc)dMOB$)A@w6$2u{k2VJSSNTyl-9x1dz_` z(e*v%8)@_noCq8bh`;DhZ44AawVp^M}O;)0;txQBPiWXUrzY`ISwo`B;4R_ z`Q-?}GC4Vsx1hjSJ?A-mPo2)b! zFJUOq=?B?C0POyG4)dq}KzELfS1#CIERE}GZ3$@4xiwJp;AV-SKEN6N2~qi z>V#2BVUXb4P`3vcyQW<|A5r@3vx5W}@lCar6&D6uG%VDfS1;+>G_9JtJ8V((OjLoZ zr{xNno?6QHgG_SA1E0oNnS++gc!28lgZzzg*b6sX>rx`dqMGQBNSYV|wAj1tz|7-5 zNw}WfZ#$NoleU&k>%5`gyG<2^t(@`%y#ur4+XdJxZ(9m^Qks~-9>+gf=aNldpk_tE zwzKS>nyss)*-Z#hXKI=wNlpj*f4)W<13OO;cIael_WzY^zOE?#r87rq#~ty+drS(m zQ->`_h0h0bU_2nmLnVHIRgE?z@t>XQ)*3Vby-n+3hi~1oa!6JD&d0hUP$Ww*BTQx! z2#5fANQ+59{#kx6*pQp;9DqH`ssR#%=^bX*8%wK!{>&+9ndvy>8yND)gtS?o*17h4 zSe3>eV(p*3e~xJgR{f6E>7d_+HAiS#F8T_L)odI}G`V0)=hOk*-)qY6L=6haNH5=~ z-%%BwucS9lG==-!8Aui?Sz29@_cej?_Vs5_wfpSTx>J&$b|xybO1?ReqOEFNg5?bM zJve*!;Fp+zyu134lKwV2K{*UhszGS)zg=>>$M^vPZYziC0% zb_{{rA82R!dSewiJ7OS`Uc7I+XhgLv(y{#j;VKlzt_|~DRl`1)CE2@r+@`A?4=H{T z)1W{3;`8rD>OftNN%65PFVl|c!iIj$@Q8_l-`MbnW?T{5j|%!PBll`1k$;eM@_}QB zP+|I|wettJ9(sO5igvcX-TJFE?;F&8-e#H!gce~NzIS=`F?F-EV~i1IM>9X~H;H^4 zL<6n7|L4wLyi#S>We%^)(TV}sol zhFbjUP+BtMm*b^Mhf}tpHPX65*)$>OECVK*2eQdKWfJM?02SPZr0%>WyCe`DBXiWegG;-Id`zk<~o=%(l0RY1>|Bec-55 zr|X(NOyF66hTy^hKgB}2wI+tOl;Fs?ik~yrN{<72;or@?)HdT+5iwP!E8Q9ot}g7^ zP!IoWjkb^57>gE>#-~yCU|Ilu-w|%mrt+Wj+U)-jvev)#I>GRx2zn;Lq-l8^=*}Rg zcJ|sg^!BfBu;kMPj)*EnF5(d9Z`tgyFPeG4`LGgO;`&dzs z0W^zhC-=%PqG?at;#bedkJN0gIZKXl71meM(NF3)enm-|*Xb`_7ic ziDYkq2HmTY_094+!p5*sy-GVAK1T=(RzZ7U1#W?u^CDU0!)*%yHsHFQq4I_?)^*a? z$X<~G!aq9HOfpf%3U9C!WsV`uqEgK$II`Hzz6ZZOo#SipVYO`j+?$8{sXH}hGjQ9l z`ZC42)~QOS$S@Hs_<~S$_HT21Hu%7aGTm2yBn)}Dh4{A7QqlDMf0hKYLVHK z?Ecw&KdIbbTViq=+G)t;TTbEHp0St=drsAHR@`Oiv&a+~3D(JGKkNP}xRlQ+G{jSA zFj(kP<|%}|bpAejWUgzl5B(=S=OusE!+~X5zQ$mi>dE>Nhb5tHrq6Wv>cFg6r;(PM zy8y1aS5?9$*C7PE5I8AyyO;dfr}x8n>x>n-aXQYSGV-qo0qUhZ-Flf%FT2 z_EEH$adVGvQZvGlM14UpY7wHt?(Wq9xaN&!F`K``ogHv;LY9fHlWI}Paigg;TK#dv zH&%Ucb4B3B`{(ErB)P`MAlNG2HeIFg&2!Hm&=a%W?k5x_R;3B4rbO{La9ah=Tu@O5 zp?s?@&DsCQ-dje+wPo$UAtaCl55X-25AF^L2@pKEySsYZ)jx~Sn^>nLQi^?c~)%5Vdbvs^X<&aoJ9wK+At*t+?6?C5Dk^Gxu#w3ombp)n8L~ zYbM?+zBTVjEe&|_miZPa>L8*K-qF59!5Cp~x&lfKvCn;MR`tJCYBVFC_HTl?QnwiJ z4l1GM0P06M&<63PS!u@$nT*q6V{s!ya~|=U{u<-s-FapCgl#w5T(2B^`XFL~324k# z@hL6o%08J6pN8#kh>(yWd9^`m*9xv*5L!L} z|7x?!_k``P8RzR^C?y|PNZjTNgBD1a)iyIwpE&x)Wm@!DoMRK0lZR-?+Zob26$WMiCS52+!8 zJyt+=CucJcJw&Pm&;!@r+^HupyY_61h`BvjS(*g|SvwR8UtRba@!d z2@QyVOV@15e$^;yq?}aKPIaPx&`a#iqCS48%-&V3$XjV=gWf-RAoS-lM6~7i z47?v76%?+jEj7_mE_kbmXBC;$t7;3hUP5i=$ct)47YWz9WEMsEw=BLg^dE5|6=E^+}=S*|UR9 ziP8BiKQxvvv^W;+S@dRp-&pl}GAQ^8J_ZcX5fqOk&sIwVsjBz-M866-F&kU2suU2B zSb7gGmn9DCiM=not?BG*JaX-vaAyc(=${JZ&l?QRMSd*Z2Ia>yhIIiA!lmQ`+Ac1dMfa=m?rndN*#a89L(q2)^Fr6w&2HDIvhTD52B*^qb8 z5m3mq_oBxtoM&+E1(EmGN5y(i=3|d-*_4Fq-$uYT)Vmyn_!IY^At>w>gc|VMSysb2 z{B~ACU`rZT{GH89$dOp^Z~}nqzKZWMxGj)5cU^6Fg?kIJ4Jbg!_whQuy;b!fne8tY zPJph2qJ1rm&^zf!vMPx&Sx5l}^QIW~=p*-ClXS_tOShjgN+s)IYqiu3@w;v;<7%ZjKg>^#jbkQHe_hw zKjRO~N#as_SiqC){9#HIy;KTfMM6;gUVX;c7g@ZQCTTmc=Ug#$u6p6~kO!Atg+w|M zsU2-CN0NHgOPP|{cC3*`Gu`KY&&?(pNspm3xk2n!@rBdx+trmBBCShYj5Xq?S`Cm% z0k%93O#y4ksUpHb$ugMl9aRAUG-1gAty!CC)VBE7G`1rT2T99rcPYPbOC8E2r9bdu zuu}|JZUZ;J3m?C!_x0rDTh@wat~+iqLuZ$)%W*_rnL0C1H$94y;!*t7h|`7l)Q$qp zr{GI~^2)NKuV))-mjon5y|H-Ye7-P#@@UaO0M`dDgFyycjrVsKXC*>ACBn9*n=(bp z;v){$X3CJ>pv%T&PwTkjhc_E5Hgc7()!-=wSI|z-L+s82z3sdw=VHCrYLh{f z!2tF(`*{f>ULYg*I7$x|_e$mQ9z*W&9>I0$y7RQ?GrT^n@3~-_ty(F6S7HvVokAX^aoD~)Sk=S#> z6Z`nhuQ8zt`4q-vsgR5wd1m~{(>$@*kF(wjp(Osl4>aVLNQh}34Q}zK-2iIZBJ=gx z)eD3!`9H9}r}9lJ;>DHa9$D%5xQEbbZV82_aEGGn*P(QT#;Il39qFw+(vJ2}>w_EA zPLxBoy$Iz88)efEjF;#I+k%iu8zny&_2gTKeQ1oX2@!QfckEM)XR*&29MLXf^QLK@ z>|#XnU22{jW_AW>RMD&7^d#F^C7FKEm71EteR^9S$POaogEYoxH!TcXhMu^<^jt<;zxKfelocSFMtNZ1yFX z2ziW}GU+J6QoU})J-4TIgN=jB9a;#GxHN;t(+6yxB?7-mR5c4_QXQ{1Hy^K`#)400 zS8bF~T$Zz?YCSwt zWd`r87`T#F|K!E?M^vDXCG@_56_C9iEl=vD*Ur~|O>pU)t;7O>6 z>yucWP7_sudWPn*535(!iX?AG6Gk6&?G8GJ8}BaNLF>hjZQE2io9U_Q1}LM9Vb4V( z7Kc>XtqD0Sl3!IrLW~uCZKLVod3WVQK_D^dJtodlyGk>BQS#|L9vQufm-n0T(%}2z zfomVO4Xpk&)bw11jgCN3lr+O_Gl`e=^z@?NOHa7s-RAwl^v^c`lw1{5xU^3l&LS{? zKonuK55YX9pXm@;r2Qll+bMek^obkF?#Pu+#uB!z2^H(r5%;;u#LPh`2;Wh;{VF`; zxJ|m*PV>QY2a<2bCb3=dNVkuZ$(JnVBSmR^3moB zLl9>l*>43k#*}*Ij$toT+R0olD(h$H22b$TtWq--))BW33(nr7`;zunfJ^-hcU#xG z=>}9`VZg8z=`Y<9Nz{x;O-b3WMW_GNS&lYaOr3&eiq*}(awJcFzItDYn z6OcDXLN|s7x-Sk$C@7QKiThe5z8AgK8sAc1mvDLD4c7J zvhHaSOElF~D>f$mBwZ_HxDIH zkrdcufN$zey?zNKlsZqQ6fSJL-CYr~9*>D-d=>qm=>AsSLO8V&LNWUEP*uEf+WmE2avlpQ9 z16rZUd5e0yikiX8bA90Cy|f`o&{nd8ODz5A7QqD*k0jM_#je7IzQX&7m{Bqh2{ZQk zVK8Yy#}t+qKLG8Bui>^$x=BRs$GmNBs5YG;rliriVM0q~_V!jKOQmnw_-=Pzmbo?H z26S|SWWi4?(8%~>;3kJ+BeqJJ*7C3^-8iw@-v)TgX?4N%e=j+k09=46;D$$h_nk=f01&E=MVC-2VqN(~c-fwt2U z0Y{#pu&||24U<6)^Xpdd`&YE0pZKz*Fn*0q{HR%iJw(fxr5gEKz0t{Z;F`LwkXQ<8Q(q%kK&UGfbeC7_ z5mD=5&!kQVj`ns;6G_;$L|GNhCIuY{FNqDyZj-*UUY2n&={t?gc3A?X{L-W-oTM%6?<^0j zeY|x(mdi8)ms)<_7V^GbF?dtpo{c=63gLy6O=WyL%m@(p59O(q{+m5ejOi_N1P$7-NDZSmh$!b zLK1RP1lfcMBSmv{mNa*7wq&o4HGHJzonH-aMp%UoWLzT&jikuafNM$u9m-PEP#)&x zDq$4>zK&Z_$CWsJ4pfe3vEgNtZd-xzY4f*K72!IvF(fPkg@BWx@Lk1xxBKWl^;o`$ z`}5^qT2ZBqpJ^UPcP^C$V+-?tCAI0#-FH;pq$P_C7yDFR7DolIi5fvwYJP@yhDP{@ zss4%wvq-=U2~riC$Eu}KG@M|;g@fuRVhh+@K;xY}a&&0RwGo=I4*-h7a>JPgIhv*= z#JwK>9~Pkar+II&C7t8bytmJ)l2`bX7hbof~4XAFbhJvGZzrL9Plgqnk#TeLGsJH4=F-%5l!s zt`V&wXnEGLkMGc&)10RvUS6W=r~_hH1@g4aMo84wSt4f-6W; z;f@NVL-Su?0sVv%mE(=N1H(7|#G@5G2Hv4g#EalEXz4R>eYmoen)5oB4Xq(US0?WD z#fSS~Tb`7m_qWcwGQpuXYT#ibSrYIw;tu6Wrz7W@7n9klh5&C(2^(GosYaI?>-ASi zh8w%pAYLsVnS!Od$Ap>#%sMrd$EL906i5eRrP84^qz+rHnKn^%L?dvtex|RBRp-iZ zNWG3Piq|GCyJuFlk)Mw~&Nd@Nf4(br&WW&H>~IP%X?BZ0kfo1!! zxh3TL>Oth1+@ucUtD7@d1&0O7+&}CQ__r(4W9er$0q?*4pXC8Q3QaT^ciLI$>5Ycl zBj_VmDr`4-?oLk-b-AQ*hPiAm?$3ljg-xfjcW6YtvSH}mM!w0@;W!SDb@(E9n4hkDFwU6=s#C0V%u~-woI0*2SQ;9ITp&0dhX{r zW>4+ui&v|SfA}Nd+3j#dzwN9`>;by+i14cvUmVMZ5qXZyg0U`t>UL%ehko3DbM+-m z?Itk?kxQi*9slbcTVq<87-m^!ODvy`s52Md=5uOH;Bcn{p<8IAW? z&xMujZ>z-oS9QMZLUy%2!TN{E)Kd?PSKeRbo4Q~57P}GtNbc-a%{0Beg^qIH$$OIs zMFv)>R|HXA@R2^4z2`hPe*+kZegjyl!gPRoEo#x%gIzu34Nu>f59w%Hi>u#u@zs0X z$o?fr$|F0=Mc*54c+~OQ$Q{a8U^zcWJq#&PtKDv?S2+q+2bwW+2^7v`BOu^DBf3+y zr9O+X^Gi4Mx$jvn-J}h2<#<>j_*C|c#um*pVBBrfQyO0$&vqDv&j6=f+U3dix#N|Q z!Sf(p!Q*7uYq>=g#3!wl@w!#bJOV-S)Liz$P>xt?Nj`j@Bp?LDs3rBLY*M-cR8M{@m;GLBj`s+fbU8G$DLS6* z^bU`Qa;;fZXi!w*L`i$4nXKWFb%cd_iL-l{$@FQl^FntLhjnz-tXY>D9-Hp?l-hS* z{_t9{YP)$XG(5(?sUz(js;Hwm^h+!7)!jg7;i3sGxn}cM2FLe z6Z9>!yy8XsG+Y1F4uKw<49nzI>5(@~Dgqjc1wkWf-M!XlxgH50N=*T_vvvQ%eKd$q^shoS1cc71}24VC*~^38Z(+XpdyL8(1hdL2FOxAa%H zWqk_n>4W1-$lJL@ew$aL9yV8$2{MnqLoMtV@J4)`jW)Zq9Xa1WHG44u>w-D8?Jiv1BnznTyi`f& zrds3$w+rs1KI{rCdz5m@Pugb1B>JuqeJ-$9+&w*{izF40O_3w`sD$3TV4#;@YSN(ICE#ip zgSc^msX?{f>!2dF{0sU^FCC-!wBf--9YgQF`aJ^u9+vx`0JFF3e2j*F$X6sUvZbAf znik*5sJX!RQr=qXddFv?Z=PqnPp=A-Mdr#Rmzo+2*9EweJtX+cLWikCP^ekFB^8(5 z8wfLkFgVFP9kFGvgO3^Ev&tX&^Z(R@x0bq7FC*j5(S@K5_%T`L6po!f)RgI}r1XDIIQW&L#qjH7H& zlNFU7p_YHFz$I9UyN(8U^gu)Ku%?rCy?&W@+?xBM!9P789?Xq)(jpGu6YEvIouO$zYX>VXz#AyQqaMMUbCrjn!A*8E)6 zoaER2{sf@7`euyPS~&<-=Drnw?C^%BI?L^6Qhb(JFQ)Ce#UV=Lr9^>+eU(T4D;h2` zNh%0xiFphuC=3LG1#sIQ4Pl=g+wXCKKCFkQ=LmyMrRHHYX%Q6gkzefyD9fd z&u++2h@c_JD1w6>d-iN=2(@_&kqd9&VGdlZrF|Y&Uecf2 zywC+EpDhtyWqt$is4P?0r$NcptkHw2o-eqP|*E|yDZKCHl%nA-f_DbjZ$kAo*uX#q15k^YgxEs z(s|S&dxQDs0tMZ1xC(-ESRX%7;rAbPZ<|!HsZv78-!10QuoC)Y;&gHovz^$5<73$wiAmLt%g=f28)g!$uM+RK)|tZ!`3RJp3eEGST2I! z(eZfWrq04X!Q*vLTeVP+1=MGL-LsVEsrm{_Cc8+vna%hM;^|doY(`9f$b;wH z9~JhFQ` z`ihg+lOaLSQU&Ne^!iO>re*me?9NuA3EddVfGyEgH?CCiHWZW?0i=P;%{TbCN`=?g z0S!TJNQ$-2^n{|SAG*QsFT3@x?puS5Q}?$)&89!DRbcIFR$EGPGMKn3@)FN@#j-u!TPK?IHPc7u*= zpnoZ5fA}!!_OGuK&2H(|{F%cR&za=I6OxVa3PZq?>GMn>GgBA@2y>n>fXn@9E?eG} ztZDk@`-`(#IMF|6_($9;CN=B+Z*u?c5mbR8&ff>IXuG$YW^C9PK-)fjS;~Avv^ND7 ztrh{dnyk-Qc!6u|y{(lolNP%SfBkyT|0QEzH`ysrK@;%%r~nidwa}?n|5)Yt)HkcI zI2V_j`=2w+^{>+HVsG!VE7|6;aO!xj7fMQv`KO2!96HVXciuv{R;IB$RX)u8Pa2~! zwkAL`W8ytgZHLb+fY)UTNV-y=Pi83}y0O-=*-5!<7QUr^WEX50DpbG{Hm{t6>ZR+bHol1t{2`a zO@z`C1uv7dr-NPtS_Ufp=Vktn@#E4v7aXj{8CZ?UO^U>#&UByo=_w$H z^Vd(C2ZFgqJVENnbyyWWk3MtUN0bPxnNaeTmL5Syzw{t2$2ur#es2WEsX?Mmji&?c zM%&_0?PS=LNKdoQ_SQ}lraCWX)ID_&JqP$dKeb!tpP#BOLaTY)`7Ap*d3c<0Vb1jw z|25sJCYN8~wll5n#}e!UeMXranw1@rgD(PXCTMi-wVaF+0wt5S^Iwn634XqOhV40_ zqXO3GUr9BPkgi?T%_6+?yvX$eR=W|v^=VlLp0T_nJ~Xxr-}-@FmIL~j-*bfIv3#{u z*ymf!GX5f||KsvWr9IRLpWVPcwOP@A0ne%;8e~LW4K1psdbo7q2|#S0(?@CX;I6c>hV+x$o?ji1yaN7MQ%@%9L(>~wzn;yA z2J44D@MsNpdvCwEe^z5T z{n?aJ9zhNClJP#9G7v(4mPA?p5CRtKt@@{k`)epg_rZ0h7%CBf>W-oG?^nqamnfOb z<(n+Y$)KqsqR+{g!VJj;5zaTk})4gc4hWSubuq^yK)zoS#v2AKwQrrC!^!)$-&ymA+6a6L*5repF?M z$SjDMQtS&^Y6~2(d|XqSpFUn`uU8NxtS=M)8mI^0=^AKVn5%RmL6*hY`IXoY8cr6K zZn2IIBg!;B@KjX0ZC`Mrw?;s{pRdhpyVjBfOv*&ezLAjb&qC6yi0Lj5h-@6w5eHLsSKF?0S+V2?Bq#mQ1eQtup zYIu9P+U>RJLt)Z>He23-`#$!_@Xp~W^=b1~jiL}&Pz-e=1L3GhWuAft5I0hjXJ@xo z54c|pW>~cE?I|$C^jPT|_O9p5_>p{pO;IA;Rn0-KPl9wddvmv^CH(5(B{F*z&57Zb zIJVotG!?PEho=R`U=1h?xeqm4GpyQ;2v7B3R&Jl}icxnsTTW;rO!q1pl3p%o>m9bz zwi{9p)JB-yxXE(GpuUe@d8kf@5Y0R?k$SA@G;uPAL8$szs?z62s-tLFVP%2D{f=&o zN`ONYla`5Ba_!!y^g*0}c$qB52I!?Jft#bub0*^N!|cd6&`Z|v%!Z&BiPTX)-_c7D zsnxT_+N8HUt9+sK8<6x8J^mDX{pX^nLF^j`x>stgQc!(Is<&IA>ckc0(f2 zQ_9aCvuUxH&pB_-!w4Dy$==xwaTA8p7gt$oCrEMVL zN{uHdy&wMi7F(M3=w^Na+4eneR4{(q@{UlPo{7W0TFgwgpF`Ow`=68kKgM)+qwlw-B*AD1?uH!`Tv+2omZUI}p}Or7u#lG=@tnvD1s) z^@JP4K;Do3N!m2uuF(oi{voX-r^h$Bk-rNuZLl`+X`5a1_%-vY^h;uvzu9{H0|`;H zg7;dpXdja!!`E^uTmeMCS0A@Nn4pAm( zo~!11>3R8{;YY0vN9}#v*3I4`sa~E#tj7zZih?3dHPHOD?8bMizO06Z{l5qK6D}#9 z&&F_!dnu1SV(Xw9i-6^1mFo-Lwpwnmtc%!pem>XN!*5EH2&~dCiK<4=BOXWa&-bu7 z=+c)ExFWZhQFr18h$MJ??gq%t>>r*^?_I$YS=Bfc?^AqFP*N{e+lZ za#9K;I#URw3C|GVfy)^@Z(w;rCzc{3oBf#jzxcBUl(&b?2`YZ;s(LEXk%-A&r zYJXbsb&?->0wm$lmN&-*=u7QtYhLwC_=uRacsJNpF6Jgj>Xm0GJ2stBnE*cV@RNHI za9=*S=F|;;3QrYE>YnuaXuydINJTa29t%WGt~}_hvUdqThm04sNUTpt+`EqTSqXSA zqg>BZng~1!okUoR{(6a;>9Oc0Cukx7qFZ?g{fV6*>EyY?ZWC_vDgL`HDk95KZaPnU z% zZx|nJkHx-btV#L*C|@#{9~_5qSW|$WoMt6>W+X3zGqz_mG9tfQu@UJKb^=dy-#LxU zqZHHwx}>hXN0%>3iHJS9kHJez&zTPf2p-YQC#J%>du-VbgH)3|?ZFCEwora{n+uFL2gREQ{-LCqzGuyfoy@D*=gR zaNjQq^DCcoJ~kTd%DFM|G*M}XoaSHs@~uJbf1(Z2*CT}mo*@c1vFyHSCvcG^?1LNt zoAES?#@cY=D;gBb`ax-&aT@8V@|?q1NiSF!+=7*T*-}YMtis-4K^PB0pDL>Z4{l_y;j}N@ewFFsqCdqI?)l+RS z@?NlK^5|CCkAXQEUAE&nDoP!Sjqexb2LE-w^;Ji!ZD!ue^1Y2Ay=b$#l zfRS4|CwpjshhsKbem~KKYLdPA@s&WtKhA@kT7u?>0hb?n!Y7oTv}MuJN)OySjb9fB zD&J023o|pjB`)ma4u_Z&O{I#9b9DAqsT5e>_jDq14%mnOP;3M{3`RA z6Jp1h$_uOI2p1gUEOqt>=gf!pjQeA2+YF;EW}^On3*k_oRNSen!6%7y!;Kq^N$uvN z8g_!B+y!6`H>py&B2@uY*WR#bEWDQ!_+^0W3alLmzDY?-c_JbLn-pkEjW>9u-)HJ> z3#l;8CW`MudZUj#tDC5w2dNTtia4eU!V8G*+n7Xcu(*(Gv~6GN^ga1DR!F>!WIdr^ z<8^{5YgaX&l2>@}`9 zfG~Zg0gYx{WngTv$(x0z<*uFx3ucpyQ2hgUaD`LrH<5D!9d<9UyWx78AZy6vTx|Pk z7X*BX03^Fj$;!GkBT0H-PC?jk-M-WMT8>kfv?riZfro+NooWlx{XAUG6QeLOF#L^* z>hb*o-sl=M_|EbA5?jrFc3VvxO@QKLbv*dPKn{lMa`-30h>Z5i(a-95NS+@ob#It6 zD@HtDGFE$31Yvy|;TJoQG3J-XszgDl=Z_{o1+NzyI_hA^E z+O`_cO}3t(sey%qrH0znN*GuCF4E_!B$;5SgD^XdlU338jA~e*w&Gif z!HPQ_G3uR^L5<2sO9ibDs$l7#R98xE^3E|cfKe!7K^3=PK?Iw;|Lw$Tb(5|sG<&eM zg8lf6PrW{#dvZ?r5)=+QB|GO|3d>(E(L*cfYSx9G#rf709|!9T_gqyhZ^MqcCz=hg z%TcUHFVSvU*Nd2M?!NQouSx8g)U%#L_5{XjoyvV7Z@JM$*9Ui!|6JGmV|?GdMQA;& z#-5pnOwvIDPH}1}>ZKwg%Bz4Q{7D6w)asZYGg5ws}nD!wv0ZluG8{|YjyiPX7ZxVwbTc%t`K2}beLyH$Lv3}`3nC}@i zR9j-%Hv>!Z&wKphpwAEBYJd}m$t{(^<@@K@%T{uQFIlXKGx;M6#2?oHg3RaeK%(I! z^-RS>n(lw2Ilr!cySpTCGgiQ$(U`a)VQ!QUuf^-44i&WYZjc0>FKw4{PMn6V*G1|6 zHVrJ%-3;?44i#A&`CFdbc(s`^FR{{{%^Tbw8tU==A-`M+^gFQV^Y`mx4wU{-wG=uy zynaBJW}<@1+|t6FLt$B7nh2rdponn7BTU<-v&LIlu}7xR$>=M`PEHgO?(%b;Bjjkx z6OQr{j3sa+%X9ZWl3!}?g0|rGNz*Wn)Lwk|*l!%I4=SJ`eR;+f@@%8Y|T3pdPpn^zTH$(^f2 z1=R(ZUj*vHhaGkCm)$&xRy%=k6WLK1C^`CbKHp(T+JyLyoZpBAr0oy($qnASD)Ui$)2j^lAV<*qB0R;GBFl8;Qqo2^yp(s+an&X z4N3p18AiM07AZ*)7gLaSvb!*ottdZwUwub%X6#miH-I`VXhbkde-!AQ@=N4 zIN`lHkh$7lHN#Qnhibo!E6T@|OPYbkz@XvDG`@_;5IiKk1^!G7(ubHFD#qzpmo#mZ z?Mlmzw9*PeNv!cY6bJ@szjdqSTRqlih@gaP_Cg3o%c-G4M}Rfdn1+fRwpfMg^IVbu zGj(O7<&=3V9eaio8(V_TydtC@v=~V4j2PBadj%1?yn_zbe=)~5LXe)QgKK!bvDJ6B zJqLBoTUEi5Tf5%?)p#w#i1%>#IZA65z0%Z_(4+<#Nw0`h0*ASr?NB3b<9kD>#ox1{ z+MbvlIre&>*(hnXN25lOhoWn-=QI2lt|y-rpGv2G3F;Gs;ZpBl9S(1fm4AtR_)%-{ za!?DUF=Ln#ogsZh{Naw(?DWZ=hriM2b+-F;kQ=TBIkiNUIBzQOM!yhm>rtp;Mbmz5n1KIp z^s`eE_ntn1;&dPSoe5~YFwk}=>o4}KHWl(7gvBKZ$*~zZEVD{mY)XwCd=HOTQYBlR z9j$r=&kt3eZ_iO>G$=gI77tl#D$IJ?yIst%Y%fTvE z_B%gV-(bDnb}UD%OWrp4#>FhkiyoL`_V*jPmZiY?LVKMxLYQF?=G+*fK=I_-wuZEz zLYV9k0N5#frO1$ZiD!B~C->jr&1jwXayjt*SYBrV#8Vf5wL(&({M{O8?ldYw%(?VO z{Kr;4MX&oh0h?ND%!yW_2mpyWz2_UEr^R07<>b3!g3lwn*H&^Lw}z#O`fve*iO>xc zX8}>kfcAuoHpUM$vo`rwMoU49U;`%D|YFwiuEh$i6 zpPgIwYe&_0G4mgI0A7bBSGkX? zpW(9PgK?FQmt>y`s^`GtvJ=(Qhb^=~_3GY^)n+? zcsdP&E#JEY63)4y`}w@mZkOm9({DaNW=l+YtotIj6ZRt+5Bt=3ZL#5~-e_+|Q6_Lb zE~g?hkAq~zI}CX~zHL*K>z<-R)HrUoI%W8MYqeQFzXGg?NSM_UlO;t=ZYy&4@4PVay#kOkS-+0dASDWt^wzjGlX3ThGs0%v_I?{}y5&R->P& z^;V!npbDS3CPqQl(lw=UyZd%ra3*jjW_!J3I5%Fp?b^#?)ICj%9G9U3ZTEW4s@%uq zU%hR3VkSy&UIu^97h~M7F%$7eGCr%t|3uASzDuV=Lh(m!kVIxdpMn<)nBYv$RpoU+ z9+@dkK1u(xS{}{pkKQXE5UY5exBrt^(mHmuj?13gplE2Ti_O9)l-)YCCHl-kB$P1j zs-5zqI#a6yFUM!?$5HkPS^AJ?j_1Vc1A2g3``g7hPoI=7XV9qOK(%&v=GFUL6AtTJ zmjeBOD2idDn5I3)Wq_+Ed9b5sH-Y7M88|T(dJq&z$AF}Nuc&zQZY-` z%R)SG)i^DTA|=_Lf#6Jl-E}VQ1nBMR!G+%cx{XL^dXN1`{I)?OF1fqKn*{&P(j#l? zfUE9d*yHvKAJ{Vu$rlM_iLa7VS}&&*g0be{ItDA{NMRCJ}T54<_R$6UGJLI(%<|F+HcWWyAwT}5Uva3~`% zmA%)vilG;!-JKm@E~X7k(|)3pyO!8$G?$ymD&`K-r(!8agNVbyr2D~2JkbMw+WVZezIbx){u{~-bTVnolmWH z=Ce3f-QKLB@@pq&w~iyPs|fKk@xu~tzX>(B91xc?psV`$^x zr2h~&`X}h7Z`hFl{x3(SBLx*_UrYG$pO%E)fbdKtO1M+g1M*JTOQ+vcORIPIb}vF) zCNm@#b%ohrSDFO(H_sz^tbMc<`u<|q`^OGq?k0=I%PGcABWk@fs!|)oeNojMfE#Ap zA6v2T)$Qi}UH!4%^)YrXj&-1V>0CWjDK3CatPYAF_Ur%)NE!=K@+Co9Fm~HE002c9K zR8-TWL$Q!F)Amp7LlK3Z)M8n=4mC2-t?Z?1@*!UPqW#8GC6H@6Lik!lsw{!{#?Qfvq~Cu-!r&di{WIU@!6J@c^9`9C|K32s&lO-Q zNr@qk-TwaQ)^jps*Qo=7Z;gDlA;qe2OYgi_r&YhA9YPH~pWyuh@Zkg-=-x|5t8QW2 zA7@2Ue#-JK9QefdZBtm;B6GC9(W$jYT=TZ&vl8@M{3ob z{C|S&AAcPf<>8T)JIi)jG9AzVbob25rc^vBYR!l!HeB&e?P+62nvgSA+qB|BO=!x; z94R(fd@b24sb(v@f;}KJG8rbkSVBAk?@sh6?lJy58v~anFXb-;gS~zQx($A?Upa&# zp#Q*jJ!qlXUK?UIh^%3+k;jCuUfv>_7k9IDK7)A=Vt z*CpnfBlC1eR~O{)st=5GrZrDrSI8d0A&8X^DYqj;3KbI0A`iqH2LECqJQ_Pz{N^j- zQ@{Z54tS>1qr;J-$}~ULg)ccEM?7SyD&4w7rRqFUc(#H2C4?;&DeKy6_|R2z!#x?y zq}iK|x92!?oi^+^f%63I{5RX-Zsqj)-*CrdU|HE#%B1b#X^ON4T zlr5X}sL;9hZQx~-an+RjPjG^;WLcX394~C^yN%Zkh5|FRA>o(D1>xljMC7_$gCDWA zeZ%kdq5l_kU*Qzh`}VD(AR!1Uol+v*%_1pMillURcPxlVm!LE(N_RKAbjPlAv-HwS zH_N+zfAhTWe0`qh4|wmHojtQNXV09u>$Z^FQL<_2W&pn(>sl-&%Tm~_#|O4RVM z9Ll#>`IdazB}xNj+;(7nP{<~?^K)3Nemiy&k^X)fK_U-3>B*rj-ZmxZe8*86yI?V$ zqKEI7!Y1w2t!Kt}uB_k)Psq?9%$=^vg_c^2E7PTLRQVbbW^8nL(jz!c0bk-8m?~4P zPIJt@Yj}>&aS;;R>o=5T&(clXr}+RHDYf+RPNiq~hSUDFj}YJ-4;K|jMJ5FKW0(L9 zBhiX`f5M5bnyJO&k2lb2*wQt)GG_r%MyKXE`VgCif#ej}SYwga{!Z+&f|HvnX24it z9TLzV#*Maf&`}uuQheUggtPDGI(5Syj`$b&F08Ni-9M0?-bfp1rlUm)d+a*109cnl z6DR<+IDnd#;VEf(m{=@Riy5c>7)rmkhSO?=2@(BFzNl3}0^2@G6NvNuglW=W#%s-4 zY{|ZcRIX&m`*cwLKKcr6S>B?f1IFX>EB<{7=}(Vo{Vqt%utsR+(kaxCZJ_wrd{Cd% zji2|saFe;{H9j>x-6Ow2xu+9g{B=chcYVH+T9mL$eV~2L?k8x6p}^C$P1%$`4i2V3 zDF|N0Lp5Z$rx(i^2PnM?$s>vuT*##?EiVW5>(za&&(`Nv)#My>vKExH@(l?P41sD? zceXHXg$f+?!Uktr5YM+3>C|mzK~Tu{vyBXXe0LgPn${|Ex$@j&dG-TNnSqx6+2uyr zrS1b5?ti6;|M`xyP3b68+x<6&r)<7Ydu?`kY?sb}e2R&=&jK{lyE0|48pJXv{$6j0luUkv2OF3da=7P8+}eAl=c4uqfWKclRg<}VQ009i=1$x?3?Euv!#%gY zkpI13NJ7RBe)J1%o*^+#dG?av#)tJ;5@L~j#QwOwTBKeTm?1IqMr*96@WoilL6mhx+E9mq7hQ;6K;ld#icRFWE9Xk_>1s2)_ETAm1o>NgKJ-7*~0G<@%8 z)gySAS{UBwLCbE9kd28dRe~&Y`-^?MD==rubQ7fZD7yS~I&Qe6E28*NUd-?XuKdVL!i<#i%|mLVL_OpvU0=K;Ds6Bd#6 zGSYrveKEv4e=~+LG>@2-%{JN=R}BO??x(f}yBYiy8xKDBNLXP$eKnUz&A$CwCL*Nv zb$t}gTV)#g%obj5KleE!rn{6)uyi_m@U%JLXt|12G~IKoJ<6wq*kC#=Ljbg8ui;wot+ia+<&T%#XWnp#=KE8?em7fI2qw*!?u~OKww$sJwxlY@GUdBqgHrw*zG`>td?RZK$Eyu3`oY5-o1}b!@9NX{-6_ zjm5rmu6#j8HSPt@2Co&<@|N{aQ4kzwI_V}gSsziZ)C@x{`Z22lo#3$JHE)KQ3}N4m^pwLC&j4n?*9H?FRT7-0jT;H`Q)aPSNmn%fg^&5G zb}s7_d?Ln>T6a~>e9=YBA(-^*2~yhDlF|Np190gjhx_T2^mD6m?%C2JpSK5P*IeB`odAiY=(|K#X z;eGzHVr_xdti0eai)2nYp9k{wjuy8oq%nDk<^bZGZMjjNCbH$_iNY1fZcu7pKd^ba z#fH8)HYXD>80n*x`zExm#)?ekvT-z=c9fN@ zdVbz`IYmV4u!_$Own@E%$?AZ9mu33i*^`^?+ALtK=GQ6g_9XR{GduHz4*uO(03wo6 zk#Tiw6|-#rhpshma4jr)^>*cpo#kNp3qc5X>RUq$`CZDzDF88Xoi0=!syY1M_``Wf64YDty5}aivGUcz@Kkg}Kz$mn!p4EibcK$ok z#EWO-7}D(Mj1P>u=IoM8t6N`Ofcil*?{;9!g{Ww!vmdqN{H;dio2f6m@|s73oZYt* z?Qw<@E34J^hQanzPtJq{AjdS+6|JxzNzFuN_p{$wI{23mxH@L4sy#B;e?0*{AZ2}0 zMlMIu3T_`nmizS^Bq<9hBp*w_%P592Mhr-`3tiWpvJAC50fl7-hWDeMHt7*5t9Xn< zmxBkxbe^gzq3y3P$+aoY^L(p4r^!Da-3GyBft%XhGguawJNbV7i7($w`H|U5n9p)f z9d(2mv1u2I->7AQ^gD}2H~fTar|iu$96{u*2@Zd)HF_8Cu$z58K&IU1Q72ZI9%X^C z%SDr!;q>Alh$6}aH6EM7ZONHC)yDI<*&L^OaW~`%*RudKz3g7?bmQDl$dUQ;(b5xJ zs@)Pz4I_Ypug+P!3=^c`b0_0o*W;Y!3JNmdoyJKF`fqcsnw8~W%WDlG^3NqU=4R$t z8fI$<#ga(`lyd^Y9u2XXfT*E+j=S!TjV1_RtgqUi?dl`|Uv!ywU!OtboOm$-ZUlmh z8d(MGV^pY#n#X}{X=K!Q|$(6?vLlyn+f^eThR#&=_5x+M|ZtRn{hz}#>1CMrbwkc zzpDD_cfTxqLpMw_m#^wtNMnSE#co9+RYf+{6O~C(7g!syzQ+f2F$qGG%A+Q+@tgMm zte*oNB~_ftok;tnZ-NKdKp3v4&i3f-N*|a) zyZ6`SkgmGaz*tMtCERhPZZmzSK37iF$D`^H*5O*y;+Md2vb&Y(+KLeB-@erC6Y+_7 z!R;a9e(*ro)CV8m^3xEAXkWDSYrU0Eianu1Mr zhb9`<3G#!v5Oa-?%Z>BBP$nIt1bN0W~3 z#c}A0R2636h~VMCXt|e<9e}(Mt&dDPDcOMqA2`OfI9OUDXQZl7OSiw%Jn<4zL=*g} zy)soTV^>F^Wq-UZCX3=usLwGoZ%{(w+YRpDj3mP$NdELX)|3ks%%Y$s;5Euv$`7|3~_D) zA87BH8#(hgicb*8ap{M;LyNcHoe}%rd4z^aO^|#m*}FMfXR6A;f*#na!XsG^z<$4A zxPoLFvqTdjg4;InEjBN7-<7CrFZGS`Q}$CWC30mo$)#8C6yC`Kx0b-dJkPW${mhxB z2`aYBZ=u3gx&4fg0(}#VQuXpQz}e8QHvT>*5HsH%o&YD96sg(PzPez~JOkbLM5N{S zI5MV@{Of%F@{S;)ba#7f-2rs%NKoW0OyiVKZ8lue$Yxj@NzXjybn#HF)&nEeZC5hi z|HC*wo9_DxE{s-(a~^*afXy@>SG74_&e2hp>*qzS| z6aOhfsP8j|92HPM=jM8l=DJ=Z7}mN*ETOa0gLYq{@R+>aDLh=7u20X*&o^TI_JaRG z>Ti25-43YNTb)ASLL~UgPVlHnkQIbx%nf{z6%_)l$WAbQaHHY=R&dpVMt9ajVn|xT z5c_nT5wlkydQj$^%ua3EG^=7uTF>HyNg zs7K&D+f42~?iE}-)e1QepPq>m3w7L};WRaTYi@!an`O?9Xsf5OJT;`op}neG3K4Y3 zZp;i!0Y}8rWOOg{Dy*69nT>6mQAU5Myf>Nnf$#V(^%+Yyr2XCPrsl&9=-`QRiT;P0 zoT}cdcIiEKUd6{6oh9CePF!QLE+;Z07)zD@2hmy5B&|2=jpt=Xr_{3giio8?_?=VR zVvL-@=Vj?g(1#2E6RdTv*YDhyhj4N0+Kj#wd&OlWw-v6H@MyHu$O&YBFv$81ZK<7m zwO3^iC-01L`+jvF(Q*+RF3_+pbdsCv^g$jS_<-^b!l#jJ#E4)NLs^QSMv z@N<5ld5P;u;N#!EL;BC;e<98iCCTxq`;+rOHQ~#WYHXgZJapLF#cN9SP`MFKK_F4d z36&hPr(Q2o^|(W>dt=>Za>gIER-LTn>ZvV!F}797{I%>SP`JkUDS8QXvP1LZnFLq$ zwAe9$i9X|p2O=)L@}}YV!=?9pe=x$t+AjKc6kc6i_0w?Wp7M)b4zHe+Id(seG8x|y zJUnkBn9)d_@PyR`hzn8g?z9+ph{K;>W#qOiD^dGw7;9y3bkB>ns2cp9h^t7oo{C=2 zmUVs<11A~s+A`(vMN&~y8;}$mFOi6cgzkgTPBWhGKUs7Y011J%Y3ZLX@MB{@9aqmD z;m;2u57uuTk0+1yvFkoB&qV|*Oh<5L&cEOGSieiFRWR*vg$>MzPS*)jR6L!|0?_Zb zd})E`j{H>N5|DqLkH2S$h*)Pso%#A-nsOF8L>Oz4;}6_M&ALt+`Mmhem$SOJ~tuOaB2RN5==y4i>DYz9ec1 z-_0t`Fonmo8^=U1=TvNhAQ5xenkfA|;VF-)K(#zGbNzI^qp50>LbfDYMa_05-NzuH z0`lOhd=!SOD2~~J;=YTpkq+XbL1S>K%+lv!b&QcZ2kZ4WCp#+fiZeevWq?J9B);`c zRCcZwY)GghE*sy<>=B;b@@au}Fb>S?x-Q?cwFr7W&Q=FCRLDA>HY+t4Nc|dzhh1%S zAv01PKnhnCHBfpw-32l?d4Or;HHQPVF3^@(Tmyz;;=VcEerhG@=e(MY_K38YPx}+v zA$m`x{%^MUpNpeLjx}pr1-JTpmfZz*b*jKf!w77F2gxj0I{fZprg!3L4ZL0h_77_q zn}P)MQK@a>Nihh-7}_~2#_a4?B!T%F&~F$n>uD0%lH%)7pa``&!jC6u?U{~4Hsd!v z5F(nn++*lDgM{kn>gA}bX>#4w&pYLi#C2j48JJv*(mJ)#ifB1&N(^hmG6B>sgl!e0 zo6yk4tdwE02xXI7SI>isx%@}^u60radF5XF8`iu1L*aZEHLZa=SeOap-?G?6=TN0f zhfW?PeB?l0l66M?Rqcb!yL4i=VS9v!1^o1}q5Uuv+mcj*uhsOq{sTpLq%kSl_v&D@ zfB*+e|NdL^Ytg&)$lAtW@f)=6KeB zOx(TM8ckB!Snr=x5JXEc38*Ts4{OQ&Zu#(jv7Utu#cw6Ct733<`siw+?g5iEg~V!W zEa#GXoat8o>x|*8X`hfVdH;pt0&lLfguzs9fEZd;R+Z(`%?)F3RN{&w56{!tl z{1>l+?63*)V@hiTbq;Wg6{4FQP~+J2Kl=nYKYJTL$<}F&kx_TW)W461D>3mhjNAj` z94P$V(LdmafAqUgY>QG8vlA6-E%3Ou@N9=DgKO*f?Ed7NtZP3fsr z==JyAj1uklv;}E(A?t&q$NTbG99t?*mG?D8v6+dEzT}3mM^zN%=IfWb+k7l-LkJXU zW!qpYGW{_0g?&pq9{#v`K5xkR?@^dTJC&dhEGA6UIvs`{V)VW%PjyeE%uhU66wj1h z_Qg&&*Cz8A$-QX8N6Ja6d`TF3;i}{HTeket-K?vW{lKGz!}gYssiir)t3u~0LiKV5 z7ANNHd@qFY2T044`5>v0AqjPOl&*0H3OaP^0jG`z$b1B+2jX6DlqU-udmZj6>-3U1 za#&3J0w43Wsbv#jTs^EXQ>ok;7cs8}ZmScMC>IEOf-{2w_LfSanqK!O``*6vGm6!$ zNiZQ(FQiUCfAWx*F8t??!E$5LQStM%GbOd}JO?;d5`X1-}lnZKp=L#>ikdf%O%eEb&2$9CpvH6@9(iwC_;YCU%wZ%v>IVBUBJs5 zj`95PWaQ(m(bMxxU5Cp^r<@ktGl#MXwr^?cbsa` zSyNCilfJ53v{{`WcX(1}ioBf#bVn`zXwLefR==)8i}t$9JkC+p%r}E^w-DvM>%>YI z%fP8rg_4xkxDv(YC416~vrWbPC7R1l^E(lt26q~dYE6}cDPo1lS2Pw>EL%NO?^gHs zusIGI*)_gAuDHjMrU%*+m)?g8sz$?)jsH5}zNMauy{aDVcUzOlInDBq#O%&78H~($ z23G|fuS?u!ZSnqY9D2=Ec-(3NJ*zfM69D!*vWe`@$CQ!*Lj~CKI@=y^DYUi(LTyUP z?2<2;d-eS&b@KyHN>-KdlgUahFMK_c>TW8@daHzY`UY4sWhKv{eCHhN;=$?WcbXSo zLavD-RMv2Ydb^YiAOOah8=$;`K($}ZFMZ9$+4HS^b$e`%G@n8GDn7oQ6$ZO|s_=8Q z9h9lmR*c5;AKeYNO~Et4Uqah~4Y!wzUq300dJM02?v$HZm7d`hz!N0+IajV&4K@%@ z6rkKzOA9BjsneXee|2k4R*zilTar$+lVFI-0|J6$iIa%8J-*Y&)ILLQ%;MJ{FUI=1 zy1Wf}q$~MCst8jgNRnwrUi0uYIql+H-!S3n!y`Aq4^Q90HsbbmWK?$Ov7S+${qiwZ zer4x+Jr#P##PrvAfO9j)z|Nv!tmT(uvIBcljNm>a}A^n4y9SLBKj=a46eXr2QURNt0M+A8chBt*9%8n&Q-^_QI+i|)D ztSqf{T0A^%RH#l6%6a6{(@_rcrBnjO7j{R`3*OInMA-#1(ss*aNQ8$N_h zFxpmo)C{rOCw6bZlM>LRBWv5_J6DGnA34bhC?vlQnmaxz!XuYqw&dzm(j5k8>T3XM z)mjzjZgO3#>#F?P6$GW^nDR~P;+h|nGnQaS+a%KIxhCG~F0U@LX{QO&zZ7gUVXAhY zaF}{Dh{LsNI<2>K99$Q{=B#6QkJr^RP{MiG`E}fn`8P%vs|cijs4%{N`+{-lCh?E! zXWh!2_ME5HZ`U`>2LbKc+kA8{yJ&59w8SJ1?n_)c<1)K4J6nkP`e(LZD58cDJG|+L zr?|g5_t%Q*%BD8zYMa94sjVY3@5j))dS5rnG9+kG_dd?)?{fz8Ae8q>$v+G=wzm~) z%FFqSeMh2vVwU%UUJ014xBDN+_%4_{9N8}npC5ziWQ;jSOu)njY9_ifQ^dyzFbxb0 zf+Z9^>Y!}ba}aGawGVw8G9B`xbNYqFTJS4t`;; zj5;h8S-FoAZ%Eyt@U>O;fm}UIngo=}mue;fcf-#>DLg}hsu@$T?!N;rJr}C2YNW;D zR9YYV7jtZz&OTVcO8ygGQoynbM=O|Bxie^>yvg#+!Naa!Y_)!l9L;Mxw}z~eHrM}{ ziXTF+FT$DGP{f4+JJx7^N~HE@LZzu?{X2-=eKrG0pvors*=9pq?Y)=g7#FNm9CSqLAKX#rWa5ggZSWK=QM?*_S zJIea=-@55USAH(BJK`lLE@3o_lDS8jABw+MIpn|Tj&jyuBIi;qUH+bLv7V8QR~~}b zP~|?-xJlK|?>Mhu8-PK1$mZ?w5;sB@3N9((-@4r(QMz<`u1|4kDzq|lb!jUmylIG6 zN-->JoIyIeVMBcd!_%Z1>f*Ee>H&^-ZdJD2{ro;DkT&)-D0Eg=#F2dd2Imga`ltod z$f64QUEcfZR$sQm^pSTSjZ#&0HLgBA1nlPGlDS^<%}ih>S09&>`NCMc`GKLqtM&YC zb8u_jW^FMK6qlDSar#27)kSJ$z*=Sn@}YI}GKV zxB_!)0rcP5O4V4`2N-zii{C#UX>rE~a44`G>RT)Nn|`qKH(&X*fL64zy73v=EX!}q zjRat#j}YL&G(#%)58hWw(^h_PTQS}yXO*xq|ES<|1K6IYf7gwE)!g#PiA($>QC(<8G7EMZnOx80T0LKTU?{ew zGNqZ1RjJG~%8R3Jb=ou6dzthx;AkVd{P;QoLb+HQ{-|864VQ*$NO+2D@#12QMsO@n zMGkF3d1O|od`>r&Vg503KpT@zv~Ya+(K7hv(lIYLZoi0(+8RmbWc#PU z1EM+p`+WD}?zw;|7CeTsUz+n~Am;mwENd<~v7ZfJ-om<|6PyHoxDGoXEv#`#?(QF; zJ!aGVP5oYDlcj`dTt6E6Ma_!WO?qw)jP zJcEh>B^Is)nA5VWRqL(HRcb*VgJEWclSyd~jvsPHbC|E%zwfbw-n@l}CSNRbZF^G) zU+dj45x~#HvF2aZ?WOp0DF}O1Nwy73y^SuyX>J-MQhIBJsvRDRVV3x1(sJpvJQQh? ze8_fZFO4EmC^@y7pT9dHiF=;;gJx0?ZSTr+gi5S9F$HY}u+Y!ycbcw}IP;0#x!-^D zsG|%!-247Wgi+yKB=sIR{dfpkc6s++>5ai?61y=lwfb4f#Fjvg1~xG5-OyLjM&DgK zk93#ktwn(;1B>OE#=H8h@zg|0sPoPhq4g{(kgX$GBR5=(-yfFt7hLJ3_d9>gG{V6u zY=X+&QX84G{Wtmhd)5=&5t>w~@9BT1(!_9sAMbc$aRwv~B?kaI90aArf>XKyKRu;2 z#bkc1pY{(bWD9NZYIlE%1|H8#KCc=Ydx4+yMtPbeW0e7*SQy-~zN@ z(n)!g^*VHt%W;vKXJD3rH|0s_K>VmsGivyw@#^x>Of?Qe03MAcWBn)qQlF{17?7r; z)6KTSiWZ$;i4`tp7VYF`CZNtA*DFeScN5O1+PY-HnocJwqz)fH_6IM{ZJR zbdI7#k+o{JPBbqwKMf|XNMxl9ovBh=o)C;}IuGZbY>y=}McWeXIx>nLl3q`7SS};r zM|1J?W@L0VIIt(psb*AR>t=Ct3Iz4_Z1c(gWN%=9m2czp_FqsA2(`z+Evr}0XP}-M zbYS^_kSMb51nIP=R%m9<9Y;OSRXoyrBed%XTdS%2&1kq+m2w5yXT~4f{CLwE{=R(% zfTUKB7z=e*RxkcMn7Q(W*{Zbd_=lbg1#=G^<)`)+2=5?hdn#;+G}8JAotF_|wI zHbSg|mgWsfS%Xlv3lY{y92Q2)Tz_?jmd0nyzJTVsX2OMfq~-Y;-cUh$QDCB69N$6S zd(ystdbMd~?h3Of04Qi5#Qs^KXR_S;YFkkIe{%;YfSg5=AHlb zkP^@60=A1t-#{P18?woJ4=`{8-oZ|S5VO7_y59-Kt`|Dp%0SQ1jQ!(hcaRm?L`wF# z@NUypIv~BRrp20maS36@*6#zU22$_T;8{rXdrm<%8w+%ZC^E`wg-ie%G<34EMZNa+ z%)?hB`W`v|aR!}D^^{xO%ze{ragkW1ZyTA&*I%q|gILPTc%j)p{doT!jN0FOLY+z{ z#mxN2+4qc(5aPCC!UZh(^yMeM#*>K)VP)&-J274Yuq z`<)YES-j!5XBv5&LvA62Y!AmvW#2ynOw~H-!Zy4Rt3I7_gKridb4&?L&)jw*cRfOs zN|dk1>Jwf>p%00R{W_QZZ_viB+HvYf9{45Ns$Z%L6Pxy|cT`BLM}^K;+gr>8Z%Bc06T|U9vprJQXF5}ah)|gxROAk+} zIJC-44d@AXmRlV(2`^`dg#@CktB| zrNDkbZ5g6E!`m=A>kysN5#GVxR!obN4m%jIA8<)5eWf;adShL#`%iArFZJ(=%FZq> zXNp2s=BoqAX+Bmq$#2Z|{I~(Jp8Hd6L|ukiRgW?i8UeA+V@u@gYBS1E(-~3i zmL;pj#-V0iK$AMVu#xHb=G)%HMuh11n&`9{Ejo{l)d#F zM}S&tFTT^}2~i~V78XI>RKHp?CjlDv8T8VQG#Tw}xe?7uV+>U}PfX+NFkNkD*zZMZ zf79)lJ;*w=_Z%(Hd;OFcmKMfM;OxFgbbwFfe-#ffVLianQa}XHeQDx488`s{u1x`h z_;>WuhjuI3uI=R)-DlP6X)t_#eDkOk_z8bedN=?XqA~1Pd0{EUJUA44Cf!7|x%-~i z{mEcb;|E+JUy8vy5s+sIWcKWr??QDmMuh-*d#T{0C$%n?Gho=*{OEGl;Ge=R(wpzE zq%K434x-Ng7ur*bmu-74-h=A)h(-=6A?XrK!u`(p@<+<<@W4#v@?0tfoK!i-a0^+Z zZYw`$!I($qI(>dHRbmfy3QoY3#u~Xa46;TdZ`OK=x$cmoS!I&jE*2_tDk>jK`5IOg z9JW@VYF#S+wP9heI(G)8&B+O7=8^%n%rj8nC*2scll6&6BlGci}d zo46DCV6-Co=uVw&c-E@Zv`u@K({<#Z zXzJK_nDu=-($R#d9pEr-FDighQb{*YjgD+2fLp3WtSA2YF2lIGUJT3a4lzWwPkbOp z5ATYSwEPn;GA*mw+BLA|X0G>Q?cNPif%!?r_Q}Z`zzMvM8`tp594uW80DF|EeJssp z=Vncgo2;!FK$ozVQk5pV`7nzorZTZEN?XOmfqJ2tUmQyluqtf%P~LT}*L3SWV?N1H z6#I~$Ba5s{GD6ySknhhnC+_=qK8$zuNVq2VjLTr2D=nNeCdamVX(dUZZX&1mSk>8^ z9>Gf4t8=!43GH@WDB(vdpCBrymG9vOE+7%L<@;H#3oF}sS05Y?sK6(PYrV1A=n7Vk zGZ<*FZYZ63Z?bRDYYCvAKDZ|$bh+bL;>;=*Hw^V;=JE9uU^2R>U+F>S@th6xJbZqQ zxA4U*eLUR*j7&x4*eesD1$ zA#OM$p%bk)s*(Y%>cX+FcuS{K+`3X6tRSrDNwLV+gyaV`X9q{F4*;=S&vZb|oJ6_~ z+P-&`Ew|ideN}Lfi6J0zhIQ=t%+lJ6Up2r7*S@$>MDarHE+XG+>ypD8A;RlR0|;1q z=@e}F2V{m`P@Jyo%Uw0pRer;j;KQ*5;657ppn?ze7Fz)KqG#$#CL^w^?a;;ZOvLwu zLTIb*W2zG&Beqe z%b;!@gpc^9W1bW%xJO3gM$gS@J%t4GK9G84ELjJ9b)Q}>r|!uO~Ryedg1a2B9s7xk^-mhgtJT(tZ$Dqj|IPNqJz=SMo@Z~BGa5`GP&~HZ}STy5qV5wXH1FjsH<;V0if*O zrVW-vi8C&m`ipP6%@2*aOVoEf2^>or@VKKt46Hkir189P>2L^{NUhCuFx$l~2o>C|S@ zFM-honb)@T$uR_Cu#ZzOl)*wPi9D-?!n*+WyCtC`UPp0vZu`Z3Wer?yDG{gR*y_g3 zdPAKGFRWfQaKS7?#6CUdh`{_5Sv-#doGTjJ(LtQq*gj4yMteg{B<21wHIvBnmt~De zgJ&&k#XBSiQ;s_`o@`XQ+CuD@csW{0vlnZawpI8?$DQ9<%S`A#18I0^{4@Pq7gf_} zG4NO?t^jRor%BaU`HxB#o-0Z}K$QRJDj*H%?>#UImcJ z$)JV?pugn!fqmD3YJgEg^Q{%r(ar?K8zo3WD5e?{8fEV%u&waY-#;ai!rPQoQ_6R7 zAmFYsq_bGF!eWnFg(2nQ>(>v}*0!_m)>0aV7#O0Viqj<@j(H5G5BR7Qi->F=jv#Lv zPegB5sjp+Obj3PdNc5Kr_Z}G8xAhsKjLopqFuBIgxbK#)N$)P-%ttqwV4ENGx{~eg z@7(7>El&v4bMG8jPqn<`FOU8>n=wAjmhR5ooZw;=~%Vth$zgJTQlM4_$jxVX1n56fY9J z<&4Zq^>mmOjGN#D_r+c>C^>>oosYA+zeN^+(xQhJ2L~QuIk_T}!&xsZy)l897bfU% zESn&s7`js5@Wr&F4Nyfg1sD9b!^G%fb*y=|D=cRcK~=s){lp1zMSODr-25!iqO5`8=5^ui5-56_yT{ErzGZ{279^p|K?_s%^`%YCuhr-O;J3V=~rXq^BQ(0Ya z<)fM>lalhHf7$c@;i|rVvgHPp7I|^bB+nz7C*Do3%0^%K71KA1Mr_7F^>EG0i86ig zLQzc^HR4rZaqxRY3%YEaf$vP8K%?D}c7GB4TWdcKq^)zdR z=jY$_Ps%~d6Y11Jwus;*s!){i`#zYo6dAKv?$!*hTTO#JapShiiy>of9JxC_Lt&h?MMn###m*mlBWU~e= zuj-Z|5@xm>92VN+GB>JNvX=zVQHM1L#h@OzpTT?BQf%=QhZ=~I7ob+kpZ+77?`?-w z+>$!*l~s=GYseC3z9~yUW0SE&cZ7>i~pPq4^K-5dx51dA^s2*t99h;bLxQ!ydGdZThLaW+jyg zv-kNse{W;TSa4Izid4{xfv&g=1s@Yi`MTbUtNau{)EfQ#Tnm7{Jh^M$l@bj-S-k{s zDygATx|84~X7K(YwH9?V6V>_Nfmj0zyQw(v$*@5LYnyLV4Z89(Wr>IMhon5z!kBXV z)?`aJCT}>npvpo2>j43%)||$NZ;1W-_R@G(uATdNkTBeJt(#d*MMJB zW)=36f^6FE$$Y-tG|-`!#;am^v1I+qQ+@**GE?Qn$!EmYdcwnh>QSt#f7PSbI?Uak z*y$X(n%Umt*X7H19>W|jeDV_YchF ziN{Ch9iaHP^C*2gLrA!OW#J2Qa`w3HSR3}a->-}4YHcxBEnUB*{E)f%g|Ryaj?NHO zjd$LECXnDJrj2bg9#Wv9T&R`*`n~`bS-y&sl`S5H=I)Z&?B(yum)+ID0y*YMlYBOM zK}+{2EPRNZug@#&FF$!uz7r+r@O*=p@$~zy6RrsZ(9vqz38CjTrb6I&@sayZF+JOMuGG($}BR0oA_7yq*WBRr^mDzs! zUES6YcUywY&~wK1;kJBDhEfr}>^Cw`iXKjL@y08JEx`8qchF1b;e#jnB%SEe@jI`- zLgmJ9Kd8}c#{Z!S%d;Vx{s`Cq_tyP~uKGVTs4mWbM9=@?3H5m^qB(v4LlbuVUu4M< z{|lvh4V?S8!2W-pm0tgcl=UsP#`5Ozz?Ad);A37}S3T#FSbp ztS}`qb_)ee@*G~hCN-bVLSrMr0i%~FbVcYrXIk$k&X=HH-hL9*@6TIcmvi1AgQ`pC z*`0%bchFlhuZ$f`^luHrXos({@7`POufDLEhgV1RHCm`6`zmys-Vfslt!NAG*=kZb zr|($&#fW{&`14P#9y$c7K9F$EP{M1(2I zoyO>)`dY1RFseKcx8@zS-PiQ$-~Bb7USs?r)#tcy6}gPC2NgODU*@m2=@~$-$oyx2 ze6^8|`1dP>h3Pf?83Azv?Zw#aS$yE;D3f_Atjr}<3bx}R3|h%sCS?|%l3+}j&0m$`(;Avf4k*Tv7ND&?qkA* zrqMcOeW@GTTN$=y%`zjP5~Yr1#m-O6I6mu1w?D16Cjm=aI}|4G@ZqQKha>z zNIY4KdSpZ%(CwK`$fZdRvyDGnVS6VkTCg8j$ytQpFcVJ-8P9vl?$asy?+${`G zpRbSHD;B7_etYFTPLrUNOJ@1V+eVD^V!~GTn!5CH`Li#ag_?-T_)`E zB>2K-Y>5>8`|Pn_g3l!}^|-%9UU~ksUbx2=QcRL+*1Kmma}t}(^aX+E@oLJn-rl%R;RIuCW~+2GY$1KTou-PFSmPqU9+ENa39kW6J3!drYiet z%tk0oaD&D?vW-I=F%4rJLrbTpE`Hl?<0LA(pVliS&d;QU0z84rRZr5oXm~k#de={} zoG90?e+8vnoTqd}LdRAuY{x=P2h!(GtI7~7((Uf8VS&AV%H(tYk07s1?>V&vJ4fMTuR9U4 zKt4(j`fOxt#Nr75fpg`vvABVe7sr=dkl0I)Gmm&2b0T{;Z-lA=#fymCKG6&?Iqx>| zN!05N*o-ZFt*3>isy?~I)EjC_@5RMwm-_bZDQ>b(mza3^9DI6b@y4meQ!?yPpT*8G zeS>>PnF(B%rDtXf>Ld0X_Nkb6s^v&1JA|LE;-ITrX#tj{1=`3J)Qo>W+FqcJpdvs8 zB>roP^*w?{50n{Pg`62Je@SD%DPXo8R%yVDKrnbZlp)U;Vr}p}E(QwzL7LHE^s_c65Em_OD$s4p+a>SW7Wu(9226-e#8ex{H_? zc#O4E2cqTuNvwR0Z@l#R@dI{Dw1qZD+RxG`Fxhh3?wKcQ%9kBq1e&@z;Xqafq z+YrXjzeAM6$;UIxN~I5Z@WAw##|MJ_>AUA_iYh<&&CQpl_6&Md-6X;U{%1OjI(6hv z>oIAzfK`5f-AuV!3AE-_kAQmRXT}^7&z5@@10E=^V^`umzCysRg#+c}Q^QWFgYM3T zf-B^Zu%=IQ^iSfGR$so6#m3gkYI<*c_z=LL2gdf2ByYe~Le!Uug&X(6`--FFQGk7^ zVyJDCS}$%JG)~r4yRmoTlqro$lsB^UGvYtwm*+SRWcQUf1;FgalO~I9J!+jH zj0SV=0H{ts^fr7Nqv+X4uHY?)y|g}4*jIL2rDY2V{E*{HMQJDyDl8|t{Mp0F)d$}`;n|P} zL)B`Mw_UXB0@HO%DMQll`J;?pg^~+@!f-7-#x9ek7yMxo(fQm!UlwaE`fnVa&Cjgv zC7YF}4SrcDgMa+6TB3(!9<�Fe-(uf9K40l;)QH~^w}#I+$1Am_@@yfYvmu-R@$k8jZGK1rJhSb|@ z4JOERBv)jhR_9=&<1U+~SYr$i`)m6rmKW6+)>ga<+Lw$-u@h?7sjd&3gq4inFwvex z8-;MZ;|>zKR5PkK*3zQoPn^8@MfI!8koQJG-S*DsAt|2rin4>Jiv^|bs$umh*cQX&*; zlx>aZ%KYw)+K<&C+c8-L7z8tEMJ=Na>Gk>J7cyHr6f;RsrMc@Fel7|Q)2Bj;hQP%0 zsiwxo+b!Q$2=DZ!C9Omd_=jSVE|=Z#7B62qjA@UhG9r&~{=5Hekjn#z1#y}s>P<;; z>_@U*oyHg}lC=NB+IvPd{budLAOa#yKzdU^kX}NsBGQ|J^eVlF-iwHUNSEFfK}33Q zp?4B`Z=r+q5+Gm_ax(Lrd1mH6bKXyH*2>Dtm#kmzd-r`^dsqFHL?gMm<9Bm$*A#Gg zTY2B4>kKow*3G3qL)-1^LZ-c~$7k;1B*EpFkqTFM8Szd?5)|u0` z$F`-KvDdS?TTgQGz+zO!#vo6#y$a!0*t<%HUDKDV8vjcedE`Z>@xO!-VQid&YcY3J z*VkH+8=ipLFxN>juW@3Z8ak|ha#ckCjjO`e^Th|Ht#2}h=w2vEfN(>AR zsJ*1Q<^u*5>qhqfBTn;n+fK)9PWi(cx@us$Qz2}{9yV|wycc0wrF7W_lHgkhNS7gV^;@;KktS(X< zBn&>X-gp4k^aWO2x#Wl~`~(TP?~%VmdLNTeBzt~?8SQvO8JaF>MBUt@26?^-11*}4 z2B)hec#XvsYOlZdl}qc~WSeJI!a^BoJ)C1fTn$EtC4%RxjL-D_H7w3KEs$B+^U*R@ zCt8+SCc#wFbDh}UOkTKTqW1295Y_@&fe5qxeh>&N=*Uywtpls4vZ%&x_ugi1-r;N@ zRy(Wu14=r*-;yAD?6LEd7ArnN(pGVyG>=@pA%9HzFjLCISJ_%0B1aguluv!oq@+Z? zp$qxJH62j$$rJe>e$;ed+m8{0+_L3;CbQ?&xL#f&CgVEavOTS@mMu7UpuFWja>@S^Hr^3Jy*?~ zg~eWR4|8$Sl5PnUF6IGGstVt|8l<|(Yvk_i*1iaUFhf4y<^k>Ez79BMm^s5lg|GbZ z)^$XyJ*F2U8OC>fpmUB-WDdnyt?lrfW$*G=i$fRy3ASEy$vfq>G!yNB(RMKC@Zpv( z`j6tYy`|os<_3D9?eym7ugk=(f*|`T`lOFk4|p%3WtUbsWI_F{RG0Q9gq5p4vy}6- zhWc-6{2nWk;Zi!@(|LK?PVu1V#Rz0&jX6tLWgNmr(Q!J@9o7YD$8-(6#wEn|hP`;S zAh&U}IcjkeWywNR$UYG2^4QjllRZ_p`{$6(1Ycxav=yQARB~hJUs$$f=XmJ3kQ#X&ILw z4XBJ- zJe!kp99GFbn+8L7=Sfb1CJY%>5OvX0!W~bqwzeFEt)8zsQ!@AwJNY?kiPsO-v^QfqaN*t3pg^5{|6N*aOq$^Uu znR$Mu@*P0?{7mgXg))Xl%L|Rg+hv}N{0pH91O!+AAE$X#+}`cuLW)R8YwS7R?7BvY zJANdfc}M({*|n!*Q-}%o(W=j*-a(Q=ld48xXZ?cTzaM;vaLH8iARO~O+A&-3aj}N| z(V;jOzKHB1^eazmVUT=-*lu+INh# zhHW1VmB$DfiMW#B13}q%qSa0G-RpC}I@eWBzf3a4f|`(_nD+XiOtF+({8%nQ^xwr& zf5}5fjb5Xcahs!Cq=u5#-}JT5sh`$f-ris0?CjDRO9Sp_B65f~HKs)+MSJ`o>B(Lx z(N>>c`r-AFZblj0byN0kKH$=3dDhGG?pv!vI_!aJzw%~kGU4Hjh(ksi+Kl(GUcteA6r$%C$7)&bx{gqt4=;;vWcL&0P|d57@C%O57?g^4O6}q z_t)y&YwJAA#rhUDYCW6K#j)TSCZW2a>vcy|;Tq?GDO2be`M>2KrNe5(zRv)48b-)k z0HKYz(9!+JHOmbZi-!5xw~r$>U2en05`;-A)`qSbqs8;nBkR^b0Np796OLb&OQqnE{0b}&x_tUvqD#aUxY;rCcfN8)N zuftVy(i9sxzCS`~&T;K(to0WAiFEgA!3299WgRI@E_CAPUd1!0CZ6R^)lX3gB1)`; z3jR1s!J4ku#ZX#rr2lH5bf(joj_4l0GV+D_ zt`B(Tn7IP8=6?ne@D~(x(wK$L(Y6v-eVUQ}o09mHHMxl1wWJ?8p#zn_GX``RwST85d<$}EQ@`Ck$y3I+nW zV~@&_tY(OGK+v_X7U>%Av9x3P#jO6OInP>8xOe#d`?sK2zNE(+bzW2IN`OJRt?@{KMAl^k6Db1J9B@jj&i|>4JUnU zs%^}XJ5iC(`|x&ad}G|uek;kzIt!YpwBcVC>0fHuvx@O6OH@ti14+t9OO{hg*D82@ zT(J12?u@%dl{Wy0S#e--N^WYT;uE~}71>y^6XYkHj#v&JvD37w*qI_52CDls?~S4t zTJc=2X3`MM%9}i)%btu|%L`r558c+}{=h4x-1lT&R0{Y}yE75ok#Z%YtPRaW%5pCD zcXLe0Uu&EnO7j0w2^+g#%wIL<^s3jm*w$O9bgdk_h2TqhTm*ykCf8+Pd#KDOujskf z1jz0d=QI9SI1AaL-@-|oTrmah(tT4CgIYdL`#90V`tO>$^o=ATzxC()MvQwm?U|%phcH@)U^Jlq*yzX5djXKUtd}zjQ|5O*@g?gEO;)6}g zHfE$0W&3EUlbNxH++kjV%|6%f#A>wmd&&Kq+hy@n8~B%32`yXwkmgEnHyzEwwa?qD z7d~zpHfFVL0&!SN@ki>ZVukd1W1ueW>UU+0iYGS9O>%p-{EZEC14-AS0tXZ5p8WfJ z7{Rrj)8m!A#rxn#3Hfun_#hv@rDhO-YG5`-`!TgeLE@m#?OUY)w zX108BTN+g#DmrG*pWlmIL2ctKc|jYF3CatdISH%MPls(GD2c4xDS-#HffClnLEr4A zro@vGGLWU2n3|wFfOKEWoqeduVS;ainEeE91;HoAp(N~+uRsGK!XK#N7@e`LL*L!x z+oGza2>ct9PA@uAnZKm;xdbEC*RJ1BY{fjyr#I6hBOLt>^}NT;%zTeFn}Bnh&o7y% zqkWIZ!MnDir<#f60F%h66DWdK_sD)uTIm%x9@HC`8qJHg_ir+u^zH+8DmQyTg(*GH;KNp8Fs>7*!+#@kWLPeSZ)~Bd?C&%$)6@2yrqcE z67`{2q}d5*qtFJAN0Y=9WF}E}1`@{$9xZ+qgs%Y*~zc6=p=QsgzhWfBXG6UAFF4r9vlQ zFLCOn*?E8z3=%vgcKq!ES-mC^&+(sHV5@d%^XhWr8h%o!+^$P)g%d*$f1aqN(&dI_ zh#4ABwpK3gLonCBA$MmJbW#}bnxisw-`F^sy5`a?@!KIzmC*j_@q-?&)EZSxC7p|3 z$^-Wh;E87#gy5?u7ISB`fNU1yUJv*lTkq!{i`Pt-8z}c^VsS`il<(d+TGz=Xz(^WF zqA~l+3>Zq@QoeIXcF!tx)06Urp%*Dh>@wnr?l3{yDX6v=Px%bc;)LLV@x6woy?j|=C;ejWm3xR@3%RBGxB7aQLJ3p=MKDkT+mSH}1eK|8y% z^+Sfu@B314*S2jCRk7k{FSs{7t*cE34V%326^US>5@RNuvHqw(=q~~&zSbF0L1&-s zDZkE*+?d!bIcHk>CI+@=(@G8%OMQ(Fn#<;^zZYuwp8miVU-;eYd=+?WJZL0-9tw3u zP>Ee9bP2ok5w%BTWLQA53+(Fr_w`AbM*ALqW@}XBmLy@+7$@<0ARu&GQEt(rwwuTa zkPtqwI#}#_aM+b*F}96lq+bN>JlHq76-)OzgoBX3#~CCWYHrVDeO(ce76vP-ZBT5Q z#=yS*J@b{PyP75gjsyv@h^5e(u?zks{NrvMWX&35$7s{);}s(SVb?%2$g-0{1?-%!QwfKI|N?u@V7Nrj5;Hz-ydIJHzTen*{lAeC=vz zz8-rwR#+R=x_g%6tDi)=q)>e8*qH*h9yj{8y#cRZZe z7gf&C-0JtL+HE-{p~^buJ*|k|y(mbYzgUq1OQgP;QEQH5b!3#2`y=Modd=;d%AfE8 zV+cn2x_A4X#j~UyONmzBG_jT>p3}zgd@vImwU6GtCeBOVn4oA1AwUtYu`-FnK+^4+;SST;5yH z+Emit;F;vTc+E)u2mZ|(A?_4um@5s+a8=K!wi+cH$+3U_RZDm2>hR^uX$O{3eDm>s^N^b6vw$@>-^%5obYA75 zay*Z$4o-`dB&7<+wRTF)P3un&8n!gFH-1-ExEMD^RRn22zo{%l7GXFZsAYw#Q@>!? zU#bZ@6lwhbzzcA_`0H&`+Uj57{#&=}A736TcmMa@?yj8w__=>?n*QR&{jV4bm4;%= z|B9g~Z9LKo!QeDDmE#^2=Jb(op39|G4@uS*C=4>@%hr5??|1zBv;d65<@&DeFID!7 zA4AkczM4?UUxbk>M973#;U6;5gGc#Y2Y2>$WBqeZ{z6$HBdgN+Tc1BQ&nS^(&*w+w z=ro6Ka;@)EuxP=Hh%YyctAF85u&|i_Kc5=%tK+}#w%>gI|J%tt{_nH@XVX7mum%75 zDq8yc-}}ei{qys!$u;I_&t>a|8$caMc}e_xl@Y-*fsa$}hAI!h7u-q_V1J)0>&pjR zCyZ441*^jnV(2KfYse1-Z#3;>(p@Cw33J~nlfhOIgv3Vq0rzFc z``=XFM{6;=qgD?{-93aH<)p|nYPimyaF|g_yt{w3-1too@na@FhD~~bO^P?Gce$z7 zW8c`9Y53yCFX>|1f5VK+LSp)(`L%_e7OmP7%VP9UA3Gc%R`;t)4aRs*Y|&IV0tL4I z8#Mx?-d(w4*>}DEfj*GJ8pl-GNaD)tPV#l~r)Qj>M-0YXL^n@wGZPCA3`-Lv4wn%$ z0i)V)H5uaM9S%24&nGnFWIAOqvk=KnYlgI--JgLaLbu&j3&FL81(X`m9CABXLB?cB z2*E5u|Hfn9V84Z5zg1;=^gB5+WPRl3O=l98!b?)acUVHe%)wXNgdn*P{Nm7%AXnpj zKopFRw%e#zYVt{sQgZQuhAq5m`u6g+=_d4?Z0xkJXEkac70 z1TAYMp~#dx^(8-|GU}A+SELS>*O?V1P8Je8NZ4($MO#_&Qj`b*>X*%#(-oo%ym7ROh1fzAy#(OgFxH`RAy_nn{ zmO~pmN?wPksf&SmoY}i%5bSpj=b-W`Z)SOL0+vzd@KT*#FC5o9t8aX; zscbrmqcCne117x~(j}D8sf#A67WtujYN+x4D4vZ^lmDzx{@C2i(F(`C#%ogTO9+g|JOm;VGc?U@G?dY&NZ;+L#q%p&#}NG zNm0mcD+OcM-ke24Y%N3?LTQWApCpcH9hDgyUMLj4zHm1^80F`H+Pv$Y~qS7XRS36!ITZ&0I>^X zHMLrd*I7U{6AkI3XcfmX`(7@!n=XzkHwk+^^DNhpS;1(iaA;OTc1UStuV{R=Ki&%b zJ}GoM)}aksNr1plx?5AutPe~m1iFeZGkwwIR!=C3MeT^0ebZo>>i5D~5Cn^KKO|)wM&J)vU!aWWm23 zFazd*S+GZyVTr(eD;roYQMCT`8&UKbFQDTtVh7w*eQNoTB#2Frfbn<8Fr{e-?8W9w zO{c5QygjJwkY&F?Xm3?&8I(*^B?fpcKo*Bkc#z^ZmONjxzb`vc>It0~XLOGioZ13=EDW@}8O3om|LbkhHHhy)1HF+Db!! z1>rvHYIftX=~=5K<;;%17dq6L9~7Gv#>guQC{62#4Zu`2KV4x1SJo0<;1C5-jJ1W( zobn>Ww?u?WQ{wGHq$nKk>5I}+Rs66ttfF0u*E-=TNDu)VrRcB{$~$F$)iMn>(SAvU z#epvuR9pZ9^%yc@wx zEXlx-_&~Sjp-OINdIZM^xuF#+$T3Lr%0+RjS@I*E_^^tW)oGMSxp`~aHsrH;X4BXq ztRqX|U5;O4 zu-1duF4kLxJ1#3hwV%P0^BONVji)M2oHw_uV~83yq?3D+^Gp^m=baiEho0L9+}wng zd3P!hJCfo}S`SQiCP?M|vgWRnHNlM8Fu=R&%vKM=YpcupQMWma?_byaVt(YI<9GwV zg)-O@XC6um`ZF#!96UwO{MKAF+td@AQDNn2Q)Mp_i7YvKE|mABzlIigu`}{!Woa}a zJU)s#v3PkJd<&a-u|whIf4eDR3yMl954z&rgz2(^^aDhxu; zCA#~^^2*IU>tEymKK)+J!xZf}@Tse|E#u8jo5+@$i5Xj)iafS3G4oNgkakjbeL;Ip zW3BBKNrK&taw0okgOUaKme=z`eFdZ;7&SnyrN8|ayTM_pi8N1Y;0^bEp)c39KKmGt z$8>c>^TxFR$1@TVyWUH)o@0`!YoPVBU!E+yvs zzIJ7^sR=l3lfMFpzS&W6A{uK#q2%LfRaFB`YOoz7Bn9;^o4Hi34AiR2kiP@#b2{C9_b*VHWsmK=8LX)Awj zuHt?hWpqvJTo{R`Gzp-Ohe3CnF1~3Jy5CC1=wF-;cU}~n`>bGw^8NWq6^FK_+=>lY ztl6Zjq`jb$(lP@GA9`JLYYYCc)%to^vv$`jW1wZLV5}O;NX0=ew^@@zcYbnN_Ni&U z5bbImgLZl-r7aQGh0A{dj<%sQ48oX) z?*H}M-1HpT8`t?H{H%u<>#&b+R|ZWp-T_LrR>Y$Ts0r47_o~5eX695x#xDt3b()-nM*X0o|8 z-?MQ)s61`7i)*n~(Q33ZWk7F)bgY@`0G+-cr%VSRpmqybgTIDN#T*{)_X4-VRk!x1 z*_zL2>}@f0hJB4&M68BIzRti`*03Owp>`MTaX+=!CxP5g>S-02^M^j>Af}$o@VvE1 zXU)8a*3t7q*!cAM zbc6ByF1*)b!Ih~H@gm#7l* z8hL#Wn;WIfF4|JiQi6IH(KOIhcp6Lj>}_H@94A=RS=vMNNstVXxH%5477YFM;nv9Z zUp;3ZDJ8wLA?QW6Ihy=5*o%}^GQTwm-_lY^_&Yi;W0Dv2$>M|_6IbnNLJRJ&_O7^` z!lm{Y8kcx!FT21XIIQ;)=)jGt1^JjK3jOqfuC9cd6R}e*=E!ctRd7)!}P8mGWN53ODN?SmXLO&N~bY(i>Nw4C_k-^QWK{S}kasx-n#$M8Ah}frL=;n}rE+bQOwm&=) zERkelVJDpTN4O$5s7^;!=7cMCV}@r?HU4L1T+l)2NiaKn&%Ba`%7ph6C#dA4o>UAg z7GeutNYXp&r}92Yo94sVVK%-p$5Ph_$AD#LkQ54g?9{9(v+nT4-9NAZFCJpy0Gdfg zqs{t!#ID#1o10V{9tqvh4fuG#_Y?9UH>Xvp{Jz6lE0pNnQSCdWHC>!=tT?Q~Aae)~ zKyEwSiyP3PH@)|Mg;hpojZ;zaaP-r`h~~8124+dg^jT_`6jrZ!&YXcv&-T9G#3HL0 z(01=lzBQx8(2<3X=YV^!R#{E2!`mNRq7RrQzd+weRLqZFP1{C;#n=Via zI|XaOx+z7KELQXgwY82x}0^ks@to(_8IHOs{w z^$`EhL*!BYet2@x{^t#QSO9AU_azwHA|9BgCjtX#Uy}YaM`O7wN`+LG2tC*j;uzPU zQ(MKtDR)T#nt@wIf>gAk*MI*JK3)p{;S&*SWK%u4J`#3*%JJt*8>}%c50YMBe}Mpv zGH5mFN`0e14a9%d-LfPidMRZwJYB#B*vJ(#v~YQpRx*CLc~(dZ5$Jiy`vc?uXarhi z8Z65^i;^7^`nCgV;GenwABHati z7x!Ex@Aq8y$V*q|#olg2c;9${-`QJ1cnDXRkyibq!|T5ih9Zr!&^$^6Z^*6w|fjNfYko_r57k8uA|8!Fp5 zW;l{=#wp%w<+v2|NMZa?#U&}63>p<%(xGwDB6bG-Oh-jS35_}I42-1@Qr=$8n2W(F z&l8F6S8wuI|E>BUq0kL{?ae6i$&HsqfEefdLfT8<>R^AVOXpVm_4*Z}!cgA3)lPVH zMW#8e^*wV5^F_c}3GG^p@71csOhBe>r|Y$r{``8QDe6MAoTYv2y5n7#PmDLN z3$MwvG=D`#N*=dpdaKV*uU=zOdqZIo02Y0`WFuSV=v%JNdZXB-$wpA{!t&-EcyQ~8YM4kOLQv5%I&O9{N?PLMdC!(Nw?T!G+ zY4l%)9+qjBJ>3(q^4IOr#9+sQO z@=kX_9oZsGGB?`k|2h$52Q#9(lOx)$51U6v-6Yh*y4 z3*sgqAJu%emwD{A)9&RJAxK(xwuZrhey)x7#l7xK6=Xc3X;#U*l9M%k?QNJM^MkTo znY7rXL}`Y=>r1UYVT>FGX=WR(y4oDsTZ!Gb_~DbAAz=zm#@Sh>Bf>dCB zHPgz)*s6C~N{;;RCxlF$(e~!r+lJlK?RRp}6L;Fg;B$cpx@CbLDZ#s9=UeMO6PF6k zAQ}4POVm8!$^@A4=ADG@z-fI91J~Yj-vR>Q#u@xT!gkv)!ASy6g)oL2Nmq z>9om8@zVOTIqF2wmU=MsCjC}9n7P$n-9bM11o8$bkV!V z*p(CvYfZ7d>Qn$Uxa#&igE99xYzJ@Z0-e8}?bc1Bk9TN}q@Mn@IF}o)Nl%eOZM3t# zUKvr0&t%WE-X%{h+=PvXeIEbQKH@m-#!2a|U#s8PxIhkOCB1Wds-wZLPh|aFQheV^ zr5JxWues#QoMPL#NVt-&=Qc%)6e1|MT3U1NG5@N2&zIq~pEQyC!e{SkXckfiA51#| z91t#IqS{wq+xZ5z*r-y#l}s7dz1jZ4;lDpRDg01afHlZ{8bE!vTx{X>Q@TJ`6O)B# zo!s7PIdbU(@sjse`tz)bf6)r{Fnfcz@xLeu1uiq!O$j{D(V&FnU3aNwdY=yD*SugT5y z#2Gexqi>`qEC+jq{+Erko4}DY%Yg@n8ciqR4CrIr6Y*2pP;sB9q+MwPV_CZXHVdrC zbRBnzaV6Mc!`FC+j#tre>YU~ve7!8A0H+0^FkDcr!;q`Io+O!d>SPxhgR7;V{(>^UV# z4$tdEw&z6`%arLE?`RAW_Gd>oXiD?vxa;?G*wHE4`ZjIUp02_S3(*6znq2nZZ!6$l zZw|5Gg-%>C+R&a}w2W;yJyz_cNWuB`mF{A`EjFIKS=H_@GO`YkW!1yqzA}EMjC)Y6 zS;ZwJ+8w?Fq+-0mURQeu&fxciCrRDdg}izCtfO;{YxQ)d`P%X|0_l;PMsKz+Jp?S^ zVOiaqf2UKTs_tc zPx^RATMCqTWKQtDHz+^)#2JGSNp8Q!F%NdtjifqnA1v~X$V9r@nGLE)8b(Dr+mlB> zELY08I?xk(v8439jITud#QbOF77x;YY=0}RvqsC)e_Bbjitnt>SI+UIuudqUwrt(G zOsodo%v~~G!@#iCQF>N``3`rudYI0a@!r?_Feis{wfhHW)rGZS_+u>nKTLPo7esz* zvUyhI#p-&1LK=Y$tL!D>E|#EK>fk%){y=YTTe2#%FtsO-u(FPBFF5l9t?9>dv9r$J z%?mVI-%92k+%hEfV#HnNZ9NnsKGp(rDfEinm>387du?C>ymD{RPLX7Lxg~@B^5e8J zF@9Bxp40Jz0%#CgyfP z%_{lH%=d>A%*c+j)O*4``!^FgAj$n;<5$@B6c~aK+xw>F*2dOam`j5 zUBR58?oKrMriHkFcw;sCm$ZN~*Pu)2Q-iqDyy@s>9iy8iYL}{s6R;LZ=p6~bkBS-+ z@};3FtgklZWDW=%n;=|H4U*b-f-_+G+ji)F{ydknXd~Vmh}XJztK}Pd#@ETr^+UN< z=LS}+ZIq#R%T(5LwJXA5Bzx?a&`STlz1JPDf#;hDUGgRE^1KYZX2@75wk7DZgN&Z= zDNO%1urlmy29}d?FslCRAhax%qHUpGu))Q;%_sG?V2qWwsi1r?00Ui?FWoHZZy@ zsO}wUEBiK|g!tlZFGFM64cI?++o&@sv(4e1(v**-$x#UiY_fMHTqm!*B+`gYMO)&^ z>-tFc(wr&mD`11&u+nBuDb%UtD<(j zr!FzT{L$O`>;e294Hd_3@IahqTCC;IY)rcLKZ2PvnTw&u@^{6(?cZdHp?FGL9$_Fz=uB$ zT@5Cnc`VBfTPCtOq4ieX3zarh=;jY!Hl@t)cL8Am;YabPpx2$Iy7dSDR~~CI@+kkR z)4ZN&DE?-3u&_WhKOeUqYd&EutLaDWTJh%9t|l#Es1-C6?nme_nlw53IYdC@f4B_r z6^<0W=};f^q{2Hrz~7~@I_zv6mLqmBDut0oSOo+#qAw3o>{2sRVF2Ab`3$#DjvVnL zCA4>WE?B+N0Azw0h>xsAg$Q-%eHq`gXF6j3XEW&)QsXuq%aoG` ztlgM6y4*O^7xO+d8g@2ndd5FU8UK@~VnJqhEOMb84I@HQOE?#-r=w8>!WFhj?Hd3l zCJ)lnkfwarmbCOV$aR6-c9s`#Z>vwkk1=zmWnG_d-_-ju^aa8>;H-P>Bh9*`KrC8T zB5}s4^@H;gVJJTS;z?R)oui2{mCF%kJzi_0w|%zfI)pFtd@ibjrEF8_z8O(hXYy)fFIrMz)7bX$01f4U+KBYrauG#6jRw7Kmyg7jcA zqEQ}ion>ddbHR98$kg`uxd7hsr_Nh4_BgkfDtc^z8ESxof>BC?z80KyfJ!jTT3=82 zjo&rm74ctIWhzh(S^0_`jen> z^_|1vyYBDfK&C1&U}+&@bg(@2!xfd1;j3Hr`)T{**}fW2#O7v47P|giVa5noMz|QK zy}f&{VG##R+u6F!Qr`R5a!u%uU^3GLSC`7y59K3~eLte}lrf5L9iI&}qy{NXw=W6s@ZFS%}#Fm?vn={6`MRPFjX=6TG znFLqZSuFz8BaR__q^-8LeLiV=2$vG6mic(~?($juMeV#T@|@AjSKlmg-7{AQJ=U&) zMyMl=G?w@dZ#Z#y86`65mgk6vkc$E&7#B5fGqNl5VI!|Ponqsq@I;(wBtTn ze$HOC)8$L&lbCEkX*Ov6c99Mfuh$xQwS@Y+QO-?w|OcPd7Q9sE$=n zrl*{xES*YX1Z9*)PkwU5*QeI<_}V6iO5?=lYPc)CeY_Dl$AhVDYXf3FmmWiV1k<*Q zddlLTnn_MV@~(X5Nxc?Kj{^5q)5F9Hwh#FRGMvRCZR}c_Zp)+M+?@uufgEoC&al}r z|6ImHU%{L=AVtddS|_+V?T(cqXJveaocggQi5p=Iu!TkzwJ zw|H;%60BdlH3}MV#me4BvhGysYc~L5%OOWEu@%RN0XJ^)fiVT4p55Az zD}?X&j;x`45Le?gvT?Bao8NOhzNyf#cmr9OQc~yM$Ws7o-HWKX7xz}>@#oGh0(P=9 z1PVGnB}_WHAU%cnJL@?1+GxV>oTaymsLwkrGSAvygi*@d zq+%mRZfO4|qkAj)1*eazbgt|^l9Ac&;zwghPlXyj>H4J|?e<(c^rKH1;le8D~ew~SsIJgh~G0Gd5TUHoQAT}~TjfyBKSd%seLUk13IUM)@ zmCdERA`%tjn2LHoa%I2qQF%L~DlL5=Mu)UPJlV6bUMw#(m2{wK1h$HVZ2fD8^HTw* zg=fr>MlBc3sV(XAvp4!wSy{G-$Y6b?(CrekM@gNT$2+he}+ zC%jmUN2{Q+zGA(-#^WQu7v5jtm9s!v44cVWv07Y|+3x#Q{`;@@;`>HemT}5f^gKKD z>eosgkGsnUtDVPKR#0iCn7wpc+Jj}@^0Fwy?)O;KgJ|QigsFR?bz(K|@^HAJQP`QI zIpp@yGAHa-Gt8sMq`uGkunu3b=rSX}wCge2o3g~tY`-;4qUf{#NRi(ZS>9h0d3teE z?WZjVdB9&kTkgXXSue!zPd`I<;m6+YgB^d5xJz*aY0!-RQMG86HjLom;B?+QJC*`u z%K3>%U9+b+CbEcDl4-VH6`lTmLqrRW`fYWpPB<8D8;pX8x$p|%X{U|}9A5p;LT&Oe*uaMa{)NeY`uU5&H|j1#Ya@d8Jc(UHNpGx=DG#6y~w(7t;^ zx|u*`(D&)wsvS5K|81ijpqf&n_tZW0sq{+EhI@KvaEM?>zz6MJ`_PGhwRHYQ5`H>$ zT%emR2;UjEKkHE5WaVYu4sOgneP7W8U6S*;0GUx?ucorb>W3le;f}jiKPBtbjo3XU z>oealV0dEp$}%fMz*#auQaf6Y?2BuD#VzOS+ecC(?-4xs8I1K}b_Ga@>9QFH37918 zBJWyg!MO}bJ4tnQt-=FatuLgE{k9wCMt&YP;~P`dtrZk(9C>{#du^E;4KWX~JR{6l zzn4hqVV6kOVok0UE1qQ!N7SkgvR=Qo|LL>o*C3GdxW)gWcq&9C_Ae^tvEmeSvDR2l@J zPz@zErD|3{HS(bODw~Z#AWTie`Hz{Wtisrdddb-6PkP5Bk0XH~177=K_?jSUT5d6@ z=fl@-=|)$fy;oH8z(q&cf*+56`EP?!Dlvg)w^8?~GkYNF_~$d)vmV^14~JI3h17Rr zg$C9R5D>_j)7R8tepr|G!i{{QsfqWiZu`?1nJ$ zSpY9dnfAH^Vd-3LYh2wfeUVjK5JK+;hG?U^B{uUQAF4xA%2gwJAO4LU7%ajJTt0;| zqROJZJr#TTa%GPzM&8}w^(}awV=E`xKPYsU`rmAsUm9Scq0qF5QJuHC!jxts8SjYq zxcNm73yk_z`PQKJaPVFQEYN&c6)rlj^ho|GZic^U5&K>!e#&pW$;{m6zV!C2jeI^M zHMmIHel-pWPQut+MYX+aBap|QGQ-b5;{8INwdebUG-vf3V>>J&oV3x3T630lBsD(7 z3&t&8f5518nEV}->0#~k^)Et)v^KTVh2imn%CA-)&iT&v9Pp#9Ou~a!>V-N0xbfG~ z5Z8??y(n}1tH^*#hFaf~-D*O*MXX+}_M5;=7yd?O{3q42#1|jTcx(NRcVE+fFpH2f z9R(C7*Uq0%UytE zPFQ|E3>d_Aj*9je8{KO0_rA9wf%r|ngixxKf;%&zMTEu zq}60Q0+EVJw&QfP^8TeTg}+lME$O+_W*%GAKLCy~g9O(&KjR;aVq$-PEqw(X0oGmu zk#F0bFhg2-=LI|BiH~?`q}jva`oKN{8bOSEv;2twI_cbaElKOd zI@gL=LHF|Ry6L6$fzZupcuobl-&D8@b*nEKqrK86Q{}=bSXN4Qa%aA?nRUk_!3j7G z1y~ivaP`u>-(}EIuf9zoe8rDx4slwgp^D zdJp@nB6*($opxi-YI|5dNJPWLwD-ckrIhG+ z7HWj59ji+0(NeT#?Y(y*i4`MOOYJ=(Hm#l55hK<&pL=iby}iHJ@9*#T{dy6c}{KuTe}*b{-lGgX8zZ~{wCpMv}^OD<39sV zeVt-Sy)0iPZW=OEih2Ct^7s9PISeEV=?Ysf8^oSf9%-69kJV~=N*kaedG1N^tYn`) zy*;p{?4#Ilb?vg9NJ=ELvWvtAS$K5r%AEwARD?MpSx5r)=|d^YlTCv5*&M?im?#pI zl{xM7kVqol?K`o%fQ=sak<4Fsu7A#G_~GK&^arJhjFWm5j3!hbO zUT^FW7Y?k%sEBgt&lp$5 zZ$4tHQTJ)eT}%w^NUfw{P_Z@Y8FqTT??UGr`?0S2Qj6%uvp0SBo6HK$2$zGXW}=@Z zDjcPKklp&aZ9FJZnS{Ud4Iocr`uU`&-rRPuL zsDpl=D1dm-+Ep*}x#%DrT$)ww%&RW{5bf_DS)ORhY@zt&21+e>C!fBz8}X=HgK5L> z%*O9BriVguvR-}$ABZ@}9yzD2QL25z03}j=nzuP`6qCW)$uD|L+|9qL|S*52r+*&pp$SJh62z%fsPE2}?h zg`6_&xjJFj@Bcd(S9+6RNJg*>sHoF+^-=GGhx{T)Q`hyWE`+v@+l0+b6ECXE^LvJ^ zwYr;tQLKB@wSmYxC#r|J)!*n`@iz=lWfZr@4y>F7`OZ4Oeg6C)=f^{{T(ilgOP;Qh z#RtbdO@C-{H@C+PBkg8QOM1E`&2l?y<)_C@L(Mh47p|Z`)%X)ORXDcK#_Kj$9(7em z%wyNAhUM>Gy>ta6w+VVKe*d5%J0~ZAZ7m>xqAYs{dN-@+bHLcS*5I6*va%_=_xXfZ z|MHTP!)?_2Tim#iC+jAfdT^4~BKEbXZ|0~+D5xjio6!jqq{ZM47gf4?RXo*G{=2rK z*)@tN?J;w=XHGjtzD%VivyEviN@Qxe(M~4)Zj6mYU~GFC#Zh> zFUas-%+0qxLizC5S)9rJyeJG7b%N{0LKsZ$g1Dm6`f#_%HM~Lb_1vJ?O2x3H^!Jxn zXeeB3L0ByR)baMp>C%Oabb80P+^{}(cU|t~!HHMK*|UDn;;f$xnGd9;0vUL~|MY$c~sq`4Wl~8iq!7bF72mP<~O-uNq5^ z&i2I#B|Q!9iTQHYQiK*f8l9pR{7UR?qRkp#KX|n7nB)s$;iwF9^u{YlNr@Bs2{>$S zn)|1c@FDO-nR=V&Q{Sf1W7Ju_@3$s3R=}?4gK>yYC)Eia;sU+W$F#(pE}r2KnN;9@ zTXd?qyT3j8F$1L6ydiD+p;|gdjTi{jg=A9eF#WY-9k0R`r;)|YR79KzTl#H zT9hU4q-^>mrA3c^J2&6SPp9-=YpGTex<9?0Qq&pa=92iUswsIqAWZSi*9$w;&o7JU zAdO(lM`0J2oqtNI8kK4C3f2{}$bup60t!pcgSW=E#UbJ7@NLNWVXtm;w7E*x<8fp# zc%&oujvXh+6LQ*=Oe(o`hyC2Yo_9?Bmam7*E9ctNyL{fJqGs07%LV1N?t544 zOByw*`n~Ma<`3lfUMJ1f5|_R?AAg{0x2~e5(rba?7!RVqMYf#Tbjw=rkBT^qY}Hr7 zK;ANbYMJ~J93eCYUDeocvrD-4MIB)mP4lcOTU$PJpVahG>E`p$oRWCYuy>!jd^Vtd9%MNjXbp1{`{f1ew9~^<7qoM7{J`j_!9enC!J0A;G zo9!DsZ{7F(5k~eT!#=LP*G2~8xup*1uJ?9&XtVCb8^7{VWx5+7CFD0+kk_LWlb{oo zU0n>0{6Zz4?@ggZ*@a3*O0j_3evF%#GQGTvo8PYYQaJI6sO+GwPm7aM*`* zhJQ80ZGB@yFJ`;=qXE32Jc5ks^BVxC#nh*l+x6{;OKVpzr{dfkqqkq{3u&=3IrQz9 zHDxU45EPy#-It@jRkwvKe8|_wCZWDLgS_M<=30|hof#!GmNJWw))%Xr7&j*#9n9Xu zm)nWMx~AQl5b%V_<1PJ$kM2;2d!(bIBk$Pu zw_dDu>(0S%{QQvWQEHY7s*_30u}qF)Mtv$CuFEApKPn9ieG-rZIr&r#@n*{mXJ1Jz zo;X}(GQ6|dNnE-Mi5*3}>O!08t)4CeN|k~FAT%nIPzL{uuktbZ_FNGBE!WfiSaV_G z-p7=KW#|0V7_$T399dOV9U^rC1uGWxizVk3m6Cc+*1n5t+(4l8+7Abnz;VA2RE{~%hc*4mp{2uOA>^vQv=|41CE zkp1lLDi;@5wI}4eMU)*^Q!am0vmWc^PHF)`DoF4B%V&vMv)X?pZ-;Z_LczG&js zg3mijSkBQ*5!%ndmYpvJJqEjAXKPnrN6HamK7#**v1ru8^eIPxTMB_oNFMk49R1AC z34G*Wp45D1nansQ-?Z+fF+y%*n)8CO^><$O;m@KHE60~H!532ZhI!4sxu-m3rX(xm zbYzSR(=r}7eiFgT3$?9`Dl&`y=v|m5c@CtKq(ojU=G-To>6Deo?SKP;;RnJF@Q3-2 zV_v0$s*H`wt$lsL&{m+OWwgaZrx9FJ82{^732d6cV+O~8R90*7NNP1qZGB*>E@J#X zW&`yT&0S%hJfD=%KeG~a?~VTUDWNP}5O?s}3Ah*iK*tteh)RGjzi` zsI7yMIbQVGEhRf~36)gwx&z*mn!Z@RuhZTqb0mCv&s*dA>Dx-e`zN!mJ)V3{U+NY6 zU#%g<^ZB3IvdD(sd7|A_iGI0v@})1pEkOJ2du1)?)2NxLvK_p+cN}!7{9Y5@^gy@VP(Yy5h}3d?j_*DT(B;&oGwm z*G3(lY#X|ccp6tH=q&GZ;fY+6(6^&>)>@jb`m{+;O-@n{c*O3lmZN*>aK}r}O66c~ z;#{A8=B7l^4Nvz9(9{^+9IDSWIV!5|GP+H{Vp?sI$uK#8@i+c&{2UCa7mqxK?gyLZ z25-9SGKX1kPaE%_O6utMCc}eN-BxUJqPj-x^QszF{0{&67yAyUN^ph z@C~0mRs4O$jg-8na$dqLE&()WnJb9oD9nj6!l&yx{rQ^fb$)|mKMAhFct`cn(f*`D zj+0Go1-m41nPVAW!F^V?PPgiI$p(|mS&xR(4VP-t?mCoQ-;4M06VNuLyX`zu#;z?y z#wO4+{(@D{kRuF4S)C!a|28+yQ1UR`YqZlW_A>U}%u zpB0tJqUOpVmRWn)7gGe;QfCUVmM(s1`M5mJu}C`BU$=i*-==d45IW8bJGcqEpqi*P z&I^V)!mPk!rD+!)2j{uA8mn}cxd^Ye)Q!Bg0Ua_2ALI5$Rl54Uv*F(HUup;mt^I1c=+;u6oNjZLx;;jR6d_Yyy>;!i&YQ`8`MP_dhTyUM zSVNI3vAr(`GiytR%#TvTYAmHa!2Z-Wh_h+GjqSY12AADQQ{}|n!|6U}&5Dd8&FLk2 ztqRRBn#>*CR-dq1lS}q#T7|54)WdE@69q^eL_cJdAuf3mFif|Cy~n;ZF<3eFwBP0s zc2-!MK7#=OO9aE8E70FY+NLMkoFeg%yM6kWtSBHB&&qwj!Y+x!#7?okICFo|XL5F8 z4h$L=e#Yxzc7Pp-$UMtqr8d^7x7uGV!Y(b`%;!@tfsSTdw&fc2>N=Rz?A1GdOHhH2 z9EXNC{AlzH8cpBoJE(;9F)6-;puBx*CZt4kxkQkI6XfX}y3e!8BxkwKYR1DbnsBGJ zY`wvI<b1; zYN=nd>M~Z}DL3`th6qL^y57_G4W!0#)Me2b{42aAQ5C6jsB|L=Jc*Y@H-PT0=1#d6A*Q)VlEQ0oCp^Y2r z3Z{v}I|VUSKFDjn?RGjDA0N5H_uqaYDB|5dETr#r>_$391V=t=eZ`KvHr&3f{hgiU@r|9yfBcFj z#t{qCZ}+(~X?FkvSL1U@=zIL>!5 zZm07R?TVI`h`iv1ZkX9--AJiBKk_L7cYUGuq!(x`%`RElMB%;;cZ?U@L`Y4!J@#09Od_;Ka@#-YX%X!b! zyh!&*1Zo-kppK5uBwyZy_1wGd!{#j=kH9M2Ocwn8%wCkk%S&=RO26+A1GaYj-?C5Z=1o!puip$cuwy9w` z{m4`#yRl!xe8KwCGbn`yjdpw}JMNTM-ZJ`LlS=Q1%ex9{gz{~b*N0__>hQwqc14q4 zv2?~iV$Yi1-5zLW7X`n8EiJF-Mg~$>zB$YX9eZq?okz@_&IcORzOAcu$*48=chPT% zG*1y-Q2^Aut#m}N-<1zOhE)zh`m2T%LS71i$|`0CsF5lfGja3hcQ0(DSQn_DlAsy& zGC&TQmE09=MrJ{at28>T6K2Z^;&nPQ^tYj_rvfX85`h#7FMn*t?3(zKRX*RDBO$2~ z!I!}gT4c0ok9Kuhq{sCWpcjWtMp-jOM{A6zG%eit8>D%c+9`kYzo~BowSXQW(lR+U zU4KL_z?=9qf9W9BOJ}cU+ECD{=*|i$^z`vd(h!-^*)VJyp0zdLCW&Icf`{}DQ(i9% zZ|E#O>L9ESx#L4b{j^FzF#v6;S+)WM!x<|*7hh>}_~s$6cJl;sS1vV`IAte;&%D^# zMVvv|@(@?(;2EdIGqttV87$RDLwAr#ZqEtq8@Y8y;Y0LI4kM>yLMqAvW4#g@Pc@td zu)dB%QYSJ4n|6^roQd1QY@MMXV(PmYIZ zYgRa`-i$`jOtQWZgw*d&vQ7=uWTte>nI@hXu4}WY{WzJ85DGZ0VCr|E|j7$(F3@D1U z3fp!755&}Mr|&yWE2U9+Hom)k3c8V*oRzuksJF)Ky@3=`%fD@i92bx(IhC5lK+Vsh zzbxf1o2$0kVXC7D+hbNSqZ&yCy`E`r1vJom{->39YWG#)5^-MJ)*{2M?|bQ4rt$nI zv;AcXE+(<$)HQQmcs6%OQ6ayEhqD9vE$#f6?-h|@Rf)-dhEGi+2Fyi$^Zc)edR_Jo zl~?jhhN*Fn%OF7^{>d)SHV-A9lBUB~ zFcjr9SpSoL^`qact;YHGu?E6pP>3H(Hl^WAZgtyn0mR5W{;Pc=2eCwbb389Ub{12y zzBynd6xH#Tpk?)N;+1mW;_Y|aB#@V%>O@Ju6I&6z|9Ea*7#&RkIdQqL|A9Qq!OB#4 z=H$%}b%_4Zv+eW05fUpjaS6-YEFrEJG-6Yv$Y)oq-6}{`E*5v)-8iJmVYDq?!u9^@ zdsP>#7wFp+osg);!Bk(~E^v~H;eKt#x|AU-lB_D5Yx;gY>$4Xvut^I7{7eJ<(3?FX zWbKL@@LQk0o=dJb^k}guhV!e!`FJqQk6;Kj5G7J!__ z)r~X#Sxy7%DEI{3qVnr&RCD=d?s7*qkQNNeqxTs8?YXJJP>k!+%DwgJ-6)YWmrtHc z4E9jM5REVCRb@%=!n5Pp;Hp~%`P)GtBoJbnUaWChKM|-+YTHH^wkSk|M9HvaA>`L58dxVpkWRjKDDBT6?i$M7FnU*1zj?XV|0H-S#3WQ94`UE#RUAA z5$t~)>a|9##yYMmqP&-+vHK>qY{F}hgH96SSHbfDoorYIgQ+ZD$X|HI7B-%T3KQkt zv%hB**p29~x2*fUw{xJ-XKSQci%X(|@=bm@r@a>}J9NAH#8}S0;Ynt_t2y{e3Njib z3aYpdciXiWDH()W6xg+QNTb~?!l}joTE2E!F1yyL?)>tT{<*6&Lj67%uVbHvNf~3t zWR(o5SE5rQI@?Sq!l8uJ1+_`q-x#&|mK%5BG_2OLCLXdBwZ~8`lY3KFd+g zq|x2iW4QTsUT+(ij8@aFxSdLK?%cN{1K{e0_Xzj>`H9|e(~{NL>KnS-UiP9M7YH;N za|(nWWHe$;SA3pQ(+cgz4=TLmw=T|PQaSy5&MiS{1<*JD`1zIV5%_Ua@AgJ`yCXdN zdFB&L?TK@P+mX47gy-`*k{6&!FXncG_iVBDlt8I zM3V2)`3!I@Yb5<|pccApZWu>prZ92K0M|zK_oZh|r3@3|dpo>skM=Z(nyF7GVwRF| zWbeM{ky5g9od&zEW zX~LQ(lma?$BW1q)I==jS#`j9r*!zh&@&uV-RzlMVC2G4pXej7IWR_q}O)^vm7=BNC*Xa|eK@qL)bgC-z_ZJg{T*LYA$FnQL5U_BA zB|in8qf4%t7z5eD((g`~XWE)eD(nBV&$)B&Jg?MU_{T4E*T;yPH6gCczZk)z+3_hT zhaz6EVzKeKHq~x*N5)O$4ev9|UQ;6Y=PowPUrlqU-!J*^zTfPwx&04hAFw-rA0E4Ks_NASe-mx$rm2J{@y|}2i9TBz`JUya zsut8uaH<0MEvOLsUK)V}=J zXyhNWE^)sbBNkY*Yw@M#?Se=?8`b2|dgB=a^s(Mo>HxCazebl_Pqu(_81d5j7cmX%@l5K!mT_$V)&3_Ghk!lsh6EbQCJk;#{ zkcxdCDtujrmapY7JBv)N(`g?`wwbvr;D3v8{~WS4CBKOJ!+wQsgY1bDaxU$0L5!YX zQDPr8+R8`Dx!2mucHnmUJ1e6IZ3oxo7PSTZcIcm2LEfMr=^!{2LeDh9a8!G-=n2Kd znu;{Un6lboNCxJ%s;e@n=Pu12q#L2viw5Y{f@@TQE_Hx4^nPvUiq|+)h#+SR4=QsI zgKQ;GFFY;y)*FlCVdqzbnUn&lRWUw0+-}L$l(NQ<%OZs6doZ4v_qiz%uVGF7UgcJQ zu8O9%@n3GJV&Z|!uH#whlu;weIQ!J!LHEqKF}=|-ZDT`H%X`hU$>BvvBg?@~18$}~ z%_oAo$*Ic^T$tb^Xu=+uf%L7t%sFk0z-j3XGWZSzua$Jw>qM1q=)Y5JT!e zbF~;ZWb34%H)_TdsiE#C+58DX)`Oa?3YdI*LjTZXj+f&u>54XNs3t_CiqQkHU#R)& z!wEOR<&FHkhRL|EcIG-To?TxZr>6J$&1tJjLhEsEeW~!EF!z<$zg1mzFfB`eX2F|d zHAux*Qu#{WetB>vz)gsI!+4S#=h`IHKLL(O;0tlxc~P(ZSe6QAq8fqNfQryIFPPuj zfse||H>wSM;YZr6ZvUG7iD3a50geTgu!siy+c+gtYYu!;%WTHghy~Wm{C-k^;n5 zh_z9fEfluhRu(HyNP!9{sgDxbBd}jtb1eWg6vEz`{p73YMa6YZG$p>*BP@18oXD$o zw99pZ;?d(4<0If~Fn$XT5q=ec&3wT=(Ahk$43ZYFsWGy$+gl&`(+FPPWmRPf7-pyw z5Sgs%Qs}zI_6711P!dBk4%OAp1CsUyMal5rfU!&c_67LwEjrlTnHye&L!J8@&^)}W zUfR@|ex!gbyURk(bym{HB?V5ZiQLM;nK+siDDHyG#6ai%UeBcxFyD0jZYXMwSMe8Ygn5c691t7^u`;`xL}gFOd@`Nm`w{>_Ci zD*a-08k6RI3$vZNF+H({f(~WZ1@U~QN+=$WC#q|V!E#-n(o!E-1zhh-GMMEZHp`kZ zLA@sI_l+KL)Nyz1Prd(Q(5l2Hj9&3ug@vmL0;rj+ME7aE&sHk8$j8TpGOzhB@r`bKa`FR%9b%rO4%e}00e@^eg7nT1M4gtdqt>y+_ z4OC(3Ec=%MOU1ntrhQ5pVF!tyAUy zBg*gxw%_63H1FQivw~`CQ(bq9*N)YxICUJe-`RfSH)43A*dTHQfP|M@Q9yckFR@Lt z1oqFSe zrDAD-TzL>kqn-Er{ItiPwB)(+Tv;hhiqFVSqosfRQ0VL%%;|&a=xlGLa6Z#x4kM2Q z74&Muazy~biu4qI@+MB&!wFG!N3DCCQ?b?vu;))cG_9HPg>>wRExL;DvD${$q8np& zY=^-Sn&TZVd>(9NN|5G_KJovZb}_pYCV#cCKP{|^MV5-+@+Bp~I~^IIa+r>XvM?=| z(PfW5tX^5#`_8=S@DSX>OK&q!linq^l=A%won97a6h&zENW&4N=N_hJCT&^?!JQ&4;ms1%o!{B`$>_9B$R#vcqX>H_+f zHzOkDLm)jIX8K|GC#y8$gs9e@JvBA;PjM%cm=X*W8o-N2vnZZ+dwllw4tFcGXp4hG zC-SO}37!!sM=8v1_c4td+gPQJvSZ1$-!uO6GtQWTx^7}!9Gf)EgSWgSzHU@)r-3H0 zE0*Fa>*WsNZC7AOFMv>DIesA_UEJVnsa-Z6_2@p-Ue7nOp{`OutO@%Xm z3$D_S=j1QJ3iqE(U!l={?$@h#F|`;A#}H_@mLC7@qoJpFHiE+dJB@}Adx3`1)=Ffh zS`NC%s7`VSF$%@Kce!|I;aFv^in6fulXCtO7tX!cBMZOt2j|%$-S?I%FTs8SA(nL` z?kZ7(0qpKP)5$B9qjjMDDP6rl8u_wj1OTF|=q@wA4a%cyzlKd(B!m+!;%f;Z_P9Z}o$t_Rl3S}>mJv--<> zMR#Xay?E=~{6!SH)h&ZHTSiu=2fea<#}pK9$qu6ZnkUf>ks69NL$5G>8chQ`2NxQF z*?7pxn5jR0wlW-W%P+zeo8$*&OOqGIa>=QMG!|<0kHlv75X74q(~4^j36&4OAHL1B znSQ#>Nwoh<`y1_oNu-nvxq`S-T-hQGh4EpMV*R-6vS<(B=2&H>3G<}t&%Pd{r&Uma zUyXY49pwLrWN|kYQ|G)WLai@D{DG$ZH44$my{f&!t(#v4Bh$I!lED3jeT{gg2sUXA zB{ng5E40XO6Cb-T0S)JL0dS#Q`t13P%jK1B`05r|aiG?HQ7rn~?rxNT`1+qWr2L4_ z9MGTWvV#yv+{Cig;OTf+UZEndrb`4 z3JQNeEqvVX4pC!(34O5Fm+QISdj@Hs`*8@h&#l6F`En>6SXKRsz~Yuk-~pvGp6QDG z|J~EPZ)W<;`6CNK^t=A@DLf-7@*z5fyO4QT_vJ6OAx|Uz0{ARRH2_NTyB5kaX#Q#A zsBSEf*$2Hv4^t3X7RA9^{kagAlXHs(#mR^<0}f%EqTL0OK4wBA@tRw_1uV~T(>Kq| zee|n8zmv_ZpzMB?x2Z(vU0Ifa6~~6{>Az6LZ+g8YZ(COqPh?pneQMQB^$LxCBi%ng zKhTV-G+LT+kL7TSmme^l>>b@nhrG5V?8lmBW%@WUGPl*P> zPfp!)RIs6X7%ExxvoLVioqc42uducN?SU>na&he8=n3)R24;4af3R!_0IW0dq&PHw zT~xd~Zv^mPAb4#D%|gcXG_aDR2Jt?EGycor+=i8e>vHHwkM&a`7vF0pfy}u#6p2)E zO-;xWMh;$M@Es#~a;!dXVUb4pUVgjdNWdl*XJ2cBfwJ8@qZUnD9T=bXx1Go$`)s%P z$ITsH|C0(lzpAs>Ta)kgfA%}~Vd}5{?e_tD|Nm9R#Qz1re2mLtzD>?x*EH%n z^kb3_QZta^FT%8q%gC44xvN!Myd$N5@5!C)H>vWmzzI!$x|-%xbl?Pgg}g*Ho8DV5E$r)8<8r!fim8K7;+!f;SKNO!fSe~BM|b!z%cy(k zqa`n%dAqbTwc+d^>He#A`m=+}Li>-3ZB!)NwN(avH?SufdwP>3vvOuJK8r5DrP3+^ zqk{aVp5cJZv(XG4Hw}Hm`hz$v*M2-?9X<^KRx3jPdJ9ySx0~pe*gPUSSIoly~2I<1RASe zajA@aGrWm$|D;HRhpu$p+v#+Au5y2=MPUk(X3^04^688~5lvq)Sp${4W@n6JIIiM3#Wy5v+T-m0}C#kJ&sofNCfs5$y=7^X11(fuV^l+KP)^ zne20d!~H8A1r8GxXG!W@FV8PRi}bqDP)#0l#I4aM0OsSyw%3%n=tMp4>>rl`Xedkv zpQE6kR$@}35+3w~5NhzxI=#Y@-=a;Qu!maz*PF1J>CZLYbqJHL+q-&FDR4Hby=tv~ zb2T^MyvK)YIeG#OStdZCKz8Mcr`xttCsX&?y2a0hV}C5|gBQXJM{-Jl!JN8$rhc;6 zUgpu8(KJ?%^FJ& zc_r4DF)%}9t+ZXdynY3TXlJ*7WJE3?|Elfd;oW^UE(52s#l4f}_$2z)ivJw7h0d6> z`{zQNggM*pU?)0 zSJ5RvifiD_R(_JUliPasB*IrY)4%#O=Un20*V?=VF56Ver^IvSB&S?N0GW14L0m|ht%DE zC`dg1eQi*!T?6N($=#GZFH6lF)bqcN32EvI_!llVkGzP%! z1Xhgw55p7RQ6bEPp>h7GWqMQU{IhHdC;#kqWeRVHoefUI%A@PE1J&VB5vLL~5n5!Q z{M=yZ288s@vGuVjEGbYqsBWVyUJu1PS>)V8wxE5DiDDl?0C`bn1m%1NtA-$Fw~uWz zP!UyQ24)ckxtLzhW{E~YWC-TJ9KrI_4E^& zGO)SyCdE`A87-Y&niph}Gx)aQ;nq!<8C5?c`Jel~IqdBq?$ifDr`ELa?fQhg*9?Q( zcch-~_(5#{;=y~ey(8oC+Tc!R=<;vx8cE~+G2aKa zlf`_jQ?ZPKa(1wGp{UgjKQ>*d9wjFAtHT*tCxoU!u9k;3ey>b!^g_|dz7eiKST_&1 z@4UzK@U{-8oA+yZC8B;R^mG|H`D*wJMu@m7lnZcisNZ+bjxa@{ScfnTE{aVOs_ztl zpG#j-KC3?AiXS5rs`}MJy8xRoh_r5T}|EY3jcuGE- zGruH>k?8l`r$ciJ_n*CNeBZltD_f2Dt*4k}+d=|Ze|LpPIaq@k(@0Q=$UID9390#2 z8{P zI}>DCaT^E6?#smRH@V4v&zq#Bd-rW8Qy0;v7Er%5NaD|X84{?B{1w=68 zOB256wP%|X$LpC!5IKOe@+s8J^}zL|h;Z!)Op&bBZj0^?4l|Kr?KhIm(LIWLX>|*G z9iiv|UGzKuQ=9RSk}|FC^3#%XQVl7|=~0b=OasauZQ=zD126pL+UtRT16*}(GBG;H zzr6p*{c7f{l9qct{HhN7P(0IR5NBs8cl{zCi};Z$?K*@T&v&030r17ef_jmq+{1V9 zj7`x-hv?_<`Hnvaq5sYFAgbhO33!$4S^l?RQ(yb`lqI@B*rM%<)L2s>PPdEi?`8C9 z&VgWbIEbq;TIj~eGtPQdX{uV&n`8rgm(Qb;UwQ)n=B7J|E@ZNmhmK;;^`4A-KQr|- z-s@f9Y1(#($a;9J0lFMVT+8Qo;N>YDzvzWn+x9X1vDB{lEj|h#p#{?IvwCTj^2^O( zG(nBhl}BrbUH}PJV=Xci^Y5RwSD7h5Qif|0WHR)-hZx6kGjKDc0QMv>fk!%|jqIAg zfh{%bWw3ZDs6*UT5&%z4=_HP2NBm6)YoZjY<}`y6cC@^XZCg3>JZ04mxz~J3p>ngL z#64bpo}Std$!;nV%e|w`trD(}2bCBW7g-j zUJZ0?M$)Pk;aP;{cmLG(`zczhD%HCki@})R1)!_I2bOwv z*cr&vuH1fuHtk6fJEijCF4^D*Fdjg7LX%quS+Ikw*c<<ZPKMVk4qc? zaLBWxZCgNp%3p4+PM{naEBH{K?{+z5JP6a<*R=IP>7%kDZ9j7;>8C*qhS-mS7M{gI;sunX%9tY zxzP1F`R$-57pedMa7bPeqgkAxf!oFl)wMr!B}F}Dnc5U`=-u;Z-<(Y?TBUD*yU*hC z-UIJb>j$Ai9ewiapp&oCO#o{1nTIc3uo)wsTJ)-=(7hTEJsonnEOl;Gq5vO}4F}lD zi3S<@+=TSQyujZbShl;r5bUmibB?6FUP3O?Di_3zs*ilW28;734icnezD0?x(AD05 z8cS?aqy++|G=lGSx;|R`A>H~IloK>;wURh45kD6ZVNET@9^PTMd%&7r9uN^ZY$u$* zsgqK9kk{usRZM!nU;#QC^q{SMs-oAT`MiRX*po{DycwZeyeZ*w; z7mI3U61!;B?hKb#n+|@0WO&W^-YcE1cdI+pgO!vgUo|KDs~x?K-$Z1-f1&i2O`8_> zK)#A6{-$mGDdb2=kfGWZMpsx3WMEfL7{VR@xr*l# zEMI`0oQL0WV0A5Xy(NmM95}iAt6HGQ+!7W(r-+j&$UMbKYt`*|qRTjQa_(F5IAbiN z+>f5YrBfVQ=VdS>My7*FSFy!0$a|{Aj9zmQth?5kn~)LKsP>rffhF+FfpKAdTs$vI ztZDbFZhgNzLbtgsJFkz@oh;EIL&h|~_z|S=`f%nh7Wz7Wp|S^-3p|Rh29SE`sH$)6 z6#Ke<(l5;m4>cUx5!UJ7^ z;emLA7GRw!H8AYmWt9h=r5yq}DJ;|P5;Kza0vpr!)6gSZyhDO6+BJu7fDzss7@$FC zTdd*1?l99OR-vRXlK{d`Yk;qjm*UpYmQMQl6O8z?LMWL_#E`iJ{4tNty1>4% zNfE$V;&>F^X02AM{DmUV6fT2GfNb2Y0LZvVbbN(xuz6XJCwl$t+<+Plp4CU96WgFH z7eiK-CaS_AN!FD=G3UL0o=^uMVjk#xNiD!*ybD)mb#-FyH5It8_HEpw@Zu8ktCg5n>WYi?zR z7(-t~$SePobC_s6nNPr<&^7FS`w&I+{)ct7|It#58OwgR?K0p4Fy0YYOsZr=gwnA{I?P$>n=Z4ogvgs2b<8?xAht1-IP=kq~ua7Vcmw`d^%joiQ01CyzQ(? z95gqz)%VN(mRF&&V`0re$YZ2uLu^)AOX^{+k1J?2_``S(U&dq)>C+Dq=yhYG?sYnA zt7fnjFb8=YKbimf(Ru4h3sexkIrQ0c-c?mNdGIZY7aJ93g(Z@m-c)Z@rHTsP)PA@y)j9i0vzb{4?`8SDA$>`j&F^NBVX0%3w!i`m8QF@tJ{V3*kHcGLsDC zz|Y>9)X7dBKe)^pvjBOcK0I7p)#8GyLMT@2RT6S$;u}6`ukF$-({ z#WaO}dUL2MKF1+hIZNJ9n(JY^k}TN}W>D0cN|h>7i9gskwVCnT^s$05H_6?&>;p}z zD0mR20%0pLiy9%PhuvTi-cSVYK7BBZ7;`ekOzKb%uO|G1KbMSFP*(bp9z-;3%1RBC zgn8-OKw-@(upc6 zgJ@{tAJ~wrXfIIjJpr)_%3CLBgD0{PKK!q{KzKfbF~^zCubEA=wT?w(Q&_sgX7~WY z>wQ-)8mgWY&Qpqg z3Wxu$Q^}FJ?3)-)(EJ%+qMc>S6>FJdJiAStR0~oT2`AWNVm5QiIWqOfp-=7JbT+&w%by|`(zYL zS?F2wori;AMsdKH;@$Mwm56&mJ})@`J~{8M9xVy%dYqVe9ej5u_J&(d)N|-Wreb_6{Te@_hw5P+u?nby&%lp! z4*#(C_5>-Zh{CI1nALgmg=y?HNPukn{))^u1Swdby$j9+19~MH^SU#vGN@Bn^bDj} zPD^?BuxG;`^iGI-Az>(iz6|q3$(Aks7sHM?3CHqA10wagj>zA>03S*2SNijZVZeAJ zQR#h~UzHAqWm_-Ki*F`}PTtP#;ks?qYzI~J`(9MJK7oFksl+7_2|fzWHQ3zLsjGrA zkhCWFEj|@>ZfSXysM@r7i2tfwT3i;kRGabk_##Hn5X05}-T4jl4lNSND=&B^tlbnn zNst-^#hiWfd53k5AujcD5cV%v`P5$*I^(dr#J%!95R+JIx{y~ouc^HxZ^lslpJVp5 ziTXN=NId*zH0FG(WVvaG&W=%#e26&B%kF7??E9C37jiLq z3Q3R2UWX1OuWxmR5cJnh#kPAF=jX03tpCAqu8;lJ@+bK z6xmzMe1FuwdQm)X^YB|X$Jw1eDnVvh23_eXqlYO-=*&@qiGR6cn|W=o?)u4^Sc9-x z;*Oo(>$iRKLAvE_KbxMi|3l-KVL=X$a73?Tu6fz*xs%)`uk$^tl2x4-wLMJkHSkYZ z_fmi#3!2VzWOkDj0=s7gram6dmB5nzTh>UQE&Ops`(+U)ekqbeTGpD-OOE`$m8to` z(WX{toy+yOhWE~%BXIqvjLpzMil>{u;0^y_0x-lzt5J4_dW_y?)Eo(GZFV1#;vo#S zU!6EuqE4ygS$!WSt!2pCrn# zKc?aV@l5vOE^+=~T6h@`{ZV6)ypwpZ^9n6+%QIQX9bRogK03TQ+@0Lx;t7XcSmvFl z^QQwfb?qI3zy1!;KX??n|8YKZx^9^T?S$|_u=rs63w zeevSjweJBr9|8h|S3k{oe;By1y6))jKb5~fwI07!Zz^zlw|26px2JETPjbKKNlMTU z7mv=zR(_s+JC=;CHw2I+S?axMuR?DG%2ej5-Zm_YljlvikVOjCDKw`Pr@hnht!`3h zy#KvGt_Swlo&mDdZuyy>NyjKVgMU|$m4j1BFpXtfd^V@RXIx-ZCRNbFXd5;*s)^3lqe5 z6}yxTZOUSZTo2vNz83a|p3gJDEuQ0}D`MO{Q0<`}M5Kg3s1hmAJ3;@-y-$D1{pZ_F zCBz$j4n^}SipG(f+F{0d!!Om8QQTwkl{d9PX0%PWD&wC0is3V2{ExXx;6t zZ%MoxvdH-a?jc8h6%PJ~bV9EiFJ&FfXVzL)v=P+^9`OD8dMBM@7yI=RP!#7(&}DI# zx57%;$MY*GmXCc3Z$$YJr8y)haxZ{wG&X!o`Tf-;VgMq%86F9bEIrcqdjXT)fi z^!V9wj|<-8Rg_H2=+(JbvrPReQz!|VLG|XLTZ>Y-|E^LWF4|}M9}8Dbd?!Tw=fZK^ z0@Ipxg>t(h-*!yEA4p}WQqsHZjBq;s_&?0OWmH>j*EL$8Kq&><;$BMe;!uiHXmNLU zcXugXytun-a3_=^DH=Su6C`+W35WZBzW0&$Ip_R6=U;X*M)n@pwPenDt+jTk@sK{n zF-ar&WDpxnQ@@@mI2asd9FO>^M0>>u`1hrJT2%VjtezTelD+vt9l@@Tm?NReC+BA$ z*MA$@XV~@B#}3Qlg%DFsgA36?5?iNP{tup#)&O`6R`i0hwFvP zvu`x5qP(gby6+rY&L4V`Qkd}*H_S$-RnI2`o?1B@zRAc?3qo|x`RwQ4A3`%m`gB`^ zc&SAVuxPoKaOvOMrBfxV0@Lw99#-6mLRigH)`vXqXMd|*1ROr^+Z))#xS+a7U_ zEG8mP8#V0SZ6eg2I8&*hJo$Xp^7Me$ppSlU`J`8YqW38+{bdp z4&pSNPLvVB70T-@<|{e)2im@4pIIZtZi6pa+qAqw|8pU~I&HmlmwR^^#=t34OtKs2 zu-*YjW<-2Jd_wD3h>GZSz%Cii^d?&r*+yS6yw6_ObT8{1>c{j~t=f-yJ01yw3Vbu| zPCszj-!MljN5?D=PxwRJSDIHrqkrMSZoUBknJogRaUMlb3nIO9>HGt=7Bg*l?6-z? ziy%&+p^cd<$_lyL^~4n<&-x`jFC10n2`EIMTJh59J(MI?7GutXp-)a96;Nk<@u7Xi zY#F)Bp1gv1U3b9ab>nm6nxX=uoBG(jEk+?9s1D2ic(b(OGlaDgK=~B&e7HSza^pvo zo4#t3$JAYVyXULXU{w+7?Qvt;Rl&q;zT-i?%NnubP3tGyy!?l40$(v7nQ9WkT*^~K zwqS7_M};P#<`DJ!YH$b5Ru#P9bCIxStImHd;e<2uVX@2JLGg&;yJBN*Mz3uzg&Z-} zDm|L3z#+1@0ctNdZJr=C5t*HXr3e9z0~2?UmS}KShG5@T3m@(-iBflaymu{q)P>bIe$qn!&hpyMJx#F1GHw=o(QS zmheo)_hv~Be(u1Uww!o&{C>s6ayqSDpSa0zV+m;TVNX+hvDoqSe0$_O!3+dvm25P7 ztgUc|qkGt-Ot}qD>mgDCt44Zzt$+oM5#L7bec=mjXRZm2?5PQXeeCUv;!P_;bBr}< z{u@#UET7DniQClD1HEu*Q1bq|`Qbl|P_R72!&(dt&sjv++FCoaB@w#aj))X2nS#ly^^Oumrdiy8mi3{Oz_j({Be z*)Tz}6*|y--zGDH_k>x+U|F(?STS1FXID<2=&iat>?uSR$j%1GFW8Z;XA@6&1ef7L zmWBTOaXooa%HhtWc5gM;jQy>jlYTpMyw!f7QdXDWG1{ihnDEJ|5tP4^nA@6>dgXrZ zDlW(wXzPCPxwJ@<@9B^uCBY|o$XBj2sjM=52Uoj9XnZy2Dph%kGb2NPGBaeYW05mI zcW@`bW^>Ecv?n`1_kfGnC%zAiMqNXHl88S*@i~TV_`E|@%0q!-gkk$Ao;N##u9e0j zUwU2bv5v8_NA@{mC2cf50mNu__@1or>K|5^qbgb%Vx|unEsb-bZ7aJXDqC(V-X6B; z-&%qfc>X__;MvDR8vB)@byb*hef~>wdn*y6^&0DH3Rk-gQ6+)PlIN55LyJp<>oLfd z5qCGYg9`JB&kW!1S3=Tm?$n0gr%`3CaF(#uniE&yl%q%0YvUjt5_?))R8<9?#vd$Y z>+kA3L701YcS`WcvSbWWSDBvWJy5jG-_L5@B&LQPgxmI%#BD?Yn@}d9oWAH?iYBsc zv8ZcgQQ~)pC7ddFnHm(M_hi<|`saJPlIP+1XkB64{b@>AsXX;YHcD-0StE+c=H$H% z9A`PapnhE^Nh_V%+VukNkp4o9mKfWhQVKJrU@M+Hv;6t3=}5iZrJ-JJeOy8w+nvtf z&5~(pL9g}FmR*f06jUsib2Rf6-uP`0+4QX0IiITWr`h1Q(_uapvXV9lhGr3!W>I14z7Ho1jOEP^OlV7Lm48@)rjFBIMsTl>4?PUq`2Uv4 z|H|pN!wu@DLM>r4kHwmtbi`%6XJ6Vw@oSaKo==>%z2Q1@E$mv5`ATIvF4K8#DJOs@ z5=D1*sBvws(aiM)8dXJhMNnDzQHjA?QX}VPm3|*DzHhWkQc$y1LC{kbAaRjr%z{eF zIDWXC+SX-lPC{+$Z!iFbnZ3)^zcre@_l-V}zJ3_!^*V`Vdr4O3o9S{q$8v4{(b+(2 zIHTz_mfv1zL}Sc}>J#-tk??5yr{9H|$F3%m_I#X9)`NfpnR@jQ?LY+XQNB3vwDn~V zO3vC5fmUD|VG81R?ALI7f~HwV+JL)Pg8O&)OX|F#83!+FYTc=IU-u?mi268Fh=P1K=17MfUEb5|EeD+G_ocKnh|l z`t0R@BCi(@+j?%F3`}R}0vzYtmCM3be`C6cPq48;>Jv63y68$yH@cFSOfxSNRBV=d zqB!lTm-3#OZy?eK7aQUd_`9PB9tbI8AJWTQu@p~WpTi)uO}{`GX0yT5!J&84!9h=> zyso_;!JSuQlHN_#Aqqgca-_#FKTp%smQJ^Tn30d-+Dq6#uK@DXBTAF!X`fGLd_VP@ zJ||Bp1%*?dSf-wN4KViuD1wCo;?}WZQoh+UC!4r*8@W_b$7RM%`yDK0B;BBfRK&>| zyFN1N*!6Nc|j>NaJGee&v#^ph{VnlO+G1jRKuoz zr%)ik1H>*Ddaf~c$J;yPWoBc83i$cb%Rq(worYq^-AwR#_{YCtPFGsn{c%86qi--$ zKU2`1MRZYMfKXv0dr}InkohfU%&F*b{F_h0w^@X@35d0v|MJ(+VLCvB_`N~y$sntq zr->%2s_M{%S0K3M9C}QNFnT;CwcM_l*qUA3ohj#7DZ=%GG8)bF4y%t>T96;z_#$Wj z{+K)D>%~kd*pSw6GmvY!bnU`71#L7RNNhS$>@2?vaDboRzL7O>L1~F4PV%mG?Db}Q zK2r=!i*P093_Nd@$y)J#?0Jn(B0Y`?73aG`-dm=f7adw8Nn&1>Am5GcN+Wi}eu;Zl}$PgqsnSLSq+hE-!D z)2H$F+6U%Be!IB)ETl3~&WzTMoJToVoGtPTzr)n{Kl^^nn8QN>_H4hXocb<)s6A1o zh3cz{D!JpxK4b^<4jdN3$8dnNaUx)Sx0DEDNDoudO_&M*Qhg57!3>yLnA&h>cbJUR zbu`2$&A8l5NkODvIYxTeh50^UibeW7V@|>M@UR#f>ZJN|+P9l<*{zA`)r z?Z&V_!Ksp$FFrKdbMn2~Q|P!wDl2lfK4shp$1bv>S{0vwcaF!LqhEdeFTkS;%)B-N z_GLVhN?Z)(o=JkyK z>SIv{7=IFy`Rw&EDuvfBu593A*D245@e~DKRLvyKHaP=X{2=5T0Sf?2) zG#n0lcsBPsmOMCC*xEzN`|GYAKwgf2(5vSB)2Dr5Uwd6QpOc!)u1%}};NW!Qkl9_g zC4ulCW>?eUtu0qkzU;>imt+FqixAb3Z-*VQqoQTk)J6*=TyKGagMReZoD_vNF3t!L zf6A=Xf)FPKl*pAnJKI#EqZU#?B zOUghKx$tSv_dBm0$gVLNnEa31pC`!vy9@RO;G`T|iE}u~Cv?D*-~n73EZ(|*nIG@J z{VV`@`#COFcc>gMoNyxF{u1%CXL*R}KLnK*RA;5GCl0z>%CRJ>+2(pWO5OIK5foN^ z*}`LiZaTc4EQ4JwC4>c4DxEKC)KV8NDwXP->3e$oJ$))Htr$M&wnz# zZfQhOB_S_)TjFvD%g-&*H)$0P^fM$g#pVyzA)(LLV6X@(!O&}))CeJ`=wAM~x8x3D z6p}OH;{&6XZ>YlHrEshje==-cVRVb}T6J5>Dl71$AG^7xM1$0ZRyDvDJm8|*$>rc(NLJE&7pIq^paUjIA_ z`sdnXVSE8SwlMN?_9o2hkrV_Dnf=g zLv^8hTW*Sn;B>F`*3<=Spo%|1xe_?KkVl{&}W zpmdq`dmGBV{vsNu^lol0^4|Lk1tmyX{4E$S3=0v+RKKA5(d^K>N|opjKmSUJPVGx= zjt;uzp0s%j$5jPam!IyqvF{#aKL?$2Ua4ldH^5wejr!}ZHVd?D%N$i$J(K}tt|0CfQI}v#h%UFt$;)HQFeu0-UDRk`)2C>2A_#> zzh4N?M=m0#b=DDWHegJ$fh3TiJ>x9pIi?jy){7b~27qYH^Df1x_lSy+4T1NBtE>V# z?u3TJudI01Urd5Ss3nD66lffHI)u>wI8iy#$v_W`#-~#}8PFJ4`6E{-ZFk~!GfztN z<90-O00ybY-PjvTXDhZ4p3v_IRyHqP7vztW#Bwq;2wl7h$XhA!{hC!)hjl_#4NIQw zsLemmAKvq0xVwGoPu)j3GMS$9*_{EtU%Bz;)a@nnNw{bzfuDxMJ|D}M6AZmL90>5` zyzSN%##)=D^p>uzd@3S_*=hsHK@J()Qq^?kXQw{d80wt1%npDF$9E7hb6sX|aq>zt(y`PAw(g$Xik>-SO zAIlyuX1?7h|4t&3Xkl?8KHVk2v!jY2_Lz27ElYFxq9PNwxd9Z*;`2#|ovx|m^Ia@{ zDp1bbc3=!UYxF$AH762>PjA`vrLN->(6sWt!D8M7+#Il6els&rhnGn0&=TSQ@=bnt z#8(6r9ANH_oP?8VBX4r!mO=x1#Vu+iqc-CI-QIt@H~bYs2-%$MdwgeOZR8)>@M&t9 zdMRtb6;|O6z&mff9L(hw|K-BReem9P*Zh;g^R^2u<+8!>YHto0S32&v{TB%}!cctw zCA8}%KJVa*Cs~O_H&@t>_fL-cBVU$>Hd}I6q{VfNV89Iz7|VGp`!1vKajP(2;#32P zIdoTWubDvah0^!QTnE|$s6hWXrGWJ77yELRJl)$f9Ycl4P|V636F1%X<9eRsPlaBa z)$)HG0c>egd28n%Lh#bk|mg}D_p@D4z9Wd<@ zs+;m^wEkw|fXas3ZAS=dknr;1%Jbrv=^jtGtoKRX6#Es*q*(sh7q5#+3yqH1A9ppt z>qQ%;vm1{^$KXSIp_H&oxIff>dlUveus8yQ9F{=-+NHx=K7s(hAR#cMKaZ#!J}${O z;-4N+3A^S-mnMPZNZ#=7}=`^wf*#p6Fg ziCMJvxl9O!fHjWkS?=Rz1R_O5gZYr;Y1S4f97o{KgIxPK6IYNueg|9Qu;t2wWzZP4 z{r}VgkSj%y&@=XA=Wb-P9_2YpB}C8~Y^}AA+TK}kF3uip$!Z}=x?Yc&_uiOGw+?KV zS+d~6SGTWkUH9>A$^Xb@J)RP})9zpNyfiEuB%cyjf-G`u189yc*0Zn9hN3ULP@ag} z=~uh!P%x!;YBwOo_v4xM0}<{c04XQ!^Wy~bFXKe#rKYI z{0w+;lNw!WLW8n>D+uEBKn%-70Y3Bok?AS5#?^MW7rhwj@l5>Flo&qO zWlv{etZTc;33i^gT+R}Ysl-(uBjgqggzd;Xdcx(gC1~2d)@E&%U8|N$Y^r0G=?g}S z7@(B&Y1ZTjjm01O@H(-#g}tu(gX41TXv(?5 z&t8<6gRr?A)I$v^C4q1IPwjLfmI6kBCN;w=8Wy!z4`Tkd4m{koByt_i1^&w~uSusO zx*dZaV9k#7LloP?fIsf7tWG?!opF=J{)eB1TBqPYvB~^_^L-N#xy8G5s#e<3_ND^= zLq*>iRbz1O+S+aPWu<|H-m0znuNG~wjOH4NCTpta*F@~4AKrRwBP&V7y4W5w)X*$e zf|47TUA?h?ZklsQJd73e7g}9O(x~=_6BJ5}Z^5qSVFcjM?j#H7fbX^_2V7sx*=9LiPrfM4$qZK)c;{!KkAEr7;1$64DVjsiR0TqvaIXo$iMvX zsfg&sA>!R5&*2+yZmi!wsl~pB@gCu2KJsMatmqGi`qF)f4sq{Dod5`aU2Voc8A&t} z>BV!2#xfY$n8HzaDVf7{yyUAvKeTul4KExb%$oS68LsqrWTcAJEx&s42IoAhw|vP3 z)%u(LjpfVWeJx}?z$2V$^C%mffBl;_nd>cex7k^tT6)WYPz7o5-u>3=3fAo5JzdZh z(LGWCb1Nz!sQxDX|4<2?rS$j7jE^nfzp3b~A*F#?w)UV74Rl51)GK}wO2M#=+y<09 znjE8tj>Pm&E~F56_T5ySzl%~3$@pVvsh6QCk99H+bbeo#v){%2!Wigzm0gvWtRzBrBv;&5;_L<{hg-z9 zt=1*gECH1v(~Pd|#rjax8DD>_N(f>7v^nntSX>NpG(G*el6{R*Dx*t( z*ODoCOqrC2=xlS~>k3vIpPUo+dJzW}Q0tUcnZ3HiP!Dv)mLndbKBiAGFgfO)KH z{X)v%*bv zyqt~GVs9bL`JQruo@gCyp_e1_qN~Q?CYP||r4@zA9fohWHJ;ilk%cc%*S2uLOFw`1 zx3h=(6uI~B{1%_AU=VQ1H8RMawQ!AN106HzIa6sy-A`#;KL6XF6ZkPp%eMU5P1nm)>yPrImGIyX? z9jc#VkH)S;L-oY;e1&E_o=wSjE8sZ1yP+(()D=%;E#YV_H-HKTr#L2=#R>M@4XapP z?9+AL=wb$O;#9@A9{cE1-hP6yZmCSze+J|q-vb2ig7tU)i5=IqrR(^U>9{E~dQyes z3LP4ja6deIX)VIcC+vb#eGgAZQbe}oIC25%e+pb&*uHw^z#EpYE7-1(HmDH*{JJv~ zRuEPEl;@op^0Gq#!@4Q+s(#UCH`46E?xMexfVBLZ}`;zEGKE zS!+Y!>w0&5(xLBp6Mc=K10tYSw@nye9z|3UCBO1Hqjzzm27OV%otV6NguZ?1pPUm`uMoz zAWa}iWI|w3o*J`i02pyQ0HvSevgVFlg*iLrxPSn z{Xm%}{3^T}YvkXJKS~7O=^M?~ms(LMu0^OIH?O45#krf{Rz5~H>Ehy3Uxc5RSl4A} z_H=6qFQ9Aj{F3AUv)YY^XS=t1NACxNT%Pt^?ofdmGlj?kfXO+WfXreFFtX0cN&T^> zi*2buV2xq*>YahvY=VTPVnw*!U&(Jn`~n(}0dk10NT6uR|xw zV#1a*fP2f0stNPa{l+2LQ#DAWfgswbf6U$EvD=C#(5zWRW0>79bEQ{i>h^8YDs>f8 zbYjhKq=z2nG0gvw1q(tV{~Tmw1}T6n()hXv@dfb4CtIC-h0!mwscDfq#TbSgE-vpEtUZoej2=eA*zH})N?upyM`FRLJ+Z9_Bkoy~-98-* zr7tqVqY5S#wag{YIfE9|K0P#NHX`&+&dX$aOL81JcA`se&I$Y(}(k^Ipj<_!0oE*$&-1hrM7c_0_#OMvS0dumV1Ju(U_%Aa2P))9pB-PTw-&Dm=kO&?CM3q-qU8xK~q)!mR4@llI^ zr8{(uZ3fNp9qmFWU4>U)Zhh-RK`A#Q?`dj{xvx7&=nWR0*?+@TWBHI*GJJ5oQzoRP z*G1~|I3i)STbS4HV6hSF z8HU{u<&(BUa`k>-;-fnVQb1+&p0fwwYs!tb1(@3>{lSOwi*5@3P!joW_7<%Md7BT_ zk156cDpB4VV`8cleJ;lu@0U;}0c55RI01Eci`E*{Gq_&mM_%~C>e`EB3zR0OdXPzvsk>||1ZvkO>LxTDHGX)*ty zu-X;bM0XYtXL4L=2Ur!IH}b6o!+#m&xc++y4>jIc8Ip|FZgc zPsEJ+dqfm0wc*gz9|CfXrI2AZ1ok{YRoG{X{2$=9={HnN)QK++*|XwtxTqCTxm!sK zSD!PlzX~IBw#TWuRVjRQr0{8S%84MSxhKd;U!aB~t@!$( z!;k$^SRn?(1VtC*m?fho;}#7>!bLC9zV(#Om~^t4b1TRSXB?~dT?*Gk;9{(gb8d%0<}d?pwR7`Go} z`*Qfjw~m>#x;HtT!sDFgmqHoQSVbAp3!T{|zGqrp;dDNpZ+XSO%n<16d*?@C zZ-J#c@l^YKr&1D~4r(s*r7bdeS#(qjBI0-R>(UQ;o<3J+C?agh!}59*AX^vcL&j!p)a0kxNS~|SDzH+m@S+{(;`#1JoHUzR^`#{ez&}?GK;Mzln z%gdb!&2tllMBsvgvU54+wOWrR3>$GgpF4j$bfl|ux9=>AN+{vhs( zfwDC-J`_?IUu#Ffiv9TGFty$fq2+ROR0Byry%DMKW-JS|^K{)eM6}&Wuq1CCKjrnN z?KQrORnerHWjtgD@?p;4-}N;S0pPW^cmpRkoyc-Z;hf&9xNcDR@nxRq#fft2o7xB9 z^Sf2y4>fu{C6z7g?_|H7T1v(R&fHuECx+@rCFQZ4?~@yjzRn#8%tgFs?4DYhcyjr& z)!%a_e@B9`!!U}g^4lDT_GOL5i*_x6VsbF@ZQOQx4?YKs#~)5cyTqTVHy=E0K58~0 zsFHR%Z>Q>@Slwsl^5x8Rd14VPZ5`_T^zhl?s0;04pH*&|kEqy@tGLvRPO52WF`yVw`7u?I*_mX^NG zQQ?vakVItjHiOmEL#~F>=fY&aHrf#48tW^V+(Jl~R26;nDfdYzf*W*=6lt_1ygJzw zH)eY1De?!>M+Ozp)R~;AbRzmuhb^uzUh88Ku-C^U^_LRA85Pt-Xb3?pcrqd5g^zp% z!gI&}?(Pw9H|YIGt!5SyNgq4GQ!R4aSASt;sHZ$2ZJ4RcU_E#4VCn;!wea<8JXS+ z+l;p&g`gEP|E;*r$5YOwyIt}njt%)EipTdVk6MEsn8&4o2_gk%-7=lNWZiyo1@xAL zDV%oGkIwQxpZ)w3ujO#MT*C!>o*p=~%%>;^d6n8mTqfu(&`aW}Nf@~K!*>%y|MmkU zNZ=U>#vO1cCLHR*p@xFjsl5XB^FtxiGSu^G^!`zL3@~0U8IrvZ!|`&z-nl`qc$Q|u z+q{L6b9logOgZ9UJ$$;8Il3vI4#Ro5ZBHo;iJrky;d=CKNl%cgTMU-IAxS=MHs@wD z;0O168juKHU|`OI*9MjJtTY=0gmTcLSCB%c?M3g_ap3op2GLX{ zSRIb_iCH)Cif_}t@dFDb@5n^gJmRohq_b(iamyxa5Yf>k(fYH_tTbgTCQ-r}=yCj} zpoW*$atg7@YRU5nF4cl0aNGBIU73eU^^7^THy1b}(XG`&-7hl3Z(ke{UY$FHMgPT> z*oBTe=4D=d^A0+}qh9+P<*4P6gK%e5WTcQ!vR8U6h8B4GGzOHC_gG z`4W%Js#c3~)xWF3OQMqE+`6}kI26Ogp6K6GeogQurd=L-Wj5q3&f4=y%?)KF6Wciy zt*il6SsVEoudntbIz(&+`jR9O2=4B7r+dGonhTrK(;j=USbI~Ze3m0{eux1gM>^1e z1YVIYYr;tbAf6-^XViA&pTFn(0dVL}X%!?dVe0PIoHK5qaX&3Sn>m}>MHm`~jI4*$ z7~*1^%b#qKt1XU%u^6dd;XVa>(oQp3;a=*a`hjM1Y9DpVTU@&`b#_((sb_8D+c3!J z9box$U8lS6*krKIh(^bZul2z3=G=qEhjbrqOUT@XL$hA?BzD~%z!nKoE-}~E&sS59 zG;OO%1>Q)6gQPUp(hzI6|6-s@d^9T$6TT+Yxq=VOpr|2&)FeseOAAXAKX07sI5$E* zS#+QULLn7X|C!9(s0XFpyBzCRzh&)XLJ4Qp-JvA3C^cVS z(At}~%5iNfEI*Fx_f&h6h4z+j30O$m(oy$p7I1TP{B^i5Ry&8q!{ayADYzY8xNOo} zX0>c0C9^UN*(7l6BNO7CcJoN93uOfd9x}n>F@C2w$71NVU9Z zMi2c}n^#p`4FJ7yJnoRqoehqhWxn82VZ?xJN=iZMIlj_Ugj{$AU8Qu(?$o{?U<2GP z-JNc7ojr*l3h=3rHZsRH+2WT-+x5-Y;!$)$^27IfZT5v;4vXIEx%xeHQ@4u!oG>>G zC#}u0w|GAStXdFg(-kOQ&`^TeHIta6Wz4khk0pJ?$GY;jZwOZX2&wt8M(?l2cMaNX z(Z(eoJkSdHEgtMmAv-tQt@F-Ua*Mt9fxXV19;T%}o>znt(^1%)tRfr4v1GAK`X@i# zW3ZxTE$sVC)ompoq(8ZG@Z`?|gWm)XX^ly_kY$=biU2!g2nN5NZ?I}Pwdbwvez**{ zW#&()!nduG1Mriic=B7Nb^y>^Vcnthb{*y1x8ohvr58`a>Yf9%t~t++qe&ByS@#2_ zygBY!u|3c7hdOv^lWv3iGm|MHSAd43gDUbKTT$VKz9_XDjBz%5xb*7to{%))wrTa+ z5U@j)y&y+7#U8$d&I}H%(d#~1 z4LI!e+HP9)OVZc%1uiT@RXEdFoDsU0VV|6~ZC#;+_KivO1uW)|cu-r2U9$5!l^0>I zseW(ZLr*{WUyL+K`ZK@b#!%eq$o;+RU}XE9p>IzJe$ue@U+v|Jzq)KRitM4JOgvz~`{XmgQA zFuy0F-{7-1N>JuBc;s>eZ?AOjey3cNwn$d@vujYa8&5xH^!RI#oTyjhp-k$~?8`~z zIo@N?`&*PeJrnrZa*T!omMwnFETV$eqRmOB(t>6ll7(uyDMq> zmfmcjeL5IPK`nM6WlSlH)H?=}IEp?p+jTHJceR!fjElzq4Y$?zY8_`L94(!#_s^}) zEj~XiDcy=;(hBIgofjY(cQLF+raVttdje{E4qd5|Mqq^io$jwc6J3r9&npc0V{zPl zDJBb-&)2Kqqjk1VQm4NF=7>`F&RpZYVt(K@6b_rOW$N1Uo1=p8kQUP%G|Ds=klW}#nmwYf59L6sw5>7tkSMpyhAa`y$xtMzG{=*`VO_&zRkl3fw z2B*^-YmZEYQATmHy0-f3Ok)$Nkr(b={=uzeEj}a|b*Hny{MSSzNj&)y7|H!!Sm77w zgu<-MyO?X$`pT z4}@h0V)d;h^{i6^BRIu^3Ao1Qu%%=+f8}Nlb`4ub#O+1HtEV^RwGMD{xVi$^+&JqG zi@%MYRtK`tzXeS>i(KWQmA9&=wCo% z+(wUs)|LUJ3o^N`K7_%&vR% zNfSq%4W%4y*Q7g_BaB#b{Uq^tWjz%puK(0(dD(qP=Am3dEI6<_Id0rw5chC`={S8c z$D0({!e;yx9~NZK5ueLys$hj{P)feOmh&DcWeDtx|09#WQ&;x|#X7}Q<{M!r01b4t zI@yvLNFoQ*Y={j?aZVGvi~77LXTgi-wcHFV!8$CdEXuCB5u}bvi02EwQ{-HgsS-8p z_HS&8Mc1ChR)5SfnEgt(EU~BBeMv@na4TSX>$6SJDugdznxM`8k|xPT!ZAVOJ6`W-?RdwoqimCJTQ+&)wZ zQu33|Pw!KT-g?k>=3wCUuZ~a2dSe7wTetGL{9gYVuwI5(+y6JDqfL_;DI?r7p%nf;X(6))RorRpV|# zXV7)a=@&pD|1H?erccP@)AsOkJTGUk*X(9adZE>R<#*n&7=VJsp&4O}D~*RB(P=Fj zN}2-s+8=JtCMMJ8F*GiYFeBLW<~PCN3Cz?{7%CAcoPy=2;~kyc&GBZL(#*+I23>!) zz;T2b7_vBF*2|urP^P8T7HRW$(V^%-qvUd2^>n`MJExs3(yV(Q)Br?%?4b^}6Vw-3 z+K^r?v!!r&*~1C+bKrT-O(Zw!XtfV_v6&9#Q~T8NXpP`J)2QU~7;`mb0q@db33lTB z$MEr#2W;@J$jsyR)t-E%drX;H<$EeBBu^h=#pEDH@Q;c3Kt^nfq9f`;O9#Dc4g>A} zg5v^da#J&)e=HmlWzEe=sG$EV#7tNHo>7VYYpAiljqWjULG%%knaz9%{h~QDKH{X> z;hBQT@U06p61us=j;np^v!~qh)VewcpCT!sY|E8KWBYU@6NhHtxFJ#l%zr*%B#G`@ zy+}$2Fz5-}z1N9PN2Ka%{?}TvXJC@Hkir}O6WKPSC#kpQ@)vJBYA2PtXOzIEf zxxnuc(liwY;kv7~W_&&S{dYw3RjB$e%aBHifG0d?IkBe0Lov3ijhPSpmo6K!y7}{= zQK072FMlg_R0$q+Tkz~x{w>Tak%7|G6h&b4MNR*nnH)lLuWozLjwoTxD#t{?;jh!} zrsKb`kk8*(s9|CxZPF$CvE;n!*U0&~z?Y1Y^Jr~#Hcv0Z@5loMnQr>MP7ggz{|1E4 zrST3b(rcWzrL9UDRPTdO8ssYaaVd4OA|_Se+J7iC8hl>*9Cba{ght{oH<4Dff@8l=U-N4X{DLQdW59t#Vtd~=!Q5g_Pn4=40GFZez z(M)18{$w-gl*0I~qpR_hC|LRr5v+96#kF2E%<^;GxC4g(QBe zG4-3)TT)woMpQOehmSuaIcD~$E`C8l9=d)Bd{XiZ_%q{xakEBL{r(p}6S5MR#Ukl) z^k-L>t~28((gJUx|1r2g3s0f5v!3>t+l7OHMw^UN(dfiRHlRCm+dD74_3Tb3I`@Q? z9h=RY&jy#y9J1S*i}(eACA3n07d=OkE`=E^L&TWbTO_jkR@DUVJ`t&;*A3tU3G7i8~1 z%vtI1RliYTa0_(S5uYZH^Gaapd$J17Jk|BQ5SdT6{Zj>)M)hj_WPsiF_1VIMkTh$J zfWe_Xyqq^}!kKU;i&>_&WIMf=-69}3!!00jc~lUv{tK}-`EMD)n`ZbRBSqAjVY4fm zVjhu{A}uX_n$xI~*zjH$A1AJ=-y{181?314H^jBIJSs70@J_dz8SyDefbrV#Z26oL zF!x>?-?508#cf1bM4111X-tp1YW&pcKP%O!I(fj>v7H33ZhS&7FdXa8U6i|#-51p9 ztbp~nWS%S%k)Mu&KF@;FNf$kwE6Pw48>z|oe#WXV8xDR|?4VVPb{t#ymK2HNWo<0u zi#Pt3I$hWaLuo9QlO;)9SC-7Y_W39y;@ZN#yu`_GNW2+%|Eyb15PVc1|LtTx_YBrw@O&ZJ1W5zKUMi|2g?-4^oi5%Y zn=rZrbd70P#vQ`IR5IQbNyYSz_CM!7AnLko#@!_`e^;5Lb+BUK%NCz?74BL+aC|3d ziSLKo>i$%aVo;~8YD}`F%Qo&S{BUK#;%R_&d$HPSYs=IBgZ&O&Qf_cbzQK^BoT0o~ zkKfLcXSw*uXtdsCNw67>QQR75U+i#C;&X~t`O7a4w-W>2oca9@<`+ZA99MiWp`oLT z%HC^3hf7-(BF}P6jv*IbEUu$Zr27H#DV7C5x+Kwt(UPGslRje83ieL8^|Q>$ z$h1(7;^89WFHZ(nU)`PoJ~qABPsD3OG0oZRYXB+faP8CYAQ#h7P&9+T^X~XEZMUin z+y-kqyle3o`%`QF{&vUAmPsI@VobJAw>?a&mv71zy5BbZjdjE(qEA;!wD(L)_0jpc z*E8atnn0?gt7c#6{uUYqhp4T+4T8Z+&P4eGlZ1=z@?kp1;UB9~jOTLA9Jns)(&~a` zJt1K}qDs_Fe~-|>d-c}{4d&VRAwdR?wnruf=^d_rr@`-J){EKGgRE)&jUJ@ee+_`;NRidshFPivIU6 z`4YlXccuNty=V&+HWj&jU5t-kKm64GboA6cctreBeggu<0C|*c8M`3uw{)?WAfjqK zoSw$-FtiUgNk`np!GQq-5o<+P$I`-{J})$)H)yK$6&qt*N{8xL9O0)Hoj_c_QFpByny&NnITJ%Q&yLq?)mHD=A}8Wg4+wpHMAfjkkMWP$ukX%#E;s4(gH@4T z3~V0FJTVImBP9+NDb-dV$G5Q+2?ub0wjTeMpKiEnwu!4n?1t_;BkfB&Zv7j)asJG= z8I>Urr6IQ$td9Any7{xwdhH_mz1=vHE@n>A+NBK$7d^H4<|G9aJ1)_5ji#n(0?~%K z@MPM%(ueUGC^h=O4kIOEf35I-nK5vOWH`eCV6g zJ~_EvvadW_d!A;wJ6R`8I%|PYqe99nsOkMIsw!Q19h`$9m_@hp1Jcg8G%G9pdW5=JgB0d(QePVn=y;*++dbC z<%sYlEAhgt+%1oLEWC>6eeIeqQb@((3*oTvxmA4eNa&f62Y+l8fK4fXiLs~yJkZyD zm6#Jh=g*2&xwV^FWjn9`(yQIkxPB6UsoRZkb|*omY1aZtVdp1{mys!tE51gbdiEwD?m3F>Pa7;L+T;7d)bHCnHm z$1QR5#Up>VMWqCqI~B%Ng%8HNuFATvn?0$C>MpTqcC+oZ>tCMvVE_l>Mw+X9U#MvT zK6p_u8GbxL^lL1pJhE=w8C2p^G*O3?*mZkI9-s-2-o^uqj(Zz(J8Ut&c>2_tLQ}LZ z-^Ug|i3V7sRF;}%LrV?R8XK~tUbK6O@`MIV<~VP(Aya|GgyzrP?(6M60S=|y`v-wl zKlO`WwPM!lf9Ms{z4yz%Ihx9hsCU2)hK{AE`I)&GN#^lP*BAKMP9Hr;*o$~R<;@d1 z7**7hC@{^vF96hpz8`eBjWY2jORT!}sH<99MW}l9k2@)XRTeiD7QP}V!4tG!{yHT3 z3@*|wHEN(h);E2Rv+WwLpCbW_k1ZtPu+=o+@2iZM^Ej@Fk4o$jch9Oddo7Ao)6z}3 z`CS-=9iFux|NhpvOSXR*agML*JcCVsc~0G1_T=H!L<4Kh0{pp4F_Xq_SfdXVf4L>> zJk^d`Qu2$1x&S~;(Q+p{&xjq@XqjetD_^l!ro6M*q~EN2hzn+BZ^Y~;IpZwo-jtA> zH|O6#j(JWcHCzPN0whGn8J9H^%o}h;+vp?G_vcOnL?;=_%J__nWs2ST!H@Svqih9L z=1aU#n0KU8@iHHWZmL#WOJz0;xx}^F_Rdp`?tL!wd|cOq;=r}(gb*mw|A)?8^%qjt zn%+-UGFJ9Xdk4($TG+A_+mt$noBOUq#}`#@u5y~xejgl>?$nBrr}MA8crys8`?iyd znjvoPZQA=$8zgIsjln*h>o@n!CHJYtQCfO)NFmwtf!oE*h(qBN+O)hRdR0ow2+<+~^lDUxxDbGEND^X=hVpscuf>Tt>oG~k&s{0i;fpMTQL7?hx1 zOzH6Py+xn7_ubMrc_}W>UBEuXg?zN?6zfOaqKf0jp?E2GnKDHR*ski?{u0|xxr172 zRa1eBLJ8&QbGqrtV|t2I0e2=)drXzJmU&{1DY`yBTk=6I_m3YwrI5O6x!&1m(P1Fj zv89JCi6p7};#68#)Z(llS$_meu#huD&IbxmNE%(nLYJq&`R~tn`T2mFg4x7s=?cV` z7?>SCN#lXIsv_f^A@-l|EjH`RO)(W)n#(>!`SKCK%B?ejQm5;FCo>Fe zyxTrYA4IFg;}G~w#(nB!X=I5{XNfGebQ2jRt4bWr)@mO{L}rDn*v))C*|MgIUKsJ_ zTy9%)_J-rFj!{QWQhw~di^TIh?vby-SxC|DnjU*nT?_@+!0j9xcSmpQzqzV}W}2wn z4;_sA&$Uy8XAPc~GtzQ{ z$U2b?S{TZEJSF}=2Mfh;XPIFKx67_fg>o{KuMR9XtBibVclbGT^H1L_RWGWbdncefj_u6KTo6-m^Gc&M?Rs;#^gJaO=JvaXJyIZ{N%B*6*h0QtR*UZplQ)YeMxiTz=uT>O(=@H$yRXA zB+X5sTwWA(Zf>TLFKXYTDH1hOs1{?f|7|Pe))LQSoTkp?HpOnPAuHsJ^Cm5^uh=6Cmq4i^ALm!@G)vVsfd+fh?Eu?TK{F>Jo$&bJp@Pnxh-No zsmo>Y6QrnD4G_u;P_`Yy;C14p^6*I+%3UvdtynmAR1W!*w)LgzBf7>V#q3*V-{qd+ z<_dv~9?~p(lRE^0N^=_BgS6y!LPD5UATa|-mU8#IC*oI7$2@n$E0>L%b2Ae8f&&qV z59{wal;NR2Nug>Lze`6TP)=yh7~RY_-JsoXFp$H4%e&;z=YV?7htb4s&7Hy)vkAxA{f)fi{?4I(lriBO{kG*CIjnN0)-;| zGsCcah>#1kw|K3bs)WwoZo>{rm)f7YkspNR zKyPv4y{Jc}JSv6N%*RPY9YpDSJL9>@;Ye>-L5-%RH9oc^Xj%$n(bhOi%UyExoOP9oCFa7|l_hqa|A^$8IrR7;k2a}KoX&?)B4-5WnO!bcvFn?a^F zTh|!(TyJqeR~t7eBL!ANHaUO>qMwRAvahF+M2FWjzwFZznyjCR42WdFfog?g7flv; z1kX9edcBG2YL-sckJ{$@hVBCUb8ZtCZhvNJYKtZ~ZCD9UvZJZ1^na-97s2GR!LGwl zor-rBYKoHNLW|zvp!y&!dEBf|jMLtyI@Z?v30DkDQU+TArD}=Gz4nNo)uFtsu~3`i zN!YD;*ID7AW>D5yamNrSl%`B^=VQhpnbBvl&@~jK`*;UNVwkK;?yYD&6~!@qmMNKp z9dPY;+ktm?(L+QHlTs<>7`+=zY#kz6GMqgmkZO5s+GiyqyWINfJpboWL=4vbtkur_ zrG?bYC1`kw{BWHS+6(80s--yr2U%nFGK9N*jXxSOkkJ2ZGNz~Vc6kiJ)qPclz<>!| z(awsl*Qng)!;h@B?@=Vy=84l%bvVPoq+>%5TO8Jyb{ygjh!CXL?}&`l9b-CfOhwlC z)URUzF?F1|=(D)l*^u=ZW|(&+XEB2hyUY-SL7Rr|mafdMS^7q%S@=+k`JRaJ(Nw9B zw8OQHY5iI*yMT`$_K0D}hacYZZ|)9us5af3vbz;~T4RbjOv@hG8iWr}T^jdU=Zc(B z`gG#R@zTSNn<R{5UtUzzV6Gu_rTw5#Q^VfXS?| z2U26x(wo6ZU3PiKsJrXV3|axTRc9#O z$P5N8C!v+}IG-WINChn;{>G@&5BJNoB?8vg@+Ct+q6i3wjft7%ovt8$6#uo8g_67X zF;=PI2$&MioEnH1(^!YfJBcve_$(oBU#{$7J9%j=Lf+q(7KN?nI=Z}=SI2(6`oIPM z<0ymc&RVM?bS7=KIdOOTWH!hQb~QWbX!$(VhYxsUTMs*9n>I0uuc9-7XHemJZ}$?S zHM)Ok^Ymri0FN6wYAC;O&0H~onF6Is@!_6}#JP(e7~UMUt7&~ulRU4!WbZ+vG?jqP z2XGmIeS1=c^B2vJi9O!FW=CfE9N$52&F((Bd}7zd4K+SHGjoJoF(b&1mS^}mKT%cw zLM*eIsr3)r#dR=i(bxr-4#c|ITeR_aB3PiR^B4LGuW27FytIV)L|iwXSiss@Z)ra=e89A z%_3wwQRV52pC8*YwJa|U1Vp^v1)duNZ+x!P7L=)itC==o@IPX6IJ9~n)&!rJ=?Zl- zzShMYM`k9uI$!2MzaSMPuQuUJa2@s5%Ga~abz$PFU7}eczgjDCOi9gKpN*wv>PQ*2 z#Few9SEbaZoiJvr<37gCk+}W#rOM_rVpHj)hWrDz-Pe5k8!6%YZRJ_v7lEaE%T&C{ zTZ!83GnWRbg9(1q)wdqB%{vxIclV_f6SeF0EKlNdmKvAGZThPD^LxUJT7Up)z}C|S zd$N-}d2TOO`5f=hDHppR8;tc_sqzQ<^TeXlY&cdEk<0mrL6MX~n<2H5{w5nHzzZg_ z9aKZw?bBwF6JoDTK?#U1evk^U{$g^+y2d8JX44%>BiY&@&;I-R>~-#Er;1hQ!88Ax zlWoz?p=E{5F1de;X`Ll)mRD8ic~m> z>g}n8)KZ2^vR@TE;mn36S!mSia?)T#rJ1K^N8t7G*2p33-S~LTqUNAPTCy`(@=c1F zWqR|bpJhv8%0XyJhFjtcdS8)-6u24AcJ~%bNc^RHH~e8Osvs=*DaGp*6xqjRyW1v9 ziYiCOiM~J7i9D{pUrpOcQ#)JJDOzy%JM*=r_aI#EYjVP^eE+SsqssGSos;_HSlD*Y zbw6ki$F|!JAb!1vA$(cI%DMQR6#vC1lwcckdE*tT+s$&(=>^~nFXc&Xn0A`QvMBZm zN>)Bc*>wkV#qHRAp$Wx#r|lskEr6~~*B<(39MS89X77SKZUux;hl5N z5?$&FaDQEn>`qXm8w)p=VKfE!?T&5Sne5mv8Klm`Qg|hSWmf$kWl|;R+nJLK2qEKo zXaI|1>)BxjFb|>POYb=@YhpF7aey~ZzSn(?6TGL&Eu!j3Hl-sdoz?Wx;cTK2ii6VL zsV$dy=T-WUB_`bBYbGAlsLreEBwfTMwR={D$N!6Dym3KLT3a@o6a8)c>)aaERVlOE zhIG3L-gtEhEku1U{INpwZJoE~shhO+9E8#4DljQH+C<^UL%GAZ7xG)1uugoNiM4xL zE9tVy2knqBCkV0DMcV<-l=b>}IIWb7#vQ}>)Iv|Psm_hT;cCC8k51v7hq* zI4Kq}mBU|hO)FQgFYj9>(k4~@2>pD>bPfg9HRuq+2)P~=jCvjyzoSbX_89O>Pq~=C z+I-7lLqsdE^vHuKS9Y+dp_LDL_dZQ}=v$ua_v>mK*X!y&*wwjdN|0*HajyJc6M@+R zaQI+w;`4ydnfBr};$m!Y+f0W9I86povLi^+9?cBbiBP zHC;+qgU-9vl^XZ&b@tDN?rFXb3swW-E{;*jqREwY@2q+2+ytGk&ix@#kXv$xd&JW! zbI*>0G{17)Weq4HMpt#6X+%w2<*EX9n%E2V)qIH0GctkhfN7_A@b}&G*1!>sbo3b0%Iv7}=pyNDkIWU9Wt$e3vRp;U= z!witil2aX0-g4pb8ZeMTdW2~}b>5|2drn8$@ICPsjw;2LT`zGa<$BQ0nY#H|bi~v2 zux;%9`yDs!+k)3egm%AU>nNHO&8NUkg933EocJx3akcLBvP)MO!Fc;r3)6q=RIye> z zkw<}8F_oJd>AS8Nvaok)kvIxH7upSGvCft_g-o*O4o=vJWsipNYHGvlkDK#=__?5_ z-gpYC_fIp#%!3?&R6?DC(Db7q?cqN$bf?q12HBfX5or&+6IPSs&Sq=9`O0{~n%?=5 z&s&gw*D|5nFuBk}29Ia^+w=pKgE#zcAR`CsrS$<=O&xGkSoqeBH4kC^G#QDYPPw32 zyI$x3khln>DU}F{FCG&{pV!6_`OLI|&jgYFW?MHUSe`=WJyslx5~$H2`J0Cvid(sy|HS^31!cDXS4>SEhQ)#Lra825tI z*2FFgjQ3di&c@wyCxT=dc`}o5Fr9{W<*0U(O%raxU6O3KVYjhsM#ibH4(F2Uj;^EO z$%@xjGk|^STdNlkc?TaIDp7ApXBYhi;l`Fg9{??q)t%PXiMk0a>B<*Q6QQUQQ>~ zWi7nvY6SRKX{?D$l`OUo8mGoh z^>zewx7A7Ok^nTE+kuN@@HyFG$l&stGF~OIy1Fmnla?ncT^rsy z)LZwhQFDW#4UdO^+o%9iLpMm?=A%AK!yp}i^`#5wMoZ37&SJH9&ncP(gaY<&OnQe8 zPoA^iPs?q5s6*QXga$3ve{9T z&D=l`Dd;^|bH1RiP?L!Ye(I=fUUinq4 zAQ)%u$cmYrJ7zqXsLV!^8sw7BSQRngP?v#qpAuMhCh<-h~YM|sF2eblb> zzJOy>$VXhGOV9r3S$_?;;K>~EVB?8GorGn)#blPy)Y<0X+L@~T_ok6IUc8zdOn z8!CTkA4n)3m&bUXzClfu>MhJNFc$XT2it4i@6jj+Go@*a{*rm^_lPY!Lue?I9irN- z&z#raDUDb?%{_4t;a-UmE5e?MzUect$}krHB;H^3?0of|FqH-w>(6pM!7P1Y<0Qgk z!mN);QSWRhKZZ1mpPqc)*RJzJ=Nu%K=U^-d(I#cz*6-DB$n`I)zdM>j$8xsep2;wu z9CK8&skTn}->05*AN1`z#c$1m4d2RcLH4=^E5?S{zRX|n#%bF`0(HYt5g<@g|CFRW z(#_lx`s_J8$XV<6A2r?w@ z5Dc#|V}Jl@olfg#q1}W3vRPVxWK(QL0Gd1s8Sb!?L-E}j4;pWrYtOu0wX~=AjBM_W z>jYej_lalJ!{Dkg0eLN=3KYeZIxTeQd?$Tkhb!h8XS|squ2U zK?uGT4)|8}`=b&K5Bd?Wm@DKF)pBQlQr*}Mnb`DC$>y9est3mHNMyf*NMzr9J{YLM zN^=i*pO`YFyA@l-xWm2a8K1evbeK?^+^f?OWBn+I%;{U?FNYAh*{XU^(|hf%3E8l} z9a&;mnLWMo`XbfI7uXT&k#sH&i_vWvc-d$_kugDXb>C9g7emx(tkNKr7t6$jh}um ze39UVKR47zn$=}HNeTvF7@rE{u7QZ5IiNTp(#ShQkM-t@1gX?biAEp|VM% zca4_$8PESx0EuJlJGRMZzCcxS`jRNUf&kaOF=rMkSVx~?>1V2CmrJ6K%3d!3s{XaB zbvI^U6ZGaOnlAOMz##d z`JGv|PE+BdmrVcOEcBQ}R617Tn{Dqp^hG-jK}g4UAT^+rW;UKlaZYa`Uu^#M#*1)l zD0RI9T^m%hI^Z>6vrGq=`?ptz1Ax6anKEu->2Mw4l^6jHhEusD$C-?mVFxSZnQ6UM zDvP~}DC)g((mC!!^q=B7lK%aw{~GFGZ)U0ndX@(^1ErQ=KX-Dp{(beNY+kMqTf3F` zohjW%e@Q;cl@U6+(=jg+_|IQrD*v^_mC5(A!hi2`ent6KV?(WxoW^WNWuElb=7i1D zkCSgnagU+8k}}KdO+uDBO#xSNd1ygL&@4yG=)V>sCgKsm=98oR+y46BSI|(7W6RNc z_&n!Hx&5Y9Du=pQC9~)M~zY*~7=lef{7&u4wzhB@#i?TqU@4x-8ZTZjCDA7;r z>;G8$uaoQn`TxB7f8QY_UJYHk^grwV-$@LQWNXQPmi@1Zys=w?GnT~X7HPWUtpM%I z2b*;if@&za7uoKc(8yij$z9i$=^kY11=AMO(y%XRE9MaVHWYnNf0vx=cSmD>?oO;E z+g8k19mL2Ioxz~<6UI5o2Iv2}!~dDyOV#;lu`AwRB;9avP#*11@2B4Gyb8Lpcp{GV ze05<1m+9V)inoZr6z<~Hx^S?lA4kvk^@~JIf;y2`w+kol@Xqm|p>kAX3Jx>%>-!lY z8-a*>NR4Qa-oyQFZ)3{|L`Uw!DeMUCV|PA>w_UY9zowb+CU8*|_bJS> z(3gk{A5A38GF7;Xma1<(5c z;I2K&v%DlRK6vfXQ(4$@d9}CIRz&?<`^<&Y6^aqLR#Zoimh2x&^3HZl4AS}z1gIvm zv*&P0x3{bwg!_mY6#%Wj&Fux*&RN(TuhiE`UGKcQ{(`0dTssU$EgxPAI_I2Oc=5^o zJUdXwY%>34dVZ-+1Gk{JjiJmh3%R8wOY*f%+2uPuX-+~%EG{<1kp0%4Qf5SJIL%`g zjmS^`z~J?YEu|Uk#GfT+hYB%n&5S=ytl!=9oi){)>&Zb^bz?NA1tVitk*G<)v)uBW{UGeQF?;V!s8zx_*?6? z{wzS-DJ&%32_0p@Ur&|6p7uG`p%A9?(rfUOe)Lk?FY^~IpAM3p z6|b7#N6q59|{BFEiwz#>aAW29|#g}6p#hz$YX@NP9SAtw~a zz|ZQG)2V# zRa`U)z>KArAzK`e9S1#c=XXN|FGdFTe;H=!i(1#^kqW;qZ2BKWAARyZ-}gBY93aa4RxC|(PHPesEx@C%{1|N zm2hU$SGKoglAWxkc9Tv7Zp$*?galJwwKtHEu55Yc*7#ip$TuhuT`vjy&Cyb{_u$y9 z)xXj*rCvEoIsN)&p2dr*e>1J+3Hif=I&0N5ed9G5y^{1Lci#xQK3C5x}0V8_4R6GWvR@7=`5rc@N z3VvS1OVvG>Jcq1DkL-m87tYUZ`a)bpiFZtNMQ%Vx%@wH~^jblx`Ce;I?=D=Z6;XD^ z(U6WsL50DT)T>((Z}p^=$GKJV20brlF`qYI{}Z)VVUHXN(EeZJl^=TZip;*%{sxc1 zH>`OMSh0%R{Juj8Y`P+!_v3RIzuwR7L>E&7Qn`ntb8Q+5E;XMC8tQ`@<2o}Ux3X$B z%kE)uo!ApO(+aZoM`>w)Xm#seUQP_WA4Jl10S-^-;9G`vbrh|U+ZR1v^L+F{^?KS# znAIuOj86uxehjDI1@PMqDXV_z7$1VVuR7AZ&&$wo+E7N4e}bOe8+|sH>*p3%FZ+4k z>$Od->-#!cc_HgKrLMZgXor1r?pYemvguk&YA1CW=@Fz>MF7PFYn5g2)^!AM60ikm zgCc}E2Ml&I%HkVSD3MI_ypJu+7-0)~&jn9J-*}=ZFlC;7;a)N>&Nt#K!^`(F09SE9lbuQ@{3J+1SkW&+G8ni9(cJ zF%k>3c4mhl8~9`4njZh;V=O6tB-c4sx`#HwyMor zx=hmwsYI4Wy53DC%2X|T!V~+4)i*Bd9m8~wGP3j2$7qgJ^kZVz1;jacA5?x@@B*Ol zdyk^EnBrzEYyRGRYDqQdyhPr3mzh32smU5|oTk4~V^?$Vm+;5Vx7~tu@kWJXP=HpD z%if3LM-3MKcxj-q_l0XUG_RbI&V7bOtRJqOjiqM4d{aiM-%p+G>$u;Ve5gtP0_x%f z=#iGQj<-1bDj|>{WM%?3hQS~0Zvw6^Q`32q&93{IOqZ0-&o*J*I3jeM_ANzn<*(UO zi~{RZvWwI^r{et<^|rRhc&P`?%Mv9jw^io{dj5&N^3MZ1OtYa*yE@brKUS8agl}0* zaNH?hX>Bh6$5bNHKkzoxJ6y}Fy*x1&gDbb1&MMg)u-QG7NF7lJmzfFBf+PtfVT zn3Z+@%(Uk6wi0uio*EJB_qoCnHR{6PrG?6vGNXshcNfx&up|C1p}U77Xg{M}cg4Ur zU4tOcD-H$?6#o_Op%LcSQPiRw!M*Izk}opT4SPSVMqvN$r!7ghq+u$mi)iZ9xuVdaI0CjKW(<3H$qWy12VjnQDi za`UMA$BJlwt5Ck>L$1fWa~V%sv1f+M9L?Fb(k{6eU?LaIv;JJ&d9%Ys#;Z=#JiD7J zSF7!GR|ow^SJzE@`2F!GRmLdp?)bp2nCf%s-Th)W;7lbRY>q79%R_fZVdj11f+;)~4ib&P>iOF%|4KW>8L4QKlkplmg)JYz z;YR4q=_KdP^(Zq8WdFY19oBY(!H0H;TekMNeEYsM2PPsWK6w|XFz%!84Wr>YKC)xD zrFx$J5hc&}WKe5>RuopSG9F)@hJhf;y|9*YOj_P=641N*#7CMe4n2cccb7ARYDZg| z#SP2_P?Z6!S5>(dr6U7gCb>|nLy2e9lM*_O@w_HHSwe(MFB}iF^3Ui@Z8f~SLT-lf z=cl@e+NJdNG)lRRL{IE7P!n~2Z`H^!jM09fHT>Zx5cl}36EzJA*XaUrS34P+|3Lol z2}^97f$(?@m(8`XCHbZEu4v8f3$9Pwg0&>11Z5ejElc>RxK35_HWQ}xMxyoU`giR_ zh^yZUNtj8=Th)Mr^lmVWodM8ZxFWgMAUgJntJLNxx(w9^hzR!U8XkahV2Fe;|v+nFQuMZNrJSSI-t5{`ix<`2$KZt%EQSuoS zr{Y->BDN!G(yyWbPceU_Y`*Boa1cJOXJlyrpErnVLYFH^dh%tHg24RgRF%*HzAF0u0GCIzL`&;RmS^P<`hD`Z=kOa~jD6<7a~aeXvC_;+ z#%;2c;d$&%NAGE04N6LOAk@QDVI0XCja*CQ#wVmLt)OvVtdJNI5dT3U;c6Mpl^iI zn-W1Sjj6FnEtWtTUm+Xv zPeMc+&j$b_w(KXzS7kr=24_(&HB2dX@*}0o|&Q;eddx8Iyb8Ep4#2^Y#GSG?{9Zfj5)M0Rc=$r8nnT5FW z2nA%8=vvK(2&g@LW&CVT`5?FKrvl>4xZ2RfANp+SUF-=08j}gIJEp3wJVzd5H!#V%4~S|#s;iM# z5Wq6JtOGhiC-!Rn68h`kt0ak13iflim7s#)`>NmZ!)1^ zy?F2LEcx^RyRvgBt7>2+>e?Mh)4CR2)3NjbHLB)$yh{H}lAP7*q;FvcAm)NrP?*Dd z=E#pxvD}I*jNzHy3jV5BOTQPb^Xq_5O+8=c#G~PMM@p}McxnadL2og)PZ;pvWlHAbs4a?7u9lkM3 z68m1q^ejHj0aqN$#Ekb5ZgDCUR-QAayU1S-HVO^AXNO;s4|-KBZb$nBjt!!XfAYU3 z2&L}bsdku_m4Y8;fnEqP`~kNVsO^dMM>xO!rGS%ixw+UP@ukLj!WJ`f7Rjb}g|v^= znLTEH`EvR*KGHB2pUzaXTRv?FgOZ=NiK|x?9r>S!plltR;c>wVRRebxI_6k_G6bs~ z07=u+;qDiL-OU1+y1L9}cfl|q|033pF(!1rU>#58_41l2bQ9j}c2w<>+^Ekf>TIQd zR~zhlh42Swm|ZUKLB?sNJn-`WUwmVWf!oRhe80!XLml%9`Pm2IVwq{C2z7JYlCb#n zNABOT0FfrSF)6#6%GH(eskZ9vb5y1Cq4I>BF#G18iMx-h--d{*o2$QSEfK!Ftfyxv zB(>m9#3Um=p{7DH-}dYxm~=Bwe`PwB%1_)4ub;sJEu9=&68jE9tP=|nfKO%%jhPG& z#=6S+@!@LPk1*L=0|GV(5{$ddWHMOXl;!e7gz1Agiec15Ix{o8F8tky&@(Bu>VWg! zsCu=+5wi~pVKI3%Uo^FJvSxX2P$yHDRJ!Rp!%5Ni9j}r)#3797G6T3N=uIW)8zLD7 z=ja@J9u<<$?=q!(IuV~x@Xskd?jzlIXZn$S8uMHAAGWV-P1aA;5s?^6O>Kyi^OJ5y zgZsC9@9Fl!*o9OK<5u>1B9vWQen$JqxPJf$34vy&P99OzWodYMp0K?8){o9;3w{Iy zjEfB`mE0avZVtdd6E~G)r84qSD?g72iMK*BKrhcPa7=v^$P>fRm6l8~p&}8S>J-6= zWsY%?%A?xLPFPm;QymrVI>uajfm2^<91Q!bGxR(6rL!AWJZb}F2@6z)ah8mq!k*;3* z*zErWB|%pCF(9fu^5%A-OW!jhEgKM8mxy2TLH8@h6H=rw-CM=8!6#C@==3%&{}i{V z04QIDfzn&-XSxsZ2n5TYl2faSmF@A%WQesh0^Uqjo4lY~x@Q+9t$9nI!~lpfBDqJ0 zCeDSFU5|OL>(Yzd)|wj4AfcW^)`wBnA*8G^3vLnC52VVPKr3s}WbYf%SA#_8b?yPE#SS!4 ziS|#epxH=S@S|Y&40FZdC(gX$OFq5^|F7>yas^WBC*GEsmD(Tn@+53IYJzS1@*ig7sz;uQqKRajN-IW zg#I_Y(2(Z}v!^l|<(@qG^%S>O%nHhiwt|(d^5d{H zv|&H1wzRZ7T*|*Udf=EF3%$_KEw>sKkQY4L`DQCx@7yc!gb8`}ZcG6m)!#FqjJK>5 zJ#@S4YW|DU<-E!&+Q7wpDp(_bp@7&s5Y|mO8GsE3TUb%gSB}T9DF5SI_%fd_2G){n zt9Al;0DdJ9>0*x6YIzj_YF;D$mUkMn@M1G!!GJ7R>dq zxD?=AfJ8=US#00qqi+PYb!x~DX1PTMcLQr!CU=gRzK24gPaH9oXNS4=rkY^yvVM+8 zlp4=NS3SOcQGdvKj??jdElrF1>;qEHWn#L?>(MySO96GsEb*e zV@*`g%PuHw;l8i=g3c+R%oVs%KrDU2a;DXp(3+oRQURDy-OhDeEtVnjD6csxsJE~2@6Q~)p1|uS>x{zp|88{iGv^r#f?d^)Lya}tZjCz z+RZrFIM>Q>=Vm-UT2+N^cmC?}XNaK@S&hSHjrfgIk3weTT_x>E%!lq*T*IZWoWUgg zMu63jv?S7>7R!?rIB5)mqiabu*N>_{vq9<@Q1xxkM#F`*@Olo+i=|7cQdSb2)a*TR z!&SD+|Ls%#50qq&H3tm7Q9I-vGBU7oRP_!<)Hg}o^&8?;8eU>(;r8IY<>$_*L&R~w z2fS@L91`;c{PO-Cp&xIKIc~Pk_1iO5H5KK_rqDDmQx8_NK4{2T(V=;irZY9yOzE`y zzM(O4!D6i^3T`zYtUU-zyUl(?@sTjJ&Vc;$Vc*r>I8$~=e_VKv$PYK;SL5Eoel%P+ z>SOsDo%BdjmLdpgKfh?R+gB;juH&H?pm{*vl8UT+)t0}$Crkh`H?O!<**D%qy|MWkeMsKxZW}t<{o7kK@8e;ZD zE_c}N3l~SihU{nG0>w&h*PG|(XJX`r{|?=sj-5qj4P5JDchiU)u&>;@t&hjND{++z zNtFB1)1b2zx-yw|Ge)t0b1a>EFw}w>;hQ^;L}1cV%V*$See+Q}E?r2xi*D`;h-^mO zM6SqShk;Ym0FX&F3lER4Jp1EY-GX(H=d1^=GzoEkxI)ahwD9Hlx1fdY z{?g9xSAM=K{=QQTt}6>a6zv&x$z5v{>-_;M^#9f8E*zTv2WfH27xE&`{7Hw=juHX@eAHTMN)kMO`@cI6K$K7VO zem8F85vII-NSG`1q3}C?;?GTjTm82f=@CXJMUx!|5zMb9B|eC~QQ8I47kE*n087J~ zJ0I|bu(maEubG@)iPnrE%inksVl4e>6|6aSmEzZ+Lcba%L}FUooBWPu8`u*IpEjm`i7eH6Kj(Bs=oypIydN(6 zk?N{)ttrQ$?&x>YcoJ+V5~KNb(=S~~3oGmN@>B?ZelQpoT^D^9tkMip)m_li(Bb@z z?U)tDP!=1}Sd{AXz{FuS1(#Ko79tTX>`hL7>9q3^HIhFdxKUDBxl!CyCn6YQ#Cb;t zb%#mct1A;Q`1WwGfhQce9qv@STcN^=W~1Q6>FQth%F3=Dj|z3XBX>~!9~1TeLtFlu z#PkP<`fIgzW!lhgog80KTsVS06=S1K4l&d7pVoQZtZZUo8)VwF!s*gj?4*_=UGw*X zU(g0jJC8hFHuB3U%XZ&#D{|YFAjaZLx>BE|fcqvYZQJj$pHF9p-5kH;$@=gn9y)n) z6R>_H1Oa~OZH4y@WvL^7WA0{`InDdSo}F;ZF3A&#U~s{3i(Y)xEh)C_ZFTi{vz8We zg8`2|p6QiCDFtA@@yszv0+7Hj zyR;%G&tCYH2*{|X6MtcnOiqOdj(I0a3+{t*$_!U$L+uv=;x@S64+=*s>b zzz$|CzCsi4#TF7yX_K|q=ckc+ywYP)2fCB%xG=kRU0P2Ybe`zJ12K`FRF5G0xe*|k z#{dIol|+1yONh$wHT}IUlga%}D1UGmRkHckwodp=g58RR$tI_vBv=!%(2*YVO`?q@ z%6e&FcPaOH0TAurv$r_!*{(5g>iCOQ*J{k4!wEW;J!hY2jbGPjDT28B>_NKZ@F zT_(;FWT}cCb;Sw-FVn|ijtf`em9%9~aY6YY4*`N<_JnJf zLQweG(@&2cehMD(AZ7`ZC*An-I1J134|Vy)BI(!t5Uhq@zo?&oeU8QYi+(JdA#LQ7 z_)&v{u;KU|wsTzMjn8x^P9hxcvgEea8(Vh)J~Ur=tN#Cy_a0tNZc869>QNK~js-b% z6;V101f;8oNUs3_1ESQ>ODLhK96^yHU3xDG5D*BVg$Rg9Zz&Q8kxmkd0-*=KyzjT} zy~q3h1K(O-)_T@T_On->*?Z43^P8DHvkSL9MAs+FI5cGmU(u9XLN;YsXhMq&8}){#Qe*^YHJv- za$6E@Mt*jJL6t=s@9q#M*`5rlGdb|yXOT1u0&u>R(-vf`w9E~ zcGP+)>jVL(vYV7AQ+BI%J19#Tn-XpNs*!eu%8NB#5^X(?<0Ds8a{X%PN9au; zSLEA;TGtqZiWr$`Vg4*8bD3;$UWJY5mWny?T-+xs9?0_agQN~c-egBnO7{CR%fGxO z_o`Wr_o)!ZLy&)$4%#ir$| z)XuLeO#nSEl+G9}x+5wnud9>8-3CvCuh;xlatkBAOmW#gEujD}WF}x$-k*~lR}L?7 zQo6x=4eUQ&-{MYx19csm8PhT?L_=0LdElQUm%&awl%7_vvGH{;nXwD_>3IXv-8t*{ zwJX=P$~sx6;ghn*+Cgm?V{Q&!mLqj3uDXS5gL?&Qs&+|4yc=jB4+f8?A&~SbZ{Sht@Z4tGDLuZXB3q6x+3j z)$z%OKMV(FC-at>d!e5SYGzg;GkfTYECtcEsmx96(4Dm?Vr>`_UrnmKZ1qiHcNWp| zi#Yea^g>~K@qy$(0Q$kU^I+S|-o^WbsYnAHUP*WiJB}Ia66tj6`(RJ22Q<+XJoi(! ziN7&lpR?r-i{64-S(>qWnKl2$N%LrrPp2Tz12eC7PxqpZj6kb`N{;HZMuU2LTiff) zSr{70p4w&1rqY}>IE?v$O-RI>00{>i7+D18s- zuUl4_p$J56YH3_P23NBXm_0bQ->fNy`BlIRGmn8t6?aRs6c{|JeXYe_1jiqRtISKe zgj(w*s#cU7dY(TW{l~@MCAW!;@KCUg-dp1j=E;W1=d(Mh*kqU^_mJ7??G*%m`>_2r zu{kU^xxc?fug?x;lRfrT;^x2yM{}$=2shIBpjE$lGtl=@uoB3p=UMl>QYK{iCzr6> z_H1xk_OFvKBl#sw-bw zzf@xpblj#RK+KY}&zDK4+3jWd>7C3-#52;ayS(?wO!in+tu*}UA!sB`JIDOn>HCF< zPA{jbarlDQWxLM$i8r|pScYRH>{%BUg7K>X51sM z7S{YOMmF|bJI9UhWK6^Kp~N(D1ASRe8k7eNke7dII(oD>`7Pw3&aAZ3c79I*Z)M(aM)n7=)8XNg(@R1bah6)Ak0g;bHg?!yMvD1ZabH?HF$~d`{$dz zX8S~5=*Cfg>atPqnSYMd$857QI7Q<5_j!9sy?B*CE3$qMbxdDXvU71?NfN|w7jD&` z9IJWDzyTTe8lyU%mNjdXaZjJp>y}Jy+P(y@^(mfmVPVm85T(_`e5Gx_FkbFxWvy^9 zDQTJh`F9y(=Z;f(sTiQ&!l(Mry-7d!sx*ItY-dB<{DHIki<7#)!x8SvyxPhOMTPlQ zVsE4|4KIf1Z9dB^O*hwypofA7TKxe5vucKz@Yuy_y({S7(UirPTV05B5#8f3 z6~kf8-A{M{30X8Xwb-Q9s%tAHe-KUhxryR|b_gsuUTQx1Qm>9bi=Res^9sOnxfhLTR$bpud{fJTh_X!t+e;=+) zy^S4C4>w!6@AiyghtNa=s_4pmn$F1@$Bef*&eGj>eu;JmtJ3kS7vOTC2%F5lZ(;%| z8yO@!{MR?UbqAB|+wM-{k||BxEEWkMY43>piA}E&egb2|N}w{@&9>rh%<9NYiNS(n zskp-uR63l-OOkt5q`^?tO1~k}gnVQe`OpPWl_#wAPbOpmCxG>NmF31#nYoL%v~>@H zzzl-C!d}iNXyE;}9685Ig6$<|wF-G^+Jzuwa$1gsSPA%y#N98sC`U0dg^G~~v((gw zOrR;zRU!YxibS-&AALhPZHvB{rS>ADuK)XVh-+pUNzGgpB-lJyArDw<32dKuF7Qtp zUJn06r$en<6^_5&Li*%uC10x_^&xF6Q`jV>*Qo~BnJDYisLzTb+}4)28{gi;!K^A? zRL#|L3I{-+{bcdkIIH~Ptx>!WNlP zw<4ho@xk-&gIl$d9d}V%M`Cg6hf}y|5Jnog?yj+qVz2NR)3J|9sXD$Aw}CD_u$@ zeS#J#uB7TD;qg5fG*#?)fQjRlM2f1)K65g!H&=ldO{Ln$$bfm6Z>g4!Wfd+$r`h-f z%Vh^~Rr%H*HO`*6cL5>|;6EgOdXA7A)wHmhD3)^$>_66=#+ke5 zWXm09QQBIE@hf*W1HN0kRyLK|YzHN@Y}NLEDExfF1r~`7lYT2Jspe#Lu*q8KVE4jm z$&_)lO74?+L7R>Hnb)#>RF4&lToi~sUK1RA68v~*BkwHBv4oM}6yzQ!;2Q(~cXd+B zmfO|M8uS98^M?#|4EaMR5#pM-T^brv@lWztzzXD5<^t;-YUf8Z8FE?$5k{^Nzy^sg zi};lTDF=yG660x}fCu~FfxkJ~#MPMAZ4#z$oDy>cJ#kYf!XSP4a)#6pc4@LPP zppN_wlKcB?pHp|6??QYg*t!>*j=zlf4Jk+wFlt9fyM4`l$l$NfZCKi&O(D_QY3 zVtmXb)K~Z}BULye1_a`XXECA?mkfEbn*ASw-AoK)Q5x;Pxf%td@0_ZE{-fq~Vs6_XeHL zzyiC>sDTgf6@G3neEAJ3ih>E>4E;n>#)lz7BAvx*5*sHDud5$+;HVd!%ynOZ!)COy zwt~IiG}#UGO_bJnq9FR`I`A&p`(yKjO+S zj9QrPA%+h%oz7C(AP_|8x&Eb>?N{Igx!$==0Sss3dQxh=; zhxqD!s}*m!FOL*fc+6-aq^QdR-JRT)w#v6U@-u%kCSPu(@&(eRVKztmF2umzhuuUD zCd>FM927Oj4jaR4^q$<+4I=R&mhAs3iartTxGPyVc`uhx9qg!|oTGO%Crt`pkXr2( zU&uef;?--jnXuM=!-RPr2h*MFA?Nt53ilK(L>9Ug%VLqJuyV!(P{rgo#ssAL zf?W?H9cIVNK9mVE&2!uTG0%N`)FBwZh86;<_k&|;6=CJmF|P7`!Z+)gb`}<|%|w%w zwOm}8=L_8X^T3z+=GNlxRO;Mg8jp~clRGXZLBEa3q7S36Vq^BK{1eo9nnVRDp)#2O z9=M6fED+!1`er)#;=lYU_b49mhcpcvQd9+-atxlDVPB7>v`vHmbo^GU10Q@VkM&uS zwQ1CNXB73@P*_+rzE!QAuqbUvTc_r|pB^E09vjs@D-v-m5A$YlZ76t}++k;7wS=S~F>25xYJgn)OsoNJ3zA(jUTgFuX(^l4&l zpPojU*5vW%;sOnM@bQ-2R`62cZLMu02d)9c4sg3r45fE+JEBr^vkJOYgIsC$k?set z;{5XLUrA)LdjI-zc!g-|s1KnMtfjMb#Ib7@`!ft;^P+4u$lruwhH5-U+VG=pehV5>T*v2T$#ZFk1Tmm*)eY#ANFa@K&_WiT z{@|3MQ@McoVS*>|a0$Aqk_?$@U2MYLvwO+~h{@2@37ODENBoP|eqL1SM3i-^aE^ae zMT|ratyWarE1tiZZcb@E9-G~9c3sSVkCZQM=;w&OUXzCywr8f{^VT?8b1pcRh%flG zZF^6YaP+5BmXyhE3)>cHY38;0J8kX9%TlNRN}lXE8!Ag`Ulfe@@<_SibgW^WasJ?* zFYwRkpZH+3GS@4f_L4X2TuG5mJDJlMOuWY%;YKNC|GL!7tg63H zv0g@nq$nQ!iiyZReve;yEbh<2QXxf!4myf84L$?;tq12udc}+-W=lBDzjVFJuZ;F%olK1!#uN^>PgWWP*%!eVJ8adO zhCJw@l%k&XvG44YlAW0E?Na4t$9vlTLZL>&1Cy{=BUVhr4yC?E)+*fgH2;;Ek?D*c zN;0dJWO?$78r}+3=ZCBum?cY)ueNvh-$aY&e_J+QJhH1A&I{9($4tx}g{+uP*Qn;@ z#lV%x@MnxXG0fTSJce+tJoK#*|CBXHr(UM5Zn}tdXvD{IlV%4#ylzXZb3ek?aAjf1e=CV2u`={-&|2~a=||PHm%|y zcX&A|bRTZ_i{-vUG_0Tu9k5oc4p_-rSD;j>(g)Z2ZpEGNx*2Nn@j73Oc#$tVQvp8Z znGVWm9Eh+Uo<4vn-rTjYIF;x8m)z zdYR0&!UEucprhF=R{+M#Fx@Wf-HNu`AFqWz^w0I9m>@%EBZ*mdnR|fCiYrFyVYWPP zGMf{hSf-n8f2I5wRalr8KJr*qpRL%i@pzaEr@5%td@9Gv0H!iAx#&ilJXO0yPQIce zlrc7iS)SPsJ!}*pbP_uQ&xTAipWwzM^Oo#~%Z_}}(1PG&5~+EUJYtB~9hM}u;P*>- z>t{XsTk|=gIHj*O*xMh@n_v6nvK-45TCt>$S{~9l{!_GEu0r+5QwH1RFNFQMU*_vFWA0iAV`%{0mi>X!mip zW+hQ~4nv_ES?PV=j~tZK8kfAQpVr-P-4^ZTwXjW!x2>3SBVq4$8b_a>;BPP+f2>7Z z%{;*uF$4CW(1KR`%afFz*B#yMeDg9T-XR*^YduHbr9@%Zx3d9mi-E%Dm`ih>xilMA zn>70|$&>?FW9H6BQ5WJhPuoxTTW2q(d9s^5VLSp)i(7KIXIuh{CtfHVM+lX*AY?*^ z)Y$=hMLJu!mU-%WmFK^`vf@E(QpzcTu6tA3HlGxY-tm?_;8V2fad4q*<nJRFF8G`uwxW;Jd(U+&5HLN z8&dzJ_c-Y_6<|uUTg4VwdR5DNK(O>}D|(UVYp>v4<1}Khy#R`@_dTuSmhz>-|!n5?~f5|Y>8%FM(PlvQ-(Ws?-OD&laGxW&@2R<@do)WoJ@KT zx8Y?M8m(wdi3P=ds_ZFIhAzLZo$9GS!eZsua63Bhg#;w!VY4#w384MpT`_d?*tPLt z`&UR2ZcWpezI?~P!_U=sP*T)!3Urk*vU|%kr&3> zgBTyL(p7BYuv#&JY*U1Wb>AGi0{^HH1(?lJ6isx~KD2>>mRPK+hey+Q>^9$h&U6%e zJ}=$U5_$i#d^lYP`lCpE-!pRb`|L`uId>?-A38H}B34ecYx>EOhO*ISM3{E=l;K+N zGa-R@tw=xo@V;Be#p)+@ma22|`q2iSGqHk@sI`bSGf3QR(JrQMPQ8MLc~a=oZVWpL z`FX{-N?ydETrCU}tu)BqQsz#x4O&hJjkF9qLJyWxtA(uG##~Tv7f|+j`KM^1W_ywL z+ydDtidioD{(7!I(_mk`A?&-EP6e<+ZapTgF*{_nzJ6`lII9tA5j4H^Ca{cqdJTfc z`8KTOGhPmP%--rcZ=P|mHXdT1K*-4j9>}zr2fJf zLm%qGP!{~X#>)Bonu3$V4Q!X%;i2uBHHE_l7U8G|`so4_nSFg?IW6BTEN$Qych<+g zuD6d7iiOAfy%S90NFN;GLvP&yb>j~V?e&cj5@~rAyG>u7bL~rtSC6Fe{wu=Zdjl_} zKQv(A@Mg_>h4nocb(gF|-+gfk_nG`*hNoISKk|V{hJCs6q&4fy z)(RRRwK`MgW564mW}9C&ZZv>D7MZo}{+uGHA}AhfyYXiFiJw4?6W(m~ZJVV9r{Hwx zH`bFb$#G}tf+Wn3D@zmEfX(P7{Xf+FkM96&$zhMpjgR%X63=FcYeiHK<9{H=8z{y~ zb8l~{ak<&3&4fuKT;+9&8iyhtp@e@CLj7gT;tZ3k+t1iTqV4Pz?B8{B`2dHbP{Ge~ zGYyBwxnJ;I{yQSWc)UydHKIAO{D__>)Lvb!vHvq?YQ($R95NBv3hoG(BrX4$UyY55 z2@qSmQDuk~EWRGp1~+E)_3#DdKIg;Z;22v41O@lKcj^Wdge(O_KMkNtg}}e zNP6nah9_*?#LNE6+5wiO*1EqnLeKF**dg>~AEEJ@9nAt|r_)7j`sBiOuV$UH!Al)4 zrMknnOK$Ga3VCYYC5iXeqOfjD4cJz9ln*;G<;`3L-Vm*YQl1{!axnsa6z4@6jRJRa z3*Qi=E#*j8PbirXD&y^?0Thxgs71jv_fM@%oo)9eIJXUYzd&xP!u|w9hM5r^=hKxUok4r5NF9bku`vqR#{6Tn7J;@!w>F@EhBwsQ0()Wf0@I)JTpIXp| z*XMN`_rVj0^f=AL7G-a{oS2JJqQ|?1?N8Osj#p=H^_rN}ti>XS39+-{#S;0f-pZqU zNEl^#6DuxOlh;jabpWE2nxHyljgX9^J-~OVVUWn@r4h1#XXZGA)7@X- z;X8`pu@VIM^yK_w{jns_WP?*TnHX`k*d~D+qQj=LdY)djz?YE*1B`2qe!Y0V<7B3! z=EwoN%;+VNE~mb-GfT`PG7}NlSU9-uiTD~JH|s7JurxsffR1&iSl_Zo**+oNrD zy#g=Q?9VN}*{2B?mZ`T?fDUfL`7cPx-vXWsMY(T!OCptJ(6fvXH}zo8wzTJB8=4}x zngm)uhw@~D51y>@%Xu?xR{;3H&<^u2=E!N&`x_MB5(POrO;07ia2xkPE0xQE(gpmW zQlC=cY4tp`&uL84_@-`IR)AOaJx^27thWVU`|$`={qB_0KmoKdbCWznJ}GNI~#v`xATg5keyyrQHkM_GF&-fvI}#|3M3#`E#bk{zo$>Mk0^V=Z z!*a%V^8o|UIqCX6R|&k*G>&Ua$d6qg^(%25E}FfNW8FuP9@?ePsOKQbb3(UpsT1lF`JhQToX zy<2-iT!T?*6<_{-a4m`%vb6Zw_rC#4yOF-pPrEnrn*dOXN8JHW4vVE_mYkmc<3rj~ zhx(&x8S2y>St?lkMK_zL6zYAhIcs_~_EnoQKm@mkb;fOPk}F8o83S3BTpDerTA`0J z_!^RLbe*?knCmPPcQi-Q$LFeAPM8JVn6OYP`ypSHcBvfQDw!Z%>OEmuJLJKED=e3d z91193!+s8__(emVw5&_Ml>6o+IeOq?3XL@9{E&+P%2zLRuel5I? zkah!&c8on5Z_H7#)b1JFq!UbBcYL9N6H@A7w0lS~(Yq45(b5)%(V*g<$o^q%Aw~;TBIFjc2g_y>& z0YdrvIkb}F=UYrM{&e1iq#$ZD8rbU_@IF1F(L|EaBPrjMXOrvO7|rQccsrZv#Vs5P zqlk8Uwi=Ae|%lz9)W5{Qf)fiJr&-Emay^*_)Cz?Qr^j zL>>jL)Z|UH?{20YhfV7NBfnR0{`x!J2fJBVTl!gnJ4lrkNG6p(mHs)@)r!f(5YyqDH&UCm`Q)M%2YM5_{@|6-MA7u`@a;z1~*45kx#tw zMfGd;dzrQkZvONm_;ii7KcG*zRCFn+SZ}Mh^6O81bv(7n8+BxT6q6pdttq`5 zQ9jmZ$ZDV^O=;DZiw1trw;WEmv zck9&QCbNVe3?!D?=?>{Pe=!wyVz^*!J^`2H*_+JVx^uW%*SaYf&l0&1S*};s$p^5g zxF)DN^RO+Z859Kf5mF0i5AkN~pAl(`fd+^#d*n8UHRCfLA6Rwx`U~K`Cb}%ApfDq{ zkwO+sH(TgR2(sCsE-!LFKszh?Fw4Fm+ne#UU-)VDtn$MeVI^-Y`9ogIQLZhH#4UiV z;{sDeP!yP(d#cHJqmZ$jUAltsQ^3Ez=gG)hB+3V3q zsGZ7_(LF6c@G#ldB3J7HppRG+{s!?h(=e;TA#(?P=IXl^JHx65U{(a=6H}jv+p@Mp zZ4%pR?@y%P;H?W1EPg+9UZyb@reesy7&H$NB9)&K?*IpNRqihnc<5wFBlAgT$mMzO z6#Hi%uiFNgtlwdl6*d%f1UT=JiOtM>*USx&?wbv)TN_sA5J)HUV)h}+n2X0jn@7HzVGJa!`|s8O2mVkm zB~wXjccE6h9gL0C-7M7$dLSkJL=e3_&~f3>hf^&eM1S<(z`JW3&84eu@Gq9OXJ zJm=+HVj@7b@spjoRXOMM?poGf

qj#vWymIR)?D$I_`ku%p9dT$|jakqadsr7<#g z!|HlR+?X5BwwXzi=Xo5Q-)6*1ehDN1$%8*GoL3&D>S$E@7a$4pK?y1Fz+R8Ov zg8RlHE4Ic9LeMcDh*NlV8ldB?Lv1y8odbqsbe#$e_O~GsrEtmfPjT zmz;K`oLoLta-5J~esQm@6TG1o{C8pP+t;=x{d%T)ei0@57s`!C<~1X6MIiDrPB)5s z&b}4|_UkxM>U0<1dZ_g_+VBEemT-xTt}eKG?fPdOfWI7tKF}r?!n(N7y30 zOv1{iHN5O9F!$8~8(D!HL4xmB0PF63?zZ{?hf-26_E`6KV;ZJ@WkayXMeS_W!DpTP zBGoN6g9oGe)DSffp2mFp6#nXpROP(Yl-iy${A;!Szm%BC)6rUY*V@-?#|U*%AjQc^ z#Nz}T+c_DAu}{IF#9=P_8s_EmVRjDl)klN4Un}po$eH!@p@tdo2aHI0g7im3v->e9 zNzCGP8lnOFfPL(|nU4<=IdBsE2NepaHEZze*zjw$wARU>5Xf>*dC-P$b||Ik+AZ2l z6NJ+;BC7SLd;n?gltoWNYP)CBbc;voBkX!vN4znPceI>R$DTh*aNm@f=HKvEx7kq- zN~c;Z(ftz#w5~C8DF9AIG8ErjyJJ6>H5EmJ2`w-z@L|QFXY>tyWmZeqZRA`)SEP&E zi`R*)YSjXQb{lzX>+%m!>PPk4857T|K19x`H4U2Fdv@S`Pmt?OL8Bp00ljdJ{8baXJy8cwSMY>cvuXN8g?9-YlQ^-IZ zC;V0i*bOov*!7|#IB@7d4on~2<54nC5(+o!Mz0|jm_hqoGEk*tT^vla@9^&vxaWk|q9+hMR39vD2QsD_{Te*Ct%%Eyv;t9r z!%G!<72E%jC3<2|7OR>f}yL1=d|?!d(Hm^ki_#Zs^Sod9$U6 z7K5c0fPsEywg%1|6<*zi!zcZT8y43|V)u{Okd(X(Jj@K$#DDpgh`uBC8O5}I?6ws> zpQP9wDOx5sRuM&(tgD?;9KU{nhcp#JV%Rwo^3hiVpDTKlI@hJ5-O|^Dj~XHpWfk14 zdm83i6SuI!G{_bW1qsPu;gtg}wVmT5EvTZKTe2QNgtB779#z5qU4G{Afj9Imbl7;_RDYh+o5=;z)gDwV@TVW{xjA zt(t?(D0?WZ{ueg)>ChmVI+;d*0oSK?xbJKzl5V>oZ$^eTIqKZAZ7|9|d^zOybR`h0 zm>srp2A5W$I?aq1p2~_6-N3W96dEC1#+H8JMtC;tbW(=h}0XV*!4Vx?7N*mv5kq^F7CU~!0`?ZrXtRay2qwUSl-{w7=l7Zc!m z*6G`80IYlw!UfUj%%ja21NEKnd(??W#DkNEj$D=HS5E0r(q%P9FJ&kueV{>2O??ee3mB z4t#f_dZ7y@v)EWkdUSRdV3XBggb}S7KCbteELIVX3{N}K*7g=s8Yb9hKetgD;yS6C zf^zTC*qP17&vfa13$1icr9QISF5mc76$2LV{s!Z_(_$K7MT)a;$IgO!EOBG>& z7yq%%lVcB!NP32n&`{dQpM*E5aNam4<%Iz~tG*d6A`0?eQO35+en|$qG1!~8%JMp^ z_bgM<6u$Qu3MeiAB=>T|gwX%oJUeEQn%iodHrYI&2P!zonl8QsCZQrtG@G038XMbn9HZaCPI31(#wMz2Q=^Ucl?Bk(~?QYqYz4ulS)AD=F zQgs0Ez6cS!t{6V&+|<=7r95UH5YPiiV+^7_ z@{QhhSygPv&*_jE-gG&6Irbs;F8d?IpRsAXQFcNHMFY2FEXu!K_B`@{+d*Bty>jmy za{p;ki;!}OmNTy1wA~k8rxdS0qRsesG3_A!Z5&L{!v+oCEmGpX!fiU@Fd1U;ONeVP zl-gt(#I~v{uGwCoa|?;n=L#^d5a+!gkKNfWXI^7=-wU6%UwF7bc)$loIbM&UrXzG%YRf%}7-||m_F%{7oIr@ohX}q;(aN6-D zB4Y0ZNKWk0iL_o#zA}U2`Q}CR|9EBH|CYEearL4g_BYCs7z3EwP9%JUAt-p=XH;<6# z!oWS7($(U!!7kt1UBFhO`%BW2OCJ+NcemcY0_!kkkacFCpMUF0+X0+kac_?s{W34> zgW0nB-L$6XrGb{0NBXCW4Eg;W0iPDmytYv=kIQc$RG$d>7_->%e>eDAWr7BBV$$7+ zgYuBR9YRHOzT=}@m@ff2zLSW^U|sj1@hZn4XL~0BqJan@^i{Jm2di;+FKn~7UG;D* zMDxIEIBHT#RJfm`Z}LX(S)l8Y^c!hWsZguL^H9|Mc-+W!lu+ZvxL;A{2vmL9*BeWzyn9r7HZP{dpN+(c1jRs}P^fu*zRFrTjvo%QQ^fJGcDNQS^=S z>j|?uj!cTw3H$(ekJ#wAuT`~Lr(l}AY@BF$ z!b*J2uDZ3GO5j~kx?gK>X5j@kl>S;NMr}Z3^Xi+NI}ghrHGwf{a82i969=o-D&Qv4 zwTru@u|EGsHXoO&)?ohAn(FqZwd$^YHzI*}obMa^N%xV7Rj#k4I%9qCl=e_mBtxyK z8}{OWizkQ2*;AWIK6iJJrzxBtw7l8Xp~d9-)1790L+W}BB`wi6x-3?q7vz9-?vOy| z!b>ARGe~Riek3WG1nDuM_5%690V<*jTqNEAgg>e$AqbFJlgAHSwc>dIXnqe0a-M;Br_hCI9T z8#mNRK@+V+i-`Jf$!g(K1CLsMq#bNTy|y%Jvusz?u0hwC@N=ad<;xXMyP^ZU?mwx$ zu`Z;PNU)dGpLj!k<4@L;wrS8x^ZMal>RN3pxMb|3Zm}2apZQ57zzfrsP{%a%!}5-* zR`wyG9$xr*zNUX~61b{VqtXzc;BE zp5+nZ-Ujd#N{Fh4*dotME&oVw_pRT?e*esAqFq6A@^Rg~v;;f8bZh^CD6fM9+b3P8 z#;JgFXSt-$+A61{ralSyObOi)?-0lNgPsi!1;~=}Q-Hy$*&Y#2UQ}RSCTlidPy#T5 zGCWc3X*>1S^=_d1LArr;F5{_S{c{}FUW0#Qzz$8n zZ%m?ikJ*}v60O2ZU5gkvTVwy(zbC%MTxIE#^gccQe=a}8x*oV;DCW?IFm0$R`Ar(Q zzGvx667N;r{2>%Hgo0aRrq#V(3Ca@8>u34N@PQLrW)vLZV$3J1m~$+MHgW>@#zYR%OM*Rt8=x6g#k+y1I7PTY37`rM*}{ z%WBCmm+z5(nRX>y)16Oep$BILotDv34G4E6M6nzLR?xmTQlUw=g!Sl zhxjaoRZ?h#X>f^9u78FRESukBVER{DUfWMMY4v4Bn#C_)d2SM_{P|Pi-W;7PZi|x( ztfxhL@l`Os)Cq3+45h^EC7O4_wihXz+&q3|yfz1sf9tC}t&sXPTXy$yZm8)ct9T}V zZR>}(e(fhK`px-U7(V@nwtVo%0f&}_RbJ0t1ZIQ zFq-yiSQt;aqsUt~|Go7|ijGE=c57wPC5&p%poN72>F`67KS6O7@zXZu*k!%FxEPLC zO3+*yFo2#F{qyS9^n+!z?uCx`*Porzu39!;VY?!=3`AUyMTO_-=ZLIrmxNNV44Cm!0g)FKwqk-R_ma+#AQU*6Iw+?E| zV=TSCh2(;xm8=};MaGF&NKmo7y*vl8rX3bg8bgPmH(s{x(l^=P^n#B!t(^WRN3%rJm%su)L5-4%tre7tArQ`ucs<>Irwb0NJI*x0j;fnV zAlRXy?(J4Kf--T5;HiTWufFDnGJRi6WKFXUn=tUs0~^XBL5KG4*M$ABtLu6)iF8p+ zVujW{aMq%~-ScuP`I-~4@l+rWUu zIT3I4CK`aOGKB0UZ8O%4A}#?{e;H5U<%xE)j?kccCD*faDQ7*p1Z^*;`KK8huc=OR~4j!vULoOdm3SHesE1msu^XY*{6J_^OFV=dl>?r|F)0kt(30h_KAs%++tK`rZQKsuMZ7)Ka!w_F&;NCFV2u>H?Tq-i2?=_oi1tRQsi@({B)4AQkebCx zu_;-jDiFA_Z1p_dvTLS>hU?*XMDsou_g`Ai0$vtum)QwJTeBYjna0Is{-(ZNqQc@H zV&TQESkr8C&)=}HipY-KrSn?bFaI~jQ()niR%vO@(F&?QGEN-*8z@g7tt0_QZ_zlO zxWrx2RJIup%oq@AxTo9QZr2v~(z#`+-<0}SgxBLvL~VPq!eZ|uB9wb8wfMOd_=$jA}2r!$lYjsV75xq9YDh9rH4PvNGDHtV_}P zbQ&C+)8l||)0GW_WL($Uw8@QttNaX!4{xqMSug)bChJxYPdB%kJU7uLRfDuuu5q;G zKwk&hFC40U(K?h6d%o<7*4z|F%7othz^56S96^E}k((qdIj0wh{||TX9nW^Z{||Tf zDn*yJD6ST@iq_uRDr&Dp2%)W=7&StTP}S0E%^J05L_!ipY|&N~u}7q0*G}wNEAE_g z?(@B`>wJIb-}{f>U-?Ho^7*{i>-~DJqjKWz+Rs!Y#FEZ@5~WsTv&7L!Bxl#8K{gjp zaH*BR`q7N8c^hTbbnEr?k}8=z2v!hWxw! zFr$EkUA~uYW3DE&Y;>%GW|sY3I_iymb=vRySGSV}zIOj`r6vCg&ZIRUTChd*iP5>? zq>7N}xa8WEmMXZtxxZqM-e~bSUyz|set7qOC`PqWECp2B@(5y@t&sBU+EG4YDA#nn zuy$!F7n4odqAuk^)z={nyAA@oB<14#LSg2_)oMI)g6^F4q){4D%b0wMX!w3b_ttz4 zRVmb`+nLlDC=QXK$svfY5w`XB7VB4wb_T=!XM@z!Y3$4R2;z|NZ1q*djz2M!XQS3` zWziY5{Qse$&Ks-C0cn2qTh%`X_szP;0b1-$arxT*YODI!*v4usV9B18_)iWs!TO>V zMgoNSK>+8;&J2b7B_FOJ3$9T#D0Nq4W+V)`M{GF!{z$lKev@~(-fzP5Jb7{PTqw>41ZtaK75BPY5Sh*pgCHYl@^}=>gG!spi~3!f zq0j?!nJej(@ec7)c3b<*!S#Dby^9`9&InYVOZz2l?1)Hggua_S{A?RJg#Zv;&u&%Y z&<_!1{etJf8zk7?QqJ5p!OA#g+QEbfJmh(Sv~kP)m)TKe?>k2Cbh)I%Nf_^@+fWny zhu{x3&Jr?U*6bTt*E*ILj?bE5Lo&v$&zg+k{=-jFnQHS<9hYy^2Tb*lD%O&q(<|KX zu$yeQOWnHC+mH@c5EYM%FFdVfQpWm?__5jK{-`AP`g18ZzU-VL>@dO8?!%>ojIv08 z3Bkz_h}V14HO}9L>~9|lgn2y!$AHJ7q1>?4HI+=STYQMY#)u?6)(WA&v+;B)O;lbI{%E_`f}6_>fw!XKlu zYtCB%PH&PsZ1Z{R@{{76%8nfXT`?L~pU`?~D z%Gk}xpnu6utNHy){n=FX;bIize=Y?Rce1Rt9JO7SPc|i9IxNfzx@(4XXWG;qMb5+?Ijk~y>`MY*`@#^imn_MBBfW{p-W_-BG;G-Ws zHj=NeZsT%PUllXTaM(40xz>C2gISXUj> zQ^|W{tX8yEF6om}bgM*jFG5Oo-9jTe(-(hB)14_Py#qAIMh*aA#tPm-P)fm^9Ia+X z8NtQ+RPhz?ULKO^Z0(<@ubW;--x{{-AJNRxv;leZMx}Ae7*BcdqWb!?%WZo16N^Xk#@G>gu`t$T#XI@lC1FLb~TG5C(w+Lg=Hwv))RVP07(muG`_9-?0 z9SYU-#6x|pnw1PW9EfJJ$*eK`4g+=LGZjE3v#Ro7Luf4J%MS+g+3;YVEgrJ=VHb~# zd9REux!k@$$3t$Pyq)(<*uU1FhyEK#aZ4H?AqUley)l~|ZsA8Hm7W=u+$YsA` zXUi95y-ABb3$Ky%UBd3#y22IHg+NZVW|xg-oajx$M_X@Bb?ZSQP6a=t8wzZ3H}H9!k*kI6m4AAzgyJM}&r zkMFCPn&Yo{zGaG3R69IQG8rih*xMVkmff7iSIA9z*PCr^6U_(UA~DSKyf);0g8v|% zv;46qL=AAX`waWcY(sg>Y<3NswsC-{b)J`9YRt(C+^Oe0WWYqdmK@ZEJ#w-_b`{)} zGiYlB%pLuOdvz6oq&aSd=P&sX&Yqteaoe=uXm46z3y`I4k0Kei zyNGNk2Du*oxQ>K4+$Nb;7IG8Z^S}gg5y76D;#(C)d?P4c@j+J*K(Fn<!(blP{JBeo=ox>YygRvH0qfLWQcDZudMFrBdRDZ=mKRtI)JhUd0X{c*8O&?t zqZa&lU%{;pcZgyga+l=*>G4K|;rFxJi=AbQR0R1*jZ z8#KiR0)d~si}X_16SJ98D*;2@me_^XZFg!}NcOKN!OUl*Mm>a_hm|q8_>Gw27fENc zSIpm>p9zzPb;nO%@?kKlc6c?3oOFy=J5#%u|4Q$)vI9~`Y`OKp5~R*{c;~jwfzMB# z`pT6Tx$MoG!OT0MXZ+gO9y}47tIeB+Hj4IUVZWPnk-u2LS^?HX8dV0K3M%mb$Bomj z-!3=$)1MBA@@4b6?K)|7Fwnf5AUnObI%X;9$Xz z;h@Br6~qdHDhhY#ScPlOFP0c5JVK1i)mJ3>buIb~)XWb>CG<9^qmyMaGj@dBTf_1I z!m2dOs+Z^zpoE<%T8@Toi>erYhDr)$Atxhx$X5*R5?lJH+TDjWv zuikqmWf|w{sI>Vmt$w&&!M+a8ypf)I#u(=1{WP*{EeFv3Z!Q3O*{6SVrhm)fqaa1M zvl^XZo0$n7@X_{KUn;z|O~k7sKh%eMT6rtUB>E{E3A6L|jBU{?pg-<(X4aXnoj#>@ zc*dw_QMX~*=VCv}QrSXbJF~MME-D`X)Tvc&OpUN=gJ(xd;2#!kCynCG-mEh^BxX zJyoFB-b9cKAG}=*D6tclvE6qZM%coT}6s}#ynO1uq_o>Q=vSz%?s@c`V z#+I+E4Ty{yXcDjE_XB~O7rZ-tDyauH8f>9EaV_9uW6olWwa#+eqv-=V8HR7Hh`8U; zX{x`EBkC26va-3xxGC2E?<05&{97sik5!uJ=B!*tLXY|+i_P|eF%BOcSj{&i2@cj} znr{|#ToXNz$(@T<56NaI^CkS%Gecx(v8J)jvd&WcS!7YDmU#l21_2izg!;`}-Yh#qbRQP`CZ%HesnD?e z$TaxoBRMIo*aQYP4)>=8tuGE+ zyPi{?bcLKDkQ=7XVxSg1WRhauX=Akl%I#$nGu+b~WL)3dHaz{uY#38KRIr~Pcu?6K z_{4#!6&pSf+iLJBJu^z}XnGXZ{zVVC z**7ld_zjeH&olc{_pWSs=qNF8IvwN-vDaVb1Pu-Yv8NP z|5ywE-rSj=_s=~ARKkE;h;y2%97;0`VrktJbP=4cN zDaFPOSjc#jEF@(6StpOvXAs1^i}`5jV0|;y{g&_bMyX{b=zIYr7y#?H)VHu6?X`%A z?fRWW+udJ_ImLpGH*9kJJ%TikzB@)coGI?iuR_1`JMK`J21YLV*kU@Npz}J8i5$TKT4ewGE7B;Oc*4X`E?Fd z506u36BjvQ=D&fwfM$!4sX!*jJ^x>O=3m~?%Y@F5?J7~QLQs;K$1?K1ru4xrav2uy zbeNiUBvP3M9jbX|l+#-?hqcAN=_E|%NAL$pb#mS!ZOqjBJ_ascwlV z&PCsMJ8RXe@6Rc{=#C{eW2D~1qSHM-7M3z{YoZY!o6drW- z@4J!9+-r!0kR-T-O;nA1^GjUE1vxs^WLtwW6U1dzZYEAB$sP@9M{8~*&&NeRe>JtY zmbTw1x4t!ScaRgia2QDMZ05T5+eGm60AR;CyElBN5`;WvUbFIzq{{5qj8o|OAw6NN z{8rkJ5nCbNuRZvwT5|1F`0}QNa~^wVI;J~o-*UrG@R~QE?qS{e!9%Grv@KTL0nVld z11K3U4JLGtQ6i`27M?tfZ0N8t-G?g9fsL6w&P)@_%$)7|iN*4;{PFT6sNy_w zj(rHl^@WuhfM*1&RXzin(L;rwYo7u1;{ zUBfw3?5A?`=Fzw9f4Ji6P>3Rx;Uv<+-JrphB}YJLt4qUp&MR%>?Zk>sABWR~YKT7GT#Sz3@)5lRLuH6gl#vGGLIed8)KdGRtwO2B_9Rh}zlT=Pp3w-BFo`&h1Du$bV+M#Tg) z8xwZkqTXiW(|lEYhF*X+%_OW>lrkrjOCN>?XA0`zf>9SOb(Z#ZFd3awo$JmKXv)6l z97`=&PowMsYsi6qx_?W%7z7x-w+Y-~PvY$P#is^gyx`*)Y08HMMlv=wE<>&4#UH|y zl1km*G15Nk)p6Wz<+8Muqxu1NU;=q*E&4ref=vNdFty^=;X)4%P2pgt{#pJ`v$-4mx66q3;FE%Eh5c^ z_@BJ(J&s8|0Yl_^bkLLYr9}{q<0-Hdt>BR)#w;E-eL#n!%S6Ci^Y1%BL1k z{Fm34?H62fl6f`D9u`Up-LWa@+_jk;tzn#v=2>g9(vT_UAV(fb$}d8T0NFa5J@6Ty z*$X?A;lSYe6l$`#(7nS^W4Rfk!sZ}-6p#zHoQz<1Lr&VMVy;@f1BeTN_3eTU@FCR? zylQw#LVmwfhk2K4AFYg22ss!lijqWJ;|)Vw0FfdBfL0@uF=a5H8llYTlmZ;RDBO&kW)${;$+7B5o=wTJn*wjkDXJ*Rn1TF$t*Dk+|)c&8EMWUbW!jbQV~yFRf3< z^m4#v*FS?V>SqHFu6+iYv?!<#mWP*R=~xuz5s#|W(|nkW-0S~!=|AU64LIx}9C~7Z zW^-PWJs?+Do9|A;a|FvRcnBqFNoItE3cqb`PS}*Gj17XXR(qx_JpZ1LEMu%Z95>`M zytN?1t)hg9O9`t&4;~u3K0!q8uWt>yx?h*E=_dL!--udug1m9>Clz5=yIP^PIM0n? z*#gk>u_^XC^Gu%voG@Yaop9H=S|{TGR0!fpVxe2N?^*Tpc5>H4BB^IMX-vQJr=*Hu zhxWsnDb&O&HL5R7Gc-J%) zkomz#**w&`=wTC+XRN5u(O18?@uy}?L=|+T%yV;5pkbi2Ec^U%kYbP)Jae_iwxKHP zfKAt1)MN66+TnLQiN4B*T&Sd z%K|FRdB`N%FIQv7k z`zs{Isw`kWuzlQ|JzKZFWRh09E{@m%m8o2CO$MUUmFMeiFb!FO=Zl1cwvz&@jCu>Q zpQ)XF{&A3A=btPh3L`na@t_S!luj^X5A!M4552telQT$L@MqDpZbydv#(c;Jnct@G z-l<8*ub__DSH4Zbk zkxqDC&6udpftB&go6VKTJw&r$!(LZY?aW{o)I{h@(`8nXTsKbg(rEYAIxk@c_P8uV znK0q5rv95}Nl$Whuvz23tZb3vst)VY6^`4|nIzJ=G%~eF%g~yyJO!|lexu= zLCiDvW=D}~O&&7bL0Bih;-H(C)G>DzcgVW6Gy6))eZ^%|KU2)vzr?Vg`mUd#&=@Av zri`AE85D0CjR1!6Qj5MSNqBXVumkl*eiE6=t1Sm7eh_@2+cxfx+l=ax%{C<29Hmw- z+T6BCV>%^kt&BW>&&Q(%ihPeoGu=H`ld|oW-xfTrHS!bUbEh95%^$4_YxT@ z>eZVb82r=0U2R5{-bad9KzT=li}xW-TdgR5*ssCHE&4D2%^ryRm0ACDv1diDhDBXJ z%(AV=7OISRD#4#EE2??7KXTWvPdps4{;$Q}SNoHsikbI>+v?&=P<(N6CMhcv?brAR z!R^`vqb3Q>>yteH!VA8Z$Rrgv=q{Bk7bhfDY7SRjiF*v^>)U4Y%2Gi)MOzcx3rLR6 zW-^tq=5Lq96hoehhP1P5IY+*3C!1nR%k|x0CU$aVA2AkrmH4sZS9w)~XbEbq%DzZC zNgh`$EkuA9O&0f5D$IY@aE8%hIf~q0c>VA;9P%E`oFM_6zScPDnmGmH03}aJPTj>{ zjI@r{_}M&tBFBEOdNO;Vvo*KRGdpsfLC{r|%W`n?+1h;l8rJ1an({#=-yMqOQh0e| z8g%|==4l+~#KYZ6E}S!;t4QOb{@Wutqu8;Kg-p)>dO(PXsyeSGShx-)T=Dt-c0}|&=SX~P9@`9_WPf>7EPsRltd@(MjkkgkZC1zRFB}@Jdr5LkV zQcT~pU3xDzB2XT~53=#Q)q5VeTRL*_sU{NhTJ&xmEmg;N{{GEh@4G~JmE%$`m&l5P z%-*zL@$3AwQNNc+;{E1U!hW-poNwXDwxnRjA6x^z=bFqkr#whIE9WuWPO}xlM=6omXb;`YS}9t_Vms6qJ6mjXjUz~@Y+^eq)f##Na7`#u4p3n zI=0ua2^ESor6e~_d=Z~Ogh7MG?*qGOF-yww`ko^)R&`QJ&u6h-4R8L@T>0l2`j?&R ze%oZtQejdKxtt#xn;IL>+Js-`j%~&ff&zk%9tepjxz_Y0-3YR*yUI6+dn2^tV=KoL4O{9nUsv`eCMVZrse3D`@@2li85t`;2~hIC!Qr2BDSxTF&~?AWH9=73 zPXoemm3bt6A3bj94pi!CkY{U5+oSB~6Fipe_cmPCetZoJR-%cO89ukTFpT^ddQc+N z@_ypdum9e&_;dHsZHY{9spW9r=Nd7E@?g-a_i#d%wTk7h7-H?)A zEF1QL8yOm#2ZjZl(_e61IQ<%en4W@7`#-ulpdGgiTCXbf4W83BEL4#dPX^g-dh>vD!J!DqSUTnZ|%0e zs`{~Y^LTJk9 zBp-S4d_|{>^rYO5kp&BWyWK0-PsmMA(er~j%KMVyVK$%vaA$gzvFq-L;ELBJ!adre zQC8Onn|Du|Kty6hV(2DQ?K-s*aVR#*$MMsL=g z_3cZzfu*9YN>xfGqV{##88s6k^T_gq#0x8l%nHRJnZ@j-=*;-74uHvHljBb*f|<%$ zlKRUgB|hVBh|L}c-n!|yJpFfGsP0GB(f3a$Nf!v59_`WzLk9cIx;3|me|~-THsmaa zh**=^$;g{+g<@=&_T0>CeAkJ8aw%78fF;S^{x{?`f$ANFC$7GBXjkv*)O&9>AjBj~ zw<*hm%4gN`yA;mUQJgcW7-$MR>e>tQzmDdyV{yoruyz%KXO|f-qAUyZ^Zagw2deeU z0yxgadTc5G6kl&%QqlN}#Ri&+4L>|A|_5=`8=gwyuR%(M*r z1?8y{`M#fSG!Rd<65OSo4{1$cy3$F91B!JyQ_FW|$4v+EING2Z^Lxu9xd^QU zl41Ynlb^jsE#-Bc?&q9vJ3aP?h&s-4=~LZfqI$yg0P}Au;{R)@YVzvJ1-~R|g##>S zO9V!*+EDwsKEZ9F=T)OPCSVS_yqs#0CeNsO_%3BetRj`#=vviRLR(i5Y^uuQmMw!V zlMK2QJ_F60oNhALY^HLBcla+GwkuFQrx!k27FDkO%O@V9-%N~Dn(xaXozxRP{3+)G zv98ppni&on5E-_Nra7RPGoc1vWt6;HT5LspMN;w(>=ly{wOVK{)i$%hl5rnwp)>(q zd9q9Yqpv?~o9BGSw{OQoug9r@P%(HLyg*6_Mu=;DZA+;?UO^4GAaqba8EWuD8KL(3 z2$%8Sw(oOhPksVjc`AY*)O%E8OAM@w()`il`^kGy&PgIViKyFtHLK6DtyWLZmT0EZ zS67^hzd7`cyqvbT=Q49OAgV4+t`6-p^=cUARPqTg)Zzg+A{9=%l)xh82RWqN29UO4 zW1IvqeMga8l53b?NZ$7YVLuO7wA-swOj3)3(5ayo4t+^QY@wbTzklnU(R9SqO3uVA zu0W!w@KVd!Mdf-xd!+-$)u&ESGX#P&MXfi|W3L;F59k1iN`k ztTqk*Q}tW4_FJk(NHO=`^|_;En>O*y{({Wf?bI{g*X4oy>R|svuM~Bwc+2W1_)snK zANm7fhjTZwYcw}P$FRfP11Y0%ZqXjSr92@$^ZfQ3dh4U{IEdI#HO|i;_K7IDo z>G_Z%^gusWwcm7T*T#{rZ+$3eeaY6^A!A?GlXPZus010z*7nC@jx|H-;f5WxXh4-+ zp_rjxH40Wj_2XejcRIm=jCiL`YuUY|b@0LZBME9M;H}&f%mJH#uaA$8#ZNMNsvQIi zy=h*d&Gd$@nX@3Qwgo()!h>sxnY`xJ$6r3Y`>mEX2HHnsAPIMxyZ0?U4aa=ijw|7# z>Bza$@YVHTbhv*D@~_atMCkpS-*9H@2?9s0->i=E=v7QrPA+%K8et1w_q?6HjnCg;`$x92oc($?`~4YT? zhm6~6^wdTphD!E*G6zXj!Z06Ov}KbWaDNO>*&8Y1l_`Yn&Gm0l%9`}LU`zWzK=Gt8 zY}!W+s9tPel0>u)4JK%F1>9h_k<@-7LE!7rF}#s54cJ`cm(3OV=4Sg}aTUQgEOlVv zquw%3SK0v6G%eJ6YuU5*WPWO7#WRnaDt_F^0b0B;8TEkAp9>&f+>|0H@CeHnWYx`~ zu3<=1t}z%_)zCcKU8|}8&S$(osL{9YQy8ZM36Pp)Oq=3V+oW)?RyZ2hmq-K2)!hWz zSZ>)tB6>bt7(fJDhUJ39pVp|?Z>W)Xw=T+w+mr6_=BD3fl&E)?^dt&w zh(NSwU=&c3yu_7@rVki}H$}}=R6o#_w-i&Z-p%u_p8*||nT2ONM!dUD ziP~=(iYeJJthBslM1Bcvnfb1@FYH`jg34kU{7u1|eTu!w#h&zT9uunN+d1_-l$g##>mccFhSkWo zW@_4Uv@1kW{At5&TMf%@V@r%cn`9=B3_*IT!hT(x+p9^3nm8}|I&H31&ql?nyOO5} zWydf~XGf9jBgkcmeC7JZs9>De$+Ppl{@a1em!O%0g{u11jl z7#}()u)g5*95Pt{gnlv;Ec&k4ygY>4k_qjt%GaSCF(unw`;Cp~A5h%T)Q#Qz+Vbwk zt@b{9y$zG`j;UeUouspHP~#3^IdKj?sjO%ft38I9(+dXk8E{I1f?+ z^NmB*R!5#SCx18pE7pDN*cZ+(F4qR;A0}jjPRSIc-BPYHX;6O3;_Wc+JHY1-sW2O6 zM`c`l)v)^YkdrF%Fgkq@fGoYRt74<4fdHfO*cZMG@=_c2Uq?Q13~^(@apo3c>1e$f zX%^q=#H*n;ro&Y(QzVFT{JNcgR8*>A7?~UGI(!CWfh;+}D~bHkJinT3NUetAe@Wgd zbwBeO18q{qnBp;{6jqD-FJs+OUG-GCisu{Z2=+^oTEYp>|Hmchp9ScI*&!(6?=e2d zj;)S={r`ojfeAaoq!Rwi7vOK@@c$*Ojbks5wB>g}Xw(l08X8P6IlVuem>LXT)8T-Q zM@Ud>gpIgTf};NR!Sq!S zYR@+q8yjTe7LGnpCbk`=p4~t8Wya~cywX89un;OCN9zTJXB}0IL1aLeHHcPh`a!=} zo0K}%-F+#>jV`~w?~}`skxY!1Oa3$&iiO!-M^SXN`~IG8be|Y?+uq?5OG#+7_5m0) zS;f(xUT#bhO9(rDEam-E4%5S%-SqH6xlMycg^FLV=Q2l2dVt0UYsnri6R5=x#WMzd zp9#ek9;3F+Kdjfqc`#9I0_1uiJZb&Y>%T3Pm*t$-nXRFngfr|6!?hY|qlkt4mpylZ z(Jn~IrU>Z$m49AL8Y}l2HS~&y)v%{`U*|Ed)oh2LgwqZrBU^$~G~=ZZ_!N&eT1@(9 zT|1kNPa!J5c9lnP1?HsHZ#KsdQhYLdRus0#>F>Rl{w#zrZsuZc^y+0sjubvPq>KHO z735lvtv2j#BL{mtYLRv7+if=(KXua=19ZzL;Ptm z7ZE^e(-m`PV$lP(AGAe@{@i4H)WR;$t5FSOJLFEC8n#BbJ1a3MIp=a#)!!5AK%ZWz z)YWE?9vhnPSy|=DB6(H+1;KX=fz>9h8v>l+L+9Jh(~%Ft$9nUO^SW3S4~^Rm2GK>+ z!R*gPB-;XJt;ID_`T3k5jhUGqk}*Qg+G%~}J}oo-iCGy1zw^&lxOc7WP5rnRIQKDx z#s7MOLDG^x{>-C}L1&)Kxet!!zel_2#ICQ)IYb*}IsX#A8|w|*$XUcnmw8=<%RxDK zeW{x^`U@}?DsJ|RGf0q;!M`VRQ;E`RQESBi-ooGxKkjdfi;>QG!%yPve}T9q5h2&w z3SVN_1_~S@KrGnkE5om*F9E*y+mv^xMyF0;iP8r6W}4Dr3YMc*nL9kH*4x)dx5T=1 zW0I()Gtj$4@UWh>4pAX&gP;h=p*inQZ@+hjMI!WFq(!l*(*HdLy!v3qL&TR^CM~ zpIn*8Qk2C8vOukJ={NiFCD5(wMHp&<9pfkdBS>4ay*yDaP3|yfPXlZ&pUSeZAw`gC zFxi-533=i)lT}+=-l#I|h=dIfXjDiws(xN%Y20E{V?y0;l-7&NCrxSw!)3iKWXO#h zw$bTaA9sXE-6QuONO(|K$5jiB$iZE#=`%MRSWuYkCQ2_wnwYv9tVUkVuE*<$^Ip`#gTd z=xUD@8PH7M7wgj~b`mX?^VypWZnANjp>O`K)?@K`oZ3>gu);qJJ=~ts<0vT#1Z|n( zv*fNga7&+{&DVC1^)QxqTxc-{u~WP94Vp*0yXOdZ(nq=a(68O~Hz^;0RK(o}U0>EQ zy_Lnac09T3#E1_$XSiiuUm0yR=-c7Mzr*{=|4-mFtJ;}W4dL$PpEbYbfbEqGrKhHI zj3n^KP%!b42oCYLmoj%ZquE0v-T1e*b9_ zr<}1po)^tk*T1f=S{el&3yk!D#oK0-Z!#sN8nQlH&(%i8bRsSm4Jzya<`em^q}zFu zEg)(h9}?>x6e9~xH@ZFrFc7SrJtUTo$O7Qytxbs)7q|iL$?$$Qy^hDOZ`>Wi@>fJ-z z31NSK5!Jswo#uP`NqR|s$UAiRx2R%$EHm;xp~1iNl<*D)FPCD;X7=!gO`0$=?RRcznu-O(nB*nB-fXoV{1X3 zN1JI|LB*hgCWrzkBNPE(XAR z3I9T)4@>r|j9^WlCIhz{d`>to2LbwKMY(`Pusk|y(Gc-%$vK3-Zso1sw$2~_v7?HchCK!8;P35CO>0XiCDG~MxjTm# z>FQakdg01%+MZf^3+cF$>zFy9`_8kY&Hd>I4}p@Xx*YS*k!6FuL|DBqJ(o?~N1hsxNx|iu>;KvO%WHt{z7eg3Oucxd#CE zRn|c0<7Szl)}wO&>`7A~jHhL#!TGkDV6GrjNxpLYq0||Zn5ZJl&*7(oR=gLtUf4Fr z6cy#XxbUbzpBWg>nrU(}SFoU~BSgdD(uo3tKjZ#98i#knCA0(sO5>l0_yFLv#Ct-Deo;odF!7Mr#g2I=Nyh?~%y~at4L%J4K zF-8o%BkL(;clIOcuo^~XVp^gZrU^Ij_tT@#myv4H1e(dmk?qa*<|EMK^9?$UW{YFr z`ZSyG*rO^v;b>=>I>B_z_vpe2 zS%+-zPhN&20|!=L6Ejrg5Q~@-bv(1JBYDv%>T=8|DI z-xG5}(X1HL!$r>L<1=UewOfMD3%~a|mFHtDtpdM3}H3PWz@r# z<0+@hGniIkxwv&`DRR@WaXz|*F)-Orghnj-)qSw(R?OUm2TQU^hK&A|^Lxxhm1Su$ zO=(GJOQPSB_m#1lazq87n|iOd;i!LabJ$Ti(L50p_5~*Je<>p&7He1s8H-7|a!j}k3z{bE;$KEA5#N;Ex@yT-;L&e2&(Gt^&u*7We zSdadsJk##AW|qScG6i|uiMV6CAH#W-!-xJQ=1^%klP4?*@Y!ZLHq8)71DS85zGqfI zRJH@I-6&`0AN;)du5%r8;pr7czArW9HoDa9c9lSpSvCnp-`Sn9g#HCpi966LyJ@PO z;C~FN8Qf^sxIHsB@%4-5&KS_hVq2P`(Chzq^p5 zcJ&zJy4ZnywRRG-8kavk=!WFev_`j>=q6h)oKvar`g*JL&@tn-VT^g7#^-x>KLQQ8 zcu)_dWD_Cs!1_HgLLfZ#nYnlLIaY@DIW z`AX&$@b>q(ZO0?tg3@)O67S7U?~|LolgGy-{#593}qD5H?Aj}ee}L|Rg>}cxOC#YR@K0z`+px}Nvk)0ei9kT%d9%{ zOR)-K{iHPM>K0z^p7mQhriQWo`0z6eZJ|fVYJsluCW{Es@l|E0*Ah1HH_B^oxBDrS z$M!tRj#gb^zxc>~wW3B&d>dd;^~L7MlW8cC6t(ROCYUA9n9fd{rtlmwcW}*ILEE9kR?*J%2gHJg3Zdo(UNWmaE!V1JXTWb6l z;%%8?!3sHVCu<<@C>hj%>xB+~MsioV7XHA}#Z+7U#9ll81mgNHG^i% zE+AVPU{Q=T4FkiVz4p7BWLFdC(?@bzy3Dv=gI)zci{}>BAR*)IqMEbsdm?)&Aa1)gMxa$5)#OZd{%qb>2&C*j@%JSN!j-GM+MR;@uHrH>cSK4r?ObC zS?lU4$J75D)a6ynxT&u$Em-4cKz44@rXSnAH<9pbtj2WMsG6LUA=(ya{81a@n4QPg zm0)Yd_rD&g=Vue@E#*Y7(S7t!Br=aqO)g+}d&A8+YVe>>UaEpmPx(y{?CI6~jr1}f7 zOdc3hA~F8!kGk2`o-w;|@{W(R>f{~sFrb)NwPmcSo}k3Q^ts6CWWgqogh%~DPF~=^ z)LwGdg);rMM#BMG-m=27OOAFYf|bwS!sKvvQYjMxVJ>GGft_P1+jp$6_E`tTR!C$T zKseKJY%Wj78`CT=bo*242Ri`cw`bk z5|Gm;&*^QQiatM1#X#y_=EP$$%rIQkZ%uQHgc!cAj;$(W=&aYG*qSdOPvy3JgXQ#i*DO>PNSgUINhK?lfr$&Q)8&quDhfMVD z(0-%}H=Q)h(~c^FiJlNBi1rAc(^6Y|usx+T%6HlJ12#IWJs$b}-IT1#&AQTI;9OQN z?V_o2=!55<>3>DBj=lV4ddEsbr?Yso)u(s8yGuc7$EA0`U0GmUgfc5ZP0D;pAe*KX zud|jj^Sr!JxOR4@t3Mh`NRcnN-fzJQuDTznZPdcu_W6XY-_*}uN~-zAVL18;{*^8J z@#yb0EOg?opcTnRNw;4iYnMHssc@BNWiUx@qi!q&D!X}=(8ai`=ucl+qj&$IuCMow ze5CDeT9_o~9DP%YhAaAWWz17|9bc3t*TC<2iULXr^62}rZ=@DTVA*24IJWMpFeg`q z9-G`ecYautMqg`yXOXjo|H2yh%yR^H*p9 zRI83!t9`}wuspT$tWL@$*rP-kHf6Yk4-?yg6T>J9=*z{K_ZKgpS0}XnER3NBUmil3;^3;#96o z&i9&%jg84YKi8--5&>NXQpFpGhT5ocpP32xcrP4^nJT4hHh;tWdQ!>)#@`-j0&?|U ze=`nxU2Je4*BcT0N<2&H_Crq6{+y3b{(nsRTR~4x-cYma`+X=Cpp=Q+>}*`}xMw^*1}ux{^;>*$fM^&~xlXD9;SQx-vK0tUcgUyG_SHcltwL6TiErtb?M zW?rk`LZCW1FS4OvJ~C@Ceyg4EY?9lZdI8_`uFZSzwMc!4xfWA?>$mla=^#FB(Kl{K z+qXj7W(^}-v52YLE;Fp1WXTDDYP%i;gTA~ij;|`};fh@Zy$aRRs;JTDmslIF60KP?a`vf2v-u}VU9xBf!4W@env36L#AcHMSn&%&FIA=iEjp}xrXqj)-~F` zP06e=1lMWJL3>ndcs%zh#sn31YnWixUpV;>4Zo4QdgRv0>SKKhu+52Uz>V%IGO9B6 zd{bdr=JFu@z2^YKQNQBZ3_Gt;*)$K>=;Bs-krn{*W|vEPx{ncDxJlnh`w>l($bJ<= z@3-KP$UsMSy-@b~YF$elpI2Sc{ErQ?&)nrF**bZnsPe-hU`0*Cf_{19y=m>0qfV7Q zk1tiVc=w$1x)FA-)5f0HrH#b}m7Q-sI*1zkjjK`Tv;6+%I#G;Lz!bJE6pigl4q^*t z#%XVYTA(XRe*c)3JI0jh)YxM8qbJ4+51AMiQ)3-&ThY-8BQ{*#%#LT<{C;*6KnyOZ z)wqCO4){=R=I_$iBfPb-_iBH(5O<($4Eb1ijTmGDf$fp*GU1hRQv}@~F#c`umwJ`Y zF_VLlHS;a)#w$X>8xbB0mPYbS!;1%)8)VI-e7!(8>OW^#a=m|`WSmg@l9|)`Jt4Q@ z0^r9`O01MR_H5eDV5VlMhz3A_@e@5z{!lY{q-9EW3b`-4`4z1^?|6aS z5L!zNUiUG~sI@)&(T1cD5ZriX=g>s|vSfHq0vx~^0?GL}grceAwCtVf zzm>9U&4Y+R4>?A5!!4HE75Bl`sd_dH4Mfk1MT6Q}&b3i{2_gJ-?A)*&$~n?_fcjf@ zCS9ofj^mA}{<3>^^alEw`QF&7Y;%9`v1<{q6;*`a44J9xrld98Hcj0PNrC8%5dj~M z4lA+({0SM?;7OPMzBN@>Twe<3PBLMJSx*FF2jrE_fK4Y0DyH;))*Fe>V*9@)ZdfKr zm|To!1ENCMfVdl&m&f>1Q_*_L6~=XgSn?j>Y+|65P?8&Fe+19H6?}3?u!bXs(%)$} z@QNAbt-ek;w?|t8m+5-rb|3ls?ivi+Bm}o(JbZ)X+-p#XA2|gDA@iP4_OVNBE`u`C3-a$>RZTqMV3kq%( zL_k0VL_kET0sNAY09P3Vl?b~X`>{VU$|4rDc(uUjp+sUJ4i5n*j%@?7&Eq4^J zUb!)N0K}>TLzupTRX|{E3wdL!2!lA_5A>;jUU!H22lnV%UNEX3Fp{T|{>E8GXW`YA zjRp%uxgz3y!s&mW{W9__;qVGLVVF}WzA6Bbpmz=3eI%8q$pA2r}|Ah`N_CFrO9o+xX#uf#M$24Q~RHV6) zT4*>5f-4O6XR*r@pzXXRQp)7?%^1}Ht-0z>*ZwLCe$z*YS*4iaGn-~d*NDopWdo)c z$Bkb!U4nq6Z#kpkOCrn{FERuRjAxr|gcC6pAe`zP}NH+1JnixO( zSQO6K$?;Ioirg}e_4otrSM06LY|(nSbS}h-m8$?jV7nrkt7q#ZhVW+Q3;n-sh5qj1 z)H=x@SQ%2lGAIIG`_`p8qvCFL8mG2$QjX0TR>zfDJ<2t1v|8*ebTiVn`uwB!NbQT^ zJp(?F*68(Av&NtR-IFWHS<@#h7d{4>+8oyC#e0Y~484rulr!PyxBz&U`VJrekgj6k zb7Y>&zTfk?@h)S)9>x&(I5Q(7T3f8ArJG_zo$4y?ninty>GrmL`ixt5{FL+an0cuS z%KaA%@BKIq;_ZTkOSd*zPlI3DRs+0V{$@Ma`aVbZg4FbU_PnlooCUs+=djW(4U#tX zWvIMb3$M@XWA+2$lzi<|CyhHNJnz}4hsfaR*gNr+TJAznQb7sCJ9C_H|372wUmq`# z>)0~1b(7~#i*g_6Flz>gw7!C?hG)74D1!a+BDz81LAgoXv7funb(y>@Qi5_wsFp>} z$u@x6u}isO75j&EH)Mbj3n=->;m#!HrEZJ%{aL7%uqJ^kOU$!XmT$gxlzshN^=x>Y zSXJpQo6{l^vryc{g`%5BBLK~B*N(Zb1E&=D`o%!ISLJ)|9{@|HaPZ=KcIx^88b15G znUXTm{@{ek%5o4bqVk1FJ402&zUh*w&XQ~Gljx&Ey;sP&A=Rz57bcDf=M3M+n>-A6 z_l=Q0tMbLiT1Cu%-f2~F&3VFX{rz_)#7;%7(cJuOzMyAr2UQl9cG&_+S!sP`AkKq> zp-(lcrH7clb=1qGxzhV(+438(0!fcj`=>%)(5bGNUBQ}=@*vd_iCK>kKAu#Mz=XXV z*1~6b(rBNBu5KyL<2RYn#sC(I3}8w7P~!ZBpC{gI-}MQN*o%C1F_jLpf~l^b*C- zJV_>q2~>!J=+ThwUFU}~{|gIH5Ov4t&s#lK1AI9^!s#e)au0 zq+LArowN4x;_tsiVC-gyOsp61=(QjuqHvDAKS`6P$Z(#L| z8q01#(r9{m>dJ5+ER>2PE6g$~GjVD3y`{>{!Nhg;ptW$sK(I-!s`zFd;cblgyQ>U6 z;XgZTe4kO|#NDm#?TwF~jfumSPYdvT9z(XT+dKX^lTpxoPz2p^Ry1jD{9+wtxXl1P z9oJdzty;qF=j)mw!Ler4yzlXB1N`bdKMs5PeoFF#4x!SSDJ{t{UjAA39R&#P6%pEu zFemfUpq`$88T~X~rmd%m?-uDHKJmnb6p>$^z!4D037|C-aRT909}vCvK%m)}+GDUe z@J75`P`Zx8R^-);RD`9R-qc~QmeEY?`tp0TFXbmG4?hH8-?oCn>{*HGlIOW+<*`*I zZnWS0N_6jG7SxMaTXcb?k9V(w$a8e+(iXFfAVODRWY=6)kLZ>cv!-X8 z=e)F&M14iWpxF9U6m5P;EzSMxRW*9M)K_KzPI9OH3>OYlJbdH2;ccN_kWI%KoAxs@ z)*e@#4i*uRj8OpW;KCr)6WDkE z?1h+_A;+)eabZ$KOO3pMXU~6Gc!ae1J%sb^W`>QXKsGYP-`pyV+t}Vtu7bW zgW#x{J@=|(gDGcZOFoICSYWoue`c24~s&ZkudW@OrHLeETuv1w+`?W% z<&V~>@;L8iSo+;f)Gy3Htz-m(Y-R=7;FiD4f_TE{wh2nt%}wnCRTsUaUN3X8bj;PI zi7NNQM(X-HPH?z_Lhi}ZY7nq~o;$$Qu77+3z^Zw>O<|v+mx5FxE~U>^W1y@ibsV+_1)+ z^Xyo&8uhPhUDX808NZ5Nr4ro|9Z3N)uh^bnrBHz_U?kAOWMRK^|N7Up^DXRo5-z2K z>pE%{_bxEu@D?j~uOkxqma-_mrx?tIVkRWKZG=l{?~EcNYcqWjd#T6xWTpomWo!Ad?x0 zS}MRYC@}GcPRu1POgfPP!Nl0~<~cgW57OGQ8-!RB08*;U%A z4UXOVZ>0)79@Kv3E5Cc*ZlFrz$c#tqqc}3K*YqtHM}~>N#sE87xlUPF1A9M)0c7UE zIx*DQ+}e_VY4>#Odf=Yt%XS+QewdYmM$Ve0XOSd#Ww;9~8zXT0> z#3irPq@S#EIfXMai7T2@aA~d%L{!XmxAYc{>PCpbnLmrFv>rtg%JKLF!jt;ZD8u)v z?t-q}!B+XuU-VPCXWpY-D{JxDJ}{=C3!F;vqyy!n?on8!M=~C%XU!zT2{)cYL+Y)ez219zRDYB?0&rN>hB1+VG)rRvzE?h@l5zPNrIJep_X_lENG<{)=YDTI+ zoB{c&!(q`YY;xY<&>30iG}v$3$mHYA>L}>M&cKNiWxlJC`+aN4#Y0a&Rm|?Z?cK)g z0KUhwPjiE{xYg0V$#AW;BcLXrvXkcnNYm=Xoz`_Kq@7WW zPdXjsNtoR_G#(nz;l5)T8AhddNK(xP0r!WSA!`RFOPz?PgVg?JlwHhpc<(HWK0~ne zmLK%nJ2KLb+<-EYd&=q)7P&Hn!RWk@J|d4v`O@`s&4(vbge1#jZrKXMHlyIb8@9`z zyJ1uM8K%OXM;sdyO>a>%4o$1*e3g~qJfvb>SB3vFjPYD8D2G!Da9X2e|FC00?NTC- zw_eu|A1l(FXy?rrF7FFkRmtc)1BN&Z(JgcozviSzNlxEh{f2@D^gbVlg-D&{KtT9Ar zkH$No&&mq-0~pT9Olb3^hA=MA#Pn<1ga+N~c?H_J2%UoXQe&$TFZI8X4ak#0HAs-s zaAsQ}_lN6Zrx!w(j|&#Jod4FXiIKPUiSUB9l8;E!{=tlaGA*23S~S*$xTxEdpZ5>U zHHZH1_ZB0Ko{RIyJ8=_t(8XOpUZmWE?FQhz{Ge(?Cpd7MFUE7LfTb{>4JeL z2UbT>wS!fUgMLQ#aPuh_s>pho)}xO9+Bj`-Dv}F7TQ)&8tG}%pvp4@D08SQy#HMz> zK9b>j>625M%nWHUSJYk!;>r)^!sUtW>MOTvH#0M**pU`OBbh~Jdym~aU%`d%e|Uc+sPj}>{7HtrwFuy*K)yP-F^0{5 zT>*ZCaYfCi0L{ceq58Z!#`s=&lG@PAf+v+X;XXa*!?1=uzXK~TH%9fn_aFM=;7Cgx zb5imC{IaY@-imB`$N>xB_+sT7f4(_z+gy9V<4)9L$v=lELwPoXgdS~ROS3sq9Lh0UfIL`}g>{O% zCA4ezWox0`Pqh9E_h9fK?ZlhA34jlCLFhS!4YE!BoMTXJyAsPT)2V4sgv=BCLhXbF zu)jPG4-2iWvCE``T#%L|3fk;RRSX*H)X^&jKDP zGKKnLufx5bn3*m&BOUDH;V9r`i|?P!8uVeE3xTeHO+Tx?cL&b4q*s52WM-Uj-eQjM zkki{c`Ke4x8X5JuOIPxo%<^n^`izO1R-Ep_)@vph5rH62`4rSIPk@-gw|;H2+<_2Z z-SP0=7`YFMS%LsNxHqWm@^BQz_jzt$kbS{v5-peD`dZB@{A z6o#Ffd_r2ML@WBqsph+xP38*Hcs~z=Rl+Cv5RHWvdxqM;o-(PA{gXR5w5^ysE91;suYhPt8}(r@DtnQ8p!!69|mEVX|6oC z#x>@iRM<*mZW#IjEHQRM{g`XQSx7cu0r-5=l>Zfca(~3~&%iVkVKa#QwF-Lt7~7Sl z07MFphH#9e* zo>2Ehx4E!OD~&?GIRJqp7PmT`5-zN9>)4vJuRczn zUwxC!H@1-&lquYu!T#_f1DxBeJzQe>*{6^^`Jm3R$Aga=A!LRQcLKexwBj?}xWj>z+Qj-#VAuK=$UffNyM$K0@4=&!`bT4jvfdC%vy z=)gx>NeR<7ni9e6`v&uiB8lEXzfnp{O2$!JcSU=@*bcBgD~pF;fH6=nf?(HNp~bES zrq5E_iLa8SXhuV@wpsaOG(ho88PM=UZG?`+oV-tq%CGj+%oW~R zjdu{1ZlCi}X_O}ff1eS5jggp@W$1h4LCOj>vj4b*0C-_&qm@`M?NP*`b*ElKsrJ{U zt7Xlm!3votk2E{#AzSn*c+ijNLcGmGld0top1_DQbVsvIbM+6bE00yh6NR*pwv1}E z6q$z}mJa<`q=+E!y<`t@&st3#@6P4QEi<@u^u8FKv+?ji-tkNN{J*wPmWf-}~@jf9^hxMf^A6;-9bZJ=I@V9;_h?wW}+B7C`b*+#2R*q9TFU~M!)*ZF#jv=1hPa}2GvqKH&G5-WmRAV?K9I?}Gax)Ek%;Oj49`6nwc5L&#rvMV)&Xrt&_v6^?X ze|DB(c$nZK6g_6>#qjlhtUXc%_6z%q*NYE=1}kF~E8lTjL&-&I{aZbA-%Qb;<<$;B z++=u3qSv_+R$a1{WIAAu_O9%QwHK`**&<9lt2o+iov=yW8v%C22_&)__*)bOmj2o+ zO!hL?xaiOn6Lmog%LCXPEbf+n?8<&eT7q0euFyq>>BU$BFt-^7+{?x2IIcQ_#$n^C z{CanLyic9`OZ#bFhgi&ec>>q>@66>|BDRCIH&ya~V3)7RzuM_jYw`wTYx?57SDz|BpJD6`)tKD2c4gQEB6e|0`q;a`C7U>XCa=G8>AaW4`6}eSf z5zMSZpok#hu(4G5>}Z|UIkovymW9{7*avTAjz3W6B)NeV=j<0otHDFD+Qs?E)?7fH zSaGxWcO8e&twTDtHY0XoVHzq@tgXhBVI%tw&5m*NREGhQhV)8+v8+_7U$BlV&)$@S z-k4uW%-2KcU?nm5k%M(^q2b~!ad?ZotmxLt;$!ShzjasYvs|`qZyXTV%X%sQhc#i! zD^oMg4fwUKhBe^|TzdOi zCj|GHuaSZOmxI^(m5uKc*CJ~lYn33W@rpq4J9aZCCva(ZLaO`bD*xo3)!e zD9P3$W}}wlt1qkt7v?A0%!AB0md=R%8QX6>m8?O2U}^5Hw@P%MK4h`entQ-@|J}5z zXL>99W=KG%VW-x!7V`>2(MUg!Rb5yrFZxP)dE`Xz(=<4!>dtqKvi=*yNARzB(#4Gz z1UD@S-~{D^r;W}BhZ>y^9Adp4pSda|*oHSBumqI=`~6^c&~4dH8YW{q&2L;-(DaBK zr#a0$f8Qj{q^C%&)&im|F;9rfyf2$2*eJKOqc4rR zJzC-RElYo0gS zteUloK&HdGAr=9NSl2IB&-*mImWirqs*ilmkdL?T!a^kOQRp4BO4o}H24%%1HT4uU zTK0ToIF3q1_$KYlp4F*CyBm)Bq97G3M(*ciD*vRVeVfT2_s_R(;{EH!(YIqvap!>5 zR244+0!6hRL{sqJb(PSZY7^QaCW`xmRxrlM@hqqQt0o^B2H zM>d7w82UP4h9Tf3$Po;#j!HkAv?pyyGdAX6dXih ziZ@KRDrgm%^Qk;zzY~XC3&`kBbk3h&7w7Aje!ybVA)etD zX;qaxdYG-0v>!)FXvbn)c{1d+ugFYW3`r(Nb$^x90-FtUE1?cnKfqy{Il&$B?=w(i z8z@V^Dog#3{Y1Z*z_hOgF>QKW$HhE5D?9JtTaw;_2)F2p2y(j1G`J|Y+)jCc%w*|# zvF?jRztgW6lTPU}Bz2_I$x%1=mQRC>s0BurtlPEc#%jEG`euvl7WJdy#%iJ`4Ayzp zC-~|CZZ>=L-9KVHcw4!G)Ro!%S-RkhR$-q-@@`)}e5!G$tg^cH`C{1OketvdkvV;_ zG-4x7AJP9w5OO^Q>q^|WZTe#6DT_PLT_pU8L&)21Ck{}5e0w3}_%=1k#t-M)%(XNS z@Ju5ruVqaPvF629pylZSJdaCb_JlZ5=nFLZD=!A`I6?xt-V{q6x*t_3RZxMlQ--}S zXt$}~>}hw%$X9rk75LgI#yjKpZA!A2sfk^u{N2i)R?i;(BicC07a}z4^b<4Y7{J?n zhxYSUMAiC{iO&yy=q3A>3osP7dL(FU?uB#^)t!N%Fcj*6F`s1tgvhh74PxVyWmppH zoHX5`WgyXGH?%^7)THa2J|>SzJvDH+y)d4~V7=}Y>t9zuUSNpI1D#Qpt#sy#EkF<3 zW}HKq(rakUu?>I=vSBXV-(hM;}dNyH@?$-icl8?KsQB4h7B$-)c zi!{p;kk0lshyKv_3Hjzb9-UEp1A&5HFNHb6V@ECe)YlvJNoy?UT2|4S@*@m{)mM2D zoAv!ne!F+-$_MMZI3$dBD<+!{4$b3PBHb&eW#KZtjS^y0V0OkI@H z&aB%CLI=ZH+_zi=KiuSGl%J5!U-H5Y@Hbl9C)!N%dV7!u-&qD#(Wu|-426fx8GNME zL4N1LF4^=>r_0xgA}-=#ZlJ~a!7j=wAj0yYb7k;VmH9nUmX8~S*Zmn zcV&v*`0*a8FMezu>D;sN!zT*utY^h=FZsTyfO0#Cq?!#3yKZ%#?g*JUYx*k>11xb; zK@7`RZ^;rphEkGi^T#R4)eC2cFU@?3M+a%p>(IZ=Rv=sb-x7%H*WGegNQ?QreP@E= zT|gk7q`aeA3;l}`RmSl@sOZ)%IyI*8zGy&Hp3+8@I;JTfP5Xta;}vr7&jgL3F_jrm zD`~4*KUl z@0fruk2h}G+g^lU5ZuVRNFHrZ!3oQlem?w)h#!irS5?mWRc~9)#(qUkgrNW?DTG4#LRT#Cv|M~JV#-TV>zQnF1FJLo5JqnWj zZejF#eNRc9a^E|P1lnxkhXs0&w8=AGuXgu*$FbDk^a)(&M&lYHcYKf5lQm}9b)Ye| z*>=XKAb_Q=x9IasUkNEzn!2#PLCgMWTx@xf_TH5ED&IA|M}s`fizr}a0TSW9@#kJP zN!4goYA+x3ZqCH2P)S7$z7Iy=UR;!%+ln|K?gc@eV3A9F#6x_+J8rR)tA2k zK#o!1;I=nw47LwQng#-dxPNCpyBgh|gq{9*if6bMm;(B_@O;g#PrY*kw5zU1ssf7v z!_G1=GMMlj0iIULZ;Sth1sJK+Ajr7%_hU5TI)tu1^x3+h)>nk4;?OALnNns}<}XEz zYo@2_9XadF{R%Dp#)yL3C<`I;n{QdcPFmW-MAZ}{D*&fJ#_4%EpCMB3vg$cU3s1Hu1_G~&-i*N_rD;PTJJ9WIZ6NP?ce>MMi>9TAe;V=!leF>Hug8>%Df|(w4Epi zuP!PQ5dIIysUweP;%e!e*7pc=9_a|zJPNKwv}wYWI&na$hFA9&Di!n@A1Hbj_K1y?S8@l^^)%=4aoe?b4(_gLOQE7G8|j^UBx*BGTP% zm@Vgv>9{(C_88n=7=S~o&uhCA2>C8f ze5^eyf9zY53;Pf?bi!bvg2pA>*7)#1S*%juH4Xi$h?u`}d(i~ieD^_+dI6;F(Gxet zQQhx(Mmw)(?M7RzE&rHJ-UK~dVQfA_bw3pCs|L*QK`i6BWQ?aTBkQ)`zk-)_kNvH!uX@RWwuFef3bL9Kfpq2}5lbc7($r}GEw7F)} z&hBEbXanZdjvPb3`y}i>yT5h|@LR)NtxD*D__}4w`kw`){dHds@AePYr;+dsKdBlW z12yHt;!*l5f<%o!hLfVD=ojBTRyR|oR}>_=StDXTTYa~!uz)pyo7(1Iz}d}6yFDzo z4P0>g09PqGd2ISc+zM8x;Zc4h(1ihgY|WjnVoHAC6u-s0Z4@%wP4xv{iC=#pAo(nU z??AI?i3pAgss>v<`NWwJt%e^xi&VTKZyf2?cGq^g+Ss%N|ugC`R| zN2-Ghe;OL1iIgT4k z6>z?z7xZ5HRPnlz4OG)JkO!RLPoR3DV_@kCr%LE~Jeo6^P(N^ z!0K7ngOy^5tc`pk>6tgyC024DiT`mw-|WVRhc>G(VIXt$hG!}X?hdbL*S zI3;*uc$5KWEMhMG3)AD-zxvIaI+$OmTOBWD6ZRsCWNG4L-a57QZU@yPT3Jhq^h5t_I~kK9A)Y3~vG^j|E2F2$!u`$0SuQxIq4ZsADY z8)yVWsn6MSq?q8<*sJQf$6vWXF@qy5h9Dv|(D<2S-_#rc5ws#xKv1VoZ(63)%6?&5 zM@<7nUS`4L0N=83jyz=IJ1z}nf7yN2>=+==7gbv|@{66_)^e5@H^-P*jygvJLN$yI zVAty$yk!;3Pn|e*R%~N5#WUxQHlXDAR`4M^`&~8_CKJQsgj=RhPw_o2>0#^eY-@W;As?|QC_chPghw8m_UD8SE zMG0HfD$ECcuv#ZIrAfINxx?;(+vys?qYE>*HXM;pl-v3S+2r((WYS1?o8rvp$|sK! zgi(jFNh`*08H4B#@dpp7@)+s!6~c!%20clCtieJX2r1M}t=`>I)M3j=| z3!{Ey>=?${v-DshRC~_iZ&$$RY(~4@ilmi^P)?KG_D7QCX{S51FL^2wpJZ&j7U@+1 zA)Z@ozJ_I{^CoS;^?fQ^pQoLWJ&Sk@BcY<%yl`w~a9l-hik8>HPAN`)+4G2e!5DMf zt_W{+4_ePH?%YGwVWb2CNqyT>`6fa)h~7LRN7FQTDeOfTN&8`;RYsu2dX(i#2Je4| z<= zLAu&OFa?hC`i%H<+G#?SKhCK2+LQou}Mt%Mhc;OtEf!Dggl&>;eFvPJZ# zeUl{tmmC_1Uig6Z`6iOzpb|4<()S%kDR;qCxGJI~^TqG z^|>5W-S}a1aB5A&?6qC|w!U_{g4gpA)?5Ht`gg&SS-EI~u~U$0j9xA@v{i$E9VXjU zIIr2PAP+Od(bhKsk*ICwRqoc!t~a^(FD>4a7IQaQc+IU2Lj^?j_mp_Ee|vP~U4B;S z`KtBF0ll5qq9k_U3GfJeTg1trCUPpMvmX_34!NPELhx?vzPB{`eCZz(yN~aUMG-ll z<*wRmIx{C3Aol9=5_+#bz=WW|%@nkUxMd!CLc!B`DmZjn z9&IOSnseo=UWkDO&e7~)2|bg&Xz9GE;2G>B`ED=8gGiEVikq;|5u-3IE{f)X3rnmI<(n`TvtQ#%ONQ~WGqKCxI2^VYlP$aF;zN$Isba+l*4gP6?SC`=$t}MoQ$ey@=hCD|&R-ntYK< zuAroLCx;lUOL(#u(N%VSxrbJlTI(LPS%f&F9B2YOxS*AgO%BW>TUq6PM!Nn;=n&g6 zcN43k!d6nGOC5}#-B(el<3$<)#O+=Ga!mAI4nVEG3H-y zb+HpoMH3nYLW?fAk*GboOc8xU%|PDkXK4HG#Ie%Pyn%~=b{)}q5iQdkZ-1$0Q$ccb zynHFzUMq!Er}iX7GF5sLH2G4{aqj0ix>&^DY;BH<&YM4@;akmM`Pc}l5{#|}ao-~> zlUDKzzOvCK^CYdkG+L^DTGP|NMtrq}3~~m@s!!PBpx^Bi_4n^TaWjx( zxL9*mT&C`f-uDuTpX~g6so-?|SeGJ+Xs79(W|eEew`V?79slQeddxQ1;RhD$A@8&0 zey8Ono6*D*PC>?UQO+ZsRX{V85m-dQr}hk`Vdo`r)q82XHj(|d!ZavorbCQg7VF_wvWKsuu_WutT|R%UGEE2cz~$ zpb7FRSL=77XDUpwN**GUUCnI)8ZrZ|p8ew)Vy}n1x#s8f3BhOUSXF|lwMz?PJ#e{iK3YQ z-M};s^qZCIer<|2g{9e@`ZJ1uV{2Q(YTs&dt~0nB8D+s z@_uph2?iJN@)z5C=dmX@TRR3^a5VXSBK-^y)cY+5B(}*(1ayQOFs&)}x-uByr+yJ+ zHOgFoAq<$8!?#t_T>M07b(BYgnRX5qbErz&@ zIEI2ou^&pE)jYG;l)m(j2zac`E7Ij;$sVk2_=BX?Ya)m9>1XJ;KLf|tv0BQK zSeZz$+^rD^(h`9Z0b#3xgHOy%!D5|q;tlnLG8c!@y8v0VLA(5#jDTcu9^Bwi&Eld< zSD11FbhC7mQfmKsi#OcFn^)I0VLlTSh{ed;U0)W73%%f$QuvV9ht>9j=uNxAp?e?p zEPa;7P-zb4B35lYuW>KRVKb~gXNT{l)N2}D>h~o@aT#AJBJ#2J?}_El|B6(zrH7#P zPi`-C4@vKq_~5A5wLt&q5Aak{3pTW9f8Y519gFU;+PLZuCIs)c-i#E0KtZa|=61MO zSZm96ieO)%hnle^mO9?s%*;opd$}Y#0(|t}hEY=int%M%qi=!@ZpKRy5BKU1Utm zYx#%f(KwC%lq@k)P2>nqj3=}+IEXD;yFe~3T&Re z82%1pE=%0nbB3}*by&k8=Zxg5VIQ`Zl#%O3uiW#lb7b6?UNqHE8!>ojAjB;CYq_U< zl<2h+baLbA5i9DNZQa`I`Z%8wW>haRnCeNx_$BK1eXQk!lA7B>(#6Zzh_6t!=B9v) zN~RV{p~w?U>AmAZW-CJQ043>D29&zvL@jVk}{@3zXh) zh9*$nSwcD?vp{#W!_+0`md#zT`--S(&AqfJQ&XQmkt_^({zvTIn0 zj(LDj`aJYy79u=5*xycgVjK!M6vH)5hfOV|e+p1;QR|X&xs${33 zO#_0x+x0eMs?x;FbS66fuPhv2B@#GAVh}dfbfDn+f+?*Da7ESHtzd*iX_Ty+8M}b8Sy6akLlJ6lOfq z49_mfxWNAyccw0Dk7K(xXY)Z4>;l83E4QpsU(nP}Mj$DK?fRsk9oz3pCbXkwSir$g ztu*gE5C6+$1(fgr)e(`Lvp-WpNVe0Oo0hx9x80hdMCd=_<;aVdVeu*gG-AqVjX!~? zT_rrXz$k%-?T`%<#$AF0E@8f?9aJk zF6>GNY98SCb3hK3hW6Bb{FXlX3-1b)#HrJxujjiKAieASh01>4c86p*7M0OQ-CBW8 z{R--K7+1c#{OzV%RgSTYR;a~IyC=3#D|);sF%13GeSl=V|6Nr+X+p*c(7dPN_`K+^ zq?o&NPCUx9MU%Pywx{}g`7;!eEV5C(_RH4)=m+qsTXrN_$|xIX6R#cC4tYg6+=kDX z9_S&SU%@#p)37%jo7wsDcLu^O&wT>2GB6sJ?eYnmKOdWGO>l!#EE_r+|b5cyvNg|Bo9jy|Kq>pke2= zY{6gsw2(11oIm=gS@$CI%q|X!pldB?kmcH|i;^Ix$$Ce?eanDK1?1B#J$Cs@6D@=H zN=4j&?^$Z`p69S9{XFm2mzU_xq>%M4kK_kt`zEyN zHxE-KKhc_t(HpA#8@lDcl4s`!|e5$ZqOOWU&xo8@W76rhW!-mcllwt>$Q`=P(qbo=7;2C~?L1ex4f2AwNS z%ewY?h^6;9p2kniiI0W%WCm{b%|yDdy>1|f^$h}qp)C;8t>8{9f)4v;2J!r%XtrhO z#d#D)iEpC#b24$yV=F`FcYR_k#nA?hFHSA5j*UOco(Wns4&JM?cQnJfrR!5jSMw?q zotx4(6oNnILP{mlAl>T6rQxu+Wy}n}YK)iD=i0zYept;NErvk&8ZcsS({s}D`FSJ4 zwk3dEx=ZKyC!5>7yRMTy__?6JcV(VP=C)!@QO9(X5l4MYq6YxQhRSqEx#)E5S)JeL zM5m?QHh++l#$-VCfZR8T^n|Uw-Zk(u9PL-Dl$m=)P zM#EH9Dv8-3-g}}9Lzb|fP$Siic08N%qnA8nzLTR}`~06|OE^7a(+|swL|*&Nul@PF zj}3SAboy|GE(Qj!ZEnX}y?w$g0?-GVjTO3zM|4#I>lUV>VZdS02ds8-2GHN$VvC6{GJZ$ zuhq-wp&=k5J7M3@H|y{LQvT)~Ox14pMBPJbj~vq`dmPCZXxPM2g4#_P)fk=Vpordr zi#`?}V`f6L3nLEp^P0g7wV$BgL-^*5{MzO`@Xzi1v8qeC5VClZm@YABPlG7w_%lo? zlj-D9est_b(d}%kH*m>6HtcpT&y_7_(ht=Z+kRkwNIcy^tq@YV9eq<(soe~1G{H%q z(shl`-MmZQ%Gfwq*@h0l)EHg`Su&lpSavaKx0VyI0dM06ktdk3&j5OOFB63JTM733 z1HjxJlm>{j+ENw3U!0n(tZ34n!Ijf%*6+Af8bPVuEY?r^WE>1qVMm()A_$DjRv#yY z#$AK&>5lDlE+w5W?|?3})=QTSHc~A|@73SU{CoE}qQl$gW7ZQkaa_}04Z8bLRyu!^ zKs9N&UB3$0nc{QQP7zE#qQQBTR&uj($;c5Hj&1Sq(mQjG^4U!z`tEZs1vH3hWV88( zw8zo>H}(?4sf=`#>N%wMAPe#o^)lPRSi99)-|$iq&S8cmh5vk+B~`6Yl9CnkQ`Eg@lr?6xdERL;O^IA zCdM?yAWZX7oY8VUBiHA`x`vOdnjn5CP{AXz55Ae@MdND+G%bph`n9`04f1ccoLxv- zCi^EV)tjkECnHrO?5SV+A0+5RJJI3%W>LNI8KjH?9=HU=;`0k+2oz{H%;c0BR7p zi=t$9!^?f2Pbw`w%3>0)$s6lsZqSZlM@sXDI=1M=dIssaaYo+v+I#u2zdX<=h_`dm z;GY5#`{b{je7Cd`%+I|FtA458=;PX*Y;|Z5`=O>FpH=}_Nndf)cRgbvIEccTuDfe7 za`o~@+Y+!&jVh1zT;(X?>G3fm~rgb za8vm0MFH<+dsoS-LBl17!ymu;s-F zo^S5#)Uo$3zh%HndfT2-ZrA%t4Ul=MiQ~FjJ&@ol7Ju=4OkE$fOo6jobcEE zrI7~wS*u7XfqrI9jUax>r8U@n+454pzuV4ct0xopv;BngdYsd_1u)Ew8JGZZX z{zaay(M?)8bZaG9ma)JVntti#_Lur6f|Paded*N%0UeIp+hk0X_~qYCsn%?;%ppFP z!sjUhWKPTx6lKaMbo~9%J+t4T_;uV{e4T?d`xYKrRB4W{@+--Ux`rrnSOQ5c*;kMF zf46)#k>=Jb=HuC=x5P+uXX+sjNR_G4@*8t^^?YfP2c?tcsnw(1&LzB;XDSc>oOWzK zbPHrAd;L=246y*%RC~UfGuMN|a{`|Qs{cIOt1%$$lSJYrC+eh|3Vz6WuSmzkd;&^4 zJfg)!2)C4fBVSdHem#sy4x#*C?7ekRn_b&IS}N3#(xSzR7k9U|Sn=W-+}(p~f#TL+ z!5xBoaVhQ^+#N!&;BxYO^84QB>HE!`IWy;vGjs00GRb6;d+%#s`?{~S_F78l+|-U@ z76bLBXB|g4e5Sd;53ntMAz4nnzbq?@<*%V>{+j~X=<7O@(a`waGv`~15%;e@m;a_5 z5DlnR)Da8TJ&tTH))br&TV2w4fkfS#{OOTi8>)0o_L1@M%pi65@V57Rw?_0JC+Jkf z;u*Ll-OD%H4udt6>s#G(Li%3nK4(?Jmb9X&9>Iib)>l*0j8Qe;?7@`#S+%lPaswDF z;9igL6)t!C^RG4ZFpOReIWd@K)^bZ)?lYg_iTkQ0I-ShKTdN==SBU!hbdi(0-&P54 z{@Ee^+>mLL#qN0~K$%I*8c0v3*u!lzxurr_?FM5a<|UvnTlw(J6^uN9l2;bW-K>G` z1vOvl$PR79STqsb82Q_{yM74*0oJ!M)kZwLr?0Rj2=Wqc{Y8`;OrF#nKqZhmuzk?f z2xp|U9ht1oxNd%ka!z1>=`Z?x`o{kJ`XVl-{6|W$IFNVy(A^JIXVFl6KMR5ThqJ>c zRL01SpahUEShlUJne*fOM!<&d14=~NdGUK}S9d&4GO z1M{j1=_>L+7m0z1>-T}e%S83AT-PogWLX~2SaS>|J!?CzR|5E(lm^HMdr%&Q(kh04rMB^IANPj=2jfckZmOf5YN zgEkH%UTwGh1I~*x&=5g(Siam|?>XaK<$VUzt*O-V)>#w538Mg8yApxd>>c_ZlKVCtl)@xw#+q7&1=$#n!0Q&#HJSyYB2 zYOc56!t$1EY>r2T+|2#D5t`9`@bKyBm+9tXo$w+NETveoR@@<#&KtE&9ecHDk^6&x z&2+xoM#4z5C1w!avcu!XELs2O*R*Jd@X@!8eYWU-HJ zNp05U)(ILfU5UTgIaVY6797;I;XK5m?vS=vLY3~55w(<57NCLNf{#K6yt&)E7$mk( zDNAe`vh1040E~x#UY`y0#Bo?p%jGI^d}Di}lBe>#20r_ly9QDwm3c3fi-aQI4j)XP z*tU#r8=6`|ud2zbF*xs696#SUL)dV|a4OxJ^nikridV(T&&D(Cp3ZF5#jotm;Qte?4@iFk6^7JXvJ9^w>9 z16Woc{KmPDoz}G47|Aqx+d6T@>+jf#dt0r636<6sHmSh17#q#>!3ETmnsaZvd8Xgu zF%?7S&|mJ2Qdck)5PVal@|WGUW3t9B4i>#C`TNS-h$)B2F!K`8-jfnSF149c_Og#? zy7w0qkVGMQ_*H+9OgcH;a%sMlWt_=5*MdfHVUAIR}zjV;SS8G`f|t28Jn!ryu7?&>_G-AOW5DgXXfRS8e@$HD=^6uY2AlQXu~Lz)$$W~tRs z7cJeBP%>q2N(`!%{13ZKjifLa0Bd?RSV(>W&&?pcnjFV%iAvf5P8>AI;iC{OHB!lo z!QJoPTY5YFH~k*@ldm&VIwe6JangVWCmT4K*p`FuWTU-LgzLJ)=v0G~x~1vhSm>08 z)2&mr_7V!6Hn|KA;iOt;ZH+Q{Tm0v+s~B0TLxSA8U~UyG;G4csVaTGR5458dNV}E= z2&A7@tRf*+hEzwP5u8B0qI4eai?4zA>ho+;+^Cb|PbP#XL?WKV(dTO>L|P7{JkzYO zD2q%FiT8X}r0(iG1)GsV&^_(vKqE+cDTdshh!^r?2c|9dr?dGSul)^VeQ;>iBx`JJ z@~JsPdu{Mf=;jrk_=Jz|)^^sMO-_Xk=Kk2M;+#KfB_A}$F3!RQKp@eyea@d7y}@NL zbIO#|=e6{7sYrCB{7b91<8f$4*{ZIBY~z&bXseDQuYfJGM^KdC?pKp6{+B3FoPyx2W~j zeiPGG@ky>cP)+q}LvXI}$-avbzKM&?au$F!Uh0N0){0JrU0Y}__vf=fb8at^R%U@p z`TN%Q`M4I;*iBVIt_)UvTUwDE(*ibB#uT`2_o5OTBEwQ0ian1VGfDIaB(N|fl7 zs+C=K=%%WB>qRYwQwK-Db|f2M>zEwnS_M`?WXtYRJAwD7h?yCH)Yq!x#}<4R;UrcY z7!8_osn@3z&C@=ys>G7UM7p#Yp@O7i>=YjP%jVkz);IV3vPnEjVQQ)0irBfy;hty?g%uV__@-26v>`UowJgM%~j6uY?Wx`tT|I|A@aQL9q%nis5&8FHfK^RM8q0PT7*IEZ$_CXR7) zU_inz6G!>xT*&=C!Iq=IF=pe@Mqd(q2NL+}_J^AU=agl#@(}xHej2$^Tl4?ir_~`e zkw~4VVa4~)gigu8{TCjexT$EUN6c1FM|5!#WsCp|g&Oir1*Ek!y|I50Pe}G5-6-aD zhXohDX${QE$o`6UsOCv&8thJT56QD}0E`}*!*^!6nRh$j*d^J?0heAU2X#n@x zUBl&X7h|Qqc-gn49ub7Tg&kD>#?*MrLitKGY*=T&wDG=o32K=HZyi)Jz%GuRh-DgU zI%iv3&X-P@Df>dG1QxC_>PHd<0+~;WNde3lV8nE-pF;WNX=Vl|ibtGZ#%CU>Y z-|cPgKI`7;iq5Uu**CY4wu<5{`cYM>Th`dZ(!1O5EzZ8mgPbY3Y`Uz*p-)<-(qxR1 zQzcZ*rlvXe0(AHaQu)c*xz(l#7?V!wvSP2bnnLM7y*!ZA?aiI0+@Be#?`SL}HD7XW ze`dTk&W|-HuRXE@A%<^_KQOcZ-@?@q2G|T4zVtS{b3-^#w{#9^=cx(i$(SfIBpC?m z11U2}RXMzq`~F99rZF2OpkBdOW02PkgqNU*Y$5kPq*^LvSI@^`5cXC%#A%9Uc0zEq z+c4fLqX8$bzlPU&1^R3be8|4Tn3~EiPnZ>_sPIg=&Uk9FX9(GB1%5oxVTfm9y!>1R z+`4CvHUu7wXXaQ|;;2+a-=@#>Ay*Zxerx>5V!E8(aG~M+1zq=I@v4e)`{yekPcq4% zCa)%t@*73fl(mJ;&k_Exvq7~xXC)B%3A3U&`gLnASya3|lB23*WfdBoG9(D08Z0@ZU0%%D1dLLk&vm~VHOSJ!N? zD&4Xx#%aPm4X;ku1IRyB8V`t-H6|g*x`*ganpRo%(q zbSacnX;nMdDGe>ItLC`ZzXC&W>{ZCeRewbm;n+XVD|WR`ZPeZ_>#WzrW7;uLT>cAM zf4@tTPP(zt1hAJ|(19(cdJmMIhO86n#Ti}pL77k2{}V~~N=LSA8)ZEDx8rF(E)lD8 z45+HOu43zd?bHJ81r6i{IM99`c9o9*iC@lcRS7`g1-MKA1rJ|G!Q#X-_RxqS`E$53 zXtE8%mO~KcT?nVfgP8RHmfs1)N!GS8rmUOq<+l~T6QWN@w|wrx)@-hQe^IXlTu{>O zYVyLFD{Sq)Rn_dYB`6Zc+&0*h-_{vQZ_x0vn2xCBvYABURGMnA`QQ#CKuT)e&*Fm9hG+%O?#u7k7s0e(##FLfwb&<~a27~CN;TzmKAKp)E+3*4oG<+$BNF>xFuAK>jhA-prlQqv zA#1G$?J%E|(%Dv+qERV=IX~%yS^0xlB9)W^9@{&Kh>5OK*2)beETT}J@wn7&t){ST zrK6ID^N=t5M_gK117Cmjc)_`lVK^8FP$a$fSpS`Av9Xgj4`k&G|b;6YyQgs8~mveS_?jP0H#+L z&cMnn4ie*Eu~BItcVPrNI#IQ{Y1sft#~_;KyRFd9%m<(F5vE$UU(#Yi}t%xlj--+yF;8-{aZT?zGkgQ)&U0m+)j! z9Jlz|)ZtfF&|hpB_5`WWun_6{x>Y-EjGDZaqw|I=m@8&3we@DhZq2f;66BlKiMGZ+ zfcO-W{|4f5Bu+ghRr+V$p1jov6(FQdpafGApuGC(UEQ4PI6%ES)dmlx;IhVJnKqN!_aMkRCVD2aXKwaeGj#3>!9d< z+T#jZ)ZU}m&Q(97Tn(v3nx+(@mCCo|)#sfR*LEF5*8EA=0hJL3K(mFC1QBg+|CAK& z4>8iFQK_*5iUucR3^^-rm?a5r@!N5Wv&a&pQ5)q_CU<|zE95VxBtUL*$R2{Cc86@? zO7*&HLQdwe1ox{Y$vJOUFf@j-`1u9*X4T(sTvZ%Xc!UGm{)8`C`*rj#L+mTApccH@Q=I(O ziyg&%uJ*}qP#ctR``T5v6xa3MlMZKDJ5<2=+#zwCAuFq^QrMuDuX(ac?9&=h>%S0p zBbQ_pCZt^dZK@yHyFA|1#;rl0yoUK zO*glMc9qtJB2wmbYhEWKt;lWCi}Zn!6LVEiVaY$6W-F3uVHkX=4mxtMzC$=r_S=-y zRk3j}pcc)+tM<#Y&Fv6Mi;ic++8iS3OTheV!4q&KH5K=ezlEzK%qrNSOjsH(k_g(B zvsx2e1#2hi1uALE;nYBnC>mcYCYr5JOo!Ku%eu`^+w! z#0Tld{$JMl=QM&tGcUe@f^dE+EcJa<1WeH(bNpmdNux70$STgU2i00||Gv$xU~WiO zg+-dUta3iwoE6$u~VNnr-y224Xg&-Hehz zKeBYBj}+(P&y>P7*?jPh;a5B6yrndOj7a%&q!(8k+BCgMfsZ!gt|3~BdVzh}j<#Vu{Bw0QDW2x-xB zHuJ8p=*DZr@OjXQSqls_kol9AW6z!2Rg7fQIYsuT7p6=MWXsP3edOz8_gPWJ~ScOH`_5; zYl=P_7q9VKkx?T_N>*V`R?P(-ma;@M^w#>UGyD0(f2qS9=bV004NyzYOy+D04Zia3)Wtt2>H= z9?uD7q#=m=l~ELy$zL%g_PIv=2Ts0k1I=~xz@+W(UrD+BpdXAoI8*ZUqGqNDq1t*V zHbElXI@hqBgnJ_Wsy#)s;Mwc!ah}U8RnL2$Inwl*`c);7i>Up4yXEjkV^pXDtY0mk z8^Gswq<(VEGQh1GP=2;GB0rTzSTNsz-{@PiKdDdLxDv?$Mk#VrQsEsJrn=bJY1$;A zD38??@~m?gbibNv(a9WKZ0&s}2TMP%;*w~@Oa4zRzFawAD(IE7n+$MTa?U69KcV=n z#p-!k!1-ZV$|6m~ipllz8PX-JvJR=H!VWn*P8~77jz=5lG0`~xYv>R=e;>M7`C4l~#C_Cxz!%ymRV<~c3a86t?Mv}Eb#YC2o6iiMe&WvPv>@nB3y-kl(#tEXI<0_6QSd zo!(Tkq;e2qXxn9e@!Pig_RE}Cxb97^P>QFhoOG~U6z_5fFt ztL@xGP18oYOjtut9cUSMXjXV)IUkYg(yhEqcQ1;u!#vXcE z9DFA6P zaaHo2fW(89Kpwt?KFGo@P7_#=)p7o@mK>l$37gC?vXE{I{^td5v{OnMJpWb`d zY;kO}k-mG@`BqI9e@>Ef**3Q$&j`1u?74SZF8KZu@H*jOyf4fJr&L{f$bF%B&e(H- z%ut$@uys7~czy!&PIJ5FVAC&I8Nn_%$@48wU&(GhpTowQ+4|nFMYVV?`<W2 zyo}B*CHoYWvVy$E^EWto;u_#gXBek{Nqc*Y$&owscId&m86*8daZGb)@~~F()j6`; z@A=tMhcq`GGp{yt!bWbO&}&?E-Ndnu$&rb?L_E+n-Wq*tohl+pBim5 zZ3;TLM1rz^bTlqOj!FxOZE4;2<=yYnFspglqqrjH3Ya|v^JYeR_9`N&Hd!b;%jzx8 z>*YB;a_l2P;-JrJ-?uREv)~KK~^qfb&sJ1Tg?`)9z@Hy=JH}l_G3;W$6QMsCLMC5DSXxm|M;( zR~B+04js2G(;;*jI(B=qP#ky4xpl}&$KjQc{Ik3d=_OMzF11)x-Egj_mQ&mOkP_rw zDG#0A&0Ext8Mrahxk%T=wE0IrJk1UfH_qN3yXinDw3kADZUwzwHnBK~{_I}q@?7xf z@~4x3sm++y8G<-!KU?>le$LtX)p+dwEz=MwlCc06mY`uH5OpR7=a1-IiVBa+X1$A% z%e{JPI-qzkhPd{cdAB!z1tb4Sh((U9Q>fv8~GH`#7Tn#?mn$#R123YzLt$6X;QNt3YEM|j0-zE9P&Eb z!n1T{qpopbph}r5`Xxd-)#;UI>0_1Gm?WiySM9y=q#=J%`r30meXbr>FboKfqtD*< z+KwkmEcMbcyz^#^uo(i zEWS$TDz4&%k}7Uj*L_W1LEBH{5@U8#HagOqo&Fo}P35!Qb-r@_lIb%3_!6&J7e~~a z4bZ>2T_k9a+}{2Nz5I_?pRNb@|L)z0eES3A{|j`BfBE76k5MfC4XY&n&K>wcn62}A zC%cBZs5XHT`tokFCcB#{vUO?k3u+?wiq!v3yeQpY{F@)+qpwaXGgEn?%<>azWyh*A9oFoG?8;x;-945~;`#)G zp!W9G<9tBRdpr+R%=X_UUN6yHoQKpiHXA-N`kjwtn_I4m3VHz%mVdF2k@9Y~B`wRh z85a^9s$^V$XZ%1lwmM49Sq2}0cF)Rnv@QdJ7PWMA()Muf9S6TKPL_K4Rf|ltASMr| z4Q92<+k8^67Q{y$jw+D^jCXL(fx?2+FwWD(m)A#A@@MN0*5kW8^v8Pa-=@;NDjjac4wtD9u4Vym^(L2&1bus`7(_Op0A$KZamC3!jC={-9@d#lNMz$Up9;E2!7gW ze)ox%oQ?~-SqCw9Q&X4j-t51r39r$If583uHuM8NIp^dkzu%`Xugn|STZJoMq9~G4 zq6n95&u&Y)jHK{-Yz+85e^s}kTpZJudeh;g6-D(7vv%DZ8 zk}NBFHsyk#F}VFyl%z+)Ol6_NX3DY7XLEvQ!bLLR{BGZ!`I>8YV0kSh&cBowo?mhq zO>=k!qFAzmDCE+pEze1U8pTBaapiPi17>SJrezm`9it6bBv3c)Nb6 zdLR*#l63qoq+-61OJP07m1Q#ZNCx;`(xa}*ZB~j4E-hU3ZQX6)cdPg7Iq~W&h3sVI zmv$?>X@fC<`5!mz@v!IYQo^JlZ*TUHRIIKQ%hF7nZ_GiWs>Z{367nQ!dcdIYf}jww zlP28|p>xdAf`V1t-nQ1y{CpSY64~yVm-Z>y{9G08V^S_O?!y_v&p+$KUcjeggAfp` zm6Pb0#I_dM{T~st;sz!fn9EV`SFxNT&E#Nv-uMKPzk96phGG%FNy{#A$c@`SuMc#O zw=WY?*I}@4cQMas=-C^BbJCYq!Ba5{Z&!b}eQtia{=?OU%7=(~gf;BPKd%j*QzY$f z_P`xZ)sl3E6F1M#%2LR;B^4x8<4bT8>b+5eH6Ge~W4k>kd1tOT)K`cQu$HIu*z)RX zjB4p@8hS=k93K5O4Zs~pmPtJ`TkK;%(K#vuawetd;O}^mlR(nB6bC{#Fm~d&yjXDb?pY$T^vyE zH9FrilAAVzYp%!Zz+u9;=4AhzlpkNO*hdr0m)U}Q%d=W7riRm$Q|k+&vRr0iOrCMF z_-nH3&p{|78~iZUQj>?J^sj(nk}XagnR$Nfo;**$NODa5T7Meqn4z|<1Dhw;;Z&7I zee;TSF<|WW*8Y8|r<*WEId5~fbI$dw5ju*0N760qp}s=GPUpv~MSb z?h4+Q5?t605wGtLllH_QKLydXVyo@CCZj6*Zte5Cb=3FQK*OCM?{9C%^sU-nz&fM# z$r9Bmhps@0*x+ueo#PY|?ne?LOm3EF0m(cSkv6MQq&|js@f1c^(7dY*eY8Cl=KP{! ztWW!mq$VA8(4)Ssj_KTh1~tKv*7QBGCH(<8qx}?J^?KLW!4oOU)+BQ9s`QQtoQK6OHG zUqAN3rZalJl%|8`f*4av(PwSHpfi^^CFMyZZi!j4+SW{GsS23ZX4aLf#8f&+w)4{y zyU`4h*S9l@E6#v_WOh-}4Q)CkRoHcntBRsC&pa9h|FX_n_Q|&P+Cn}D+Cl#id|6hs zW+Alb`1XE7P~c+veeB8hqvdO6cx|hHPmA|=6m(+z&#T%O{gmE>=fbtAC)lv{*FO5Y zF=6;B6Tj_hOjdTpJ^7p^VpKFEqFc~N-lixWPY#=Op8YzYrwrcFxZ`AAyhljFlOkx2 zr+2E*WZ$doAa11>Mi&X4Y6K|Ho{R>r9^B@3;WD2D z&X-rzm%6q`U4XLFbW``Y2L7wi$gnc&9=xcM%}J%lZ3Tk<{Aq-VcuqC3sw}XgMmL}1 zG0*)=MC*4hVsF>l9s7{unsQQo_TX4^iQZ4q3Z6;9!{>3Cqa=lfoLCZ?S&9cdP ze>Tm9KI#~c139w-#NKCL^Ffso}0=83L``C7@Ne!>aIp*&)eiD4<{|-3D)i1ap`&-q+84`l0i}ew@`m-N-lAbE&rz<(Oaea{j=OoXs2p_fXj!3Qi+&*jJ;^s`02cPLBRU+!)tACq$)L+sVNqYpCHjp zbt)R%T>o-1M;tMsma9=lp9ms0U)Eldn*(sKBsbVH3E#;6- z%;j-L4y=D(F>DIr&eseAG|$dbcmP)qz|nGvo_A4g!%L!P9TeCPc1~c!eeSp$O6w|j z$K+`yz205=kn5{xZr7ch*_q7mIO+>i8U07APOs0AN3V6;o0?$#QQ0aqb6Y~SOUP`Q zVobyjq3KFrM=rfXMux;+IGCuVhiXSF!tN`kzRq@Jr#@xM3Tg+%Fso%vr%$td@iLl+ zYiy5?Fj?d**Uc6~%NHJHruOFv*C}Zq%*ttxAo(Wz1=FR^RHPkeij53RN*E@N%4;qh zZ6HgLI_A!+c}NR^^{vffe2$TO{lUWH;JQ?rc_DjZRw5p3 z7o0zayoMTR_*h|3#KX#Ji|5 zE=bfi==*W*a@1~=PsYnUj%r|J5pC5qse`xKd=^Wp4-NpyX(9}16`@jiLxY3l zH^tCMNj)==r+$aBVx082Ij(sRTlsP4qwwfA{!&;jHG&dfzkYd&|5Mpus4%2rf?9m|IDKib3^|Ugqd&oZJ6g^ z(s=L&VMSk*Mvzc865?~fP&vZIsX=n8r6a(vad}6*utZyNK4j|cWYQJ6OXSdS!l=j} zHaHkt5?CDm?pU{TlMgG6Dda$=l8$MFVRM+8igi=;X!!;C$C=(62J_wL0=ygK-M|L_ z;B+mSrW?`5wiQo(XpWI0Chf*xMRscc++AR9$%0ifYgBXH;#Ue9o;fr_J@7*%=QMbay=dH08}rt&zabWv6KuSx zJB{OeFzD$lg-)tnT7^~=Kk!$udI;SKr-j=F33DAT`c$5G3^lCv4Xe>j93fQ48N314 zmp5B?)~2T0fdSIprP0xK%GhDQpS)D((tBj`k<;SlQP)p%=gABcm?so*DMF<1RrB9W zNz!)!F{9SRAY3%J?JNR38z(GboF9`I9TE!^|d9&!NuU@M!Y<@L2_IMs2}G=&9;46+E2ZMFxjE_p3(Zx!sIEDn^@~ zVxxk!jLWx;%Hu+j#U`X;=lP)^k>bCLEME=JxV6Rt{*3474u(}OxM18e>h&Bhn{%64 zX&bqIg;o+yV$;^`iF$4?mDQaG4v6`A zqxlI;7?W#Cx@V_c81t_{Y$kC{@DJv{LPSix-g7av`#DZ#>l>ObC0Cr2u9SC_Dn6B8 zFsU`&(m5vLIGlXdYEWxmZ7%J&yr8nB4{s>RN$+G0{o^SUd+VUHv{W@bWk#q>v^cdQ zIk1&PDAyn(Sa1Luz2P-cyqpwC^%4O}4u&m{=r`6C6=!e0L9KE0RfrHuaH7b+EEYRC z(A7wDz;MwWSFq9P3=)MK;WN!)z{PqsasdOD-nqqYT-_Dyb+0P1ME7lmJ|6ZGOwEne zy4lcUN^o0k7X0i>*?tQf}) zhR#xv)5UKzB+ye!ftu=gsiOIs363m&=sJ;ny3pJi*WQn|D8( zDxrp32SQAIP!Wg`f^1MPeBxo{l6M9el3S)XmLh6vH`wNbqyks-HQHIY4VC?5w0uY} zUW!+XOX$u<^?+dEb!Y4y={ucVC>Vyxoby*DIxWn#uD2TYI~9z2QeOoV>f5FJrn;-$Ip0A8Ir5suL5N@ zL8cyWHeg+HfOZMyL7G{njJspAJozw+c~ra6%dMXJOo=yCOlEt!!R6&|t1k}p+QWyI zz9j^7X9rK*t$pQt?z&zdjQbX$6(|nylDXXmi3w;rGYn3+`b!}I9Xdp!JNt{244>GAvZhT;E zS=^(|VdvIbPU#qg*&v^J&3bBE?5U*2i$rcbShnjSK@3j(h5EExng_$P4ftEt<5N4&_>;*cr`4k{(g&XSF++jjhuMZdHJz zwZX4D6~NaKO{A_E6|UE>*rdlp6aq_uizB_$&tw$7DZSrnvHm`aNHcZ4Et(jZ)s#p$ z>01}BXv~nTh^jbSeSVoS^jFJ*C2uvOdu86udGqP?9beQ=qV;dmq2K2vUxrK=cUfFN zEY7KQ8XHw+M3W^zZ&fX}uj^BvOt+_BjjU7b99=&uO4bW<(Lcdt-@VvpX3(w-6fu`8 z;=;_oxLkFz+So>?oHr^??UpXH{(_m1@U}up{rs#oQSiuEoWns6AEml}6M4|7`n#KO znF*Yg)FSA@q&Lyd;&LM%L(ZhDYUH%j24c{|^vBrxn2QAi!t}Mw4^l61;KEnJTgB3= zXEQ6i<7&+#PuAI7&$2xT-fV^G^SrQ`1;;RYuvaEpGeRgm9u+q;q*TOfH2~W|Wy*y$ z%%KIwRG|(2tMcoahWMOqDl+`S?=gu*kUD}DG$0LmT&!dnz(DXj8|u?V+65vF%mfti zPsDuTtw9*;uV=e!E5Z_k9Fms%iv=RJl1|Ivalx(y=RU5-V>Zgp`1qw3)%aeVjN*8c zOVnPSKmm^_!eYo%mC(&webVF}X|)#7ojT0Xs?sN&c5f!#AFXR*@-$a~I=Z3o&10a; z3e!X8AIt5>1}{Q#H?X{7LBoiWbW?A^UGrEoz_%gEhEVMC(9pjYVcvz0?Jh#>HuD~ ztfLqyb|3e){n5zoy&Bpfn5;F(mdO+5oO8(lxW3z)#%NzQjseAfS-TfJm@7{z(Ta+` zN_eT2bjfWd*2u(YzEFVLjv^;u-bb|998r(6KT`{L-R&$$UOtTuSji*)t0j+$D0;h7 z*((s=b}zfH(dhYFV%)hc$Ex0Dlx}^`i=X=z9zrL(ZaD{dAYF<)<-O_Ovv+9@6Cw+DWGlG(}=0(xlHgoJWu}AL=Wbz!wU>tck5?(f=h`b15lODr^O=64z^&LkYT>YG5i|w z(=wcq3E|17`~g##x*gWGhfD@UG>O(Ws{zaK43(waiM5^lI{V#{));%2&bR*@i5RIt zhU*CnQGrX0bSC!@LTaavJUg%S5-+(AySwC@`+ElR*(w#r7IC{-i0W?x3g^mct%Ji2 zI@TY0d{hTF1AwI&vSwsAxL7gcthYO!cRU?Lo6UyLWohKI)V>Z2cJK)#FbmHsWo6Lw zn7!fkHKDl^Z=-ZN`p|LL@166<8dG$`PLtws!e$ zpDbG2ky0;>zE41zA#strNnDN#E$;O4*9g|3&~mg8LMlHdv;aoRrE32u#Z&N70-K3l z2C&l_G6{IQw5^N3t9T;X*gaXiUKi%auR5yacI{YR&Kp9<>m*-ZSP`%G3`npVE1ZX1 z$k5!AaTY)`>F-Kgf?lp`zOj1m>*o`UM2|PsOB=m$C8QD~_;hC^TaT<-6Vzh>p7SZ_ z43VU?=7!~K+edSkj8Bt%QzdUTF9;%(M43KBx}&8pL;(|qDQdJ!3#px{b$X&3cv~yV zl@d-iZ@?dSr+SO$qgt?-(AtJIN74rfM{{joN`E%Y+rTz5Ps&;V} z9$f2E>?rl97}jgT4p;Wk37%cVruN3~mYt3=3$xzp)+ywRw$l>IyHINoT#0~_jELpe zXZ!$GDW?>|y5}5acR&sV098-X`b;()&H@HP-K9>6=>_NeS7z>AF^p5QiH(Si*0Tb( z3mma{%VtOz27Wswe%JQo71Qh`Fi ze;8}15U_|zK@oCUk6x@%nRdC}MVSGZG8gpxK&++OC9}}CH``iq{K@7T`q>VWc1Apz ztm!vPX`|Ub?zS)4E3my>@Iua?ur9+5oX26Mol4h!&m%yPLq%k;oQEjZZ3b_|;w>{K zRV6KJ|L<`3_G9{FhG%_Czblt5#w^Wb2YnC2o7uui^KG^lx&i24B;?oDOfViD3?e)) z&K<_3o7_GMii;sINHUzWw=dPlj(`$*<}VBPO1p7DMQJUsmW z%y!?ou~i?@r@_S~*f#6fB@Nyyx2^(Anx

Crs21;~<`Nah8+e^R56th>o%`w1a{BUs_e_kkhK^9icIM}pnqH|3f z_M=C4d)9*=D;ZibbnL8qGI_bJ;o0_Bjpk7Mid8L$*VpR4i95D&DMGmFC{0(GWbp_W z56aOlG@R9nzLl<K~^&W2yPRzyfa82CxL{4&?Qn0m$;w(Fye#z^Mc*|Nyr{lQ)3Q1NGy7jw$VoN!= zn>Cg5vf5{TxmYk^)n4LV_kz$5I%?mpige+%9>&~y1?An(%PlCp`_p_Kp#>2}J*0+L z?wE@WJnd8AEin;^wcb6v7_dJo!>?j2^7qgWMmsDG8>-G-vI|h*s@!6F<#>P78;tW! z!Q{ENVx5RLRRxSWo>y8BOJ2<~K_u$1yEp5L z(8S+i(ybdIjn5~leKY8NS6^f)Y(xlD9lbk@D0bcp%v2ZuYT7M)`UohHaMO@LA3Vw6 zEUYM0sV2W2EX-bk!hT@TjgBsqe{)Le%DB#dwNWg1!Q}1kB}6ss`vm0d^$3JN~tV1n0$p(7;BzTE#R4v#f8g z^kAV|+ffsCn_#a0!hezSFBc{&%ljBQ`I+O5ah_-GXxNPU7>#NJ3pQLSLGg`PjJO9#r#ewVV@#)y;7b@h4gKaJ7%H zyd+?a_vYL_n#e@s61D9R%p_!z@#nA2uW-cDS_p(yAHmRZDx}pa^*g;INhGN!&~h~6;=f~(ei=ERvU`szq#q-2Us*pBF z;UosNPQTY0`ffGuCNh|@Q25w+a#hjWlooHA#KfomC!sqUd2eyIZ(nBtt0iBqP8OB^ zw8^D4p27f%UR=wiQVSqSB^2X!$4Pdy}UuhA4D7iW$Pu^T$Kc3yt=NLjG-alx)*+idkq^b>J&Lt@^3R^y~soBsfK z9p50oWw4;sX|`4=8G7Z? zdR&$g18)KtC8WR$j-vuuoI#gp0 z_3EeB^Q^U&{uHK?jT|W~bvW7UwtIkv$?DRX%u&NAl(pZ-9uMF#w_p;13s zMM$-daccC=Nk!A)eyB`o_F}*Tz`MbLPebtb6GTZ}*mWqx$b!OT%2um4biu^daA)Bf zcPbFFrcRU^d)7iEtG=h1_``JFf;Vg9-1fTquk{^QA`OJqT5#q#?VU}vL{D_ukYI7SGv6_U`KNmF7S zTO*=8Z?b-AAdgtqr+devH>AL%LQukrxT7)eQRrZ6a?|cP<9{Cag>9hsbq#TIK;*96 z&tJaJF2UM%@^9e%fO+z_e5e?VX-O&b+Y9n;$F>XT-Bxx5>5+Gq+SWAY^Prj#lM&SL zjdB&o#OCH#rBhD^+L;xsWA0l;MlENT2k8|oO^HD}JyLAjtsu+_AKORN+~5G=6Su}X zIUz@iJIpP;0Q}%9($k8c2G;Ra05dAq&HPab(n@dwC)7rvgJ?AD?k>)nt zL%NZ*>Ep>~4_5v##$Z5U(nk;~b>WQ#k+;;GX1Vw|?CJf`!1>tC3sSLw%*Yz`gSCDY zF@wwRsYTL#0JfNSx=Y|cN|EZr6th;>X2>L5)~%oXnONbm8VWne4z0F8w^rPgw80Ko z53*j_>7tlYNjxihgo2f(Gw^dZ+4k=|LlL~6vk?8kFTM0nPReu7c%cnHM+Yd+2ZX14 zfxna7<7gh_l^q06tv~$qUU>6|4v`S?dnwGaen=cemNOGQb+CEZXIk>HeCK>7Sn)OG zf^}j>*GzHdEa~4FTFBQg^9(c;LZB#6bc{e6iHn?|M88z<0v(IM)Lo*3KO@g)2gU@= z*H%AgW7MR0iY!U)6=vgnHjF&0y;-@YM^^U~x|R20_wzHIQ8bY^iui=AS<3gMGK|NL+-^Zk(M-B$nNm0{*1-^F$dwK<1o zhQ?#Njtf@vSC}!(Hy#mvTHN7#U&f5g_UsZB*V{MZ?piO8neV;;!sAug2PuO0(OkR`(X>WS;(yE3wYZGJrw0x>O-hJ829Z$qmvhC<)Ca?FEc(v zc7t#Wqn9ohZBDOK+t?%z8k-ATEF^VoMJCA_f-t&Nlvxf7Y3XC)8+5}zzVw$Nq6Vzr zIK!E4$pRQY!vCbYX;F#hNVQs$+zPQU$n;fh!>8xZ8EEN^jj}=^64%EsDLbtOA9tir;#lTm z+AoimHuSJOyX!OY_YZMtz|=E z&C_hQ%4FERr^7d~O8ix!&i5n0kH724Z^xQ2?^p^ z^iVFecxm|dEV8Xkdmf{+I}ZY0MFU1hUnIhL^7$XartGf-Y_r6@I`&fu98Hu_M08k9 zoN^UF9$m94JjnDzwv7?wOFwFy&3N@Zno{kUmfcp0Sql-I(jc+ZG4(hPmp%xHx%>cFIfJ5To_+ zZ00l(@FrMUnkYK@RNo`wx7~EM3(LOpOoBT5z4q$;2jg(@4nIdimof)c9j4-`km-!E3eKN3a~BO z+g_f!5G^kb{Wx-X;zCV5#}sXxO~4^2t;D&0jQ-om|1=q_ln}3jtg%qBz0USQD5j`C z*5B#qy^A=G>fLUl#CsO%Xm_iAAns8AG%PQ7|P>iZ?uA23G`nkTu`jZ6Y%zqlHa$w_8_lVb7e0q{)U| zL1cK|Sf*9({jeH^RYql%;n3~F&P3miur0tU^wz&4(71&w{$V)zJamV`>_;YDlSFmL z&3u8g7_?;O0p%}_?|ze#$cgDYu66M6R7jgG7(*SCas=8omoT-5pJT>k_{ zUPbK*&6H6-#;Ta`u0%fpQsBD1#bAw_t#?cJBUZfRjQnNRll5V0Y$Dh$EKOV~dP1T5 zMYvj;cWRurOPKPJNBA=ejFwl0Ij4HQ&icH|Pfi4jrFlUb*=$B^r6VbP>mx;6E%LIp zv{bo_{N$LPH`r&Gu`aYE-$^)sUV{Ew(-)`T`!1(5_f=!=e!#wPpIQmO@W_9?CVatSHa6In0hK028X6Y4+x4>HOY4uDi{&zmX*9$TT6{3y2dq{0_)yWvI1<=}yrsJm#Fi8(Qa1;kRhczY7ZKa?|MEB8@;O=>$Zwe7`N7SIduuy7nRonGQh<7$7Pj6Clf{O&PrxWWqL%#HkJdnZjdX)YG0F+g`RuheiE^RIhpuF#&{y!`G%(_lw9dv>llkW_*9cvt zuFTfPCVJujZcEO?Gln9+@N%CJDHE_3a`$&mTP)JDkBxzh(DCff(D1QY9L#ke8p@~$ zn$6{M{l#-Qbb~@X<(5DO(?_yB-O>%x9Dw;m9%}(2-j%$rQ4(ZqrTz;N_G{wqzy^n5 z+ADc)H^&!!rpID_KVJB6<7_lZXh{t==kPsPIRAcV-Yk)7QibQeb=?nCt zwfbS(V1@uyDDGlx-XS7)RV>s-ct<3U789_qR~EZSjsTuqj(-a_Fq+<%zoJ~Nv`BF9|9{ohQ{h=a^7S z?HfV8-vLzQ)(A2~d(?26il9ycwS2fuD|Smw1GDz;oAxY`o`rn0QRNk0&N47335JK) zP3e$b5I*d7pcm;$$!w5CnjFko+0x3R@bu_PEE9{z&d0wyu{RuQcJQjqzEpE4XHfaB zlE3oq<5D$ng4~IvLdgQb`pD}ChXJ13Fli+4wvxZX8UZET);o;E{Z$;G^6+1>hUgkT zKdtd6TQ$zrBYIsi>d#EpUv=NWgZkZrwpjWvg7hTaFA9~`jJ9_<;c2f(ZcH>iTvy>5 zo#}mnQ=)#&VxOD%=)wD<2^FX6QvDr3h`kMW-;x)x$ia?KQO?VmXReOm$4keLY2oRo zg8#)-I21ahTGipiPwdBo6&Pr7nh{3Ag@MJ7S(L_S&hQNWGdtn=YM-H=0{a@~? zC;zRiyAe-osNrEnZ0#nyqHbXpjoI~A(l9C2=;{tjL0Yw%3E};OJY8;E*99uG+Na_C>oCa%xrn{Y1%)Kdm+%zt8 zpS!XesBU9tCj<6P|4@v3Q4^W_mybKG=~kv$^S6rxwVgA`myzVU8agbkjK`I)uf-rl z1My{yS3rHZ+cKv3T=nxA4>{~H?6*8@3}=N;KM!Dm?oW|x(Gk_4`Zf>EqH0~m#$6S1 z`$A5n12u1Wl51!U_4%D&Dgmh{`k6LO?Y9RKl3Y|wZT83le5Jk~wVa(6*mttr z`a|VVz79LRRi_V|hAXcB+P}?l{o^vt+U#>J#%qGt6ny6QpIQad@jp3BZ;AE)Q>zde z>FmUi4tT_E5YF{-@|Fsxb!h_oE;J8FXg~>N=|K+ys<_^;U4kvCKGl#js1HI-u!jR ziRK>*&Y-fGgp=uw8FP^{RzVdW4lyCS8x$_W8#`_pV5$Duv|BV6*{cB6Y#>Cna)hsL;R`~orJV+*kS;em-HqaI3tGrV@evdxF@sHSEpeF% zJQUbjk{WCmrY&i-re}jiZp$KLUUam!mX3U^S4(Xxm1JUeey(yb`SZJsX=Q0-L93Kr z*!xZ7(sb0f;1@wLzf~8B779ElNoI|Ybowi`C|2*9Oi%B3xnhZWU~0<|M<{HZAbsl; zWJ2#MGdzJH)u_NQZfhpdWqXvT5a}(ypPz^xxyc2C6cM$%D$g=cMme}ySEwx&7$Iqm z2<;`)FQ$7sjhtY}y+47sFcSG%~~gcSUPT&Z+y*`gVdrXNM(M0FlHqIjDH#?(hHi9Am_#<&_v z1hppUD{OT8>@A-tzSw@~Lh2LgPe0>bA;<9@K%g!Y@3_JcibbA=&&=W!b?ZtcZzq((JQ zT)Tt2%N91T*B$cHzEBjbc~*{cOhTZvZWgC0;(UQ;1unDpqs|RJ;+%fl#Lsi&HL}HH z^_3BjVM?LdInY6WQhKH4@FDY$3-4h7#vtha=-zs~W`7WHXX|ijrKINFPc6*0nLcv= zaepc7^9Y93KV%x04x*9SoMi%+1AK$?Y8AipHIArC>K919RZVC-3=^ zVrIXwE>DbT$RkXT3RH@gqrZdvGJ`01=3KM9+?4-agpuSKD>lCdecP1g)iZoGo~<;d zJ=JbD-+(*RQGiRTAMLfze#cv6G;~@_qx$Q&C8ey7MdiWAhGRMqOhQ%qO~~CAb(CgUTS?@phl6050BA~V2lS@~2_cg3f~2JHD21uM2aUiT2TLn^wa zAk?#DZlO#}t*ULLw9l$tuVr}e}!4}8*8dTAd;&tLCBKd)B2b`>ZJ@zw(j1BT15yyEmk z>WTRf-v^3HIvMTX6P^w&&dr}3=UnaRbs&G-)njw?IKo05*`w+zSh4@`b+^K_?j(ne zKijVS=eu5sA&#>o(fqC?f@XLhso+x6bB~@=&lmco)A{kc!*tNeuH?;${#ZnrC2@lbs&^9$*X zmNvWGi&Q=(kHh(W$5LjY=ZV727gK-{=Y>Xd*b1GKuWfD?2UHoC0bnCI3)RM&NcCAX4C$7ax;)GJO07amf zKtKm*F0@UQS{rPqihe6OEg4p3u7kM%i=|nvq3Ehg+>8!PR0N&)ohluy>x(KfArp7c zSERp8yQ2iSs0BLTad89hv1AG3PpI@GVKNKrYsi(ai3}{u>2A>7`6%i^I;>#lBrp#)*8>_c^8>w@bR zDULrkW#6wInRs4JlQyeYf}x@D^1QDfmnA&~>+oD!L1rSEn$4|2o^XE9$t?^?CJ4t0 zQOT9PVXt^_F-;*@cyDw)cBylG z|0T2AU3z_)F6360Ndi3Bt1E|TSv`te-ji^9n(?|5vng!`QW5PxFq!U7AD< z+J7+IgTC<8Kl8v@`ta3SG;;yTEI=wsf;DRHKkfugg`HuoJmk_G7(X7HSKc@S4j7Z6 zVDzm%n405NqG(8;Nu+y?s_WTCjP)@m0#l&RvimP0nT*rr;U%E}12j`dSCsUkm8>|s zFl(9ka^QhTM5^>FS3O2PGl{*WMbu)Lzd0dcT{bhDI1tZ$C?%eS-y#a9Ib^x{ICL?Wxq`JP7 zhoj--Gga?Lhgy$q23gK>aGeiy1{Z%z{;qv>=fT^N`&uocG3MLMU<*HLsMa8CKt9DZ zR4F4RSe?@q&(*p1rR_wzO^3VUPH^eZ!7HTS1SC_)&8-6?E>xDcITGbC7==bN6+CT= z^?C~HuVagDz@%MIYb#{)W7-|fuc(>Eu87XHMviBjnIX~vIBzv&K6)>U^py$3C*&Gu zvniZV@0^!5C4&a*Q~px;-MD1D9iRQC9JBGsMd76^f!|R|@3u+J>4Gc0V~~22ru~f8 z)mY~Q0yECc$W|R}8Yf+EbSUp*E`jc9$tBSzwE?(gKoYqfFfBWjhlL+Q-b}XK=#nRU zYS}LtGH~YE4T%BV#Pqvw?Hq1r%mDynR-ud$^aXg%8nDfa?QAag@Oz0yww;_umC5y8 z=If!aZ#46)* z3+UVL&ab&qs-3ND9BRP(jqf-QlJF;AsipK^QqYR5 zW)i^cI<&ZmJdD_RJ6fsFb;-%L z7zV{0=Dt*15vzber3qj5Z1$Sz8uquq{k*y)Hq>ypA0RDPxu()9Z$P=0Eir!cK&lhp64aEIoRgP0O(6gM$Tl&i63k zrRND_W+`FUj8Oll*dg?-WRz06_gtf{_oli{e5Q8q;0m;|*>%v#I(1;tPpizspXCb+ zy%8_;yDH4r?9$xUi~d-;uXoz&w00VtUGsIlL|@QcZu_GvLQvkKWYbw}IA)4shY+O2 zb&GB-pH>kmrqvwYcwVmAj>&9gm}6>1dHt5rM7VP&nYv<5q>djles zYE{vF-rk{ueI!cz(j0fsJYXd?+o6A-Bh$Mu~wJnw$6Z|J-R40QLc z+?_00kL$-B%%}SfQj)EyDkh)d?Uqo=n^2f%8B%6AeSCR*1&z~xL5El3nMhBO7il@A z89g|8Vy>}q}T}DQ(~ ze%C!`*yCmqRP&f{cK#y6@|%R)tTgZ;g>6>t8!uF_YqyI;N{%fuL=ASy?b5{m5O>;D z2h+l5i5x-gzS`(+e(~;lceoqfLT2waY1IP-rCCq4^}gz-<|!JOvqgRIO=WZAYgX8l z$1|^VQ`(!v;ZP1k&leoG+rV~lZn1wOI^n82oWNS!{SZ@^`RHQR!g_e0EG9hJL*P7>-pk zNnNQC7^rPg)4P|mox|D_JN5&SvKk-GPS;{rakYpS@w+@iN?ZHqRVR6?kL*WjJCVv( zmy2F_Z;7UiHaxE+a{8NVKRI4n^D-;dyaV`=a@>`++o7}KXn+r7zVBI`#u)rs#*}7x zVKRoHWvj;i;pVH!GFaiXdo!K0q((Hy6j8HE?x|CHq3?dRwEWU(wR=K0mv@R2F==>x ze;!*USvlh!^mUOgL*ZJs^Zr}O7&O72zOg1ryJ#OMPptt)MG&tG#S=M$_S(p=u#HMY z3n&5v$5#cN|NA)n^?&f5C19TQ5w4l3TAka1N6Bv!Yo_tO>k_v9`Fz3XW+IW$q|62IA*6#Z3u!#*P5 z0N4W$++16{_Br|`E|7P&!aumCI5OdK!*u>;_~&;`dn@nN*Gqa}^F;`w*vd&U~kJgnGIAm~{Df zo;jhy^J+iiK07$90>4QsNl&+6G>|2%sU!dIcoqIl$p-#HqVb1U5iwDvo*+c;WbxFX z8i{1ZGNB!yRTaGn(szbu2QraFy?Dy$ux@u&(*m|+Z>JPM()?p{@Xfr^sgRJ2g+ui^ zy4wEblRD>+G%bsV=HwK{%*)``)~TU5Pb`Si*P-c>vtIXj)`x@*sw=jLA<2qjzv$i4 zSgqkqnQP#QSY^u;1?02ZEhg#~jF0*aU?eK}V4%&eW2o3JyH!Vcq2kdxJ106Pw(8mG zvq5bNowEPAsI_hv^u6;H`Uo<2p6U7(iwXamOS4#II|Vyd;cR(i^yN{CSsMC$2ove8{mA<)H$t`Q=zKHx9e-2- z9AgnhyR|?hTYy}zD5dqpvkv788>gyU&{3Yex;-B;;?7A0?u@}jgm zVM$ve*|MQK`a~vOyoHLbWIQst1*E4)q-P$J zVPr6+g$`of1*~w%sGiDkhV`3Le9uw0x2G{ilb*F=1yQ8*v6q|7^UePXrT&`&m!8N4 zAD@T9yPLi$;>K*^i)gU*P7N>L!Twr<-;sFiNx=AyYBBNl$U6dvOgq#+X}RlT|2eAr zMyJcc82r|SzzT`msBk;fw7DnC{(khN=v%sC74p==sStT9yiCl>%V~My;0{bvlW*2- zpI*(qv8ZAH*L2ge?=NVVZKXP1T4CIcUvWmhoURj>z2;csN&w5S^H|^+zMS{VQ{0oS zGKWQFroclJ&o2I__NvuGYvq%`-ohUmmTmvnee}W0YK3c8lM;tU2U)Y{1byv z$TfB(=uhYsVl)kfnUT`q6-PYYv3QNwyi-_wd=ZnbAvbv$nRI`i-pT)>BapZwZGDUc z5e=PR(8b!&<9qje5|?D%sn^RmQVluL$%ZBY+!*_5hwrHBJ@G?3YdedU(bL~tZ_NJ; z|3l|&&-8x3-e(}K=xy!l^zUvtwC$u~Iyuc7eD-Qg%!eih%g5xlVsnkeGJW5FEkcRF zENV~0;{FJ8v9;Sot&Z@UbbQ zUQhWDgI6A4+Z;+I>|wkzXYyD56kaMG;)O!c$nP6lxCQXHp|0?=fL)hzk#**ZmMnYH34yW z&O7YI=DKZ64W^%<@90#++2q+2(x+22)ij5fbW3U&f7HV7)Bl|HidrpzRltdwGnapC znk}bWj|y}v`#qlo*P)e$DCRe*&-@nE69`6^RHUxjEEq|aG^Jzx={!j!?XZm&+lCqW zwHCBdfQS0MIdz$>PrYhCTk@Vn2PNO=WZFsL`m6bA@*K~1_}r+qtFO~)b9r45%hsUh z?aH3Dlskiz!$$YqTcG%$T$IVd9|La3sedO2T)}s4(~Wm{cgkEACG~u!$LB)kD~fiJ zgVM!jvw3A9Dkyin`&3Msv$gcX$@6ETv^ADB?Amznh>@#8yJ=VMmS+J#KU zD$8^a&yWi~tRMw3))ttHP2JI;!pP_^6J8p&I*9yJTSlOy{mUQ^89^OY$|uvr>QEEC zBT5#>A>O8NA=@{Dccdfn=$?5@Nr6?VQ3D#)fD-8KbD+AZTw-!ZDkQQMm#-x?(m)Kg zSZ?i1&*93Fd|PKiCD%gdbfl5nx|>Mb4hmS_Dy0;mwlYy|3bH?YlI+>7o9y9c8aKXv zM<=?#FO2?6eD3h*`s8Rbbaovg6e#%-j}e7nmA)4%Qade0V~>tJ-GxlW3VDU*vSz38 z7)WTkoQijXRw{hXvucj7zcOV`GZJPi?mas-dEp3h^ybH8Y8qKerC6&%x|$H-FBQxp z|9&K#Lg3>C)&djo?pt6totsW~nE+%HkQYQeUe0h|F(?IogRSxuEygHXkK6ry?4>7K z*xLV?7oq^BlyxtpP?4jLkCPD-AQtY9=JQ_pA^`!$6@r{DzY(kPGfkVijp<*90z}F* zDeqFc)A({L%=C!y_1$=lj@z#Yy<~(NPi4`Q-6kytrq*>=npi?XIX-cY9f7pddJN0a zQ2ry5%N$L8!bD!kkJbu*FGpB8Bjb7tTH>RV&w6pLL=!lk!QX!3(!X$m^fH6L%!7&5OJ4>uTJKINM;}I^sHt< zv5Cecx+<0v*QXJcd7I%$V~R|h2+LvV1H-YQlT!C3pT>U!DMQ1g13xa}UmCB!`nv%V=Sm_Cm@D;xT(M zkPHnTZ)V5j!L36~k;I^IX<-QtuLl;Z2Xh9?=|kGJ@0{^kNH6zzvcw|~D=iNr7r$=R zP%R|mo!}mg)caRQ^0W=H+gXlBzAn5yw(GRdz#W-fcHVysu&?p%^_FA}&k8(Qmn0#& z{!Frf!Om2@*__^-eO4oLZF!~`z3=_q)`Mu#gQt6hHUcBPG<%LkI4{{&bDrujfzNFn zTq4_8$4N)$Elja_z@v7$2Wu&}w&jl3!${U~Boh`yS(PWoMp3{~he-QUDlq|hAVE!^ z9>lT6`bRp8pnfkLo41lrEY<6b$r>)dyp6HyfMihkUJ#FEd3|8f{NCNc{e#;@XX8=& zW()d=v0PFQ4sNI;DBvtTgI+a0o=;V}Ws-Q)CF0Gq@6U^CUrt0a>k)OIxDnj%29MS) zt5g;x&E|-YZ-E*)jbH)>p3g8s6l7D!f$IRZWxFTHU}ierdK5xkdY>NW z8J&t|dg?YapLpNZ=I~8rw6)=mkS@$TnH;@}l=Q}!s?^IXW5x7$S=7~0`p{*-c^jvH zDR6(#`xT7Lw_C6&^kY#E1SmCA1pYrG=cqJA372lOW?efq0mm_|DjR~&{LI8+XWm|zqpAhqKy z=ElyB4^G+&AXhe5YJ|;WiSKu|5P4+0m*dmG5`NG}6c-#ad5C zIy&hedLR@w(n}ku-?(MkOb|bh)GOm|{D4Ls&r-V3lgiiJANiE>{;GWmKVsXfDCLsy zO8bVng^c<0-Lo_QG#%)4Zlk#!0G8=W=C)Z5tG_5v8k@&sqD;=i#U@=U^R+g}9gw%1 zw98dGi|5A5m#JcGc7vwU^rWm^FD+W>LBfyC0*ihHhe%#Xa9d9N*5qMYUElGe770>D zxvKD3RJ{zM*FGOTtAkq!YPL=S-S^V{*HU9C(5ko#rfQrhJA+mEi5fZTZCRkX>+@=t@Uw%4 z?Zwgug8$G0{MW<3CjD376aJJ~jsNckY%0x>y@k8Cy2;{*MOCT;ms4~2-+zh*wWYt$ zKEFP|=iSfz-;|vg{BN`e|2GCCj354p1wf|X0*fgmWz<@U+7i6nxh^6n#}B7zyYgJ% zs`{jlS~ZOkWk2r%Yf?Hj|;<&E{s&`f;WOT^*%ve`#EWO>GjDY6JngPHH1cD1Wkks;EBn;Z|xoj3|xqBGN-d~;Lf93s|@fkw;Vu!))G03i>JsSib${o}-{6IRy20gMltj1q-BVJPKYd6_g zi}T_73%$suekCRW9%u6)@6~lDOqV{M*k1-~Q-6fSM6JZ;cd$;jA(s59GjD5M4F~6v z&Pxx{JhW!PRX5)tGh~42bxn5#iXP<{Rr?B0ACn)&$dqI+IbJBamYZ99R%2n@&(-9_ zW3=}9PLYHtjsA1vY(!_5S(ZisSc*}@0C6bM~ySg zUZ(@apSpsn9|iYHMYPXu@ZK~?&#nT73c8+tyN|K$OQ)_*fF3qkzjJdv0as=DhnIY+ z9p3|zzk^XkR0@@I#S#4&>+uPf1`|wCU55!j3)fV{Iq@*+v{9pVPYCd5-WF`WXWdtb zXI=X0R>|bc(qOlFG4$Gwr}g~r?6$cqIz?{>vYy5Q^5AYBu z{v{xRYt0mRj@B{%s>qm%H&@4OMw|u<*0MwzFyj5gJAFR%?JH;c^SVe-eTyjF3ZOog zBEB1bEw*>G74i!iaUFhBulf4!<{&%4&GJ$jvnTzcqymJykWcS*X!B(Oq(bhTyF)!< z=i$irRrrf~qUZ(KT}R!mnx9alGX40~%yd6`Jsl439X2`W-c{aM&Npe$hAWJqg-~w( zBT3MXOoM6KpAiM>>n;5ye**zr{)$t{X7C+e3-fGy6f~t=uA`7nhXNFDZ#0LyV)$cK z{y}lXVMKwC>tQ4-x_({B6bmCezu^Uch*pDd)2#B+r9`Dwx(HiQCyUjB-3j%|KmCW2 z=Q|e`+RJ(`x6?f-Z56(z_P=rNzsinU3{Db!%Df-QBffP0-cs_Fz5q}BeL7gMKiYid zk^FuC6oIXYq)oWSt1GHKK3_(#J7PI9ZsXwR*Bmr;+lXuBl%H(ZwX#9BY=T#LiOu0c zm=T^g7<;(Gd*iSXRWMkWRNCw6yf#0sEc~WsVW0?aY=gMZwG+Gflw?U4BzSJ zID@K28sKwGMNuy@(#iSOg~#&3WZQa*Gc8bjZ5fjIn|eF>@#@;yF^N5ggTl*=EKZi4 z(aCu9q4?V1-d$eXpB5fxls19wvKU8RB=d;9?#ruBt#%gHGV0%a|4%n`Z>p^hoq|J_ zuOEuV_LON`$*6a@u^Vb2=9|1!IB#&7%U5+8h$s(;m8hW<3Jv~Qd0*RIZA=E{n!eDc z+xAEoEUp>6a#>#uO!}loZx6=jwC=RLujS!2Kfh{YTJwEeD^9L1NW&P7NF@Gy^$GK)eIryB z;72mbU5U+OiLBH%5kyTH6oYbEahJ}rFBJd1^Q5+S@4#J}03@)0Op_mNJ2Wwr`!Xt6n08{d_OX)Hr*=-jS+9YG7O)wLcbx^^-X(;+}W z!M5bZvDZ}z_-B+qtLOu~aMD}5nFdecOk9`NUpMhRyAh@)72(p0hZ{a@OLCE0%}|Hf zNr+wdq3MN?GUy_3lXQp&wD~!E0EKfc$>7{VjcCSE`c#8I$A zu4=+ytF)1KQZ!Rq&nBUuXH=Azi#sxMf_G5=7!NY*7%nTV8L~IGDd{?q{RW3D?p!!A!;WoXzD^ ze)?cVGfZ>sLvgaRp$dHg`+E~+aq|+4IOlj5OHVjgBeEnwkD%KANmVwfHFRReL2%}W z;_HYb4fJJ+b8n}2{=wSvWIhRw1)`b}{Fp2cVXo<(;Oqug7pcL}!Q&kH<+g>JZ$9;b zs@+2X*J$-LAGntD)izAw#a7G8SYBlODO(FtL-;8Y*_nQOmEA<7aXB`Fr>$ z6PKZ6oUZie)rEN}x*u_2kEL#|hmQcJ-@1f+#F5@XpI8<-QgYQr42YQ&1kKf`W*Dis z5{K3JdT&2FlWTO3e_O#6x>cnsBE7zd)76f%$5&txRjp!)TX?EL^ zNpjxss}^#xQhZaCVFYy9bU`Q3vFhh0N2wn~BHScrycEptVS95h{NFOR9a}R$Y$D}M z4@VS6X?s=eYT#Sd(B#`m{TRSbhgOzZ$(2pNebmpdB#C)Vt#Ke&X1$!ftftnFm1kuA z1P@UwybyE>n+1%Pwh4SvL+F6eC5Q@z&U6Q_D86|Z*hX6VE0$Pc@d8)k>W>X;n*>6~ zmfX;iE0_6=Bw(T8^oIGTjGrP$z&wFx7YaqhT0ymn14jKz!hcr{fznT}F%ECI-O6X? zk(yomS3S#kT738B>Ve95vU_0g|`J|$Q<6GVoa z{{pYl0C&)r)ZP3la8|h)D`~Poz{x49opc{)lr4`6eUEA}OnvK&6A^Vs{jMDb#*?wC z}_=Jb}{odI=os#U+#7~mUGgj9j9<^QId6lT2 zO}0`PoKwzQfC~1~qSv@slu+_;?V8d}?;g@wT~47Shxzh&J{bTmt;qNqcPDnb<6pg# zte#Qz2Ll&d@tn2o(%!6>9zE)7L8X)c1>c^2>1vNLd}vBeTa{WLTdy-&CG)J@+kd6| zI^34mhEfc)vnOd5!$V2hxv$<=8ah(}(?`cF{b=F3EfKk=`F0zq@5Z+k51I(6zhvFA z`C!`i;plA;OUlq9nPk!@dI6$MA0t}a@C>w0~HI?kEWVS2qQP#db+ZSp`HScP@5 z_E#Y4-Rk@^{j#*iSbIrz^1`(h+b)zvzVsnBDYCgM`XblC2FuD_x$w$*;m&Eo(T_#~ z9s{;>xGMUQa&|7bg+V%W$0TopX71H4$u_(iR0HI^?AEsm@|E0f0th-VbmRtY*5lcq z)a>am-!nAwi^7}-bEp$Ntbv4QuL^a&<7@cdiOg-znIQ0SySBAT?MJ&_Z;vb($+5-} zkN;J``Ycwkezf_1>zJI^>j{@>I`zHF47ogi1CEs=_4_<3x*msD3#25XusFqZ*uHBq z<>0O|U{xq{&O}>ri0bw#3E&NDu=+LV^29c@F%i79av?=_!qaAF+kN2=KG{(Qv-QKt zRf~b+D@ojB4$7iQ)JIPsdW@;{n$zVLu&JYA$aXV-B>j&T21$@Bs#q?@YNRe?R_kTM zaKf9+m#MmLfemw}uXI>YqFROx$9n~eFcbu?uIU-SJ#Be_%ucOAd>Wtk_!S7Qml*nP zzK!p(D2Sx)Ot5{RUfA0>zgjLk-RO!uoZX$t1a8%Q)g)}F&V50*Bv+EP+zYPD(BiT{ zx-oR6d+W9F-s4fXqo1vLBUtY;=G*pR{rHN{2q*s}S%jJA;IMQ(r#4;b_G2)3sEZvM;=HVhe+ zpi{>dQf4IdxjtTjiZx!nOADX5hxsw@ugMiyt~#k{J(Bw*7KMWCHB>(zz8jzbZ4~{` z7G(0~3q7Qe4DUW8gmxlz*kzs^|8;V^V|WY`y7Qx*vaxr}RB2&)UHKvcm)YZrV7M;_ zeje1p7}7j1HFs6n9lr~c9dDaq`tXIDF8`!&dksZj0G!SOn*#`ge9wAa8~M-4cvBq_ zU+<^g0MZxEd8p^09B@V5lBXl6A@#t{!DndPc>jWn5nXa1f@5W=?bSk`J- z_1DVYFFmcY?$#66RyF&7Y7#LS$0{c%?4a!@7YtK$ePzUjkkVP~89r7sWMr$`C7sH# zf3v0{QK~7GXKNER8QQYMdflXaxaIG=eDP4r%%!vQbEQT>71w9%&R|WC3H2(BAnt4u zJE99TzTukyGEm|xs5kNPzj^zuhUkSecx-jL4gKc9>wmQOU13dTTc9|OItt2I07J7N zFh~=rA*{?*T|Yl z#SC9{g-%t_RbHNM__>p~@N6y|JSxz0)#jJ_SMT2?Eyem56j&@Aj)xBKxl?0^j^1TA z%vYvv+Fh^Ps<(gSZC3it?{P**g1)P?Q6K9-Gzx{6va-*Eg}<^Hf!c3J19pp4Pj~J@{eJWlMNY>5vA3&*ZLbk@*v2@Kq?$_a**Qm!SJ z3`TuJZVd6w<`yVCRtY08oTqHAh#g~cnkMZbD6+81t2*+{Wzq0j3}w>nftiI`YNMC9 z6{SahG~8f-a$&u*^7x{_A7+TgQ{NLAHy3@4>2ElAbt^Jj*P^UYKKj~1L33=yn8qj4 z`>-elSCXYe0F^T(lS2Q<_lEkSFWg->7OuD6T6$f3=$^;zOa1E0w?1o%JtuyUF@;w} zf%}Jp3V%gtX3nWBn|^U>HNVUF*4(^n-1xKL`l(=_jt?YSL447pjIf85;!!E9?iH3L zk5cl9C`BvTuT?M zzDr!My1Chwl7;(iX~Lu(nO!>>t@cV!Uz5?z0;tVE%ry?lS?*`GJyY<|6$u)Xh^`@_ z_QkRrQ4w(ezQDB?eK_U$oS|x|ukl@oaBXKStMj`e-lJ`L|;E@?{S&H2A?Z z+kK@QrWpLZSd#EwMhPfs`)CBI4k|k5XN-kR^mxmAI~oa>C~}0)Bb^G@)16vg8WRP|;{S2Fz{N%a?By6P;{GDNOvP z>Ug;1+=D~MD=t^#mT!_?B_66ywb!F||Fm%UZr;7^nWRDoo}F7H-HZqTRkTh zL|P^r0xZ?{%$|S7-$Lnp7i3xcnGuD>7MWPGx z7ljU&X{ebs$KAOvxXws6I+JE)q*0w?6QKL$@Zp8inbwWl-^|@3_KIVeFd{L>t_$ z#FrZpntXdS0&6e4yxROPMAZ-MCASO_%SLjV<1grvb1Zh7-5N%TS7^PjSVI+_FB*dn zS!@ZnP6WUnDAQkKZfwwq7nGRe$Fc}ApjklHE<@}OZe~LfW3~6`HGjuj+I$bhA)iFf z+WrbL=nl+Q7^7WE8cICyi)=c6vf3i7HR)-&$lY#@zL(kfnevll=ATX{FF5GvWZcnF zE?vUbt0bJ~P#I2w`ht4i^0{0V22?SV~s|D)@lJwy#PLN3b7 z^0^(gDF{jyM_M<1>HSdPAaFJ7VMg0mM9SPR)U42`Aa~D5_ut$5m_1nyK;+&0n-it> z%}nDPB}d3SW%Zu-PLv8}Ww)WCU&gP#v3Y|%J@Ni13K`tCHcOFsA|fxVoH~fIYA?HS zH{-cLzdUG16I+t0dcJ)OH+vnorhwTFo1*0fXme{fFF&tq^Q^nd zV%OO0^y5kN27^N$gI^`v*%`0wM%fyK4;{&w>;Ru0c~sEx(bU%aWX>l#zv{Uo18DR2ghH{y^|?`my-Xq^673q>*kAuvu|n-|YwaxH{0+Q(;uk61#An zqficP@ktxop=LhUJ@rr8vs6y7HtY+^y`sDs{n$pzSwPl}cl+~cA*-xu%M%-4jqM6; z9|NP|8-1r!|8>d52S^eU{o3!xC-eXa8=eB!^AZn|%@bSUB+}Uc!mBjHr7nrQ(4&oe#4(RI0zde-QMP>8kzJ=8J;D)GAX2)pbf%DKSoQ0 z9yk;Qshs@D@m578Qm$3N7B>_AqVBm}Qr+C>WVP8D*(XEyjAW2qky;dnc|;CBRG>mM z8ZZUG&1qL9UWiFh7{2DRw^;8>f}Pua(f4BWvdjV9Xy{KYyoWE*z6Px z(-G5me0>&wV89=qUQ)xhoE$!2J?}B>4qth7U&MQyM|1^(y=3^FR6Lpu z?j5Au>C;~x3n^Ave)V4f4!r6xDzMxfYm+?Wd;GUn`GPl2AIsgJ+&Cnwno(P0F7p>l z=LPk*^>dxC&wS*CHXZ<3@*0JL27cQ#CvgS9CrV_--oOk;YC;wXa}XX?QrOI>W5?=R zclFSmZ;5)VKojRvZ2n)HmyWmUTZkQd#ZQ8DmRT=foLRnE)=PNieLVQr(7ki!yaQVE zb93L$9w8C1XPzaPe!k3cHVx2=Ui}?9ioMRWzqSbT3a^6>jhiK2^+irJUW=iC+dTHN zt9O+uS~yW>=sJtnLGhZx5m3>d1nbh76G!kRei^@%1-<=T1)R~4<`q~{UH4YcMd~)j z9z`!ME4p$V+xW9v`z&+$!7B#=E53(9;qQ}1MY|~f*I*j3wWMw;-)>%GfNc2j`;)}qUw(st7$dB@q%AhRdUQxF9e^twF4yn|y^Gv(AQ z6lz;{s%?Gtp}#4sL}l>9KaZWDmW1+UXh*AsW2T9vVtgJ2r<7WQD9VKR4MGsI%_xf5HB>cfT^AuPz)&{ z#;rJdDi8*0}5~pny9aLzUc>y&`KTKJq_N(;Z)>jp)CA(9oKjTk4>$v#F^l?JR#X;vN z9mEHNXZX{xct&}PGYQKbV>>gpAq>{!t^IH|9yg^6(T(yKRJ{Tp*wyg^&vFPvPdWx9bgnzaYY8(y~C-q09mNBbM};LQNyptTB80_kb#& z_DdIfZ~=Qac9y~Bmfc;_0X?>cP6tswy}JPfY;*LoK!Y1FxeVcIvc~|0PUf$;0Ox)r7!SByo_>C~I=nB~e&j`nqO=!Dy=klRN}5Z-O36yP6AK3s>QwE? zLsHd?;~bWnArO5*Lj6wQEXIGa%Bkr}QkAXzEh&rM$4jn@5HIAUp=i1|B|k$bXC?FM zB6}yc`u=J05@(}rl#X9mDIh(}_$nM`LNjqt^q2Ev&JC4&Z4Uy0mDzlr$ifNfCfXzr zqr_V`UBw7UXP4g*p~W5KIsOw^Rehel-c)2)*^)pq2V*-Kg|o|6hOXIz{E#M^;&Agm4Y!aA7P}2)T|zw#TE5SA zlhsaY$=aV^Zd&^pwcWof=Y2lidA@K0#$6#1QsW2Jm&c~LkB9B?rJ1T0ii((u;P$e7 zB^-7seIsH9^OWzqletOg7`q;aArt+(-^icm87pOw1rOp*_FI}PM(&k>8modT%X>ak zCQu%{>uTOaJ+!ffEin4Fq%nxhFaAOS6CV*_6;)P!pLH@kbReL$IU*tk?~ij`!ZMAB zKI*|oLP2w-gt>&^$hWyyWVxAK(P(FXL`!H$F(B99T(2RlEdYGGC5!Tqcbs&+uJhvCz^cTto z1~3trs@GQ;MnvZv$_d*d)9xOYs?UnL?`_RWZo9y25Isd2dQ)BoOVROgWK8D5Z&aQ% znQL@P?uZ~XXh?5IolOaye7sEd+#c1VK5F}#7s{T?^Pr{eKmwj{3w3$(A+`S1cinN5 z)xkO1$#_6_`+SQYzEf;*Rond z2U(W5Zy-*pB9-enwM)n3!lUy?*m0lUrZ;+UB}k@T=A5#`Cd9~5#_#R={Js6jm@C80 z>mNq7IR+G~A7_&9t8L*-mJE|#K4?X#<11U?>_>ms!RH|6LaCgfg`yNAYTNnmPon75 zC}iDKL{9(ur9M^^7f4?-Mn`i{NRuvI4Q#2*Pr=isH{TL zSdv3z2k8cNENDBx%gwdiNKWdEU2mwirsO+e#?-5qAZW+3b0DunW)*2773HH~U?LVb zS0<~M{wU9xoiXp+T1{~_vo0$SC-@1JH(qcsh`_-XCM7h>o%HnuQ}&(KeC9hby^xG{ zgqCIv>FE=UpW)n!1ubU*NIV7Ds^Xt}*tXkHr;>7@SIfkHGJ$|qfdP#=DT*%ahIIDa zu3|2F-|Xo$CPZ(sIZ+sl+X0qCr+Nw9^+|CNm7qqFD|4t9xqw*3LWTWXKOyGg3UrY< z(yCf&J1*d*Mh}DNt;F20wRd_F{NJbEvJrXatwGZtS4obsLgIa`X(Rcb5adTgehXw! z>UgzjeSoE_l<(oc(e*yGcIHE!H*{}LC}JJcmR04E+&Hrtz4Z(&BLAkg_e1Vp0d0;G zJ6;~|>-|odGw$NvGc%$ca2BkCHWYVg2}|{m)Z&tnLHiq8{!@h=!)d@Bw9w#-!U=Y9 zqIOWuNp8zMlL`cv$?VioENsb4HxV23>G%V*Hz^Uk`z@-_Fs|_1leo=UA|-1qV4w&b z@O9g%NOutNz}`RSPQF8`B^4et(o0$Z7OkTFvgPZ#7Kaew=Jo3{j#?YZd+qExN>BZ} zOeAgJs!iTWq#i4h1&rAo&ebUNQIfhOWzXp#(qmnS#@Ng&^C6&>!G@WqDb(qfUZ-L6 zG3G9PGatdcV}4=4x`dR#AcMbVvtLVEE3fxhDc83)I%qJQxFk7wljUr;?Rt@*R`{)v z2^?kzuwEj&xlKP>)NLSCexH@MfsN)^9kwFvhN1Bi>XR+kjzns4tDmB93ynGGsL<3+ zvMwsXS^KicsbU2TecyxXV~b8-9XW+7mj~o5SH`Hk%hvEHAwL2!-qIG}?27n2A6e+! z4$(Q6i=wE)S^nH{J*{nXk{MqXT*3>-y#%Q90~yO#YHLVYlu=l9WQkVFl-B$etiajL zbxMv*+@Zfw8{XQ-2cbF5a|g5wVub4=pkwigGq!^Vxb+S=5o70SCyI zT}(t*CzD-VqrB8}KX^K(snCPUl0(z2rS`t~H z&PpJ;2v}wanqSigE_vV-43=KpL>iD z@<)u?t~3*;-}u(Cw2-8f-;FMVc~JR<94A+9465gjrzSF4i1`#l zrPF zf?T-@%hxw8+Yf=}k%VrU9Q@la4L(CJnQhr<r+DPX~j#M z{>yLe>l!g8+-VG8s5AAXnLtRR#(EQ>sds5w>?1(F$dZ{kp14(=cf)7nYk zzD~SQ^kpPvdm(FSytr<9%`vhKvlv4^0t#t2F~OKL@@Xm;@o5Il#)@RRZ8G$bm3!Y8 zhj9Cr6uT~XHTT&epEnd%NyKjDGA+llPG_cn#U6QOxQDZ>r_AWpObZ|!T=>BpvoK4&^oPFkd6QuZi& z?ANqjAwpo3RG;%psK|(e{5wI@?pAfjoLiJAZ(TYb;1$_-4+tjqOqEpLX&4y1N%S6t(#}IOYWxfV(K0(}2c46+m%QwCFV{(HD^ub#FTFkBE+% z9MIrk0^8L`#t$h$1Zhr=#M~SLn{_#yA|Em($bW_z4nR-WD*){4~PYPdkJ+~uUSbbzrpG%#vxOW z)$JI*G|z13^-CSZho);`NK^VX#Fi%a_SGhgm6JYL-aZw%(ANdDFzwu0RD{k$q%>uV z2K%Di^A9sjxiR^rKR^S~k_(t&- zjeb_h@dQcx-IYO<^3GS&oTld6m%?wAF6&%li?swC-J&RvYb&A5=fru`M#@~3A=SO1 za!)-^>6R3>BZ4v#nlA`a{CnLWM2tn+D+0_g3avqF7b-eI$3o0yuw}WrnzryQNQ3*t zn>y6R<2=5nt7QLf@q4!nZu_UH1n*lHx>adx-x#eJdv6&`tGM(+FpR6KhhNxho}CsO zEa|ksz$5v@Gj&+I9Y3JV^AL9`&53-wWO+I_XdbrN$vA05wVVy`vBy6MLuac<&=(Xt zOC8!n7O_q&z6seT#_9_c1*GGF@ChpQYk0tO_pnscPk;ATD_Z4~v-~2@0FB~DEar+L zZqtS`)ZihZOIQ%2pSBP`TwbHeGmVXR`f)!yB#&aRjbqrlaqgeB_`46q@pV3Yq~#H( zOMYt=X9<>%R8s6gGnz}5(~ML9pu`^&%2d9%+5V^XiCYJ~|M-4){m<0@ zt1HQ{BR}f#yn6qi=^RKbGDije{p;bUGyet{56^A8d;c5k|G~)r!1%un^8bg&1 | grep -q "clang version" && echo clang || echo gcc) + CFLAGS := -std=gnu11 -O2 -g -fdata-sections -ffunction-sections -fplan9-extensions -fstack-protector -fno-strict-aliasing -fvisibility=hidden \ + -Wall -Wextra -Wcast-align -Wpointer-arith -Wmissing-prototypes -Wnonnull \ + -Wwrite-strings -Wlogical-op -Wformat=2 -Wmissing-format-attribute -Winit-self -Wshadow \ + -Wstrict-prototypes -Wunreachable-code -Wconversion -Wsign-conversion \ +- -Wno-unknown-warning-option -Wno-format-extra-args -Wno-gnu-alignof-expression $(CFLAGS) ++ -Wno-format-extra-args $(if $(filter clang,$(CCNAME)),-Wno-unknown-warning-option -Wno-gnu-alignof-expression,) \ ++ -I=/usr/include/tirpc $(CFLAGS) + LDFLAGS := -Wl,-zrelro -Wl,-znow -Wl,-zdefs -Wl,--gc-sections $(LDFLAGS) + LDLIBS := $(LDLIBS) ++CFLAGS := -Wno-error $(CFLAGS) ++LDLIBS := -Wno-error $(LDLIBS) ++CPPFLAGS := -Wno-error $(CPPFLAGS) + + # Library flags (recursively expanded to handle target-specific flags) + LIB_CPPFLAGS = -DNV_LINUX -isystem $(DEPS_DIR)$(includedir) -include $(BUILD_DEFS) + LIB_CFLAGS = -fPIC +-LIB_LDFLAGS = -L$(DEPS_DIR)$(libdir) -shared -Wl,-soname=$(LIB_SONAME) +-LIB_LDLIBS_STATIC = -l:libnvidia-modprobe-utils.a +-LIB_LDLIBS_SHARED = -ldl -lcap ++LIB_LDFLAGS = -shared -Wl,-soname=$(LIB_SONAME) ++LIB_LDLIBS_STATIC = -L$(DEPS_DIR)$(libdir) -l:libnvidia-modprobe-utils.a -L=$(libdir) -l:libtirpc.a ++LIB_LDLIBS_SHARED = -ldl -lcap -lpthread + ifeq ($(WITH_NVCGO), yes) + LIB_CPPFLAGS += -DWITH_NVCGO + LIB_LDLIBS_SHARED += -lpthread +@@ -205,7 +210,11 @@ $(BUILD_DEFS): + @printf '#define BUILD_FLAGS "%s"\n' '$(strip $(CPPFLAGS) $(CFLAGS) $(LDFLAGS))' >>$(BUILD_DEFS) + @printf '#define BUILD_REVISION "%s"\n' '$(strip $(REVISION))' >>$(BUILD_DEFS) + @printf '#define BUILD_PLATFORM "%s"\n' '$(strip $(PLATFORM))' >>$(BUILD_DEFS) +- ++ifeq ($(EXCLUDE_BUILD_FLAGS),) ++ @printf '#define BUILD_FLAGS "%s"\n' '$(strip $(CPPFLAGS) $(CFLAGS) $(LDFLAGS))' >>$(BUILD_DEFS) ++else ++ @printf '#define BUILD_FLAGS ""\n' >>$(BUILD_DEFS) ++endif + $(LIB_RPC_SRCS): $(LIB_RPC_SPEC) + $(RM) $@ + cd $(dir $@) && $(RPCGEN) $(RPCGENFLAGS) -C -M -N -o $(notdir $@) $(LIB_RPC_SPEC) +@@ -224,17 +233,14 @@ $(LIB_SHARED): $(LIB_OBJS) + $(OBJCPY) --only-keep-debug $@ $(LIB_SONAME) + $(OBJCPY) --add-gnu-debuglink=$(LIB_SONAME) $@ + $(MV) $(LIB_SONAME) $(DEBUG_DIR) +- $(STRIP) --strip-unneeded -R .comment $@ + + $(LIB_STATIC_OBJ): $(LIB_OBJS) + # FIXME Handle user-defined LDFLAGS and LDLIBS + $(LD) -d -r --exclude-libs ALL -L$(DEPS_DIR)$(libdir) $(OUTPUT_OPTION) $^ $(LIB_LDLIBS_STATIC) + $(OBJCPY) --localize-hidden $@ +- $(STRIP) --strip-unneeded -R .comment $@ + + $(BIN_NAME): $(BIN_OBJS) + $(CC) $(BIN_CFLAGS) $(BIN_CPPFLAGS) $(BIN_LDFLAGS) $(OUTPUT_OPTION) $^ $(BIN_SCRIPT) $(BIN_LDLIBS) +- $(STRIP) --strip-unneeded -R .comment $@ + + ##### Public rules ##### + +@@ -244,7 +250,6 @@ all: shared static tools + # Run with ASAN_OPTIONS="protect_shadow_gap=0" to avoid CUDA OOM errors + debug: CFLAGS += -pedantic -fsanitize=undefined -fno-omit-frame-pointer -fno-common -fsanitize=address + debug: LDLIBS += -lubsan +-debug: STRIP := @echo skipping: strip + debug: shared static tools + + tools: $(BIN_NAME) +diff --git a/mk/common.mk b/mk/common.mk +index f6d5fbf2..d46584b7 100644 +--- a/mk/common.mk ++++ b/mk/common.mk +@@ -28,7 +28,7 @@ else + DATE := $(shell date -u --iso-8601=minutes) + endif + REVISION ?= $(shell git rev-parse HEAD) +-COMPILER := $(realpath $(shell which $(CC))) ++COMPILER := $(realpath $(shell which $(firstword $(CC)))) + PLATFORM ?= $(shell uname -m) + + ifeq ($(DATE),) diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch new file mode 100644 index 000000000..46c5e3ad3 --- /dev/null +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch @@ -0,0 +1,13 @@ +diff --git a/Makefile b/Makefile +index a374cc09..b6632580 100644 +--- a/Makefile ++++ b/Makefile +@@ -20,7 +20,7 @@ + WITH_NVCGO ?= yes + WITH_LIBELF ?= no + WITH_TIRPC ?= no +-WITH_SECCOMP ?= yes ++WITH_SECCOMP := yes + + ##### Global definitions ##### + diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch new file mode 100644 index 000000000..b6723794b --- /dev/null +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch @@ -0,0 +1,24 @@ +From 13cc971cf7ab0bb8fe8528f2ebd65685a81b7ae3 Mon Sep 17 00:00:00 2001 +From: Atharva Nandanwar +Date: Fri, 19 Aug 2022 15:25:18 -0600 +Subject: [PATCH] fix: remove buildpath for package qa + +* get EXTRA_CFLAGS for nvidia-modprobe +--- + mk/nvidia-modprobe.mk | 1 ++ + 1 file changed, 1 insertions(+) + +diff --git a/mk/nvidia-modprobe.mk b/mk/nvidia-modprobe.mk +index 3406222..c7af214 100644 +--- a/mk/nvidia-modprobe.mk ++++ b/mk/nvidia-modprobe.mk +@@ -26,6 +26,7 @@ PATCH_FILE := $(MAKE_DIR)/nvidia-modprobe.patch + ARFLAGS := -rU + CPPFLAGS := -D_FORTIFY_SOURCE=2 -DNV_LINUX + CFLAGS := -O2 -g -fdata-sections -ffunction-sections -fstack-protector -fno-strict-aliasing -fPIC ++CFLAGS += $(NVIDIA_MODPROBE_EXTRA_CFLAGS) + + ##### Private rules ##### + +-- +2.34.1 diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb new file mode 100644 index 000000000..2d8013672 --- /dev/null +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -0,0 +1,65 @@ +require libnvidia-container.inc +inherit features_check + +SUMMARY = "libNVIDIA Container for Yocto" + +PACKAGECONFIG ??= "seccomp" +PACKAGECONFIG[seccomp] = "WITH_SECCOMP=yes,WITH_SECCOMP=no,libseccomp" + +# We need to link with libelf, otherwise we need to +# include bmake-native which does not exist at the moment. +EXTRA_OEMAKE = "EXCLUDE_BUILD_FLAGS=1 PLATFORM=${HOST_ARCH} WITH_NVCGO=yes WITH_LIBELF=yes WITH_SECCOMP=yes COMPILER=${@d.getVar('CC').split()[0]} REVISION=${SRCREV_libnvidia} ${PACKAGECONFIG_CONFARGS} \ + NVIDIA_MODPROBE_EXTRA_CFLAGS=${NVIDIA_MODPROBE_EXTRA_CFLAGS} \ + GOCACHE=${WORKDIR}/go-cache GOPATH=${WORKDIR}/go GOMODCACHE=${WORKDIR}/go-mod" +NVIDIA_MODPROBE_EXTRA_CFLAGS ?= "-ffile-prefix-map=${WORKDIR}=/usr/src/debug/${PN}/${EXTENDPE}${PV}-${PR}" +CFLAGS:prepend = " -I${RECIPE_SYSROOT_NATIVE}/usr/include/tirpc " + +export OBJCPY="${OBJCOPY}" +GO_IMPORT = "github.com/NVIDIA/nvidia-container-toolkit" +SECURITY_LDFLAGS = "" +LDFLAGS += "-Wl,-z,lazy" +GO_LINKSHARED = "" +REQUIRED_DISTRO_FEATURES = "virtualization" +do_configure:append() { + # Mark Nvidia modprobe as downloaded + touch ${S}/deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION}/.download_stamp +} + +do_compile() { + # Ensure the copied bmake is used during the build + export PATH=${WORKDIR}:$PATH + + #go fix + export GOPATH="${WORKDIR}/go" + export GOCACHE="${WORKDIR}/go-cache" + export GOMODCACHE="${WORKDIR}/go-mod" + mkdir -p ${GOPATH} ${GOCACHE} ${GOMODCACHE} + + # Ensure reproducible builds + export SOURCE_DATE_EPOCH="${@d.getVar('SOURCE_DATE_EPOCH') or '0'}" + export CGO_LDFLAGS="${CGO_LDFLAGS} -Wl,--build-id=none" + + export CURL="curl --insecure" + + oe_runmake +} + +do_install() { + oe_runmake install DESTDIR=${D} + install -d ${D}${sysconfdir}/nvidia-container-runtime + # install -m 0644 ${S}/src/${GO_IMPORT}/config/config.toml.ubuntu ${D}${sysconfdir}/nvidia-container-runtime/config.toml + # sed -i -e's,ldconfig\.real,ldconfig,' ${D}${sysconfdir}/nvidia-container-runtime/config.toml + # sed -i -e's,mode = "auto",mode = "legacy",' ${D}${sysconfdir}/nvidia-container-runtime/config.toml + ln -sf nvidia-container-runtime-hook ${D}${bindir}/nvidia-container-toolkit +} + +FILES_${PN} += "/usr/local/bin /usr/local/lib" + +RDEPENDS:${PN}:append = " ldconfig" + +do_compile[network] = "1" + +# Added to skip buildpath QA errors for files generated by rpcgen +INSANE_SKIP:${PN}-src = "buildpaths" +# Skip QA checks for Go shared library +INSANE_SKIP:${PN} = "already-stripped ldflags buildpaths" diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb new file mode 100644 index 000000000..631e2dbf5 --- /dev/null +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb @@ -0,0 +1,31 @@ +SUMMARY = "Transport-Independent RPC library" +DESCRIPTION = "Libtirpc is a port of Suns Transport-Independent RPC library to Linux" +SECTION = "libs/network" +HOMEPAGE = "http://sourceforge.net/projects/libtirpc/" +BUGTRACKER = "http://sourceforge.net/tracker/?group_id=183075&atid=903784" +LICENSE = "BSD-3-Clause" +LIC_FILES_CHKSUM = "file://COPYING;md5=f835cce8852481e4b2bbbdd23b5e47f3 \ + file://src/netname.c;beginline=1;endline=27;md5=f8a8cd2cb25ac5aa16767364fb0e3c24" + +SRC_URI = "${SOURCEFORGE_MIRROR}/libtirpc/libtirpc-${PV}.tar.bz2" +SRC_URI[sha256sum] = "1e0b0c7231c5fa122e06c0609a76723664d068b0dba3b8219b63e6340b347860" + +# SRC_URI += "file://0001-__rpc_dtbsize-rlim_cur-instead-of-rlim_max.patch" + +S = "${WORKDIR}/libtirpc-${PV}" + +inherit autotools pkgconfig + +DISABLE_STATIC = "" +EXTRA_OECONF = "--disable-gssapi --enable-static" + +# Append -fPIC to CFLAGS +CFLAGS:append = " -fPIC" + +do_install:append() { + rm -r ${D}${sysconfdir} ${D}${datadir} ${D}${libdir}/pkgconfig + rm ${D}${libdir}/*.so* + cp -r ${D}${includedir}/tirpc ${D}${includedir}/tirpc-1.3.4 +} + +inherit nopackages diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch new file mode 100644 index 000000000..124258b9a --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch @@ -0,0 +1,27 @@ +From 00529c2b30cf74993bb124ba82a6947703540584 Mon Sep 17 00:00:00 2001 +From: Matt Madison +Date: Sun, 17 Mar 2024 09:33:12 -0700 +Subject: [PATCH] Fix cgo LDFLAGS for go 1.21 and later + +Upstream-Status: Pending + +Signed-off-by: Matt Madison +--- + internal/cuda/cuda.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/internal/cuda/cuda.go b/internal/cuda/cuda.go +index 2c70a82..3fc7952 100644 +--- a/internal/cuda/cuda.go ++++ b/internal/cuda/cuda.go +@@ -23,7 +23,7 @@ import ( + ) + + /* +-#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files ++#cgo LDFLAGS: -Wl,--export-dynamic -Wl,--unresolved-symbols=ignore-in-object-files + + #ifdef _WIN32 + #define CUDAAPI __stdcall +-- +2.34.1 diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml new file mode 100644 index 000000000..e09dec2a3 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml @@ -0,0 +1,26 @@ +[nvidia-container-cli] +#root = "/run/nvidia/driver" +#path = "/usr/bin/nvidia-container-cli" +environment = [] +# debug = "/var/log/nvidia-container-toolkit.log" +#ldcache = "/etc/ld.so.cache" +load-kmods = true +no-cgroups = false +#user = "root:video" +ldconfig = "@/sbin/ldconfig" +#alpha-merge-visible-devices-envvars = false + +[nvidia-container-runtime] +# debug = "/var/log/nvidia-container-runtime.log" +log-level = "debug" + +# Specify the runtimes to consider. This list is processed in order and the PATH +# searched for matching executables unless the entry is an absolute path. +runtimes = [ + "runc", +] + +mode = "auto" + +[nvidia-container-runtime.modes.csv] +mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d" diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc new file mode 100644 index 000000000..1c6efdf3c --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc @@ -0,0 +1,18 @@ +DESCRIPTION = "NVIDIA Container Toolkit" +HOMEPAGE = "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/NVIDIA/nvidia-container-toolkit" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://src/${GO_IMPORT}/LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57" + +SRC_URI = "git://github.com/NVIDIA/nvidia-container-toolkit.git;protocol=https;branch=release-1.14" +SRCREV = "4668c511de4b311c96bc3dd0310bff40b75083bd" + +SRC_URI += "file://config.toml" +SRC_URI += "file://0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch;patchdir=src/${GO_IMPORT}" +GO_IMPORT = "github.com/NVIDIA/nvidia-container-toolkit" + +DEPENDS = " \ + curl-native ca-certificates-native go-native \ + coreutils-native \ + " + +S = "${WORKDIR}/git" diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb new file mode 100644 index 000000000..9106fce63 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb @@ -0,0 +1,54 @@ +require nvidia-container-toolkit.inc + +SUMMARY = "NVIDIA Container Toolkit for Yocto" + +GO_INSTALL = "${GO_IMPORT}/cmd/..." +# The go-nvml symbol lookup functions *require* lazy dynamic symbol resolution +SECURITY_LDFLAGS = "" +LDFLAGS += "-Wl,-z,lazy" +GO_LINKSHARED = "" + +REQUIRED_DISTRO_FEATURES = "virtualization" + +inherit go go-mod features_check + +do_compile() { + echo "Current directory: $(pwd)" + ls -alh + oe_runmake -C ${S}/src/${GO_IMPORT} cmds + +} + +install_bin_stripped() { + install -m 0755 ${S}/src/${GO_IMPORT}/$1 ${D}${bindir}/$1 + ${STRIP} --remove-section=.note.gnu.build-id ${D}${bindir}/$1 + ${STRIP} --remove-section=.note.go.buildid ${D}${bindir}/$1 +} + +do_install() { + # Create the target directories in the image file system + install -d ${D}${bindir} + + # Copy each binary to the target directory + install_bin_stripped nvidia-container-runtime + install_bin_stripped nvidia-container-runtime.cdi + install_bin_stripped nvidia-container-runtime-hook + install_bin_stripped nvidia-container-runtime.legacy + install_bin_stripped nvidia-ctk + + ln -sf nvidia-container-runtime-hook ${D}${bindir}/nvidia-container-toolkit + + # create config.toml + # Ensure the installation directory exists + install -d ${D}/etc/nvidia-container-runtime + # Install the config.toml file + install -m 0644 ${WORKDIR}/config.toml ${D}/etc/nvidia-container-runtime/config.toml +} + +INSANE_SKIP:${PN} += "already-stripped buildpaths textrel" +INSANE_SKIP:${PN}:append = "already-stripped buildpaths textrel" +FILES_${PN} += "/usr/local/*" + +RDEPENDS:${PN} = "\ + libnvidia-container \ + " \ No newline at end of file diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service new file mode 100644 index 000000000..07e956b35 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service @@ -0,0 +1,14 @@ +[Unit] +Description=NVIDIA Persistence Daemon +Wants=syslog.target +Before=app-compose.service docker.service +After=systemd-modules-load.service modprobe@.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/env nvidia-persistenced --uvm-persistence-mode +ExecStartPost=/usr/bin/nvidia-smi conf-compute -srs 1 +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf b/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf new file mode 100644 index 000000000..e0b90c5d4 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf @@ -0,0 +1 @@ +options nvidia NVreg_RegistryDwords="RmEnableProtectedPcie=0x1" diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb new file mode 100644 index 000000000..db47484cd --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb @@ -0,0 +1,39 @@ +SUMMARY = "NVIDIA NSCQ library" +DESCRIPTION = "NVIDIA NSCQ (NVIDIA System Communication Queue) library for NVIDIA GPU systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "b8cc1c4e37794eaf738c488d419a7c90" +SRC_URI[sha256sum] = "66d1c4303700f19bd86bddda071340e070511a7070ce60a8641f16db7c184cfa" + +S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +do_install() { + # Create directories + install -d ${D}${libdir} + install -d ${D}${bindir} + + # Install libraries + install -m 0755 ${S}/lib/libnvidia-nscq.so.${PV} ${D}${libdir} + ln -sf libnvidia-nscq.so.${PV} ${D}${libdir}/libnvidia-nscq.so.2.0 + ln -sf libnvidia-nscq.so.2.0 ${D}${libdir}/libnvidia-nscq.so.2 + ln -sf libnvidia-nscq.so.2 ${D}${libdir}/libnvidia-nscq.so + + # Install binaries + install -m 0755 ${S}/bin/nscq-cli ${D}${bindir} +} + +FILES:${PN} = "\ + ${libdir}/libnvidia-nscq.so.${PV} \ + ${libdir}/libnvidia-nscq.so.2.0 \ + ${libdir}/libnvidia-nscq.so.2 \ + ${libdir}/libnvidia-nscq.so \ + ${bindir}/nscq-cli \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb new file mode 100644 index 000000000..3d91e4cd9 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb @@ -0,0 +1,86 @@ +SUMMARY = "NVIDIA Fabric Manager for NVSwitch systems" +DESCRIPTION = "NVIDIA Fabric Manager provides NVSwitch management for NVIDIA HGX and DGX systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "a71788f11f6edabf69df32a7b9dcfa68" +SRC_URI[sha256sum] = "8d24cacde4554d471899ad426f46a349d5ca0a2e8acd45c2a76381c8f496491e" + +S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" + +DEPENDS = "" +RDEPENDS:${PN} = "bash zlib" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +inherit systemd + +SYSTEMD_AUTO_ENABLE = "enable" +SYSTEMD_SERVICE:${PN} = "nvidia-fabricmanager.service" + +do_install() { + # Create directories + install -d ${D}${bindir} + install -d ${D}${libdir} + install -d ${D}${datadir}/nvidia/nvswitch + install -d ${D}${systemd_system_unitdir} + + # Install binaries + install -m 0755 ${S}/bin/nv-fabricmanager ${D}${bindir} + install -m 0755 ${S}/bin/nvidia-fabricmanager-start.sh ${D}${bindir} + install -m 0755 ${S}/bin/nvswitch-audit ${D}${bindir} + + # Install libraries + install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} + ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so + + # Install config files + install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ + + # Install topology files + install -m 0644 ${S}/share/nvidia/nvswitch/dgx2_hgx2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxa100_hgxa100_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxh100_hgxh100_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxh800_hgxh800_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/mgxh20_nvl16_topology ${D}${datadir}/nvidia/nvswitch/ + + # Install multi-node topology files + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_8gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gh200_nvlink_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl576r16_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_nso_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl4r1_c2g2_etf_topology ${D}${datadir}/nvidia/nvswitch/ + + # Install systemd service + install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} +} + +FILES:${PN} = "\ + ${bindir}/nv-fabricmanager \ + ${bindir}/nvidia-fabricmanager-start.sh \ + ${bindir}/nvswitch-audit \ + ${libdir}/libnvfm.so.1 \ + ${libdir}/libnvfm.so \ + ${datadir}/nvidia/nvswitch/* \ + ${systemd_system_unitdir}/nvidia-fabricmanager.service \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc b/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc new file mode 100644 index 000000000..b26d64681 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc @@ -0,0 +1,16 @@ +inherit module + +S = "${NVIDIA_SRC}/kernel-open" + +do_compile[depends] += "virtual/kernel:do_shared_workdir" +EXTRA_OEMAKE:append = " SYSSRC=${STAGING_KERNEL_DIR} KERNEL_OUTPUT=${STAGING_KERNEL_BUILDDIR}" + +export ARCH = "${@bb.utils.contains('TARGET_ARCH', 'x86_64', 'x86_64', '', d)}" + +RPROVIDES:${PN} += " \ + kernel-module-nvidia-${KERNEL_VERSION} \ + kernel-module-nvidia-drm-${KERNEL_VERSION} \ + kernel-module-nvidia-modeset-${KERNEL_VERSION} \ + kernel-module-nvidia-peermem-${KERNEL_VERSION} \ + kernel-module-nvidia-uvm-${KERNEL_VERSION} \ +" \ No newline at end of file diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc b/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc new file mode 100644 index 000000000..a1ded2ae0 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc @@ -0,0 +1,76 @@ +DESCRIPTION = "Linker and Installer of Nvidia Kernel modules, libs and binaries" +SECTION = "Installer Tools" +MAINTAINER = "Nauman Shakir " + +RDEPENDS:${PN} += "${PN}-firmware ${PN}-private-libs" + +addlib() { + cp ${NVIDIA_SRC}/${1}.so.${PV} ${D}${libdir}/ + ln -sf ${1}.so.${PV} ${D}${libdir}/${1}.so.1 + ln -sf ${1}.so.1 ${D}${libdir}/${1}.so +} + +addbin() { + install -m 755 -D ${NVIDIA_SRC}/${1} ${D}${bindir}/${1} +} + +do_install:append() { + install -d ${D}${libdir} + install -d ${D}${bindir} + + addlib libnvidia-ml + addlib libnvidia-allocator + addlib libnvidia-eglcore + addlib libnvidia-encode + addlib libnvidia-glcore + addlib libnvidia-gpucomp + addlib libnvidia-ngx + addlib libnvidia-nvvm + addlib libnvidia-opencl + addlib libnvidia-rtcore + addlib libnvidia-tls + addlib libnvidia-cfg + addlib libnvidia-opticalflow + addlib libnvidia-glsi + addlib libnvidia-glvkspirv + addlib libcuda + addlib libnvcuvid + addlib libnvidia-pkcs11-openssl3 + addlib libnvidia-pkcs11 + addlib libnvidia-ptxjitcompiler + + cp ${NVIDIA_SRC}/libnvidia-api.so.1 ${D}${libdir}/ + ln -sf libnvidia-api.so.1 ${D}${libdir}/libnvidia-api.so + ln -sf libnvidia-nvvm.so.${PV} ${D}${libdir}/libnvidia-nvvm.so.4 + + addbin nvidia-smi + addbin nvidia-debugdump + addbin nvidia-persistenced + addbin nvidia-modprobe + + # Copy the firmware to the target directory + install -d ${D}${libdir}/firmware/nvidia/${PV} + cp ${NVIDIA_SRC}/firmware/* ${D}${libdir}/firmware/nvidia/${PV}/ +} + +FILES:${PN}:append= " ${libdir} ${bindir}" + + +INHIBIT_PACKAGE_DEBUG_SPLIT = "1" +INHIBIT_PACKAGE_STRIP = "1" + +INSANE_SKIP:${PN}:append = " ldflags already-stripped dev-so" +INSANE_SKIP_libgl-nvidia:append = " ldflags" +INSANE_SKIP_:append = " ldflags" + +# Create a separate package for GSP firmware files +PACKAGES =+ "${PN}-firmware" +FILES:${PN}-firmware = "${libdir}/firmware/nvidia/${PV}/*" +FILES:${PN}:remove = "${libdir}/firmware/nvidia/${PV}/*" +INSANE_SKIP:${PN}-firmware += "arch" + +PACKAGES =+ "${PN}-private-libs" +FILES:${PN}-private-libs = "${libdir}/libnvidia-pkcs11.so*" +INSANE_SKIP:${PN}-private-libs:append = " ldflags already-stripped dev-so" +EXCLUDE_PACKAGES_FROM_SHLIBS = "${PN}-private-libs" +SKIP_FILEDEPS:${PN}-private-libs = "1" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb new file mode 100644 index 000000000..91b654077 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb @@ -0,0 +1,16 @@ +SUMMARY = "NVIDIA kernel module configuration" +DESCRIPTION = "Configuration for NVIDIA kernel modules with dynamic settings based on kernel command line" +LICENSE = "CLOSED" + +SRC_URI = "\ + file://nvidia.conf \ +" + +do_install() { + install -d ${D}${sysconfdir}/modprobe.d + install -m 0644 ${WORKDIR}/nvidia.conf ${D}${sysconfdir}/modprobe.d/ +} + +FILES:${PN} = "${sysconfdir}/modprobe.d/nvidia.conf" + +RDEPENDS:${PN} = "bash" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb new file mode 100644 index 000000000..bb73a6d11 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb @@ -0,0 +1,17 @@ +SUMMARY = "NVidia Persistenced systemd service" +LICENSE = "CLOSED" + +SRC_URI += "\ + file://nvidia-persistenced.service \ +" + +inherit systemd + +SYSTEMD_PACKAGES = "${PN}" +SYSTEMD_SERVICE:${PN} = "nvidia-persistenced.service" +SYSTEMD_AUTO_ENABLE:${PN} = "enable" + +do_install() { + install -d ${D}${systemd_unitdir}/system + install -m 0644 ${WORKDIR}/nvidia-persistenced.service ${D}${systemd_unitdir}/system +} diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb new file mode 100644 index 000000000..bacb3893b --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb @@ -0,0 +1,24 @@ +SUMMARY = "NVidia Graphics Driver" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://../LICENSE;md5=92aa2e2af6aa0bcba1c3fe49da021937" + +NVIDIA_ARCHIVE_NAME = "NVIDIA-Linux-${TARGET_ARCH}-${PV}" +NVIDIA_SRC = "${WORKDIR}/${NVIDIA_ARCHIVE_NAME}" +SRC_URI = " \ + https://us.download.nvidia.com/tesla/${PV}/${NVIDIA_ARCHIVE_NAME}.run \ +" +SRC_URI[md5sum] = "4d0264e320d09f614b086c3ad99a75c5" +SRC_URI[sha256sum] = "0256867e082caf93d7b25fa7c8e69b316062a9c6c72c6e228fad7b238c6fa17d" + +RDEPENDS:${PN} = "nvidia-modprobe-config" + +do_unpack() { + chmod +x ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run + rm -rf ${NVIDIA_SRC} + ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run -x --target ${NVIDIA_SRC} +} + +do_make_scripts[noexec] = "1" + +include nvidia-kernel-module.inc +include nvidia-libs.inc diff --git a/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg b/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg new file mode 100644 index 000000000..9e4b7776d --- /dev/null +++ b/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg @@ -0,0 +1,3 @@ +CONFIG_CRYPTO_ECC=y +CONFIG_CRYPTO_ECDSA=y +CONFIG_CRYPTO_ECDH=y diff --git a/meta-nvidia/recipes-kernel/linux/files/nvidia.scc b/meta-nvidia/recipes-kernel/linux/files/nvidia.scc new file mode 100644 index 000000000..8bb149af1 --- /dev/null +++ b/meta-nvidia/recipes-kernel/linux/files/nvidia.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "NVIDIA configuration" + +kconf non-hardware nvidia.cfg diff --git a/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend b/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend new file mode 100644 index 000000000..08a530bb1 --- /dev/null +++ b/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend @@ -0,0 +1,7 @@ + +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI += "file://nvidia.cfg \ + file://nvidia.scc" + +KERNEL_FEATURES:append = " nvidia.scc" diff --git a/repro-build/check.sh b/repro-build/check.sh index 38327ba01..74df22610 100755 --- a/repro-build/check.sh +++ b/repro-build/check.sh @@ -13,7 +13,8 @@ GREEN='\033[0;32m' RED='\033[0;31m' NC='\033[0m' -ROOTFS_PATH=tmp/work/tdx-poky-linux/dstack-rootfs/1.0/rootfs +IMAGE_NAME=${IMAGE_NAME:-dstack-rootfs} +ROOTFS_PATH=tmp/work/tdx-poky-linux/${IMAGE_NAME}/1.0/rootfs BUILD_DIR_A=${1:-${THIS_DIR}/build-a} BUILD_DIR_B=${2:-${THIS_DIR}/build-b} BB_DIR_A=${BB_DIR_A:-${BUILD_DIR_A}/bb-build} diff --git a/repro-build/repro-build.sh b/repro-build/repro-build.sh index 18a93b868..2b96bfd81 100755 --- a/repro-build/repro-build.sh +++ b/repro-build/repro-build.sh @@ -23,7 +23,7 @@ done BUILDER_NAME=dstack-build THIS_DIR=$(cd $(dirname $0); pwd) -REPO_ROOT=$(dirname $THIS_DIR) +REPO_ROOT=${REPO_ROOT:-$(dirname $THIS_DIR)} GIT_DIR=$REPO_ROOT HOST_BUILD_DIR_A=${THIS_DIR}/build-a @@ -41,7 +41,7 @@ rm -rf .dummy build_to() { mkdir -p $1 - BUILD_CMD="${2} ${GUEST_SRC_DIR}/build.sh guest ./bb-build" + BUILD_CMD="${2} ${GUEST_SRC_DIR}/${META_SUBDIR}/build.sh guest ./bb-build" docker run --platform linux/amd64 --rm \ --userns=host \ --user $(id -u):$(id -g) \ diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 9d6f3067d..5274ba465 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -272,6 +272,11 @@ def setup_instance(self, args: argparse.Namespace) -> None: gpus = { "attach_mode": "all", } + elif args.gpu == ['none'] or not args.gpu: + gpus = { + "attach_mode": "listed", + "gpus": [] + } else: gpus = { "attach_mode": "listed", diff --git a/scripts/bin/enable_vfio_passthrough.sh b/scripts/bin/enable_vfio_passthrough.sh new file mode 100644 index 000000000..941695ac8 --- /dev/null +++ b/scripts/bin/enable_vfio_passthrough.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "==> Detecting NVIDIA GPUs and NVSwitches" +mapfile -t GPU_BDFS < <( + lspci -d 10de: -nn | grep '3D controller' | awk '{print $1}' +) +mapfile -t GPU_DEV_IDS < <( + lspci -d 10de: -nn | grep '3D controller' | sed -n 's/.*\[10de:\([0-9A-Fa-f]\+\)\].*/\1/p' +) +mapfile -t SW_BDFS < <( + lspci -d 10de: -nn | grep 'Bridge' | awk '{print $1}' +) +mapfile -t SW_DEV_IDS < <( + lspci -d 10de: -nn | grep 'Bridge' | sed -n 's/.*\[10de:\([0-9A-Fa-f]\+\)\].*/\1/p' +) + +NGPU=${#GPU_BDFS[@]} +NSW=${#SW_BDFS[@]} +TOTAL=$((NGPU+NSW)) + +if [ $TOTAL -eq 0 ]; then + echo "No NVIDIA GPUs or NVSwitches found. Exiting." + exit 1 +fi + +echo "Found $NGPU GPU(s): ${GPU_BDFS[*]}" +echo "Found $NSW NVSwitch(s): ${SW_BDFS[*]}" + +echo "==> Loading VFIO modules" +sudo modprobe vfio vfio_pci + +echo "==> Tagging devices for VFIO passthrough" +# GPUs first +for idx in "${!GPU_BDFS[@]}"; do + BDF=${GPU_BDFS[$idx]}; DEV=${GPU_DEV_IDS[$idx]} + echo "Tagging GPU $BDF (DevID=10de:$DEV)" + if ! echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id > /dev/null; then + echo " -> failure, removing and retrying" + echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/remove_id > /dev/null || true + echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id > /dev/null + fi +done +# NVSwitches +for idx in "${!SW_BDFS[@]}"; do + BDF=${SW_BDFS[$idx]}; DEV=${SW_DEV_IDS[$idx]} + echo "Tagging NVSwitch $BDF (DevID=10de:$DEV)" + if ! echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id > /dev/null; then + echo " -> failure, removing and retrying" + echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/remove_id > /dev/null || true + echo "10de $DEV" | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id > /dev/null + fi +done diff --git a/scripts/bin/lsproc.py b/scripts/bin/lsproc.py deleted file mode 100755 index d9665cb99..000000000 --- a/scripts/bin/lsproc.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import sys -import os -from pathlib import Path - -def get_process_pids(process_name: str) -> list: - """Get PIDs for the specified process name""" - try: - result = subprocess.run(['pgrep', f'^{process_name}$'], - capture_output=True, text=True) - if result.returncode != 0: - print(f"No process named '{process_name}' found") - sys.exit(1) - return result.stdout.strip().split() - except subprocess.CalledProcessError: - print("Error running pgrep command") - sys.exit(1) - -def get_working_dir(pid: str) -> str: - """Get working directory for a PID""" - try: - result = subprocess.run(['sudo', 'pwdx', pid], - capture_output=True, text=True) - return result.stdout.split(':', 1)[1].strip() - except subprocess.CalledProcessError: - return "N/A" - -def get_socket_count(pid: str) -> int: - """Get socket count for a PID""" - try: - result = subprocess.run(['sudo', 'ls', '-l', f'/proc/{pid}/fd/'], - capture_output=True, text=True) - return result.stdout.count('socket') - except subprocess.CalledProcessError: - return 0 - -def get_nofile_limit(pid: str) -> str: - """Get max open files limit for a PID""" - try: - with open(f'/proc/{pid}/limits', 'r') as f: - for line in f: - if "Max open files" in line: - return line.split()[3] - except (PermissionError, FileNotFoundError): - return "N/A" - return "N/A" - -def get_memory_usage(pid: str) -> str: - """Get memory usage in human readable format""" - try: - result = subprocess.run(['sudo', 'ps', '-p', pid, '-o', 'rss='], - capture_output=True, text=True) - mem_kb = int(result.stdout.strip()) - - if mem_kb >= 1048576: # 1GB = 1048576KB - return f"{mem_kb/1048576:.2f}GB" - elif mem_kb >= 1024: # 1MB = 1024KB - return f"{mem_kb/1024:.2f}MB" - else: - return f"{mem_kb}KB" - except subprocess.CalledProcessError: - return "N/A" - -def main(): - if len(sys.argv) != 2: - print("Usage: script.py ") - sys.exit(1) - - process_name = sys.argv[1] - pids = get_process_pids(process_name) - - # Print header - print(f"{'PID':<10} {'FDs':<10} {'NOFILE':<12} {'MEMORY':<15} {'PATH'}") - print(f"{'---':<10} {'---':<10} {'------':<12} {'-------':<15} {'----'}") - - # Process each PID - for pid in pids: - if not Path(f"/proc/{pid}").exists(): - continue - - work_dir = get_working_dir(pid) - socket_count = get_socket_count(pid) - nofile = get_nofile_limit(pid) - memory = get_memory_usage(pid) - - print(f"{pid:<10} {socket_count:<10} {nofile:<12} {memory:<15} {work_dir}") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/samples/Dockerfile b/scripts/samples/Dockerfile new file mode 100644 index 000000000..0a916aa02 --- /dev/null +++ b/scripts/samples/Dockerfile @@ -0,0 +1,42 @@ +FROM nvidia/cuda:12.4.0-devel-ubuntu20.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +# Add deadsnakes PPA to get Python 3.10 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa + +# Install Python 3.10 and dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.10 \ + python3.10-distutils \ + python3.10-venv \ + curl \ + ca-certificates \ + build-essential \ + cmake \ + git \ + libjpeg-dev \ + libpng-dev \ + vim \ + && rm -rf /var/lib/apt/lists/* + +# Update alternatives to use Python 3.10 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 + +# Install pip for Python 3.10 +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + +# Install Python packages +RUN python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 +RUN python3 -m pip install jupyterlab networkx six + +# Create a working directory +WORKDIR /workspace + +# Expose the port JupyterLab will be accessible on +EXPOSE 8888 + +# Command to start JupyterLab +CMD ["jupyter", "lab", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root"] diff --git a/scripts/samples/docker-compose.yaml b/scripts/samples/docker-compose.yaml index 2798ce386..6eba3a293 100644 --- a/scripts/samples/docker-compose.yaml +++ b/scripts/samples/docker-compose.yaml @@ -1,9 +1,15 @@ -version: '3' - services: - nginx: - image: nginx:latest + jupyter: + image: kvin/cuda-notebook + user: root + privileged: true ports: - - "8080:80" - restart: always - + - "8888:8888" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + runtime: nvidia diff --git a/setup.d/nvidia-layer.sh b/setup.d/nvidia-layer.sh new file mode 100644 index 000000000..46da2ea09 --- /dev/null +++ b/setup.d/nvidia-layer.sh @@ -0,0 +1,2 @@ +LAYERS="$LAYERS \ + $THIS_DIR/meta-nvidia" \ No newline at end of file From 00e00ff0d94f60bba5f1bb1685cf71f0d7a85396 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:13:18 +0000 Subject: [PATCH 206/435] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index afab4c0dc..c6460dc58 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ *.qcow2 __pycache__/ /.target +/.vscode +/.claude From 84a43aa0bc5f1121f327288b3541db226faec7eb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:36:33 +0000 Subject: [PATCH 207/435] Update yocto to whinlatter --- .gitmodules | 14 +++- bitbake | 1 + dev-setup | 6 +- meta-dstack/conf/distro/dstack.conf | 8 ++- meta-dstack/conf/layer.conf | 2 +- .../base-files/base-files%.bbappend | 8 +-- .../recipes-core/docker/docker-moby%.bbappend | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 6 +- ...0001-Update-path-to-native-BaseTools.patch | 19 ++++-- ...ile-adjust-to-build-in-under-bitbake.patch | 25 ++++--- .../dstack-ovmf/0003-Debug-prefix-map.patch | 47 ++++++++----- .../dstack-ovmf/0004-Reproduciable.patch | 67 ++++++++++++------- ...eclare-ProcessLibraryConstructorList.patch | 37 ---------- .../dstack-ovmf/dstack-ovmf_git.bb | 8 +-- .../dstack-zfs/dstack-zfs_2.2.5.bb | 1 - .../recipes-core/images/dstack-initscript.bb | 3 +- .../images/dstack-initscript/init | 0 .../recipes-core/images/dstack-rootfs-dev.inc | 2 +- .../recipes-core/pahole/pahole_1.25.bbappend | 2 - .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 4 +- meta-openembedded | 2 +- meta-rust-bin | 2 +- meta-security | 2 +- meta-virtualization | 2 +- meta-yocto | 1 + mkimage.sh | 2 +- openembedded-core | 1 + poky | 1 - 28 files changed, 148 insertions(+), 127 deletions(-) create mode 160000 bitbake delete mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch mode change 100644 => 100755 meta-dstack/recipes-core/images/dstack-initscript/init delete mode 100644 meta-dstack/recipes-core/pahole/pahole_1.25.bbappend create mode 160000 meta-yocto create mode 160000 openembedded-core delete mode 160000 poky diff --git a/.gitmodules b/.gitmodules index 1a6548412..f80eb534a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,14 @@ -[submodule "poky"] - path = poky - url = https://github.com/yoctoproject/poky.git +[submodule "bitbake"] + path = bitbake + url = https://git.openembedded.org/bitbake +[submodule "openembedded-core"] + path = openembedded-core + url = https://git.openembedded.org/openembedded-core + branch = whinlatter +[submodule "meta-yocto"] + path = meta-yocto + url = https://git.yoctoproject.org/meta-yocto + branch = whinlatter [submodule "meta-confidential-compute"] path = meta-confidential-compute url = https://github.com/Dstack-TEE/meta-confidential-compute.git diff --git a/bitbake b/bitbake new file mode 160000 index 000000000..720df1a53 --- /dev/null +++ b/bitbake @@ -0,0 +1 @@ +Subproject commit 720df1a53452983c1c832f624490e255cf389204 diff --git a/dev-setup b/dev-setup index 376ee34e6..51d3a7a0d 100755 --- a/dev-setup +++ b/dev-setup @@ -36,14 +36,14 @@ if [ ! -f "$BUILD_DIR/conf/local.conf" ]; then cp -f "$THIS_DIR/bb-build/conf/local.conf" "$BUILD_DIR/conf/local.conf" fi -OE_INIT=$THIS_DIR/poky/oe-init-build-env +OE_INIT=$THIS_DIR/openembedded-core/oe-init-build-env for script in $THIS_DIR/setup.d/*.sh; do source "$script" done -pushd . -source $OE_INIT $BUILD_DIR +pushd "$BUILD_DIR" +BDIR="." TEMPLATECONF=$THIS_DIR/meta-yocto/meta-poky/conf/templates/default source $OE_INIT popd bitbake-layers add-layer $LAYERS diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 39042d489..6040b8a53 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -7,8 +7,6 @@ DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.5.5" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" -VOLATILE_LOG_DIR = "no" -VOLATILE_TMP_DIR = "yes" PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" @@ -24,4 +22,8 @@ SERIAL_CONSOLES = "115200;ttyS0" PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" -BAD_RECOMMENDATIONS = "busybox-syslog systemd-extra-utils" +BAD_RECOMMENDATIONS = "busybox-syslog systemd-compat-units" + +# Skip unused components that fail metadata checks under walnascar +SKIP_RECIPE[ostree] = "not required for dstack" + diff --git a/meta-dstack/conf/layer.conf b/meta-dstack/conf/layer.conf index 15ad1890e..7258ad32b 100644 --- a/meta-dstack/conf/layer.conf +++ b/meta-dstack/conf/layer.conf @@ -5,4 +5,4 @@ BBFILE_COLLECTIONS += "dstack" BBFILE_PATTERN_dstack := "^${LAYERDIR}/" BBFILE_PRIORITY_dstack = "20" LAYERVERSION_dstack = "4" -LAYERSERIES_COMPAT_dstack = "scarthgap" +LAYERSERIES_COMPAT_dstack = "whinlatter" diff --git a/meta-dstack/recipes-core/base-files/base-files%.bbappend b/meta-dstack/recipes-core/base-files/base-files%.bbappend index 66848558b..ed7056d5d 100644 --- a/meta-dstack/recipes-core/base-files/base-files%.bbappend +++ b/meta-dstack/recipes-core/base-files/base-files%.bbappend @@ -3,11 +3,11 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" SRC_URI += "file://dstack-motd" do_install:append() { - if [ -f ${WORKDIR}/dstack-motd ];then + if [ -f ${UNPACKDIR}/dstack-motd ];then bbnote "Installing custom dstack motd file" - install -m 0644 ${WORKDIR}/dstack-motd ${D}${sysconfdir}/motd + install -m 0644 ${UNPACKDIR}/dstack-motd ${D}${sysconfdir}/motd else - bbwarn "Custom dstack-motd file not found in ${WORKDIR}" - ls -la ${WORKDIR} + bbwarn "Custom dstack-motd file not found in ${UNPACKDIR}" + ls -la ${UNPACKDIR} fi } diff --git a/meta-dstack/recipes-core/docker/docker-moby%.bbappend b/meta-dstack/recipes-core/docker/docker-moby%.bbappend index 871cfbdf6..8a6e2d028 100644 --- a/meta-dstack/recipes-core/docker/docker-moby%.bbappend +++ b/meta-dstack/recipes-core/docker/docker-moby%.bbappend @@ -8,6 +8,6 @@ FILES:${PN} += "${systemd_system_unitdir}/docker.service.d/override.conf" do_install:append() { if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir}/docker.service.d - install -m 0644 ${WORKDIR}/docker.service.d_override.conf ${D}${systemd_system_unitdir}/docker.service.d/override.conf + install -m 0644 "${UNPACKDIR}/docker.service.d_override.conf" ${D}${systemd_system_unitdir}/docker.service.d/override.conf fi } diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 93f6d6b27..06cab0baa 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -9,7 +9,7 @@ REPO_ROOT = "${THISDIR}/../../.." SRC_DIR = '${REPO_ROOT}/dstack' -S = "${WORKDIR}/dstack" +S = "${UNPACKDIR}/dstack" RDEPENDS:${PN} += "bash" @@ -83,3 +83,7 @@ do_install() { } FILES:${PN} += "${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf" + +# Cargo embeds build paths into binaries; allow TMPDIR references. +INSANE_SKIP:${PN} += "buildpaths" +INSANE_SKIP:${PN}-dbg += "buildpaths" diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch index 01f54b104..890e58621 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch @@ -1,8 +1,16 @@ -From 77c79b5aeabe614ebaa993c6a0e6490798d92059 Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Mon, 4 Nov 2024 04:27:21 +0000 -Subject: [PATCH 1/5] Update path to native BaseTools +From 0f859c6de2914cd984f8a0dd61471b91a2371bfa Mon Sep 17 00:00:00 2001 +From: Ricardo Neri +Date: Thu, 9 Jun 2016 02:23:01 -0700 +Subject: [PATCH] ovmf: update path to native BaseTools +BaseTools is a set of utilities to build EDK-based firmware. These utilities +are used during the build process. Thus, they need to be built natively. +When cross-compiling, we need to provide a path to the location of these +tools. The BBAKE_EDK_TOOLS_PATH string is used as a pattern to be replaced +with the appropriate location before building. + +Signed-off-by: Ricardo Neri +Upstream-Status: Inappropriate [oe-core cross compile specific] --- OvmfPkg/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) @@ -20,6 +28,3 @@ index 279f0d099a..285f061bf4 100755 echo $EDK_TOOLS_PATH source edksetup.sh BaseTools else --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch index 16db65bf4..4a5552857 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch @@ -1,17 +1,27 @@ -From a8f873986e7d817d6f0db2824c2348e5120a1094 Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Mon, 4 Nov 2024 04:27:56 +0000 -Subject: [PATCH 2/5] BaseTools makefile adjust to build in under bitbake +From 9dbd082516fdd7100f99f0f389b8eef5ececab19 Mon Sep 17 00:00:00 2001 +From: Ricardo Neri +Date: Fri, 26 Jul 2019 17:34:26 -0400 +Subject: [PATCH] BaseTools: makefile: adjust to build in under bitbake +Prepend the build flags with those of bitbake. This is to build +using the bitbake native sysroot include and library directories. + +Note from Alex: this is not appropriate for upstream submission as +the recipe already does lots of similar in-place fixups elsewhere, so +this patch shold be converted to follow that pattern. We're not going +to fight against how upstream wants to configure the build. + +Signed-off-by: Ricardo Neri +Upstream-Status: Inappropriate [needs to be converted to in-recipe fixups] --- BaseTools/Source/C/Makefiles/header.makefile | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile -index d369908a09..22c670f316 100644 +index 55a7307a5e..e36e6a8f04 100644 --- a/BaseTools/Source/C/Makefiles/header.makefile +++ b/BaseTools/Source/C/Makefiles/header.makefile -@@ -85,35 +85,34 @@ endif +@@ -154,35 +154,34 @@ endif INCLUDE = $(TOOL_INCLUDE) -I $(MAKEROOT) -I $(MAKEROOT)/Include/Common -I $(MAKEROOT)/Include/ -I $(MAKEROOT)/Include/IndustryStandard -I $(MAKEROOT)/Common/ -I .. -I . $(ARCH_INCLUDE) INCLUDE += -I $(EDK2_PATH)/MdePkg/Include @@ -54,6 +64,3 @@ index d369908a09..22c670f316 100644 ifeq ($(HOST_ARCH), IA32) # # Snow Leopard is a 32-bit and 64-bit environment. uname -m returns i386, but gcc defaults --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch index a793d63f7..1e79c23ae 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch @@ -1,17 +1,31 @@ -From aa8d288279ef96ffe576a0e434c1d9f435fef1e7 Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Mon, 4 Nov 2024 04:29:03 +0000 -Subject: [PATCH 3/5] Debug prefix map +From eb6db21ed0a845dce149d17acf8673d5188adea4 Mon Sep 17 00:00:00 2001 +From: Alexander Kanavin +Date: Mon, 14 Jun 2021 19:56:28 +0200 +Subject: [PATCH] debug prefix map +We want to pass ${DEBUG_PREFIX_MAP} to gcc commands and also pass in + --debug-prefix-map to nasm (we carry a patch to nasm for this). The +tools definitions file is built by ovmf-native so we need to pass this in +at target build time when we know the right values so we use the environment. + +By using determininistc file paths during the ovmf build, it removes the +opportunitity for gcc/ld to change the output binaries due to path lengths +overflowing section sizes and causing small changes in the binary output. +Previously we relied on the stripped output being the same which isn't always +the case if the size of the debug symbols varies. + +Upstream-Status: Submitted [https://github.com/tianocore/edk2/pull/2202] +Signed-off-by: Richard Purdie +Signed-off-by: Alexander Kanavin --- BaseTools/Conf/tools_def.template | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template -index 0f110fbb4a..35205db748 100755 +index 0caa1c9001..dd86b3249d 100755 --- a/BaseTools/Conf/tools_def.template +++ b/BaseTools/Conf/tools_def.template -@@ -896,7 +896,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ +@@ -920,7 +920,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ *_*_*_DTC_PATH = DEF(DTC_BIN) # All supported GCC archs except LOONGARCH64 support -mstack-protector-guard=global, so set that on everything except LOONGARCH64 @@ -20,18 +34,18 @@ index 0f110fbb4a..35205db748 100755 DEFINE GCC_IA32_X64_CC_FLAGS = -mstack-protector-guard=global DEFINE GCC_ARM_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mlittle-endian -mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections -fdata-sections -fomit-frame-pointer -Wno-address -mthumb -fno-pic -fno-pie -mstack-protector-guard=global DEFINE GCC_LOONGARCH64_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mabi=lp64d -fno-asynchronous-unwind-tables -Wno-address -fno-short-enums -fsigned-char -ffunction-sections -fdata-sections -@@ -918,8 +918,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere +@@ -938,8 +938,8 @@ DEFINE GCC_ARM_AARCH64_ASLDLINK_FLAGS = -Wl,--defsym=PECOFF_HEADER_SIZE=0 DEF(GC + DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) DEFINE GCC_AARCH64_ASLDLINK_FLAGS = DEF(GCC_AARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) DEFINE GCC_LOONGARCH64_ASLDLINK_FLAGS = DEF(GCC_LOONGARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) - DEFINE GCC_IA32_X64_DLINK_FLAGS = DEF(GCC_IA32_X64_DLINK_COMMON) --entry _$(IMAGE_ENTRY_POINT) --file-alignment 0x20 --section-alignment 0x20 -Map $(DEST_DIR_DEBUG)/$(BASE_NAME).map -DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h -DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h +DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) +DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) - DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h + DEFINE GCC_VFRPP_FLAGS = -x c -E -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h DEFINE GCC_ASLCC_FLAGS = -x c -@@ -1072,7 +1072,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( +@@ -1092,7 +1092,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( *_GCC48_IA32_DLINK2_FLAGS = DEF(GCC48_IA32_DLINK2_FLAGS) *_GCC48_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC48_IA32_OBJCOPY_FLAGS = @@ -40,7 +54,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1100,7 +1100,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set +@@ -1120,7 +1120,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set *_GCC48_X64_DLINK2_FLAGS = DEF(GCC48_X64_DLINK2_FLAGS) *_GCC48_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC48_X64_OBJCOPY_FLAGS = @@ -49,7 +63,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) RELEASE_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1209,7 +1209,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s +@@ -1229,7 +1229,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s *_GCC49_IA32_DLINK2_FLAGS = DEF(GCC49_IA32_DLINK2_FLAGS) *_GCC49_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC49_IA32_OBJCOPY_FLAGS = @@ -58,7 +72,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1237,7 +1237,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set +@@ -1257,7 +1257,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set *_GCC49_X64_DLINK2_FLAGS = DEF(GCC49_X64_DLINK2_FLAGS) *_GCC49_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC49_X64_OBJCOPY_FLAGS = @@ -67,7 +81,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) RELEASE_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1496,7 +1496,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 +@@ -1516,7 +1516,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 *_GCC5_IA32_DLINK2_FLAGS = DEF(GCC5_IA32_DLINK2_FLAGS) -no-pie *_GCC5_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC5_IA32_OBJCOPY_FLAGS = @@ -76,7 +90,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC5_IA32_CC_FLAGS = DEF(GCC5_IA32_CC_FLAGS) -flto DEBUG_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl,-m,elf_i386,--oformat=elf32-i386 -@@ -1528,7 +1528,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, +@@ -1548,7 +1548,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, *_GCC5_X64_DLINK2_FLAGS = DEF(GCC5_X64_DLINK2_FLAGS) *_GCC5_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC5_X64_OBJCOPY_FLAGS = @@ -85,6 +99,3 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC5_X64_CC_FLAGS = DEF(GCC5_X64_CC_FLAGS) -flto -DUSING_LTO DEBUG_GCC5_X64_DLINK_FLAGS = DEF(GCC5_X64_DLINK_FLAGS) -flto -Os --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch index 7b73762f8..73f242321 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch @@ -1,8 +1,32 @@ -From 81fc61513cf51a9ef6947dcf4bc1388a50c13f42 Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Mon, 4 Nov 2024 04:29:41 +0000 -Subject: [PATCH 4/5] Reproduciable +From 0d22e07cd9cc35ede04c01d3141068db744d7677 Mon Sep 17 00:00:00 2001 +From: Alexander Kanavin +Date: Mon, 14 Jun 2021 19:57:30 +0200 +Subject: [PATCH] reproducible +This patch fixes various things which make the build more reproducible. Some changes +here only change intermediate artefacts but that means when you have two build trees +giving differing results, the differences can be isolated more easily. The issues here +usually become apparent with longer paths. + +This was all debugged with: +TMPDIR = "${TOPDIR}/tmp" +vs. +TMPDIR = "${TOPDIR}/tmp-inital-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath" + +The patch specifically: + + * Sorts output in GNUmakefile + * Always generates indirect flags files used to avoid pathlength issues else the + compile commands suddenly change when using longer paths + * Sorts the AutoGenTimeStamp file contents + * Makes the TargetDescBlock objects from BuildEngine sortable to allow the makefile fix + * Fix ElfConvert within GenFw so that only the basename of the binary being converted + is used, else the output from "GenFw XXX.bin" differs from "GenFw /long/path/XXX.bin" + with sufficiently long paths + +Upstream-Status: Submitted [https://github.com/tianocore/edk2/pull/2176] +Signed-off-by: Richard Purdie +Signed-off-by: Alexander Kanavin --- BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- .../Source/Python/AutoGen/BuildEngine.py | 3 +++ @@ -11,19 +35,19 @@ Subject: [PATCH 4/5] Reproduciable 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c -index 9d04fc612e..83fd6c9c05 100644 +index 18594122ea..92b5f13cf6 100644 --- a/BaseTools/Source/C/GenFw/Elf64Convert.c +++ b/BaseTools/Source/C/GenFw/Elf64Convert.c -@@ -13,6 +13,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent - #ifndef __GNUC__ +@@ -15,6 +15,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent #include + #undef RUNTIME_FUNCTION #include +#else +#define _GNU_SOURCE #endif #include #include -@@ -988,7 +990,7 @@ ScanSections64 ( +@@ -990,7 +992,7 @@ ScanSections64 ( } mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + @@ -32,7 +56,7 @@ index 9d04fc612e..83fd6c9c05 100644 // // Add more space in the .debug data region for the DllCharacteristicsEx -@@ -2299,7 +2301,7 @@ WriteDebug64 ( +@@ -2322,7 +2324,7 @@ WriteDebug64 ( EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; EFI_IMAGE_DEBUG_EX_DLLCHARACTERISTICS_ENTRY *DllEntry; @@ -41,7 +65,7 @@ index 9d04fc612e..83fd6c9c05 100644 NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); DataDir = &NtHdr->Pe32Plus.OptionalHeader.DataDirectory[EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]; -@@ -2332,7 +2334,7 @@ WriteDebug64 ( +@@ -2355,7 +2357,7 @@ WriteDebug64 ( Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); Nb10->Signature = CODEVIEW_SIGNATURE_NB10; @@ -51,7 +75,7 @@ index 9d04fc612e..83fd6c9c05 100644 STATIC diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py -index 45b39d7878..3fed7d1736 100644 +index b829a2503c..afe826b78f 100644 --- a/BaseTools/Source/Python/AutoGen/BuildEngine.py +++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py @@ -70,6 +70,9 @@ class TargetDescBlock(object): @@ -65,10 +89,10 @@ index 45b39d7878..3fed7d1736 100644 if Input not in self.Inputs: self.Inputs.append(Input) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py -index 6d9c60b702..83ac267647 100755 +index e5f282c4ac..27a34ef7c6 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py -@@ -576,7 +576,7 @@ cleanlib: +@@ -571,7 +571,7 @@ cleanlib: os.remove(RespFileList) # convert source files and binary files to build targets @@ -77,7 +101,7 @@ index 6d9c60b702..83ac267647 100755 if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0: EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build", ExtraData="[%s]" % str(MyAgo)) -@@ -727,7 +727,7 @@ cleanlib: +@@ -722,7 +722,7 @@ cleanlib: OutputFile = '' DepsFileList = [] @@ -86,7 +110,7 @@ index 6d9c60b702..83ac267647 100755 if Cmd[2]: for CopyCmd in Cmd[2]: Src, Dst = CopyCmd -@@ -760,7 +760,7 @@ cleanlib: +@@ -755,7 +755,7 @@ cleanlib: self.BuildTargetList.append('\t%s' % CmdString) self.ParseSecCmd(DepsFileList, Cmd[1]) @@ -95,7 +119,7 @@ index 6d9c60b702..83ac267647 100755 self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile))) self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd)) self.FfsOutputFileList = [] -@@ -799,13 +799,13 @@ cleanlib: +@@ -794,13 +794,13 @@ cleanlib: def CommandExceedLimit(self): FlagDict = { @@ -116,7 +140,7 @@ index 6d9c60b702..83ac267647 100755 } RespDict = {} -@@ -1008,9 +1008,9 @@ cleanlib: +@@ -1014,9 +1014,9 @@ cleanlib: if not self.ObjTargetDict.get(T.Target.SubDir): self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) @@ -129,10 +153,10 @@ index 6d9c60b702..83ac267647 100755 if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros: self.FileListMacros[T.FileListMacro] = [] diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py -index 65a2176ca9..318b9611ce 100755 +index aa0b71632e..57c9cad117 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py -@@ -1474,6 +1474,9 @@ class ModuleAutoGen(AutoGen): +@@ -1429,6 +1429,9 @@ class ModuleAutoGen(AutoGen): for File in Files: if File.lower().endswith('.pdb'): AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) @@ -142,7 +166,7 @@ index 65a2176ca9..318b9611ce 100755 HeaderComments = self.Module.HeaderComments StartPos = 0 for Index in range(len(HeaderComments)): -@@ -1749,7 +1752,7 @@ class ModuleAutoGen(AutoGen): +@@ -1704,7 +1707,7 @@ class ModuleAutoGen(AutoGen): if os.path.exists (self.TimeStampPath): os.remove (self.TimeStampPath) @@ -151,6 +175,3 @@ index 65a2176ca9..318b9611ce 100755 # Ignore generating makefile when it is a binary module if self.IsBinaryModule: --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch deleted file mode 100644 index 12a1c8cfc..000000000 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 596f2e3bd7292c8f22cfe01ec95b8b46ce1bc8a1 Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Mon, 4 Nov 2024 04:35:18 +0000 -Subject: [PATCH] Declare ProcessLibraryConstructorList - ---- - OvmfPkg/IntelTdx/Sec/SecMain.c | 1 + - OvmfPkg/Sec/SecMain.c | 1 + - 2 files changed, 2 insertions(+) - -diff --git a/OvmfPkg/IntelTdx/Sec/SecMain.c b/OvmfPkg/IntelTdx/Sec/SecMain.c -index 7f2d28af95..fcc6b97c28 100644 ---- a/OvmfPkg/IntelTdx/Sec/SecMain.c -+++ b/OvmfPkg/IntelTdx/Sec/SecMain.c -@@ -163,6 +163,7 @@ SecCoreStartupWithStack ( - IdtDescriptor.Base = (UINTN)&IdtTableInStack.IdtTable; - IdtDescriptor.Limit = (UINT16)(sizeof (IdtTableInStack.IdtTable) - 1); - -+ void ProcessLibraryConstructorList (); - ProcessLibraryConstructorList (); - - // -diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c -index d13a948509..58c48c967c 100644 ---- a/OvmfPkg/Sec/SecMain.c -+++ b/OvmfPkg/Sec/SecMain.c -@@ -885,6 +885,7 @@ SecCoreStartupWithStack ( - InitializeCpuExceptionHandlers (NULL); - } - -+ void ProcessLibraryConstructorList (); - ProcessLibraryConstructorList (); - - if (!SevEsIsEnabled ()) { --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 96fcda3c0..0cf1e1028 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -19,16 +19,15 @@ PACKAGECONFIG[tpm] = "-D TPM_ENABLE=TRUE,-D TPM_ENABLE=FALSE,," #see https://src.fedoraproject.org/rpms/edk2/blob/rawhide/f/0032-Basetools-turn-off-gcc12-warning.patch BUILD_CFLAGS += "-Wno-error=stringop-overflow" -SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ +SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https;tag=${PV} \ file://0001-Update-path-to-native-BaseTools.patch \ file://0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch \ file://0003-Debug-prefix-map.patch \ file://0004-Reproduciable.patch \ - file://0005-Declare-ProcessLibraryConstructorList.patch \ " -PV = "edk2-3a3b12cb" -SRCREV = "3a3b12cbdae2e89b0e365eb01c378891d0d9037c" +PV = "edk2-stable202508" +SRCREV = "d46aa46c8361194521391aa581593e556c707c6e" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" CVE_PRODUCT = "edk2" @@ -49,7 +48,6 @@ inherit deploy PARALLEL_MAKE = "" -S = "${WORKDIR}/git" DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb index 1d4e512c7..cf5074428 100644 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb @@ -11,7 +11,6 @@ SRC_URI = "git://github.com/openzfs/zfs;protocol=https;branch=zfs-2.2-release \ file://0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch \ " -S = "${WORKDIR}/git" # Using both 'module' and 'autotools' classes seems a bit odd, they both # define a do_compile function. diff --git a/meta-dstack/recipes-core/images/dstack-initscript.bb b/meta-dstack/recipes-core/images/dstack-initscript.bb index ee32b1aa6..1d0f7c612 100644 --- a/meta-dstack/recipes-core/images/dstack-initscript.bb +++ b/meta-dstack/recipes-core/images/dstack-initscript.bb @@ -9,7 +9,8 @@ FILES:${PN} = "*" inherit allarch -S = "${WORKDIR}" +UNPACKDIR = "${WORKDIR}/sources" +S = "${UNPACKDIR}" do_install() { install -d ${D}/ diff --git a/meta-dstack/recipes-core/images/dstack-initscript/init b/meta-dstack/recipes-core/images/dstack-initscript/init old mode 100644 new mode 100755 diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index 1996e0756..eacc68bf7 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,4 @@ include dstack-rootfs-base.inc IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver vim" -EXTRA_IMAGE_FEATURES += "debug-tweaks tools-profile" +EXTRA_IMAGE_FEATURES += "allow-root-login post-install-logging" diff --git a/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend b/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend deleted file mode 100644 index 9ebd86051..000000000 --- a/meta-dstack/recipes-core/pahole/pahole_1.25.bbappend +++ /dev/null @@ -1,2 +0,0 @@ -# Update pahole to 1.29 for BTF reproducible build -SRCREV = "b9cc7963051b2099795129450f9b70c81950d02f" diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb index 7f943d78c..e134f3e47 100644 --- a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb +++ b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb @@ -11,6 +11,8 @@ SRC_DIR = '${REPO_ROOT}/dstack/mod-tdx-guest' SRC_URI = 'file://${REPO_ROOT}/dstack/mod-tdx-guest' SRCREV = "${DSTACK_SRC_REV}" -S = "${WORKDIR}/${SRC_DIR}" +S = "${UNPACKDIR}/${SRC_DIR}" RPROVIDES:${PN} += "tdx-guest-ko" +INSANE_SKIP:${PN} += "buildpaths" +INSANE_SKIP:${PN}-dbg += "buildpaths" diff --git a/meta-openembedded b/meta-openembedded index 72018ca1b..a4e768dcf 160000 --- a/meta-openembedded +++ b/meta-openembedded @@ -1 +1 @@ -Subproject commit 72018ca1b1a471226917e8246e8bbf9a374ccf97 +Subproject commit a4e768dcfae181dd99e39c4286fa0729ca7f4e47 diff --git a/meta-rust-bin b/meta-rust-bin index 53974894f..678a792ed 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit 53974894fe1bf0fba26d0f6d5ded32b3c3f4a43e +Subproject commit 678a792edb50aceb491bd74bcb6d4d6cc79c8b63 diff --git a/meta-security b/meta-security index bc63d9574..a54650c74 160000 --- a/meta-security +++ b/meta-security @@ -1 +1 @@ -Subproject commit bc63d95746ef4f0ac6820165e5041d83647e8c9c +Subproject commit a54650c743b7534f60fc6ee1df9b57a8f0150a4d diff --git a/meta-virtualization b/meta-virtualization index bce9962c5..26e2b40b9 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit bce9962c515089d62082953b795c0e80083c96e2 +Subproject commit 26e2b40b91f2424b0b9318b50dbb700a67714b6f diff --git a/meta-yocto b/meta-yocto new file mode 160000 index 000000000..d02d3faaf --- /dev/null +++ b/meta-yocto @@ -0,0 +1 @@ +Subproject commit d02d3faaf4d6075ea03e9eb47654ec7639f929a0 diff --git a/mkimage.sh b/mkimage.sh index 9f986409d..c000b0741 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -46,7 +46,7 @@ VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAM echo "Loading verity env from ${VERITY_ENV_FILE}" source ${VERITY_ENV_FILE} -DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION) +DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} diff --git a/openembedded-core b/openembedded-core new file mode 160000 index 000000000..851997859 --- /dev/null +++ b/openembedded-core @@ -0,0 +1 @@ +Subproject commit 8519978592483bb096ed5192fff7af6c887b799e diff --git a/poky b/poky deleted file mode 160000 index cd44e6bd4..000000000 --- a/poky +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd44e6bd40b0c1f498b3feaeb5e9b72f8bf32d41 From c7e3d5217fe5cee1a1821375c595239c1a9ec17d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:10:44 +0000 Subject: [PATCH 208/435] Adding minimal stdio.h stub to satisfy libgcc-initial configure --- .../gcc/libgcc-initial_%.bbappend | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend diff --git a/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend b/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend new file mode 100644 index 000000000..c006d6efb --- /dev/null +++ b/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend @@ -0,0 +1,19 @@ +python dstack_stub_stdio() { + import os + staging_incdir = d.getVar('STAGING_INCDIR') + stdio = os.path.join(staging_incdir, 'stdio.h') + if not os.path.exists(stdio): + bb.note("Adding minimal stdio.h stub to satisfy libgcc-initial configure") + with open(stdio, 'w') as fh: + fh.write('#ifndef __YOCTO_DUMMY_STDIO__\n') + fh.write('#define __YOCTO_DUMMY_STDIO__\n') + fh.write('typedef int FILE;\n') + fh.write('extern FILE *stdin;\n') + fh.write('extern FILE *stdout;\n') + fh.write('extern FILE *stderr;\n') + fh.write('static inline int printf(const char *fmt, ...) { (void)fmt; return 0; }\n') + fh.write('#endif\n') +} + +do_configure[prefuncs] += "dstack_stub_stdio" +DEBUG_FLAGS = "" From 400b67cf6f2430c0d164db1cd7723e713f1470ec Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:06:25 +0000 Subject: [PATCH 209/435] systemd handle volatile deps --- .../systemd/systemd/blacklist-autofs4.conf | 3 --- .../recipes-core/systemd/systemd_%.bbappend | 17 ++++++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) delete mode 100644 meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf diff --git a/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf b/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf deleted file mode 100644 index 0b88b129f..000000000 --- a/meta-dstack/recipes-core/systemd/systemd/blacklist-autofs4.conf +++ /dev/null @@ -1,3 +0,0 @@ -# Disable autofs4 module loading -blacklist autofs4 -install autofs4 /bin/true diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index 8c49ea4ec..f73be27d0 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -1,16 +1,15 @@ -FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" - -SRC_URI += "file://blacklist-autofs4.conf" - do_install:append() { # Disable systemd-vconsole-setup.service rm -f ${D}${systemd_system_unitdir}/sysinit.target.wants/systemd-vconsole-setup.service - # Install modprobe blacklist for autofs4 - install -d ${D}${sysconfdir}/modprobe.d - install -m 0644 ${WORKDIR}/blacklist-autofs4.conf ${D}${sysconfdir}/modprobe.d/ + # Ensure systemd-resolved waits for /var/volatile tmpfs and tmpfiles setup + install -d ${D}${systemd_system_unitdir}/systemd-resolved.service.d + cat <<'EOF' > ${D}${systemd_system_unitdir}/systemd-resolved.service.d/10-var-volatile.conf +[Unit] +After=systemd-tmpfiles-setup.service var-volatile.mount +Requires=var-volatile.mount +EOF } SYSTEMD_SERVICE:${PN}-vconsole-setup = "" - -FILES:${PN} += "${sysconfdir}/modprobe.d/blacklist-autofs4.conf" +PACKAGECONFIG:remove = "sysvinit logind" From 2e5695898afdee90e33ddedd267a9ca236f62421 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:05:37 +0000 Subject: [PATCH 210/435] Volatile /etc/ --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4f4acc1bc..1acf99a68 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -142,11 +142,4 @@ mkdirs() { mkdir -p ${IMAGE_ROOTFS}/etc/wireguard mkdir -p ${IMAGE_ROOTFS}/var/lib/docker ln -sf dstack ${IMAGE_ROOTFS}/tapp - - # Aliases passwd files to a subdirectory - mkdir -p ${IMAGE_ROOTFS}/etc/users - mv ${IMAGE_ROOTFS}/etc/passwd ${IMAGE_ROOTFS}/etc/users/ - mv ${IMAGE_ROOTFS}/etc/shadow ${IMAGE_ROOTFS}/etc/users/ - ln -s users/passwd ${IMAGE_ROOTFS}/etc/passwd - ln -s users/shadow ${IMAGE_ROOTFS}/etc/shadow } From dd579e0e40cc29e872eaf7e72bc565bd8c95b489 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:05:09 +0000 Subject: [PATCH 211/435] Rust 1.92.0 --- meta-dstack/conf/distro/dstack.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 6040b8a53..71a028e2b 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -19,8 +19,8 @@ VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" SERIAL_CONSOLES = "115200;ttyS0" -PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" -PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.92.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" BAD_RECOMMENDATIONS = "busybox-syslog systemd-compat-units" From ce97ff87e8683d045d2d5c12f70cdbb96e9d8d93 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:54:50 +0000 Subject: [PATCH 212/435] Suppress the warn Text file busy in build.sh --- build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 054c59f34..1e1d207ad 100755 --- a/build.sh +++ b/build.sh @@ -113,7 +113,10 @@ EOF build_host() { echo "Building binaries" (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) - cp ${RUST_BUILD_DIR}/release/{dstack-gateway,dstack-kms,dstack-vmm,supervisor} . + for bin in dstack-gateway dstack-kms dstack-vmm supervisor; do + cp "${RUST_BUILD_DIR}/release/${bin}" ".${bin}.new" + mv -f ".${bin}.new" "./${bin}" + done } # Step 2: build guest images From d583c53b13d3c2e8d65216025f73e02a54258ff8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:54:32 +0000 Subject: [PATCH 213/435] Remove CF_ZONE_ID --- build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/build.sh b/build.sh index 1e1d207ad..a70ce8538 100755 --- a/build.sh +++ b/build.sh @@ -85,7 +85,6 @@ GATEWAY_PUBLIC_DOMAIN= # for certbot CERTBOT_ENABLED=false CF_API_TOKEN= -CF_ZONE_ID= ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory EOF if [ -f $CONFIG_FILE ]; then @@ -224,8 +223,6 @@ workdir = "$CERBOT_WORKDIR" acme_url = "$ACME_URL" # Cloudflare API token cf_api_token = "$CF_API_TOKEN" -# Cloudflare zone ID -cf_zone_id = "$CF_ZONE_ID" # Auto set CAA record auto_set_caa = true # Domain to issue certificates for From afe57867c474fd7467f17a31c6dd2157e5aaaa17 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:42:11 +0000 Subject: [PATCH 214/435] Rename DStack to dstack --- README.md | 4 ++-- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-core/base-files/files/dstack-motd | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 2 +- .../recipes-core/images/dstack-initscript.bb | 2 +- .../linux/files/dstack-docker.scc | 2 +- .../recipes-kernel/linux/files/dstack-tdx.scc | 2 +- .../recipes-kernel/linux/files/dstack.scc | 2 +- scripts/bin/dstack.py | 18 +++++++++--------- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 65790d7dc..74c6b7cc2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Yocto support for DStack Guest +# Yocto support for dstack guest OS -This project implements Yocto layer and the overall build scripts for DStack Base OS image. +This project implements Yocto layer and the overall build scripts for dstack Base OS image. ## Build diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 71a028e2b..0cbe1368c 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,6 +1,6 @@ require conf/distro/cvm.conf DISTRO = "dstack" -DISTRO_NAME = "DStack" +DISTRO_NAME = "dstack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" diff --git a/meta-dstack/recipes-core/base-files/files/dstack-motd b/meta-dstack/recipes-core/base-files/files/dstack-motd index 047731e70..5376caf7e 100644 --- a/meta-dstack/recipes-core/base-files/files/dstack-motd +++ b/meta-dstack/recipes-core/base-files/files/dstack-motd @@ -5,4 +5,4 @@ ██████╔╝███████║ ██║ ██║ ██║╚██████╗██║ ██╗ ██║ ███████╗███████╗ ╚██████╔╝███████║ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝ ╚═════╝ ╚══════╝ -Welcome to Dstack! +Welcome to dstack! diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 06cab0baa..4c6e90363 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -1,4 +1,4 @@ -SUMMARY = "Guest binaries for DStack, a decentralized computing stack" +SUMMARY = "Guest binaries for dstack, a decentralized computing stack" DESCRIPTION = "${SUMMARY}" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" diff --git a/meta-dstack/recipes-core/images/dstack-initscript.bb b/meta-dstack/recipes-core/images/dstack-initscript.bb index 1d0f7c612..61908cc0c 100644 --- a/meta-dstack/recipes-core/images/dstack-initscript.bb +++ b/meta-dstack/recipes-core/images/dstack-initscript.bb @@ -1,4 +1,4 @@ -SUMMARY = "Dstack initramfs files" +SUMMARY = "dstack initramfs files" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc b/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc index 40bf76dec..09dda98d0 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc @@ -1,3 +1,3 @@ -define KFEATURE_DESCRIPTION "DStack Docker configuration" +define KFEATURE_DESCRIPTION "dstack Docker configuration" kconf non-hardware dstack-docker.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc index e7a9aba53..5786a660e 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc +++ b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc @@ -1,3 +1,3 @@ -define KFEATURE_DESCRIPTION "DStack TDX configuration" +define KFEATURE_DESCRIPTION "dstack TDX configuration" kconf non-hardware dstack-tdx.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.scc b/meta-dstack/recipes-kernel/linux/files/dstack.scc index 4725b8084..e277ce0b2 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.scc +++ b/meta-dstack/recipes-kernel/linux/files/dstack.scc @@ -1,3 +1,3 @@ -define KFEATURE_DESCRIPTION "DStack configuration" +define KFEATURE_DESCRIPTION "dstack configuration" kconf non-hardware dstack.cfg diff --git a/scripts/bin/dstack.py b/scripts/bin/dstack.py index 5274ba465..73c810d34 100755 --- a/scripts/bin/dstack.py +++ b/scripts/bin/dstack.py @@ -144,14 +144,14 @@ def gen_vm_config(vm_dir, host_port, manifest=None, os_image_hash=None): @dataclass -class DStackConfig: - """Configuration for DStack client.""" +class DstackConfig: + """Configuration for dstack client.""" docker_registry: Optional[str] = None default_image_name: str = '' qemu_path: str = 'qemu-system-x86_64' @classmethod - def load(cls) -> 'DStackConfig': + def load(cls) -> 'DstackConfig': """Load configuration from file.""" cfgs = load_configs_merged(generate_config_paths()) @@ -167,10 +167,10 @@ def cfg_get(section, key, fallback): return me -class DStackManager: +class DstackManager: def __init__(self): self.run_path = os.path.abspath(os.getenv('RUN_PATH', './vms')) - self.config = DStackConfig.load() + self.config = DstackConfig.load() def _generate_instance_id(self) -> str: """Generate a random instance ID.""" @@ -365,7 +365,7 @@ def resolve_gpus(gpus: dict) -> dict: case 'listed': return gpus case 'all': - return DStackManager.collect_all_gpus() + return DstackManager.collect_all_gpus() case _: raise ValueError( f"Invalid GPU attach mode: {gpus['attach_mode']}") @@ -797,7 +797,7 @@ def write_to_sysfs(path, value): def main(): - parser = argparse.ArgumentParser(description='DStack VM Management Tool') + parser = argparse.ArgumentParser(description='dstack VM Management Tool') subparsers = parser.add_subparsers(dest='command', help='Commands') # Setup command @@ -849,10 +849,10 @@ def main(): args = parser.parse_args() if args.command == 'new': - manager = DStackManager() + manager = DstackManager() manager.setup_instance(args) elif args.command == 'run': - manager = DStackManager() + manager = DstackManager() thread = start_server(args.dir, args.kp_port) manager.run_instance(args.dir, thread.host_port, imgdir=args.imgdir, dry_run=args.dry_run) From 3c0e2a01ae8e2baf6202caa1bbc6456f12eaf70b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:42:44 +0000 Subject: [PATCH 215/435] More stable network in bb build --- bb-build/conf/local.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index b87c10a2a..a51d8c24b 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -157,6 +157,9 @@ DISTRO ?= "dstack" # - 'buildstats' collect build statistics USER_CLASSES ?= "buildstats" +# Prefer faster GNU mirror +GNU_MIRRORS = "https://ftpmirror.gnu.org/gnu/" + # # Runtime testing of images # @@ -286,3 +289,5 @@ BB_DISKMON_DIRS ??= "\ # track the version of this file when it was generated. This can safely be ignored if # this doesn't mean anything to you. CONF_VERSION = "2" + +FETCHCMD_wget = "wget --progress=dot --inet4-only -c" \ No newline at end of file From 8fd64a5ec31dd13a72d75ed885961962bdd714cd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:49:51 +0000 Subject: [PATCH 216/435] Use custom kernel 6.17 --- meta-dstack/conf/distro/dstack.conf | 4 +- .../images/dstack-rootfs-base.inc | 1 - ...0001-x86-tdx-select-dma-direct-remap.patch | 35 ++ .../recipes-kernel/linux/files/6.17/defconfig | 359 ++++++++++++++++++ .../linux/linux-custom_6.17.6.bb | 36 ++ 5 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch create mode 100644 meta-dstack/recipes-kernel/linux/files/6.17/defconfig create mode 100644 meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 0cbe1368c..9613503bb 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -8,8 +8,8 @@ DISTRO_VERSION = "0.5.5" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" -PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-dev" -PREFERRED_VERSION_linux-yocto-dev ?= "6.9%" +PREFERRED_PROVIDER_virtual/kernel = "linux-custom" +PREFERRED_VERSION_linux-custom ?= "6.17.6" LINUX_KERNEL_TYPE = "tiny" MACHINE_FEATURES += "numa" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 1acf99a68..657ab271c 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -12,7 +12,6 @@ IMAGE_INSTALL = "\ iptables \ docker-moby \ docker-compose \ - tdx-guest-ko \ dstack-guest \ wireguard-tools \ cryptsetup \ diff --git a/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch b/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch new file mode 100644 index 000000000..662dd175c --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch @@ -0,0 +1,35 @@ +From 48cf4656e9a23f7f5b4a2f6a0e5b5b2b6d5e5b6d Mon Sep 17 00:00:00 2001 +From: h4x +Date: Mon, 27 Oct 2025 23:52:02 +0000 +Subject: [PATCH] x86/tdx: select DMA_DIRECT_REMAP for encrypted guests + +TDX guests require DMA buffers to be mapped out of the shared (decrypted) +pool in order for shared devices such as NVMe to complete I/O. Without +DMA_DIRECT_REMAP the dma-direct layer falls back to allocations from the +regular page allocator, leaving the memory encrypted and causing every +request to wedge once it hits the host. + +Ensure the TDX guest configuration pulls in DMA_DIRECT_REMAP, which in +turn enables the coherent DMA pool machinery and honours the +`coherent_pool=` kernel parameter. + +Upstream-Status: Inappropriate [TDX guest specific selection] +Signed-off-by: h4x +--- + arch/x86/Kconfig | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index 5d57930be347..c1b55a4bf4f0 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -886,6 +886,7 @@ config INTEL_TDX_GUEST + depends on EFI_STUB + depends on PARAVIRT + select ARCH_HAS_CC_PLATFORM ++ select DMA_DIRECT_REMAP + select X86_MEM_ENCRYPT + select X86_MCE + select UNACCEPTED_MEMORY +-- +2.46.0 diff --git a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig new file mode 100644 index 000000000..7c03fce39 --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig @@ -0,0 +1,359 @@ +CONFIG_LOCALVERSION="-dstack" +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZMA=y +CONFIG_KERNEL_GZIP=y +CONFIG_DMA_COHERENT_POOL=y +CONFIG_DMA_DIRECT_REMAP=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_PREEMPT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CGROUPS=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_SCHED=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_DEBUG=y +CONFIG_USER_NS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_SYSFS_SYSCALL=y +CONFIG_SMP=y +CONFIG_X86_AMD_PLATFORM_DEVICE=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT_SPINLOCKS=y +CONFIG_INTEL_TDX_GUEST=y +CONFIG_TDX_GUEST_DRIVER=y +CONFIG_TSM_REPORTS=y +CONFIG_CONFIGFS_FS=y +CONFIG_NR_CPUS=512 +CONFIG_X86_MSR=y +CONFIG_X86_CPUID=y +CONFIG_X86_CHECK_BIOS_CORRUPTION=y +CONFIG_EFI=y +CONFIG_EFI_STUB=y +CONFIG_HZ_1000=y +# CONFIG_SUSPEND is not set +# CONFIG_ACPI_DEBUG is not set +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_X86_ACPI_CPUFREQ=y +CONFIG_INTEL_IDLE=y +CONFIG_IA32_EMULATION=y +CONFIG_KPROBES=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_XFRM_USER=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_INET_ESP=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NETFILTER_XTABLES_COMPAT=y +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_VS=m +CONFIG_NF_REJECT_IPV4=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=y +CONFIG_6LOWPAN=m +CONFIG_NET_SCHED=y +CONFIG_VSOCKETS=y +CONFIG_VIRTIO_VSOCKETS=y +CONFIG_HYPERV_VSOCKETS=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_CFG80211=m +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_PCI_HYPERV=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_DMA_DIRECT_REMAP=y +CONFIG_DMA_COHERENT_POOL=y +CONFIG_CMA=y +CONFIG_CMA_SIZE_SEL_MBYTES=y +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +CONFIG_CMA_SIZE_MBYTES=64 +CONFIG_DMA_CMA=y +CONFIG_GVE=y +CONFIG_SWIOTLB_DYNAMIC=y +CONFIG_ZONE_DMA=y +CONFIG_CONNECTOR=y +CONFIG_PARPORT=m +CONFIG_PARPORT_PC=m +CONFIG_BLK_DEV_FD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=1 +CONFIG_BLK_DEV_RAM_SIZE=6144 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_EEPROM_AT24=m +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_FC_ATTRS=y +CONFIG_SCSI_VIRTIO=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_ATA_PIIX=y +CONFIG_PATA_SCH=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_NETDEVICES=y +CONFIG_WIREGUARD=y +CONFIG_IPVLAN=m +CONFIG_VXLAN=y +CONFIG_NETCONSOLE=y +CONFIG_TUN=m +CONFIG_VETH=y +CONFIG_VIRTIO_NET=y +CONFIG_TYPHOON=m +CONFIG_PCNET32=m +CONFIG_ATL2=m +CONFIG_ATL1=m +CONFIG_ATL1E=m +CONFIG_ATL1C=m +CONFIG_ALX=m +CONFIG_TIGON3=m +CONFIG_BNX2X=m +CONFIG_BNXT=m +CONFIG_DL2K=m +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IGC=m +CONFIG_JME=m +CONFIG_SKGE=m +CONFIG_SKY2=m +CONFIG_8139CP=m +CONFIG_8139TOO=m +CONFIG_R8169=m +CONFIG_REALTEK_PHY_HWMON=y +CONFIG_USB_NET_DRIVERS=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_USBNET=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_KC2190=y +# CONFIG_WLAN is not set +CONFIG_HYPERV_NET=y +CONFIG_ISDN=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_TABLET=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_PRINTER=m +CONFIG_VIRTIO_CONSOLE=y +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_HPET=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TPM2_HMAC=y +CONFIG_TCG_TIS=y +CONFIG_TCG_CRB=y +CONFIG_I2C_I801=y +CONFIG_WATCHDOG=y +CONFIG_BCMA=m +CONFIG_BCMA_DRIVER_GMAC_CMN=y +CONFIG_AGP=y +CONFIG_AGP_INTEL=y +CONFIG_DRM=y +CONFIG_DRM_I915=m +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_DRM_CIRRUS_QEMU=m +CONFIG_FB=y +CONFIG_FB_UVESA=m +CONFIG_FB_EFI=y +CONFIG_FB_HYPERV=y +CONFIG_HID_HYPERV_MOUSE=y +CONFIG_HID_WACOM=y +CONFIG_USB_HIDDEV=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_SERIAL=y +CONFIG_USB_SERIAL_CONSOLE=y +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_FTDI_SIO=y +CONFIG_USB_SERIAL_PL2303=y +CONFIG_USB_EZUSB_FX2=y +CONFIG_RTC_CLASS=y +CONFIG_VIRT_DRIVERS=y +CONFIG_EFI_SECRET=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=y +CONFIG_HYPERV=y +CONFIG_HYPERV_UTILS=y +CONFIG_HYPERV_BALLOON=y +CONFIG_INTEL_IOMMU=y +# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set +CONFIG_IRQ_REMAP=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_XFS_FS=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_RT=y +CONFIG_BTRFS_FS=m +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_FUSE_FS=m +CONFIG_CUSE=m +CONFIG_OVERLAY_FS=y +CONFIG_ISO9660_FS=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_EFIVAR_FS=y +CONFIG_SQUASHFS=y +CONFIG_VXFS_FS=m +CONFIG_NFS_FS=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +# CONFIG_NFS_DISABLE_UDP_SUPPORT is not set +CONFIG_9P_FS=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_AES_TI=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_ARC4=y +CONFIG_CRYPTO_CTS=y +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_DEV_VIRTIO=y +CONFIG_CORDIC=m +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_DEBUG_INFO_BTF=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_UNWINDER_FRAME_POINTER=y + +CONFIG_FAT_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y diff --git a/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb b/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb new file mode 100644 index 000000000..200982db1 --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb @@ -0,0 +1,36 @@ +SUMMARY = "dstack Linux kernel 6.17.6 built from tarball" +DESCRIPTION = "Custom dstack kernel based on upstream Linux 6.17.6 with tiny Kconfig baseline tuned for TDX guests" +SECTION = "kernel" +LICENSE = "GPL-2.0-only" +LIC_FILES_CHKSUM = "file://COPYING;md5=6bc538ed5bd9a7fc9398086aedcd7e46" + +PV = "6.17.6" +LINUX_VERSION = "${PV}" + +inherit kernel + +FILESEXTRAPATHS:prepend := "${THISDIR}/files/6.17:${THISDIR}/files:" + +DEPENDS += "libyaml-native openssl-native util-linux-native elfutils-native" + +SRC_URI = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${PV}.tar.xz;downloadfilename=linux-${PV}.tar.xz \ + file://defconfig \ + file://0001-x86-tdx-select-dma-direct-remap.patch \ +" + +SRC_URI[sha256sum] = "8ecfbc6b693448abb46144a8d04d1e1631639c7661c1088425a2e5406f13c69c" + +S = "${UNPACKDIR}/linux-${PV}" + +LINUX_VERSION_EXTENSION = "-dstack" +KERNEL_VERSION_EXTENSION = "-dstack" + +# Enable BTF debug info for bpftool and out-of-tree modules (ZFS, WireGuard, etc.) +KERNEL_DEBUG = "True" + +# Keep packaging aligned with our tiny x86_64 guest machines. +COMPATIBLE_MACHINE = "(tdx|sev-snp|qemux86-64)" + +do_deploy:append() { + install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config +} From dd731b0822a313160521dcb151b21509d0868d9c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:53:28 +0000 Subject: [PATCH 217/435] Make GCP image --- Makefile | 2 +- meta-dstack/conf/distro/dstack.conf | 3 + .../images/dstack-initscript/init | 134 ++++++++++---- .../images/dstack-rootfs-base.inc | 2 + meta-dstack/recipes-core/images/dstack-uki.bb | 133 ++++++++++++++ .../gptfdisk/gptfdisk_%.bbappend | 2 + mkimage.sh | 170 +++++++++++++++++- scripts/bin/authenticode_hash.py | 138 ++++++++++++++ 8 files changed, 549 insertions(+), 35 deletions(-) create mode 100644 meta-dstack/recipes-core/images/dstack-uki.bb create mode 100644 meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend create mode 100644 scripts/bin/authenticode_hash.py diff --git a/Makefile b/Makefile index 2d0a05329..700cdb1dd 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ dist: images $(foreach dist_name,${DIST_NAMES},./mkimage.sh --dist-name $(dist_name);) images: - bitbake virtual/kernel dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) + bitbake virtual/kernel dstack-initramfs dstack-ovmf dstack-uki $(ROOTFS_IMAGE_NAMES) clean: bitbake -c cleansstate virtual/kernel dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 9613503bb..ec6e8ff41 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -27,3 +27,6 @@ BAD_RECOMMENDATIONS = "busybox-syslog systemd-compat-units" # Skip unused components that fail metadata checks under walnascar SKIP_RECIPE[ostree] = "not required for dstack" +# EFI/UKI support for GCP images +MACHINE_FEATURES:append = " efi" +EFI_PROVIDER = "systemd-boot" diff --git a/meta-dstack/recipes-core/images/dstack-initscript/init b/meta-dstack/recipes-core/images/dstack-initscript/init index fba1182a4..968671f60 100755 --- a/meta-dstack/recipes-core/images/dstack-initscript/init +++ b/meta-dstack/recipes-core/images/dstack-initscript/init @@ -1,52 +1,120 @@ #!/bin/sh set -e -export PATH=/sbin:/bin:/usr/sbin:/usr/bin +ROOT_DIR=/root +ROOT_DEV="" + +log() { + printf '[init] %s\n' "$*" >&2 +} + +mount_move_all() { + for dir in "$@"; do + mount --move "/$dir" "${ROOT_DIR}/$dir" + done +} + +find_device_by_partlabel() { + label=$1 + + log "searching for PARTLABEL=${label}" + + for entry in /sys/class/block/*; do + [ -f "$entry/partition" ] || continue + uevent="${entry}/uevent" + if [ -f "$uevent" ]; then + current=$(grep -E '^PARTNAME=' "$uevent" 2>/dev/null | head -n1 | cut -d= -f2) + if [ "${current}" = "$label" ]; then + device="/dev/$(basename "$entry")" + if [ -b "$device" ]; then + log "sysfs matched PARTLABEL=${label} at ${device}" + realpath "$device" 2>/dev/null || echo "$device" + return 0 + fi + fi + fi + done + + log "no device found for PARTLABEL=${label}" + return 1 +} + +resolve_root_device() { + if [ -z "$ROOTFS_DEVICE" ]; then + ROOTFS_DEVICE="PARTLABEL=dstack-rootfs" + fi + + if [ "${ROOTFS_DEVICE#PARTLABEL=}" != "$ROOTFS_DEVICE" ]; then + label="${ROOTFS_DEVICE#PARTLABEL=}" + if [ -n "$label" ]; then + device=$(find_device_by_partlabel "$label") + if [ -n "$device" ]; then + log "determined root device via kernel parameter: ${device}" + ROOT_DEV="$device" + return + fi + fi + elif [ "${ROOTFS_DEVICE#/dev/}" != "$ROOTFS_DEVICE" ]; then + ROOT_DEV="$ROOTFS_DEVICE" + return + fi + + log "unable to determine root device from available heuristics" +} -mkdir -p /proc -mkdir -p /sys -mkdir -p /run -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t tmpfs tmpfs /run +main() { + export PATH=/sbin:/bin:/usr/sbin:/usr/bin -mkdir -p /dev -mount -t devtmpfs devtmpfs /dev + mkdir -p /proc + mkdir -p /sys + mkdir -p /run + mount -t proc proc /proc + mount -t sysfs sysfs /sys + mount -t tmpfs tmpfs /run -mkdir -p /dev/pts + mkdir -p /dev + mount -t devtmpfs devtmpfs /dev -# Parse kernel parameters -for param in $(cat /proc/cmdline); do - case "$param" in + mkdir -p /dev/pts + + log "booting dstack initramfs" + + ROOTFS_DEVICE="" + for param in $(cat /proc/cmdline); do + case "$param" in "dstack.rootfs_hash="*) ROOT_HASH="${param#*=}" ;; "dstack.rootfs_size="*) DATA_SIZE="${param#*=}" ;; - *) + "dstack.rootfs_device="*) + ROOTFS_DEVICE="${param#*=}" ;; - esac -done -ROOT_DEV=/dev/vda -ROOT_DIR=/root + *) ;; + esac + done -echo "Setting up verity device:" -echo " Root device: ${ROOT_DEV}" -echo " Root hash: ${ROOT_HASH}" -echo " Data size: ${DATA_SIZE}" + resolve_root_device -veritysetup open ${ROOT_DEV} rootfs ${ROOT_DEV} "${ROOT_HASH}" --hash-offset="${DATA_SIZE}" + if [ -z "$ROOT_DEV" ]; then + log "failed to determine root device" + exit 1 + fi + log "resolved root device: ${ROOT_DEV}" -echo "Mounting rootfs..." -mount -t squashfs /dev/mapper/rootfs ${ROOT_DIR} - -mount_move_all() { - for dir in $@; do - mount --move /$dir ${ROOT_DIR}/$dir - done + if [ -z "${ROOT_HASH}" ] || [ -z "${DATA_SIZE}" ]; then + log "missing dm-verity parameters; refusing to continue" + exit 1 + fi + veritysetup open "${ROOT_DEV}" rootfs "${ROOT_DEV}" "${ROOT_HASH}" --hash-offset="${DATA_SIZE}" + log "mounting verified rootfs from /dev/mapper/rootfs" + mount -t squashfs /dev/mapper/rootfs ${ROOT_DIR} + mount_move_all sys proc dev run + log "switching root to ${ROOT_DIR}" + exec switch_root ${ROOT_DIR} /sbin/init + log "switch_root failed" + exit 1 } -mount_move_all sys proc dev run -echo "Switching to new root..." -exec switch_root ${ROOT_DIR} /sbin/init +main "$@" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 657ab271c..8c94032ec 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -31,6 +31,8 @@ IMAGE_INSTALL = "\ xfsprogs \ e2fsprogs \ e2fsprogs-resize2fs \ + gptfdisk \ + parted \ " IMAGE_NAME_SUFFIX ?= "" diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/meta-dstack/recipes-core/images/dstack-uki.bb new file mode 100644 index 000000000..c9883bae2 --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-uki.bb @@ -0,0 +1,133 @@ +# Unified Kernel Image (UKI) for dstack +# +# This recipe generates a UKI containing kernel, initramfs, and cmdline +# with dm-verity root hash for GCP deployment. + +SUMMARY = "dstack Unified Kernel Image" +LICENSE = "MIT" + +DEPENDS = "systemd-boot systemd-boot-native virtual/kernel python3-pefile-native" + +inherit image-artifact-names +require conf/image-uefi.conf + +# Initramfs settings +INITRAMFS_IMAGE = "dstack-initramfs" +INITRAMFS_FSTYPES = "cpio.gz" + +# Kernel settings +KERNEL_IMAGETYPE = "bzImage" + +# Output filename +UKI_FILENAME = "dstack-uki.efi" + +# Base kernel cmdline (verity hash added dynamically) +UKI_CMDLINE_BASE = "console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0 \ +mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n \ +tsc=reliable no-kvmclock" + +# Verity image to get hash from +# Override with VERITY_IMAGE = "dstack-dev-rootfs" for dev builds +VERITY_IMAGE ?= "dstack-rootfs" +VERITY_TYPE = "squashfs" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" +do_install[noexec] = "1" + +# Dependencies +do_uki[depends] += "systemd-boot:do_deploy virtual/kernel:do_deploy" +do_uki[depends] += "${INITRAMFS_IMAGE}:do_image_complete" +do_uki[depends] += "${VERITY_IMAGE}:do_image_complete" + +python do_uki() { + import os + import bb.process + + deploy_dir = d.getVar('DEPLOY_DIR_IMAGE') + target_arch = d.getVar('EFI_ARCH') + + # Find the EFI stub + stub = os.path.join(deploy_dir, f"linux{target_arch}.efi.stub") + if not os.path.exists(stub): + bb.fatal(f"EFI stub not found: {stub}") + + # Find kernel + kernel = os.path.join(deploy_dir, d.getVar('KERNEL_IMAGETYPE')) + if not os.path.exists(kernel): + bb.fatal(f"Kernel not found: {kernel}") + + # Find initramfs + initramfs_image = d.getVar('INITRAMFS_IMAGE') + machine = d.getVar('MACHINE') + initramfs_fstypes = d.getVar('INITRAMFS_FSTYPES') + initrd = os.path.join(deploy_dir, f"{initramfs_image}-{machine}.{initramfs_fstypes}") + if not os.path.exists(initrd): + bb.fatal(f"Initramfs not found: {initrd}") + + # Read verity hash + staging_verity_dir = d.getVar('STAGING_VERITY_DIR') or d.expand('${TMPDIR}/work-shared/${MACHINE}/dm-verity') + verity_image = d.getVar('VERITY_IMAGE') + verity_type = d.getVar('VERITY_TYPE') + verity_env = os.path.join(staging_verity_dir, f"{verity_image}.{verity_type}.verity.env") + + root_hash = "" + data_size = "" + + if os.path.exists(verity_env): + with open(verity_env, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('ROOT_HASH='): + root_hash = line.split('=', 1)[1] + elif line.startswith('DATA_SIZE='): + data_size = line.split('=', 1)[1] + bb.note(f"Read verity env: root_hash={root_hash}, data_size={data_size}") + else: + bb.fatal(f"Verity env file not found: {verity_env}") + + # Build cmdline + cmdline_base = d.getVar('UKI_CMDLINE_BASE') + cmdline = f"{cmdline_base} dstack.rootfs_hash={root_hash} dstack.rootfs_size={data_size}" + bb.note(f"UKI cmdline: {cmdline}") + + # Output path + output = os.path.join(deploy_dir, d.getVar('UKI_FILENAME')) + + # Build ukify command with proper Python paths + native_sysroot = d.getVar('RECIPE_SYSROOT_NATIVE') + staging_libdir = d.getVar('STAGING_LIBDIR_NATIVE') + + # Find Python version directory for native packages + python_sitepackages = os.path.join(staging_libdir, 'python3.13', 'site-packages') + + # Set environment for ukify + env = os.environ.copy() + env['PYTHONPATH'] = python_sitepackages + + ukify_path = os.path.join(native_sysroot, 'usr', 'bin', 'ukify') + ukify_cmd = f"{ukify_path} build" + ukify_cmd += f" --efi-arch {target_arch}" + ukify_cmd += f" --stub {stub}" + ukify_cmd += f" --linux={kernel}" + ukify_cmd += f" --initrd={initrd}" + ukify_cmd += f" --cmdline='{cmdline}'" + ukify_cmd += f" --tools={native_sysroot}/usr/lib/systemd/tools" + ukify_cmd += f" --output={output}" + + bb.note(f"Running: {ukify_cmd}") + bb.note(f"PYTHONPATH: {python_sitepackages}") + + import subprocess + result = subprocess.run(ukify_cmd, shell=True, capture_output=True, text=True, env=env) + if result.stdout: + bb.note(result.stdout) + if result.stderr: + bb.note(result.stderr) + if result.returncode != 0: + bb.fatal(f"ukify failed with exit code {result.returncode}") + + bb.note(f"UKI created: {output}") +} + +addtask uki before do_build diff --git a/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend b/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend new file mode 100644 index 000000000..af3122b28 --- /dev/null +++ b/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend @@ -0,0 +1,2 @@ +# Disable ncurses/cgdisk to avoid linking against libncursesw (not in our images) +PACKAGECONFIG:remove = "ncurses" diff --git a/mkimage.sh b/mkimage.sh index c000b0741..0028a4697 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -2,6 +2,7 @@ set -e DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} +ENABLE_GCP_IMAGE=${ENABLE_GCP_IMAGE:-1} # Parse command line arguments while [ $# -gt 0 ]; do @@ -41,6 +42,7 @@ INITRAMFS_IMAGE=${IMG_DIR}/dstack-initramfs.cpio.gz ROOTFS_IMAGE=${IMG_DIR}/${ROOTFS_IMAGE_NAME}-tdx.squashfs.verity KERNEL_IMAGE=${IMG_DIR}/bzImage OVMF_FIRMWARE=${IMG_DIR}/ovmf.fd +UKI_IMAGE=${IMG_DIR}/dstack-uki.efi # Always use the work-shared directory which has the correct verity env VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAME}.squashfs.verity.env echo "Loading verity env from ${VERITY_ENV_FILE}" @@ -50,11 +52,153 @@ DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} +AUTHENTICODE_HASH_SCRIPT="${BB_BUILD_DIR}/../scripts/bin/authenticode_hash.py" + verbose() { echo "$@" $@ } +align_up() { + local value=$1 + local align=$2 + echo $(( ( (value + align - 1) / align ) * align )) +} + +calc_authenticode_hash() { + local file="$1" + + if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then + return 0 + fi + + python3 "$AUTHENTICODE_HASH_SCRIPT" "$file" 2>/dev/null || true +} + +write_authenticode_hash_if_missing() { + local file="$1" + local out_file="${file}.auth_hash.txt" + + if [[ -f "$out_file" ]]; then + return 0 + fi + + if [[ ! -f "$file" ]]; then + return 0 + fi + + if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then + echo "Warning: authenticode_hash.py not found or python3 not available, skipping Authenticode hash calculation" >&2 + return 0 + fi + + echo "Calculating UKI Authenticode hash..." + local auth_hash + auth_hash=$(calc_authenticode_hash "$file") + if [[ -n "$auth_hash" ]]; then + echo "$auth_hash" > "$out_file" + echo "UKI Authenticode hash: $auth_hash" + else + echo "Warning: Failed to calculate UKI Authenticode hash" >&2 + fi +} + +create_uki_bootstrap() { + local target_dir="$1" + mkdir -p "$target_dir/EFI/BOOT" + + local uki_file="${UKI_IMAGE}" + if [[ ! -f "$uki_file" ]]; then + echo "Error: UKI image not found: $uki_file" >&2 + return 1 + fi + cp "$uki_file" "$target_dir/EFI/BOOT/BOOTX64.EFI" + + # Calculate Authenticode hash if not already present + write_authenticode_hash_if_missing "$uki_file" +} + +create_partitioned_rootfs() { + local rootfs_img="$1" + local output_img="$2" + ( + set -e + local align=$((1024 * 1024)) + local sector=512 + local rootfs_size=$(stat -c %s "$rootfs_img") + local rootfs_size_aligned=$(align_up $rootfs_size $align) + local rootfs_start=$align + # Leave extra room for GPT headers (1MB at start, 1MB at end) + local total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) $align) + + truncate -s $total_size "$output_img" + + local root_start_sector=$((rootfs_start / sector)) + local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) + + sgdisk --zap-all "$output_img" >/dev/null + sgdisk --new=1:${root_start_sector}:${root_end_sector} --typecode=1:8300 --change-name=1:'dstack-rootfs' "$output_img" >/dev/null + + dd if="$rootfs_img" of="$output_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none + ) +} + +build_gcp_disk_image() { + local disk_img="$1" + local boot_source="$2" + local rootfs_img="$3" + ( + set -e + local align=$((1024 * 1024)) + local sector=512 + local efi_size=$((256 * 1024 * 1024)) + local efi_size_aligned=$(align_up $efi_size $align) + local rootfs_size=$(stat -c %s "$rootfs_img") + local rootfs_size_aligned=$(align_up $rootfs_size $align) + local efi_start=$align + local rootfs_start=$((efi_start + efi_size_aligned)) + # Leave extra room for the backup GPT header + local total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) $align) + + truncate -s $total_size "$disk_img" + + local efi_start_sector=$((efi_start / sector)) + local efi_end_sector=$((efi_start_sector + (efi_size_aligned / sector) - 1)) + local root_start_sector=$((rootfs_start / sector)) + local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) + + sgdisk --zap-all "$disk_img" >/dev/null + sgdisk --new=1:${efi_start_sector}:${efi_end_sector} --typecode=1:ef00 --change-name=1:'EFI System Partition' "$disk_img" >/dev/null + sgdisk --new=2:${root_start_sector}:${root_end_sector} --typecode=2:8300 --change-name=2:'dstack-rootfs' "$disk_img" >/dev/null + + local tmp_dir + tmp_dir=$(mktemp -d) + trap 'rm -rf "$tmp_dir"' EXIT + local efi_img=${tmp_dir}/efi.img + mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null + (cd "$boot_source" && mcopy -s -i "$efi_img" ./* ::) >/dev/null + + dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none + dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none + ) +} + +create_gcp_artifacts() { + local gcp_dir="${OUTPUT_DIR}/gcp" + local boot_src="${gcp_dir}/efi-root" + mkdir -p "$boot_src" + echo "Installing UKI as EFI bootloader at ${boot_src}" + create_uki_bootstrap "$boot_src" + + local disk_img="${gcp_dir}/disk.raw" + echo "Building raw disk image for GCP at ${disk_img}" + build_gcp_disk_image "$disk_img" "$boot_src" "${OUTPUT_DIR}/rootfs.img.verity" + + local tarball="${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-gcp.tar.gz" + echo "Archiving GCP disk image to ${tarball}" + (cd "$gcp_dir" && tar -czvf "$tarball" disk.raw) +} + Q=verbose $Q rm -rf ${OUTPUT_DIR}/ @@ -64,6 +208,14 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity +# Copy UKI Authenticode hash if available +if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then + $Q cp "${UKI_IMAGE}.auth_hash.txt" "${OUTPUT_DIR}/dstack-uki.efi.auth_hash.txt" +fi + +echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" +create_partitioned_rootfs "${OUTPUT_DIR}/rootfs.img.verity" "${OUTPUT_DIR}/rootfs.img.parted.verity" + GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" @@ -77,7 +229,7 @@ cat < ${OUTPUT_DIR}/metadata.json "kernel": "bzImage", "cmdline": "$KARG0 $KARG1 $KARG2", "initrd": "initramfs.cpio.gz", - "rootfs": "rootfs.img.verity", + "rootfs": "rootfs.img.parted.verity", "version": "$DSTACK_VERSION", "git_revision": "$GIT_REVISION", "shared_ro": true, @@ -91,6 +243,22 @@ sha256sum ovmf.fd bzImage initramfs.cpio.gz metadata.json > sha256sum.txt sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd +if [ "$ENABLE_GCP_IMAGE" = "1" ]; then + if [[ ! -f "$UKI_IMAGE" ]]; then + echo "Skipping GCP disk image creation because UKI image not found: $UKI_IMAGE" >&2 + echo "Run 'bitbake dstack-uki' to build the UKI first" >&2 + elif command -v sgdisk >/dev/null && \ + command -v mkfs.vfat >/dev/null && \ + command -v mcopy >/dev/null; then + create_gcp_artifacts + else + echo "Error: cannot create GCP disk image because required tools are missing" >&2 + echo "Missing tools are among: sgdisk (gdisk), mkfs.vfat (dosfstools), mcopy (mtools)" >&2 + echo "Install them (e.g. apt-get install -y gdisk dosfstools mtools) or set ENABLE_GCP_IMAGE=0" >&2 + exit 1 + fi +fi + if [ x$DSTACK_TAR_RELEASE = x1 ]; then IMAGE_TAR_MR=${DIST_DIR}/mr_$(cat ${OUTPUT_DIR}/digest.txt | tr -d '\n').tar.gz IMAGE_TAR_NO_ROOTFS=${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-mr.tar.gz diff --git a/scripts/bin/authenticode_hash.py b/scripts/bin/authenticode_hash.py new file mode 100644 index 000000000..de7096fb1 --- /dev/null +++ b/scripts/bin/authenticode_hash.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib + + +def read_le_u16(data: bytes, offset: int) -> int: + import struct + + return struct.unpack(' int: + import struct + + return struct.unpack(' str: + with open(filepath, 'rb') as f: + data = f.read() + + # Read DOS header + lfanew_offset = 0x3C + lfanew = read_le_u32(data, lfanew_offset) + + # Verify PE signature + pe_sig_offset = lfanew + pe_sig = read_le_u32(data, pe_sig_offset) + IMAGE_NT_SIGNATURE = 0x00004550 # "PE\0\0" + if pe_sig != IMAGE_NT_SIGNATURE: + raise ValueError(f"Invalid PE signature in {filepath}") + + # Read COFF header + coff_header_offset = pe_sig_offset + 4 + optional_header_size = read_le_u16(data, coff_header_offset + 16) + + # Read Optional header magic + optional_header_offset = coff_header_offset + 20 + magic = read_le_u16(data, optional_header_offset) + + is_pe32_plus = (magic == 0x20B) + + # Calculate offsets for excluded regions (checksum and cert directory) + checksum_offset = optional_header_offset + 64 + checksum_end = checksum_offset + 4 + + data_dir_offset = optional_header_offset + (112 if is_pe32_plus else 96) + IMAGE_DIRECTORY_ENTRY_SECURITY = 4 + cert_dir_offset = data_dir_offset + (IMAGE_DIRECTORY_ENTRY_SECURITY * 8) + cert_dir_end = cert_dir_offset + 8 + + size_of_headers_offset = optional_header_offset + 60 + size_of_headers = read_le_u32(data, size_of_headers_offset) + + # Hash header (excluding checksum and cert directory) + hasher = hashlib.sha256() + hasher.update(data[0:checksum_offset]) + hasher.update(data[checksum_end:cert_dir_offset]) + hasher.update(data[cert_dir_end:size_of_headers]) + + sum_of_bytes_hashed = size_of_headers + + # Read section table + num_sections_offset = coff_header_offset + 2 + num_sections = read_le_u16(data, num_sections_offset) + + section_table_offset = optional_header_offset + optional_header_size + section_size = 40 + + sections = [] + for i in range(num_sections): + section_offset = section_table_offset + (i * section_size) + + ptr_raw_data_offset = section_offset + 20 + ptr_raw_data = read_le_u32(data, ptr_raw_data_offset) + + size_raw_data_offset = section_offset + 16 + size_raw_data = read_le_u32(data, size_raw_data_offset) + + if size_raw_data > 0: + sections.append((ptr_raw_data, size_raw_data)) + + # Sort sections by offset + sections.sort(key=lambda x: x[0]) + + # Hash sections + for offset, size in sections: + start = offset + end = start + size + + if end <= len(data): + hasher.update(data[start:end]) + else: + available_size = max(0, len(data) - start) + if available_size > 0: + hasher.update(data[start:start + available_size]) + + sum_of_bytes_hashed += size + + file_size = len(data) + + # Read certificate table info + cert_table_addr = read_le_u32(data, cert_dir_offset) + cert_table_size = read_le_u32(data, cert_dir_offset + 4) + + # Hash trailing data (excluding certificate table) + if cert_table_addr > 0 and cert_table_size > 0 and file_size > sum_of_bytes_hashed: + trailing_data_len = file_size - sum_of_bytes_hashed + + if trailing_data_len > cert_table_size: + hashed_trailing_len = trailing_data_len - cert_table_size + trailing_start = sum_of_bytes_hashed + + if trailing_start + hashed_trailing_len <= len(data): + hasher.update(data[trailing_start:trailing_start + hashed_trailing_len]) + + # Add padding to align to 8 bytes + remainder = file_size % 8 + if remainder != 0: + padding = bytes([0] * (8 - remainder)) + hasher.update(padding) + + return hasher.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Calculate PE/COFF Authenticode SHA256 hash (TPM Event Log compatible)' + ) + parser.add_argument('file', help='Path to PE/COFF binary (e.g., UKI .efi)') + args = parser.parse_args() + + print(authenticode_hash(args.file)) + + +if __name__ == '__main__': + main() From 624131a5bd5ba03df0ef97e23e1a4be665d6e1d2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:56:17 +0000 Subject: [PATCH 218/435] Add tpm-tools and lib --- dev-setup | 1 + meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 2 +- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev-setup b/dev-setup index 51d3a7a0d..976e79b5d 100755 --- a/dev-setup +++ b/dev-setup @@ -24,6 +24,7 @@ LAYERS="$THIS_DIR/meta-confidential-compute \ $THIS_DIR/meta-virtualization \ $THIS_DIR/meta-rust-bin \ $THIS_DIR/meta-security \ + $THIS_DIR//meta-security/meta-tpm \ $THIS_DIR/meta-dstack" if [ -z "$1" ]; then diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index ec6e8ff41..8314bd5f5 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,7 +1,7 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "dstack" -DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" +DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6 tpm2" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.5.5" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 4c6e90363..7b1769f1c 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -13,7 +13,7 @@ S = "${UNPACKDIR}/dstack" RDEPENDS:${PN} += "bash" -DEPENDS += "rsync-native" +DEPENDS += "rsync-native libtss2-dev" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 8c94032ec..435210800 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -33,6 +33,7 @@ IMAGE_INSTALL = "\ e2fsprogs-resize2fs \ gptfdisk \ parted \ + tpm2-tools \ " IMAGE_NAME_SUFFIX ?= "" From f1ce52757bca0fceb2f81c9bcf29366d37260973 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:56:34 +0000 Subject: [PATCH 219/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 7bf1843a8..72ffe344d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 7bf1843a8ddf877fbaebb9898a7f27a19a49ab73 +Subproject commit 72ffe344dacf2316c7e4a42803eda2af4e3cd15b From 0aa78d457d03d42dd69e278880defac530924f99 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:04:26 +0000 Subject: [PATCH 220/435] Update zfs to 2.4.0 --- ...eadmmap-Replace-uint_t-with-uint32_t.patch | 46 --------------- ...8a4630af60496c9d33db1d06a7d7d8983422.patch | 56 ------------------- ...stack-zfs_2.2.5.bb => dstack-zfs_2.4.0.bb} | 6 +- 3 files changed, 2 insertions(+), 106 deletions(-) delete mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch delete mode 100644 meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch rename meta-dstack/recipes-core/dstack-zfs/{dstack-zfs_2.2.5.bb => dstack-zfs_2.4.0.bb} (92%) diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch deleted file mode 100644 index f1cfab4da..000000000 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch +++ /dev/null @@ -1,46 +0,0 @@ -From 1f9a5cb860b3509791e59a8cae9d5f265e832ed0 Mon Sep 17 00:00:00 2001 -From: Khem Raj -Date: Sun, 28 May 2023 16:33:15 -0700 -Subject: [PATCH] fs-tests/cmd/readmmap: Replace uint_t with uint32_t - -Makes it portable across glibc and musl - -Upstream-Status: Pending -Signed-off-by: Khem Raj ---- - tests/zfs-tests/cmd/readmmap.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/tests/zfs-tests/cmd/readmmap.c b/tests/zfs-tests/cmd/readmmap.c -index 704ffd55c8..a2590e0e8d 100644 ---- a/tests/zfs-tests/cmd/readmmap.c -+++ b/tests/zfs-tests/cmd/readmmap.c -@@ -38,6 +38,7 @@ - * 0 : no errors - * -------------------------------------------------------------- - */ -+#include - #include - #include - #include -@@ -55,7 +56,7 @@ main(int argc, char **argv) - char *buf = NULL; - char *map = NULL; - int fd = -1, bytes, retval = 0; -- uint_t seed; -+ uint32_t seed; - - if (argc < 2 || optind == argc) { - (void) fprintf(stderr, -@@ -92,7 +93,7 @@ main(int argc, char **argv) - retval = 1; - goto end; - } -- seed = (uint_t)time(NULL); -+ seed = (uint32_t)time(NULL); - srandom(seed); - - idx = random() % size; --- -2.40.1 - diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch deleted file mode 100644 index f5504b389..000000000 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/aaf28a4630af60496c9d33db1d06a7d7d8983422.patch +++ /dev/null @@ -1,56 +0,0 @@ -From aaf28a4630af60496c9d33db1d06a7d7d8983422 Mon Sep 17 00:00:00 2001 -From: Sebastian Gottschall -Date: Tue, 23 May 2023 13:50:24 +0600 -Subject: [PATCH] fixes broken aarch64 inline assembly for gcc 13.1 - -fixes the following error - -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:549:24: error: the register specified for 'w7' is not general enough to be used as a register variable - 549 | register unsigned char w7 asm("v7") __attribute__((vector_size(16))); - | ^~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:193:9: note: in expansion of macro 'GEN_X_DEFINE_6_7' - 193 | GEN_X_DEFINE_6_7() \ - | ^~~~~~~~~~~~~~~~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' - 1360 | REC_PQR_DEFINE(); - | ^~~~~~~~~~~~~~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:551:24: error: the register specified for 'w8' is not general enough to be used as a register variable - 551 | register unsigned char w8 asm("v8") __attribute__((vector_size(16))); \ - | ^~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:194:9: note: in expansion of macro 'GEN_X_DEFINE_8_9' - 194 | GEN_X_DEFINE_8_9() \ - | ^~~~~~~~~~~~~~~~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' - 1360 | REC_PQR_DEFINE(); - | ^~~~~~~~~~~~~~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neon_common.h:552:24: error: the register specified for 'w9' is not general enough to be used as a register variable - 552 | register unsigned char w9 asm("v9") __attribute__((vector_size(16))); - | ^~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_aarch64_neonx2.c:194:9: note: in expansion of macro 'GEN_X_DEFINE_8_9' - 194 | GEN_X_DEFINE_8_9() \ - | ^~~~~~~~~~~~~~~~ -/mnt/b/yoe/master/build/tmp/work/imx8qm_var_som-yoe-linux/zfs/2.1.9-r0/build/../zfs-2.1.9/module/zfs/vdev_raidz_math_impl.h:1360:9: note: in expansion of macro 'REC_PQR_DEFINE' - 1360 | REC_PQR_DEFINE(); - -Upstream-Status: Pending [https://github.com/BrainSlayer/zfs/commit/aaf28a4630af60496c9d33db1d06a7d7d8983422] -Signed-off-by: Sebastian Gottschall -Signed-off-by: Khem Raj ---- - module/Kbuild.in | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - ---- a/module/Kbuild.in -+++ b/module/Kbuild.in -@@ -57,9 +57,9 @@ asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_ - ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) - - ifeq ($(CONFIG_ARM64),y) --CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o += -mgeneral-regs-only --CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o += -mgeneral-regs-only --CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o += -mgeneral-regs-only -+CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o = -mgeneral-regs-only -+CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o = -mgeneral-regs-only -+CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o = -mgeneral-regs-only - endif - - # Suppress unused-value warnings in sparc64 architecture headers diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb similarity index 92% rename from meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb rename to meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb index cf5074428..d06893378 100644 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.2.5.bb +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb @@ -4,11 +4,9 @@ LICENSE = "CDDL-1.0" LIC_FILES_CHKSUM = "file://LICENSE;md5=7087caaf1dc8a2856585619f4a787faa" HOMEPAGE ="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/openzfs/zfs" -SRCREV = "33174af15112ed5c53299da2d28e763b0163f428" -SRC_URI = "git://github.com/openzfs/zfs;protocol=https;branch=zfs-2.2-release \ +SRCREV = "743334913e5a5f60baf287bcc6d8a23515b02ac5" +SRC_URI = "git://github.com/openzfs/zfs;protocol=https;branch=zfs-2.4-release \ file://0001-Define-strndupa-if-it-does-not-exist.patch \ - file://aaf28a4630af60496c9d33db1d06a7d7d8983422.patch \ - file://0001-fs-tests-cmd-readmmap-Replace-uint_t-with-uint32_t.patch \ " From 92e7e548ad91af666e3a1288927d54f03dd06de5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:04:44 +0000 Subject: [PATCH 221/435] v0.6.0 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 8314bd5f5..4294ff8b9 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "dstack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6 tpm2" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.5" +DISTRO_VERSION = "0.6.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From 2193b47c27959499894b616b2745143b30b9b0d3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:08:53 +0000 Subject: [PATCH 222/435] ssh: Disable password auth --- .../openssh/files/disable-password-auth.conf | 8 ++++++++ .../recipes-connectivity/openssh/openssh_%.bbappend | 10 ++++++++++ 2 files changed, 18 insertions(+) create mode 100644 meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf create mode 100644 meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend diff --git a/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf b/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf new file mode 100644 index 000000000..47963f278 --- /dev/null +++ b/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf @@ -0,0 +1,8 @@ +# Disable all password-based authentication +PasswordAuthentication no +PermitEmptyPasswords no +KbdInteractiveAuthentication no + +# Only allow public key authentication +PubkeyAuthentication yes +PermitRootLogin prohibit-password diff --git a/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend new file mode 100644 index 000000000..81a2e54ab --- /dev/null +++ b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend @@ -0,0 +1,10 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI += "file://disable-password-auth.conf" + +do_install:append() { + install -d ${D}${sysconfdir}/ssh/sshd_config.d + install -m 0644 ${UNPACKDIR}/disable-password-auth.conf ${D}${sysconfdir}/ssh/sshd_config.d/ +} + +FILES:${PN}-sshd += "${sysconfdir}/ssh/sshd_config.d/" From 88ebc919fa1af596897e9d3686e403f48f12787c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:14:35 +0000 Subject: [PATCH 223/435] Update meta-confidential-compute --- meta-confidential-compute | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-confidential-compute b/meta-confidential-compute index 6d1355e0c..a9857da07 160000 --- a/meta-confidential-compute +++ b/meta-confidential-compute @@ -1 +1 @@ -Subproject commit 6d1355e0c684f7fdfb688ecfef0ed63feb2aac4c +Subproject commit a9857da070b542dece7dd0630b3b89804234af7e From 29f2555bed30f54e6d70884d3f251c316af9dd15 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:30:32 +0000 Subject: [PATCH 224/435] Use patched meta-rust-bin --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index f80eb534a..415c27ab0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ url = https://github.com/openembedded/meta-openembedded [submodule "meta-rust-bin"] path = meta-rust-bin - url = https://github.com/rust-embedded/meta-rust-bin + url = https://github.com/Dstack-TEE/meta-rust-bin [submodule "dstack"] path = dstack url = https://github.com/Dstack-TEE/dstack From 9b5f221fd9e380218e2a992ff2a2b03a5cdfc827 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:42:51 +0000 Subject: [PATCH 225/435] Update meta-virtualization --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index 26e2b40b9..1c64117a5 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit 26e2b40b91f2424b0b9318b50dbb700a67714b6f +Subproject commit 1c64117a5e30dbb8330e8899cae8ac722b9ce553 From 78d1eb494112b2578c288c7fa6a11cf7f3178d57 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 03:35:47 +0000 Subject: [PATCH 226/435] fixup zfs --- meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb index d06893378..fc2361daf 100644 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb @@ -66,6 +66,9 @@ FILES:${PN}-dev += "\ ${prefix}/src/zfs-${PV} \ ${prefix}/src/spl-${PV} \ " +# Skip buildpaths QA check for kernel modules +INSANE_SKIP:${PN} += "buildpaths" + # Not yet ported to rv32 COMPATIBLE_HOST:riscv32 = "null" # conflicting definition of ABS macro from asm/asm.h from kernel From ed9905484dd3a77efb0368e1f3087198f57b1e9a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 03:37:27 +0000 Subject: [PATCH 227/435] Fix QA errors in meta-virtualization --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index 1c64117a5..ebb9f7cdd 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit 1c64117a5e30dbb8330e8899cae8ac722b9ce553 +Subproject commit ebb9f7cdd5475efd0fdac840840334376db98fbb From 5a5e27e1fc1debe04940c6de5aa3ac270fa78938 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 03:38:38 +0000 Subject: [PATCH 228/435] Update nvidia receipes --- meta-nvidia/conf/layer.conf | 2 +- .../containerd-config_1.0.0.bb | 2 +- .../libnvidia-container.inc | 18 ++-- .../libnvidia-container/0001-build-fix.patch | 2 + .../libnvidia-container/0002-secomp-fix.patch | 2 + ...-fix-remove-buildpath-for-package-qa.patch | 2 + .../libnvidia-container_1.00.bb | 10 +++ .../libnvidia-container/libtirpc134_1.3.4.bb | 6 +- .../nvidia-container-toolkit.inc | 7 +- .../nvidia-container-toolkit_1.00.bb | 2 +- ...172.08.bb => libnvidia-nscq_580.105.08.bb} | 10 +-- .../nvidia/nvidia-fabricmanager_570.172.08.bb | 86 ------------------- .../nvidia/nvidia-fabricmanager_580.105.08.bb | 64 ++++++++++++++ .../nvidia/nvidia-modprobe-config_1.0.bb | 2 +- .../nvidia/nvidia-persistenced_1.0.bb | 2 +- ...dia_570.172.08.bb => nvidia_580.105.08.bb} | 5 +- 16 files changed, 102 insertions(+), 120 deletions(-) rename meta-nvidia/recipes-graphics/nvidia/{libnvidia-nscq_570.172.08.bb => libnvidia-nscq_580.105.08.bb} (76%) delete mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb rename meta-nvidia/recipes-graphics/nvidia/{nvidia_570.172.08.bb => nvidia_580.105.08.bb} (75%) diff --git a/meta-nvidia/conf/layer.conf b/meta-nvidia/conf/layer.conf index 8b179fa57..568d6d5b2 100644 --- a/meta-nvidia/conf/layer.conf +++ b/meta-nvidia/conf/layer.conf @@ -10,4 +10,4 @@ BBFILE_PATTERN_nvidia = "^${LAYERDIR}/" BBFILE_PRIORITY_nvidia = "12" LICENSE_PATH += " ${LAYERDIR}/custom-licenses" -LAYERSERIES_COMPAT_nvidia = "mickledore scarthgap" +LAYERSERIES_COMPAT_nvidia = "whinlatter" diff --git a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb index f3c094bc4..2bc9f8509 100644 --- a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb +++ b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb @@ -6,7 +6,7 @@ SRC_URI = "file://config.toml" do_install:append() { install -d ${D}${sysconfdir}/containerd - install -m 0644 ${WORKDIR}/config.toml ${D}${sysconfdir}/containerd/ + install -m 0644 ${UNPACKDIR}/config.toml ${D}${sysconfdir}/containerd/ } RDEPENDS:${PN}:append = " containerd-opencontainers" diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc index f93a1ac6b..46550957b 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc @@ -5,26 +5,27 @@ LICENSE = "Apache-2.0" LIC_FILES_CHKSUM = "\ file://LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57 \ - file://deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION}/modprobe-utils/nvidia-modprobe-utils.c;endline=22;md5=b6a3106a81660c726888d006853ada63 \ + file://deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION}/modprobe-utils/nvidia-modprobe-utils.c;endline=22;md5=2ee73384f2a7a7e246230f8a2e2d4b30 \ file://deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION}/modprobe-utils/pci-enum.h;endline=29;md5=ca948b6fabc48e616fccbf17247feebf \ file://deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION}/modprobe-utils/pci-sysfs.c;endline=25;md5=0449248350efd54938e7f8d25af965cb \ " ELF_TOOLCHAIN_VERSION = "0.7.1" -NVIDIA_MODPROBE_VERSION = "550.40.07" +NVIDIA_MODPROBE_VERSION = "580.105.08" LIBTIRPC_VERSION = "1.3.4" -SRC_URI = "git://github.com/NVIDIA/libnvidia-container.git;protocol=https;name=libnvidia;branch=release-1.14 \ - git://github.com/NVIDIA/nvidia-modprobe.git;protocol=https;branch=main;name=modprobe;destsuffix=git/deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION} \ +SRC_URI = "git://github.com/NVIDIA/libnvidia-container.git;protocol=https;name=libnvidia;branch=main \ + git://github.com/NVIDIA/nvidia-modprobe.git;protocol=https;branch=main;name=modprobe;destsuffix=nvidia-modprobe \ file://0001-build-fix.patch \ file://0002-secomp-fix.patch \ file://0003-fix-remove-buildpath-for-package-qa.patch \ " + # SRCREV = "a4ef85ebc86688eeef667271abbc7cd4f1110cf6" -SRCREV_libnvidia = "d2eb0afe86f0b643e33624ee64f065dd60e952d4" +SRCREV_libnvidia = "889a3bb5408c195ed7897ba2cb8341c7d249672f" # Nvidia modprobe version 495.44 -SRCREV_modprobe = "d6bce304f30b6661c9ab6a993f49340eafca7a7e" +SRCREV_modprobe = "0c7433d5b04ce4a97036a939c076416d47427675" SRCREV_FORMAT = "libnvidia_modprobe" DEPENDS = " \ @@ -36,7 +37,4 @@ DEPENDS = " \ libtirpc134 \ ldconfig-native \ " -RDEPENDS:${PN}:append= " ldconfig-compatibility-symlink containerd-config" - - -S = "${WORKDIR}/git" \ No newline at end of file +RDEPENDS:${PN}:append= " ldconfig-compatibility-symlink containerd-config" \ No newline at end of file diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch index 7325aa2bb..4751aa6ba 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch @@ -1,3 +1,5 @@ +Upstream-Status: Inappropriate [embedded specific] + diff --git a/Makefile b/Makefile index a374cc09..34d21e25 100644 --- a/Makefile diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch index 46c5e3ad3..1a3c46566 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch @@ -1,3 +1,5 @@ +Upstream-Status: Inappropriate [embedded specific] + diff --git a/Makefile b/Makefile index a374cc09..b6632580 100644 --- a/Makefile diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch index b6723794b..ec42da634 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch @@ -1,3 +1,5 @@ +Upstream-Status: Inappropriate [embedded specific] + From 13cc971cf7ab0bb8fe8528f2ebd65685a81b7ae3 Mon Sep 17 00:00:00 2001 From: Atharva Nandanwar Date: Fri, 19 Aug 2022 15:25:18 -0600 diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index 2d8013672..f53a02d00 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -3,6 +3,16 @@ inherit features_check SUMMARY = "libNVIDIA Container for Yocto" +fakeroot do_unpack_modprobe() { + mkdir -p ${S}/deps/src + if [ -d "${UNPACKDIR}/nvidia-modprobe" ]; then + mv ${UNPACKDIR}/nvidia-modprobe ${S}/deps/src/nvidia-modprobe-${NVIDIA_MODPROBE_VERSION} + fi +} +addtask unpack_modprobe after do_unpack before do_patch +do_unpack_modprobe[dirs] = "${S}" +do_unpack_modprobe[vardeps] += "NVIDIA_MODPROBE_VERSION" + PACKAGECONFIG ??= "seccomp" PACKAGECONFIG[seccomp] = "WITH_SECCOMP=yes,WITH_SECCOMP=no,libseccomp" diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb index 631e2dbf5..f4d1cacc3 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb @@ -12,15 +12,15 @@ SRC_URI[sha256sum] = "1e0b0c7231c5fa122e06c0609a76723664d068b0dba3b8219b63e6340b # SRC_URI += "file://0001-__rpc_dtbsize-rlim_cur-instead-of-rlim_max.patch" -S = "${WORKDIR}/libtirpc-${PV}" +S = "${UNPACKDIR}/libtirpc-${PV}" inherit autotools pkgconfig DISABLE_STATIC = "" EXTRA_OECONF = "--disable-gssapi --enable-static" -# Append -fPIC to CFLAGS -CFLAGS:append = " -fPIC" +# Append -fPIC to CFLAGS and fix GCC 15 compatibility +CFLAGS:append = " -fPIC -Wno-error=incompatible-pointer-types -Wno-error=int-conversion -std=gnu17" do_install:append() { rm -r ${D}${sysconfdir} ${D}${datadir} ${D}${libdir}/pkgconfig diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc index 1c6efdf3c..200471374 100644 --- a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc @@ -3,16 +3,13 @@ HOMEPAGE = "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/NVIDIA/nvidia-container-toolkit" LICENSE = "Apache-2.0" LIC_FILES_CHKSUM = "file://src/${GO_IMPORT}/LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57" -SRC_URI = "git://github.com/NVIDIA/nvidia-container-toolkit.git;protocol=https;branch=release-1.14" -SRCREV = "4668c511de4b311c96bc3dd0310bff40b75083bd" +SRC_URI = "git://github.com/NVIDIA/nvidia-container-toolkit.git;protocol=https;branch=main;destsuffix=${BP}/src/${GO_IMPORT}" +SRCREV = "c748619c592030519361274a2fdd43f2c2ced73a" SRC_URI += "file://config.toml" -SRC_URI += "file://0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch;patchdir=src/${GO_IMPORT}" GO_IMPORT = "github.com/NVIDIA/nvidia-container-toolkit" DEPENDS = " \ curl-native ca-certificates-native go-native \ coreutils-native \ " - -S = "${WORKDIR}/git" diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb index 9106fce63..44660559f 100644 --- a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb +++ b/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb @@ -42,7 +42,7 @@ do_install() { # Ensure the installation directory exists install -d ${D}/etc/nvidia-container-runtime # Install the config.toml file - install -m 0644 ${WORKDIR}/config.toml ${D}/etc/nvidia-container-runtime/config.toml + install -m 0644 ${UNPACKDIR}/config.toml ${D}/etc/nvidia-container-runtime/config.toml } INSANE_SKIP:${PN} += "already-stripped buildpaths textrel" diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb similarity index 76% rename from meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb rename to meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb index db47484cd..a6ffc4314 100644 --- a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_570.172.08.bb +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb @@ -5,10 +5,9 @@ LICENSE = "NVIDIA-Proprietary" LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-${PV}-archive.tar.xz" -SRC_URI[md5sum] = "b8cc1c4e37794eaf738c488d419a7c90" -SRC_URI[sha256sum] = "66d1c4303700f19bd86bddda071340e070511a7070ce60a8641f16db7c184cfa" +SRC_URI[sha256sum] = "71086dc7d9c97ac20a7d14de2a119e3b2ee97a08da452d63c2c4f5a71d80e19a" -S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" +S = "${UNPACKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" INSANE_SKIP:${PN} = "already-stripped ldflags" @@ -18,16 +17,12 @@ do_compile[noexec] = "1" do_install() { # Create directories install -d ${D}${libdir} - install -d ${D}${bindir} # Install libraries install -m 0755 ${S}/lib/libnvidia-nscq.so.${PV} ${D}${libdir} ln -sf libnvidia-nscq.so.${PV} ${D}${libdir}/libnvidia-nscq.so.2.0 ln -sf libnvidia-nscq.so.2.0 ${D}${libdir}/libnvidia-nscq.so.2 ln -sf libnvidia-nscq.so.2 ${D}${libdir}/libnvidia-nscq.so - - # Install binaries - install -m 0755 ${S}/bin/nscq-cli ${D}${bindir} } FILES:${PN} = "\ @@ -35,5 +30,4 @@ FILES:${PN} = "\ ${libdir}/libnvidia-nscq.so.2.0 \ ${libdir}/libnvidia-nscq.so.2 \ ${libdir}/libnvidia-nscq.so \ - ${bindir}/nscq-cli \ " diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb deleted file mode 100644 index 3d91e4cd9..000000000 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_570.172.08.bb +++ /dev/null @@ -1,86 +0,0 @@ -SUMMARY = "NVIDIA Fabric Manager for NVSwitch systems" -DESCRIPTION = "NVIDIA Fabric Manager provides NVSwitch management for NVIDIA HGX and DGX systems" -HOMEPAGE = "https://developer.nvidia.com/" -LICENSE = "NVIDIA-Proprietary" -LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" - -SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${PV}-archive.tar.xz" -SRC_URI[md5sum] = "a71788f11f6edabf69df32a7b9dcfa68" -SRC_URI[sha256sum] = "8d24cacde4554d471899ad426f46a349d5ca0a2e8acd45c2a76381c8f496491e" - -S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" - -DEPENDS = "" -RDEPENDS:${PN} = "bash zlib" - -INSANE_SKIP:${PN} = "already-stripped ldflags" - -do_configure[noexec] = "1" -do_compile[noexec] = "1" - -inherit systemd - -SYSTEMD_AUTO_ENABLE = "enable" -SYSTEMD_SERVICE:${PN} = "nvidia-fabricmanager.service" - -do_install() { - # Create directories - install -d ${D}${bindir} - install -d ${D}${libdir} - install -d ${D}${datadir}/nvidia/nvswitch - install -d ${D}${systemd_system_unitdir} - - # Install binaries - install -m 0755 ${S}/bin/nv-fabricmanager ${D}${bindir} - install -m 0755 ${S}/bin/nvidia-fabricmanager-start.sh ${D}${bindir} - install -m 0755 ${S}/bin/nvswitch-audit ${D}${bindir} - - # Install libraries - install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} - ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so - - # Install config files - install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ - - # Install topology files - install -m 0644 ${S}/share/nvidia/nvswitch/dgx2_hgx2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxa100_hgxa100_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxh100_hgxh100_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxh800_hgxh800_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/mgxh20_nvl16_topology ${D}${datadir}/nvidia/nvswitch/ - - # Install multi-node topology files - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_8gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gh200_nvlink_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl576r16_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_nso_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl4r1_c2g2_etf_topology ${D}${datadir}/nvidia/nvswitch/ - - # Install systemd service - install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} -} - -FILES:${PN} = "\ - ${bindir}/nv-fabricmanager \ - ${bindir}/nvidia-fabricmanager-start.sh \ - ${bindir}/nvswitch-audit \ - ${libdir}/libnvfm.so.1 \ - ${libdir}/libnvfm.so \ - ${datadir}/nvidia/nvswitch/* \ - ${systemd_system_unitdir}/nvidia-fabricmanager.service \ -" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb new file mode 100644 index 000000000..e5b3f8f25 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb @@ -0,0 +1,64 @@ +SUMMARY = "NVIDIA Fabric Manager for NVSwitch systems" +DESCRIPTION = "NVIDIA Fabric Manager provides NVSwitch management for NVIDIA HGX and DGX systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[sha256sum] = "eb3a81d004de426dee9b0f1332093828edb197125ef57a2e6500a95df3332d0b" + +S = "${UNPACKDIR}/fabricmanager-linux-x86_64-${PV}-archive" + +DEPENDS = "" +RDEPENDS:${PN} = "bash zlib" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +inherit systemd + +SYSTEMD_AUTO_ENABLE = "enable" +SYSTEMD_SERVICE:${PN} = "nvidia-fabricmanager.service" + +do_install() { + # Create directories + install -d ${D}${bindir} + install -d ${D}${libdir} + install -d ${D}${datadir}/nvidia/nvswitch + install -d ${D}${systemd_system_unitdir} + + # Install binaries + install -m 0755 ${S}/bin/nv-fabricmanager ${D}${bindir} + install -m 0755 ${S}/bin/nvidia-fabricmanager-start.sh ${D}${bindir} + install -m 0755 ${S}/bin/nvswitch-audit ${D}${bindir} + + # Install libraries + install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} + ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so + + # Install config files + install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ + + # Install topology files + for f in ${S}/share/nvidia/nvswitch/*; do + if [ -f "$f" ]; then + install -m 0644 "$f" ${D}${datadir}/nvidia/nvswitch/ + fi + done + + # Install systemd service + install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} +} + +FILES:${PN} = "\ + ${bindir}/nv-fabricmanager \ + ${bindir}/nvidia-fabricmanager-start.sh \ + ${bindir}/nvswitch-audit \ + ${libdir}/libnvfm.so.1 \ + ${libdir}/libnvfm.so \ + ${datadir}/nvidia/nvswitch/* \ + ${systemd_system_unitdir}/nvidia-fabricmanager.service \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb index 91b654077..5ed2c9567 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb @@ -8,7 +8,7 @@ SRC_URI = "\ do_install() { install -d ${D}${sysconfdir}/modprobe.d - install -m 0644 ${WORKDIR}/nvidia.conf ${D}${sysconfdir}/modprobe.d/ + install -m 0644 ${UNPACKDIR}/nvidia.conf ${D}${sysconfdir}/modprobe.d/ } FILES:${PN} = "${sysconfdir}/modprobe.d/nvidia.conf" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb index bb73a6d11..7503da705 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb @@ -13,5 +13,5 @@ SYSTEMD_AUTO_ENABLE:${PN} = "enable" do_install() { install -d ${D}${systemd_unitdir}/system - install -m 0644 ${WORKDIR}/nvidia-persistenced.service ${D}${systemd_unitdir}/system + install -m 0644 ${UNPACKDIR}/nvidia-persistenced.service ${D}${systemd_unitdir}/system } diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb similarity index 75% rename from meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb rename to meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb index bacb3893b..33f56b5bb 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia_570.172.08.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb @@ -3,12 +3,11 @@ LICENSE = "NVIDIA-Proprietary" LIC_FILES_CHKSUM = "file://../LICENSE;md5=92aa2e2af6aa0bcba1c3fe49da021937" NVIDIA_ARCHIVE_NAME = "NVIDIA-Linux-${TARGET_ARCH}-${PV}" -NVIDIA_SRC = "${WORKDIR}/${NVIDIA_ARCHIVE_NAME}" +NVIDIA_SRC = "${UNPACKDIR}/${NVIDIA_ARCHIVE_NAME}" SRC_URI = " \ https://us.download.nvidia.com/tesla/${PV}/${NVIDIA_ARCHIVE_NAME}.run \ " -SRC_URI[md5sum] = "4d0264e320d09f614b086c3ad99a75c5" -SRC_URI[sha256sum] = "0256867e082caf93d7b25fa7c8e69b316062a9c6c72c6e228fad7b238c6fa17d" +SRC_URI[sha256sum] = "d9c6e8188672f3eb74dd04cfa69dd58479fa1d0162c8c28c8d17625763293475" RDEPENDS:${PN} = "nvidia-modprobe-config" From c1355e4ba6339074bc3c4f7279d5463f893c826b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 04:33:19 +0000 Subject: [PATCH 229/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 72ffe344d..678226c61 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 72ffe344dacf2316c7e4a42803eda2af4e3cd15b +Subproject commit 678226c6188a2014129daa92a374a5facf5bfb99 From 8fc9ddb70c743dbe1c575a016a18e85b0ae0c562 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 04:36:05 +0000 Subject: [PATCH 230/435] fixup zfs --- meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb index fc2361daf..93d18bfa2 100644 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb @@ -2,7 +2,7 @@ SUMMARY = "OpenZFS on Linux and FreeBSD" DESCRIPTION = "OpenZFS on Linux and FreeBSD" LICENSE = "CDDL-1.0" LIC_FILES_CHKSUM = "file://LICENSE;md5=7087caaf1dc8a2856585619f4a787faa" -HOMEPAGE ="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/openzfs/zfs" +HOMEPAGE = "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/openzfs/zfs" SRCREV = "743334913e5a5f60baf287bcc6d8a23515b02ac5" SRC_URI = "git://github.com/openzfs/zfs;protocol=https;branch=zfs-2.4-release \ From daed2d798b54775884afd37cac942285e7ad3db5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 04:36:20 +0000 Subject: [PATCH 231/435] fixup meta-nvidia --- .../containerd-config/containerd-config_1.0.0.bb | 2 +- .../libnvidia-container/libnvidia-container_1.00.bb | 2 +- meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb index 2bc9f8509..d2757e9ca 100644 --- a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb +++ b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb @@ -10,4 +10,4 @@ do_install:append() { } RDEPENDS:${PN}:append = " containerd-opencontainers" -FILES:${PN}:append= "${sysconfdir}/containerd/config.toml" +FILES:${PN}:append = "${sysconfdir}/containerd/config.toml" diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index f53a02d00..b533f734a 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -24,7 +24,7 @@ EXTRA_OEMAKE = "EXCLUDE_BUILD_FLAGS=1 PLATFORM=${HOST_ARCH} WITH_NVCGO=yes WITH_ NVIDIA_MODPROBE_EXTRA_CFLAGS ?= "-ffile-prefix-map=${WORKDIR}=/usr/src/debug/${PN}/${EXTENDPE}${PV}-${PR}" CFLAGS:prepend = " -I${RECIPE_SYSROOT_NATIVE}/usr/include/tirpc " -export OBJCPY="${OBJCOPY}" +export OBJCPY = "${OBJCOPY}" GO_IMPORT = "github.com/NVIDIA/nvidia-container-toolkit" SECURITY_LDFLAGS = "" LDFLAGS += "-Wl,-z,lazy" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc b/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc index a1ded2ae0..622eacd2a 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc @@ -53,7 +53,7 @@ do_install:append() { cp ${NVIDIA_SRC}/firmware/* ${D}${libdir}/firmware/nvidia/${PV}/ } -FILES:${PN}:append= " ${libdir} ${bindir}" +FILES:${PN}:append = " ${libdir} ${bindir}" INHIBIT_PACKAGE_DEBUG_SPLIT = "1" From bbc89cc42cad210f7f08d208a3d99be471b4d149 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 11:52:37 +0000 Subject: [PATCH 232/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 678226c61..bccc20dd1 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 678226c6188a2014129daa92a374a5facf5bfb99 +Subproject commit bccc20dd1dfcde88fc3f4860b19333b523b1a7cf From 169cf0245594962a2bfead9a42621ce2529155f1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 11:53:02 +0000 Subject: [PATCH 233/435] Adapt docker 29 --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 11 +++++++++-- .../recipes-core/images/dstack-rootfs-base.inc | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 7b1769f1c..2f7e0fb15 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -78,11 +78,18 @@ do_install() { install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d - install -m 0644 ${S}/basefiles/docker.service.d/dstack-guest-agent.conf ${D}${sysconfdir}/systemd/system/docker.service.d/ + install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ + + install -d ${D}${sysconfdir}/systemd/system/containerd.service.d + install -m 0644 ${S}/basefiles/containerd.service.d/* ${D}${sysconfdir}/systemd/system/containerd.service.d/ fi } -FILES:${PN} += "${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf" +FILES:${PN} += " \ + ${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf \ + ${sysconfdir}/systemd/system/docker.service.d/dstack-prepare.conf \ + ${sysconfdir}/systemd/system/containerd.service.d/dstack-prepare.conf \ +" # Cargo embeds build paths into binaries; allow TMPDIR references. INSANE_SKIP:${PN} += "buildpaths" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 435210800..09605e8e1 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -143,5 +143,6 @@ mkdirs() { mkdir -p ${IMAGE_ROOTFS}/dstack mkdir -p ${IMAGE_ROOTFS}/etc/wireguard mkdir -p ${IMAGE_ROOTFS}/var/lib/docker + mkdir -p ${IMAGE_ROOTFS}/var/lib/containerd ln -sf dstack ${IMAGE_ROOTFS}/tapp } From 81da8404efba4feac0c0e6c2ce343aa6e439efa1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 23 Dec 2025 08:16:31 +0000 Subject: [PATCH 234/435] Remove mod-tdx-guest --- dstack | 2 +- .../recipes-kernel/tdx-guest-mod/tdx-guest.bb | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb diff --git a/dstack b/dstack index bccc20dd1..805b55473 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit bccc20dd1dfcde88fc3f4860b19333b523b1a7cf +Subproject commit 805b5547371971faf098b74e7def711be52b7536 diff --git a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb b/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb deleted file mode 100644 index e134f3e47..000000000 --- a/meta-dstack/recipes-kernel/tdx-guest-mod/tdx-guest.bb +++ /dev/null @@ -1,18 +0,0 @@ -SUMMARY = "TDX guest kernel module for Intel Trust Domain Extensions" -DESCRIPTION = "${SUMMARY}" -LICENSE = "MIT" -LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" - -inherit module - -REPO_ROOT = "${THISDIR}/../../.." - -SRC_DIR = '${REPO_ROOT}/dstack/mod-tdx-guest' -SRC_URI = 'file://${REPO_ROOT}/dstack/mod-tdx-guest' -SRCREV = "${DSTACK_SRC_REV}" - -S = "${UNPACKDIR}/${SRC_DIR}" - -RPROVIDES:${PN} += "tdx-guest-ko" -INSANE_SKIP:${PN} += "buildpaths" -INSANE_SKIP:${PN}-dbg += "buildpaths" From a9295ba8eb45a0253656e32b568962d2945e5944 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 24 Dec 2025 07:22:14 +0000 Subject: [PATCH 235/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 805b55473..f1a49f01a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 805b5547371971faf098b74e7def711be52b7536 +Subproject commit f1a49f01a74c03c2034587dcf624bfc9c918ef20 From a4fbb9217a0bc33563154090c9674e5b42c46280 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 24 Dec 2025 07:22:40 +0000 Subject: [PATCH 236/435] Output gcp.tar.gz inside the image dir --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 0028a4697..0bf58a674 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -194,7 +194,7 @@ create_gcp_artifacts() { echo "Building raw disk image for GCP at ${disk_img}" build_gcp_disk_image "$disk_img" "$boot_src" "${OUTPUT_DIR}/rootfs.img.verity" - local tarball="${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-gcp.tar.gz" + local tarball="${OUTPUT_DIR}/gcp.tar.gz" echo "Archiving GCP disk image to ${tarball}" (cd "$gcp_dir" && tar -czvf "$tarball" disk.raw) } From c22dfd41fc833441306dfe972f26d44ddbaef6a3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 24 Dec 2025 07:23:02 +0000 Subject: [PATCH 237/435] Add cli dstack-cloud --- scripts/bin/dstack-cloud | 1906 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1906 insertions(+) create mode 100755 scripts/bin/dstack-cloud diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud new file mode 100755 index 000000000..4f0de517e --- /dev/null +++ b/scripts/bin/dstack-cloud @@ -0,0 +1,1906 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +""" +dstack-cloud: Multi-cloud VM lifecycle management tool + +A production-grade CLI for managing dstack VMs on various cloud platforms. +Supports local configuration files similar to git's working model. + +Usage: + dstack-cloud new # Create a new project + dstack-cloud init # Initialize current directory + dstack-cloud config-edit # Edit global configuration + dstack-cloud prepare # Generate shared files + dstack-cloud deploy # Deploy VM to cloud + dstack-cloud status # Check deployment status + dstack-cloud logs [--follow] # View serial console logs + dstack-cloud stop # Stop the VM + dstack-cloud start # Start a stopped VM + dstack-cloud remove # Remove the VM and cleanup + dstack-cloud list # List all deployments +""" + +import argparse +import hashlib +import json +import logging +import os +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field, asdict +from datetime import datetime +from pathlib import Path +from typing import Optional, List, Dict, Any + +# Try to import cryptography libraries for env encryption +CRYPTO_AVAILABLE = False +try: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + from cryptography.hazmat.primitives.asymmetric import x25519 + from cryptography.hazmat.backends import default_backend + from cryptography import serialization + CRYPTO_AVAILABLE = True +except ImportError: + CRYPTO_AVAILABLE = False + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +# Configuration file names +APP_CONFIG_FILE = "app.json" +STATE_FILE = "state.json" +GLOBAL_CONFIG_PATH = os.path.expanduser("~/.config/dstack-cloud/config.json") + + +@dataclass +class App: + """Application configuration.""" + # App name + name: str = "myapp" + + # OS image + os_image: str = "dstack-0.6.0" + + # GCP cloud configuration + gcp_config: 'GcpConfig' = field(default_factory=lambda: GcpConfig()) + + # Docker compose file name (relative to project root) + docker_compose_file: str = "docker-compose.yaml" + + # Prelaunch script name (relative to project root) + prelaunch_script: str = "prelaunch.sh" + + # Environment file name (relative to project root) + env_file: str = ".env" + + # Instance identity + instance_id_seed: str = "" + app_id: str = "" + + # Gateway settings + gateway_enabled: bool = True + public_logs: bool = True + public_sysinfo: bool = True + public_tcbinfo: bool = True + + # KMS settings + key_provider: str = "kms" + + # Storage + storage_fs: str = "ext4" + + # Instance settings + no_instance_id: bool = False + secure_time: bool = False + + # Allowed environments + allowed_envs: List[str] = field(default_factory=list) + key_provider_id: str = "" + + def to_dict(self) -> Dict[str, Any]: + data = asdict(self) + # Convert GcpConfig to dict + if isinstance(data.get("gcp_config"), GcpConfig): + data["gcp_config"] = data["gcp_config"].to_dict() + return data + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'App': + known_fields = {f.name for f in cls.__dataclass_fields__.values()} + filtered = {k: v for k, v in data.items() if k in known_fields} + + # Convert gcp_config dict to GcpConfig object + if "gcp_config" in filtered and isinstance(filtered["gcp_config"], dict): + filtered["gcp_config"] = GcpConfig.from_dict(filtered["gcp_config"]) + + return cls(**filtered) + + @classmethod + def get_template(cls) -> Dict[str, Any]: + """Get default template for new projects.""" + import secrets + + # Generate random instance_id_seed (40 hex chars) + instance_id_seed = secrets.token_hex(20) + + # Generate random app_id (40 hex chars) + app_id = secrets.token_hex(20) + + return { + "name": "myapp", + "os_image": "dstack-0.6.0", + "gcp_config": GcpConfig.get_template(), + "instance_id_seed": instance_id_seed, + "app_id": app_id, + "docker_compose_file": "docker-compose.yaml", + "prelaunch_script": "prelaunch.sh", + "env_file": ".env", + "gateway_enabled": True, + "public_logs": True, + "public_sysinfo": True, + "public_tcbinfo": True, + "key_provider": "kms", + "storage_fs": "ext4", + "no_instance_id": False, + "secure_time": False, + "allowed_envs": [], + "key_provider_id": "" + } + + +@dataclass +class GcpConfig: + """GCP deployment configuration.""" + # Required settings + project: str = "" + zone: str = "us-central1-a" + + # Instance settings + instance_name: str = "" # Required, no default + machine_type: str = "c3-standard-4" + + # Boot image settings + boot_image: str = "" # GCP image name (auto-derived from app.os_image if empty) + boot_image_tar: str = "" # Explicit tar file path (overrides search) + + # Data disk settings + data_image: str = "dstack-data-disk" + data_size: int = 20 + + # Storage settings + bucket: str = "" + + # Network settings + network: str = "default" + subnet: str = "" + + # Tags and labels + tags: List[str] = field(default_factory=list) + labels: Dict[str, str] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'GcpConfig': + known_fields = {f.name for f in cls.__dataclass_fields__.values()} + filtered = {k: v for k, v in data.items() if k in known_fields} + return cls(**filtered) + + @classmethod + def get_template(cls) -> Dict[str, Any]: + """Get default template for new projects.""" + return { + "project": "", + "zone": "us-central1-a", + "instance_name": "dstack-vm", + "machine_type": "c3-standard-4", + "boot_image": "", + "boot_image_tar": "", + "data_image": "dstack-data-disk", + "data_size": 20, + "bucket": "", + "network": "default", + "subnet": "", + "tags": [], + "labels": {} + } + + +@dataclass +class DeploymentState: + """Deployment state tracking.""" + instance_name: str = "" + project: str = "" + zone: str = "" + external_ip: str = "" + internal_ip: str = "" + status: str = "" # RUNNING, STOPPED, TERMINATED, etc. + created_at: str = "" + updated_at: str = "" + boot_image: str = "" + shared_image: str = "" + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'DeploymentState': + known_fields = {f.name for f in cls.__dataclass_fields__.values()} + filtered = {k: v for k, v in data.items() if k in known_fields} + return cls(**filtered) + + +class CloudDeploymentManager: + """Manages multi-cloud VM deployments.""" + + def __init__(self, work_dir: Optional[str] = None): + self.work_dir = Path(work_dir) if work_dir else Path.cwd() + + def _load_global_config(self) -> Dict[str, Any]: + """Load global configuration.""" + if os.path.exists(GLOBAL_CONFIG_PATH): + with open(GLOBAL_CONFIG_PATH, 'r') as f: + return json.load(f) + return {} + + def _save_global_config(self, config: Dict[str, Any]) -> None: + """Save global configuration.""" + os.makedirs(os.path.dirname(GLOBAL_CONFIG_PATH), exist_ok=True) + with open(GLOBAL_CONFIG_PATH, 'w') as f: + json.dump(config, f, indent=2) + + def _get_shared_dir(self) -> Path: + """Get the shared directory path (at project root).""" + return self.work_dir / "shared" + + def load_gcp_config(self) -> 'GcpConfig': + """Load GCP configuration from app.gcp_config.""" + # Load app config which contains gcp_config + app = self.load_app_config() + + # Get local gcp_config from app + local_gcp = app.gcp_config.to_dict() + + # Merge global config with local config + global_config = self._load_global_config() + global_gcp = global_config.get("gcp", {}) + + # Global config is used as fallback for empty values in local config + merged = {**global_gcp} # Start with global config + for key, value in local_gcp.items(): + # Only override with local value if it's non-empty + if value or value is False or value == 0: + merged[key] = value + return GcpConfig.from_dict(merged) + + def save_gcp_config(self, config: GcpConfig) -> None: + """Save GCP configuration to app.gcp_config.""" + # Load the full app config + app = self.load_app_config() + + # Update gcp_config + app.gcp_config = config + + # Save the updated app config + self.save_app_config(app) + + def load_app_config(self) -> App: + """Load application configuration.""" + app_config_path = self.work_dir / APP_CONFIG_FILE + + if not app_config_path.exists(): + # Return default config if file doesn't exist + return App() + + with open(app_config_path, 'r') as f: + return App.from_dict(json.load(f)) + + def save_app_config(self, app: App) -> None: + """Save application configuration.""" + app_config_path = self.work_dir / APP_CONFIG_FILE + with open(app_config_path, 'w') as f: + json.dump(app.to_dict(), f, indent=2) + + def _generate_app_compose(self, app: App, env_names: Optional[List[str]] = None) -> Dict[str, Any]: + """Generate app-compose.json content from App configuration.""" + # Read docker-compose.yaml content + docker_compose_path = self.work_dir / app.docker_compose_file + if not docker_compose_path.exists(): + docker_compose_content = "" + else: + with open(docker_compose_path, 'r') as f: + docker_compose_content = f.read() + + # Read prelaunch script content + prelaunch_path = self.work_dir / app.prelaunch_script + if not prelaunch_path.exists(): + prelaunch_content = "" + else: + with open(prelaunch_path, 'r') as f: + prelaunch_content = f.read() + + # Merge app.allowed_envs with env_names from .env file + allowed_envs = list(app.allowed_envs) if app.allowed_envs else [] + if env_names: + allowed_envs.extend(env_names) + # Remove duplicates + allowed_envs = list(set(allowed_envs)) + + return { + "manifest_version": 2, + "name": app.name, + "runner": "docker-compose", + "docker_compose_file": docker_compose_content, + "gateway_enabled": app.gateway_enabled, + "public_logs": app.public_logs, + "public_sysinfo": app.public_sysinfo, + "public_tcbinfo": app.public_tcbinfo, + "key_provider_id": app.key_provider_id, + "allowed_envs": allowed_envs, + "no_instance_id": app.no_instance_id, + "secure_time": app.secure_time, + "key_provider": app.key_provider, + "storage_fs": app.storage_fs, + "pre_launch_script": prelaunch_content + } + + def _generate_sys_config(self, global_config: Dict[str, Any], + gcp_config: GcpConfig, app: App) -> Dict[str, Any]: + """Generate .sys-config.json content.""" + # Get services section from global config + services = global_config.get("services", {}) + + # Get KMS URLs from global config + kms_urls = services.get("kms_urls", []) + if not kms_urls: + kms_urls = ["https://kms.tdxlab.dstack.org:12001"] + + # Get gateway URLs from global config + gateway_urls = services.get("gateway_urls", []) + if not gateway_urls: + gateway_urls = ["https://gateway.tdxlab.dstack.org:12002"] + + # Get other settings + pccs_url = services.get("pccs_url", "") + + # Read OS image hash from the local image directory + os_image_hash = "" + try: + # Find the image directory and read hash file + search_paths = global_config.get("image_search_paths", []) + local_image = gcp_config.boot_image if gcp_config.boot_image else "" + if not local_image: + # Use app.os_image if boot_image is not set + local_image = app.os_image + + for search_path in search_paths: + search_path = os.path.expanduser(search_path) + if not os.path.isabs(search_path): + search_path = os.path.join(self.work_dir, search_path) + + hash_file = Path(search_path) / local_image / "dstack-uki.efi.auth_hash.txt" + if hash_file.exists(): + with open(hash_file, 'r') as f: + os_image_hash = f.read().strip() + logger.info(f"Read OS image hash from {hash_file}") + break + except Exception as e: + logger.warning(f"Could not read OS image hash: {e}") + + # Build vm_config + vm_config = { + "spec_version": 2, + "os_image_hash": os_image_hash + } + + return { + "kms_urls": kms_urls, + "gateway_urls": gateway_urls, + "pccs_url": pccs_url, + "vm_config": json.dumps(vm_config) + } + + def load_state(self) -> Optional[DeploymentState]: + """Load deployment state.""" + state_path = self.work_dir / STATE_FILE + + if not state_path.exists(): + return None + + with open(state_path, 'r') as f: + return DeploymentState.from_dict(json.load(f)) + + def save_state(self, state: DeploymentState) -> None: + """Save deployment state.""" + state_path = self.work_dir / STATE_FILE + state.updated_at = datetime.now().isoformat() + with open(state_path, 'w') as f: + json.dump(state.to_dict(), f, indent=2) + + def _run_gcloud(self, args: List[str], capture: bool = True, + check: bool = True) -> subprocess.CompletedProcess: + """Run a gcloud command.""" + cmd = ["gcloud"] + args + logger.debug(f"Running: {' '.join(cmd)}") + + if capture: + result = subprocess.run(cmd, capture_output=True, text=True) + else: + result = subprocess.run(cmd) + + if check and result.returncode != 0: + error_msg = result.stderr if capture else "Command failed" + raise RuntimeError(f"gcloud command failed: {error_msg}") + + return result + + def _run_gsutil(self, args: List[str], check: bool = True) -> subprocess.CompletedProcess: + """Run a gsutil command.""" + cmd = ["gsutil"] + args + logger.debug(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if check and result.returncode != 0: + raise RuntimeError(f"gsutil command failed: {result.stderr}") + + return result + + def new( + self, + name: str, + os_image: Optional[str] = None, + app_id: Optional[str] = None, + gateway_enabled: Optional[bool] = None, + key_provider: Optional[str] = None, + storage_fs: Optional[str] = None, + secure_time: Optional[bool] = None, + no_instance_id: Optional[bool] = None, + project: Optional[str] = None, + zone: Optional[str] = None, + instance_name: Optional[str] = None, + machine_type: Optional[str] = None, + data_size: Optional[int] = None + ) -> None: + """Create a new project directory with template configuration.""" + project_dir = Path.cwd() / name + if project_dir.exists(): + raise FileExistsError(f"Directory '{name}' already exists.") + + # Create project directory + project_dir.mkdir() + + # Update work_dir to the new project directory + self.work_dir = project_dir + + if not instance_name: + instance_name = f"dstack-{name}" + + # Initialize the project (non-interactive by default for new command) + self._init_project( + force=False, + interactive=False, + app_name=name, + os_image=os_image, + app_id=app_id, + gateway_enabled=gateway_enabled, + key_provider=key_provider, + storage_fs=storage_fs, + secure_time=secure_time, + no_instance_id=no_instance_id, + project=project, + zone=zone, + instance_name=instance_name, + machine_type=machine_type, + data_size=data_size + ) + + logger.info(f"Created new project: {name}") + logger.info(f"Project directory: {project_dir}") + logger.info("") + + def init( + self, + force: bool = False, + interactive: bool = True, + os_image: Optional[str] = None, + app_id: Optional[str] = None, + gateway_enabled: Optional[bool] = None, + key_provider: Optional[str] = None, + storage_fs: Optional[str] = None, + secure_time: Optional[bool] = None, + no_instance_id: Optional[bool] = None, + project: Optional[str] = None, + zone: Optional[str] = None, + instance_name: Optional[str] = None, + machine_type: Optional[str] = None, + data_size: Optional[int] = None + ) -> None: + """Initialize deployment configuration in current directory.""" + self._init_project( + force=force, + interactive=interactive, + app_name=None, # Will prompt in interactive mode + os_image=os_image, + app_id=app_id, + gateway_enabled=gateway_enabled, + key_provider=key_provider, + storage_fs=storage_fs, + secure_time=secure_time, + no_instance_id=no_instance_id, + project=project, + zone=zone, + instance_name=instance_name, + machine_type=machine_type, + data_size=data_size + ) + + def _init_project( + self, + force: bool = False, + interactive: bool = True, + app_name: Optional[str] = None, + os_image: Optional[str] = None, + app_id: Optional[str] = None, + gateway_enabled: Optional[bool] = None, + key_provider: Optional[str] = None, + storage_fs: Optional[str] = None, + secure_time: Optional[bool] = None, + no_instance_id: Optional[bool] = None, + project: Optional[str] = None, + zone: Optional[str] = None, + instance_name: Optional[str] = None, + machine_type: Optional[str] = None, + data_size: Optional[int] = None + ) -> None: + """Initialize project configuration (shared implementation for init and new).""" + # Interactive prompts for required fields (only if not provided via CLI) + instance_name_cli = instance_name + + if interactive: + print(f"\n=== dstack-cloud Project Initialization ===\n") + + # Prompt for app name (only for init command, new command provides it) + if app_name is None: + while True: + app_name = input("App name [myapp]: ").strip() + if not app_name: + app_name = "myapp" + if app_name: + break + print("App name cannot be empty.") + + # Prompt for instance name (required) + if not instance_name_cli: + while True: + instance_name = input("GCP instance name: ").strip() + if instance_name: + break + print("Instance name is required.") + else: + instance_name = instance_name_cli + + print("") # Empty line for readability + else: + # Non-interactive mode: use CLI provided values or fail + if app_name is None: + app_name = "myapp" + if not instance_name_cli: + raise ValueError("instance_name is required. Use --instance-name to specify it.") + instance_name = instance_name_cli + + # Create shared directory at project root (for system-generated files) + shared_dir = self.work_dir / "shared" + shared_dir.mkdir(parents=True, exist_ok=True) + + # Generate app config template with embedded gcp_config + app_template = App.get_template() + + # Apply CLI-provided values (only if specified) + if app_name: + app_template["name"] = app_name + if os_image: + app_template["os_image"] = os_image + if app_id: + # Validate app_id format (40 hex chars) + if len(app_id) != 40 or not all(c in '0123456789abcdef' for c in app_id.lower()): + raise ValueError("app_id must be exactly 40 hexadecimal characters") + app_template["app_id"] = app_id + if gateway_enabled is not None: + app_template["gateway_enabled"] = gateway_enabled + if key_provider is not None: + app_template["key_provider"] = key_provider + + # Auto-disable gateway when key_provider is not "kms" + if app_template["key_provider"] != "kms": + app_template["gateway_enabled"] = False + # Also set no_instance_id=True when KMS is not available + app_template["no_instance_id"] = True + if storage_fs is not None: + app_template["storage_fs"] = storage_fs + if secure_time is not None: + app_template["secure_time"] = secure_time + if no_instance_id is not None: + app_template["no_instance_id"] = no_instance_id + + # Apply GCP config values + if instance_name: + app_template["gcp_config"]["instance_name"] = instance_name + if project: + app_template["gcp_config"]["project"] = project + if zone: + app_template["gcp_config"]["zone"] = zone + if machine_type: + app_template["gcp_config"]["machine_type"] = machine_type + if data_size is not None: + app_template["gcp_config"]["data_size"] = data_size + + # Create app.json at project root (not in .dstack/) + app_config_path = self.work_dir / APP_CONFIG_FILE + if not app_config_path.exists() or force: + with open(app_config_path, 'w') as f: + json.dump(app_template, f, indent=2) + + # Note: app-compose.json and .sys-config.json will be generated during deploy + + # Create docker-compose.yaml template at project root + docker_compose = self.work_dir / "docker-compose.yaml" + if not docker_compose.exists() or force: + with open(docker_compose, 'w') as f: + f.write("services:\n") + f.write(" nginx:\n") + f.write(" image: nginx:alpine\n") + f.write(" ports:\n") + f.write(" - \"80:80\"\n") + f.write(" restart: unless-stopped\n") + + # Create prelaunch.sh template at project root + prelaunch = self.work_dir / "prelaunch.sh" + if not prelaunch.exists() or force: + with open(prelaunch, 'w') as f: + f.write("#!/bin/sh\n") + f.write("# Prelaunch script - runs before starting containers\n") + os.chmod(prelaunch, 0o755) + + # Create .env template at project root + env_file = self.work_dir / ".env" + if not env_file.exists() or force: + with open(env_file, 'w') as f: + f.write("# Environment variables\n") + + # Create user-config template at project root + user_config = self.work_dir / ".user-config" + if not user_config.exists() or force: + with open(user_config, 'w') as f: + json.dump({}, f, indent=2) + + logger.info(f"Initialized project in {self.work_dir}") + logger.info("") + if interactive: + logger.info("Configuration:") + logger.info(f" App name: {app_name}") + logger.info(f" Instance name: {instance_name}") + logger.info("") + logger.info("Created files:") + logger.info(f" {app_config_path.name} - Application configuration (with embedded GCP config)") + logger.info(f" shared/ - System-generated files") + logger.info(f" {docker_compose.name} - Docker compose file") + logger.info(f" {prelaunch.name} - Prelaunch script") + logger.info(f" {env_file.name} - Environment variables") + logger.info(f" .user-config - User configuration") + logger.info("") + logger.info("Edit the configuration files to customize your deployment.") + + def config_edit(self) -> None: + """Edit global configuration with $EDITOR.""" + # Create global config if it doesn't exist + global_config_dir = os.path.dirname(GLOBAL_CONFIG_PATH) + os.makedirs(global_config_dir, exist_ok=True) + + if not os.path.exists(GLOBAL_CONFIG_PATH): + # Create template with organized sections and comments + template = { + "_comment_services": "Service endpoints configuration", + "services": { + "kms_urls": ["https://kms.tdxlab.dstack.org:12001"], + "gateway_urls": ["https://gateway.tdxlab.dstack.org:12002"], + "pccs_url": "", + }, + "_comment_images": "System image search paths (for boot_image_tar auto-discovery)", + "image_search_paths": [ + "~/.dstack/images" + ], + "_comment_gcp": "GCP cloud platform defaults", + "gcp": { + "project": "", + "zone": "us-central1-a", + "bucket": "" + } + } + with open(GLOBAL_CONFIG_PATH, 'w') as f: + json.dump(template, f, indent=2, ensure_ascii=False) + f.write("\n") # Add trailing newline + + # Get editor + editor = os.environ.get('EDITOR', 'vi') + + # Open editor + logger.info(f"Opening {GLOBAL_CONFIG_PATH} with {editor}...") + subprocess.run([editor, GLOBAL_CONFIG_PATH]) + + def prepare(self) -> None: + """Generate all files in shared directory.""" + import secrets + + # Load app config + app = self.load_app_config() + + # Ensure instance_id_seed and app_id exist + if not app.instance_id_seed: + app.instance_id_seed = secrets.token_hex(20) + logger.info(f"Generated instance_id_seed: {app.instance_id_seed}") + + if not app.app_id: + app.app_id = secrets.token_hex(20) + logger.info(f"Generated app_id: {app.app_id}") + + # Save updated app config + self.save_app_config(app) + + # Get shared directory + shared_dir = self._get_shared_dir() + shared_dir.mkdir(parents=True, exist_ok=True) + + # Generate .instance_info (instance_id will be generated in CVM) + instance_info = { + "instance_id_seed": app.instance_id_seed, + "app_id": app.app_id + } + instance_info_path = shared_dir / ".instance_info" + with open(instance_info_path, 'w') as f: + json.dump(instance_info, f, indent=2) + logger.info(f"Generated {instance_info_path}") + + # Load GCP config for sys-config generation + try: + gcp_config = self.load_gcp_config() + global_config = self._load_global_config() + + # Generate .sys-config.json + sys_config_content = self._generate_sys_config(global_config, gcp_config, app) + sys_config_path = shared_dir / ".sys-config.json" + with open(sys_config_path, 'w') as f: + json.dump(sys_config_content, f, indent=2) + logger.info(f"Generated {sys_config_path}") + except FileNotFoundError as e: + logger.warning(f"Could not generate .sys-config.json: {e}") + + # Process .env file to collect env_names + env_path = self.work_dir / app.env_file + env_names = [] + if env_path.exists(): + envs = self._parse_env_file(env_path) + if envs: + env_names = list(envs.keys()) + logger.info(f"Found {len(env_names)} environment variable(s) in {app.env_file}") + else: + logger.info(f"{app.env_file} is empty") + + # Generate app-compose.json + app_compose_content = self._generate_app_compose(app, env_names=env_names if env_names else None) + app_compose_path = shared_dir / "app-compose.json" + with open(app_compose_path, 'w') as f: + json.dump(app_compose_content, f, indent=2) + logger.info(f"Generated {app_compose_path}") + + logger.info("") + logger.info(f"Shared files generated in: {shared_dir}") + logger.info("These files will be included in the shared disk image during deploy.") + + def _find_boot_image_tar(self, local_image: str) -> Optional[Path]: + """Search for boot image tar file in configured search paths.""" + global_config = self._load_global_config() + search_paths = global_config.get("image_search_paths", []) + + logger.debug(f"Image search paths: {search_paths}") + logger.debug(f"Looking for image: {local_image}") + + # Expand ~ and convert to Path objects + expanded_paths = [] + for path in search_paths: + expanded = os.path.expanduser(path) + if not os.path.isabs(expanded): + expanded = os.path.join(self.work_dir, expanded) + expanded_paths.append(Path(expanded)) + + # Try different file name patterns + patterns = [ + f"{local_image}/gcp.tar.gz", + ] + + for search_path in expanded_paths: + logger.debug(f"Checking search path: {search_path}, exists: {search_path.exists()}") + if not search_path.exists(): + continue + for pattern in patterns: + tar_file = search_path / pattern + logger.debug(f"Checking: {tar_file}, exists: {tar_file.exists()}") + if tar_file.exists(): + logger.info(f"Found boot image: {tar_file}") + return tar_file + + return None + + def pull(self, os_image: str) -> None: + """Download OS image from remote repository.""" + global_config = self._load_global_config() + search_paths = global_config.get("image_search_paths", []) + + if not search_paths: + logger.error("No image_search_paths configured in global config") + logger.error("Please set image_search_paths in: ~/.config/dstack-cloud/config.json") + return + + # Use the first search path + target_dir = Path(os.path.expanduser(search_paths[0])) + target_dir.mkdir(parents=True, exist_ok=True) + + # Download tar file (e.g., dstack-0.6.0.tar.gz) + download_tar = target_dir / f"{os_image}.tar.gz" + + if download_tar.exists(): + logger.info(f"Download file already exists: {download_tar}") + response = input("Download again to overwrite? [y/N]: ").strip().lower() + if response != 'y': + logger.info("Download cancelled") + return + + # Download URL + download_url = f"https://download.dstack.org/os-images/{os_image}.tar.gz" + + logger.info(f"Downloading {os_image} from {download_url}...") + logger.info(f"Target: {download_tar}") + + try: + # Use curl to download with progress bar + subprocess.run( + ["curl", "-L", "-o", str(download_tar), download_url], + check=True + ) + logger.info(f"Successfully downloaded to {download_tar}") + except subprocess.CalledProcessError as e: + logger.error(f"Failed to download image: {e}") + # Clean up partial download + if download_tar.exists(): + download_tar.unlink() + raise + + # Extract the tar file + logger.info(f"Extracting {download_tar}...") + try: + subprocess.run( + ["tar", "-xzf", str(download_tar), "-C", str(target_dir)], + check=True + ) + logger.info(f"Successfully extracted to {target_dir / os_image}") + except subprocess.CalledProcessError as e: + logger.error(f"Failed to extract image: {e}") + raise + + # Verify the expected structure + expected_dir = target_dir / os_image + expected_tar = expected_dir / "gcp.tar.gz" + if expected_tar.exists(): + logger.info(f"Image ready: {expected_tar}") + else: + logger.warning(f"Expected file not found: {expected_tar}") + logger.warning(f"Downloaded structure may be incorrect") + + def _check_and_upload_boot_image(self, config: GcpConfig, app: App, force: bool = False) -> str: + """Check and upload boot image if needed. Returns the image name.""" + tar_path = None + + # Derive GCP image name from app.os_image + gcp_image = config.boot_image + if not gcp_image: + # Convert OS image name (dstack-0.6.0 -> dstack-0-6-0) + gcp_image = app.os_image.replace(".", "-") + + # If boot_image_tar is specified, use it + if config.boot_image_tar: + tar_path = Path(os.path.expanduser(config.boot_image_tar)) + if not tar_path.exists(): + logger.error("") + logger.error(f"Boot image tar not found: {tar_path}") + logger.error("") + logger.error(f"Please download the image using:") + logger.error(f" dstack-cloud pull {app.os_image}") + logger.error("") + raise FileNotFoundError(f"Boot image tar not found: {tar_path}") + else: + # Auto-discover from search paths using OS image name + logger.info(f"Searching for boot image '{app.os_image}'...") + tar_path = self._find_boot_image_tar(app.os_image) + if not tar_path: + logger.error("") + logger.error(f"Boot image '{app.os_image}' not found locally.") + logger.error("") + logger.error(f"Please download the image using:") + logger.error(f" dstack-cloud pull {app.os_image}") + logger.error("") + raise FileNotFoundError( + f"Boot image '{app.os_image}' not found. " + f"Run 'dstack-cloud pull {app.os_image}' to download it." + ) + + # Use gcp_image as the GCP image name + image_name = gcp_image + local_mtime = tar_path.stat().st_mtime + + # Check if GCP image exists and is up-to-date + result = self._run_gcloud([ + "compute", "images", "describe", image_name, + f"--project={config.project}", + "--format=value(creationTimestamp)" + ], check=False) + + need_upload = False + gcp_creation_time = result.stdout.strip() if result.returncode == 0 else "" + + if force: + logger.info("Force enabled: will re-upload boot image") + need_upload = True + elif not gcp_creation_time: + logger.info(f"GCP image '{image_name}' does not exist, will upload") + need_upload = True + else: + # Parse GCP timestamp and compare + try: + from datetime import datetime + gcp_dt = datetime.fromisoformat(gcp_creation_time.replace('Z', '+00:00')) + gcp_epoch = gcp_dt.timestamp() + if local_mtime > gcp_epoch: + logger.info("Local image is newer than GCP image, will re-upload") + logger.info(f" Local: {datetime.fromtimestamp(local_mtime).isoformat()}") + logger.info(f" GCP: {gcp_creation_time}") + need_upload = True + else: + logger.info(f"GCP image '{image_name}' is up-to-date") + except Exception as e: + logger.warning(f"Could not parse GCP timestamp: {e}") + need_upload = True + + if need_upload: + logger.info("Uploading boot image to GCS...") + self._run_gsutil([ + "cp", str(tar_path), f"{config.bucket}/{image_name}.tar.gz" + ]) + + # Delete existing image if present + if gcp_creation_time: + logger.info("Deleting existing GCP image...") + self._run_gcloud([ + "compute", "images", "delete", image_name, + f"--project={config.project}", + "--quiet" + ]) + + logger.info("Creating GCP image with TDX support...") + self._run_gcloud([ + "compute", "images", "create", image_name, + f"--project={config.project}", + f"--source-uri={config.bucket}/{image_name}.tar.gz", + "--guest-os-features=UEFI_COMPATIBLE,TDX_CAPABLE,GVNIC" + ]) + + return image_name + + def _create_shared_disk_image(self, config: GcpConfig, app: App) -> str: + """Create and upload shared disk image. Returns the image name.""" + import secrets + import shutil + + # Check if mcopy (mtools) is installed + if not shutil.which("mcopy"): + logger.error("") + logger.error("Error: 'mcopy' command not found.") + logger.error("") + logger.error("Please install mtools:") + logger.error(" Ubuntu/Debian: sudo apt-get install mtools") + logger.error(" Fedora/RHEL: sudo dnf install mtools") + logger.error(" Arch Linux: sudo pacman -S mtools") + logger.error("") + raise FileNotFoundError("mcopy not found. Please install mtools package.") + + # Ensure instance_id_seed and app_id exist + if not app.instance_id_seed: + app.instance_id_seed = secrets.token_hex(20) + logger.info(f"Generated instance_id_seed: {app.instance_id_seed}") + self.save_app_config(app) + + if not app.app_id: + app.app_id = secrets.token_hex(20) + logger.info(f"Generated app_id: {app.app_id}") + self.save_app_config(app) + + shared_dir = self._get_shared_dir() + + # Ensure shared directory exists and generate all required files + shared_dir.mkdir(parents=True, exist_ok=True) + + shared_image_name = f"{config.instance_name}-shared" + + # Process .env file: encrypt and save to shared/.encrypted-env + env_path = self.work_dir / app.env_file + env_names = [] # Collect environment variable names for allowed_envs + + if env_path.exists(): + if app.key_provider != "kms": + raise ValueError(f"{app.env_file} found but KMS is not enabled. " + f"Enable KMS with --key-provider kms or remove {app.env_file}") + + if not app.app_id: + raise ValueError(f"{app.env_file} found but app_id is not set. " + f"Run 'dstack-cloud prepare' to generate app_id") + + # Parse .env file + envs = self._parse_env_file(env_path) + if envs: + env_names = list(envs.keys()) + + # Get KMS URL from global config + global_config = self._load_global_config() + kms_urls = global_config.get("services", {}).get("kms_urls", []) + if not kms_urls: + raise ValueError("KMS enabled but no kms_urls configured in global config") + + # Get encryption public key from KMS + kms_url = kms_urls[0] + pubkey = self._get_app_encrypt_pub_key(app.app_id, kms_url) + + # Encrypt environment variables + encrypted_env = self._encrypt_env(envs, pubkey) + + # Save to shared/.encrypted-env + encrypted_file = shared_dir / ".encrypted-env" + with open(encrypted_file, 'w') as f: + f.write(encrypted_env) + logger.info(f"Encrypted {app.env_file} -> {encrypted_file}") + else: + logger.info(f"{app.env_file} is empty, skipping") + + # Regenerate app-compose.json with env_names (if any) + # This must be done after .env processing and before creating the disk image + global_config = self._load_global_config() + sys_config_content = self._generate_sys_config(global_config, config, app) + sys_config_path = shared_dir / ".sys-config.json" + with open(sys_config_path, 'w') as f: + json.dump(sys_config_content, f, indent=2) + logger.info(f"Generated {sys_config_path}") + + # Generate .instance_info + instance_info = { + "instance_id_seed": app.instance_id_seed, + "app_id": app.app_id + } + instance_info_path = shared_dir / ".instance_info" + with open(instance_info_path, 'w') as f: + json.dump(instance_info, f, indent=2) + logger.info(f"Generated {instance_info_path}") + + # Generate app-compose.json with env_names (from .env file) + app_compose_content = self._generate_app_compose(app, env_names=env_names if env_names else None) + app_compose_path = shared_dir / "app-compose.json" + with open(app_compose_path, 'w') as f: + json.dump(app_compose_content, f, indent=2) + logger.info(f"Generated {app_compose_path}") + + with tempfile.TemporaryDirectory() as work_dir: + work_path = Path(work_dir) + raw_file = work_path / "disk.raw" + + # Create FAT32 disk image (no root required) + logger.info("Creating shared disk image...") + disk_size = "8M" # 8MB FAT32 disk + subprocess.run( + ["truncate", "-s", disk_size, str(raw_file)], + check=True + ) + subprocess.run( + ["mkfs.fat", "-F", "32", "-n", "DSTACKSHR", str(raw_file)], + check=True, capture_output=True + ) + + # Use mtools to copy files without mounting (no root required) + # Copy generated system files from shared directory + required_files = ["app-compose.json", ".sys-config.json", ".instance_info"] + for f in required_files: + src = shared_dir / f + if src.exists(): + subprocess.run( + ["mcopy", "-i", str(raw_file), str(src), "::"], + check=True + ) + else: + raise FileNotFoundError(f"Required file {f} not found in {shared_dir}") + + # Copy optional system files from shared directory + optional_files = [".encrypted-env"] + for f in optional_files: + src = shared_dir / f + if src.exists(): + subprocess.run( + ["mcopy", "-i", str(raw_file), str(src), "::"], + check=True + ) + + # Copy other user-editable files from project root + user_files = { + ".user-config": ".user-config", + } + + for src_name, dst_name in user_files.items(): + src_path = self.work_dir / src_name + if src_path.exists(): + subprocess.run( + ["mcopy", "-i", str(raw_file), str(src_path), f"::{dst_name}"], + check=True + ) + logger.info(f"Included {src_name}") + else: + logger.warning(f"{src_name} not found, skipping") + + # Create tar + tar_file = work_path / "shared-disk.tar.gz" + subprocess.run( + ["tar", "-C", str(work_path), "-czvf", str(tar_file), "disk.raw"], + check=True, capture_output=True + ) + + # Upload to GCS + logger.info("Uploading shared disk image to GCS...") + self._run_gsutil([ + "cp", str(tar_file), f"{config.bucket}/{shared_image_name}.tar.gz" + ]) + + # Delete existing image if present + result = self._run_gcloud([ + "compute", "images", "describe", shared_image_name, + f"--project={config.project}" + ], check=False) + + if result.returncode == 0: + logger.info("Deleting existing shared disk image...") + self._run_gcloud([ + "compute", "images", "delete", shared_image_name, + f"--project={config.project}", + "--quiet" + ]) + + # Create GCP image + logger.info("Creating GCP image from shared disk...") + self._run_gcloud([ + "compute", "images", "create", shared_image_name, + f"--project={config.project}", + f"--source-uri={config.bucket}/{shared_image_name}.tar.gz", + "--guest-os-features=GVNIC" + ]) + + return shared_image_name + + def deploy(self, delete_existing: bool = False, + force_boot_image: bool = False) -> None: + """Deploy VM to GCP.""" + config = self.load_gcp_config() + app = self.load_app_config() + + # Auto-detect bucket if not specified + if not config.bucket and config.project: + config.bucket = f"gs://{config.project}-dstack" + + shared_dir = self._get_shared_dir() + + logger.info("=== GCP TDX VM Deployment ===") + logger.info(f"Project: {config.project}") + logger.info(f"Zone: {config.zone}") + logger.info(f"Instance: {config.instance_name}") + logger.info(f"Shared Directory: {shared_dir}") + logger.info(f"GCS Bucket: {config.bucket}") + + # Delete existing instance if requested + if delete_existing: + self._delete_instance_if_exists(config) + + # Check and upload boot image + boot_image = self._check_and_upload_boot_image(config, app, force=force_boot_image) + + # Create shared disk image + shared_image = self._create_shared_disk_image(config, app) + + # Create TDX instance + logger.info("Creating TDX instance...") + + create_args = [ + "compute", "instances", "create", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}", + f"--machine-type={config.machine_type}", + "--confidential-compute-type=TDX", + f"--image={boot_image}", + "--boot-disk-size=10GB", + f"--create-disk=name={config.instance_name}-data,size={config.data_size}GB,type=pd-balanced,image={config.data_image},auto-delete=yes", + f"--create-disk=name={config.instance_name}-shared,size=1GB,type=pd-balanced,image={shared_image},auto-delete=yes", + "--maintenance-policy=TERMINATE", + ] + + if config.network != "default": + create_args.append(f"--network={config.network}") + if config.subnet: + create_args.append(f"--subnet={config.subnet}") + if config.tags: + create_args.append(f"--tags={','.join(config.tags)}") + if config.labels: + labels_str = ",".join(f"{k}={v}" for k, v in config.labels.items()) + create_args.append(f"--labels={labels_str}") + + self._run_gcloud(create_args) + + # Get instance details + result = self._run_gcloud([ + "compute", "instances", "describe", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}", + "--format=json" + ]) + + instance_info = json.loads(result.stdout) + external_ip = "" + internal_ip = "" + + for iface in instance_info.get("networkInterfaces", []): + internal_ip = iface.get("networkIP", "") + for access in iface.get("accessConfigs", []): + external_ip = access.get("natIP", "") + break + + # Save state + state = DeploymentState( + instance_name=config.instance_name, + project=config.project, + zone=config.zone, + external_ip=external_ip, + internal_ip=internal_ip, + status="RUNNING", + created_at=datetime.now().isoformat(), + boot_image=boot_image, + shared_image=shared_image, + ) + self.save_state(state) + + logger.info("") + logger.info("=== Deployment Complete ===") + logger.info(f"Instance: {config.instance_name}") + logger.info(f"External IP: {external_ip}") + logger.info(f"Internal IP: {internal_ip}") + logger.info("") + logger.info("To check serial output:") + logger.info(f" dstack-cloud logs") + + def _delete_instance_if_exists(self, config: GcpConfig) -> None: + """Delete instance if it exists.""" + result = self._run_gcloud([ + "compute", "instances", "describe", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}" + ], check=False) + + if result.returncode == 0: + logger.info(f"Deleting existing instance: {config.instance_name}") + self._run_gcloud([ + "compute", "instances", "delete", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}", + "--quiet" + ]) + + def _parse_env_file(self, file_path: Path) -> Dict[str, str]: + """Parse an environment file where each line is formatted as KEY=Value.""" + if not file_path or not file_path.exists(): + return {} + + envs = {} + with open(file_path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' not in line: + continue + key, value = line.split('=', 1) + envs[key.strip()] = value.strip() + return envs + + def _encrypt_env(self, envs: Dict[str, str], hex_public_key: str) -> str: + """ + Encrypt environment variables using X25519 key exchange and AES-GCM. + + Args: + envs: Environment variables dictionary + hex_public_key: Remote encryption public key in hexadecimal format + + Returns: + Hex string of (ephemeral public key || IV || ciphertext) + """ + if not CRYPTO_AVAILABLE: + raise ImportError( + "Cryptography libraries not available. Please install:\n" + "pip install cryptography" + ) + + # Serialize environment variables to JSON + envs_json = json.dumps({"env": envs}).encode("utf-8") + + # Remove "0x" prefix if present + if hex_public_key.startswith("0x"): + hex_public_key = hex_public_key[2:] + + # Convert hex public key to bytes + remote_pubkey_bytes = bytes.fromhex(hex_public_key) + + # Generate ephemeral X25519 key pair + ephemeral_private_key = x25519.X25519PrivateKey.generate() + ephemeral_public_key = ephemeral_private_key.public_key() + + # Compute shared secret using X25519 + peer_public_key = x25519.X25519PublicKey.from_public_bytes(remote_pubkey_bytes) + shared = ephemeral_private_key.exchange(peer_public_key) + + # Use shared secret as key for AES-GCM (32 bytes for AES-256) + aesgcm = AESGCM(shared) + iv = os.urandom(12) # 12-byte nonce for AES-GCM + ciphertext = aesgcm.encrypt(iv, envs_json, None) + + # Serialize ephemeral public key to raw bytes + ephemeral_public_bytes = ephemeral_public_key.public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw + ) + + # Combine ephemeral public key, IV, and ciphertext + result = ephemeral_public_bytes + iv + ciphertext + return result.hex() + + def _get_app_encrypt_pub_key(self, app_id: str, kms_url: str) -> str: + """Get encryption public key for the specified app_id from KMS.""" + try: + import urllib.request + import urllib.error + + path = f"{kms_url}/prpc/GetAppEnvEncryptPubKey?json" + data = json.dumps({"app_id": app_id}).encode("utf-8") + + req = urllib.request.Request( + path, + data=data, + headers={"Content-Type": "application/json"} + ) + + logger.info(f"Getting encryption public key for {app_id} from {kms_url}") + with urllib.request.urlopen(req, timeout=10) as response: + response_data = json.loads(response.read().decode("utf-8")) + + if "public_key" not in response_data: + raise ValueError(f"No public_key in response: {response_data}") + + return response_data["public_key"] + + except Exception as e: + logger.warning(f"Failed to get encryption public key: {e}") + raise + + def _derive_instance_id(self, instance_id_seed: str, app_id: str) -> str: + """Derive instance_id from instance_id_seed and app_id. + + Both instance_id_seed and app_id are hex strings that need to be + decoded to bytes before concatenation, matching the Rust implementation. + """ + # Decode hex strings to bytes + seed_bytes = bytes.fromhex(instance_id_seed) + app_bytes = bytes.fromhex(app_id) + + # Concatenate bytes (not hex strings) + id_path = seed_bytes + app_bytes + + # Compute SHA256 and take first 20 bytes, then convert to hex string + instance_id = hashlib.sha256(id_path).digest()[:20] + return instance_id.hex() + + def _get_gateway_urls(self, app: App, instance_id: str) -> Dict[str, str]: + """Construct gateway URLs for app access. + + Returns: + Dict with 'app_url' and 'instance_url' keys + """ + if not app.gateway_enabled: + return {} + + global_config = self._load_global_config() + gateway_urls = global_config.get("services", {}).get("gateway_urls", []) + if not gateway_urls: + return {} + + # Try to get gateway info from RPC + gateway_info = None + for gateway_url in gateway_urls: + try: + info_url = f"{gateway_url}/prpc/Info" + result = subprocess.run( + ["curl", "-sk", info_url], + capture_output=True, + text=True, + timeout=5 + ) + if result.returncode == 0: + gateway_info = json.loads(result.stdout) + break + except Exception as e: + logger.debug(f"Failed to get gateway info from {gateway_url}: {e}") + continue + + if not gateway_info: + return {} + + base_domain = gateway_info.get("base_domain") + external_port = gateway_info.get("external_port") + + if not base_domain or not external_port: + return {} + + # Construct URLs: one with instance_id, one with app_id + app_id = app.app_id + app_url = f"https://{app_id}-8090.{base_domain}:{external_port}/" + instance_url = f"https://{instance_id}-8090.{base_domain}:{external_port}/" + + return { + "app_url": app_url, + "instance_url": instance_url + } + + def status(self) -> None: + """Check deployment status.""" + state = self.load_state() + if not state or not state.instance_name: + logger.info("No deployment found. Run 'dstack-cloud deploy' first.") + return + + # Get current status from GCP + result = self._run_gcloud([ + "compute", "instances", "describe", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}", + "--format=json" + ], check=False) + + if result.returncode != 0: + logger.info(f"Instance '{state.instance_name}' not found in GCP") + state.status = "NOT_FOUND" + self.save_state(state) + return + + instance_info = json.loads(result.stdout) + status = instance_info.get("status", "UNKNOWN") + + # Update IPs + external_ip = "" + internal_ip = "" + for iface in instance_info.get("networkInterfaces", []): + internal_ip = iface.get("networkIP", "") + for access in iface.get("accessConfigs", []): + external_ip = access.get("natIP", "") + break + + state.status = status + state.external_ip = external_ip + state.internal_ip = internal_ip + self.save_state(state) + + print(f"Instance: {state.instance_name}") + print(f"Project: {state.project}") + print(f"Zone: {state.zone}") + print(f"Status: {status}") + print(f"External IP: {external_ip or 'N/A'}") + print(f"Internal IP: {internal_ip or 'N/A'}") + print(f"Boot Image: {state.boot_image}") + print(f"Created: {state.created_at}") + print(f"Updated: {state.updated_at}") + + # Display gateway URLs if enabled + # Try to load from .instance_info first (actual deployed values) + app = self.load_app_config() + if not app.gateway_enabled: + return + + instance_id_seed = None + app_id = None + + # Try to read from shared/.instance_info (actual deployed values) + instance_info_path = self._get_shared_dir() / ".instance_info" + if instance_info_path.exists(): + try: + with open(instance_info_path, 'r') as f: + instance_info_data = json.load(f) + instance_id_seed = instance_info_data.get("instance_id_seed") + app_id = instance_info_data.get("app_id") + except Exception as e: + logger.debug(f"Failed to read .instance_info: {e}") + + # Fallback to app.json if .instance_info not found or missing values + if not instance_id_seed or not app_id: + instance_id_seed = app.instance_id_seed + app_id = app.app_id + + if instance_id_seed and app_id: + instance_id = self._derive_instance_id(instance_id_seed, app_id) + gateway_urls = self._get_gateway_urls(app, instance_id) + if gateway_urls: + print("") + print("Gateway URLs:") + print(f" App URL: {gateway_urls.get('app_url', 'N/A')}") + print(f" Instance URL: {gateway_urls.get('instance_url', 'N/A')}") + + def logs(self, follow: bool = False, lines: int = 100) -> None: + """View serial console logs.""" + state = self.load_state() + if not state or not state.instance_name: + raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") + + if follow: + # Tail logs continuously + logger.info(f"Following serial output for {state.instance_name}...") + logger.info("Press Ctrl+C to stop") + + last_output = "" + while True: + try: + result = self._run_gcloud([ + "compute", "instances", "get-serial-port-output", + state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ], check=False) + + if result.returncode == 0: + output = result.stdout + if output != last_output: + # Print only new content + if last_output: + new_content = output[len(last_output):] + if new_content: + print(new_content, end="", flush=True) + else: + # First time, print last N lines + lines_list = output.split('\n') + print('\n'.join(lines_list[-lines:]), flush=True) + last_output = output + + time.sleep(2) + except KeyboardInterrupt: + print("\nStopped following logs.") + break + else: + # Get logs once + result = self._run_gcloud([ + "compute", "instances", "get-serial-port-output", + state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ]) + + output = result.stdout + lines_list = output.split('\n') + print('\n'.join(lines_list[-lines:])) + + def stop(self) -> None: + """Stop the VM.""" + state = self.load_state() + if not state or not state.instance_name: + raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") + + logger.info(f"Stopping instance {state.instance_name}...") + self._run_gcloud([ + "compute", "instances", "stop", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ]) + + state.status = "STOPPED" + self.save_state(state) + logger.info("Instance stopped.") + + def start(self) -> None: + """Start a stopped VM.""" + state = self.load_state() + if not state or not state.instance_name: + raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") + + logger.info(f"Starting instance {state.instance_name}...") + self._run_gcloud([ + "compute", "instances", "start", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ]) + + # Update state with new IP + self.status() + logger.info("Instance started.") + + def remove(self, keep_images: bool = False) -> None: + """Remove the VM and cleanup.""" + state = self.load_state() + if not state or not state.instance_name: + logger.info("No deployment found.") + return + + # Delete instance + logger.info(f"Deleting instance {state.instance_name}...") + self._run_gcloud([ + "compute", "instances", "delete", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}", + "--quiet" + ], check=False) + + if not keep_images and state.shared_image: + # Delete shared disk image + logger.info(f"Deleting shared disk image {state.shared_image}...") + self._run_gcloud([ + "compute", "images", "delete", state.shared_image, + f"--project={state.project}", + "--quiet" + ], check=False) + + # Clear state + state.status = "REMOVED" + state.external_ip = "" + state.internal_ip = "" + self.save_state(state) + + logger.info("Instance removed.") + + def list_deployments(self, project: Optional[str] = None) -> None: + """List all dstack deployments in a project.""" + if not project: + # Try to get from config + try: + config = self.load_gcp_config() + project = config.project + except FileNotFoundError: + # Try global config + global_config = self._load_global_config() + project = global_config.get("gcp", {}).get("project", "") + + if not project: + raise ValueError("Project is required. Specify with --project or configure it.") + + result = self._run_gcloud([ + "compute", "instances", "list", + f"--project={project}", + "--filter=name~^dstack-", + "--format=table(name,zone,status,networkInterfaces[0].accessConfigs[0].natIP:label=EXTERNAL_IP,creationTimestamp)" + ], capture=False) + + +def main(): + parser = argparse.ArgumentParser( + description="Multi-cloud VM lifecycle management tool", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Create a new project + dstack-cloud new myproject + + # Initialize current directory + dstack-cloud init + + # Edit global configuration + dstack-cloud config-edit + + # Download OS image + dstack-cloud pull dstack-0.6.0 + + # Deploy VM + dstack-cloud deploy + + # Check status + dstack-cloud status + + # View logs + dstack-cloud logs --follow + + # Stop/Start/Remove + dstack-cloud stop + dstack-cloud start + dstack-cloud remove +""" + ) + + parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") + parser.add_argument("-C", "--directory", type=str, help="Change to directory before running") + + subparsers = parser.add_subparsers(dest="command", help="Commands") + + # new command + new_parser = subparsers.add_parser("new", help="Create a new project") + new_parser.add_argument("name", type=str, help="Project name") + + # App configuration options + new_parser.add_argument("--os-image", type=str, help="OS image (e.g., dstack-0.6.0)") + new_parser.add_argument("--app-id", type=str, help="Application ID (40 hex chars)") + new_parser.add_argument("--gateway-enabled", "--gw", dest="gateway_enabled", action="store_true", help="Enable dstack-gateway") + new_parser.add_argument("--no-gateway-enabled", "--no-gw", dest="gateway_enabled", action="store_false") + new_parser.set_defaults(gateway_enabled=None) + new_parser.add_argument("--key-provider", "--kp", type=str, choices=["kms", "local", "tpm", "none"], help="Key provider type") + new_parser.add_argument("--storage-fs", "--fs", dest="storage_fs", type=str, choices=["ext4", "zfs"], help="Storage filesystem") + new_parser.add_argument("--secure-time", action="store_true", help="Enable secure time synchronization") + new_parser.add_argument("--no-secure-time", dest="secure_time", action="store_false") + new_parser.set_defaults(secure_time=None) + new_parser.add_argument("--no-instance-id", action="store_true", help="Disable instance ID generation") + + # GCP configuration options + new_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + new_parser.add_argument("--zone", "-z", type=str, help="GCP zone (e.g., us-central1-a)") + new_parser.add_argument("--instance-name", type=str, help="GCP instance name") + new_parser.add_argument("--machine-type", "-m", type=str, help="Machine type (e.g., c3-standard-4)") + new_parser.add_argument("--data-size", type=int, help="Data disk size in GB") + + # init command + init_parser = subparsers.add_parser("init", help="Initialize deployment configuration") + init_parser.add_argument("--force", "-f", action="store_true", help="Overwrite existing config") + init_parser.add_argument("--non-interactive", "-n", action="store_true", help="Skip interactive prompts") + + # App configuration options (same as new) + init_parser.add_argument("--os-image", type=str, help="OS image (e.g., dstack-0.6.0)") + init_parser.add_argument("--app-id", type=str, help="Application ID (40 hex chars)") + init_parser.add_argument("--gateway-enabled", "--gw", dest="gateway_enabled", action="store_true", help="Enable dstack-gateway") + init_parser.add_argument("--no-gateway-enabled", "--no-gw", dest="gateway_enabled", action="store_false") + init_parser.set_defaults(gateway_enabled=None) + init_parser.add_argument("--key-provider", "--kp", type=str, choices=["kms", "local", "tpm", "none"], help="Key provider type") + init_parser.add_argument("--storage-fs", "--fs", dest="storage_fs", type=str, choices=["ext4", "zfs"], help="Storage filesystem") + init_parser.add_argument("--secure-time", action="store_true", help="Enable secure time synchronization") + init_parser.add_argument("--no-secure-time", dest="secure_time", action="store_false") + init_parser.set_defaults(secure_time=None) + init_parser.add_argument("--no-instance-id", action="store_true", help="Disable instance ID generation") + + # GCP configuration options (same as new) + init_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + init_parser.add_argument("--zone", "-z", type=str, help="GCP zone (e.g., us-central1-a)") + init_parser.add_argument("--instance-name", type=str, help="GCP instance name") + init_parser.add_argument("--machine-type", "-m", type=str, help="Machine type (e.g., c3-standard-4)") + init_parser.add_argument("--data-size", type=int, help="Data disk size in GB") + + # config-edit command + subparsers.add_parser("config-edit", help="Edit global configuration") + + # prepare command + subparsers.add_parser("prepare", help="Generate shared files") + + # pull command + pull_parser = subparsers.add_parser("pull", help="Download OS image") + pull_parser.add_argument("image", type=str, help="OS image name (e.g., dstack-0.6.0)") + + # deploy command + deploy_parser = subparsers.add_parser("deploy", help="Deploy VM to cloud") + deploy_parser.add_argument("--delete", "-d", action="store_true", + help="Delete existing instance first") + deploy_parser.add_argument("--force-boot-image", action="store_true", + help="Force re-upload boot image") + + # status command + subparsers.add_parser("status", help="Check deployment status") + + # logs command + logs_parser = subparsers.add_parser("logs", help="View serial console logs") + logs_parser.add_argument("--follow", "-f", action="store_true", help="Follow log output") + logs_parser.add_argument("--lines", "-n", type=int, default=100, help="Number of lines to show") + + # stop command + subparsers.add_parser("stop", help="Stop the VM") + + # start command + subparsers.add_parser("start", help="Start a stopped VM") + + # remove command + remove_parser = subparsers.add_parser("remove", help="Remove the VM and cleanup") + remove_parser.add_argument("--keep-images", action="store_true", + help="Keep disk images in GCP") + + # list command + list_parser = subparsers.add_parser("list", help="List all deployments") + list_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + work_dir = args.directory if args.directory else None + manager = CloudDeploymentManager(work_dir) + + try: + if args.command == "new": + manager.new( + args.name, + os_image=args.os_image, + app_id=args.app_id, + gateway_enabled=args.gateway_enabled, + key_provider=args.key_provider, + storage_fs=args.storage_fs, + secure_time=args.secure_time, + no_instance_id=args.no_instance_id, + project=args.project, + zone=args.zone, + instance_name=args.instance_name, + machine_type=args.machine_type, + data_size=args.data_size + ) + elif args.command == "init": + manager.init( + force=args.force, + interactive=not args.non_interactive, + os_image=args.os_image, + app_id=args.app_id, + gateway_enabled=args.gateway_enabled, + key_provider=args.key_provider, + storage_fs=args.storage_fs, + secure_time=args.secure_time, + no_instance_id=args.no_instance_id, + project=args.project, + zone=args.zone, + instance_name=args.instance_name, + machine_type=args.machine_type, + data_size=args.data_size + ) + elif args.command == "config-edit": + manager.config_edit() + elif args.command == "prepare": + manager.prepare() + elif args.command == "pull": + manager.pull(args.image) + elif args.command == "deploy": + manager.deploy( + delete_existing=args.delete, + force_boot_image=args.force_boot_image + ) + elif args.command == "status": + manager.status() + elif args.command == "logs": + manager.logs(follow=args.follow, lines=args.lines) + elif args.command == "stop": + manager.stop() + elif args.command == "start": + manager.start() + elif args.command == "remove": + manager.remove(keep_images=args.keep_images) + elif args.command == "list": + manager.list_deployments(project=args.project) + else: + parser.print_help() + except Exception as e: + logger.error(str(e)) + if args.verbose: + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() From 9163e801ada76eaa6e7cb7f9e20d65bd61f44f3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 24 Dec 2025 09:21:17 +0000 Subject: [PATCH 238/435] Sync dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index f1a49f01a..448e606cb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f1a49f01a74c03c2034587dcf624bfc9c918ef20 +Subproject commit 448e606cb970231d56a4dfed211ef37f827b5e55 From 373ba086584d81a028a65655093af4f7d8109fee Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 8 Jan 2026 12:23:09 +0000 Subject: [PATCH 239/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 448e606cb..ffb41d9a1 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 448e606cb970231d56a4dfed211ef37f827b5e55 +Subproject commit ffb41d9a14bbff196a5b77b43a988a7507d56191 From 44a44446a8710c24db9a29604e20e42d0b301233 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 8 Jan 2026 13:28:32 +0000 Subject: [PATCH 240/435] Update url of dstack submodule --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 415c27ab0..fb2a5ce18 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,7 +23,7 @@ url = https://github.com/Dstack-TEE/meta-rust-bin [submodule "dstack"] path = dstack - url = https://github.com/Dstack-TEE/dstack + url = git@github.com:Phala-Network/dstack-gcp.git [submodule "meta-security"] path = meta-security url = https://github.com/Dstack-TEE/meta-security.git From ea76d255756be93e4f28faf65a9ba30523f7122e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Jan 2026 02:10:13 +0000 Subject: [PATCH 241/435] Fix reproducibility issues --- dstack | 2 +- .../dstack-zfs/dstack-zfs_2.4.0.bb | 1 + meta-security | 2 +- repro-build/Dockerfile.repro | 6 ++- repro-build/check.sh | 44 ++++++++++++++++--- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/dstack b/dstack index ffb41d9a1..23ccf0080 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit ffb41d9a14bbff196a5b77b43a988a7507d56191 +Subproject commit 23ccf00807806f1dbec9eaba81e8105e260e0979 diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb index 93d18bfa2..ced85263a 100644 --- a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb +++ b/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb @@ -68,6 +68,7 @@ FILES:${PN}-dev += "\ " # Skip buildpaths QA check for kernel modules INSANE_SKIP:${PN} += "buildpaths" +INSANE_SKIP:${PN}-dbg += "buildpaths" # Not yet ported to rv32 COMPATIBLE_HOST:riscv32 = "null" diff --git a/meta-security b/meta-security index a54650c74..323f5a448 160000 --- a/meta-security +++ b/meta-security @@ -1 +1 @@ -Subproject commit a54650c743b7534f60fc6ee1df9b57a8f0150a4d +Subproject commit 323f5a448dcaba65e36d8d63cf20eb127cf91a5a diff --git a/repro-build/Dockerfile.repro b/repro-build/Dockerfile.repro index 00c3084d1..b6951f59a 100644 --- a/repro-build/Dockerfile.repro +++ b/repro-build/Dockerfile.repro @@ -26,7 +26,11 @@ RUN apt update && apt install -y \ libclang-dev \ xorriso \ cpio \ - zstd + zstd \ + gdisk \ + dosfstools \ + mtools \ + libsqlite3-dev # Generate locale for bitbake RUN locale-gen en_US.UTF-8 diff --git a/repro-build/check.sh b/repro-build/check.sh index 74df22610..e83e257b3 100755 --- a/repro-build/check.sh +++ b/repro-build/check.sh @@ -22,6 +22,30 @@ BB_DIR_B=${BB_DIR_B:-${BUILD_DIR_B}/bb-build} ROOTFS_A=${BB_DIR_A}/${ROOTFS_PATH} ROOTFS_B=${BB_DIR_B}/${ROOTFS_PATH} +# Only compare these image paths (relative to $BUILD_DIR_*/images). +# Everything else is ignored to avoid known non-reproducible artifacts. +COMPARE_IMAGE_WHITELIST=( + "bzImage" + "digest.txt" + "initramfs.cpio.gz" + "metadata.json" + "ovmf.fd" + "rootfs.img.verity" + "sha256sum.txt" + "gcp/efi-root/EFI/BOOT/BOOTX64.EFI" +) + +is_whitelisted_image() { + local rel_path="$1" + local item + for item in "${COMPARE_IMAGE_WHITELIST[@]}"; do + if [ "$rel_path" = "$item" ] || [[ "$rel_path" == */"$item" ]]; then + return 0 + fi + done + return 1 +} + check_files() { local path_a="$1" local path_b="$2" @@ -121,24 +145,30 @@ analyze() { check_images() { echo -e "${YELLOW}Checking image files...${NC}" - find $BUILD_DIR_A/images -type f | while read file_a; do - rel_path=$(echo ${file_a} | sed "s#${BUILD_DIR_A}/images/##g") - file_b=$BUILD_DIR_B/images/$rel_path + local differences=0 + while IFS= read -r file_a; do + rel_path=$(echo "$file_a" | sed "s#${BUILD_DIR_A}/images/##g") + file_b="$BUILD_DIR_B/images/$rel_path" + if ! is_whitelisted_image "$rel_path"; then + continue + fi if [ ! -f "$file_b" ]; then echo -e "${RED}$rel_path is not found in $BUILD_DIR_B/images/${NC}" + differences=$((differences + 1)) continue fi - hash_a=$(md5sum $file_a | cut -d' ' -f 1) - hash_b=$(md5sum $file_b | cut -d' ' -f 1) + hash_a=$(md5sum "$file_a" | cut -d' ' -f 1) + hash_b=$(md5sum "$file_b" | cut -d' ' -f 1) if [ "$hash_a" != "$hash_b" ]; then echo -e "${RED}Hash mismatch for $rel_path:${NC}" echo -e "${RED}$hash_a $file_a${NC}" echo -e "${RED}$hash_b $file_b${NC}" - return 1 + differences=$((differences + 1)) else echo -e "${GREEN}Match for $rel_path${NC}" fi - done + done < <(find "$BUILD_DIR_A/images" -type f) + return $differences } check() { From 268ce48628b2cb93c27d2d2df447a6d0284affd1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Jan 2026 08:05:32 +0000 Subject: [PATCH 242/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 23ccf0080..9df8aa1fa 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 23ccf00807806f1dbec9eaba81e8105e260e0979 +Subproject commit 9df8aa1fa8f807e0a74181831b210dca21b8b2d0 From d06a4d66510db9f8cd51cbcb87e3005451e7453d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Jan 2026 12:32:53 +0000 Subject: [PATCH 243/435] dstack-cloud: Add fw command group --- scripts/bin/dstack-cloud | 356 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 336 insertions(+), 20 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 4f0de517e..09e540fe3 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -22,6 +22,10 @@ Usage: dstack-cloud start # Start a stopped VM dstack-cloud remove # Remove the VM and cleanup dstack-cloud list # List all deployments + dstack-cloud fw allow # Allow traffic on a port + dstack-cloud fw deny # Block traffic on a port + dstack-cloud fw remove # Remove a firewall rule + dstack-cloud fw list # List firewall rules """ import argparse @@ -1216,9 +1220,27 @@ class CloudDeploymentManager: logger.info(f"Shared Directory: {shared_dir}") logger.info(f"GCS Bucket: {config.bucket}") - # Delete existing instance if requested - if delete_existing: - self._delete_instance_if_exists(config) + # Check if instance already exists + result = self._run_gcloud([ + "compute", "instances", "describe", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}" + ], check=False) + + if result.returncode == 0: + if delete_existing: + logger.info(f"Deleting existing instance: {config.instance_name}") + self._run_gcloud([ + "compute", "instances", "delete", config.instance_name, + f"--zone={config.zone}", + f"--project={config.project}", + "--quiet" + ]) + else: + raise RuntimeError( + f"Instance '{config.instance_name}' already exists. " + f"Use --delete to replace it." + ) # Check and upload boot image boot_image = self._check_and_upload_boot_image(config, app, force=force_boot_image) @@ -1295,23 +1317,6 @@ class CloudDeploymentManager: logger.info("To check serial output:") logger.info(f" dstack-cloud logs") - def _delete_instance_if_exists(self, config: GcpConfig) -> None: - """Delete instance if it exists.""" - result = self._run_gcloud([ - "compute", "instances", "describe", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}" - ], check=False) - - if result.returncode == 0: - logger.info(f"Deleting existing instance: {config.instance_name}") - self._run_gcloud([ - "compute", "instances", "delete", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}", - "--quiet" - ]) - def _parse_env_file(self, file_path: Path) -> Dict[str, str]: """Parse an environment file where each line is formatted as KEY=Value.""" if not file_path or not file_path.exists(): @@ -1698,6 +1703,247 @@ class CloudDeploymentManager: "--format=table(name,zone,status,networkInterfaces[0].accessConfigs[0].natIP:label=EXTERNAL_IP,creationTimestamp)" ], capture=False) + def _get_firewall_rule_name(self, instance_name: str, port: int, protocol: str, + action: str = "allow") -> str: + """Generate firewall rule name for an instance port.""" + return f"{instance_name}-{action}-{protocol}-{port}" + + def _parse_port_spec(self, port_spec: str) -> tuple: + """Parse port specification like '8080' or '53/udp'. + + Returns: + tuple: (port: int, protocol: str) + """ + if "/" in port_spec: + port_str, protocol = port_spec.split("/", 1) + protocol = protocol.lower() + if protocol not in ("tcp", "udp"): + raise ValueError(f"Invalid protocol '{protocol}'. Must be 'tcp' or 'udp'.") + else: + port_str = port_spec + protocol = "tcp" + + try: + port = int(port_str) + except ValueError: + raise ValueError(f"Invalid port number '{port_str}'.") + + if not (1 <= port <= 65535): + raise ValueError(f"Port {port} out of range (1-65535).") + + return port, protocol + + def _get_project_for_firewall(self) -> str: + """Get project ID for firewall operations.""" + # Try state first + state = self.load_state() + if state and state.project: + return state.project + + # Try config + try: + config = self.load_gcp_config() + if config.project: + return config.project + except FileNotFoundError: + pass + + # Try global config + global_config = self._load_global_config() + project = global_config.get("gcp", {}).get("project", "") + + if not project: + raise ValueError("Project is required. Deploy first or specify with --project.") + + return project + + def _get_instance_name_for_firewall(self) -> str: + """Get instance name for firewall operations.""" + state = self.load_state() + if state and state.instance_name: + return state.instance_name + + try: + config = self.load_gcp_config() + if config.instance_name: + return config.instance_name + except FileNotFoundError: + pass + + raise ValueError("Instance name is required. Deploy first or specify with --instance.") + + def _ensure_instance_tag(self, instance_name: str, project: str) -> str: + """Ensure instance has the firewall tag, return the tag name.""" + instance_tag = f"fw-{instance_name}" + + state = self.load_state() + zone = state.zone if state else "us-central1-a" + + result = self._run_gcloud([ + "compute", "instances", "describe", instance_name, + f"--project={project}", + f"--zone={zone}", + "--format=value(tags.items)" + ], check=False) + + current_tags = result.stdout.strip().split(";") if result.stdout.strip() else [] + current_tags = [t.strip() for t in current_tags if t.strip()] + + if instance_tag not in current_tags: + logger.info(f"Adding tag '{instance_tag}' to instance '{instance_name}'...") + self._run_gcloud([ + "compute", "instances", "add-tags", instance_name, + f"--project={project}", + f"--zone={zone}", + f"--tags={instance_tag}" + ]) + logger.info(f"Added tag '{instance_tag}' to instance") + else: + logger.debug(f"Instance already has tag '{instance_tag}'") + + return instance_tag + + def fw_allow(self, port_spec: str, + source_ranges: Optional[List[str]] = None, + instance_name: Optional[str] = None, + project: Optional[str] = None) -> None: + """Add firewall rule to open a port for the instance.""" + port, protocol = self._parse_port_spec(port_spec) + project = project or self._get_project_for_firewall() + instance_name = instance_name or self._get_instance_name_for_firewall() + + if source_ranges is None: + source_ranges = ["0.0.0.0/0"] + + # Always ensure instance has the tag first + instance_tag = self._ensure_instance_tag(instance_name, project) + + rule_name = self._get_firewall_rule_name(instance_name, port, protocol) + + # Check if rule already exists + result = self._run_gcloud([ + "compute", "firewall-rules", "describe", rule_name, + f"--project={project}" + ], check=False) + + if result.returncode == 0: + logger.info(f"Firewall rule '{rule_name}' already exists") + return + + # Create firewall rule targeting this instance's tag + logger.info(f"Creating firewall rule '{rule_name}'...") + self._run_gcloud([ + "compute", "firewall-rules", "create", rule_name, + f"--project={project}", + f"--allow={protocol}:{port}", + f"--source-ranges={','.join(source_ranges)}", + f"--target-tags={instance_tag}", + f"--description=Allow {protocol.upper()} port {port} for {instance_name}" + ]) + + logger.info(f"Opened {protocol.upper()} port {port} for instance '{instance_name}'") + + def fw_deny(self, port_spec: str, + source_ranges: Optional[List[str]] = None, + instance_name: Optional[str] = None, + project: Optional[str] = None) -> None: + """Create a deny firewall rule to block traffic on a port.""" + port, protocol = self._parse_port_spec(port_spec) + project = project or self._get_project_for_firewall() + instance_name = instance_name or self._get_instance_name_for_firewall() + + if source_ranges is None: + source_ranges = ["0.0.0.0/0"] + + # Always ensure instance has the tag first + instance_tag = self._ensure_instance_tag(instance_name, project) + + rule_name = self._get_firewall_rule_name(instance_name, port, protocol, action="deny") + + # Check if rule already exists + result = self._run_gcloud([ + "compute", "firewall-rules", "describe", rule_name, + f"--project={project}" + ], check=False) + + if result.returncode == 0: + logger.info(f"Firewall rule '{rule_name}' already exists") + return + + # Create deny firewall rule with high priority (low number = high priority) + logger.info(f"Creating deny firewall rule '{rule_name}'...") + self._run_gcloud([ + "compute", "firewall-rules", "create", rule_name, + f"--project={project}", + "--action=DENY", + f"--rules={protocol}:{port}", + f"--source-ranges={','.join(source_ranges)}", + f"--target-tags={instance_tag}", + "--priority=900", + f"--description=Deny {protocol.upper()} port {port} for {instance_name}" + ]) + + logger.info(f"Blocked {protocol.upper()} port {port} for instance '{instance_name}'") + + def fw_remove(self, port_spec: str, + instance_name: Optional[str] = None, + project: Optional[str] = None) -> None: + """Remove a firewall rule (allow or deny) for a port.""" + port, protocol = self._parse_port_spec(port_spec) + project = project or self._get_project_for_firewall() + instance_name = instance_name or self._get_instance_name_for_firewall() + + # Try to delete both allow and deny rules + deleted = False + for action in ["allow", "deny"]: + rule_name = self._get_firewall_rule_name(instance_name, port, protocol, action=action) + + # Check if rule exists + result = self._run_gcloud([ + "compute", "firewall-rules", "describe", rule_name, + f"--project={project}" + ], check=False) + + if result.returncode == 0: + # Delete the rule + logger.info(f"Deleting firewall rule '{rule_name}'...") + self._run_gcloud([ + "compute", "firewall-rules", "delete", rule_name, + f"--project={project}", + "--quiet" + ]) + logger.info(f"Removed {action} rule for {protocol.upper()} port {port}") + deleted = True + + if not deleted: + logger.info(f"No firewall rules found for {protocol.upper()} port {port} on instance '{instance_name}'") + + def fw_list(self, instance_name: Optional[str] = None, + project: Optional[str] = None) -> None: + """List firewall rules for an instance.""" + project = project or self._get_project_for_firewall() + + if instance_name: + # List rules for specific instance + instance_tag = f"fw-{instance_name}" + filter_expr = f"name~^{instance_name}-allow- OR targetTags:{instance_tag}" + else: + # Try to get instance name from state + try: + instance_name = self._get_instance_name_for_firewall() + filter_expr = f"name~^{instance_name}-allow-" + except ValueError: + # List all dstack-related firewall rules + filter_expr = "name~^dstack-" + + logger.info(f"Firewall rules for project '{project}':") + self._run_gcloud([ + "compute", "firewall-rules", "list", + f"--project={project}", + f"--filter={filter_expr}", + "--format=table(name,direction,priority,allowed[].map().firewall_rule().list():label=ALLOW,sourceRanges.list():label=SRC_RANGES,targetTags.list():label=TARGET_TAGS)" + ], capture=False) + def main(): parser = argparse.ArgumentParser( @@ -1730,6 +1976,15 @@ Examples: dstack-cloud stop dstack-cloud start dstack-cloud remove + + # Firewall management + dstack-cloud fw allow 8080 # Allow TCP port 8080 + dstack-cloud fw allow 53/udp # Allow UDP port 53 + dstack-cloud fw allow 443 -s 10.0.0.0/8 # Allow port 443 from specific range + dstack-cloud fw deny 22 # Block TCP port 22 + dstack-cloud fw deny 22 -s 0.0.0.0/0 # Block port 22 from all sources + dstack-cloud fw remove 8080 # Remove firewall rule for port 8080 + dstack-cloud fw list # List firewall rules """ ) @@ -1827,6 +2082,39 @@ Examples: list_parser = subparsers.add_parser("list", help="List all deployments") list_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + # fw command group + fw_parser = subparsers.add_parser("fw", help="Firewall management") + fw_subparsers = fw_parser.add_subparsers(dest="fw_command", help="Firewall commands") + + # fw allow + fw_allow_parser = fw_subparsers.add_parser("allow", help="Open a port for the instance") + fw_allow_parser.add_argument("port", type=str, help="Port to open (e.g., 8080, 53/udp)") + fw_allow_parser.add_argument("--source", "-s", type=str, action="append", + dest="source_ranges", + help="Source IP ranges (default: 0.0.0.0/0). Can be specified multiple times.") + fw_allow_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") + fw_allow_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + + # fw deny + fw_deny_parser = fw_subparsers.add_parser("deny", help="Block traffic on a port") + fw_deny_parser.add_argument("port", type=str, help="Port to block (e.g., 8080, 53/udp)") + fw_deny_parser.add_argument("--source", "-s", type=str, action="append", + dest="source_ranges", + help="Source IP ranges to block (default: 0.0.0.0/0). Can be specified multiple times.") + fw_deny_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") + fw_deny_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + + # fw remove + fw_remove_parser = fw_subparsers.add_parser("remove", help="Remove a firewall rule") + fw_remove_parser.add_argument("port", type=str, help="Port to remove rule for (e.g., 8080, 53/udp)") + fw_remove_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") + fw_remove_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + + # fw list + fw_list_parser = fw_subparsers.add_parser("list", help="List firewall rules for instance") + fw_list_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") + fw_list_parser.add_argument("--project", "-p", type=str, help="GCP project ID") + args = parser.parse_args() if args.verbose: @@ -1892,6 +2180,34 @@ Examples: manager.remove(keep_images=args.keep_images) elif args.command == "list": manager.list_deployments(project=args.project) + elif args.command == "fw": + if args.fw_command == "allow": + manager.fw_allow( + port_spec=args.port, + source_ranges=args.source_ranges, + instance_name=args.instance, + project=args.project + ) + elif args.fw_command == "deny": + manager.fw_deny( + port_spec=args.port, + source_ranges=args.source_ranges, + instance_name=args.instance, + project=args.project + ) + elif args.fw_command == "remove": + manager.fw_remove( + port_spec=args.port, + instance_name=args.instance, + project=args.project + ) + elif args.fw_command == "list": + manager.fw_list( + instance_name=args.instance, + project=args.project + ) + else: + fw_parser.print_help() else: parser.print_help() except Exception as e: From 3dea1d50349b8d250512b33a56540a78a4db934b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Jan 2026 12:37:15 +0000 Subject: [PATCH 244/435] dstack-cloud: Disable .env when kms is off --- scripts/bin/dstack-cloud | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 09e540fe3..025e83fe4 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -675,11 +675,12 @@ class CloudDeploymentManager: f.write("# Prelaunch script - runs before starting containers\n") os.chmod(prelaunch, 0o755) - # Create .env template at project root - env_file = self.work_dir / ".env" - if not env_file.exists() or force: - with open(env_file, 'w') as f: - f.write("# Environment variables\n") + # Create .env template at project root (only for KMS mode) + if app_template["key_provider"] == "kms": + env_file = self.work_dir / ".env" + if not env_file.exists() or force: + with open(env_file, 'w') as f: + f.write("# Environment variables\n") # Create user-config template at project root user_config = self.work_dir / ".user-config" @@ -699,7 +700,8 @@ class CloudDeploymentManager: logger.info(f" shared/ - System-generated files") logger.info(f" {docker_compose.name} - Docker compose file") logger.info(f" {prelaunch.name} - Prelaunch script") - logger.info(f" {env_file.name} - Environment variables") + if app_template["key_provider"] == "kms": + logger.info(f" .env - Environment variables") logger.info(f" .user-config - User configuration") logger.info("") logger.info("Edit the configuration files to customize your deployment.") From 8aa54095dfd24d04dd7ebfd8255bdc36d8498ef9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 16 Jan 2026 02:39:09 +0000 Subject: [PATCH 245/435] dstack-cloud: Auto upload data-disk image --- scripts/bin/dstack-cloud | 81 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 025e83fe4..30fea2cba 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -232,6 +232,7 @@ class DeploymentState: created_at: str = "" updated_at: str = "" boot_image: str = "" + data_image: str = "" shared_image: str = "" def to_dict(self) -> Dict[str, Any]: @@ -459,6 +460,80 @@ class CloudDeploymentManager: return result + def _ensure_data_disk_image(self, config: GcpConfig) -> str: + """Ensure the data disk image exists, creating it if necessary. + + Creates a minimal disk image with GPT partition table and a partition + labeled 'dstack-data' so the guest can discover it. + + Returns the image name to use. + """ + image_name = config.data_image + + # Check if image already exists + result = self._run_gcloud([ + "compute", "images", "describe", image_name, + f"--project={config.project}" + ], check=False) + + if result.returncode == 0: + logger.debug(f"Data disk image '{image_name}' already exists") + return image_name + + logger.info(f"Data disk image '{image_name}' not found, creating...") + + # Create a minimal raw disk image with GPT partition table + with tempfile.TemporaryDirectory() as tmpdir: + raw_file = os.path.join(tmpdir, "disk.raw") + + # Create a 10MB sparse file (enough for GPT) + disk_size_bytes = 10 * 1024 * 1024 + with open(raw_file, 'wb') as f: + f.truncate(disk_size_bytes) + + # Create GPT partition table with dstack-data partition using sgdisk + # -o: clear and create new GPT + # -n 1:0:0: create partition 1, start at first available, end at last available + # -c 1:dstack-data: set partition 1 name (PARTLABEL) to dstack-data + result = subprocess.run( + ["sgdisk", "-o", "-n", "1:0:0", "-c", "1:dstack-data", raw_file], + capture_output=True, text=True + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to create GPT partition table: {result.stderr}") + + logger.debug("Created GPT partition table with dstack-data label") + + # Compress to tar.gz for upload + tar_file = os.path.join(tmpdir, "disk.tar.gz") + result = subprocess.run( + ["tar", "-czf", tar_file, "-C", tmpdir, "disk.raw"], + capture_output=True, text=True + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") + + # Upload to GCS + gcs_path = f"{config.bucket}/{image_name}.tar.gz" + logger.info(f"Uploading data disk image to {gcs_path}...") + self._run_gsutil(["cp", tar_file, gcs_path]) + + # Create GCP image from the uploaded file + logger.info(f"Creating GCP image '{image_name}'...") + self._run_gcloud([ + "compute", "images", "create", image_name, + f"--project={config.project}", + f"--source-uri={gcs_path}", + "--guest-os-features=GVNIC" + ]) + + # Clean up GCS file + self._run_gsutil(["rm", gcs_path], check=False) + + logger.info(f"Created data disk image '{image_name}'") + + return image_name + def new( self, name: str, @@ -1250,6 +1325,9 @@ class CloudDeploymentManager: # Create shared disk image shared_image = self._create_shared_disk_image(config, app) + # Ensure data disk image exists (with GPT partition labeled 'dstack-data') + data_image = self._ensure_data_disk_image(config) + # Create TDX instance logger.info("Creating TDX instance...") @@ -1261,7 +1339,7 @@ class CloudDeploymentManager: "--confidential-compute-type=TDX", f"--image={boot_image}", "--boot-disk-size=10GB", - f"--create-disk=name={config.instance_name}-data,size={config.data_size}GB,type=pd-balanced,image={config.data_image},auto-delete=yes", + f"--create-disk=name={config.instance_name}-data,size={config.data_size}GB,type=pd-balanced,image={data_image},auto-delete=yes", f"--create-disk=name={config.instance_name}-shared,size=1GB,type=pd-balanced,image={shared_image},auto-delete=yes", "--maintenance-policy=TERMINATE", ] @@ -1306,6 +1384,7 @@ class CloudDeploymentManager: status="RUNNING", created_at=datetime.now().isoformat(), boot_image=boot_image, + data_image=data_image, shared_image=shared_image, ) self.save_state(state) From 5858bc0725bed0371f85257360ff24c08f9fd36a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 16 Jan 2026 02:45:09 +0000 Subject: [PATCH 246/435] Update dstack submodule url --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index fb2a5ce18..114b2e75a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,7 +23,7 @@ url = https://github.com/Dstack-TEE/meta-rust-bin [submodule "dstack"] path = dstack - url = git@github.com:Phala-Network/dstack-gcp.git + url = https://github.com/Phala-Network/dstack-gcp.git [submodule "meta-security"] path = meta-security url = https://github.com/Dstack-TEE/meta-security.git From 14b119f3ffa5e179987837115df257d54735e523 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 16 Jan 2026 02:48:58 +0000 Subject: [PATCH 247/435] Update dstack --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 9df8aa1fa..68558a9e3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 9df8aa1fa8f807e0a74181831b210dca21b8b2d0 +Subproject commit 68558a9e378e25b8e3c8a2bd82649745971685a8 From 2f1b299dd100d47f32a38925a2d8485fd3f103da Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:05:37 +0000 Subject: [PATCH 248/435] Volatile /etc/ --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4f4acc1bc..1acf99a68 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -142,11 +142,4 @@ mkdirs() { mkdir -p ${IMAGE_ROOTFS}/etc/wireguard mkdir -p ${IMAGE_ROOTFS}/var/lib/docker ln -sf dstack ${IMAGE_ROOTFS}/tapp - - # Aliases passwd files to a subdirectory - mkdir -p ${IMAGE_ROOTFS}/etc/users - mv ${IMAGE_ROOTFS}/etc/passwd ${IMAGE_ROOTFS}/etc/users/ - mv ${IMAGE_ROOTFS}/etc/shadow ${IMAGE_ROOTFS}/etc/users/ - ln -s users/passwd ${IMAGE_ROOTFS}/etc/passwd - ln -s users/shadow ${IMAGE_ROOTFS}/etc/shadow } From 642c225f71738bd5c2095679430e297caa1623b4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:54:50 +0000 Subject: [PATCH 249/435] Suppress the warn Text file busy in build.sh --- build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 054c59f34..1e1d207ad 100755 --- a/build.sh +++ b/build.sh @@ -113,7 +113,10 @@ EOF build_host() { echo "Building binaries" (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) - cp ${RUST_BUILD_DIR}/release/{dstack-gateway,dstack-kms,dstack-vmm,supervisor} . + for bin in dstack-gateway dstack-kms dstack-vmm supervisor; do + cp "${RUST_BUILD_DIR}/release/${bin}" ".${bin}.new" + mv -f ".${bin}.new" "./${bin}" + done } # Step 2: build guest images From 5beaebead54e4bb74b9476f748fe2219c5a661ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 01:54:32 +0000 Subject: [PATCH 250/435] Remove CF_ZONE_ID --- build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/build.sh b/build.sh index 1e1d207ad..a70ce8538 100755 --- a/build.sh +++ b/build.sh @@ -85,7 +85,6 @@ GATEWAY_PUBLIC_DOMAIN= # for certbot CERTBOT_ENABLED=false CF_API_TOKEN= -CF_ZONE_ID= ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory EOF if [ -f $CONFIG_FILE ]; then @@ -224,8 +223,6 @@ workdir = "$CERBOT_WORKDIR" acme_url = "$ACME_URL" # Cloudflare API token cf_api_token = "$CF_API_TOKEN" -# Cloudflare zone ID -cf_zone_id = "$CF_ZONE_ID" # Auto set CAA record auto_set_caa = true # Domain to issue certificates for From c195dc67f99338020795fbf670c00b2f5ad97963 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Dec 2025 02:08:53 +0000 Subject: [PATCH 251/435] ssh: Disable password auth --- .../openssh/files/disable-password-auth.conf | 8 ++++++++ .../recipes-connectivity/openssh/openssh_%.bbappend | 10 ++++++++++ 2 files changed, 18 insertions(+) create mode 100644 meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf create mode 100644 meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend diff --git a/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf b/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf new file mode 100644 index 000000000..47963f278 --- /dev/null +++ b/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf @@ -0,0 +1,8 @@ +# Disable all password-based authentication +PasswordAuthentication no +PermitEmptyPasswords no +KbdInteractiveAuthentication no + +# Only allow public key authentication +PubkeyAuthentication yes +PermitRootLogin prohibit-password diff --git a/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend new file mode 100644 index 000000000..81a2e54ab --- /dev/null +++ b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend @@ -0,0 +1,10 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI += "file://disable-password-auth.conf" + +do_install:append() { + install -d ${D}${sysconfdir}/ssh/sshd_config.d + install -m 0644 ${UNPACKDIR}/disable-password-auth.conf ${D}${sysconfdir}/ssh/sshd_config.d/ +} + +FILES:${PN}-sshd += "${sysconfdir}/ssh/sshd_config.d/" From 8b06b6ba6d256d51d7909d9c1781c7923e5e3cb4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 16 Jan 2026 02:56:21 +0000 Subject: [PATCH 252/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 7bf1843a8..3917376cb 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 7bf1843a8ddf877fbaebb9898a7f27a19a49ab73 +Subproject commit 3917376cbef1919041809088c3f587e2380a8673 From 8980c0901df699cdbfa28f1f77401eb5d0a7ec3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 01:47:57 +0000 Subject: [PATCH 253/435] Update submodules --- dstack | 2 +- meta-confidential-compute | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 68558a9e3..1791fcf68 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 68558a9e378e25b8e3c8a2bd82649745971685a8 +Subproject commit 1791fcf689875755d6024447a553b6ee7d949e91 diff --git a/meta-confidential-compute b/meta-confidential-compute index a9857da07..33d19bc72 160000 --- a/meta-confidential-compute +++ b/meta-confidential-compute @@ -1 +1 @@ -Subproject commit a9857da070b542dece7dd0630b3b89804234af7e +Subproject commit 33d19bc72b43876884ddb1626e8c0cdf41032038 From 4da51e905c1e7c69dd62017bbe3e6e42831cc3a3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 01:48:48 +0000 Subject: [PATCH 254/435] Update LICENSE --- LICENSE | 109 +++++++++++++++++++++++++++++++++++++++++++----------- README.md | 2 +- 2 files changed, 89 insertions(+), 22 deletions(-) diff --git a/LICENSE b/LICENSE index 7eb781026..fd94e8eaf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,88 @@ -MIT License - -Copyright (c) 2024 Phala Network - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Business Source License 1.1 + +Parameters + +Licensor: Phala Network +Licensed Work: dstack-cloud + The Licensed Work is (c) Phala Network +Additional Use Grant: None + +Change Date: Four years from the date a MINOR version (SemVer) is published. + +Change License: GNU Affero General Public License Version 3 (AGPL-3.0-only) + +Notice + +License text copyright (c) 2023 MariaDB plc, All Rights Reserved. +“Business Source License” is a trademark of MariaDB plc. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. \ No newline at end of file diff --git a/README.md b/README.md index 74c6b7cc2..f41d95a06 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,4 @@ cd meta-dstack/repro-build/ ## License -This project is licensed under the MIT License. See the LICENSE file for more details. +See the LICENSE file for more details. From 4f72c0ca734b36d6e2d91b94db8ed206f70f2ab8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 01:49:39 +0000 Subject: [PATCH 255/435] Update submodule URL --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 114b2e75a..850b16654 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,7 +23,7 @@ url = https://github.com/Dstack-TEE/meta-rust-bin [submodule "dstack"] path = dstack - url = https://github.com/Phala-Network/dstack-gcp.git + url = https://github.com/Phala-Network/dstack-cloud.git [submodule "meta-security"] path = meta-security url = https://github.com/Dstack-TEE/meta-security.git From d90c9fa06df443f3e84289a592d317d366178b15 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 01:52:12 +0000 Subject: [PATCH 256/435] Update URL in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f41d95a06..616469d22 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This project implements Yocto layer and the overall build scripts for dstack Bas ## Build -See https://github.com/Dstack-TEE/dstack for more details. +See https://github.com/Phala-Network/dstack-cloud for more details. ## Reproducible Build The Guest Image @@ -15,7 +15,7 @@ See https://github.com/Dstack-TEE/dstack for more details. ### Build commands ```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git +git clone https://github.com/Phala-Network/meta-dstack-cloud.git cd meta-dstack/repro-build/ ./repro-build.sh ``` From 9a88cf4d7b334ad4198dade18d216b755b0f0f8f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 03:08:58 +0000 Subject: [PATCH 257/435] Update dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 1791fcf68..4854ccc2f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 1791fcf689875755d6024447a553b6ee7d949e91 +Subproject commit 4854ccc2f8fa33838d3029c60803bda2bd6d2d08 From 78082be39a470980856eb89bb44ebcd3de755b06 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 12:08:15 +0000 Subject: [PATCH 258/435] Fix random fakeroot bb build error --- .../libnvidia-container/libnvidia-container_1.00.bb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index b533f734a..147a6dcfe 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -12,6 +12,8 @@ fakeroot do_unpack_modprobe() { addtask unpack_modprobe after do_unpack before do_patch do_unpack_modprobe[dirs] = "${S}" do_unpack_modprobe[vardeps] += "NVIDIA_MODPROBE_VERSION" +# Ensure pseudo (fakeroot worker) is staged before this fakeroot task runs. +do_unpack_modprobe[depends] += "pseudo-native:do_populate_sysroot" PACKAGECONFIG ??= "seccomp" PACKAGECONFIG[seccomp] = "WITH_SECCOMP=yes,WITH_SECCOMP=no,libseccomp" From 04288d031dd3bf633685c7896dbb48a2332da113 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 19 Jan 2026 12:53:41 +0000 Subject: [PATCH 259/435] bb: Rename image from dstack to dstack-cloud --- Makefile | 2 +- .../{dstack-nvidia-rootfs.bb => dstack-cloud-nvidia-rootfs.bb} | 0 .../images/{dstack-rootfs.bb => dstack-cloud-rootfs.bb} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename meta-dstack/recipes-core/images/{dstack-nvidia-rootfs.bb => dstack-cloud-nvidia-rootfs.bb} (100%) rename meta-dstack/recipes-core/images/{dstack-rootfs.bb => dstack-cloud-rootfs.bb} (100%) diff --git a/Makefile b/Makefile index 700cdb1dd..05de00917 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ DIST_DIR ?= ${BB_BUILD_DIR}/dist export BB_BUILD_DIR export DIST_DIR -DIST_NAMES ?= dstack dstack-dev dstack-nvidia dstack-nvidia-dev +DIST_NAMES ?= dstack-cloud dstack-cloud-nvidia ROOTFS_IMAGE_NAMES = $(addsuffix -rootfs,${DIST_NAMES}) all: dist diff --git a/meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb b/meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-nvidia-rootfs.bb rename to meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs.bb rename to meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb From 7cc276ff0ef82650c65b86ba000cfa35a604818a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 05:09:12 +0000 Subject: [PATCH 260/435] bb: Refactor image recipes to use multiconfig - Consolidate four separate image recipes (dstack-cloud-rootfs, dstack-dev-rootfs, dstack-cloud-nvidia-rootfs, dstack-nvidia-dev-rootfs) into a single dstack-rootfs.bb with conditional includes - Add multiconfig support with separate TMPDIR per flavor to avoid build conflicts - Build flavors serially in Makefile to avoid BitBake runqueue deadlock - Update mkimage.sh to support --flavor parameter for multiconfig builds - Copy multiconfig to new build directories in dev-setup --- .gitignore | 1 + Makefile | 29 +++++++++---- bb-build/conf/local.conf | 3 +- bb-build/conf/multiconfig/dev.conf | 7 ++++ bb-build/conf/multiconfig/nvidia-dev.conf | 7 ++++ bb-build/conf/multiconfig/nvidia.conf | 7 ++++ bb-build/conf/multiconfig/prod.conf | 7 ++++ dev-setup | 3 ++ .../recipes-core/dstack-guest/dstack-guest.bb | 4 +- .../dstack-ovmf/dstack-ovmf_git.bb | 6 +-- .../images/dstack-cloud-nvidia-rootfs.bb | 3 -- .../images/dstack-cloud-rootfs.bb | 1 - .../recipes-core/images/dstack-dev-rootfs.bb | 1 - .../images/dstack-nvidia-dev-rootfs.bb | 2 - .../recipes-core/images/dstack-rootfs-dev.inc | 2 - .../images/dstack-rootfs-prod.inc | 1 - .../recipes-core/images/dstack-rootfs.bb | 17 ++++++++ meta-dstack/recipes-core/images/dstack-uki.bb | 15 ++++--- .../ldconfig-compatibility-symlink_1.0.0.bb | 2 +- .../libnvidia-container.inc | 2 +- mkimage.sh | 42 ++++++++++++++----- 21 files changed, 118 insertions(+), 44 deletions(-) create mode 100644 bb-build/conf/multiconfig/dev.conf create mode 100644 bb-build/conf/multiconfig/nvidia-dev.conf create mode 100644 bb-build/conf/multiconfig/nvidia.conf create mode 100644 bb-build/conf/multiconfig/prod.conf delete mode 100644 meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb delete mode 100644 meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb delete mode 100644 meta-dstack/recipes-core/images/dstack-dev-rootfs.bb delete mode 100644 meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb create mode 100644 meta-dstack/recipes-core/images/dstack-rootfs.bb diff --git a/.gitignore b/.gitignore index c6460dc58..8e7fb513b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /build /bb-build +!/bb-build/conf/multiconfig/ *.qcow2 __pycache__/ /.target diff --git a/Makefile b/Makefile index 05de00917..65e868613 100644 --- a/Makefile +++ b/Makefile @@ -2,31 +2,44 @@ ifeq ($(BBPATH),) $(error BBPATH is not set. Run `source dev-setup` first) endif -.PHONY: all dist clean-dstack clean-initrd images +.PHONY: all dist clean-dstack clean-initrd images images-common images-flavors BB_BUILD_DIR ?= bb-build DIST_DIR ?= ${BB_BUILD_DIR}/dist export BB_BUILD_DIR export DIST_DIR -DIST_NAMES ?= dstack-cloud dstack-cloud-nvidia -ROOTFS_IMAGE_NAMES = $(addsuffix -rootfs,${DIST_NAMES}) +# Flavor names map to multiconfig names: prod, dev, nvidia, nvidia-dev +FLAVORS ?= prod dev nvidia nvidia-dev + +# Map flavor to dist name for mkimage.sh +flavor_to_dist = $(if $(filter prod,$1),dstack,$(if $(filter dev,$1),dstack-dev,$(if $(filter nvidia,$1),dstack-nvidia,$(if $(filter nvidia-dev,$1),dstack-nvidia-dev,$1)))) all: dist -include $(wildcard mk.d/*.mk) dist: images - $(foreach dist_name,${DIST_NAMES},./mkimage.sh --dist-name $(dist_name);) + $(foreach flavor,$(FLAVORS),./mkimage.sh --dist-name $(call flavor_to_dist,$(flavor)) --flavor $(flavor);) + +# Build common artifacts (shared across all flavors) +# dstack-guest is built here to avoid concurrent build conflicts in multiconfig +images-common: + bitbake virtual/kernel dstack-initramfs dstack-ovmf dstack-guest + +# Build flavor-specific artifacts using multiconfig (serial to avoid deadlock warnings) +images-flavors: + $(foreach flavor,$(FLAVORS),bitbake mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) -images: - bitbake virtual/kernel dstack-initramfs dstack-ovmf dstack-uki $(ROOTFS_IMAGE_NAMES) +images: images-common images-flavors clean: - bitbake -c cleansstate virtual/kernel dstack-initramfs dstack-ovmf $(ROOTFS_IMAGE_NAMES) + bitbake -c cleansstate virtual/kernel dstack-initramfs dstack-ovmf + $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) clean-dstack: - bitbake -c cleansstate dstack-guest $(ROOTFS_IMAGE_NAMES) + bitbake -c cleansstate dstack-guest + $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs;) clean-initrd: bitbake -c cleansstate dstack-initramfs diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index a51d8c24b..0e3e9b07a 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -290,4 +290,5 @@ BB_DISKMON_DIRS ??= "\ # this doesn't mean anything to you. CONF_VERSION = "2" -FETCHCMD_wget = "wget --progress=dot --inet4-only -c" \ No newline at end of file +FETCHCMD_wget = "wget --progress=dot --inet4-only -c" +BBMULTICONFIG = "prod dev nvidia nvidia-dev" \ No newline at end of file diff --git a/bb-build/conf/multiconfig/dev.conf b/bb-build/conf/multiconfig/dev.conf new file mode 100644 index 000000000..fb9dff407 --- /dev/null +++ b/bb-build/conf/multiconfig/dev.conf @@ -0,0 +1,7 @@ +# Development flavor configuration +DSTACK_FLAVOR = "dev" +DSTACK_NVIDIA = "0" +DSTACK_DEV = "1" + +# Use separate TMPDIR to avoid conflicts between multiconfigs +TMPDIR = "${TOPDIR}/tmp-mc-dev" diff --git a/bb-build/conf/multiconfig/nvidia-dev.conf b/bb-build/conf/multiconfig/nvidia-dev.conf new file mode 100644 index 000000000..65c6cce50 --- /dev/null +++ b/bb-build/conf/multiconfig/nvidia-dev.conf @@ -0,0 +1,7 @@ +# NVIDIA development flavor configuration +DSTACK_FLAVOR = "nvidia-dev" +DSTACK_NVIDIA = "1" +DSTACK_DEV = "1" + +# Use separate TMPDIR to avoid conflicts between multiconfigs +TMPDIR = "${TOPDIR}/tmp-mc-nvidia-dev" diff --git a/bb-build/conf/multiconfig/nvidia.conf b/bb-build/conf/multiconfig/nvidia.conf new file mode 100644 index 000000000..4f3f0e625 --- /dev/null +++ b/bb-build/conf/multiconfig/nvidia.conf @@ -0,0 +1,7 @@ +# NVIDIA production flavor configuration +DSTACK_FLAVOR = "nvidia" +DSTACK_NVIDIA = "1" +DSTACK_DEV = "0" + +# Use separate TMPDIR to avoid conflicts between multiconfigs +TMPDIR = "${TOPDIR}/tmp-mc-nvidia" diff --git a/bb-build/conf/multiconfig/prod.conf b/bb-build/conf/multiconfig/prod.conf new file mode 100644 index 000000000..f39fe60b9 --- /dev/null +++ b/bb-build/conf/multiconfig/prod.conf @@ -0,0 +1,7 @@ +# Production flavor configuration +DSTACK_FLAVOR = "prod" +DSTACK_NVIDIA = "0" +DSTACK_DEV = "0" + +# Use separate TMPDIR to avoid conflicts between multiconfigs +TMPDIR = "${TOPDIR}/tmp-mc-prod" diff --git a/dev-setup b/dev-setup index 976e79b5d..1c6fa203c 100755 --- a/dev-setup +++ b/dev-setup @@ -36,6 +36,9 @@ if [ ! -f "$BUILD_DIR/conf/local.conf" ]; then mkdir -p "$BUILD_DIR/conf" cp -f "$THIS_DIR/bb-build/conf/local.conf" "$BUILD_DIR/conf/local.conf" fi +if [ ! -d "$BUILD_DIR/conf/multiconfig" ] && [ -d "$THIS_DIR/bb-build/conf/multiconfig" ]; then + cp -rf "$THIS_DIR/bb-build/conf/multiconfig" "$BUILD_DIR/conf/" +fi OE_INIT=$THIS_DIR/openembedded-core/oe-init-build-env diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 2f7e0fb15..0e6dacaab 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -37,10 +37,8 @@ do_unpack() { fi } -# Force the configure task to run every time to detect source changes -do_unpack[nostamp] = "1" - # Add source directory to configure task dependencies +# Note: removed nostamp to avoid concurrent build conflicts in multiconfig do_unpack[vardeps] += "SRC_DIR" do_configure() { diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 0cf1e1028..dcdf1601a 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -51,13 +51,13 @@ PARALLEL_MAKE = "" DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" -EDK_TOOLS_DIR="edk2_basetools" +EDK_TOOLS_DIR = "edk2_basetools" # OVMF has trouble building with the default optimization of -O2. BUILD_OPTIMIZATION = "" # OVMF supports IA only, although it could conceivably support ARM someday. -COMPATIBLE_HOST:class-target='(i.86|x86_64).*' +COMPATIBLE_HOST:class-target = '(i.86|x86_64).*' # Additional build flags for OVMF with Secure Boot. # Fedora also uses "-D SMM_REQUIRE -D EXCLUDE_SHELL_FROM_FD". @@ -138,7 +138,7 @@ fix_toolchain:append:class-native() { export NASM_PREFIX_MAP = "--debug-prefix-map=${WORKDIR}=${TARGET_DBGSRC_DIR}" export GCC_PREFIX_MAP = "${DEBUG_PREFIX_MAP} -Wno-stringop-overflow -Wno-maybe-uninitialized" -GCC_VER="$(${CC} -v 2>&1 | tail -n1 | awk '{print $3}')" +GCC_VER = "$(${CC} -v 2>&1 | tail -n1 | awk '{print $3}')" fixup_target_tools() { case ${1} in diff --git a/meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb b/meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb deleted file mode 100644 index a351ef920..000000000 --- a/meta-dstack/recipes-core/images/dstack-cloud-nvidia-rootfs.bb +++ /dev/null @@ -1,3 +0,0 @@ -include dstack-rootfs-prod.inc -include dstack-rootfs-nvidia.inc - diff --git a/meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb b/meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb deleted file mode 100644 index 7dc317dfa..000000000 --- a/meta-dstack/recipes-core/images/dstack-cloud-rootfs.bb +++ /dev/null @@ -1 +0,0 @@ -include dstack-rootfs-prod.inc diff --git a/meta-dstack/recipes-core/images/dstack-dev-rootfs.bb b/meta-dstack/recipes-core/images/dstack-dev-rootfs.bb deleted file mode 100644 index 59662e6d5..000000000 --- a/meta-dstack/recipes-core/images/dstack-dev-rootfs.bb +++ /dev/null @@ -1 +0,0 @@ -include dstack-rootfs-dev.inc diff --git a/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb b/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb deleted file mode 100644 index 01afbf25c..000000000 --- a/meta-dstack/recipes-core/images/dstack-nvidia-dev-rootfs.bb +++ /dev/null @@ -1,2 +0,0 @@ -include dstack-rootfs-dev.inc -include dstack-rootfs-nvidia.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc index eacc68bf7..1481a98ac 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc @@ -1,4 +1,2 @@ -include dstack-rootfs-base.inc - IMAGE_INSTALL += "packagegroup-core-ssh-openssh strace tcpdump gdb gdbserver vim" EXTRA_IMAGE_FEATURES += "allow-root-login post-install-logging" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc b/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc index e3f805d0c..a446e0396 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc @@ -1,2 +1 @@ -include dstack-rootfs-base.inc IMAGE_FEATURES += "nologin" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb new file mode 100644 index 000000000..930df27fb --- /dev/null +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -0,0 +1,17 @@ +# Unified dstack rootfs image +# Use DSTACK_FLAVOR (via multiconfig) to select variant: +# prod, dev, nvidia, nvidia-dev + +# Default flavor settings (can be overridden by multiconfig) +DSTACK_FLAVOR ?= "prod" +DSTACK_NVIDIA ?= "0" +DSTACK_DEV ?= "0" + +# Base configuration +include dstack-rootfs-base.inc + +# Production or development mode +include ${@'dstack-rootfs-dev.inc' if d.getVar('DSTACK_DEV') == '1' else 'dstack-rootfs-prod.inc'} + +# NVIDIA support (optional) +include ${@'dstack-rootfs-nvidia.inc' if d.getVar('DSTACK_NVIDIA') == '1' else ''} diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/meta-dstack/recipes-core/images/dstack-uki.bb index c9883bae2..47d392f6a 100644 --- a/meta-dstack/recipes-core/images/dstack-uki.bb +++ b/meta-dstack/recipes-core/images/dstack-uki.bb @@ -18,19 +18,21 @@ INITRAMFS_FSTYPES = "cpio.gz" # Kernel settings KERNEL_IMAGETYPE = "bzImage" -# Output filename -UKI_FILENAME = "dstack-uki.efi" - # Base kernel cmdline (verity hash added dynamically) UKI_CMDLINE_BASE = "console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0 \ mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n \ tsc=reliable no-kvmclock" -# Verity image to get hash from -# Override with VERITY_IMAGE = "dstack-dev-rootfs" for dev builds -VERITY_IMAGE ?= "dstack-rootfs" +# Flavor settings (should match dstack-rootfs.bb, set via multiconfig) +DSTACK_FLAVOR ?= "prod" + +# Verity image to get hash from - always use dstack-rootfs (same PN, different multiconfig) +VERITY_IMAGE = "dstack-rootfs" VERITY_TYPE = "squashfs" +# Output filename includes flavor to avoid conflicts between multiconfigs +UKI_FILENAME = "${@'dstack-uki.efi' if d.getVar('DSTACK_FLAVOR') == 'prod' else 'dstack-uki-' + d.getVar('DSTACK_FLAVOR') + '.efi'}" + do_configure[noexec] = "1" do_compile[noexec] = "1" do_install[noexec] = "1" @@ -39,6 +41,7 @@ do_install[noexec] = "1" do_uki[depends] += "systemd-boot:do_deploy virtual/kernel:do_deploy" do_uki[depends] += "${INITRAMFS_IMAGE}:do_image_complete" do_uki[depends] += "${VERITY_IMAGE}:do_image_complete" +do_uki[depends] += "systemd-boot-native:do_populate_sysroot python3-pefile-native:do_populate_sysroot" python do_uki() { import os diff --git a/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb b/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb index e5aff7973..4071fbd6d 100644 --- a/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb +++ b/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb @@ -19,4 +19,4 @@ do_install() { # Ensure that the package is always installed PACKAGES = "${PN}" -FILES:${PN}:append= "${base_sbindir}/ldconfig.real" +FILES:${PN}:append = "${base_sbindir}/ldconfig.real" diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc index 46550957b..ca7fb4e11 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc @@ -37,4 +37,4 @@ DEPENDS = " \ libtirpc134 \ ldconfig-native \ " -RDEPENDS:${PN}:append= " ldconfig-compatibility-symlink containerd-config" \ No newline at end of file +RDEPENDS:${PN}:append = " ldconfig-compatibility-symlink containerd-config" \ No newline at end of file diff --git a/mkimage.sh b/mkimage.sh index 0bf58a674..5a32690be 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -11,9 +11,13 @@ while [ $# -gt 0 ]; do DIST_NAME="$2" shift 2 ;; + --flavor) + FLAVOR="$2" + shift 2 + ;; *) echo "Unknown option: $1" - echo "Usage: $0 --dist-name NAME" + echo "Usage: $0 --dist-name NAME --flavor FLAVOR" exit 1 ;; esac @@ -25,6 +29,11 @@ if [ -z "$DIST_NAME" ]; then exit 1 fi +if [ -z "$FLAVOR" ]; then + echo "Error: --flavor is required (prod, dev, nvidia, nvidia-dev)" + exit 1 +fi + if [[ "$DIST_NAME" == *-dev ]]; then IS_DEV=true @@ -35,16 +44,27 @@ fi BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) -IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/tdx -ROOTFS_IMAGE_NAME=${DIST_NAME}-rootfs +# Common artifacts are in tmp/, flavor-specific artifacts are in tmp-mc-/ +COMMON_IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/tdx +FLAVOR_IMG_DIR=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/deploy/images/tdx + +# Common artifacts (shared across all flavors) +INITRAMFS_IMAGE=${COMMON_IMG_DIR}/dstack-initramfs.cpio.gz +KERNEL_IMAGE=${COMMON_IMG_DIR}/bzImage +OVMF_FIRMWARE=${COMMON_IMG_DIR}/ovmf.fd + +# Flavor-specific artifacts (from multiconfig build) +ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-tdx.squashfs.verity + +# UKI filename depends on flavor +if [[ "$FLAVOR" == "prod" ]]; then + UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi +else + UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki-${FLAVOR}.efi +fi -INITRAMFS_IMAGE=${IMG_DIR}/dstack-initramfs.cpio.gz -ROOTFS_IMAGE=${IMG_DIR}/${ROOTFS_IMAGE_NAME}-tdx.squashfs.verity -KERNEL_IMAGE=${IMG_DIR}/bzImage -OVMF_FIRMWARE=${IMG_DIR}/ovmf.fd -UKI_IMAGE=${IMG_DIR}/dstack-uki.efi -# Always use the work-shared directory which has the correct verity env -VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp/work-shared/tdx/dm-verity/${ROOTFS_IMAGE_NAME}.squashfs.verity.env +# Verity env is in the flavor-specific work-shared directory +VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/tdx/dm-verity/dstack-rootfs.squashfs.verity.env echo "Loading verity env from ${VERITY_ENV_FILE}" source ${VERITY_ENV_FILE} @@ -246,7 +266,7 @@ popd if [ "$ENABLE_GCP_IMAGE" = "1" ]; then if [[ ! -f "$UKI_IMAGE" ]]; then echo "Skipping GCP disk image creation because UKI image not found: $UKI_IMAGE" >&2 - echo "Run 'bitbake dstack-uki' to build the UKI first" >&2 + echo "Run 'bitbake mc:${FLAVOR}:dstack-uki' to build the UKI first" >&2 elif command -v sgdisk >/dev/null && \ command -v mkfs.vfat >/dev/null && \ command -v mcopy >/dev/null; then From 37531fe3e6bb8d2977927c05862c7a03bb5943c3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 05:26:08 +0000 Subject: [PATCH 261/435] bb: Move build config from bb-build to meta-dstack/conf - Move local.conf and multiconfig/ to meta-dstack/conf/ - Update dev-setup to always sync config to build directory - bb-build is now fully excluded from git (build artifact only) --- .gitignore | 1 - dev-setup | 13 ++++++------- {bb-build => meta-dstack}/conf/local.conf | 0 {bb-build => meta-dstack}/conf/multiconfig/dev.conf | 0 .../conf/multiconfig/nvidia-dev.conf | 0 .../conf/multiconfig/nvidia.conf | 0 .../conf/multiconfig/prod.conf | 0 7 files changed, 6 insertions(+), 8 deletions(-) rename {bb-build => meta-dstack}/conf/local.conf (100%) rename {bb-build => meta-dstack}/conf/multiconfig/dev.conf (100%) rename {bb-build => meta-dstack}/conf/multiconfig/nvidia-dev.conf (100%) rename {bb-build => meta-dstack}/conf/multiconfig/nvidia.conf (100%) rename {bb-build => meta-dstack}/conf/multiconfig/prod.conf (100%) diff --git a/.gitignore b/.gitignore index 8e7fb513b..c6460dc58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /build /bb-build -!/bb-build/conf/multiconfig/ *.qcow2 __pycache__/ /.target diff --git a/dev-setup b/dev-setup index 1c6fa203c..4be80749e 100755 --- a/dev-setup +++ b/dev-setup @@ -32,13 +32,12 @@ if [ -z "$1" ]; then else BUILD_DIR=$(realpath "$1") fi -if [ ! -f "$BUILD_DIR/conf/local.conf" ]; then - mkdir -p "$BUILD_DIR/conf" - cp -f "$THIS_DIR/bb-build/conf/local.conf" "$BUILD_DIR/conf/local.conf" -fi -if [ ! -d "$BUILD_DIR/conf/multiconfig" ] && [ -d "$THIS_DIR/bb-build/conf/multiconfig" ]; then - cp -rf "$THIS_DIR/bb-build/conf/multiconfig" "$BUILD_DIR/conf/" -fi + +# Sync build config from meta-dstack/conf (always overwrite) +CONF_SRC=$THIS_DIR/meta-dstack/conf +mkdir -p "$BUILD_DIR/conf" +cp -f "$CONF_SRC/local.conf" "$BUILD_DIR/conf/local.conf" +cp -rf "$CONF_SRC/multiconfig" "$BUILD_DIR/conf/" OE_INIT=$THIS_DIR/openembedded-core/oe-init-build-env diff --git a/bb-build/conf/local.conf b/meta-dstack/conf/local.conf similarity index 100% rename from bb-build/conf/local.conf rename to meta-dstack/conf/local.conf diff --git a/bb-build/conf/multiconfig/dev.conf b/meta-dstack/conf/multiconfig/dev.conf similarity index 100% rename from bb-build/conf/multiconfig/dev.conf rename to meta-dstack/conf/multiconfig/dev.conf diff --git a/bb-build/conf/multiconfig/nvidia-dev.conf b/meta-dstack/conf/multiconfig/nvidia-dev.conf similarity index 100% rename from bb-build/conf/multiconfig/nvidia-dev.conf rename to meta-dstack/conf/multiconfig/nvidia-dev.conf diff --git a/bb-build/conf/multiconfig/nvidia.conf b/meta-dstack/conf/multiconfig/nvidia.conf similarity index 100% rename from bb-build/conf/multiconfig/nvidia.conf rename to meta-dstack/conf/multiconfig/nvidia.conf diff --git a/bb-build/conf/multiconfig/prod.conf b/meta-dstack/conf/multiconfig/prod.conf similarity index 100% rename from bb-build/conf/multiconfig/prod.conf rename to meta-dstack/conf/multiconfig/prod.conf From 6a0ae2a7eaff5f7d0fa7c8ea206898ee66f2e37f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 05:49:42 +0000 Subject: [PATCH 262/435] Rename dist image prefix from dstack to dstack-cloud - dstack-0.6.0/ -> dstack-cloud-0.6.0/ - dstack-nvidia-0.6.0.tar.gz -> dstack-cloud-nvidia-0.6.0.tar.gz - Update download URL to meta-dstack-cloud repo --- Makefile | 2 +- build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 65e868613..5c4547413 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ export DIST_DIR FLAVORS ?= prod dev nvidia nvidia-dev # Map flavor to dist name for mkimage.sh -flavor_to_dist = $(if $(filter prod,$1),dstack,$(if $(filter dev,$1),dstack-dev,$(if $(filter nvidia,$1),dstack-nvidia,$(if $(filter nvidia-dev,$1),dstack-nvidia-dev,$1)))) +flavor_to_dist = $(if $(filter prod,$1),dstack-cloud,$(if $(filter dev,$1),dstack-cloud-dev,$(if $(filter nvidia,$1),dstack-cloud-nvidia,$(if $(filter nvidia-dev,$1),dstack-cloud-nvidia-dev,$1)))) all: dist diff --git a/build.sh b/build.sh index a70ce8538..4481aefbf 100755 --- a/build.sh +++ b/build.sh @@ -306,11 +306,11 @@ download_image() { TAG=v$VERSION if [ x"$IS_DEV" = x"1" ]; then - BASENAME=dstack-dev-$VERSION + BASENAME=dstack-cloud-dev-$VERSION else - BASENAME=dstack-$VERSION + BASENAME=dstack-cloud-$VERSION fi - URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/$TAG/$BASENAME.tar.gz + URL=https://github.com/Dstack-TEE/meta-dstack-cloud/releases/download/$TAG/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME ]; then echo "Image already exists" else From 04100cda5225411a2471ce87183fdaddb8791c38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 06:11:25 +0000 Subject: [PATCH 263/435] mkimage: Fix UKI auth_hash not being written to output Move write_authenticode_hash_if_missing call to main flow instead of only in create_gcp_artifacts. This ensures the hash file is always generated when UKI exists, regardless of GCP image creation. --- mkimage.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 5a32690be..74824c350 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -57,11 +57,7 @@ OVMF_FIRMWARE=${COMMON_IMG_DIR}/ovmf.fd ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-tdx.squashfs.verity # UKI filename depends on flavor -if [[ "$FLAVOR" == "prod" ]]; then - UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi -else - UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki-${FLAVOR}.efi -fi +UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi # Verity env is in the flavor-specific work-shared directory VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/tdx/dm-verity/dstack-rootfs.squashfs.verity.env @@ -228,9 +224,12 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity -# Copy UKI Authenticode hash if available -if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then - $Q cp "${UKI_IMAGE}.auth_hash.txt" "${OUTPUT_DIR}/dstack-uki.efi.auth_hash.txt" +# Calculate and copy UKI Authenticode hash +if [[ -f "$UKI_IMAGE" ]]; then + write_authenticode_hash_if_missing "$UKI_IMAGE" + if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then + $Q cp "${UKI_IMAGE}.auth_hash.txt" "${OUTPUT_DIR}/dstack-uki.efi.auth_hash.txt" + fi fi echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" From 0ff95e23881f5d13c1fb732a9b2e8cdeb0de5c07 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 06:21:51 +0000 Subject: [PATCH 264/435] Update LICENSE --- LICENSE | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index fd94e8eaf..4a58670f1 100644 --- a/LICENSE +++ b/LICENSE @@ -2,14 +2,20 @@ Business Source License 1.1 Parameters -Licensor: Phala Network -Licensed Work: dstack-cloud - The Licensed Work is (c) Phala Network -Additional Use Grant: None +Licensor: Hashforest Technology LLC -Change Date: Four years from the date a MINOR version (SemVer) is published. +Licensed Work: dstack-cloud + The Licensed Work is (c) Hashforest Technology LLC -Change License: GNU Affero General Public License Version 3 (AGPL-3.0-only) +Additional Use Grant: Notwithstanding the foregoing, the Licensor grants + to certain commercial partners a license to use the + Licensed Work for production and commercial purposes + pursuant to separate agreements. + +Change Date: Two years from the date a MINOR version (SemVer) is + published. + +Change License: GNU Affero General Public License Version 3 (AGPL-3.0) Notice From 8e59b37e448e410f81104bd87bfcdbadfbb9ade6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 06:22:21 +0000 Subject: [PATCH 265/435] Fix uki filename in bb files --- Makefile | 2 +- meta-dstack/recipes-core/images/dstack-uki.bb | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5c4547413..da4ba8eb5 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ dist: images # Build common artifacts (shared across all flavors) # dstack-guest is built here to avoid concurrent build conflicts in multiconfig images-common: - bitbake virtual/kernel dstack-initramfs dstack-ovmf dstack-guest + bitbake virtual/kernel dstack-initramfs dstack-ovmf # Build flavor-specific artifacts using multiconfig (serial to avoid deadlock warnings) images-flavors: diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/meta-dstack/recipes-core/images/dstack-uki.bb index 47d392f6a..4a2254b33 100644 --- a/meta-dstack/recipes-core/images/dstack-uki.bb +++ b/meta-dstack/recipes-core/images/dstack-uki.bb @@ -30,8 +30,7 @@ DSTACK_FLAVOR ?= "prod" VERITY_IMAGE = "dstack-rootfs" VERITY_TYPE = "squashfs" -# Output filename includes flavor to avoid conflicts between multiconfigs -UKI_FILENAME = "${@'dstack-uki.efi' if d.getVar('DSTACK_FLAVOR') == 'prod' else 'dstack-uki-' + d.getVar('DSTACK_FLAVOR') + '.efi'}" +UKI_FILENAME = "dstack-uki.efi" do_configure[noexec] = "1" do_compile[noexec] = "1" From 1ee65396daa70e70217f79c3098826139fc98f3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 06:22:36 +0000 Subject: [PATCH 266/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 4854ccc2f..a30e29817 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 4854ccc2f8fa33838d3029c60803bda2bd6d2d08 +Subproject commit a30e29817d6268b09c618cb2f7f3fd14078d52a4 From 2be071485df0252e0bc284cf3eccc8c2dc6082fc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 06:48:42 +0000 Subject: [PATCH 267/435] mkimage: Split output into bare-metal and UKI tarballs Split the single output tarball into two separate archives: 1. {name}-{version}.tar.gz (bare metal deployment) - bzImage, initramfs, ovmf.fd, rootfs.img.parted.verity - metadata.json, sha256sum.txt, digest.txt 2. {name}-{version}-uki.tar.gz (GCP/cloud deployment) - disk.raw (GPT disk with EFI partition containing UKI) - auth_hash.txt (Authenticode hash of UKI) Also update dstack-cloud CLI: - pull: Download -uki.tar.gz instead of full tarball - _find_boot_image_tar: Search for disk.raw in -uki directory - _check_and_upload_boot_image: Handle both disk.raw and legacy gcp.tar.gz - _generate_sys_config: Search auth_hash.txt in new location --- mkimage.sh | 105 +++++++++++++++++++-------------------- scripts/bin/dstack-cloud | 99 ++++++++++++++++++++++++------------ 2 files changed, 117 insertions(+), 87 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 74824c350..67328838a 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -2,7 +2,7 @@ set -e DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} -ENABLE_GCP_IMAGE=${ENABLE_GCP_IMAGE:-1} +ENABLE_UKI_IMAGE=${ENABLE_UKI_IMAGE:-1} # Parse command line arguments while [ $# -gt 0 ]; do @@ -56,7 +56,7 @@ OVMF_FIRMWARE=${COMMON_IMG_DIR}/ovmf.fd # Flavor-specific artifacts (from multiconfig build) ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-tdx.squashfs.verity -# UKI filename depends on flavor +# UKI filename UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi # Verity env is in the flavor-specific work-shared directory @@ -65,8 +65,12 @@ echo "Loading verity env from ${VERITY_ENV_FILE}" source ${VERITY_ENV_FILE} DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) + +# Output directory contains all artifacts; tarballs contain subsets OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} +IMAGE_TAR_UKI="${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-uki.tar.gz" +TAR_DIR_NAME="${DIST_NAME}-${DSTACK_VERSION}" AUTHENTICODE_HASH_SCRIPT="${BB_BUILD_DIR}/../scripts/bin/authenticode_hash.py" @@ -119,21 +123,6 @@ write_authenticode_hash_if_missing() { fi } -create_uki_bootstrap() { - local target_dir="$1" - mkdir -p "$target_dir/EFI/BOOT" - - local uki_file="${UKI_IMAGE}" - if [[ ! -f "$uki_file" ]]; then - echo "Error: UKI image not found: $uki_file" >&2 - return 1 - fi - cp "$uki_file" "$target_dir/EFI/BOOT/BOOTX64.EFI" - - # Calculate Authenticode hash if not already present - write_authenticode_hash_if_missing "$uki_file" -} - create_partitioned_rootfs() { local rootfs_img="$1" local output_img="$2" @@ -159,9 +148,9 @@ create_partitioned_rootfs() { ) } -build_gcp_disk_image() { +build_uki_disk_image() { local disk_img="$1" - local boot_source="$2" + local uki_file="$2" local rootfs_img="$3" ( set -e @@ -190,50 +179,43 @@ build_gcp_disk_image() { local tmp_dir tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' EXIT + + # Create EFI filesystem with UKI as bootloader local efi_img=${tmp_dir}/efi.img mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null - (cd "$boot_source" && mcopy -s -i "$efi_img" ./* ::) >/dev/null + mmd -i "$efi_img" ::EFI ::EFI/BOOT + mcopy -i "$efi_img" "$uki_file" ::EFI/BOOT/BOOTX64.EFI dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none ) } -create_gcp_artifacts() { - local gcp_dir="${OUTPUT_DIR}/gcp" - local boot_src="${gcp_dir}/efi-root" - mkdir -p "$boot_src" - echo "Installing UKI as EFI bootloader at ${boot_src}" - create_uki_bootstrap "$boot_src" +create_uki_artifacts() { + local uki_dir="$1" + mkdir -p "$uki_dir" - local disk_img="${gcp_dir}/disk.raw" - echo "Building raw disk image for GCP at ${disk_img}" - build_gcp_disk_image "$disk_img" "$boot_src" "${OUTPUT_DIR}/rootfs.img.verity" + echo "Building UKI disk image at ${uki_dir}/disk.raw" + build_uki_disk_image "${uki_dir}/disk.raw" "$UKI_IMAGE" "$ROOTFS_IMAGE" - local tarball="${OUTPUT_DIR}/gcp.tar.gz" - echo "Archiving GCP disk image to ${tarball}" - (cd "$gcp_dir" && tar -czvf "$tarball" disk.raw) + # Calculate and copy auth hash + write_authenticode_hash_if_missing "$UKI_IMAGE" + if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then + cp "${UKI_IMAGE}.auth_hash.txt" "${uki_dir}/auth_hash.txt" + fi } Q=verbose +# Create bare metal image directory $Q rm -rf ${OUTPUT_DIR}/ $Q mkdir -p ${OUTPUT_DIR}/ $Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ -$Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity - -# Calculate and copy UKI Authenticode hash -if [[ -f "$UKI_IMAGE" ]]; then - write_authenticode_hash_if_missing "$UKI_IMAGE" - if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then - $Q cp "${UKI_IMAGE}.auth_hash.txt" "${OUTPUT_DIR}/dstack-uki.efi.auth_hash.txt" - fi -fi echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" -create_partitioned_rootfs "${OUTPUT_DIR}/rootfs.img.verity" "${OUTPUT_DIR}/rootfs.img.parted.verity" +create_partitioned_rootfs "$ROOTFS_IMAGE" "${OUTPUT_DIR}/rootfs.img.parted.verity" GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" @@ -262,28 +244,43 @@ sha256sum ovmf.fd bzImage initramfs.cpio.gz metadata.json > sha256sum.txt sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd -if [ "$ENABLE_GCP_IMAGE" = "1" ]; then +# Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR +UKI_CREATED=0 +if [ "$ENABLE_UKI_IMAGE" = "1" ]; then if [[ ! -f "$UKI_IMAGE" ]]; then - echo "Skipping GCP disk image creation because UKI image not found: $UKI_IMAGE" >&2 + echo "Skipping UKI disk image creation because UKI image not found: $UKI_IMAGE" >&2 echo "Run 'bitbake mc:${FLAVOR}:dstack-uki' to build the UKI first" >&2 elif command -v sgdisk >/dev/null && \ command -v mkfs.vfat >/dev/null && \ - command -v mcopy >/dev/null; then - create_gcp_artifacts + command -v mcopy >/dev/null && \ + command -v mmd >/dev/null; then + create_uki_artifacts "${OUTPUT_DIR}" + UKI_CREATED=1 else - echo "Error: cannot create GCP disk image because required tools are missing" >&2 - echo "Missing tools are among: sgdisk (gdisk), mkfs.vfat (dosfstools), mcopy (mtools)" >&2 - echo "Install them (e.g. apt-get install -y gdisk dosfstools mtools) or set ENABLE_GCP_IMAGE=0" >&2 + echo "Error: cannot create UKI disk image because required tools are missing" >&2 + echo "Missing tools are among: sgdisk (gdisk), mkfs.vfat (dosfstools), mcopy/mmd (mtools)" >&2 + echo "Install them (e.g. apt-get install -y gdisk dosfstools mtools) or set ENABLE_UKI_IMAGE=0" >&2 exit 1 fi fi if [ x$DSTACK_TAR_RELEASE = x1 ]; then - IMAGE_TAR_MR=${DIST_DIR}/mr_$(cat ${OUTPUT_DIR}/digest.txt | tr -d '\n').tar.gz - IMAGE_TAR_NO_ROOTFS=${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-mr.tar.gz OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) - rm -rf ${IMAGE_TAR} ${IMAGE_TAR_MR} ${IMAGE_TAR_NO_ROOTFS} - echo "Archiving the output directory to ${IMAGE_TAR}" - (cd $(dirname ${OUTPUT_DIR}) && tar -czvf ${IMAGE_TAR} $(basename $OUTPUT_DIR)) + PARENT_DIR=$(dirname ${OUTPUT_DIR}) + + # Bare metal tarball: all files except disk.raw and auth_hash.txt + rm -rf ${IMAGE_TAR} + echo "Archiving bare metal image to ${IMAGE_TAR}" + BARE_METAL_FILES="rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json" + (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo + + # UKI tarball: only disk.raw and auth_hash.txt + if [[ "$UKI_CREATED" = "1" ]]; then + rm -rf ${IMAGE_TAR_UKI} + echo "Archiving UKI image to ${IMAGE_TAR_UKI}" + UKI_FILES="disk.raw auth_hash.txt" + (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR_UKI} $(for f in $UKI_FILES; do echo "$TAR_DIR_NAME/$f"; done)) + echo + fi fi diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 30fea2cba..0e0a22a3c 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -393,6 +393,15 @@ class CloudDeploymentManager: if not os.path.isabs(search_path): search_path = os.path.join(self.work_dir, search_path) + # Try new format first (auth_hash.txt in -uki directory) + hash_file = Path(search_path) / f"{local_image}-uki" / "auth_hash.txt" + if hash_file.exists(): + with open(hash_file, 'r') as f: + os_image_hash = f.read().strip() + logger.info(f"Read OS image hash from {hash_file}") + break + + # Fall back to legacy format hash_file = Path(search_path) / local_image / "dstack-uki.efi.auth_hash.txt" if hash_file.exists(): with open(hash_file, 'r') as f: @@ -888,7 +897,7 @@ class CloudDeploymentManager: logger.info("These files will be included in the shared disk image during deploy.") def _find_boot_image_tar(self, local_image: str) -> Optional[Path]: - """Search for boot image tar file in configured search paths.""" + """Search for boot image disk.raw file in configured search paths.""" global_config = self._load_global_config() search_paths = global_config.get("image_search_paths", []) @@ -903,9 +912,10 @@ class CloudDeploymentManager: expanded = os.path.join(self.work_dir, expanded) expanded_paths.append(Path(expanded)) - # Try different file name patterns + # Try different file name patterns (new -uki format first, then legacy gcp.tar.gz) patterns = [ - f"{local_image}/gcp.tar.gz", + f"{local_image}-uki/disk.raw", + f"{local_image}/gcp.tar.gz", # Legacy format ] for search_path in expanded_paths: @@ -913,16 +923,16 @@ class CloudDeploymentManager: if not search_path.exists(): continue for pattern in patterns: - tar_file = search_path / pattern - logger.debug(f"Checking: {tar_file}, exists: {tar_file.exists()}") - if tar_file.exists(): - logger.info(f"Found boot image: {tar_file}") - return tar_file + file_path = search_path / pattern + logger.debug(f"Checking: {file_path}, exists: {file_path.exists()}") + if file_path.exists(): + logger.info(f"Found boot image: {file_path}") + return file_path return None def pull(self, os_image: str) -> None: - """Download OS image from remote repository.""" + """Download UKI image from remote repository.""" global_config = self._load_global_config() search_paths = global_config.get("image_search_paths", []) @@ -935,8 +945,8 @@ class CloudDeploymentManager: target_dir = Path(os.path.expanduser(search_paths[0])) target_dir.mkdir(parents=True, exist_ok=True) - # Download tar file (e.g., dstack-0.6.0.tar.gz) - download_tar = target_dir / f"{os_image}.tar.gz" + # Download UKI tar file (e.g., dstack-cloud-nvidia-0.6.0-uki.tar.gz) + download_tar = target_dir / f"{os_image}-uki.tar.gz" if download_tar.exists(): logger.info(f"Download file already exists: {download_tar}") @@ -946,9 +956,9 @@ class CloudDeploymentManager: return # Download URL - download_url = f"https://download.dstack.org/os-images/{os_image}.tar.gz" + download_url = f"https://download.dstack.org/os-images/{os_image}-uki.tar.gz" - logger.info(f"Downloading {os_image} from {download_url}...") + logger.info(f"Downloading {os_image} UKI image from {download_url}...") logger.info(f"Target: {download_tar}") try: @@ -972,46 +982,46 @@ class CloudDeploymentManager: ["tar", "-xzf", str(download_tar), "-C", str(target_dir)], check=True ) - logger.info(f"Successfully extracted to {target_dir / os_image}") + logger.info(f"Successfully extracted to {target_dir / os_image}-uki") except subprocess.CalledProcessError as e: logger.error(f"Failed to extract image: {e}") raise # Verify the expected structure - expected_dir = target_dir / os_image - expected_tar = expected_dir / "gcp.tar.gz" - if expected_tar.exists(): - logger.info(f"Image ready: {expected_tar}") + expected_dir = target_dir / f"{os_image}-uki" + expected_disk = expected_dir / "disk.raw" + if expected_disk.exists(): + logger.info(f"Image ready: {expected_disk}") else: - logger.warning(f"Expected file not found: {expected_tar}") + logger.warning(f"Expected file not found: {expected_disk}") logger.warning(f"Downloaded structure may be incorrect") def _check_and_upload_boot_image(self, config: GcpConfig, app: App, force: bool = False) -> str: """Check and upload boot image if needed. Returns the image name.""" - tar_path = None + image_path = None # Derive GCP image name from app.os_image gcp_image = config.boot_image if not gcp_image: - # Convert OS image name (dstack-0.6.0 -> dstack-0-6-0) + # Convert OS image name (dstack-cloud-nvidia-0.6.0 -> dstack-cloud-nvidia-0-6-0) gcp_image = app.os_image.replace(".", "-") - # If boot_image_tar is specified, use it + # If boot_image_tar is specified, use it (legacy config name, can be disk.raw or tar.gz) if config.boot_image_tar: - tar_path = Path(os.path.expanduser(config.boot_image_tar)) - if not tar_path.exists(): + image_path = Path(os.path.expanduser(config.boot_image_tar)) + if not image_path.exists(): logger.error("") - logger.error(f"Boot image tar not found: {tar_path}") + logger.error(f"Boot image not found: {image_path}") logger.error("") logger.error(f"Please download the image using:") logger.error(f" dstack-cloud pull {app.os_image}") logger.error("") - raise FileNotFoundError(f"Boot image tar not found: {tar_path}") + raise FileNotFoundError(f"Boot image not found: {image_path}") else: # Auto-discover from search paths using OS image name logger.info(f"Searching for boot image '{app.os_image}'...") - tar_path = self._find_boot_image_tar(app.os_image) - if not tar_path: + image_path = self._find_boot_image_tar(app.os_image) + if not image_path: logger.error("") logger.error(f"Boot image '{app.os_image}' not found locally.") logger.error("") @@ -1023,9 +1033,12 @@ class CloudDeploymentManager: f"Run 'dstack-cloud pull {app.os_image}' to download it." ) + # Determine if we have disk.raw (new format) or gcp.tar.gz (legacy format) + is_raw_disk = image_path.name == "disk.raw" + # Use gcp_image as the GCP image name image_name = gcp_image - local_mtime = tar_path.stat().st_mtime + local_mtime = image_path.stat().st_mtime # Check if GCP image exists and is up-to-date result = self._run_gcloud([ @@ -1061,10 +1074,30 @@ class CloudDeploymentManager: need_upload = True if need_upload: - logger.info("Uploading boot image to GCS...") - self._run_gsutil([ - "cp", str(tar_path), f"{config.bucket}/{image_name}.tar.gz" - ]) + if is_raw_disk: + # New format: disk.raw needs to be compressed to tar.gz for upload + logger.info("Compressing disk.raw to tar.gz for upload...") + import tempfile + with tempfile.TemporaryDirectory() as tmpdir: + tar_file = os.path.join(tmpdir, "disk.tar.gz") + result = subprocess.run( + ["tar", "-czvf", tar_file, "-C", str(image_path.parent), "disk.raw"], + capture_output=True, + text=True + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") + + logger.info("Uploading boot image to GCS...") + self._run_gsutil([ + "cp", tar_file, f"{config.bucket}/{image_name}.tar.gz" + ]) + else: + # Legacy format: gcp.tar.gz can be uploaded directly + logger.info("Uploading boot image to GCS...") + self._run_gsutil([ + "cp", str(image_path), f"{config.bucket}/{image_name}.tar.gz" + ]) # Delete existing image if present if gcp_creation_time: From cd84be8df4936353546ddd16d9ac19124d1f4060 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 07:43:38 +0000 Subject: [PATCH 268/435] Fix filenames in dstack-cloud.py --- scripts/bin/dstack-cloud | 72 +++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 0e0a22a3c..44ff9a2c0 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -393,16 +393,7 @@ class CloudDeploymentManager: if not os.path.isabs(search_path): search_path = os.path.join(self.work_dir, search_path) - # Try new format first (auth_hash.txt in -uki directory) - hash_file = Path(search_path) / f"{local_image}-uki" / "auth_hash.txt" - if hash_file.exists(): - with open(hash_file, 'r') as f: - os_image_hash = f.read().strip() - logger.info(f"Read OS image hash from {hash_file}") - break - - # Fall back to legacy format - hash_file = Path(search_path) / local_image / "dstack-uki.efi.auth_hash.txt" + hash_file = Path(search_path) / local_image / "auth_hash.txt" if hash_file.exists(): with open(hash_file, 'r') as f: os_image_hash = f.read().strip() @@ -912,10 +903,9 @@ class CloudDeploymentManager: expanded = os.path.join(self.work_dir, expanded) expanded_paths.append(Path(expanded)) - # Try different file name patterns (new -uki format first, then legacy gcp.tar.gz) + # Look for disk.raw in the image directory patterns = [ - f"{local_image}-uki/disk.raw", - f"{local_image}/gcp.tar.gz", # Legacy format + f"{local_image}/disk.raw", ] for search_path in expanded_paths: @@ -955,8 +945,18 @@ class CloudDeploymentManager: logger.info("Download cancelled") return - # Download URL - download_url = f"https://download.dstack.org/os-images/{os_image}-uki.tar.gz" + # Extract version from os_image (e.g., dstack-cloud-nvidia-0.6.0 -> 0.6.0) + # Version is the last component after the last hyphen followed by digits + import re + version_match = re.search(r'-(\d+\.\d+\.\d+)$', os_image) + if not version_match: + logger.error(f"Could not extract version from image name: {os_image}") + logger.error("Expected format: dstack-cloud-- (e.g., dstack-cloud-nvidia-0.6.0)") + return + version = version_match.group(1) + + # Download from GitHub releases + download_url = f"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/Phala-Network/meta-dstack-cloud/releases/download/v{version}/{os_image}-uki.tar.gz" logger.info(f"Downloading {os_image} UKI image from {download_url}...") logger.info(f"Target: {download_tar}") @@ -982,13 +982,13 @@ class CloudDeploymentManager: ["tar", "-xzf", str(download_tar), "-C", str(target_dir)], check=True ) - logger.info(f"Successfully extracted to {target_dir / os_image}-uki") + logger.info(f"Successfully extracted to {target_dir / os_image}") except subprocess.CalledProcessError as e: logger.error(f"Failed to extract image: {e}") raise # Verify the expected structure - expected_dir = target_dir / f"{os_image}-uki" + expected_dir = target_dir / os_image expected_disk = expected_dir / "disk.raw" if expected_disk.exists(): logger.info(f"Image ready: {expected_disk}") @@ -1033,9 +1033,6 @@ class CloudDeploymentManager: f"Run 'dstack-cloud pull {app.os_image}' to download it." ) - # Determine if we have disk.raw (new format) or gcp.tar.gz (legacy format) - is_raw_disk = image_path.name == "disk.raw" - # Use gcp_image as the GCP image name image_name = gcp_image local_mtime = image_path.stat().st_mtime @@ -1074,29 +1071,22 @@ class CloudDeploymentManager: need_upload = True if need_upload: - if is_raw_disk: - # New format: disk.raw needs to be compressed to tar.gz for upload - logger.info("Compressing disk.raw to tar.gz for upload...") - import tempfile - with tempfile.TemporaryDirectory() as tmpdir: - tar_file = os.path.join(tmpdir, "disk.tar.gz") - result = subprocess.run( - ["tar", "-czvf", tar_file, "-C", str(image_path.parent), "disk.raw"], - capture_output=True, - text=True - ) - if result.returncode != 0: - raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") + # Compress disk.raw to tar.gz for upload + logger.info("Compressing disk.raw to tar.gz for upload...") + import tempfile + with tempfile.TemporaryDirectory() as tmpdir: + tar_file = os.path.join(tmpdir, "disk.tar.gz") + result = subprocess.run( + ["tar", "-czvf", tar_file, "-C", str(image_path.parent), "disk.raw"], + capture_output=True, + text=True + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") - logger.info("Uploading boot image to GCS...") - self._run_gsutil([ - "cp", tar_file, f"{config.bucket}/{image_name}.tar.gz" - ]) - else: - # Legacy format: gcp.tar.gz can be uploaded directly logger.info("Uploading boot image to GCS...") self._run_gsutil([ - "cp", str(image_path), f"{config.bucket}/{image_name}.tar.gz" + "cp", tar_file, f"{config.bucket}/{image_name}.tar.gz" ]) # Delete existing image if present @@ -2075,7 +2065,7 @@ Examples: dstack-cloud config-edit # Download OS image - dstack-cloud pull dstack-0.6.0 + dstack-cloud pull dstack-cloud-0.6.0 # Deploy VM dstack-cloud deploy From 0a002d4511867321274eb09bd5056a6960b7bec8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 07:47:54 +0000 Subject: [PATCH 269/435] Fix url in repro-build.sh --- repro-build/repro-build.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/repro-build/repro-build.sh b/repro-build/repro-build.sh index 2b96bfd81..c89680b03 100755 --- a/repro-build/repro-build.sh +++ b/repro-build/repro-build.sh @@ -51,13 +51,16 @@ build_to() { $BUILDER_NAME bash -e -c "$BUILD_CMD" } -build_to $HOST_BUILD_DIR_A DSTACK_TAR_RELEASE=1 +# Only build production flavors (no dev) for reproducible builds +RELEASE_FLAVORS="prod nvidia" + +build_to $HOST_BUILD_DIR_A "FLAVORS='$RELEASE_FLAVORS' DSTACK_TAR_RELEASE=1" DIST_DIR=${THIS_DIR}/dist mkdir -p $DIST_DIR mv $HOST_BUILD_DIR_A/images/*.tar.gz $DIST_DIR/ if [ $NO_CHECK -eq 0 ]; then - build_to $HOST_BUILD_DIR_B + build_to $HOST_BUILD_DIR_B "FLAVORS='$RELEASE_FLAVORS'" ${THIS_DIR}/check.sh $HOST_BUILD_DIR_A $HOST_BUILD_DIR_B fi @@ -72,8 +75,8 @@ cat < Date: Tue, 20 Jan 2026 08:42:03 +0000 Subject: [PATCH 270/435] Update dstack and meta-rust-bin submodules - dstack: Update to latest with vm_config fix and other improvements - meta-rust-bin: Update to Rust 1.92.0 --- dstack | 2 +- meta-rust-bin | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 7bf1843a8..127f5de46 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 7bf1843a8ddf877fbaebb9898a7f27a19a49ab73 +Subproject commit 127f5de46af6a8bfce10a4c5392a3339f16aa3b5 diff --git a/meta-rust-bin b/meta-rust-bin index 53974894f..79c077fac 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit 53974894fe1bf0fba26d0f6d5ded32b3c3f4a43e +Subproject commit 79c077fac9694eb5fbcee7b15e800c21e887bb5d From 82555d3692ac34424cfff4095f1cefcffc07d764 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 08:44:10 +0000 Subject: [PATCH 271/435] mkimage: Fix authenticode_hash.py path resolution Use script's directory to find authenticode_hash.py instead of relative path from BB_BUILD_DIR. This fixes the path resolution in repro-build Docker environment where BB_BUILD_DIR points to a different location than the source tree. --- mkimage.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 67328838a..053335de3 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -72,7 +72,9 @@ IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} IMAGE_TAR_UKI="${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-uki.tar.gz" TAR_DIR_NAME="${DIST_NAME}-${DSTACK_VERSION}" -AUTHENTICODE_HASH_SCRIPT="${BB_BUILD_DIR}/../scripts/bin/authenticode_hash.py" +# Use script's directory to find authenticode_hash.py +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +AUTHENTICODE_HASH_SCRIPT="${SCRIPT_DIR}/scripts/bin/authenticode_hash.py" verbose() { echo "$@" From 50ff7046f01ffacc7b3d6f33c7b20fa329870a12 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 09:20:00 +0000 Subject: [PATCH 272/435] Update Rustc to 1.92.0 --- meta-dstack/conf/distro/dstack.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 39042d489..5b2f6cc51 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -21,7 +21,7 @@ VIRTUAL-RUNTIME_initscripts = "" VIRTUAL-RUNTIME_dev_manager = "systemd" SERIAL_CONSOLES = "115200;ttyS0" -PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.86.0" -PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.86.0" +PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.92.0" +PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" BAD_RECOMMENDATIONS = "busybox-syslog systemd-extra-utils" From b9643688cedc249876e249753e174aead4a733a5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 13:48:37 +0000 Subject: [PATCH 273/435] dstack-guest: Restore do_unpack nostamp flag This ensures source changes are detected on every build, preventing stale cached builds. --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 0e6dacaab..2f7e0fb15 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -37,8 +37,10 @@ do_unpack() { fi } +# Force the configure task to run every time to detect source changes +do_unpack[nostamp] = "1" + # Add source directory to configure task dependencies -# Note: removed nostamp to avoid concurrent build conflicts in multiconfig do_unpack[vardeps] += "SRC_DIR" do_configure() { From c65b000fc098d402bc2c7b21e2b8da3f002ad6c3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 13:49:37 +0000 Subject: [PATCH 274/435] mkimage: Use stat -L to follow symlinks when getting rootfs size The rootfs image path may be a symlink. Use stat -L to dereference symlinks and get the actual file size, preventing incorrect disk image sizing. --- mkimage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 053335de3..1547bc05a 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -132,7 +132,7 @@ create_partitioned_rootfs() { set -e local align=$((1024 * 1024)) local sector=512 - local rootfs_size=$(stat -c %s "$rootfs_img") + local rootfs_size=$(stat -L -c %s "$rootfs_img") local rootfs_size_aligned=$(align_up $rootfs_size $align) local rootfs_start=$align # Leave extra room for GPT headers (1MB at start, 1MB at end) @@ -160,7 +160,7 @@ build_uki_disk_image() { local sector=512 local efi_size=$((256 * 1024 * 1024)) local efi_size_aligned=$(align_up $efi_size $align) - local rootfs_size=$(stat -c %s "$rootfs_img") + local rootfs_size=$(stat -L -c %s "$rootfs_img") local rootfs_size_aligned=$(align_up $rootfs_size $align) local efi_start=$align local rootfs_start=$((efi_start + efi_size_aligned)) From 4775271e5031090da90794b0cf25aecc798d4361 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 14:16:40 +0000 Subject: [PATCH 275/435] Remove tpm2 from DISTRO_FEATURES Remove tpm2 feature from distro configuration while keeping kernel TPM drivers available. User commands access TPM directly without needing libtss2 library. - Remove tpm2 from DISTRO_FEATURES in dstack.conf - Remove libtss2-dev dependency from dstack-guest.bb --- meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 4294ff8b9..c7cf93eb6 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,7 +1,7 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "dstack" -DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6 tpm2" +DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" DISTRO_VERSION = "0.6.0" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 2f7e0fb15..50362e4fe 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -13,7 +13,7 @@ S = "${UNPACKDIR}/dstack" RDEPENDS:${PN} += "bash" -DEPENDS += "rsync-native libtss2-dev" +DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" From f1e28796ee3846c8d4c7c0a6bd3653ec25e07efe Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 14:16:57 +0000 Subject: [PATCH 276/435] Remove unused systemd components completely Remove vconsole-setup binary, udev rules, and gpt-auto-generator entirely rather than just disabling the services. These components are not needed in the dstack guest environment. - Remove systemd-vconsole-setup binary - Remove vconsole udev rules (90-vconsole.rules) - Remove systemd-gpt-auto-generator (causes UNSUPP error) --- meta-dstack/recipes-core/systemd/systemd_%.bbappend | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index f73be27d0..7c1ea9028 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -1,6 +1,12 @@ do_install:append() { - # Disable systemd-vconsole-setup.service + # Remove systemd-vconsole-setup entirely (no virtual console needed) rm -f ${D}${systemd_system_unitdir}/sysinit.target.wants/systemd-vconsole-setup.service + rm -f ${D}${systemd_system_unitdir}/systemd-vconsole-setup.service + rm -f ${D}${rootlibexecdir}/systemd/systemd-vconsole-setup + rm -f ${D}${nonarch_libdir}/udev/rules.d/90-vconsole.rules + + # Disable EFI System Partition automount (not needed, causes UNSUPP error) + rm -f ${D}${nonarch_libdir}/systemd/system-generators/systemd-gpt-auto-generator # Ensure systemd-resolved waits for /var/volatile tmpfs and tmpfiles setup install -d ${D}${systemd_system_unitdir}/systemd-resolved.service.d From d82f4d09187bdb9b5f5200f6ec312c2176a581b9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 14:17:10 +0000 Subject: [PATCH 277/435] dstack-cloud: Remove init command and improve UX Simplify the CLI by removing the redundant 'init' command (use 'new' instead) and improve user experience with better error messages. - Remove 'init' command, consolidate functionality in 'new' - Add DEFAULT_OS_IMAGE constant for consistency - Add required parameter to load_app_config() for better error messages - Validate required GCP configuration before deploy - Update help text and examples --- scripts/bin/dstack-cloud | 133 +++++++++++---------------------------- 1 file changed, 38 insertions(+), 95 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 44ff9a2c0..9185a54a2 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -12,7 +12,6 @@ Supports local configuration files similar to git's working model. Usage: dstack-cloud new # Create a new project - dstack-cloud init # Initialize current directory dstack-cloud config-edit # Edit global configuration dstack-cloud prepare # Generate shared files dstack-cloud deploy # Deploy VM to cloud @@ -63,6 +62,7 @@ logger = logging.getLogger(__name__) APP_CONFIG_FILE = "app.json" STATE_FILE = "state.json" GLOBAL_CONFIG_PATH = os.path.expanduser("~/.config/dstack-cloud/config.json") +DEFAULT_OS_IMAGE = "dstack-cloud-0.6.0" @dataclass @@ -72,7 +72,7 @@ class App: name: str = "myapp" # OS image - os_image: str = "dstack-0.6.0" + os_image: str = DEFAULT_OS_IMAGE # GCP cloud configuration gcp_config: 'GcpConfig' = field(default_factory=lambda: GcpConfig()) @@ -141,7 +141,7 @@ class App: return { "name": "myapp", - "os_image": "dstack-0.6.0", + "os_image": DEFAULT_OS_IMAGE, "gcp_config": GcpConfig.get_template(), "instance_id_seed": instance_id_seed, "app_id": app_id, @@ -299,11 +299,20 @@ class CloudDeploymentManager: # Save the updated app config self.save_app_config(app) - def load_app_config(self) -> App: - """Load application configuration.""" + def load_app_config(self, required: bool = False) -> App: + """Load application configuration. + + Args: + required: If True, raise error when app.json doesn't exist + """ app_config_path = self.work_dir / APP_CONFIG_FILE if not app_config_path.exists(): + if required: + raise FileNotFoundError( + f"No {APP_CONFIG_FILE} found in {self.work_dir}. " + f"Run 'dstack-cloud new ' to create a project." + ) # Return default config if file doesn't exist return App() @@ -587,42 +596,6 @@ class CloudDeploymentManager: logger.info(f"Project directory: {project_dir}") logger.info("") - def init( - self, - force: bool = False, - interactive: bool = True, - os_image: Optional[str] = None, - app_id: Optional[str] = None, - gateway_enabled: Optional[bool] = None, - key_provider: Optional[str] = None, - storage_fs: Optional[str] = None, - secure_time: Optional[bool] = None, - no_instance_id: Optional[bool] = None, - project: Optional[str] = None, - zone: Optional[str] = None, - instance_name: Optional[str] = None, - machine_type: Optional[str] = None, - data_size: Optional[int] = None - ) -> None: - """Initialize deployment configuration in current directory.""" - self._init_project( - force=force, - interactive=interactive, - app_name=None, # Will prompt in interactive mode - os_image=os_image, - app_id=app_id, - gateway_enabled=gateway_enabled, - key_provider=key_provider, - storage_fs=storage_fs, - secure_time=secure_time, - no_instance_id=no_instance_id, - project=project, - zone=zone, - instance_name=instance_name, - machine_type=machine_type, - data_size=data_size - ) - def _init_project( self, force: bool = False, @@ -641,14 +614,14 @@ class CloudDeploymentManager: machine_type: Optional[str] = None, data_size: Optional[int] = None ) -> None: - """Initialize project configuration (shared implementation for init and new).""" + """Initialize project configuration.""" # Interactive prompts for required fields (only if not provided via CLI) instance_name_cli = instance_name if interactive: print(f"\n=== dstack-cloud Project Initialization ===\n") - # Prompt for app name (only for init command, new command provides it) + # Prompt for app name if not provided if app_name is None: while True: app_name = input("App name [myapp]: ").strip() @@ -822,8 +795,8 @@ class CloudDeploymentManager: """Generate all files in shared directory.""" import secrets - # Load app config - app = self.load_app_config() + # Load app config (required) + app = self.load_app_config(required=True) # Ensure instance_id_seed and app_id exist if not app.instance_id_seed: @@ -1304,8 +1277,23 @@ class CloudDeploymentManager: def deploy(self, delete_existing: bool = False, force_boot_image: bool = False) -> None: """Deploy VM to GCP.""" + # Load app config first (required) - validates project exists + app = self.load_app_config(required=True) config = self.load_gcp_config() - app = self.load_app_config() + + # Validate required configuration + missing = [] + if not config.instance_name: + missing.append("instance_name") + if not config.project: + missing.append("project") + if not config.zone: + missing.append("zone") + if missing: + raise ValueError( + f"Missing required GCP configuration: {', '.join(missing)}. " + f"Run 'dstack-cloud new ' to create a project or edit dstack-app.json." + ) # Auto-detect bucket if not specified if not config.bucket and config.project: @@ -2053,19 +2041,16 @@ def main(): parser = argparse.ArgumentParser( description="Multi-cloud VM lifecycle management tool", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" + epilog=f""" Examples: # Create a new project dstack-cloud new myproject - # Initialize current directory - dstack-cloud init - # Edit global configuration dstack-cloud config-edit # Download OS image - dstack-cloud pull dstack-cloud-0.6.0 + dstack-cloud pull {DEFAULT_OS_IMAGE} # Deploy VM dstack-cloud deploy @@ -2102,7 +2087,7 @@ Examples: new_parser.add_argument("name", type=str, help="Project name") # App configuration options - new_parser.add_argument("--os-image", type=str, help="OS image (e.g., dstack-0.6.0)") + new_parser.add_argument("--os-image", type=str, help=f"OS image (e.g., {DEFAULT_OS_IMAGE})") new_parser.add_argument("--app-id", type=str, help="Application ID (40 hex chars)") new_parser.add_argument("--gateway-enabled", "--gw", dest="gateway_enabled", action="store_true", help="Enable dstack-gateway") new_parser.add_argument("--no-gateway-enabled", "--no-gw", dest="gateway_enabled", action="store_false") @@ -2121,31 +2106,6 @@ Examples: new_parser.add_argument("--machine-type", "-m", type=str, help="Machine type (e.g., c3-standard-4)") new_parser.add_argument("--data-size", type=int, help="Data disk size in GB") - # init command - init_parser = subparsers.add_parser("init", help="Initialize deployment configuration") - init_parser.add_argument("--force", "-f", action="store_true", help="Overwrite existing config") - init_parser.add_argument("--non-interactive", "-n", action="store_true", help="Skip interactive prompts") - - # App configuration options (same as new) - init_parser.add_argument("--os-image", type=str, help="OS image (e.g., dstack-0.6.0)") - init_parser.add_argument("--app-id", type=str, help="Application ID (40 hex chars)") - init_parser.add_argument("--gateway-enabled", "--gw", dest="gateway_enabled", action="store_true", help="Enable dstack-gateway") - init_parser.add_argument("--no-gateway-enabled", "--no-gw", dest="gateway_enabled", action="store_false") - init_parser.set_defaults(gateway_enabled=None) - init_parser.add_argument("--key-provider", "--kp", type=str, choices=["kms", "local", "tpm", "none"], help="Key provider type") - init_parser.add_argument("--storage-fs", "--fs", dest="storage_fs", type=str, choices=["ext4", "zfs"], help="Storage filesystem") - init_parser.add_argument("--secure-time", action="store_true", help="Enable secure time synchronization") - init_parser.add_argument("--no-secure-time", dest="secure_time", action="store_false") - init_parser.set_defaults(secure_time=None) - init_parser.add_argument("--no-instance-id", action="store_true", help="Disable instance ID generation") - - # GCP configuration options (same as new) - init_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - init_parser.add_argument("--zone", "-z", type=str, help="GCP zone (e.g., us-central1-a)") - init_parser.add_argument("--instance-name", type=str, help="GCP instance name") - init_parser.add_argument("--machine-type", "-m", type=str, help="Machine type (e.g., c3-standard-4)") - init_parser.add_argument("--data-size", type=int, help="Data disk size in GB") - # config-edit command subparsers.add_parser("config-edit", help="Edit global configuration") @@ -2154,7 +2114,7 @@ Examples: # pull command pull_parser = subparsers.add_parser("pull", help="Download OS image") - pull_parser.add_argument("image", type=str, help="OS image name (e.g., dstack-0.6.0)") + pull_parser.add_argument("image", type=str, help=f"OS image name (e.g., {DEFAULT_OS_IMAGE})") # deploy command deploy_parser = subparsers.add_parser("deploy", help="Deploy VM to cloud") @@ -2244,23 +2204,6 @@ Examples: machine_type=args.machine_type, data_size=args.data_size ) - elif args.command == "init": - manager.init( - force=args.force, - interactive=not args.non_interactive, - os_image=args.os_image, - app_id=args.app_id, - gateway_enabled=args.gateway_enabled, - key_provider=args.key_provider, - storage_fs=args.storage_fs, - secure_time=args.secure_time, - no_instance_id=args.no_instance_id, - project=args.project, - zone=args.zone, - instance_name=args.instance_name, - machine_type=args.machine_type, - data_size=args.data_size - ) elif args.command == "config-edit": manager.config_edit() elif args.command == "prepare": From 5652ee2d6d3e471d80200678e83d693c42e7fb11 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 14:18:20 +0000 Subject: [PATCH 278/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index a30e29817..2eb52646a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit a30e29817d6268b09c618cb2f7f3fd14078d52a4 +Subproject commit 2eb52646a233ebf0d2d6e63493844eba65d768ed From 4f1d55af123e4be5c8208fbea7a3eeede51f8bea Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 14:43:14 +0000 Subject: [PATCH 279/435] Suppress meta-tpm warning --- meta-dstack/conf/distro/dstack.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index c7cf93eb6..4142d0f84 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -27,6 +27,9 @@ BAD_RECOMMENDATIONS = "busybox-syslog systemd-compat-units" # Skip unused components that fail metadata checks under walnascar SKIP_RECIPE[ostree] = "not required for dstack" +# Suppress meta-tpm warning (we use TPM drivers directly without tpm2 DISTRO_FEATURES) +SKIP_META_TPM_SANITY_CHECK = "1" + # EFI/UKI support for GCP images MACHINE_FEATURES:append = " efi" EFI_PROVIDER = "systemd-boot" From f899c7e5c9bbe7ddbdf44f45febe17871cba5b8d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 20 Jan 2026 15:16:48 +0000 Subject: [PATCH 280/435] Always calculate auth hash --- mkimage.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 1547bc05a..f6323fcc3 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -97,14 +97,10 @@ calc_authenticode_hash() { python3 "$AUTHENTICODE_HASH_SCRIPT" "$file" 2>/dev/null || true } -write_authenticode_hash_if_missing() { +write_authenticode_hash() { local file="$1" local out_file="${file}.auth_hash.txt" - if [[ -f "$out_file" ]]; then - return 0 - fi - if [[ ! -f "$file" ]]; then return 0 fi @@ -201,7 +197,7 @@ create_uki_artifacts() { build_uki_disk_image "${uki_dir}/disk.raw" "$UKI_IMAGE" "$ROOTFS_IMAGE" # Calculate and copy auth hash - write_authenticode_hash_if_missing "$UKI_IMAGE" + write_authenticode_hash "$UKI_IMAGE" if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then cp "${UKI_IMAGE}.auth_hash.txt" "${uki_dir}/auth_hash.txt" fi From 479f3bf691e9150a34b042fe9f34338f71f2ccc1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 00:31:46 +0000 Subject: [PATCH 281/435] Add gptfdisk --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4f4acc1bc..e87dab9a5 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -32,6 +32,7 @@ IMAGE_INSTALL = "\ xfsprogs \ e2fsprogs \ e2fsprogs-resize2fs \ + gptfdisk \ " IMAGE_NAME_SUFFIX ?= "" diff --git a/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend b/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend new file mode 100644 index 000000000..a366cc382 --- /dev/null +++ b/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend @@ -0,0 +1,2 @@ +# Only build sgdisk, skip cgdisk (requires ncurses) +PACKAGECONFIG = "popt" From 53a37fab47d4ec0a00853e22f646873bf36f549a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 00:45:37 +0000 Subject: [PATCH 282/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 127f5de46..6f14cb368 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 127f5de46af6a8bfce10a4c5392a3339f16aa3b5 +Subproject commit 6f14cb368804f95c920a983ba26594f386bf2f10 From 0b8d9c370aef0739b2409bb3e74c0951ef93719e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 08:24:11 +0000 Subject: [PATCH 283/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 2eb52646a..590b8f318 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 2eb52646a233ebf0d2d6e63493844eba65d768ed +Subproject commit 590b8f3186f4671398d5307f091e6b8c3a6bd7a0 From b3519ba4a3139068978fad2844fde6581b00c591 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 08:27:19 +0000 Subject: [PATCH 284/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 6f14cb368..6d146c37a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6f14cb368804f95c920a983ba26594f386bf2f10 +Subproject commit 6d146c37ab8883c51dc040a069be34ad10ca6a4f From e8656201e2988ea0980b97ef271d43cee71f1e6c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 08:28:46 +0000 Subject: [PATCH 285/435] Fix openssh build --- meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend index 81a2e54ab..1552932fa 100644 --- a/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend +++ b/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend @@ -4,7 +4,7 @@ SRC_URI += "file://disable-password-auth.conf" do_install:append() { install -d ${D}${sysconfdir}/ssh/sshd_config.d - install -m 0644 ${UNPACKDIR}/disable-password-auth.conf ${D}${sysconfdir}/ssh/sshd_config.d/ + install -m 0644 ${WORKDIR}/disable-password-auth.conf ${D}${sysconfdir}/ssh/sshd_config.d/ } FILES:${PN}-sshd += "${sysconfdir}/ssh/sshd_config.d/" From 037b9120393f324940bbd7ee9436783b94fe8e15 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 08:29:01 +0000 Subject: [PATCH 286/435] bump version to 0.5.6 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 5b2f6cc51..358352456 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.5" +DISTRO_VERSION = "0.5.6" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 6a7da729d545d88bb636f83f63c45e7438c4d544 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 14:35:29 +0000 Subject: [PATCH 287/435] Update meta-virtualization: fix runc reproducible build Update submodule to include fix for git describe --abbrev=8. Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Co-Authored-By: Happy --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index bce9962c5..806c998d8 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit bce9962c515089d62082953b795c0e80083c96e2 +Subproject commit 806c998d836a6adce32d01f32cf3b9c6871d5485 From d6b3acd98d92d28cd60b353faebe3cf1d6f16b2a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 21 Jan 2026 14:38:03 +0000 Subject: [PATCH 288/435] Update meta-virtualization/ submodule --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index 806c998d8..52cd8a290 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit 806c998d836a6adce32d01f32cf3b9c6871d5485 +Subproject commit 52cd8a290cf3989bc99bd9eb475b5db88000245a From 6afa83d7800eb49bb920a4888046371cfb1ab03e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 22 Jan 2026 13:30:26 +0000 Subject: [PATCH 289/435] Update meta-virtualization --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index ebb9f7cdd..809318cf9 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit ebb9f7cdd5475efd0fdac840840334376db98fbb +Subproject commit 809318cf945102726b6c2afbf1f977e2a607865b From 65d6ffddf0e01fe0e75c61d3e7089a7165e89b59 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 23 Jan 2026 01:01:14 +0000 Subject: [PATCH 290/435] Fix env issue in dstack-cloud --- scripts/bin/dstack-cloud | 119 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 9185a54a2..f6c92e6e0 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -43,14 +43,24 @@ from typing import Optional, List, Dict, Any # Try to import cryptography libraries for env encryption CRYPTO_AVAILABLE = False +ETH_CRYPTO_AVAILABLE = False try: from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.asymmetric import x25519 - from cryptography.hazmat.backends import default_backend - from cryptography import serialization + from cryptography.hazmat.primitives import serialization CRYPTO_AVAILABLE = True -except ImportError: - CRYPTO_AVAILABLE = False +except Exception: + pass + +try: + from eth_keys import keys + from eth_utils import keccak + ETH_CRYPTO_AVAILABLE = True +except Exception: + pass + +# Default whitelist file location +DEFAULT_KMS_WHITELIST_PATH = os.path.expanduser("~/.config/dstack-cloud/kms-whitelist.json") logging.basicConfig( level=logging.INFO, @@ -1440,7 +1450,7 @@ class CloudDeploymentManager: if not CRYPTO_AVAILABLE: raise ImportError( "Cryptography libraries not available. Please install:\n" - "pip install cryptography" + "pip install cryptography eth-keys 'eth-hash[pycryptodome]'" ) # Serialize environment variables to JSON @@ -1481,6 +1491,7 @@ class CloudDeploymentManager: try: import urllib.request import urllib.error + import ssl path = f"{kms_url}/prpc/GetAppEnvEncryptPubKey?json" data = json.dumps({"app_id": app_id}).encode("utf-8") @@ -1491,19 +1502,115 @@ class CloudDeploymentManager: headers={"Content-Type": "application/json"} ) + # Allow self-signed certificates for KMS server + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + logger.info(f"Getting encryption public key for {app_id} from {kms_url}") - with urllib.request.urlopen(req, timeout=10) as response: + with urllib.request.urlopen(req, timeout=10, context=ssl_context) as response: response_data = json.loads(response.read().decode("utf-8")) if "public_key" not in response_data: raise ValueError(f"No public_key in response: {response_data}") + # Verify signature if available + if "signature" not in response_data: + if not self._confirm_untrusted_signer("none"): + raise ValueError("Aborted due to missing signature") + return response_data["public_key"] + + public_key = bytes.fromhex(response_data["public_key"]) + signature = bytes.fromhex(response_data["signature"]) + + signer_pubkey = self._verify_signature(public_key, signature, app_id) + if signer_pubkey: + whitelist = self._load_whitelist() + if whitelist and signer_pubkey not in whitelist: + logger.warning(f"Signer {signer_pubkey} is not in the trusted whitelist!") + if not self._confirm_untrusted_signer(signer_pubkey): + raise ValueError("Aborted due to untrusted signer") + else: + logger.info(f"Verified signature from: {signer_pubkey}") + else: + logger.warning("Could not verify signature!") + if not self._confirm_untrusted_signer("unknown"): + raise ValueError("Aborted due to invalid signature") + return response_data["public_key"] except Exception as e: logger.warning(f"Failed to get encryption public key: {e}") raise + def _verify_signature(self, public_key: bytes, signature: bytes, app_id: str) -> Optional[str]: + """Verify the signature of a public key. + + Args: + public_key: The public key bytes to verify + signature: The signature bytes + app_id: The application ID + + Returns: + The compressed public key if valid, None otherwise + """ + if not ETH_CRYPTO_AVAILABLE: + logger.warning("eth-keys not available, skipping signature verification. " + "Install with: pip install eth-keys 'eth-hash[pycryptodome]'") + return None + + if len(signature) != 65: + return None + + # Create the message to verify + prefix = b"dstack-env-encrypt-pubkey" + if app_id.startswith("0x"): + app_id = app_id[2:] + message = prefix + b":" + bytes.fromhex(app_id) + public_key + + # Hash the message with Keccak-256 and recover the public key + try: + message_hash = keccak(message) + sig = keys.Signature(signature_bytes=signature) + recovered_key = sig.recover_public_key_from_msg_hash(message_hash) + return '0x' + recovered_key.to_compressed_bytes().hex() + except Exception as e: + error_msg = str(e) + if "hashing backends" in error_msg or "pycryptodome" in error_msg: + raise ImportError( + "Ethereum hashing backend not available. Please install:\n" + "pip install cryptography eth-keys 'eth-hash[pycryptodome]'" + ) + logger.debug(f"Signature verification failed: {e}") + return None + + def _confirm_untrusted_signer(self, signer: str) -> bool: + """Ask user to confirm using an untrusted signer.""" + try: + response = input(f"Continue with untrusted signer '{signer}'? (y/N): ") + return response.lower() in ('y', 'yes') + except EOFError: + # Non-interactive mode, reject untrusted signers + return False + + def _load_whitelist(self) -> List[str]: + """Load the whitelist of trusted signers from a file.""" + if not os.path.exists(DEFAULT_KMS_WHITELIST_PATH): + return [] + + try: + with open(DEFAULT_KMS_WHITELIST_PATH, 'r') as f: + data = json.load(f) + return data.get('trusted_signers', []) + except (json.JSONDecodeError, FileNotFoundError): + return [] + + def _save_whitelist(self, whitelist: List[str]) -> None: + """Save the whitelist of trusted signers to a file.""" + os.makedirs(os.path.dirname(DEFAULT_KMS_WHITELIST_PATH), exist_ok=True) + with open(DEFAULT_KMS_WHITELIST_PATH, 'w') as f: + json.dump({'trusted_signers': whitelist}, f, indent=2) + def _derive_instance_id(self, instance_id_seed: str, app_id: str) -> str: """Derive instance_id from instance_id_seed and app_id. From b49e84b79416ab9b890d7a1a97e905b2d281ae78 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 23 Jan 2026 01:01:30 +0000 Subject: [PATCH 291/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 590b8f318..702740d49 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 590b8f3186f4671398d5307f091e6b8c3a6bd7a0 +Subproject commit 702740d49a4d67d0ce186ed7afaf1e09adc6aeb6 From c92e0a87babc3efda162864d776305a7b78139b9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 26 Jan 2026 06:27:45 +0000 Subject: [PATCH 292/435] Upgrade kernel to 6.18.7 - Add linux-custom_6.18.7.bb recipe - Update PREFERRED_VERSION_linux-custom to 6.18.7 - Copy defconfig from 6.17 with TDX DMA patch still applied --- meta-dstack/conf/distro/dstack.conf | 2 +- .../recipes-kernel/linux/files/6.18/defconfig | 359 ++++++++++++++++++ .../linux/linux-custom_6.18.7.bb | 36 ++ 3 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 meta-dstack/recipes-kernel/linux/files/6.18/defconfig create mode 100644 meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 4142d0f84..86873c988 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -9,7 +9,7 @@ DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" PREFERRED_PROVIDER_virtual/kernel = "linux-custom" -PREFERRED_VERSION_linux-custom ?= "6.17.6" +PREFERRED_VERSION_linux-custom ?= "6.18.7" LINUX_KERNEL_TYPE = "tiny" MACHINE_FEATURES += "numa" diff --git a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig new file mode 100644 index 000000000..7c03fce39 --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig @@ -0,0 +1,359 @@ +CONFIG_LOCALVERSION="-dstack" +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZMA=y +CONFIG_KERNEL_GZIP=y +CONFIG_DMA_COHERENT_POOL=y +CONFIG_DMA_DIRECT_REMAP=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_PREEMPT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CGROUPS=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_SCHED=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_DEBUG=y +CONFIG_USER_NS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_SYSFS_SYSCALL=y +CONFIG_SMP=y +CONFIG_X86_AMD_PLATFORM_DEVICE=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT_SPINLOCKS=y +CONFIG_INTEL_TDX_GUEST=y +CONFIG_TDX_GUEST_DRIVER=y +CONFIG_TSM_REPORTS=y +CONFIG_CONFIGFS_FS=y +CONFIG_NR_CPUS=512 +CONFIG_X86_MSR=y +CONFIG_X86_CPUID=y +CONFIG_X86_CHECK_BIOS_CORRUPTION=y +CONFIG_EFI=y +CONFIG_EFI_STUB=y +CONFIG_HZ_1000=y +# CONFIG_SUSPEND is not set +# CONFIG_ACPI_DEBUG is not set +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_X86_ACPI_CPUFREQ=y +CONFIG_INTEL_IDLE=y +CONFIG_IA32_EMULATION=y +CONFIG_KPROBES=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_XFRM_USER=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_INET_ESP=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NETFILTER_XTABLES_COMPAT=y +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_VS=m +CONFIG_NF_REJECT_IPV4=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=y +CONFIG_6LOWPAN=m +CONFIG_NET_SCHED=y +CONFIG_VSOCKETS=y +CONFIG_VIRTIO_VSOCKETS=y +CONFIG_HYPERV_VSOCKETS=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_CFG80211=m +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_PCI_HYPERV=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_DMA_DIRECT_REMAP=y +CONFIG_DMA_COHERENT_POOL=y +CONFIG_CMA=y +CONFIG_CMA_SIZE_SEL_MBYTES=y +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +CONFIG_CMA_SIZE_MBYTES=64 +CONFIG_DMA_CMA=y +CONFIG_GVE=y +CONFIG_SWIOTLB_DYNAMIC=y +CONFIG_ZONE_DMA=y +CONFIG_CONNECTOR=y +CONFIG_PARPORT=m +CONFIG_PARPORT_PC=m +CONFIG_BLK_DEV_FD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=1 +CONFIG_BLK_DEV_RAM_SIZE=6144 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_EEPROM_AT24=m +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_FC_ATTRS=y +CONFIG_SCSI_VIRTIO=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_ATA_PIIX=y +CONFIG_PATA_SCH=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_NETDEVICES=y +CONFIG_WIREGUARD=y +CONFIG_IPVLAN=m +CONFIG_VXLAN=y +CONFIG_NETCONSOLE=y +CONFIG_TUN=m +CONFIG_VETH=y +CONFIG_VIRTIO_NET=y +CONFIG_TYPHOON=m +CONFIG_PCNET32=m +CONFIG_ATL2=m +CONFIG_ATL1=m +CONFIG_ATL1E=m +CONFIG_ATL1C=m +CONFIG_ALX=m +CONFIG_TIGON3=m +CONFIG_BNX2X=m +CONFIG_BNXT=m +CONFIG_DL2K=m +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IGC=m +CONFIG_JME=m +CONFIG_SKGE=m +CONFIG_SKY2=m +CONFIG_8139CP=m +CONFIG_8139TOO=m +CONFIG_R8169=m +CONFIG_REALTEK_PHY_HWMON=y +CONFIG_USB_NET_DRIVERS=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_USBNET=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_KC2190=y +# CONFIG_WLAN is not set +CONFIG_HYPERV_NET=y +CONFIG_ISDN=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_TABLET=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_PRINTER=m +CONFIG_VIRTIO_CONSOLE=y +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_HPET=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TPM2_HMAC=y +CONFIG_TCG_TIS=y +CONFIG_TCG_CRB=y +CONFIG_I2C_I801=y +CONFIG_WATCHDOG=y +CONFIG_BCMA=m +CONFIG_BCMA_DRIVER_GMAC_CMN=y +CONFIG_AGP=y +CONFIG_AGP_INTEL=y +CONFIG_DRM=y +CONFIG_DRM_I915=m +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_DRM_CIRRUS_QEMU=m +CONFIG_FB=y +CONFIG_FB_UVESA=m +CONFIG_FB_EFI=y +CONFIG_FB_HYPERV=y +CONFIG_HID_HYPERV_MOUSE=y +CONFIG_HID_WACOM=y +CONFIG_USB_HIDDEV=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_SERIAL=y +CONFIG_USB_SERIAL_CONSOLE=y +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_FTDI_SIO=y +CONFIG_USB_SERIAL_PL2303=y +CONFIG_USB_EZUSB_FX2=y +CONFIG_RTC_CLASS=y +CONFIG_VIRT_DRIVERS=y +CONFIG_EFI_SECRET=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=y +CONFIG_HYPERV=y +CONFIG_HYPERV_UTILS=y +CONFIG_HYPERV_BALLOON=y +CONFIG_INTEL_IOMMU=y +# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set +CONFIG_IRQ_REMAP=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_XFS_FS=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_RT=y +CONFIG_BTRFS_FS=m +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_FUSE_FS=m +CONFIG_CUSE=m +CONFIG_OVERLAY_FS=y +CONFIG_ISO9660_FS=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_EFIVAR_FS=y +CONFIG_SQUASHFS=y +CONFIG_VXFS_FS=m +CONFIG_NFS_FS=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +# CONFIG_NFS_DISABLE_UDP_SUPPORT is not set +CONFIG_9P_FS=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_AES_TI=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_ARC4=y +CONFIG_CRYPTO_CTS=y +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_DEV_VIRTIO=y +CONFIG_CORDIC=m +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_DEBUG_INFO_BTF=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_UNWINDER_FRAME_POINTER=y + +CONFIG_FAT_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y diff --git a/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb b/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb new file mode 100644 index 000000000..96c0f7aa4 --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb @@ -0,0 +1,36 @@ +SUMMARY = "dstack Linux kernel 6.18.7 built from tarball" +DESCRIPTION = "Custom dstack kernel based on upstream Linux 6.18.7 with tiny Kconfig baseline tuned for TDX guests" +SECTION = "kernel" +LICENSE = "GPL-2.0-only" +LIC_FILES_CHKSUM = "file://COPYING;md5=6bc538ed5bd9a7fc9398086aedcd7e46" + +PV = "6.18.7" +LINUX_VERSION = "${PV}" + +inherit kernel + +FILESEXTRAPATHS:prepend := "${THISDIR}/files/6.18:${THISDIR}/files:" + +DEPENDS += "libyaml-native openssl-native util-linux-native elfutils-native" + +SRC_URI = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${PV}.tar.xz;downloadfilename=linux-${PV}.tar.xz \ + file://defconfig \ + file://0001-x86-tdx-select-dma-direct-remap.patch \ +" + +SRC_URI[sha256sum] = "b726a4d15cf9ae06219b56d87820776e34d89fbc137e55fb54a9b9c3015b8f1e" + +S = "${UNPACKDIR}/linux-${PV}" + +LINUX_VERSION_EXTENSION = "-dstack" +KERNEL_VERSION_EXTENSION = "-dstack" + +# Enable BTF debug info for bpftool and out-of-tree modules (ZFS, WireGuard, etc.) +KERNEL_DEBUG = "True" + +# Keep packaging aligned with our tiny x86_64 guest machines. +COMPATIBLE_MACHINE = "(tdx|sev-snp|qemux86-64)" + +do_deploy:append() { + install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config +} From 44f2945cdb8c1a3bd1e0b4118f3a9bbb47f3a79d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 22 Jan 2026 15:58:28 +0000 Subject: [PATCH 293/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 6d146c37a..aca96b6b1 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 6d146c37ab8883c51dc040a069be34ad10ca6a4f +Subproject commit aca96b6b1d147cdf7645c950e90db17ca4ff074a From 7cf15f5313345147ead01c3bf4a47fbe4d9a3809 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 23 Jan 2026 01:01:30 +0000 Subject: [PATCH 294/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 702740d49..5b3fa9a1a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 702740d49a4d67d0ce186ed7afaf1e09adc6aeb6 +Subproject commit 5b3fa9a1a34ef69e6785742ee0cd9ddb995594f9 From 2b082d6367425b9a61b68ae66c63675a633ca48b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 27 Jan 2026 05:23:51 +0000 Subject: [PATCH 295/435] feat: Enable systemd socket proxy for dstack backward compatibility - Remove systemd-extra-utils from BAD_RECOMMENDATIONS to include systemd-socket-proxyd binary - Install dstack-socket.socket/service and tappd-socket.socket/service for backward compatibility with containers mounting socket files directly --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 8 ++++++-- meta-dstack/recipes-core/systemd/systemd_%.bbappend | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 93f6d6b27..3756dd048 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -11,14 +11,14 @@ SRC_DIR = '${REPO_ROOT}/dstack' S = "${WORKDIR}/dstack" -RDEPENDS:${PN} += "bash" +RDEPENDS:${PN} += "bash systemd-socket-proxyd" DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" -DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service" +DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service dstack-socket.socket dstack-socket.service tappd-socket.socket tappd-socket.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -76,6 +76,10 @@ do_install() { install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-socket.socket ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-socket.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tappd-socket.socket ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tappd-socket.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d install -m 0644 ${S}/basefiles/docker.service.d/dstack-guest-agent.conf ${D}${sysconfdir}/systemd/system/docker.service.d/ diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index 8c49ea4ec..79af92e98 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -14,3 +14,9 @@ do_install:append() { SYSTEMD_SERVICE:${PN}-vconsole-setup = "" FILES:${PN} += "${sysconfdir}/modprobe.d/blacklist-autofs4.conf" + +# Create a minimal package with only systemd-socket-proxyd +# This avoids pulling in all of systemd-extra-utils +PACKAGES =+ "${PN}-socket-proxyd" +FILES:${PN}-socket-proxyd = "${nonarch_libdir}/systemd/systemd-socket-proxyd" +RDEPENDS:${PN}-socket-proxyd = "${PN}" From edc26be1129a4f8f5079a5b86895f8f0f912d627 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 27 Jan 2026 08:13:32 +0000 Subject: [PATCH 296/435] feat: Add dstack-prepare service drop-in configuration - Install drop-in configurations for docker.service and containerd.service - Remove systemd-extra-utils from BAD_RECOMMENDATIONS to allow its installation --- meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 358352456..55fc15b2d 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -24,4 +24,4 @@ SERIAL_CONSOLES = "115200;ttyS0" PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.92.0" PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" -BAD_RECOMMENDATIONS = "busybox-syslog systemd-extra-utils" +BAD_RECOMMENDATIONS = "busybox-syslog" diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 3756dd048..87cf99f8c 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -82,8 +82,15 @@ do_install() { install -m 0644 ${S}/basefiles/tappd-socket.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d - install -m 0644 ${S}/basefiles/docker.service.d/dstack-guest-agent.conf ${D}${sysconfdir}/systemd/system/docker.service.d/ + install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ + + install -d ${D}${sysconfdir}/systemd/system/containerd.service.d + install -m 0644 ${S}/basefiles/containerd.service.d/* ${D}${sysconfdir}/systemd/system/containerd.service.d/ fi } -FILES:${PN} += "${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf" +FILES:${PN} += " \ + ${sysconfdir}/systemd/system/docker.service.d/dstack-guest-agent.conf \ + ${sysconfdir}/systemd/system/docker.service.d/dstack-prepare.conf \ + ${sysconfdir}/systemd/system/containerd.service.d/dstack-prepare.conf \ +" From 51ba7eabb09c0b579733d176cdb96866d5409a28 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 27 Jan 2026 08:47:04 +0000 Subject: [PATCH 297/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index aca96b6b1..c92c2b416 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit aca96b6b1d147cdf7645c950e90db17ca4ff074a +Subproject commit c92c2b41647a1e7cb2569f0644bcfb1c8b3dc6f4 From f9db23adffc5eae2fa08397e107d0ec2b20b37a0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 27 Jan 2026 06:58:36 +0000 Subject: [PATCH 298/435] feat: Enable systemd socket proxy for dstack backward compatibility - Add systemd-socket-proxyd subpackage to avoid pulling full systemd-extra-utils - Add systemd-extra-utils to BAD_RECOMMENDATIONS - Install dstack-socket.socket/service and tappd-socket.socket/service for backward compatibility with containers mounting socket files directly --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 8 ++++++-- meta-dstack/recipes-core/systemd/systemd_%.bbappend | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 50362e4fe..282595c23 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -11,14 +11,14 @@ SRC_DIR = '${REPO_ROOT}/dstack' S = "${UNPACKDIR}/dstack" -RDEPENDS:${PN} += "bash" +RDEPENDS:${PN} += "bash systemd-socket-proxyd" DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" -DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service" +DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service dstack-socket.socket dstack-socket.service tappd-socket.socket tappd-socket.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -76,6 +76,10 @@ do_install() { install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-socket.socket ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-socket.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tappd-socket.socket ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/tappd-socket.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index 7c1ea9028..041ce6c37 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -19,3 +19,9 @@ EOF SYSTEMD_SERVICE:${PN}-vconsole-setup = "" PACKAGECONFIG:remove = "sysvinit logind" + +# Create a minimal package with only systemd-socket-proxyd +# This avoids pulling in all of systemd-extra-utils +PACKAGES =+ "${PN}-socket-proxyd" +FILES:${PN}-socket-proxyd = "${nonarch_libdir}/systemd/systemd-socket-proxyd" +RDEPENDS:${PN}-socket-proxyd = "${PN}" From 7d30c7b769aac274e64b2bf9881aaceaf2e9e26f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 27 Jan 2026 13:44:18 +0000 Subject: [PATCH 299/435] Add ephemeral-docker.sh --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index 5b3fa9a1a..cd158f1dc 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 5b3fa9a1a34ef69e6785742ee0cd9ddb995594f9 +Subproject commit cd158f1dc47d4addf5e220b13f2bacee714d7486 diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 282595c23..cbeb6729d 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -60,6 +60,7 @@ do_install() { install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} + install -m 0755 ${S}/basefiles/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} install -m 0755 ${S}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json From 0e8062ed9808f41725ea455f87c607c52310098c Mon Sep 17 00:00:00 2001 From: Longarithm Date: Tue, 27 Jan 2026 19:03:29 +0400 Subject: [PATCH 300/435] fix nvidia --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 7 +------ meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 93f6d6b27..6a5c662f0 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -29,12 +29,7 @@ inherit cargo_bin do_unpack() { mkdir -p ${S} rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ - - if ${@bb.utils.contains('IMAGE_INSTALL', 'nvidia-container-toolkit', 'true', 'false', d)}; then - cp ${THISDIR}/files/docker-daemon-nvidia.json ${S}/docker-daemon.json - else - cp ${THISDIR}/files/docker-daemon.json ${S}/docker-daemon.json - fi + cp ${THISDIR}/files/docker-daemon.json ${S}/docker-daemon.json } # Force the configure task to run every time to detect source changes diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 3dc7883da..69ff82bf7 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -10,3 +10,8 @@ NVIDIA_GROUP = "acpid \ " KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm video" IMAGE_INSTALL:append = " ${NVIDIA_GROUP}" + +install_nvidia_docker_config() { + install -m 0644 ${THISDIR}/../dstack-guest/files/docker-daemon-nvidia.json ${IMAGE_ROOTFS}${sysconfdir}/docker/daemon.json +} +ROOTFS_POSTPROCESS_COMMAND:append = " install_nvidia_docker_config;" From 83f8d58b02d0290f3fa9ef392cfd4a7b4be88832 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 Jan 2026 05:13:40 +0000 Subject: [PATCH 301/435] fix(dstack-cloud): fix env encryption format for CVM decryption - Write .encrypted-env as binary instead of hex text - Serialize env vars as key/value pair array matching CVM's expected format - Change _encrypt_env return type from str (hex) to bytes (raw) --- scripts/bin/dstack-cloud | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index f6c92e6e0..7c110b8f6 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -1159,7 +1159,7 @@ class CloudDeploymentManager: # Save to shared/.encrypted-env encrypted_file = shared_dir / ".encrypted-env" - with open(encrypted_file, 'w') as f: + with open(encrypted_file, 'wb') as f: f.write(encrypted_env) logger.info(f"Encrypted {app.env_file} -> {encrypted_file}") else: @@ -1436,7 +1436,7 @@ class CloudDeploymentManager: envs[key.strip()] = value.strip() return envs - def _encrypt_env(self, envs: Dict[str, str], hex_public_key: str) -> str: + def _encrypt_env(self, envs: Dict[str, str], hex_public_key: str) -> bytes: """ Encrypt environment variables using X25519 key exchange and AES-GCM. @@ -1445,7 +1445,7 @@ class CloudDeploymentManager: hex_public_key: Remote encryption public key in hexadecimal format Returns: - Hex string of (ephemeral public key || IV || ciphertext) + Raw bytes of (ephemeral public key || IV || ciphertext) """ if not CRYPTO_AVAILABLE: raise ImportError( @@ -1453,8 +1453,9 @@ class CloudDeploymentManager: "pip install cryptography eth-keys 'eth-hash[pycryptodome]'" ) - # Serialize environment variables to JSON - envs_json = json.dumps({"env": envs}).encode("utf-8") + # Serialize environment variables to JSON (format: {"env": [{"key": k, "value": v}, ...]}) + env_pairs = [{"key": k, "value": v} for k, v in envs.items()] + envs_json = json.dumps({"env": env_pairs}).encode("utf-8") # Remove "0x" prefix if present if hex_public_key.startswith("0x"): @@ -1484,7 +1485,7 @@ class CloudDeploymentManager: # Combine ephemeral public key, IV, and ciphertext result = ephemeral_public_bytes + iv + ciphertext - return result.hex() + return result def _get_app_encrypt_pub_key(self, app_id: str, kms_url: str) -> str: """Get encryption public key for the specified app_id from KMS.""" From 6da8d8623c9bb4756e8f46b79df622efc39225e5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 Jan 2026 05:16:30 +0000 Subject: [PATCH 302/435] feat: built-in systemd socket activation for dstack-guest-agent Remove dependency on systemd-socket-proxyd by using native socket activation in dstack-guest-agent. Replace separate socket/service proxy units with a single dstack-guest-agent.socket unit. --- dstack | 2 +- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dstack b/dstack index cd158f1dc..a046ae985 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit cd158f1dc47d4addf5e220b13f2bacee714d7486 +Subproject commit a046ae985b1971d6751e3a823f31e5dd545263de diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index cbeb6729d..fc70f718a 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -11,14 +11,14 @@ SRC_DIR = '${REPO_ROOT}/dstack' S = "${UNPACKDIR}/dstack" -RDEPENDS:${PN} += "bash systemd-socket-proxyd" +RDEPENDS:${PN} += "bash" DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" -DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service dstack-socket.socket dstack-socket.service tappd-socket.socket tappd-socket.service" +DSTACK_SERVICES = "dstack-guest-agent.service dstack-guest-agent.socket dstack-prepare.service app-compose.service wg-checker.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -77,10 +77,7 @@ do_install() { install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-socket.socket ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-socket.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/tappd-socket.socket ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/tappd-socket.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-guest-agent.socket ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ From 7ef2b873fc3fb9e6fb63868cc275f21402e3032a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 Jan 2026 15:02:47 +0000 Subject: [PATCH 303/435] Move docker daemon config from package to image level The docker daemon.json is an image-level concern, not a package-level one. Use a DOCKER_DAEMON_JSON variable in dstack-rootfs-base.inc that nvidia inc can simply override, instead of conditionals in dstack-guest.bb or cross-recipe path references. --- .../recipes-core/dstack-guest/dstack-guest.bb | 3 --- .../recipes-core/images/dstack-rootfs-base.inc | 8 ++++++++ .../recipes-core/images/dstack-rootfs-nvidia.inc | 5 +---- .../images/files/docker-daemon-nvidia.json | 13 +++++++++++++ .../recipes-core/images/files/docker-daemon.json | 7 +++++++ 5 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json create mode 100644 meta-dstack/recipes-core/images/files/docker-daemon.json diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 6a5c662f0..e60141df0 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -29,7 +29,6 @@ inherit cargo_bin do_unpack() { mkdir -p ${S} rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ - cp ${THISDIR}/files/docker-daemon.json ${S}/docker-daemon.json } # Force the configure task to run every time to detect source changes @@ -50,14 +49,12 @@ do_compile[network] = "1" do_install() { install -d ${D}${bindir} - install -d ${D}${sysconfdir}/docker install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} - install -m 0755 ${S}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4f4acc1bc..1fa93449b 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -64,6 +64,7 @@ ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" ROOTFS_POSTPROCESS_COMMAND += "symlink_lib64;" IMAGE_FEATURES[validitems] += "nologin" ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_login", "",d)}' +ROOTFS_POSTPROCESS_COMMAND += "install_docker_config;" ROOTFS_POSTPROCESS_COMMAND += "mkdirs;" @@ -133,6 +134,13 @@ disable_login() { > ${IMAGE_ROOTFS}/etc/securetty } +DOCKER_DAEMON_JSON ?= "${THISDIR}/files/docker-daemon.json" + +install_docker_config() { + install -d ${IMAGE_ROOTFS}${sysconfdir}/docker + install -m 0644 ${DOCKER_DAEMON_JSON} ${IMAGE_ROOTFS}${sysconfdir}/docker/daemon.json +} + mkdirs() { mkdir -p ${IMAGE_ROOTFS}/dev mkdir -p ${IMAGE_ROOTFS}/proc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 69ff82bf7..2b2355b23 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -11,7 +11,4 @@ NVIDIA_GROUP = "acpid \ KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm video" IMAGE_INSTALL:append = " ${NVIDIA_GROUP}" -install_nvidia_docker_config() { - install -m 0644 ${THISDIR}/../dstack-guest/files/docker-daemon-nvidia.json ${IMAGE_ROOTFS}${sysconfdir}/docker/daemon.json -} -ROOTFS_POSTPROCESS_COMMAND:append = " install_nvidia_docker_config;" +DOCKER_DAEMON_JSON = "${THISDIR}/files/docker-daemon-nvidia.json" diff --git a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json new file mode 100644 index 000000000..7ed24bdfa --- /dev/null +++ b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json @@ -0,0 +1,13 @@ +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "100m", + "max-file": "10" + }, + "runtimes": { + "nvidia": { + "args": [], + "path": "nvidia-container-runtime" + } + } +} diff --git a/meta-dstack/recipes-core/images/files/docker-daemon.json b/meta-dstack/recipes-core/images/files/docker-daemon.json new file mode 100644 index 000000000..4d6c550d7 --- /dev/null +++ b/meta-dstack/recipes-core/images/files/docker-daemon.json @@ -0,0 +1,7 @@ +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "100m", + "max-file": "10" + } +} From 7e7934ab6b488331a28a1af217e4860118e4a008 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 Jan 2026 15:05:38 +0000 Subject: [PATCH 304/435] Remove docker daemon json from dstack-guest/files (moved to images/files) --- .../dstack-guest/files/docker-daemon-nvidia.json | 13 ------------- .../dstack-guest/files/docker-daemon.json | 7 ------- 2 files changed, 20 deletions(-) delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json delete mode 100644 meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json deleted file mode 100644 index 7ed24bdfa..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "log-driver": "json-file", - "log-opts": { - "max-size": "100m", - "max-file": "10" - }, - "runtimes": { - "nvidia": { - "args": [], - "path": "nvidia-container-runtime" - } - } -} diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json b/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json deleted file mode 100644 index 4d6c550d7..000000000 --- a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "log-driver": "json-file", - "log-opts": { - "max-size": "100m", - "max-file": "10" - } -} From cb2c347802908fa8f5e97ab25c9cc247e900a5b0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 29 Jan 2026 15:09:39 +0000 Subject: [PATCH 305/435] Move docker daemon config from package to image level The docker daemon.json is an image-level concern, not a package-level one. Package recipes can't access image-level variables like IMAGE_INSTALL, so the nvidia condition in dstack-guest.bb never worked. Use a DOCKER_DAEMON_JSON variable in dstack-rootfs-base.inc that nvidia inc can simply override. --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 8 -------- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 8 ++++++++ meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc | 2 ++ .../files/docker-daemon-nvidia.json | 0 .../{dstack-guest => images}/files/docker-daemon.json | 0 5 files changed, 10 insertions(+), 8 deletions(-) rename meta-dstack/recipes-core/{dstack-guest => images}/files/docker-daemon-nvidia.json (100%) rename meta-dstack/recipes-core/{dstack-guest => images}/files/docker-daemon.json (100%) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index fc70f718a..be680ff9d 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -29,12 +29,6 @@ inherit cargo_bin do_unpack() { mkdir -p ${S} rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ - - if ${@bb.utils.contains('IMAGE_INSTALL', 'nvidia-container-toolkit', 'true', 'false', d)}; then - cp ${THISDIR}/files/docker-daemon-nvidia.json ${S}/docker-daemon.json - else - cp ${THISDIR}/files/docker-daemon.json ${S}/docker-daemon.json - fi } # Force the configure task to run every time to detect source changes @@ -55,7 +49,6 @@ do_compile[network] = "1" do_install() { install -d ${D}${bindir} - install -d ${D}${sysconfdir}/docker install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} @@ -63,7 +56,6 @@ do_install() { install -m 0755 ${S}/basefiles/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} - install -m 0755 ${S}/docker-daemon.json ${D}${sysconfdir}/docker/daemon.json install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 09605e8e1..99ca19e18 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -66,6 +66,7 @@ ROOTFS_POSTPROCESS_COMMAND += "remove_sysvinit_files;" ROOTFS_POSTPROCESS_COMMAND += "symlink_lib64;" IMAGE_FEATURES[validitems] += "nologin" ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'nologin' ], "disable_login", "",d)}' +ROOTFS_POSTPROCESS_COMMAND += "install_docker_config;" ROOTFS_POSTPROCESS_COMMAND += "mkdirs;" @@ -135,6 +136,13 @@ disable_login() { > ${IMAGE_ROOTFS}/etc/securetty } +DOCKER_DAEMON_JSON ?= "${THISDIR}/files/docker-daemon.json" + +install_docker_config() { + install -d ${IMAGE_ROOTFS}${sysconfdir}/docker + install -m 0644 ${DOCKER_DAEMON_JSON} ${IMAGE_ROOTFS}${sysconfdir}/docker/daemon.json +} + mkdirs() { mkdir -p ${IMAGE_ROOTFS}/dev mkdir -p ${IMAGE_ROOTFS}/proc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 3dc7883da..2b2355b23 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -10,3 +10,5 @@ NVIDIA_GROUP = "acpid \ " KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm video" IMAGE_INSTALL:append = " ${NVIDIA_GROUP}" + +DOCKER_DAEMON_JSON = "${THISDIR}/files/docker-daemon-nvidia.json" diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json similarity index 100% rename from meta-dstack/recipes-core/dstack-guest/files/docker-daemon-nvidia.json rename to meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json diff --git a/meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json b/meta-dstack/recipes-core/images/files/docker-daemon.json similarity index 100% rename from meta-dstack/recipes-core/dstack-guest/files/docker-daemon.json rename to meta-dstack/recipes-core/images/files/docker-daemon.json From 4263d51018671ed78a1c7ade7d684bdd798d9816 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 2 Feb 2026 06:12:25 +0000 Subject: [PATCH 306/435] feat(guest): install conntrack sysctl config into CVM --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 1d9b30843..33a7a829c 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -60,6 +60,9 @@ do_install() { install -d ${D}${sysconfdir}/ install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf + install -d ${D}${sysconfdir}/sysctl.d + install -m 0644 ${S}/basefiles/sysctl.d/99-dstack.conf ${D}${sysconfdir}/sysctl.d/99-dstack.conf + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir} \ ${D}${sysconfdir}/systemd/resolved.conf.d From f4aedf4b69bf11759eca1fc9b6b475c5b3a81402 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 2 Feb 2026 08:36:17 +0000 Subject: [PATCH 307/435] feat(guest): switch to native systemd socket activation Replace systemd-socket-proxyd with native socket activation in dstack-guest-agent, matching dstack commit c6d37b1c. - Remove systemd-socket-proxyd runtime dependency - Replace dstack-socket.{socket,service} and tappd-socket.{socket,service} with dstack-guest-agent.socket --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 33a7a829c..a4a51b913 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -11,14 +11,14 @@ SRC_DIR = '${REPO_ROOT}/dstack' S = "${WORKDIR}/dstack" -RDEPENDS:${PN} += "bash systemd-socket-proxyd" +RDEPENDS:${PN} += "bash" DEPENDS += "rsync-native" # Ensure rsync-native is built before unpack runs do_unpack[depends] += "rsync-native:do_populate_sysroot" -DSTACK_SERVICES = "dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service dstack-socket.socket dstack-socket.service tappd-socket.socket tappd-socket.service" +DSTACK_SERVICES = "dstack-guest-agent.socket dstack-guest-agent.service dstack-prepare.service app-compose.service wg-checker.service" SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}','',d)}" SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${DSTACK_SERVICES}','',d)}" SYSTEMD_AUTO_ENABLE:${PN} = "enable" @@ -71,10 +71,7 @@ do_install() { install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-socket.socket ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-socket.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/tappd-socket.socket ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/tappd-socket.service ${D}${systemd_system_unitdir} + install -m 0644 ${S}/basefiles/dstack-guest-agent.socket ${D}${systemd_system_unitdir} install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ From 5a368c8eeee8b55fd725365e4f443ed04b6fa42b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Feb 2026 05:44:04 +0000 Subject: [PATCH 308/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index c92c2b416..fcbdf9b6a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit c92c2b41647a1e7cb2569f0644bcfb1c8b3dc6f4 +Subproject commit fcbdf9b6abe32decef0fcebfea2f1bfd99087973 From e50408efb0fabc2ada79c6c445f8866a8a4e61b8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 4 Feb 2026 08:10:16 +0000 Subject: [PATCH 309/435] Faster kernel download url --- bb-build/conf/local.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bb-build/conf/local.conf b/bb-build/conf/local.conf index b87c10a2a..4d565128a 100644 --- a/bb-build/conf/local.conf +++ b/bb-build/conf/local.conf @@ -286,3 +286,10 @@ BB_DISKMON_DIRS ??= "\ # track the version of this file when it was generated. This can safely be ignored if # this doesn't mean anything to you. CONF_VERSION = "2" + +# Prefer kernel.org yocto source mirror to avoid slow downloads.yoctoproject.org paths +INHERIT += "own-mirrors" +SOURCE_MIRROR_URL = "https://mirrors.kernel.org/yocto-sources/" + +# Improve resilience for large files on unstable links +FETCHCMD_wget = "wget --progress=dot --inet4-only -c" From 0db86d15099dcb9116f1fd41fba1a39c3a0b5aaf Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 5 Feb 2026 02:03:09 +0000 Subject: [PATCH 310/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index a046ae985..17c23e2b6 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit a046ae985b1971d6751e3a823f31e5dd545263de +Subproject commit 17c23e2b6bfa3ffcdd929b7aad9095930b906d32 From b12b5c6b54ace0dc99724030f0059e0b21e3fe2c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 2 Feb 2026 06:12:25 +0000 Subject: [PATCH 311/435] feat(guest): install conntrack sysctl config into CVM --- meta-dstack/recipes-core/dstack-guest/dstack-guest.bb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index be680ff9d..2996bcaea 100644 --- a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -61,6 +61,9 @@ do_install() { install -d ${D}${sysconfdir}/ install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf + install -d ${D}${sysconfdir}/sysctl.d + install -m 0644 ${S}/basefiles/sysctl.d/99-dstack.conf ${D}${sysconfdir}/sysctl.d/99-dstack.conf + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir} \ ${D}${sysconfdir}/systemd/resolved.conf.d From 71d9054e11cce42da1a14a0feab82a4b7e30edc5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 7 Feb 2026 17:01:41 +0000 Subject: [PATCH 312/435] feat(guest): add sysbox container runtime v0.6.7 Add Nestybox Sysbox as an alternative container runtime, enabling rootless containers to run workloads like Systemd, Docker, and Kubernetes inside containers. Recipe builds sysbox-runc, sysbox-fs, and sysbox-mgr from source with reproducible builds verified (identical binaries across clean builds). Key design decisions: - Network access only in do_configure (go mod vendor); do_compile is fully offline with -mod=vendor - Pre-generated protobuf .pb.go files to avoid protoc build dependency - Orphaned bazil/fuse commit forked to Dstack-TEE/fuse with named branch - Kernel configs: CONFIG_USER_NS=y, CONFIG_CONFIGFS_FS=m - Docker daemon configured with sysbox-runc runtime --- .../dstack-sysbox/dstack-sysbox_0.6.7.bb | 172 + .../dstack-sysbox/files/50-sysbox-mod.conf | 1 + .../dstack-sysbox/files/99-sysbox-sysctl.conf | 7 + .../dstack-sysbox/files/sysbox-fs.service | 18 + .../dstack-sysbox/files/sysbox-mgr.service | 17 + .../dstack-sysbox/files/sysbox.service | 13 + .../files/sysboxFsProtobuf.pb.go | 703 ++++ .../files/sysboxMgrProtobuf.pb.go | 3203 +++++++++++++++++ .../images/dstack-rootfs-base.inc | 1 + .../images/files/docker-daemon-nvidia.json | 3 + .../images/files/docker-daemon.json | 5 + .../linux/files/dstack-sysbox.cfg | 2 + .../linux/files/dstack-sysbox.scc | 3 + .../linux/linux-yocto%.bbappend | 3 + 14 files changed, 4151 insertions(+) create mode 100644 meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go create mode 100644 meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg create mode 100644 meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc diff --git a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb new file mode 100644 index 000000000..13d19d6e0 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb @@ -0,0 +1,172 @@ +SUMMARY = "Sysbox container runtime" +DESCRIPTION = "An open-source, next-generation runc that empowers rootless containers \ +to run workloads such as Systemd, Docker, Kubernetes, just like VMs." +HOMEPAGE = "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/nestybox/sysbox" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=cf0915b5e4f1337cf5b929ba1e388c42" + +SYSBOX_VERSION = "0.6.7" + +# Pin all submodule revisions from the v0.6.7 tag for reproducibility. +SRCREV_sysbox = "3a69811f54f8f83264ebb36dcaf51708e80b9e84" +SRCREV_sysbox-runc = "c58eba1be027c762c495bc4eeba7c0984beda1ab" +SRCREV_sysbox-fs = "6a8d71f54e7570e5297af89ff24ed3bafa61659f" +SRCREV_sysbox-mgr = "aaeff6c5dc70c137e62166474a309ca5fc42d044" +SRCREV_sysbox-ipc = "f05151f4b4c1df63d7fd241577ca032905c1bd0e" +SRCREV_sysbox-libs = "6faf00c74d45f7bdff0d1930fe8590c17a2d0a8b" +# bazil/fuse commit 45cd9a3 is orphaned in nestybox/fuse (not on any branch/tag). +# We forked it to Dstack-TEE/fuse with a named branch so BitBake can fetch it. +SRCREV_sysbox-fuse = "45cd9a3d884448418546d8eaa54ee7d772e576d6" + +SRCREV_FORMAT = "sysbox" + +SRC_URI = " \ + git://github.com/nestybox/sysbox.git;nobranch=1;name=sysbox;protocol=https;destsuffix=sysbox \ + git://github.com/nestybox/sysbox-runc.git;nobranch=1;name=sysbox-runc;protocol=https;destsuffix=sysbox-runc \ + git://github.com/nestybox/sysbox-fs.git;nobranch=1;name=sysbox-fs;protocol=https;destsuffix=sysbox-fs \ + git://github.com/nestybox/sysbox-mgr.git;nobranch=1;name=sysbox-mgr;protocol=https;destsuffix=sysbox-mgr \ + git://github.com/nestybox/sysbox-ipc.git;nobranch=1;name=sysbox-ipc;protocol=https;destsuffix=sysbox-ipc \ + git://github.com/nestybox/sysbox-libs.git;nobranch=1;name=sysbox-libs;protocol=https;destsuffix=sysbox-libs \ + git://github.com/Dstack-TEE/fuse.git;branch=sysbox-v0.6.7;name=sysbox-fuse;protocol=https;destsuffix=sysbox-fuse \ + file://sysbox.service \ + file://sysbox-fs.service \ + file://sysbox-mgr.service \ + file://99-sysbox-sysctl.conf \ + file://50-sysbox-mod.conf \ + file://sysboxFsProtobuf.pb.go \ + file://sysboxMgrProtobuf.pb.go \ +" + +S = "${WORKDIR}/sysbox" + +PV = "${SYSBOX_VERSION}+git${SRCPV}" + +DEPENDS += "libseccomp" +RDEPENDS:${PN} += "libseccomp" + +inherit go goarch pkgconfig systemd + +GO_IMPORT = "github.com/nestybox/sysbox" + +SYSBOX_LDFLAGS = " \ + -X 'main.edition=Community Edition (CE)' \ + -X main.version=${SYSBOX_VERSION} \ + -X main.commitId=${SRCREV_sysbox} \ + -X 'main.builtAt=1970-01-01T00:00:00Z' \ + -X 'main.builtBy=dstack' \ +" + +# Kernel >= 5.12 supports idmapped mounts +SYSBOX_RUNC_BUILDTAGS = "seccomp idmapped_mnt" +SYSBOX_MGR_BUILDTAGS = "idmapped_mnt" + +do_configure() { + # Arrange the source tree so that go.mod replace directives work. + # All components expect to find siblings in ../ relative to themselves. + # The git fetcher places them in ${WORKDIR}/sysbox-{runc,fs,mgr,ipc,libs}. + # This is already the correct layout since they are all at the same level + # under ${WORKDIR}. + + # sysbox-fs expects a 'bazil' subdirectory (submodule of nestybox/fuse). + # Remove the empty submodule placeholder left by git checkout, then symlink. + rm -rf ${WORKDIR}/sysbox-fs/bazil + ln -sfn ${WORKDIR}/sysbox-fuse ${WORKDIR}/sysbox-fs/bazil + + # Install pre-generated protobuf Go files. The upstream repo only ships + # .proto files and expects protoc + protoc-gen-go at build time. We + # pre-generate them to avoid the protoc native toolchain dependency. + install -m 0644 ${WORKDIR}/sysboxFsProtobuf.pb.go \ + ${WORKDIR}/sysbox-ipc/sysboxFsGrpc/sysboxFsProtobuf/ + install -m 0644 ${WORKDIR}/sysboxMgrProtobuf.pb.go \ + ${WORKDIR}/sysbox-ipc/sysboxMgrGrpc/sysboxMgrProtobuf/ + + # Vendor dependencies for each component so that do_compile needs no + # network access. go.sum in each repo guarantees content integrity. + # Use -modcacherw so cached modules are writable (BitBake needs to + # clean ${B}/pkg/mod between tasks). + for mod in sysbox-runc sysbox-fs sysbox-mgr; do + cd ${WORKDIR}/$mod + ${GO} mod vendor -modcacherw + done +} + +do_configure[network] = "1" + +do_compile() { + export CGO_ENABLED="1" + export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" + export CGO_LDFLAGS="${LDFLAGS} -Wl,--build-id=none --sysroot=${STAGING_DIR_TARGET}" + export CFLAGS="" + export LDFLAGS="" + + # Set reproducible build environment + export SOURCE_DATE_EPOCH=0 + export TZ=UTC + + # Build sysbox-runc + cd ${WORKDIR}/sysbox-runc + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -tags "${SYSBOX_RUNC_BUILDTAGS}" \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-runc-bin . + + # Build sysbox-fs + cd ${WORKDIR}/sysbox-fs + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-fs-bin ./cmd/sysbox-fs + + # Build sysbox-mgr + cd ${WORKDIR}/sysbox-mgr + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -tags "${SYSBOX_MGR_BUILDTAGS}" \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-mgr-bin . +} + +do_install() { + # Install binaries + install -d ${D}${bindir} + install -m 0755 ${WORKDIR}/sysbox-runc-bin ${D}${bindir}/sysbox-runc + install -m 0755 ${WORKDIR}/sysbox-fs-bin ${D}${bindir}/sysbox-fs + install -m 0755 ${WORKDIR}/sysbox-mgr-bin ${D}${bindir}/sysbox-mgr + + # Install systemd services + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/sysbox.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/sysbox-fs.service ${D}${systemd_system_unitdir} + install -m 0644 ${WORKDIR}/sysbox-mgr.service ${D}${systemd_system_unitdir} + fi + + # Install sysctl config + install -d ${D}${sysconfdir}/sysctl.d + install -m 0644 ${WORKDIR}/99-sysbox-sysctl.conf ${D}${sysconfdir}/sysctl.d/ + + # Install module autoload config + install -d ${D}${sysconfdir}/modules-load.d + install -m 0644 ${WORKDIR}/50-sysbox-mod.conf ${D}${sysconfdir}/modules-load.d/ + + # Create sysbox data directory + install -d ${D}/var/lib/sysbox +} + +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES', 'systemd', '${PN}', '', d)}" +SYSTEMD_SERVICE:${PN} = "sysbox.service sysbox-fs.service sysbox-mgr.service" +SYSTEMD_AUTO_ENABLE:${PN} = "enable" + +FILES:${PN} += " \ + ${bindir}/sysbox-runc \ + ${bindir}/sysbox-fs \ + ${bindir}/sysbox-mgr \ + ${systemd_system_unitdir}/sysbox.service \ + ${systemd_system_unitdir}/sysbox-fs.service \ + ${systemd_system_unitdir}/sysbox-mgr.service \ + ${sysconfdir}/sysctl.d/99-sysbox-sysctl.conf \ + ${sysconfdir}/modules-load.d/50-sysbox-mod.conf \ + /var/lib/sysbox \ +" + +INSANE_SKIP:${PN} += "ldflags already-stripped" + +COMPATIBLE_HOST = "x86_64.*-linux" diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf b/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf new file mode 100644 index 000000000..214e86b00 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf @@ -0,0 +1 @@ +configfs diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf new file mode 100644 index 000000000..96c18eeaa --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf @@ -0,0 +1,7 @@ +kernel.unprivileged_userns_clone = 1 +fs.inotify.max_queued_events = 1048576 +fs.inotify.max_user_watches = 1048576 +fs.inotify.max_user_instances = 1048576 +kernel.keys.maxkeys = 20000 +kernel.keys.maxbytes = 1400000 +kernel.pid_max = 4194304 diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service new file mode 100644 index 000000000..3ce213b2b --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service @@ -0,0 +1,18 @@ +[Unit] +Description=sysbox-fs (part of the Sysbox container runtime) +PartOf=sysbox.service +After=sysbox-mgr.service + +[Service] +Type=notify +ExecStart=/usr/bin/sysbox-fs --log /var/log/sysbox-fs.log +TimeoutStartSec=10 +TimeoutStopSec=10 +StartLimitInterval=0 +NotifyAccess=main +OOMScoreAdjust=-500 +LimitNOFILE=infinity +LimitNPROC=infinity + +[Install] +WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service new file mode 100644 index 000000000..11dbd1262 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -0,0 +1,17 @@ +[Unit] +Description=sysbox-mgr (part of the Sysbox container runtime) +PartOf=sysbox.service + +[Service] +Type=notify +ExecStart=/usr/bin/sysbox-mgr --log /var/log/sysbox-mgr.log +TimeoutStartSec=45 +TimeoutStopSec=90 +StartLimitInterval=0 +NotifyAccess=main +OOMScoreAdjust=-500 +LimitNOFILE=infinity +LimitNPROC=infinity + +[Install] +WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service new file mode 100644 index 000000000..55e15a5c3 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service @@ -0,0 +1,13 @@ +[Unit] +Description=Sysbox container runtime +Documentation=https://github.com/nestybox/sysbox +BindsTo=sysbox-mgr.service sysbox-fs.service +After=sysbox-mgr.service sysbox-fs.service +Before=docker.service containerd.service + +[Service] +Type=exec +ExecStart=/bin/sh -c "/usr/bin/sysbox-runc --version && /usr/bin/sysbox-mgr --version && /usr/bin/sysbox-fs --version && /bin/sleep infinity" + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go new file mode 100644 index 000000000..d3b0a668f --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go @@ -0,0 +1,703 @@ +// +// Sysbox-fs Protobuffer Definitions. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: sysboxFsProtobuf.proto + +package sysboxFsProtobuf + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Request message sent by runC to sysbox-fs process during container +// registration, unregistration and update phases. +type ContainerData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` + InitPid int32 `protobuf:"varint,2,opt,name=InitPid,proto3" json:"InitPid,omitempty"` + Hostname string `protobuf:"bytes,3,opt,name=Hostname,proto3" json:"Hostname,omitempty"` + Ctime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=Ctime,proto3" json:"Ctime,omitempty"` + UidFirst int32 `protobuf:"varint,5,opt,name=UidFirst,proto3" json:"UidFirst,omitempty"` + UidSize int32 `protobuf:"varint,6,opt,name=UidSize,proto3" json:"UidSize,omitempty"` + GidFirst int32 `protobuf:"varint,7,opt,name=GidFirst,proto3" json:"GidFirst,omitempty"` + GidSize int32 `protobuf:"varint,8,opt,name=GidSize,proto3" json:"GidSize,omitempty"` + ProcRoPaths []string `protobuf:"bytes,9,rep,name=ProcRoPaths,proto3" json:"ProcRoPaths,omitempty"` + ProcMaskPaths []string `protobuf:"bytes,10,rep,name=ProcMaskPaths,proto3" json:"ProcMaskPaths,omitempty"` + Netns string `protobuf:"bytes,11,opt,name=Netns,proto3" json:"Netns,omitempty"` +} + +func (x *ContainerData) Reset() { + *x = ContainerData{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerData) ProtoMessage() {} + +func (x *ContainerData) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerData.ProtoReflect.Descriptor instead. +func (*ContainerData) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{0} +} + +func (x *ContainerData) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ContainerData) GetInitPid() int32 { + if x != nil { + return x.InitPid + } + return 0 +} + +func (x *ContainerData) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +func (x *ContainerData) GetCtime() *timestamppb.Timestamp { + if x != nil { + return x.Ctime + } + return nil +} + +func (x *ContainerData) GetUidFirst() int32 { + if x != nil { + return x.UidFirst + } + return 0 +} + +func (x *ContainerData) GetUidSize() int32 { + if x != nil { + return x.UidSize + } + return 0 +} + +func (x *ContainerData) GetGidFirst() int32 { + if x != nil { + return x.GidFirst + } + return 0 +} + +func (x *ContainerData) GetGidSize() int32 { + if x != nil { + return x.GidSize + } + return 0 +} + +func (x *ContainerData) GetProcRoPaths() []string { + if x != nil { + return x.ProcRoPaths + } + return nil +} + +func (x *ContainerData) GetProcMaskPaths() []string { + if x != nil { + return x.ProcMaskPaths + } + return nil +} + +func (x *ContainerData) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +// Response message sent from sysbox-fs to runC process. +type Response struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` +} + +func (x *Response) Reset() { + *x = Response{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Response) ProtoMessage() {} + +func (x *Response) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Response.ProtoReflect.Descriptor instead. +func (*Response) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{1} +} + +func (x *Response) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +type MountpointReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *MountpointReq) Reset() { + *x = MountpointReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountpointReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountpointReq) ProtoMessage() {} + +func (x *MountpointReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountpointReq.ProtoReflect.Descriptor instead. +func (*MountpointReq) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{2} +} + +type MountpointResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Mountpoint string `protobuf:"bytes,1,opt,name=Mountpoint,proto3" json:"Mountpoint,omitempty"` +} + +func (x *MountpointResp) Reset() { + *x = MountpointResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountpointResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountpointResp) ProtoMessage() {} + +func (x *MountpointResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountpointResp.ProtoReflect.Descriptor instead. +func (*MountpointResp) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{3} +} + +func (x *MountpointResp) GetMountpoint() string { + if x != nil { + return x.Mountpoint + } + return "" +} + +var File_sysboxFsProtobuf_proto protoreflect.FileDescriptor + +var file_sysboxFsProtobuf_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x46, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0xd1, 0x02, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x50, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x50, 0x69, 0x64, 0x12, + 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x43, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x43, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1a, 0x0a, + 0x08, 0x55, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x08, 0x55, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x55, 0x69, 0x64, + 0x53, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x55, 0x69, 0x64, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x47, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x47, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x47, 0x69, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x07, 0x47, 0x69, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x50, 0x72, 0x6f, + 0x63, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, + 0x50, 0x72, 0x6f, 0x63, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x50, + 0x72, 0x6f, 0x63, 0x4d, 0x61, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0a, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0d, 0x50, 0x72, 0x6f, 0x63, 0x4d, 0x61, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0x24, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x0f, 0x0a, + 0x0d, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x22, 0x30, + 0x0a, 0x0e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x32, 0xf9, 0x02, 0x0a, 0x12, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x44, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4d, 0x6f, + 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x49, 0x0a, + 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x65, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x55, 0x6e, 0x72, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0f, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x15, 0x5a, 0x13, + 0x2e, 0x2f, 0x3b, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x46, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_sysboxFsProtobuf_proto_rawDescOnce sync.Once + file_sysboxFsProtobuf_proto_rawDescData = file_sysboxFsProtobuf_proto_rawDesc +) + +func file_sysboxFsProtobuf_proto_rawDescGZIP() []byte { + file_sysboxFsProtobuf_proto_rawDescOnce.Do(func() { + file_sysboxFsProtobuf_proto_rawDescData = protoimpl.X.CompressGZIP(file_sysboxFsProtobuf_proto_rawDescData) + }) + return file_sysboxFsProtobuf_proto_rawDescData +} + +var file_sysboxFsProtobuf_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sysboxFsProtobuf_proto_goTypes = []interface{}{ + (*ContainerData)(nil), // 0: protobuf.ContainerData + (*Response)(nil), // 1: protobuf.Response + (*MountpointReq)(nil), // 2: protobuf.MountpointReq + (*MountpointResp)(nil), // 3: protobuf.MountpointResp + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp +} +var file_sysboxFsProtobuf_proto_depIdxs = []int32{ + 4, // 0: protobuf.ContainerData.Ctime:type_name -> google.protobuf.Timestamp + 2, // 1: protobuf.sysboxStateChannel.GetMountpoint:input_type -> protobuf.MountpointReq + 0, // 2: protobuf.sysboxStateChannel.ContainerPreRegistration:input_type -> protobuf.ContainerData + 0, // 3: protobuf.sysboxStateChannel.ContainerRegistration:input_type -> protobuf.ContainerData + 0, // 4: protobuf.sysboxStateChannel.ContainerUnregistration:input_type -> protobuf.ContainerData + 0, // 5: protobuf.sysboxStateChannel.ContainerUpdate:input_type -> protobuf.ContainerData + 3, // 6: protobuf.sysboxStateChannel.GetMountpoint:output_type -> protobuf.MountpointResp + 1, // 7: protobuf.sysboxStateChannel.ContainerPreRegistration:output_type -> protobuf.Response + 1, // 8: protobuf.sysboxStateChannel.ContainerRegistration:output_type -> protobuf.Response + 1, // 9: protobuf.sysboxStateChannel.ContainerUnregistration:output_type -> protobuf.Response + 1, // 10: protobuf.sysboxStateChannel.ContainerUpdate:output_type -> protobuf.Response + 6, // [6:11] is the sub-list for method output_type + 1, // [1:6] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_sysboxFsProtobuf_proto_init() } +func file_sysboxFsProtobuf_proto_init() { + if File_sysboxFsProtobuf_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_sysboxFsProtobuf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Response); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountpointReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountpointResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_sysboxFsProtobuf_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sysboxFsProtobuf_proto_goTypes, + DependencyIndexes: file_sysboxFsProtobuf_proto_depIdxs, + MessageInfos: file_sysboxFsProtobuf_proto_msgTypes, + }.Build() + File_sysboxFsProtobuf_proto = out.File + file_sysboxFsProtobuf_proto_rawDesc = nil + file_sysboxFsProtobuf_proto_goTypes = nil + file_sysboxFsProtobuf_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// SysboxStateChannelClient is the client API for SysboxStateChannel service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SysboxStateChannelClient interface { + // Queries sysbox-fs for the FUSE mountpoint + GetMountpoint(ctx context.Context, in *MountpointReq, opts ...grpc.CallOption) (*MountpointResp, error) + // Generates a container-preregistration message + ContainerPreRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-registration message + ContainerRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-unregistration message + ContainerUnregistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-update message + ContainerUpdate(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) +} + +type sysboxStateChannelClient struct { + cc grpc.ClientConnInterface +} + +func NewSysboxStateChannelClient(cc grpc.ClientConnInterface) SysboxStateChannelClient { + return &sysboxStateChannelClient{cc} +} + +func (c *sysboxStateChannelClient) GetMountpoint(ctx context.Context, in *MountpointReq, opts ...grpc.CallOption) (*MountpointResp, error) { + out := new(MountpointResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/GetMountpoint", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerPreRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerPreRegistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerRegistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerUnregistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerUnregistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerUpdate(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerUpdate", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SysboxStateChannelServer is the server API for SysboxStateChannel service. +type SysboxStateChannelServer interface { + // Queries sysbox-fs for the FUSE mountpoint + GetMountpoint(context.Context, *MountpointReq) (*MountpointResp, error) + // Generates a container-preregistration message + ContainerPreRegistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-registration message + ContainerRegistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-unregistration message + ContainerUnregistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-update message + ContainerUpdate(context.Context, *ContainerData) (*Response, error) +} + +// UnimplementedSysboxStateChannelServer can be embedded to have forward compatible implementations. +type UnimplementedSysboxStateChannelServer struct { +} + +func (*UnimplementedSysboxStateChannelServer) GetMountpoint(context.Context, *MountpointReq) (*MountpointResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMountpoint not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerPreRegistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerPreRegistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerRegistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerRegistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerUnregistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerUnregistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerUpdate(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerUpdate not implemented") +} + +func RegisterSysboxStateChannelServer(s *grpc.Server, srv SysboxStateChannelServer) { + s.RegisterService(&_SysboxStateChannel_serviceDesc, srv) +} + +func _SysboxStateChannel_GetMountpoint_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountpointReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).GetMountpoint(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/GetMountpoint", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).GetMountpoint(ctx, req.(*MountpointReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerPreRegistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerPreRegistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerPreRegistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerPreRegistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerRegistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerRegistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerRegistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerRegistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerUnregistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerUnregistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerUnregistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerUnregistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerUpdate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerUpdate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerUpdate", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerUpdate(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +var _SysboxStateChannel_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protobuf.sysboxStateChannel", + HandlerType: (*SysboxStateChannelServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetMountpoint", + Handler: _SysboxStateChannel_GetMountpoint_Handler, + }, + { + MethodName: "ContainerPreRegistration", + Handler: _SysboxStateChannel_ContainerPreRegistration_Handler, + }, + { + MethodName: "ContainerRegistration", + Handler: _SysboxStateChannel_ContainerRegistration_Handler, + }, + { + MethodName: "ContainerUnregistration", + Handler: _SysboxStateChannel_ContainerUnregistration_Handler, + }, + { + MethodName: "ContainerUpdate", + Handler: _SysboxStateChannel_ContainerUpdate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "sysboxFsProtobuf.proto", +} diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go new file mode 100644 index 000000000..1c0364d2f --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go @@ -0,0 +1,3203 @@ +// +// SysboxMgr Protobuffer Definitions. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: sysboxMgrProtobuf.proto + +package sysboxMgrProtobuf + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type IDMapping struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerID uint32 `protobuf:"varint,1,opt,name=containerID,proto3" json:"containerID,omitempty"` + HostID uint32 `protobuf:"varint,2,opt,name=hostID,proto3" json:"hostID,omitempty"` + Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *IDMapping) Reset() { + *x = IDMapping{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IDMapping) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IDMapping) ProtoMessage() {} + +func (x *IDMapping) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IDMapping.ProtoReflect.Descriptor instead. +func (*IDMapping) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{0} +} + +func (x *IDMapping) GetContainerID() uint32 { + if x != nil { + return x.ContainerID + } + return 0 +} + +func (x *IDMapping) GetHostID() uint32 { + if x != nil { + return x.HostID + } + return 0 +} + +func (x *IDMapping) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +type RegisterReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Rootfs string `protobuf:"bytes,2,opt,name=rootfs,proto3" json:"rootfs,omitempty"` + Userns string `protobuf:"bytes,3,opt,name=userns,proto3" json:"userns,omitempty"` + Netns string `protobuf:"bytes,4,opt,name=netns,proto3" json:"netns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,5,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,6,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` +} + +func (x *RegisterReq) Reset() { + *x = RegisterReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RegisterReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterReq) ProtoMessage() {} + +func (x *RegisterReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterReq.ProtoReflect.Descriptor instead. +func (*RegisterReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{1} +} + +func (x *RegisterReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *RegisterReq) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +func (x *RegisterReq) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *RegisterReq) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +func (x *RegisterReq) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *RegisterReq) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +type ContainerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + AliasDns bool `protobuf:"varint,1,opt,name=aliasDns,proto3" json:"aliasDns,omitempty"` + ShiftfsOk bool `protobuf:"varint,2,opt,name=ShiftfsOk,proto3" json:"ShiftfsOk,omitempty"` + ShiftfsOnOverlayfsOk bool `protobuf:"varint,3,opt,name=ShiftfsOnOverlayfsOk,proto3" json:"ShiftfsOnOverlayfsOk,omitempty"` + IDMapMountOk bool `protobuf:"varint,4,opt,name=IDMapMountOk,proto3" json:"IDMapMountOk,omitempty"` + OverlayfsOnIDMapMountOk bool `protobuf:"varint,5,opt,name=OverlayfsOnIDMapMountOk,proto3" json:"OverlayfsOnIDMapMountOk,omitempty"` + NoRootfsCloning bool `protobuf:"varint,6,opt,name=noRootfsCloning,proto3" json:"noRootfsCloning,omitempty"` + IgnoreSysfsChown bool `protobuf:"varint,7,opt,name=ignoreSysfsChown,proto3" json:"ignoreSysfsChown,omitempty"` + AllowTrustedXattr bool `protobuf:"varint,8,opt,name=allowTrustedXattr,proto3" json:"allowTrustedXattr,omitempty"` + HonorCaps bool `protobuf:"varint,9,opt,name=honorCaps,proto3" json:"honorCaps,omitempty"` + SyscontMode bool `protobuf:"varint,10,opt,name=syscontMode,proto3" json:"syscontMode,omitempty"` + Userns string `protobuf:"bytes,11,opt,name=userns,proto3" json:"userns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,12,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,13,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` + FsuidMapFailOnErr bool `protobuf:"varint,14,opt,name=fsuidMapFailOnErr,proto3" json:"fsuidMapFailOnErr,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,15,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` + NoShiftfsOnFuse bool `protobuf:"varint,16,opt,name=noShiftfsOnFuse,proto3" json:"noShiftfsOnFuse,omitempty"` + RelaxedReadOnly bool `protobuf:"varint,17,opt,name=relaxedReadOnly,proto3" json:"relaxedReadOnly,omitempty"` +} + +func (x *ContainerConfig) Reset() { + *x = ContainerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerConfig) ProtoMessage() {} + +func (x *ContainerConfig) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerConfig.ProtoReflect.Descriptor instead. +func (*ContainerConfig) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{2} +} + +func (x *ContainerConfig) GetAliasDns() bool { + if x != nil { + return x.AliasDns + } + return false +} + +func (x *ContainerConfig) GetShiftfsOk() bool { + if x != nil { + return x.ShiftfsOk + } + return false +} + +func (x *ContainerConfig) GetShiftfsOnOverlayfsOk() bool { + if x != nil { + return x.ShiftfsOnOverlayfsOk + } + return false +} + +func (x *ContainerConfig) GetIDMapMountOk() bool { + if x != nil { + return x.IDMapMountOk + } + return false +} + +func (x *ContainerConfig) GetOverlayfsOnIDMapMountOk() bool { + if x != nil { + return x.OverlayfsOnIDMapMountOk + } + return false +} + +func (x *ContainerConfig) GetNoRootfsCloning() bool { + if x != nil { + return x.NoRootfsCloning + } + return false +} + +func (x *ContainerConfig) GetIgnoreSysfsChown() bool { + if x != nil { + return x.IgnoreSysfsChown + } + return false +} + +func (x *ContainerConfig) GetAllowTrustedXattr() bool { + if x != nil { + return x.AllowTrustedXattr + } + return false +} + +func (x *ContainerConfig) GetHonorCaps() bool { + if x != nil { + return x.HonorCaps + } + return false +} + +func (x *ContainerConfig) GetSyscontMode() bool { + if x != nil { + return x.SyscontMode + } + return false +} + +func (x *ContainerConfig) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *ContainerConfig) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *ContainerConfig) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +func (x *ContainerConfig) GetFsuidMapFailOnErr() bool { + if x != nil { + return x.FsuidMapFailOnErr + } + return false +} + +func (x *ContainerConfig) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +func (x *ContainerConfig) GetNoShiftfsOnFuse() bool { + if x != nil { + return x.NoShiftfsOnFuse + } + return false +} + +func (x *ContainerConfig) GetRelaxedReadOnly() bool { + if x != nil { + return x.RelaxedReadOnly + } + return false +} + +type RegisterResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerConfig *ContainerConfig `protobuf:"bytes,1,opt,name=containerConfig,proto3" json:"containerConfig,omitempty"` +} + +func (x *RegisterResp) Reset() { + *x = RegisterResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RegisterResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterResp) ProtoMessage() {} + +func (x *RegisterResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterResp.ProtoReflect.Descriptor instead. +func (*RegisterResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{3} +} + +func (x *RegisterResp) GetContainerConfig() *ContainerConfig { + if x != nil { + return x.ContainerConfig + } + return nil +} + +type UpdateReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Userns string `protobuf:"bytes,2,opt,name=userns,proto3" json:"userns,omitempty"` + Netns string `protobuf:"bytes,3,opt,name=netns,proto3" json:"netns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,4,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,5,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,6,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` +} + +func (x *UpdateReq) Reset() { + *x = UpdateReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateReq) ProtoMessage() {} + +func (x *UpdateReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateReq.ProtoReflect.Descriptor instead. +func (*UpdateReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{4} +} + +func (x *UpdateReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *UpdateReq) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *UpdateReq) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +func (x *UpdateReq) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *UpdateReq) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +func (x *UpdateReq) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +type UpdateResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *UpdateResp) Reset() { + *x = UpdateResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateResp) ProtoMessage() {} + +func (x *UpdateResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateResp.ProtoReflect.Descriptor instead. +func (*UpdateResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{5} +} + +type UnregisterReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *UnregisterReq) Reset() { + *x = UnregisterReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UnregisterReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UnregisterReq) ProtoMessage() {} + +func (x *UnregisterReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UnregisterReq.ProtoReflect.Descriptor instead. +func (*UnregisterReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{6} +} + +func (x *UnregisterReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type UnregisterResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *UnregisterResp) Reset() { + *x = UnregisterResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UnregisterResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UnregisterResp) ProtoMessage() {} + +func (x *UnregisterResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UnregisterResp.ProtoReflect.Descriptor instead. +func (*UnregisterResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{7} +} + +type SubidAllocReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Size uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *SubidAllocReq) Reset() { + *x = SubidAllocReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SubidAllocReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubidAllocReq) ProtoMessage() {} + +func (x *SubidAllocReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubidAllocReq.ProtoReflect.Descriptor instead. +func (*SubidAllocReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{8} +} + +func (x *SubidAllocReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SubidAllocReq) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +type SubidAllocResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Uid uint32 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,2,opt,name=gid,proto3" json:"gid,omitempty"` +} + +func (x *SubidAllocResp) Reset() { + *x = SubidAllocResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SubidAllocResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubidAllocResp) ProtoMessage() {} + +func (x *SubidAllocResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubidAllocResp.ProtoReflect.Descriptor instead. +func (*SubidAllocResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{9} +} + +func (x *SubidAllocResp) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *SubidAllocResp) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +type MountPrepInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Exclusive bool `protobuf:"varint,2,opt,name=exclusive,proto3" json:"exclusive,omitempty"` +} + +func (x *MountPrepInfo) Reset() { + *x = MountPrepInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepInfo) ProtoMessage() {} + +func (x *MountPrepInfo) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepInfo.ProtoReflect.Descriptor instead. +func (*MountPrepInfo) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{10} +} + +func (x *MountPrepInfo) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *MountPrepInfo) GetExclusive() bool { + if x != nil { + return x.Exclusive + } + return false +} + +type MountPrepReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Uid uint32 `protobuf:"varint,2,opt,name=uid,proto3" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,3,opt,name=gid,proto3" json:"gid,omitempty"` + PrepList []*MountPrepInfo `protobuf:"bytes,4,rep,name=prepList,proto3" json:"prepList,omitempty"` +} + +func (x *MountPrepReq) Reset() { + *x = MountPrepReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepReq) ProtoMessage() {} + +func (x *MountPrepReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepReq.ProtoReflect.Descriptor instead. +func (*MountPrepReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{11} +} + +func (x *MountPrepReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *MountPrepReq) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *MountPrepReq) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +func (x *MountPrepReq) GetPrepList() []*MountPrepInfo { + if x != nil { + return x.PrepList + } + return nil +} + +type MountPrepResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *MountPrepResp) Reset() { + *x = MountPrepResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepResp) ProtoMessage() {} + +func (x *MountPrepResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepResp.ProtoReflect.Descriptor instead. +func (*MountPrepResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{12} +} + +type MountReqInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind uint32 `protobuf:"varint,1,opt,name=kind,proto3" json:"kind,omitempty"` + Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` +} + +func (x *MountReqInfo) Reset() { + *x = MountReqInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountReqInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountReqInfo) ProtoMessage() {} + +func (x *MountReqInfo) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountReqInfo.ProtoReflect.Descriptor instead. +func (*MountReqInfo) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{13} +} + +func (x *MountReqInfo) GetKind() uint32 { + if x != nil { + return x.Kind + } + return 0 +} + +func (x *MountReqInfo) GetDest() string { + if x != nil { + return x.Dest + } + return "" +} + +type MountReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,2,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` + ReqList []*MountReqInfo `protobuf:"bytes,3,rep,name=reqList,proto3" json:"reqList,omitempty"` +} + +func (x *MountReq) Reset() { + *x = MountReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountReq) ProtoMessage() {} + +func (x *MountReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountReq.ProtoReflect.Descriptor instead. +func (*MountReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{14} +} + +func (x *MountReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *MountReq) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +func (x *MountReq) GetReqList() []*MountReqInfo { + if x != nil { + return x.ReqList + } + return nil +} + +type Mount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` + Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` + Opt []string `protobuf:"bytes,4,rep,name=opt,proto3" json:"opt,omitempty"` +} + +func (x *Mount) Reset() { + *x = Mount{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Mount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mount) ProtoMessage() {} + +func (x *Mount) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mount.ProtoReflect.Descriptor instead. +func (*Mount) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{15} +} + +func (x *Mount) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *Mount) GetDest() string { + if x != nil { + return x.Dest + } + return "" +} + +func (x *Mount) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *Mount) GetOpt() []string { + if x != nil { + return x.Opt + } + return nil +} + +type MountResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Mounts []*Mount `protobuf:"bytes,1,rep,name=mounts,proto3" json:"mounts,omitempty"` +} + +func (x *MountResp) Reset() { + *x = MountResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountResp) ProtoMessage() {} + +func (x *MountResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountResp.ProtoReflect.Descriptor instead. +func (*MountResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{16} +} + +func (x *MountResp) GetMounts() []*Mount { + if x != nil { + return x.Mounts + } + return nil +} + +type ShiftfsMark struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Readonly bool `protobuf:"varint,2,opt,name=readonly,proto3" json:"readonly,omitempty"` +} + +func (x *ShiftfsMark) Reset() { + *x = ShiftfsMark{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMark) ProtoMessage() {} + +func (x *ShiftfsMark) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMark.ProtoReflect.Descriptor instead. +func (*ShiftfsMark) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{17} +} + +func (x *ShiftfsMark) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *ShiftfsMark) GetReadonly() bool { + if x != nil { + return x.Readonly + } + return false +} + +type ShiftfsMarkReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ShiftfsMarks []*ShiftfsMark `protobuf:"bytes,2,rep,name=shiftfsMarks,proto3" json:"shiftfsMarks,omitempty"` +} + +func (x *ShiftfsMarkReq) Reset() { + *x = ShiftfsMarkReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMarkReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMarkReq) ProtoMessage() {} + +func (x *ShiftfsMarkReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMarkReq.ProtoReflect.Descriptor instead. +func (*ShiftfsMarkReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{18} +} + +func (x *ShiftfsMarkReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ShiftfsMarkReq) GetShiftfsMarks() []*ShiftfsMark { + if x != nil { + return x.ShiftfsMarks + } + return nil +} + +type ShiftfsMarkResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ShiftfsMarks []*ShiftfsMark `protobuf:"bytes,1,rep,name=shiftfsMarks,proto3" json:"shiftfsMarks,omitempty"` +} + +func (x *ShiftfsMarkResp) Reset() { + *x = ShiftfsMarkResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMarkResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMarkResp) ProtoMessage() {} + +func (x *ShiftfsMarkResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMarkResp.ProtoReflect.Descriptor instead. +func (*ShiftfsMarkResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{19} +} + +func (x *ShiftfsMarkResp) GetShiftfsMarks() []*ShiftfsMark { + if x != nil { + return x.ShiftfsMarks + } + return nil +} + +type FsStateReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Rootfs string `protobuf:"bytes,2,opt,name=rootfs,proto3" json:"rootfs,omitempty"` +} + +func (x *FsStateReq) Reset() { + *x = FsStateReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsStateReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsStateReq) ProtoMessage() {} + +func (x *FsStateReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsStateReq.ProtoReflect.Descriptor instead. +func (*FsStateReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{20} +} + +func (x *FsStateReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *FsStateReq) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +type FsEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind uint32 `protobuf:"varint,1,opt,name=kind,proto3" json:"kind,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + Dst string `protobuf:"bytes,4,opt,name=dst,proto3" json:"dst,omitempty"` +} + +func (x *FsEntry) Reset() { + *x = FsEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsEntry) ProtoMessage() {} + +func (x *FsEntry) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsEntry.ProtoReflect.Descriptor instead. +func (*FsEntry) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{21} +} + +func (x *FsEntry) GetKind() uint32 { + if x != nil { + return x.Kind + } + return 0 +} + +func (x *FsEntry) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *FsEntry) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *FsEntry) GetDst() string { + if x != nil { + return x.Dst + } + return "" +} + +type FsStateResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FsEntries []*FsEntry `protobuf:"bytes,1,rep,name=fsEntries,proto3" json:"fsEntries,omitempty"` +} + +func (x *FsStateResp) Reset() { + *x = FsStateResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsStateResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsStateResp) ProtoMessage() {} + +func (x *FsStateResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsStateResp.ProtoReflect.Descriptor instead. +func (*FsStateResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{22} +} + +func (x *FsStateResp) GetFsEntries() []*FsEntry { + if x != nil { + return x.FsEntries + } + return nil +} + +type PauseReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *PauseReq) Reset() { + *x = PauseReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PauseReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PauseReq) ProtoMessage() {} + +func (x *PauseReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PauseReq.ProtoReflect.Descriptor instead. +func (*PauseReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{23} +} + +func (x *PauseReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type PauseResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *PauseResp) Reset() { + *x = PauseResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PauseResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PauseResp) ProtoMessage() {} + +func (x *PauseResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PauseResp.ProtoReflect.Descriptor instead. +func (*PauseResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{24} +} + +type ResumeReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ResumeReq) Reset() { + *x = ResumeReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResumeReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeReq) ProtoMessage() {} + +func (x *ResumeReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeReq.ProtoReflect.Descriptor instead. +func (*ResumeReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{25} +} + +func (x *ResumeReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ResumeResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ResumeResp) Reset() { + *x = ResumeResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResumeResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeResp) ProtoMessage() {} + +func (x *ResumeResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeResp.ProtoReflect.Descriptor instead. +func (*ResumeResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{26} +} + +type CloneRootfsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *CloneRootfsReq) Reset() { + *x = CloneRootfsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CloneRootfsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloneRootfsReq) ProtoMessage() {} + +func (x *CloneRootfsReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloneRootfsReq.ProtoReflect.Descriptor instead. +func (*CloneRootfsReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{27} +} + +func (x *CloneRootfsReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type CloneRootfsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rootfs string `protobuf:"bytes,1,opt,name=rootfs,proto3" json:"rootfs,omitempty"` +} + +func (x *CloneRootfsResp) Reset() { + *x = CloneRootfsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CloneRootfsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloneRootfsResp) ProtoMessage() {} + +func (x *CloneRootfsResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloneRootfsResp.ProtoReflect.Descriptor instead. +func (*CloneRootfsResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{28} +} + +func (x *CloneRootfsResp) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +type ChownClonedRootfsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + UidOffset int32 `protobuf:"varint,2,opt,name=uidOffset,proto3" json:"uidOffset,omitempty"` + GidOffset int32 `protobuf:"varint,3,opt,name=gidOffset,proto3" json:"gidOffset,omitempty"` +} + +func (x *ChownClonedRootfsReq) Reset() { + *x = ChownClonedRootfsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChownClonedRootfsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChownClonedRootfsReq) ProtoMessage() {} + +func (x *ChownClonedRootfsReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChownClonedRootfsReq.ProtoReflect.Descriptor instead. +func (*ChownClonedRootfsReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{29} +} + +func (x *ChownClonedRootfsReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ChownClonedRootfsReq) GetUidOffset() int32 { + if x != nil { + return x.UidOffset + } + return 0 +} + +func (x *ChownClonedRootfsReq) GetGidOffset() int32 { + if x != nil { + return x.GidOffset + } + return 0 +} + +type ChownClonedRootfsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ChownClonedRootfsResp) Reset() { + *x = ChownClonedRootfsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChownClonedRootfsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChownClonedRootfsResp) ProtoMessage() {} + +func (x *ChownClonedRootfsResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChownClonedRootfsResp.ProtoReflect.Descriptor instead. +func (*ChownClonedRootfsResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{30} +} + +type RevertClonedRootfsChownReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *RevertClonedRootfsChownReq) Reset() { + *x = RevertClonedRootfsChownReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RevertClonedRootfsChownReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertClonedRootfsChownReq) ProtoMessage() {} + +func (x *RevertClonedRootfsChownReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertClonedRootfsChownReq.ProtoReflect.Descriptor instead. +func (*RevertClonedRootfsChownReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{31} +} + +func (x *RevertClonedRootfsChownReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type RevertClonedRootfsChownResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RevertClonedRootfsChownResp) Reset() { + *x = RevertClonedRootfsChownResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RevertClonedRootfsChownResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertClonedRootfsChownResp) ProtoMessage() {} + +func (x *RevertClonedRootfsChownResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertClonedRootfsChownResp.ProtoReflect.Descriptor instead. +func (*RevertClonedRootfsChownResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{32} +} + +var File_sysboxMgrProtobuf_proto protoreflect.FileDescriptor + +var file_sysboxMgrProtobuf_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x4d, 0x67, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x22, 0x59, 0x0a, 0x09, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x6f, 0x73, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x06, 0x68, 0x6f, 0x73, 0x74, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0xd1, + 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, + 0x65, 0x74, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, + 0x6e, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, + 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, + 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x73, 0x22, 0xd9, 0x05, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x44, + 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x44, + 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6b, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6b, + 0x12, 0x32, 0x0a, 0x14, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x4f, 0x76, 0x65, + 0x72, 0x6c, 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, + 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x79, + 0x66, 0x73, 0x4f, 0x6b, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x4f, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x49, 0x44, 0x4d, 0x61, + 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x4f, 0x6b, 0x12, 0x38, 0x0a, 0x17, 0x4f, 0x76, 0x65, 0x72, + 0x6c, 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, 0x6e, + 0x74, 0x4f, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x4f, 0x76, 0x65, 0x72, 0x6c, + 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x4f, 0x6b, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x6f, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x6c, + 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6e, 0x6f, 0x52, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x6c, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x2a, 0x0a, 0x10, + 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x79, 0x73, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x79, + 0x73, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, + 0x77, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x58, 0x61, 0x74, 0x74, 0x72, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, + 0x64, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x68, 0x6f, 0x6e, 0x6f, 0x72, 0x43, + 0x61, 0x70, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x68, 0x6f, 0x6e, 0x6f, 0x72, + 0x43, 0x61, 0x70, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x79, 0x73, 0x63, 0x6f, 0x6e, 0x74, 0x4d, + 0x6f, 0x64, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x79, 0x73, 0x63, 0x6f, + 0x6e, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x35, + 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0c, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, + 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, + 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x11, + 0x66, 0x73, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x46, 0x61, 0x69, 0x6c, 0x4f, 0x6e, 0x45, 0x72, + 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x66, 0x73, 0x75, 0x69, 0x64, 0x4d, 0x61, + 0x70, 0x46, 0x61, 0x69, 0x6c, 0x4f, 0x6e, 0x45, 0x72, 0x72, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x6f, + 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, + 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x6f, + 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x46, 0x75, 0x73, 0x65, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6e, 0x6f, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, + 0x46, 0x75, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x65, 0x6c, 0x61, 0x78, 0x65, 0x64, 0x52, + 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, + 0x65, 0x6c, 0x61, 0x78, 0x65, 0x64, 0x52, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x53, + 0x0a, 0x0c, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x43, + 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x22, 0xe7, 0x01, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, + 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x6e, 0x73, 0x12, + 0x35, 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x52, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2e, 0x0a, + 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x0c, 0x0a, + 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x1f, 0x0a, 0x0d, 0x55, + 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x10, 0x0a, 0x0e, + 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x33, + 0x0a, 0x0d, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, 0x71, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x22, 0x34, 0x0a, 0x0e, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, + 0x63, 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x45, 0x0a, 0x0d, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, + 0x22, 0x77, 0x0a, 0x0c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x71, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x75, + 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x03, 0x67, 0x69, 0x64, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x70, 0x4c, 0x69, 0x73, 0x74, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x08, 0x70, 0x72, 0x65, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x0a, 0x0d, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x36, 0x0a, 0x0c, 0x4d, 0x6f, + 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, + 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x64, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x65, + 0x73, 0x74, 0x22, 0x7c, 0x0a, 0x08, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2e, + 0x0a, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x30, + 0x0a, 0x07, 0x72, 0x65, 0x71, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x72, 0x65, 0x71, 0x4c, 0x69, 0x73, 0x74, + 0x22, 0x59, 0x0a, 0x05, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x64, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x70, 0x74, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x70, 0x74, 0x22, 0x34, 0x0a, 0x09, 0x4d, + 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x27, 0x0a, 0x06, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x73, 0x22, 0x41, 0x0a, 0x0b, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x61, 0x64, + 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, + 0x6f, 0x6e, 0x6c, 0x79, 0x22, 0x5b, 0x0a, 0x0e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, + 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x52, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x73, 0x22, 0x4c, 0x0a, 0x0f, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x12, 0x39, 0x0a, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, + 0x6b, 0x52, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x73, 0x22, + 0x34, 0x0a, 0x0a, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x22, 0x57, 0x0a, 0x07, 0x46, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x64, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x73, 0x74, 0x22, 0x3e, + 0x0a, 0x0b, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2f, 0x0a, + 0x09, 0x66, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x09, 0x66, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x1a, + 0x0a, 0x08, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x0b, 0x0a, 0x09, 0x50, 0x61, + 0x75, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x1b, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x0c, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x20, 0x0a, 0x0e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x29, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, + 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x22, + 0x62, 0x0a, 0x14, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, + 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x69, 0x64, 0x4f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x75, 0x69, 0x64, 0x4f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x67, 0x69, 0x64, 0x4f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x67, 0x69, 0x64, 0x4f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x22, 0x17, 0x0a, 0x15, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, + 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x2c, 0x0a, 0x1a, + 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, + 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, + 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x32, 0x86, 0x07, 0x0a, 0x15, 0x73, 0x79, + 0x73, 0x62, 0x6f, 0x78, 0x4d, 0x67, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x12, 0x3b, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x1a, + 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0a, 0x55, 0x6e, 0x72, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x18, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0a, 0x53, 0x75, + 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, + 0x71, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x75, 0x62, + 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, + 0x0a, 0x50, 0x72, 0x65, 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, + 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, + 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x36, + 0x0a, 0x09, 0x52, 0x65, 0x71, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x1a, + 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x53, 0x68, 0x69, + 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x52, + 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, + 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, + 0x3b, 0x0a, 0x0a, 0x52, 0x65, 0x71, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, + 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x05, + 0x50, 0x61, 0x75, 0x73, 0x65, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x1a, + 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x43, 0x6c, + 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, + 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, + 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x56, 0x0a, 0x11, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x17, 0x52, 0x65, 0x76, 0x65, + 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x68, + 0x6f, 0x77, 0x6e, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, + 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, + 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x42, 0x16, 0x5a, 0x14, 0x2e, 0x2f, 0x3b, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x4d, + 0x67, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_sysboxMgrProtobuf_proto_rawDescOnce sync.Once + file_sysboxMgrProtobuf_proto_rawDescData = file_sysboxMgrProtobuf_proto_rawDesc +) + +func file_sysboxMgrProtobuf_proto_rawDescGZIP() []byte { + file_sysboxMgrProtobuf_proto_rawDescOnce.Do(func() { + file_sysboxMgrProtobuf_proto_rawDescData = protoimpl.X.CompressGZIP(file_sysboxMgrProtobuf_proto_rawDescData) + }) + return file_sysboxMgrProtobuf_proto_rawDescData +} + +var file_sysboxMgrProtobuf_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_sysboxMgrProtobuf_proto_goTypes = []interface{}{ + (*IDMapping)(nil), // 0: protobuf.IDMapping + (*RegisterReq)(nil), // 1: protobuf.RegisterReq + (*ContainerConfig)(nil), // 2: protobuf.ContainerConfig + (*RegisterResp)(nil), // 3: protobuf.RegisterResp + (*UpdateReq)(nil), // 4: protobuf.UpdateReq + (*UpdateResp)(nil), // 5: protobuf.UpdateResp + (*UnregisterReq)(nil), // 6: protobuf.UnregisterReq + (*UnregisterResp)(nil), // 7: protobuf.UnregisterResp + (*SubidAllocReq)(nil), // 8: protobuf.SubidAllocReq + (*SubidAllocResp)(nil), // 9: protobuf.SubidAllocResp + (*MountPrepInfo)(nil), // 10: protobuf.MountPrepInfo + (*MountPrepReq)(nil), // 11: protobuf.MountPrepReq + (*MountPrepResp)(nil), // 12: protobuf.MountPrepResp + (*MountReqInfo)(nil), // 13: protobuf.MountReqInfo + (*MountReq)(nil), // 14: protobuf.MountReq + (*Mount)(nil), // 15: protobuf.Mount + (*MountResp)(nil), // 16: protobuf.MountResp + (*ShiftfsMark)(nil), // 17: protobuf.ShiftfsMark + (*ShiftfsMarkReq)(nil), // 18: protobuf.ShiftfsMarkReq + (*ShiftfsMarkResp)(nil), // 19: protobuf.ShiftfsMarkResp + (*FsStateReq)(nil), // 20: protobuf.FsStateReq + (*FsEntry)(nil), // 21: protobuf.FsEntry + (*FsStateResp)(nil), // 22: protobuf.FsStateResp + (*PauseReq)(nil), // 23: protobuf.PauseReq + (*PauseResp)(nil), // 24: protobuf.PauseResp + (*ResumeReq)(nil), // 25: protobuf.ResumeReq + (*ResumeResp)(nil), // 26: protobuf.ResumeResp + (*CloneRootfsReq)(nil), // 27: protobuf.CloneRootfsReq + (*CloneRootfsResp)(nil), // 28: protobuf.CloneRootfsResp + (*ChownClonedRootfsReq)(nil), // 29: protobuf.ChownClonedRootfsReq + (*ChownClonedRootfsResp)(nil), // 30: protobuf.ChownClonedRootfsResp + (*RevertClonedRootfsChownReq)(nil), // 31: protobuf.RevertClonedRootfsChownReq + (*RevertClonedRootfsChownResp)(nil), // 32: protobuf.RevertClonedRootfsChownResp +} +var file_sysboxMgrProtobuf_proto_depIdxs = []int32{ + 0, // 0: protobuf.RegisterReq.uidMappings:type_name -> protobuf.IDMapping + 0, // 1: protobuf.RegisterReq.gidMappings:type_name -> protobuf.IDMapping + 0, // 2: protobuf.ContainerConfig.uidMappings:type_name -> protobuf.IDMapping + 0, // 3: protobuf.ContainerConfig.gidMappings:type_name -> protobuf.IDMapping + 2, // 4: protobuf.RegisterResp.containerConfig:type_name -> protobuf.ContainerConfig + 0, // 5: protobuf.UpdateReq.uidMappings:type_name -> protobuf.IDMapping + 0, // 6: protobuf.UpdateReq.gidMappings:type_name -> protobuf.IDMapping + 10, // 7: protobuf.MountPrepReq.prepList:type_name -> protobuf.MountPrepInfo + 13, // 8: protobuf.MountReq.reqList:type_name -> protobuf.MountReqInfo + 15, // 9: protobuf.MountResp.mounts:type_name -> protobuf.Mount + 17, // 10: protobuf.ShiftfsMarkReq.shiftfsMarks:type_name -> protobuf.ShiftfsMark + 17, // 11: protobuf.ShiftfsMarkResp.shiftfsMarks:type_name -> protobuf.ShiftfsMark + 21, // 12: protobuf.FsStateResp.fsEntries:type_name -> protobuf.FsEntry + 1, // 13: protobuf.sysboxMgrStateChannel.Register:input_type -> protobuf.RegisterReq + 4, // 14: protobuf.sysboxMgrStateChannel.Update:input_type -> protobuf.UpdateReq + 6, // 15: protobuf.sysboxMgrStateChannel.Unregister:input_type -> protobuf.UnregisterReq + 8, // 16: protobuf.sysboxMgrStateChannel.SubidAlloc:input_type -> protobuf.SubidAllocReq + 11, // 17: protobuf.sysboxMgrStateChannel.PrepMounts:input_type -> protobuf.MountPrepReq + 14, // 18: protobuf.sysboxMgrStateChannel.ReqMounts:input_type -> protobuf.MountReq + 18, // 19: protobuf.sysboxMgrStateChannel.ReqShiftfsMark:input_type -> protobuf.ShiftfsMarkReq + 20, // 20: protobuf.sysboxMgrStateChannel.ReqFsState:input_type -> protobuf.FsStateReq + 23, // 21: protobuf.sysboxMgrStateChannel.Pause:input_type -> protobuf.PauseReq + 25, // 22: protobuf.sysboxMgrStateChannel.Resume:input_type -> protobuf.ResumeReq + 27, // 23: protobuf.sysboxMgrStateChannel.ReqCloneRootfs:input_type -> protobuf.CloneRootfsReq + 29, // 24: protobuf.sysboxMgrStateChannel.ChownClonedRootfs:input_type -> protobuf.ChownClonedRootfsReq + 31, // 25: protobuf.sysboxMgrStateChannel.RevertClonedRootfsChown:input_type -> protobuf.RevertClonedRootfsChownReq + 3, // 26: protobuf.sysboxMgrStateChannel.Register:output_type -> protobuf.RegisterResp + 5, // 27: protobuf.sysboxMgrStateChannel.Update:output_type -> protobuf.UpdateResp + 7, // 28: protobuf.sysboxMgrStateChannel.Unregister:output_type -> protobuf.UnregisterResp + 9, // 29: protobuf.sysboxMgrStateChannel.SubidAlloc:output_type -> protobuf.SubidAllocResp + 12, // 30: protobuf.sysboxMgrStateChannel.PrepMounts:output_type -> protobuf.MountPrepResp + 16, // 31: protobuf.sysboxMgrStateChannel.ReqMounts:output_type -> protobuf.MountResp + 19, // 32: protobuf.sysboxMgrStateChannel.ReqShiftfsMark:output_type -> protobuf.ShiftfsMarkResp + 22, // 33: protobuf.sysboxMgrStateChannel.ReqFsState:output_type -> protobuf.FsStateResp + 24, // 34: protobuf.sysboxMgrStateChannel.Pause:output_type -> protobuf.PauseResp + 26, // 35: protobuf.sysboxMgrStateChannel.Resume:output_type -> protobuf.ResumeResp + 28, // 36: protobuf.sysboxMgrStateChannel.ReqCloneRootfs:output_type -> protobuf.CloneRootfsResp + 30, // 37: protobuf.sysboxMgrStateChannel.ChownClonedRootfs:output_type -> protobuf.ChownClonedRootfsResp + 32, // 38: protobuf.sysboxMgrStateChannel.RevertClonedRootfsChown:output_type -> protobuf.RevertClonedRootfsChownResp + 26, // [26:39] is the sub-list for method output_type + 13, // [13:26] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_sysboxMgrProtobuf_proto_init() } +func file_sysboxMgrProtobuf_proto_init() { + if File_sysboxMgrProtobuf_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_sysboxMgrProtobuf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IDMapping); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UnregisterReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UnregisterResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubidAllocReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubidAllocResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountReqInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Mount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMark); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMarkReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMarkResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsStateReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsStateResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PauseReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PauseResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResumeReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResumeResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CloneRootfsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CloneRootfsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChownClonedRootfsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChownClonedRootfsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RevertClonedRootfsChownReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RevertClonedRootfsChownResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_sysboxMgrProtobuf_proto_rawDesc, + NumEnums: 0, + NumMessages: 33, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sysboxMgrProtobuf_proto_goTypes, + DependencyIndexes: file_sysboxMgrProtobuf_proto_depIdxs, + MessageInfos: file_sysboxMgrProtobuf_proto_msgTypes, + }.Build() + File_sysboxMgrProtobuf_proto = out.File + file_sysboxMgrProtobuf_proto_rawDesc = nil + file_sysboxMgrProtobuf_proto_goTypes = nil + file_sysboxMgrProtobuf_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// SysboxMgrStateChannelClient is the client API for SysboxMgrStateChannel service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SysboxMgrStateChannelClient interface { + // Container registration + Register(ctx context.Context, in *RegisterReq, opts ...grpc.CallOption) (*RegisterResp, error) + // Container Update + Update(ctx context.Context, in *UpdateReq, opts ...grpc.CallOption) (*UpdateResp, error) + // Container Unregistration + Unregister(ctx context.Context, in *UnregisterReq, opts ...grpc.CallOption) (*UnregisterResp, error) + // Subuid(gid) allocation request + SubidAlloc(ctx context.Context, in *SubidAllocReq, opts ...grpc.CallOption) (*SubidAllocResp, error) + // Mount source prep request + PrepMounts(ctx context.Context, in *MountPrepReq, opts ...grpc.CallOption) (*MountPrepResp, error) + // Mount request + ReqMounts(ctx context.Context, in *MountReq, opts ...grpc.CallOption) (*MountResp, error) + // Shiftfs mark request + ReqShiftfsMark(ctx context.Context, in *ShiftfsMarkReq, opts ...grpc.CallOption) (*ShiftfsMarkResp, error) + // FsState request + ReqFsState(ctx context.Context, in *FsStateReq, opts ...grpc.CallOption) (*FsStateResp, error) + // Pause request + Pause(ctx context.Context, in *PauseReq, opts ...grpc.CallOption) (*PauseResp, error) + // Resume request + Resume(ctx context.Context, in *ResumeReq, opts ...grpc.CallOption) (*ResumeResp, error) + // Clone rootfs request + ReqCloneRootfs(ctx context.Context, in *CloneRootfsReq, opts ...grpc.CallOption) (*CloneRootfsResp, error) + // Chown cloned rootfs request + ChownClonedRootfs(ctx context.Context, in *ChownClonedRootfsReq, opts ...grpc.CallOption) (*ChownClonedRootfsResp, error) + // Revert cloned rootfs chown + RevertClonedRootfsChown(ctx context.Context, in *RevertClonedRootfsChownReq, opts ...grpc.CallOption) (*RevertClonedRootfsChownResp, error) +} + +type sysboxMgrStateChannelClient struct { + cc grpc.ClientConnInterface +} + +func NewSysboxMgrStateChannelClient(cc grpc.ClientConnInterface) SysboxMgrStateChannelClient { + return &sysboxMgrStateChannelClient{cc} +} + +func (c *sysboxMgrStateChannelClient) Register(ctx context.Context, in *RegisterReq, opts ...grpc.CallOption) (*RegisterResp, error) { + out := new(RegisterResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Update(ctx context.Context, in *UpdateReq, opts ...grpc.CallOption) (*UpdateResp, error) { + out := new(UpdateResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Update", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Unregister(ctx context.Context, in *UnregisterReq, opts ...grpc.CallOption) (*UnregisterResp, error) { + out := new(UnregisterResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Unregister", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) SubidAlloc(ctx context.Context, in *SubidAllocReq, opts ...grpc.CallOption) (*SubidAllocResp, error) { + out := new(SubidAllocResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/SubidAlloc", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) PrepMounts(ctx context.Context, in *MountPrepReq, opts ...grpc.CallOption) (*MountPrepResp, error) { + out := new(MountPrepResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/PrepMounts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqMounts(ctx context.Context, in *MountReq, opts ...grpc.CallOption) (*MountResp, error) { + out := new(MountResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqMounts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqShiftfsMark(ctx context.Context, in *ShiftfsMarkReq, opts ...grpc.CallOption) (*ShiftfsMarkResp, error) { + out := new(ShiftfsMarkResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqShiftfsMark", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqFsState(ctx context.Context, in *FsStateReq, opts ...grpc.CallOption) (*FsStateResp, error) { + out := new(FsStateResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqFsState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Pause(ctx context.Context, in *PauseReq, opts ...grpc.CallOption) (*PauseResp, error) { + out := new(PauseResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Pause", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Resume(ctx context.Context, in *ResumeReq, opts ...grpc.CallOption) (*ResumeResp, error) { + out := new(ResumeResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Resume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqCloneRootfs(ctx context.Context, in *CloneRootfsReq, opts ...grpc.CallOption) (*CloneRootfsResp, error) { + out := new(CloneRootfsResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqCloneRootfs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ChownClonedRootfs(ctx context.Context, in *ChownClonedRootfsReq, opts ...grpc.CallOption) (*ChownClonedRootfsResp, error) { + out := new(ChownClonedRootfsResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ChownClonedRootfs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) RevertClonedRootfsChown(ctx context.Context, in *RevertClonedRootfsChownReq, opts ...grpc.CallOption) (*RevertClonedRootfsChownResp, error) { + out := new(RevertClonedRootfsChownResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/RevertClonedRootfsChown", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SysboxMgrStateChannelServer is the server API for SysboxMgrStateChannel service. +type SysboxMgrStateChannelServer interface { + // Container registration + Register(context.Context, *RegisterReq) (*RegisterResp, error) + // Container Update + Update(context.Context, *UpdateReq) (*UpdateResp, error) + // Container Unregistration + Unregister(context.Context, *UnregisterReq) (*UnregisterResp, error) + // Subuid(gid) allocation request + SubidAlloc(context.Context, *SubidAllocReq) (*SubidAllocResp, error) + // Mount source prep request + PrepMounts(context.Context, *MountPrepReq) (*MountPrepResp, error) + // Mount request + ReqMounts(context.Context, *MountReq) (*MountResp, error) + // Shiftfs mark request + ReqShiftfsMark(context.Context, *ShiftfsMarkReq) (*ShiftfsMarkResp, error) + // FsState request + ReqFsState(context.Context, *FsStateReq) (*FsStateResp, error) + // Pause request + Pause(context.Context, *PauseReq) (*PauseResp, error) + // Resume request + Resume(context.Context, *ResumeReq) (*ResumeResp, error) + // Clone rootfs request + ReqCloneRootfs(context.Context, *CloneRootfsReq) (*CloneRootfsResp, error) + // Chown cloned rootfs request + ChownClonedRootfs(context.Context, *ChownClonedRootfsReq) (*ChownClonedRootfsResp, error) + // Revert cloned rootfs chown + RevertClonedRootfsChown(context.Context, *RevertClonedRootfsChownReq) (*RevertClonedRootfsChownResp, error) +} + +// UnimplementedSysboxMgrStateChannelServer can be embedded to have forward compatible implementations. +type UnimplementedSysboxMgrStateChannelServer struct { +} + +func (*UnimplementedSysboxMgrStateChannelServer) Register(context.Context, *RegisterReq) (*RegisterResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Update(context.Context, *UpdateReq) (*UpdateResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Update not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Unregister(context.Context, *UnregisterReq) (*UnregisterResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Unregister not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) SubidAlloc(context.Context, *SubidAllocReq) (*SubidAllocResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubidAlloc not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) PrepMounts(context.Context, *MountPrepReq) (*MountPrepResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method PrepMounts not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqMounts(context.Context, *MountReq) (*MountResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqMounts not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqShiftfsMark(context.Context, *ShiftfsMarkReq) (*ShiftfsMarkResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqShiftfsMark not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqFsState(context.Context, *FsStateReq) (*FsStateResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqFsState not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Pause(context.Context, *PauseReq) (*PauseResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Pause not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Resume(context.Context, *ResumeReq) (*ResumeResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Resume not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqCloneRootfs(context.Context, *CloneRootfsReq) (*CloneRootfsResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqCloneRootfs not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ChownClonedRootfs(context.Context, *ChownClonedRootfsReq) (*ChownClonedRootfsResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ChownClonedRootfs not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) RevertClonedRootfsChown(context.Context, *RevertClonedRootfsChownReq) (*RevertClonedRootfsChownResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method RevertClonedRootfsChown not implemented") +} + +func RegisterSysboxMgrStateChannelServer(s *grpc.Server, srv SysboxMgrStateChannelServer) { + s.RegisterService(&_SysboxMgrStateChannel_serviceDesc, srv) +} + +func _SysboxMgrStateChannel_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RegisterReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Register(ctx, req.(*RegisterReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Update", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Update(ctx, req.(*UpdateReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Unregister_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UnregisterReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Unregister(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Unregister", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Unregister(ctx, req.(*UnregisterReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_SubidAlloc_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubidAllocReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).SubidAlloc(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/SubidAlloc", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).SubidAlloc(ctx, req.(*SubidAllocReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_PrepMounts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountPrepReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).PrepMounts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/PrepMounts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).PrepMounts(ctx, req.(*MountPrepReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqMounts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqMounts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqMounts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqMounts(ctx, req.(*MountReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqShiftfsMark_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ShiftfsMarkReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqShiftfsMark(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqShiftfsMark", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqShiftfsMark(ctx, req.(*ShiftfsMarkReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqFsState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FsStateReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqFsState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqFsState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqFsState(ctx, req.(*FsStateReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Pause_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PauseReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Pause(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Pause", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Pause(ctx, req.(*PauseReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Resume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResumeReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Resume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Resume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Resume(ctx, req.(*ResumeReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqCloneRootfs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CloneRootfsReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqCloneRootfs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqCloneRootfs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqCloneRootfs(ctx, req.(*CloneRootfsReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ChownClonedRootfs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ChownClonedRootfsReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ChownClonedRootfs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ChownClonedRootfs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ChownClonedRootfs(ctx, req.(*ChownClonedRootfsReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_RevertClonedRootfsChown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevertClonedRootfsChownReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).RevertClonedRootfsChown(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/RevertClonedRootfsChown", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).RevertClonedRootfsChown(ctx, req.(*RevertClonedRootfsChownReq)) + } + return interceptor(ctx, in, info, handler) +} + +var _SysboxMgrStateChannel_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protobuf.sysboxMgrStateChannel", + HandlerType: (*SysboxMgrStateChannelServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _SysboxMgrStateChannel_Register_Handler, + }, + { + MethodName: "Update", + Handler: _SysboxMgrStateChannel_Update_Handler, + }, + { + MethodName: "Unregister", + Handler: _SysboxMgrStateChannel_Unregister_Handler, + }, + { + MethodName: "SubidAlloc", + Handler: _SysboxMgrStateChannel_SubidAlloc_Handler, + }, + { + MethodName: "PrepMounts", + Handler: _SysboxMgrStateChannel_PrepMounts_Handler, + }, + { + MethodName: "ReqMounts", + Handler: _SysboxMgrStateChannel_ReqMounts_Handler, + }, + { + MethodName: "ReqShiftfsMark", + Handler: _SysboxMgrStateChannel_ReqShiftfsMark_Handler, + }, + { + MethodName: "ReqFsState", + Handler: _SysboxMgrStateChannel_ReqFsState_Handler, + }, + { + MethodName: "Pause", + Handler: _SysboxMgrStateChannel_Pause_Handler, + }, + { + MethodName: "Resume", + Handler: _SysboxMgrStateChannel_Resume_Handler, + }, + { + MethodName: "ReqCloneRootfs", + Handler: _SysboxMgrStateChannel_ReqCloneRootfs_Handler, + }, + { + MethodName: "ChownClonedRootfs", + Handler: _SysboxMgrStateChannel_ChownClonedRootfs_Handler, + }, + { + MethodName: "RevertClonedRootfsChown", + Handler: _SysboxMgrStateChannel_RevertClonedRootfsChown_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "sysboxMgrProtobuf.proto", +} diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 2628535de..214249a24 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -21,6 +21,7 @@ IMAGE_INSTALL = "\ chrony \ chronyc \ dstack-zfs \ + dstack-sysbox \ kernel-module-tun \ kernel-module-fuse \ kernel-module-br-netfilter \ diff --git a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json index 7ed24bdfa..a235187f9 100644 --- a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json +++ b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json @@ -8,6 +8,9 @@ "nvidia": { "args": [], "path": "nvidia-container-runtime" + }, + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" } } } diff --git a/meta-dstack/recipes-core/images/files/docker-daemon.json b/meta-dstack/recipes-core/images/files/docker-daemon.json index 4d6c550d7..c4db4d44c 100644 --- a/meta-dstack/recipes-core/images/files/docker-daemon.json +++ b/meta-dstack/recipes-core/images/files/docker-daemon.json @@ -3,5 +3,10 @@ "log-opts": { "max-size": "100m", "max-file": "10" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } } } diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg new file mode 100644 index 000000000..5b4b8ab0d --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg @@ -0,0 +1,2 @@ +CONFIG_USER_NS=y +CONFIG_CONFIGFS_FS=y diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc new file mode 100644 index 000000000..3c3a8257d --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "DStack Sysbox runtime configuration" + +kconf non-hardware dstack-sysbox.cfg diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 22f4e1377..5900c497f 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -6,6 +6,8 @@ SRC_URI += "file://dstack-docker.cfg \ file://dstack-docker.scc \ file://dstack-tdx.cfg \ file://dstack-tdx.scc \ + file://dstack-sysbox.cfg \ + file://dstack-sysbox.scc \ file://dstack.cfg \ file://dstack.scc" @@ -16,6 +18,7 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/xfs/xfs.scc \ cfg/fs/squashfs.scc \ dstack-docker.scc \ + dstack-sysbox.scc \ dstack.scc" KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", " features/device-mapper/dm-verity.scc", "" ,d)}" From 5d6a81ff526164303fafc4a027d80b865d98df81 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 7 Feb 2026 17:08:51 +0000 Subject: [PATCH 313/435] feat(guest): add sysbox container runtime v0.6.7 Add Nestybox Sysbox as an alternative container runtime, enabling rootless containers to run workloads like Systemd, Docker, Kubernetes, just like VMs. Recipe builds sysbox-runc, sysbox-fs, and sysbox-mgr from source with reproducible builds verified (identical binaries across clean builds). Key design decisions: - Network access only in do_configure (go mod vendor); do_compile is fully offline with -mod=vendor - Pre-generated protobuf .pb.go files to avoid protoc build dependency - Orphaned bazil/fuse commit forked to Dstack-TEE/fuse with named branch - Kernel configs already present in linux-custom defconfig - Docker daemon configured with sysbox-runc runtime - Adapted for Yocto 5.3: UNPACKDIR for source paths, WORKDIR for build artifacts --- .../dstack-sysbox/dstack-sysbox_0.6.7.bb | 172 + .../dstack-sysbox/files/50-sysbox-mod.conf | 1 + .../dstack-sysbox/files/99-sysbox-sysctl.conf | 7 + .../dstack-sysbox/files/sysbox-fs.service | 18 + .../dstack-sysbox/files/sysbox-mgr.service | 17 + .../dstack-sysbox/files/sysbox.service | 13 + .../files/sysboxFsProtobuf.pb.go | 703 ++++ .../files/sysboxMgrProtobuf.pb.go | 3203 +++++++++++++++++ .../images/dstack-rootfs-base.inc | 1 + .../images/files/docker-daemon-nvidia.json | 3 + .../images/files/docker-daemon.json | 5 + .../linux/files/dstack-sysbox.cfg | 2 + .../linux/files/dstack-sysbox.scc | 3 + .../linux/linux-yocto%.bbappend | 3 + 14 files changed, 4151 insertions(+) create mode 100644 meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go create mode 100644 meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go create mode 100644 meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg create mode 100644 meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc diff --git a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb new file mode 100644 index 000000000..a8dddd061 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb @@ -0,0 +1,172 @@ +SUMMARY = "Sysbox container runtime" +DESCRIPTION = "An open-source, next-generation runc that empowers rootless containers \ +to run workloads such as Systemd, Docker, Kubernetes, just like VMs." +HOMEPAGE = "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/nestybox/sysbox" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=cf0915b5e4f1337cf5b929ba1e388c42" + +SYSBOX_VERSION = "0.6.7" + +# Pin all submodule revisions from the v0.6.7 tag for reproducibility. +SRCREV_sysbox = "3a69811f54f8f83264ebb36dcaf51708e80b9e84" +SRCREV_sysbox-runc = "c58eba1be027c762c495bc4eeba7c0984beda1ab" +SRCREV_sysbox-fs = "6a8d71f54e7570e5297af89ff24ed3bafa61659f" +SRCREV_sysbox-mgr = "aaeff6c5dc70c137e62166474a309ca5fc42d044" +SRCREV_sysbox-ipc = "f05151f4b4c1df63d7fd241577ca032905c1bd0e" +SRCREV_sysbox-libs = "6faf00c74d45f7bdff0d1930fe8590c17a2d0a8b" +# bazil/fuse commit 45cd9a3 is orphaned in nestybox/fuse (not on any branch/tag). +# We forked it to Dstack-TEE/fuse with a named branch so BitBake can fetch it. +SRCREV_sysbox-fuse = "45cd9a3d884448418546d8eaa54ee7d772e576d6" + +SRCREV_FORMAT = "sysbox" + +SRC_URI = " \ + git://github.com/nestybox/sysbox.git;nobranch=1;name=sysbox;protocol=https;destsuffix=sysbox \ + git://github.com/nestybox/sysbox-runc.git;nobranch=1;name=sysbox-runc;protocol=https;destsuffix=sysbox-runc \ + git://github.com/nestybox/sysbox-fs.git;nobranch=1;name=sysbox-fs;protocol=https;destsuffix=sysbox-fs \ + git://github.com/nestybox/sysbox-mgr.git;nobranch=1;name=sysbox-mgr;protocol=https;destsuffix=sysbox-mgr \ + git://github.com/nestybox/sysbox-ipc.git;nobranch=1;name=sysbox-ipc;protocol=https;destsuffix=sysbox-ipc \ + git://github.com/nestybox/sysbox-libs.git;nobranch=1;name=sysbox-libs;protocol=https;destsuffix=sysbox-libs \ + git://github.com/Dstack-TEE/fuse.git;branch=sysbox-v0.6.7;name=sysbox-fuse;protocol=https;destsuffix=sysbox-fuse \ + file://sysbox.service \ + file://sysbox-fs.service \ + file://sysbox-mgr.service \ + file://99-sysbox-sysctl.conf \ + file://50-sysbox-mod.conf \ + file://sysboxFsProtobuf.pb.go \ + file://sysboxMgrProtobuf.pb.go \ +" + +S = "${UNPACKDIR}/sysbox" + +PV = "${SYSBOX_VERSION}+git${SRCPV}" + +DEPENDS += "libseccomp" +RDEPENDS:${PN} += "libseccomp" + +inherit go goarch pkgconfig systemd + +GO_IMPORT = "github.com/nestybox/sysbox" + +SYSBOX_LDFLAGS = " \ + -X 'main.edition=Community Edition (CE)' \ + -X main.version=${SYSBOX_VERSION} \ + -X main.commitId=${SRCREV_sysbox} \ + -X 'main.builtAt=1970-01-01T00:00:00Z' \ + -X 'main.builtBy=dstack' \ +" + +# Kernel >= 5.12 supports idmapped mounts +SYSBOX_RUNC_BUILDTAGS = "seccomp idmapped_mnt" +SYSBOX_MGR_BUILDTAGS = "idmapped_mnt" + +do_configure() { + # Arrange the source tree so that go.mod replace directives work. + # All components expect to find siblings in ../ relative to themselves. + # The git fetcher places them in ${UNPACKDIR}/sysbox-{runc,fs,mgr,ipc,libs}. + # This is already the correct layout since they are all at the same level + # under ${UNPACKDIR}. + + # sysbox-fs expects a 'bazil' subdirectory (submodule of nestybox/fuse). + # Remove the empty submodule placeholder left by git checkout, then symlink. + rm -rf ${UNPACKDIR}/sysbox-fs/bazil + ln -sfn ${UNPACKDIR}/sysbox-fuse ${UNPACKDIR}/sysbox-fs/bazil + + # Install pre-generated protobuf Go files. The upstream repo only ships + # .proto files and expects protoc + protoc-gen-go at build time. We + # pre-generate them to avoid the protoc native toolchain dependency. + install -m 0644 ${UNPACKDIR}/sysboxFsProtobuf.pb.go \ + ${UNPACKDIR}/sysbox-ipc/sysboxFsGrpc/sysboxFsProtobuf/ + install -m 0644 ${UNPACKDIR}/sysboxMgrProtobuf.pb.go \ + ${UNPACKDIR}/sysbox-ipc/sysboxMgrGrpc/sysboxMgrProtobuf/ + + # Vendor dependencies for each component so that do_compile needs no + # network access. go.sum in each repo guarantees content integrity. + # Use -modcacherw so cached modules are writable (BitBake needs to + # clean ${B}/pkg/mod between tasks). + for mod in sysbox-runc sysbox-fs sysbox-mgr; do + cd ${UNPACKDIR}/$mod + ${GO} mod vendor -modcacherw + done +} + +do_configure[network] = "1" + +do_compile() { + export CGO_ENABLED="1" + export CGO_CFLAGS="${CFLAGS} --sysroot=${STAGING_DIR_TARGET}" + export CGO_LDFLAGS="${LDFLAGS} -Wl,--build-id=none --sysroot=${STAGING_DIR_TARGET}" + export CFLAGS="" + export LDFLAGS="" + + # Set reproducible build environment + export SOURCE_DATE_EPOCH=0 + export TZ=UTC + + # Build sysbox-runc + cd ${UNPACKDIR}/sysbox-runc + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -tags "${SYSBOX_RUNC_BUILDTAGS}" \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-runc-bin . + + # Build sysbox-fs + cd ${UNPACKDIR}/sysbox-fs + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-fs-bin ./cmd/sysbox-fs + + # Build sysbox-mgr + cd ${UNPACKDIR}/sysbox-mgr + ${GO} build -mod=vendor -buildvcs=false -trimpath \ + -tags "${SYSBOX_MGR_BUILDTAGS}" \ + -ldflags "-buildid= -s -w -linkmode external -extldflags '-Wl,--build-id=none' ${SYSBOX_LDFLAGS}" \ + -o ${WORKDIR}/sysbox-mgr-bin . +} + +do_install() { + # Install binaries + install -d ${D}${bindir} + install -m 0755 ${WORKDIR}/sysbox-runc-bin ${D}${bindir}/sysbox-runc + install -m 0755 ${WORKDIR}/sysbox-fs-bin ${D}${bindir}/sysbox-fs + install -m 0755 ${WORKDIR}/sysbox-mgr-bin ${D}${bindir}/sysbox-mgr + + # Install systemd services + if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then + install -d ${D}${systemd_system_unitdir} + install -m 0644 ${UNPACKDIR}/sysbox.service ${D}${systemd_system_unitdir} + install -m 0644 ${UNPACKDIR}/sysbox-fs.service ${D}${systemd_system_unitdir} + install -m 0644 ${UNPACKDIR}/sysbox-mgr.service ${D}${systemd_system_unitdir} + fi + + # Install sysctl config + install -d ${D}${sysconfdir}/sysctl.d + install -m 0644 ${UNPACKDIR}/99-sysbox-sysctl.conf ${D}${sysconfdir}/sysctl.d/ + + # Install module autoload config + install -d ${D}${sysconfdir}/modules-load.d + install -m 0644 ${UNPACKDIR}/50-sysbox-mod.conf ${D}${sysconfdir}/modules-load.d/ + + # Create sysbox data directory + install -d ${D}/var/lib/sysbox +} + +SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES', 'systemd', '${PN}', '', d)}" +SYSTEMD_SERVICE:${PN} = "sysbox.service sysbox-fs.service sysbox-mgr.service" +SYSTEMD_AUTO_ENABLE:${PN} = "enable" + +FILES:${PN} += " \ + ${bindir}/sysbox-runc \ + ${bindir}/sysbox-fs \ + ${bindir}/sysbox-mgr \ + ${systemd_system_unitdir}/sysbox.service \ + ${systemd_system_unitdir}/sysbox-fs.service \ + ${systemd_system_unitdir}/sysbox-mgr.service \ + ${sysconfdir}/sysctl.d/99-sysbox-sysctl.conf \ + ${sysconfdir}/modules-load.d/50-sysbox-mod.conf \ + /var/lib/sysbox \ +" + +INSANE_SKIP:${PN} += "ldflags already-stripped" + +COMPATIBLE_HOST = "x86_64.*-linux" diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf b/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf new file mode 100644 index 000000000..214e86b00 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf @@ -0,0 +1 @@ +configfs diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf new file mode 100644 index 000000000..96c18eeaa --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf @@ -0,0 +1,7 @@ +kernel.unprivileged_userns_clone = 1 +fs.inotify.max_queued_events = 1048576 +fs.inotify.max_user_watches = 1048576 +fs.inotify.max_user_instances = 1048576 +kernel.keys.maxkeys = 20000 +kernel.keys.maxbytes = 1400000 +kernel.pid_max = 4194304 diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service new file mode 100644 index 000000000..3ce213b2b --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service @@ -0,0 +1,18 @@ +[Unit] +Description=sysbox-fs (part of the Sysbox container runtime) +PartOf=sysbox.service +After=sysbox-mgr.service + +[Service] +Type=notify +ExecStart=/usr/bin/sysbox-fs --log /var/log/sysbox-fs.log +TimeoutStartSec=10 +TimeoutStopSec=10 +StartLimitInterval=0 +NotifyAccess=main +OOMScoreAdjust=-500 +LimitNOFILE=infinity +LimitNPROC=infinity + +[Install] +WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service new file mode 100644 index 000000000..11dbd1262 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -0,0 +1,17 @@ +[Unit] +Description=sysbox-mgr (part of the Sysbox container runtime) +PartOf=sysbox.service + +[Service] +Type=notify +ExecStart=/usr/bin/sysbox-mgr --log /var/log/sysbox-mgr.log +TimeoutStartSec=45 +TimeoutStopSec=90 +StartLimitInterval=0 +NotifyAccess=main +OOMScoreAdjust=-500 +LimitNOFILE=infinity +LimitNPROC=infinity + +[Install] +WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service new file mode 100644 index 000000000..55e15a5c3 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service @@ -0,0 +1,13 @@ +[Unit] +Description=Sysbox container runtime +Documentation=https://github.com/nestybox/sysbox +BindsTo=sysbox-mgr.service sysbox-fs.service +After=sysbox-mgr.service sysbox-fs.service +Before=docker.service containerd.service + +[Service] +Type=exec +ExecStart=/bin/sh -c "/usr/bin/sysbox-runc --version && /usr/bin/sysbox-mgr --version && /usr/bin/sysbox-fs --version && /bin/sleep infinity" + +[Install] +WantedBy=multi-user.target diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go new file mode 100644 index 000000000..d3b0a668f --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go @@ -0,0 +1,703 @@ +// +// Sysbox-fs Protobuffer Definitions. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: sysboxFsProtobuf.proto + +package sysboxFsProtobuf + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Request message sent by runC to sysbox-fs process during container +// registration, unregistration and update phases. +type ContainerData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` + InitPid int32 `protobuf:"varint,2,opt,name=InitPid,proto3" json:"InitPid,omitempty"` + Hostname string `protobuf:"bytes,3,opt,name=Hostname,proto3" json:"Hostname,omitempty"` + Ctime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=Ctime,proto3" json:"Ctime,omitempty"` + UidFirst int32 `protobuf:"varint,5,opt,name=UidFirst,proto3" json:"UidFirst,omitempty"` + UidSize int32 `protobuf:"varint,6,opt,name=UidSize,proto3" json:"UidSize,omitempty"` + GidFirst int32 `protobuf:"varint,7,opt,name=GidFirst,proto3" json:"GidFirst,omitempty"` + GidSize int32 `protobuf:"varint,8,opt,name=GidSize,proto3" json:"GidSize,omitempty"` + ProcRoPaths []string `protobuf:"bytes,9,rep,name=ProcRoPaths,proto3" json:"ProcRoPaths,omitempty"` + ProcMaskPaths []string `protobuf:"bytes,10,rep,name=ProcMaskPaths,proto3" json:"ProcMaskPaths,omitempty"` + Netns string `protobuf:"bytes,11,opt,name=Netns,proto3" json:"Netns,omitempty"` +} + +func (x *ContainerData) Reset() { + *x = ContainerData{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerData) ProtoMessage() {} + +func (x *ContainerData) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerData.ProtoReflect.Descriptor instead. +func (*ContainerData) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{0} +} + +func (x *ContainerData) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ContainerData) GetInitPid() int32 { + if x != nil { + return x.InitPid + } + return 0 +} + +func (x *ContainerData) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +func (x *ContainerData) GetCtime() *timestamppb.Timestamp { + if x != nil { + return x.Ctime + } + return nil +} + +func (x *ContainerData) GetUidFirst() int32 { + if x != nil { + return x.UidFirst + } + return 0 +} + +func (x *ContainerData) GetUidSize() int32 { + if x != nil { + return x.UidSize + } + return 0 +} + +func (x *ContainerData) GetGidFirst() int32 { + if x != nil { + return x.GidFirst + } + return 0 +} + +func (x *ContainerData) GetGidSize() int32 { + if x != nil { + return x.GidSize + } + return 0 +} + +func (x *ContainerData) GetProcRoPaths() []string { + if x != nil { + return x.ProcRoPaths + } + return nil +} + +func (x *ContainerData) GetProcMaskPaths() []string { + if x != nil { + return x.ProcMaskPaths + } + return nil +} + +func (x *ContainerData) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +// Response message sent from sysbox-fs to runC process. +type Response struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` +} + +func (x *Response) Reset() { + *x = Response{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Response) ProtoMessage() {} + +func (x *Response) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Response.ProtoReflect.Descriptor instead. +func (*Response) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{1} +} + +func (x *Response) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +type MountpointReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *MountpointReq) Reset() { + *x = MountpointReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountpointReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountpointReq) ProtoMessage() {} + +func (x *MountpointReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountpointReq.ProtoReflect.Descriptor instead. +func (*MountpointReq) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{2} +} + +type MountpointResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Mountpoint string `protobuf:"bytes,1,opt,name=Mountpoint,proto3" json:"Mountpoint,omitempty"` +} + +func (x *MountpointResp) Reset() { + *x = MountpointResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxFsProtobuf_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountpointResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountpointResp) ProtoMessage() {} + +func (x *MountpointResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxFsProtobuf_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountpointResp.ProtoReflect.Descriptor instead. +func (*MountpointResp) Descriptor() ([]byte, []int) { + return file_sysboxFsProtobuf_proto_rawDescGZIP(), []int{3} +} + +func (x *MountpointResp) GetMountpoint() string { + if x != nil { + return x.Mountpoint + } + return "" +} + +var File_sysboxFsProtobuf_proto protoreflect.FileDescriptor + +var file_sysboxFsProtobuf_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x46, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0xd1, 0x02, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x50, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x50, 0x69, 0x64, 0x12, + 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x43, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x43, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x1a, 0x0a, + 0x08, 0x55, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x08, 0x55, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x55, 0x69, 0x64, + 0x53, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x55, 0x69, 0x64, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x47, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x47, 0x69, 0x64, 0x46, 0x69, 0x72, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x47, 0x69, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x07, 0x47, 0x69, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x50, 0x72, 0x6f, + 0x63, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, + 0x50, 0x72, 0x6f, 0x63, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x50, + 0x72, 0x6f, 0x63, 0x4d, 0x61, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0a, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0d, 0x50, 0x72, 0x6f, 0x63, 0x4d, 0x61, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0x24, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x0f, 0x0a, + 0x0d, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x22, 0x30, + 0x0a, 0x0e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x32, 0xf9, 0x02, 0x0a, 0x12, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x44, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4d, 0x6f, + 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x49, 0x0a, + 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x65, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x48, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x55, 0x6e, 0x72, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0f, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x17, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x15, 0x5a, 0x13, + 0x2e, 0x2f, 0x3b, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x46, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_sysboxFsProtobuf_proto_rawDescOnce sync.Once + file_sysboxFsProtobuf_proto_rawDescData = file_sysboxFsProtobuf_proto_rawDesc +) + +func file_sysboxFsProtobuf_proto_rawDescGZIP() []byte { + file_sysboxFsProtobuf_proto_rawDescOnce.Do(func() { + file_sysboxFsProtobuf_proto_rawDescData = protoimpl.X.CompressGZIP(file_sysboxFsProtobuf_proto_rawDescData) + }) + return file_sysboxFsProtobuf_proto_rawDescData +} + +var file_sysboxFsProtobuf_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sysboxFsProtobuf_proto_goTypes = []interface{}{ + (*ContainerData)(nil), // 0: protobuf.ContainerData + (*Response)(nil), // 1: protobuf.Response + (*MountpointReq)(nil), // 2: protobuf.MountpointReq + (*MountpointResp)(nil), // 3: protobuf.MountpointResp + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp +} +var file_sysboxFsProtobuf_proto_depIdxs = []int32{ + 4, // 0: protobuf.ContainerData.Ctime:type_name -> google.protobuf.Timestamp + 2, // 1: protobuf.sysboxStateChannel.GetMountpoint:input_type -> protobuf.MountpointReq + 0, // 2: protobuf.sysboxStateChannel.ContainerPreRegistration:input_type -> protobuf.ContainerData + 0, // 3: protobuf.sysboxStateChannel.ContainerRegistration:input_type -> protobuf.ContainerData + 0, // 4: protobuf.sysboxStateChannel.ContainerUnregistration:input_type -> protobuf.ContainerData + 0, // 5: protobuf.sysboxStateChannel.ContainerUpdate:input_type -> protobuf.ContainerData + 3, // 6: protobuf.sysboxStateChannel.GetMountpoint:output_type -> protobuf.MountpointResp + 1, // 7: protobuf.sysboxStateChannel.ContainerPreRegistration:output_type -> protobuf.Response + 1, // 8: protobuf.sysboxStateChannel.ContainerRegistration:output_type -> protobuf.Response + 1, // 9: protobuf.sysboxStateChannel.ContainerUnregistration:output_type -> protobuf.Response + 1, // 10: protobuf.sysboxStateChannel.ContainerUpdate:output_type -> protobuf.Response + 6, // [6:11] is the sub-list for method output_type + 1, // [1:6] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_sysboxFsProtobuf_proto_init() } +func file_sysboxFsProtobuf_proto_init() { + if File_sysboxFsProtobuf_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_sysboxFsProtobuf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Response); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountpointReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxFsProtobuf_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountpointResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_sysboxFsProtobuf_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sysboxFsProtobuf_proto_goTypes, + DependencyIndexes: file_sysboxFsProtobuf_proto_depIdxs, + MessageInfos: file_sysboxFsProtobuf_proto_msgTypes, + }.Build() + File_sysboxFsProtobuf_proto = out.File + file_sysboxFsProtobuf_proto_rawDesc = nil + file_sysboxFsProtobuf_proto_goTypes = nil + file_sysboxFsProtobuf_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// SysboxStateChannelClient is the client API for SysboxStateChannel service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SysboxStateChannelClient interface { + // Queries sysbox-fs for the FUSE mountpoint + GetMountpoint(ctx context.Context, in *MountpointReq, opts ...grpc.CallOption) (*MountpointResp, error) + // Generates a container-preregistration message + ContainerPreRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-registration message + ContainerRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-unregistration message + ContainerUnregistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) + // Generates a container-update message + ContainerUpdate(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) +} + +type sysboxStateChannelClient struct { + cc grpc.ClientConnInterface +} + +func NewSysboxStateChannelClient(cc grpc.ClientConnInterface) SysboxStateChannelClient { + return &sysboxStateChannelClient{cc} +} + +func (c *sysboxStateChannelClient) GetMountpoint(ctx context.Context, in *MountpointReq, opts ...grpc.CallOption) (*MountpointResp, error) { + out := new(MountpointResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/GetMountpoint", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerPreRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerPreRegistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerRegistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerRegistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerUnregistration(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerUnregistration", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxStateChannelClient) ContainerUpdate(ctx context.Context, in *ContainerData, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxStateChannel/ContainerUpdate", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SysboxStateChannelServer is the server API for SysboxStateChannel service. +type SysboxStateChannelServer interface { + // Queries sysbox-fs for the FUSE mountpoint + GetMountpoint(context.Context, *MountpointReq) (*MountpointResp, error) + // Generates a container-preregistration message + ContainerPreRegistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-registration message + ContainerRegistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-unregistration message + ContainerUnregistration(context.Context, *ContainerData) (*Response, error) + // Generates a container-update message + ContainerUpdate(context.Context, *ContainerData) (*Response, error) +} + +// UnimplementedSysboxStateChannelServer can be embedded to have forward compatible implementations. +type UnimplementedSysboxStateChannelServer struct { +} + +func (*UnimplementedSysboxStateChannelServer) GetMountpoint(context.Context, *MountpointReq) (*MountpointResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMountpoint not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerPreRegistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerPreRegistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerRegistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerRegistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerUnregistration(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerUnregistration not implemented") +} +func (*UnimplementedSysboxStateChannelServer) ContainerUpdate(context.Context, *ContainerData) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerUpdate not implemented") +} + +func RegisterSysboxStateChannelServer(s *grpc.Server, srv SysboxStateChannelServer) { + s.RegisterService(&_SysboxStateChannel_serviceDesc, srv) +} + +func _SysboxStateChannel_GetMountpoint_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountpointReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).GetMountpoint(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/GetMountpoint", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).GetMountpoint(ctx, req.(*MountpointReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerPreRegistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerPreRegistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerPreRegistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerPreRegistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerRegistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerRegistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerRegistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerRegistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerUnregistration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerUnregistration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerUnregistration", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerUnregistration(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxStateChannel_ContainerUpdate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerData) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxStateChannelServer).ContainerUpdate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxStateChannel/ContainerUpdate", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxStateChannelServer).ContainerUpdate(ctx, req.(*ContainerData)) + } + return interceptor(ctx, in, info, handler) +} + +var _SysboxStateChannel_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protobuf.sysboxStateChannel", + HandlerType: (*SysboxStateChannelServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetMountpoint", + Handler: _SysboxStateChannel_GetMountpoint_Handler, + }, + { + MethodName: "ContainerPreRegistration", + Handler: _SysboxStateChannel_ContainerPreRegistration_Handler, + }, + { + MethodName: "ContainerRegistration", + Handler: _SysboxStateChannel_ContainerRegistration_Handler, + }, + { + MethodName: "ContainerUnregistration", + Handler: _SysboxStateChannel_ContainerUnregistration_Handler, + }, + { + MethodName: "ContainerUpdate", + Handler: _SysboxStateChannel_ContainerUpdate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "sysboxFsProtobuf.proto", +} diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go new file mode 100644 index 000000000..1c0364d2f --- /dev/null +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go @@ -0,0 +1,3203 @@ +// +// SysboxMgr Protobuffer Definitions. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v3.21.12 +// source: sysboxMgrProtobuf.proto + +package sysboxMgrProtobuf + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type IDMapping struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerID uint32 `protobuf:"varint,1,opt,name=containerID,proto3" json:"containerID,omitempty"` + HostID uint32 `protobuf:"varint,2,opt,name=hostID,proto3" json:"hostID,omitempty"` + Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *IDMapping) Reset() { + *x = IDMapping{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IDMapping) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IDMapping) ProtoMessage() {} + +func (x *IDMapping) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IDMapping.ProtoReflect.Descriptor instead. +func (*IDMapping) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{0} +} + +func (x *IDMapping) GetContainerID() uint32 { + if x != nil { + return x.ContainerID + } + return 0 +} + +func (x *IDMapping) GetHostID() uint32 { + if x != nil { + return x.HostID + } + return 0 +} + +func (x *IDMapping) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +type RegisterReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Rootfs string `protobuf:"bytes,2,opt,name=rootfs,proto3" json:"rootfs,omitempty"` + Userns string `protobuf:"bytes,3,opt,name=userns,proto3" json:"userns,omitempty"` + Netns string `protobuf:"bytes,4,opt,name=netns,proto3" json:"netns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,5,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,6,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` +} + +func (x *RegisterReq) Reset() { + *x = RegisterReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RegisterReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterReq) ProtoMessage() {} + +func (x *RegisterReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterReq.ProtoReflect.Descriptor instead. +func (*RegisterReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{1} +} + +func (x *RegisterReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *RegisterReq) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +func (x *RegisterReq) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *RegisterReq) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +func (x *RegisterReq) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *RegisterReq) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +type ContainerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + AliasDns bool `protobuf:"varint,1,opt,name=aliasDns,proto3" json:"aliasDns,omitempty"` + ShiftfsOk bool `protobuf:"varint,2,opt,name=ShiftfsOk,proto3" json:"ShiftfsOk,omitempty"` + ShiftfsOnOverlayfsOk bool `protobuf:"varint,3,opt,name=ShiftfsOnOverlayfsOk,proto3" json:"ShiftfsOnOverlayfsOk,omitempty"` + IDMapMountOk bool `protobuf:"varint,4,opt,name=IDMapMountOk,proto3" json:"IDMapMountOk,omitempty"` + OverlayfsOnIDMapMountOk bool `protobuf:"varint,5,opt,name=OverlayfsOnIDMapMountOk,proto3" json:"OverlayfsOnIDMapMountOk,omitempty"` + NoRootfsCloning bool `protobuf:"varint,6,opt,name=noRootfsCloning,proto3" json:"noRootfsCloning,omitempty"` + IgnoreSysfsChown bool `protobuf:"varint,7,opt,name=ignoreSysfsChown,proto3" json:"ignoreSysfsChown,omitempty"` + AllowTrustedXattr bool `protobuf:"varint,8,opt,name=allowTrustedXattr,proto3" json:"allowTrustedXattr,omitempty"` + HonorCaps bool `protobuf:"varint,9,opt,name=honorCaps,proto3" json:"honorCaps,omitempty"` + SyscontMode bool `protobuf:"varint,10,opt,name=syscontMode,proto3" json:"syscontMode,omitempty"` + Userns string `protobuf:"bytes,11,opt,name=userns,proto3" json:"userns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,12,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,13,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` + FsuidMapFailOnErr bool `protobuf:"varint,14,opt,name=fsuidMapFailOnErr,proto3" json:"fsuidMapFailOnErr,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,15,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` + NoShiftfsOnFuse bool `protobuf:"varint,16,opt,name=noShiftfsOnFuse,proto3" json:"noShiftfsOnFuse,omitempty"` + RelaxedReadOnly bool `protobuf:"varint,17,opt,name=relaxedReadOnly,proto3" json:"relaxedReadOnly,omitempty"` +} + +func (x *ContainerConfig) Reset() { + *x = ContainerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerConfig) ProtoMessage() {} + +func (x *ContainerConfig) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerConfig.ProtoReflect.Descriptor instead. +func (*ContainerConfig) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{2} +} + +func (x *ContainerConfig) GetAliasDns() bool { + if x != nil { + return x.AliasDns + } + return false +} + +func (x *ContainerConfig) GetShiftfsOk() bool { + if x != nil { + return x.ShiftfsOk + } + return false +} + +func (x *ContainerConfig) GetShiftfsOnOverlayfsOk() bool { + if x != nil { + return x.ShiftfsOnOverlayfsOk + } + return false +} + +func (x *ContainerConfig) GetIDMapMountOk() bool { + if x != nil { + return x.IDMapMountOk + } + return false +} + +func (x *ContainerConfig) GetOverlayfsOnIDMapMountOk() bool { + if x != nil { + return x.OverlayfsOnIDMapMountOk + } + return false +} + +func (x *ContainerConfig) GetNoRootfsCloning() bool { + if x != nil { + return x.NoRootfsCloning + } + return false +} + +func (x *ContainerConfig) GetIgnoreSysfsChown() bool { + if x != nil { + return x.IgnoreSysfsChown + } + return false +} + +func (x *ContainerConfig) GetAllowTrustedXattr() bool { + if x != nil { + return x.AllowTrustedXattr + } + return false +} + +func (x *ContainerConfig) GetHonorCaps() bool { + if x != nil { + return x.HonorCaps + } + return false +} + +func (x *ContainerConfig) GetSyscontMode() bool { + if x != nil { + return x.SyscontMode + } + return false +} + +func (x *ContainerConfig) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *ContainerConfig) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *ContainerConfig) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +func (x *ContainerConfig) GetFsuidMapFailOnErr() bool { + if x != nil { + return x.FsuidMapFailOnErr + } + return false +} + +func (x *ContainerConfig) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +func (x *ContainerConfig) GetNoShiftfsOnFuse() bool { + if x != nil { + return x.NoShiftfsOnFuse + } + return false +} + +func (x *ContainerConfig) GetRelaxedReadOnly() bool { + if x != nil { + return x.RelaxedReadOnly + } + return false +} + +type RegisterResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerConfig *ContainerConfig `protobuf:"bytes,1,opt,name=containerConfig,proto3" json:"containerConfig,omitempty"` +} + +func (x *RegisterResp) Reset() { + *x = RegisterResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RegisterResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterResp) ProtoMessage() {} + +func (x *RegisterResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterResp.ProtoReflect.Descriptor instead. +func (*RegisterResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{3} +} + +func (x *RegisterResp) GetContainerConfig() *ContainerConfig { + if x != nil { + return x.ContainerConfig + } + return nil +} + +type UpdateReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Userns string `protobuf:"bytes,2,opt,name=userns,proto3" json:"userns,omitempty"` + Netns string `protobuf:"bytes,3,opt,name=netns,proto3" json:"netns,omitempty"` + UidMappings []*IDMapping `protobuf:"bytes,4,rep,name=uidMappings,proto3" json:"uidMappings,omitempty"` + GidMappings []*IDMapping `protobuf:"bytes,5,rep,name=gidMappings,proto3" json:"gidMappings,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,6,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` +} + +func (x *UpdateReq) Reset() { + *x = UpdateReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateReq) ProtoMessage() {} + +func (x *UpdateReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateReq.ProtoReflect.Descriptor instead. +func (*UpdateReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{4} +} + +func (x *UpdateReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *UpdateReq) GetUserns() string { + if x != nil { + return x.Userns + } + return "" +} + +func (x *UpdateReq) GetNetns() string { + if x != nil { + return x.Netns + } + return "" +} + +func (x *UpdateReq) GetUidMappings() []*IDMapping { + if x != nil { + return x.UidMappings + } + return nil +} + +func (x *UpdateReq) GetGidMappings() []*IDMapping { + if x != nil { + return x.GidMappings + } + return nil +} + +func (x *UpdateReq) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +type UpdateResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *UpdateResp) Reset() { + *x = UpdateResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateResp) ProtoMessage() {} + +func (x *UpdateResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateResp.ProtoReflect.Descriptor instead. +func (*UpdateResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{5} +} + +type UnregisterReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *UnregisterReq) Reset() { + *x = UnregisterReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UnregisterReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UnregisterReq) ProtoMessage() {} + +func (x *UnregisterReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UnregisterReq.ProtoReflect.Descriptor instead. +func (*UnregisterReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{6} +} + +func (x *UnregisterReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type UnregisterResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *UnregisterResp) Reset() { + *x = UnregisterResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UnregisterResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UnregisterResp) ProtoMessage() {} + +func (x *UnregisterResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UnregisterResp.ProtoReflect.Descriptor instead. +func (*UnregisterResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{7} +} + +type SubidAllocReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Size uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *SubidAllocReq) Reset() { + *x = SubidAllocReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SubidAllocReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubidAllocReq) ProtoMessage() {} + +func (x *SubidAllocReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubidAllocReq.ProtoReflect.Descriptor instead. +func (*SubidAllocReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{8} +} + +func (x *SubidAllocReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SubidAllocReq) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +type SubidAllocResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Uid uint32 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,2,opt,name=gid,proto3" json:"gid,omitempty"` +} + +func (x *SubidAllocResp) Reset() { + *x = SubidAllocResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SubidAllocResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubidAllocResp) ProtoMessage() {} + +func (x *SubidAllocResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubidAllocResp.ProtoReflect.Descriptor instead. +func (*SubidAllocResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{9} +} + +func (x *SubidAllocResp) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *SubidAllocResp) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +type MountPrepInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Exclusive bool `protobuf:"varint,2,opt,name=exclusive,proto3" json:"exclusive,omitempty"` +} + +func (x *MountPrepInfo) Reset() { + *x = MountPrepInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepInfo) ProtoMessage() {} + +func (x *MountPrepInfo) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepInfo.ProtoReflect.Descriptor instead. +func (*MountPrepInfo) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{10} +} + +func (x *MountPrepInfo) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *MountPrepInfo) GetExclusive() bool { + if x != nil { + return x.Exclusive + } + return false +} + +type MountPrepReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Uid uint32 `protobuf:"varint,2,opt,name=uid,proto3" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,3,opt,name=gid,proto3" json:"gid,omitempty"` + PrepList []*MountPrepInfo `protobuf:"bytes,4,rep,name=prepList,proto3" json:"prepList,omitempty"` +} + +func (x *MountPrepReq) Reset() { + *x = MountPrepReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepReq) ProtoMessage() {} + +func (x *MountPrepReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepReq.ProtoReflect.Descriptor instead. +func (*MountPrepReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{11} +} + +func (x *MountPrepReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *MountPrepReq) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *MountPrepReq) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +func (x *MountPrepReq) GetPrepList() []*MountPrepInfo { + if x != nil { + return x.PrepList + } + return nil +} + +type MountPrepResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *MountPrepResp) Reset() { + *x = MountPrepResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountPrepResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountPrepResp) ProtoMessage() {} + +func (x *MountPrepResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountPrepResp.ProtoReflect.Descriptor instead. +func (*MountPrepResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{12} +} + +type MountReqInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind uint32 `protobuf:"varint,1,opt,name=kind,proto3" json:"kind,omitempty"` + Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` +} + +func (x *MountReqInfo) Reset() { + *x = MountReqInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountReqInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountReqInfo) ProtoMessage() {} + +func (x *MountReqInfo) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountReqInfo.ProtoReflect.Descriptor instead. +func (*MountReqInfo) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{13} +} + +func (x *MountReqInfo) GetKind() uint32 { + if x != nil { + return x.Kind + } + return 0 +} + +func (x *MountReqInfo) GetDest() string { + if x != nil { + return x.Dest + } + return "" +} + +type MountReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + RootfsUidShiftType uint32 `protobuf:"varint,2,opt,name=rootfsUidShiftType,proto3" json:"rootfsUidShiftType,omitempty"` + ReqList []*MountReqInfo `protobuf:"bytes,3,rep,name=reqList,proto3" json:"reqList,omitempty"` +} + +func (x *MountReq) Reset() { + *x = MountReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountReq) ProtoMessage() {} + +func (x *MountReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountReq.ProtoReflect.Descriptor instead. +func (*MountReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{14} +} + +func (x *MountReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *MountReq) GetRootfsUidShiftType() uint32 { + if x != nil { + return x.RootfsUidShiftType + } + return 0 +} + +func (x *MountReq) GetReqList() []*MountReqInfo { + if x != nil { + return x.ReqList + } + return nil +} + +type Mount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Dest string `protobuf:"bytes,2,opt,name=dest,proto3" json:"dest,omitempty"` + Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` + Opt []string `protobuf:"bytes,4,rep,name=opt,proto3" json:"opt,omitempty"` +} + +func (x *Mount) Reset() { + *x = Mount{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Mount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mount) ProtoMessage() {} + +func (x *Mount) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mount.ProtoReflect.Descriptor instead. +func (*Mount) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{15} +} + +func (x *Mount) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *Mount) GetDest() string { + if x != nil { + return x.Dest + } + return "" +} + +func (x *Mount) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *Mount) GetOpt() []string { + if x != nil { + return x.Opt + } + return nil +} + +type MountResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Mounts []*Mount `protobuf:"bytes,1,rep,name=mounts,proto3" json:"mounts,omitempty"` +} + +func (x *MountResp) Reset() { + *x = MountResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MountResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MountResp) ProtoMessage() {} + +func (x *MountResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MountResp.ProtoReflect.Descriptor instead. +func (*MountResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{16} +} + +func (x *MountResp) GetMounts() []*Mount { + if x != nil { + return x.Mounts + } + return nil +} + +type ShiftfsMark struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Readonly bool `protobuf:"varint,2,opt,name=readonly,proto3" json:"readonly,omitempty"` +} + +func (x *ShiftfsMark) Reset() { + *x = ShiftfsMark{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMark) ProtoMessage() {} + +func (x *ShiftfsMark) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMark.ProtoReflect.Descriptor instead. +func (*ShiftfsMark) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{17} +} + +func (x *ShiftfsMark) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *ShiftfsMark) GetReadonly() bool { + if x != nil { + return x.Readonly + } + return false +} + +type ShiftfsMarkReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ShiftfsMarks []*ShiftfsMark `protobuf:"bytes,2,rep,name=shiftfsMarks,proto3" json:"shiftfsMarks,omitempty"` +} + +func (x *ShiftfsMarkReq) Reset() { + *x = ShiftfsMarkReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMarkReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMarkReq) ProtoMessage() {} + +func (x *ShiftfsMarkReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMarkReq.ProtoReflect.Descriptor instead. +func (*ShiftfsMarkReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{18} +} + +func (x *ShiftfsMarkReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ShiftfsMarkReq) GetShiftfsMarks() []*ShiftfsMark { + if x != nil { + return x.ShiftfsMarks + } + return nil +} + +type ShiftfsMarkResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ShiftfsMarks []*ShiftfsMark `protobuf:"bytes,1,rep,name=shiftfsMarks,proto3" json:"shiftfsMarks,omitempty"` +} + +func (x *ShiftfsMarkResp) Reset() { + *x = ShiftfsMarkResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShiftfsMarkResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShiftfsMarkResp) ProtoMessage() {} + +func (x *ShiftfsMarkResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShiftfsMarkResp.ProtoReflect.Descriptor instead. +func (*ShiftfsMarkResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{19} +} + +func (x *ShiftfsMarkResp) GetShiftfsMarks() []*ShiftfsMark { + if x != nil { + return x.ShiftfsMarks + } + return nil +} + +type FsStateReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Rootfs string `protobuf:"bytes,2,opt,name=rootfs,proto3" json:"rootfs,omitempty"` +} + +func (x *FsStateReq) Reset() { + *x = FsStateReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsStateReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsStateReq) ProtoMessage() {} + +func (x *FsStateReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsStateReq.ProtoReflect.Descriptor instead. +func (*FsStateReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{20} +} + +func (x *FsStateReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *FsStateReq) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +type FsEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind uint32 `protobuf:"varint,1,opt,name=kind,proto3" json:"kind,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + Dst string `protobuf:"bytes,4,opt,name=dst,proto3" json:"dst,omitempty"` +} + +func (x *FsEntry) Reset() { + *x = FsEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsEntry) ProtoMessage() {} + +func (x *FsEntry) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsEntry.ProtoReflect.Descriptor instead. +func (*FsEntry) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{21} +} + +func (x *FsEntry) GetKind() uint32 { + if x != nil { + return x.Kind + } + return 0 +} + +func (x *FsEntry) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *FsEntry) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *FsEntry) GetDst() string { + if x != nil { + return x.Dst + } + return "" +} + +type FsStateResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FsEntries []*FsEntry `protobuf:"bytes,1,rep,name=fsEntries,proto3" json:"fsEntries,omitempty"` +} + +func (x *FsStateResp) Reset() { + *x = FsStateResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FsStateResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FsStateResp) ProtoMessage() {} + +func (x *FsStateResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FsStateResp.ProtoReflect.Descriptor instead. +func (*FsStateResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{22} +} + +func (x *FsStateResp) GetFsEntries() []*FsEntry { + if x != nil { + return x.FsEntries + } + return nil +} + +type PauseReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *PauseReq) Reset() { + *x = PauseReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PauseReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PauseReq) ProtoMessage() {} + +func (x *PauseReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PauseReq.ProtoReflect.Descriptor instead. +func (*PauseReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{23} +} + +func (x *PauseReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type PauseResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *PauseResp) Reset() { + *x = PauseResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PauseResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PauseResp) ProtoMessage() {} + +func (x *PauseResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PauseResp.ProtoReflect.Descriptor instead. +func (*PauseResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{24} +} + +type ResumeReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ResumeReq) Reset() { + *x = ResumeReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResumeReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeReq) ProtoMessage() {} + +func (x *ResumeReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeReq.ProtoReflect.Descriptor instead. +func (*ResumeReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{25} +} + +func (x *ResumeReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ResumeResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ResumeResp) Reset() { + *x = ResumeResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResumeResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeResp) ProtoMessage() {} + +func (x *ResumeResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeResp.ProtoReflect.Descriptor instead. +func (*ResumeResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{26} +} + +type CloneRootfsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *CloneRootfsReq) Reset() { + *x = CloneRootfsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CloneRootfsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloneRootfsReq) ProtoMessage() {} + +func (x *CloneRootfsReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloneRootfsReq.ProtoReflect.Descriptor instead. +func (*CloneRootfsReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{27} +} + +func (x *CloneRootfsReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type CloneRootfsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rootfs string `protobuf:"bytes,1,opt,name=rootfs,proto3" json:"rootfs,omitempty"` +} + +func (x *CloneRootfsResp) Reset() { + *x = CloneRootfsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CloneRootfsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CloneRootfsResp) ProtoMessage() {} + +func (x *CloneRootfsResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CloneRootfsResp.ProtoReflect.Descriptor instead. +func (*CloneRootfsResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{28} +} + +func (x *CloneRootfsResp) GetRootfs() string { + if x != nil { + return x.Rootfs + } + return "" +} + +type ChownClonedRootfsReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + UidOffset int32 `protobuf:"varint,2,opt,name=uidOffset,proto3" json:"uidOffset,omitempty"` + GidOffset int32 `protobuf:"varint,3,opt,name=gidOffset,proto3" json:"gidOffset,omitempty"` +} + +func (x *ChownClonedRootfsReq) Reset() { + *x = ChownClonedRootfsReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChownClonedRootfsReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChownClonedRootfsReq) ProtoMessage() {} + +func (x *ChownClonedRootfsReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChownClonedRootfsReq.ProtoReflect.Descriptor instead. +func (*ChownClonedRootfsReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{29} +} + +func (x *ChownClonedRootfsReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ChownClonedRootfsReq) GetUidOffset() int32 { + if x != nil { + return x.UidOffset + } + return 0 +} + +func (x *ChownClonedRootfsReq) GetGidOffset() int32 { + if x != nil { + return x.GidOffset + } + return 0 +} + +type ChownClonedRootfsResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ChownClonedRootfsResp) Reset() { + *x = ChownClonedRootfsResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChownClonedRootfsResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChownClonedRootfsResp) ProtoMessage() {} + +func (x *ChownClonedRootfsResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChownClonedRootfsResp.ProtoReflect.Descriptor instead. +func (*ChownClonedRootfsResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{30} +} + +type RevertClonedRootfsChownReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *RevertClonedRootfsChownReq) Reset() { + *x = RevertClonedRootfsChownReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RevertClonedRootfsChownReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertClonedRootfsChownReq) ProtoMessage() {} + +func (x *RevertClonedRootfsChownReq) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertClonedRootfsChownReq.ProtoReflect.Descriptor instead. +func (*RevertClonedRootfsChownReq) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{31} +} + +func (x *RevertClonedRootfsChownReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type RevertClonedRootfsChownResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RevertClonedRootfsChownResp) Reset() { + *x = RevertClonedRootfsChownResp{} + if protoimpl.UnsafeEnabled { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RevertClonedRootfsChownResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevertClonedRootfsChownResp) ProtoMessage() {} + +func (x *RevertClonedRootfsChownResp) ProtoReflect() protoreflect.Message { + mi := &file_sysboxMgrProtobuf_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevertClonedRootfsChownResp.ProtoReflect.Descriptor instead. +func (*RevertClonedRootfsChownResp) Descriptor() ([]byte, []int) { + return file_sysboxMgrProtobuf_proto_rawDescGZIP(), []int{32} +} + +var File_sysboxMgrProtobuf_proto protoreflect.FileDescriptor + +var file_sysboxMgrProtobuf_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x4d, 0x67, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x22, 0x59, 0x0a, 0x09, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x6f, 0x73, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x06, 0x68, 0x6f, 0x73, 0x74, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0xd1, + 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, + 0x65, 0x74, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, + 0x6e, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, + 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, + 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x73, 0x22, 0xd9, 0x05, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x44, + 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x44, + 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6b, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6b, + 0x12, 0x32, 0x0a, 0x14, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x4f, 0x76, 0x65, + 0x72, 0x6c, 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, + 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x79, + 0x66, 0x73, 0x4f, 0x6b, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x4f, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x49, 0x44, 0x4d, 0x61, + 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x4f, 0x6b, 0x12, 0x38, 0x0a, 0x17, 0x4f, 0x76, 0x65, 0x72, + 0x6c, 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, 0x6e, + 0x74, 0x4f, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x4f, 0x76, 0x65, 0x72, 0x6c, + 0x61, 0x79, 0x66, 0x73, 0x4f, 0x6e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x4f, 0x6b, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x6f, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x6c, + 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6e, 0x6f, 0x52, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x6c, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x2a, 0x0a, 0x10, + 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x79, 0x73, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x53, 0x79, + 0x73, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, + 0x77, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, 0x64, 0x58, 0x61, 0x74, 0x74, 0x72, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x75, 0x73, 0x74, 0x65, + 0x64, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x68, 0x6f, 0x6e, 0x6f, 0x72, 0x43, + 0x61, 0x70, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x68, 0x6f, 0x6e, 0x6f, 0x72, + 0x43, 0x61, 0x70, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x79, 0x73, 0x63, 0x6f, 0x6e, 0x74, 0x4d, + 0x6f, 0x64, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x79, 0x73, 0x63, 0x6f, + 0x6e, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x35, + 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0c, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, + 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, + 0x69, 0x6e, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, + 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x11, + 0x66, 0x73, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x46, 0x61, 0x69, 0x6c, 0x4f, 0x6e, 0x45, 0x72, + 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x66, 0x73, 0x75, 0x69, 0x64, 0x4d, 0x61, + 0x70, 0x46, 0x61, 0x69, 0x6c, 0x4f, 0x6e, 0x45, 0x72, 0x72, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x6f, + 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, + 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x6f, + 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, 0x46, 0x75, 0x73, 0x65, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6e, 0x6f, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4f, 0x6e, + 0x46, 0x75, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x72, 0x65, 0x6c, 0x61, 0x78, 0x65, 0x64, 0x52, + 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x72, + 0x65, 0x6c, 0x61, 0x78, 0x65, 0x64, 0x52, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x53, + 0x0a, 0x0c, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x43, + 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x22, 0xe7, 0x01, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, + 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x6e, 0x73, 0x12, + 0x35, 0x0a, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x52, 0x0b, 0x75, 0x69, 0x64, 0x4d, 0x61, + 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x44, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, + 0x52, 0x0b, 0x67, 0x69, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x2e, 0x0a, + 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x0c, 0x0a, + 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x1f, 0x0a, 0x0d, 0x55, + 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x10, 0x0a, 0x0e, + 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x33, + 0x0a, 0x0d, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, 0x71, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x22, 0x34, 0x0a, 0x0e, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, + 0x63, 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x45, 0x0a, 0x0d, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x73, 0x69, 0x76, 0x65, + 0x22, 0x77, 0x0a, 0x0c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x71, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x75, + 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x03, 0x67, 0x69, 0x64, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x70, 0x4c, 0x69, 0x73, 0x74, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x08, 0x70, 0x72, 0x65, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x0a, 0x0d, 0x4d, 0x6f, 0x75, + 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x36, 0x0a, 0x0c, 0x4d, 0x6f, + 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, + 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x64, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x65, + 0x73, 0x74, 0x22, 0x7c, 0x0a, 0x08, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2e, + 0x0a, 0x12, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x72, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x55, 0x69, 0x64, 0x53, 0x68, 0x69, 0x66, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x30, + 0x0a, 0x07, 0x72, 0x65, 0x71, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x72, 0x65, 0x71, 0x4c, 0x69, 0x73, 0x74, + 0x22, 0x59, 0x0a, 0x05, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x64, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6f, 0x70, 0x74, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x6f, 0x70, 0x74, 0x22, 0x34, 0x0a, 0x09, 0x4d, + 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x27, 0x0a, 0x06, 0x6d, 0x6f, 0x75, 0x6e, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x06, 0x6d, 0x6f, 0x75, 0x6e, 0x74, + 0x73, 0x22, 0x41, 0x0a, 0x0b, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x61, 0x64, + 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, + 0x6f, 0x6e, 0x6c, 0x79, 0x22, 0x5b, 0x0a, 0x0e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, + 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x52, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x73, 0x22, 0x4c, 0x0a, 0x0f, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x12, 0x39, 0x0a, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, + 0x61, 0x72, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, + 0x6b, 0x52, 0x0c, 0x73, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x73, 0x22, + 0x34, 0x0a, 0x0a, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x22, 0x57, 0x0a, 0x07, 0x46, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x64, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x73, 0x74, 0x22, 0x3e, + 0x0a, 0x0b, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x2f, 0x0a, + 0x09, 0x66, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x09, 0x66, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x1a, + 0x0a, 0x08, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x0b, 0x0a, 0x09, 0x50, 0x61, + 0x75, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x1b, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x0c, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x22, 0x20, 0x0a, 0x0e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x29, 0x0a, 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, + 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x22, + 0x62, 0x0a, 0x14, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, + 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x69, 0x64, 0x4f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x75, 0x69, 0x64, 0x4f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x67, 0x69, 0x64, 0x4f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x67, 0x69, 0x64, 0x4f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x22, 0x17, 0x0a, 0x15, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, + 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x2c, 0x0a, 0x1a, + 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, + 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, + 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x32, 0x86, 0x07, 0x0a, 0x15, 0x73, 0x79, + 0x73, 0x62, 0x6f, 0x78, 0x4d, 0x67, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x12, 0x3b, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x1a, + 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0a, 0x55, 0x6e, 0x72, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x18, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x72, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0a, 0x53, 0x75, + 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x75, 0x62, 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, + 0x71, 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x75, 0x62, + 0x69, 0x64, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x3f, 0x0a, + 0x0a, 0x50, 0x72, 0x65, 0x70, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, + 0x52, 0x65, 0x71, 0x1a, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, + 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x72, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x36, + 0x0a, 0x09, 0x52, 0x65, 0x71, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x1a, + 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x53, 0x68, 0x69, + 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x52, + 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x68, + 0x69, 0x66, 0x74, 0x66, 0x73, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, + 0x3b, 0x0a, 0x0a, 0x52, 0x65, 0x71, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, + 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x05, + 0x50, 0x61, 0x75, 0x73, 0x65, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x1a, + 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x43, 0x6c, + 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, + 0x52, 0x65, 0x71, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x43, + 0x6c, 0x6f, 0x6e, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x12, 0x56, 0x0a, 0x11, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, + 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x17, 0x52, 0x65, 0x76, 0x65, + 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x68, + 0x6f, 0x77, 0x6e, 0x12, 0x24, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, + 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, + 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x43, 0x6c, 0x6f, 0x6e, 0x65, + 0x64, 0x52, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x43, 0x68, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x22, 0x00, 0x42, 0x16, 0x5a, 0x14, 0x2e, 0x2f, 0x3b, 0x73, 0x79, 0x73, 0x62, 0x6f, 0x78, 0x4d, + 0x67, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_sysboxMgrProtobuf_proto_rawDescOnce sync.Once + file_sysboxMgrProtobuf_proto_rawDescData = file_sysboxMgrProtobuf_proto_rawDesc +) + +func file_sysboxMgrProtobuf_proto_rawDescGZIP() []byte { + file_sysboxMgrProtobuf_proto_rawDescOnce.Do(func() { + file_sysboxMgrProtobuf_proto_rawDescData = protoimpl.X.CompressGZIP(file_sysboxMgrProtobuf_proto_rawDescData) + }) + return file_sysboxMgrProtobuf_proto_rawDescData +} + +var file_sysboxMgrProtobuf_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_sysboxMgrProtobuf_proto_goTypes = []interface{}{ + (*IDMapping)(nil), // 0: protobuf.IDMapping + (*RegisterReq)(nil), // 1: protobuf.RegisterReq + (*ContainerConfig)(nil), // 2: protobuf.ContainerConfig + (*RegisterResp)(nil), // 3: protobuf.RegisterResp + (*UpdateReq)(nil), // 4: protobuf.UpdateReq + (*UpdateResp)(nil), // 5: protobuf.UpdateResp + (*UnregisterReq)(nil), // 6: protobuf.UnregisterReq + (*UnregisterResp)(nil), // 7: protobuf.UnregisterResp + (*SubidAllocReq)(nil), // 8: protobuf.SubidAllocReq + (*SubidAllocResp)(nil), // 9: protobuf.SubidAllocResp + (*MountPrepInfo)(nil), // 10: protobuf.MountPrepInfo + (*MountPrepReq)(nil), // 11: protobuf.MountPrepReq + (*MountPrepResp)(nil), // 12: protobuf.MountPrepResp + (*MountReqInfo)(nil), // 13: protobuf.MountReqInfo + (*MountReq)(nil), // 14: protobuf.MountReq + (*Mount)(nil), // 15: protobuf.Mount + (*MountResp)(nil), // 16: protobuf.MountResp + (*ShiftfsMark)(nil), // 17: protobuf.ShiftfsMark + (*ShiftfsMarkReq)(nil), // 18: protobuf.ShiftfsMarkReq + (*ShiftfsMarkResp)(nil), // 19: protobuf.ShiftfsMarkResp + (*FsStateReq)(nil), // 20: protobuf.FsStateReq + (*FsEntry)(nil), // 21: protobuf.FsEntry + (*FsStateResp)(nil), // 22: protobuf.FsStateResp + (*PauseReq)(nil), // 23: protobuf.PauseReq + (*PauseResp)(nil), // 24: protobuf.PauseResp + (*ResumeReq)(nil), // 25: protobuf.ResumeReq + (*ResumeResp)(nil), // 26: protobuf.ResumeResp + (*CloneRootfsReq)(nil), // 27: protobuf.CloneRootfsReq + (*CloneRootfsResp)(nil), // 28: protobuf.CloneRootfsResp + (*ChownClonedRootfsReq)(nil), // 29: protobuf.ChownClonedRootfsReq + (*ChownClonedRootfsResp)(nil), // 30: protobuf.ChownClonedRootfsResp + (*RevertClonedRootfsChownReq)(nil), // 31: protobuf.RevertClonedRootfsChownReq + (*RevertClonedRootfsChownResp)(nil), // 32: protobuf.RevertClonedRootfsChownResp +} +var file_sysboxMgrProtobuf_proto_depIdxs = []int32{ + 0, // 0: protobuf.RegisterReq.uidMappings:type_name -> protobuf.IDMapping + 0, // 1: protobuf.RegisterReq.gidMappings:type_name -> protobuf.IDMapping + 0, // 2: protobuf.ContainerConfig.uidMappings:type_name -> protobuf.IDMapping + 0, // 3: protobuf.ContainerConfig.gidMappings:type_name -> protobuf.IDMapping + 2, // 4: protobuf.RegisterResp.containerConfig:type_name -> protobuf.ContainerConfig + 0, // 5: protobuf.UpdateReq.uidMappings:type_name -> protobuf.IDMapping + 0, // 6: protobuf.UpdateReq.gidMappings:type_name -> protobuf.IDMapping + 10, // 7: protobuf.MountPrepReq.prepList:type_name -> protobuf.MountPrepInfo + 13, // 8: protobuf.MountReq.reqList:type_name -> protobuf.MountReqInfo + 15, // 9: protobuf.MountResp.mounts:type_name -> protobuf.Mount + 17, // 10: protobuf.ShiftfsMarkReq.shiftfsMarks:type_name -> protobuf.ShiftfsMark + 17, // 11: protobuf.ShiftfsMarkResp.shiftfsMarks:type_name -> protobuf.ShiftfsMark + 21, // 12: protobuf.FsStateResp.fsEntries:type_name -> protobuf.FsEntry + 1, // 13: protobuf.sysboxMgrStateChannel.Register:input_type -> protobuf.RegisterReq + 4, // 14: protobuf.sysboxMgrStateChannel.Update:input_type -> protobuf.UpdateReq + 6, // 15: protobuf.sysboxMgrStateChannel.Unregister:input_type -> protobuf.UnregisterReq + 8, // 16: protobuf.sysboxMgrStateChannel.SubidAlloc:input_type -> protobuf.SubidAllocReq + 11, // 17: protobuf.sysboxMgrStateChannel.PrepMounts:input_type -> protobuf.MountPrepReq + 14, // 18: protobuf.sysboxMgrStateChannel.ReqMounts:input_type -> protobuf.MountReq + 18, // 19: protobuf.sysboxMgrStateChannel.ReqShiftfsMark:input_type -> protobuf.ShiftfsMarkReq + 20, // 20: protobuf.sysboxMgrStateChannel.ReqFsState:input_type -> protobuf.FsStateReq + 23, // 21: protobuf.sysboxMgrStateChannel.Pause:input_type -> protobuf.PauseReq + 25, // 22: protobuf.sysboxMgrStateChannel.Resume:input_type -> protobuf.ResumeReq + 27, // 23: protobuf.sysboxMgrStateChannel.ReqCloneRootfs:input_type -> protobuf.CloneRootfsReq + 29, // 24: protobuf.sysboxMgrStateChannel.ChownClonedRootfs:input_type -> protobuf.ChownClonedRootfsReq + 31, // 25: protobuf.sysboxMgrStateChannel.RevertClonedRootfsChown:input_type -> protobuf.RevertClonedRootfsChownReq + 3, // 26: protobuf.sysboxMgrStateChannel.Register:output_type -> protobuf.RegisterResp + 5, // 27: protobuf.sysboxMgrStateChannel.Update:output_type -> protobuf.UpdateResp + 7, // 28: protobuf.sysboxMgrStateChannel.Unregister:output_type -> protobuf.UnregisterResp + 9, // 29: protobuf.sysboxMgrStateChannel.SubidAlloc:output_type -> protobuf.SubidAllocResp + 12, // 30: protobuf.sysboxMgrStateChannel.PrepMounts:output_type -> protobuf.MountPrepResp + 16, // 31: protobuf.sysboxMgrStateChannel.ReqMounts:output_type -> protobuf.MountResp + 19, // 32: protobuf.sysboxMgrStateChannel.ReqShiftfsMark:output_type -> protobuf.ShiftfsMarkResp + 22, // 33: protobuf.sysboxMgrStateChannel.ReqFsState:output_type -> protobuf.FsStateResp + 24, // 34: protobuf.sysboxMgrStateChannel.Pause:output_type -> protobuf.PauseResp + 26, // 35: protobuf.sysboxMgrStateChannel.Resume:output_type -> protobuf.ResumeResp + 28, // 36: protobuf.sysboxMgrStateChannel.ReqCloneRootfs:output_type -> protobuf.CloneRootfsResp + 30, // 37: protobuf.sysboxMgrStateChannel.ChownClonedRootfs:output_type -> protobuf.ChownClonedRootfsResp + 32, // 38: protobuf.sysboxMgrStateChannel.RevertClonedRootfsChown:output_type -> protobuf.RevertClonedRootfsChownResp + 26, // [26:39] is the sub-list for method output_type + 13, // [13:26] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_sysboxMgrProtobuf_proto_init() } +func file_sysboxMgrProtobuf_proto_init() { + if File_sysboxMgrProtobuf_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_sysboxMgrProtobuf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IDMapping); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UnregisterReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UnregisterResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubidAllocReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubidAllocResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountPrepResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountReqInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Mount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MountResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMark); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMarkReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShiftfsMarkResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsStateReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FsStateResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PauseReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PauseResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResumeReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResumeResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CloneRootfsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CloneRootfsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChownClonedRootfsReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChownClonedRootfsResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RevertClonedRootfsChownReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sysboxMgrProtobuf_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RevertClonedRootfsChownResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_sysboxMgrProtobuf_proto_rawDesc, + NumEnums: 0, + NumMessages: 33, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_sysboxMgrProtobuf_proto_goTypes, + DependencyIndexes: file_sysboxMgrProtobuf_proto_depIdxs, + MessageInfos: file_sysboxMgrProtobuf_proto_msgTypes, + }.Build() + File_sysboxMgrProtobuf_proto = out.File + file_sysboxMgrProtobuf_proto_rawDesc = nil + file_sysboxMgrProtobuf_proto_goTypes = nil + file_sysboxMgrProtobuf_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// SysboxMgrStateChannelClient is the client API for SysboxMgrStateChannel service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type SysboxMgrStateChannelClient interface { + // Container registration + Register(ctx context.Context, in *RegisterReq, opts ...grpc.CallOption) (*RegisterResp, error) + // Container Update + Update(ctx context.Context, in *UpdateReq, opts ...grpc.CallOption) (*UpdateResp, error) + // Container Unregistration + Unregister(ctx context.Context, in *UnregisterReq, opts ...grpc.CallOption) (*UnregisterResp, error) + // Subuid(gid) allocation request + SubidAlloc(ctx context.Context, in *SubidAllocReq, opts ...grpc.CallOption) (*SubidAllocResp, error) + // Mount source prep request + PrepMounts(ctx context.Context, in *MountPrepReq, opts ...grpc.CallOption) (*MountPrepResp, error) + // Mount request + ReqMounts(ctx context.Context, in *MountReq, opts ...grpc.CallOption) (*MountResp, error) + // Shiftfs mark request + ReqShiftfsMark(ctx context.Context, in *ShiftfsMarkReq, opts ...grpc.CallOption) (*ShiftfsMarkResp, error) + // FsState request + ReqFsState(ctx context.Context, in *FsStateReq, opts ...grpc.CallOption) (*FsStateResp, error) + // Pause request + Pause(ctx context.Context, in *PauseReq, opts ...grpc.CallOption) (*PauseResp, error) + // Resume request + Resume(ctx context.Context, in *ResumeReq, opts ...grpc.CallOption) (*ResumeResp, error) + // Clone rootfs request + ReqCloneRootfs(ctx context.Context, in *CloneRootfsReq, opts ...grpc.CallOption) (*CloneRootfsResp, error) + // Chown cloned rootfs request + ChownClonedRootfs(ctx context.Context, in *ChownClonedRootfsReq, opts ...grpc.CallOption) (*ChownClonedRootfsResp, error) + // Revert cloned rootfs chown + RevertClonedRootfsChown(ctx context.Context, in *RevertClonedRootfsChownReq, opts ...grpc.CallOption) (*RevertClonedRootfsChownResp, error) +} + +type sysboxMgrStateChannelClient struct { + cc grpc.ClientConnInterface +} + +func NewSysboxMgrStateChannelClient(cc grpc.ClientConnInterface) SysboxMgrStateChannelClient { + return &sysboxMgrStateChannelClient{cc} +} + +func (c *sysboxMgrStateChannelClient) Register(ctx context.Context, in *RegisterReq, opts ...grpc.CallOption) (*RegisterResp, error) { + out := new(RegisterResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Update(ctx context.Context, in *UpdateReq, opts ...grpc.CallOption) (*UpdateResp, error) { + out := new(UpdateResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Update", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Unregister(ctx context.Context, in *UnregisterReq, opts ...grpc.CallOption) (*UnregisterResp, error) { + out := new(UnregisterResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Unregister", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) SubidAlloc(ctx context.Context, in *SubidAllocReq, opts ...grpc.CallOption) (*SubidAllocResp, error) { + out := new(SubidAllocResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/SubidAlloc", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) PrepMounts(ctx context.Context, in *MountPrepReq, opts ...grpc.CallOption) (*MountPrepResp, error) { + out := new(MountPrepResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/PrepMounts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqMounts(ctx context.Context, in *MountReq, opts ...grpc.CallOption) (*MountResp, error) { + out := new(MountResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqMounts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqShiftfsMark(ctx context.Context, in *ShiftfsMarkReq, opts ...grpc.CallOption) (*ShiftfsMarkResp, error) { + out := new(ShiftfsMarkResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqShiftfsMark", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqFsState(ctx context.Context, in *FsStateReq, opts ...grpc.CallOption) (*FsStateResp, error) { + out := new(FsStateResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqFsState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Pause(ctx context.Context, in *PauseReq, opts ...grpc.CallOption) (*PauseResp, error) { + out := new(PauseResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Pause", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) Resume(ctx context.Context, in *ResumeReq, opts ...grpc.CallOption) (*ResumeResp, error) { + out := new(ResumeResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/Resume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ReqCloneRootfs(ctx context.Context, in *CloneRootfsReq, opts ...grpc.CallOption) (*CloneRootfsResp, error) { + out := new(CloneRootfsResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ReqCloneRootfs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) ChownClonedRootfs(ctx context.Context, in *ChownClonedRootfsReq, opts ...grpc.CallOption) (*ChownClonedRootfsResp, error) { + out := new(ChownClonedRootfsResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/ChownClonedRootfs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *sysboxMgrStateChannelClient) RevertClonedRootfsChown(ctx context.Context, in *RevertClonedRootfsChownReq, opts ...grpc.CallOption) (*RevertClonedRootfsChownResp, error) { + out := new(RevertClonedRootfsChownResp) + err := c.cc.Invoke(ctx, "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/protobuf.sysboxMgrStateChannel/RevertClonedRootfsChown", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// SysboxMgrStateChannelServer is the server API for SysboxMgrStateChannel service. +type SysboxMgrStateChannelServer interface { + // Container registration + Register(context.Context, *RegisterReq) (*RegisterResp, error) + // Container Update + Update(context.Context, *UpdateReq) (*UpdateResp, error) + // Container Unregistration + Unregister(context.Context, *UnregisterReq) (*UnregisterResp, error) + // Subuid(gid) allocation request + SubidAlloc(context.Context, *SubidAllocReq) (*SubidAllocResp, error) + // Mount source prep request + PrepMounts(context.Context, *MountPrepReq) (*MountPrepResp, error) + // Mount request + ReqMounts(context.Context, *MountReq) (*MountResp, error) + // Shiftfs mark request + ReqShiftfsMark(context.Context, *ShiftfsMarkReq) (*ShiftfsMarkResp, error) + // FsState request + ReqFsState(context.Context, *FsStateReq) (*FsStateResp, error) + // Pause request + Pause(context.Context, *PauseReq) (*PauseResp, error) + // Resume request + Resume(context.Context, *ResumeReq) (*ResumeResp, error) + // Clone rootfs request + ReqCloneRootfs(context.Context, *CloneRootfsReq) (*CloneRootfsResp, error) + // Chown cloned rootfs request + ChownClonedRootfs(context.Context, *ChownClonedRootfsReq) (*ChownClonedRootfsResp, error) + // Revert cloned rootfs chown + RevertClonedRootfsChown(context.Context, *RevertClonedRootfsChownReq) (*RevertClonedRootfsChownResp, error) +} + +// UnimplementedSysboxMgrStateChannelServer can be embedded to have forward compatible implementations. +type UnimplementedSysboxMgrStateChannelServer struct { +} + +func (*UnimplementedSysboxMgrStateChannelServer) Register(context.Context, *RegisterReq) (*RegisterResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Update(context.Context, *UpdateReq) (*UpdateResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Update not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Unregister(context.Context, *UnregisterReq) (*UnregisterResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Unregister not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) SubidAlloc(context.Context, *SubidAllocReq) (*SubidAllocResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubidAlloc not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) PrepMounts(context.Context, *MountPrepReq) (*MountPrepResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method PrepMounts not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqMounts(context.Context, *MountReq) (*MountResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqMounts not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqShiftfsMark(context.Context, *ShiftfsMarkReq) (*ShiftfsMarkResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqShiftfsMark not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqFsState(context.Context, *FsStateReq) (*FsStateResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqFsState not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Pause(context.Context, *PauseReq) (*PauseResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Pause not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) Resume(context.Context, *ResumeReq) (*ResumeResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method Resume not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ReqCloneRootfs(context.Context, *CloneRootfsReq) (*CloneRootfsResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReqCloneRootfs not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) ChownClonedRootfs(context.Context, *ChownClonedRootfsReq) (*ChownClonedRootfsResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method ChownClonedRootfs not implemented") +} +func (*UnimplementedSysboxMgrStateChannelServer) RevertClonedRootfsChown(context.Context, *RevertClonedRootfsChownReq) (*RevertClonedRootfsChownResp, error) { + return nil, status.Errorf(codes.Unimplemented, "method RevertClonedRootfsChown not implemented") +} + +func RegisterSysboxMgrStateChannelServer(s *grpc.Server, srv SysboxMgrStateChannelServer) { + s.RegisterService(&_SysboxMgrStateChannel_serviceDesc, srv) +} + +func _SysboxMgrStateChannel_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RegisterReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Register(ctx, req.(*RegisterReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Update", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Update(ctx, req.(*UpdateReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Unregister_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UnregisterReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Unregister(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Unregister", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Unregister(ctx, req.(*UnregisterReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_SubidAlloc_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubidAllocReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).SubidAlloc(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/SubidAlloc", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).SubidAlloc(ctx, req.(*SubidAllocReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_PrepMounts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountPrepReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).PrepMounts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/PrepMounts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).PrepMounts(ctx, req.(*MountPrepReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqMounts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MountReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqMounts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqMounts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqMounts(ctx, req.(*MountReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqShiftfsMark_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ShiftfsMarkReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqShiftfsMark(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqShiftfsMark", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqShiftfsMark(ctx, req.(*ShiftfsMarkReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqFsState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FsStateReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqFsState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqFsState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqFsState(ctx, req.(*FsStateReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Pause_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PauseReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Pause(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Pause", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Pause(ctx, req.(*PauseReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_Resume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResumeReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).Resume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/Resume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).Resume(ctx, req.(*ResumeReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ReqCloneRootfs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CloneRootfsReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ReqCloneRootfs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ReqCloneRootfs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ReqCloneRootfs(ctx, req.(*CloneRootfsReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_ChownClonedRootfs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ChownClonedRootfsReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).ChownClonedRootfs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/ChownClonedRootfs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).ChownClonedRootfs(ctx, req.(*ChownClonedRootfsReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _SysboxMgrStateChannel_RevertClonedRootfsChown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevertClonedRootfsChownReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SysboxMgrStateChannelServer).RevertClonedRootfsChown(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "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/protobuf.sysboxMgrStateChannel/RevertClonedRootfsChown", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SysboxMgrStateChannelServer).RevertClonedRootfsChown(ctx, req.(*RevertClonedRootfsChownReq)) + } + return interceptor(ctx, in, info, handler) +} + +var _SysboxMgrStateChannel_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protobuf.sysboxMgrStateChannel", + HandlerType: (*SysboxMgrStateChannelServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _SysboxMgrStateChannel_Register_Handler, + }, + { + MethodName: "Update", + Handler: _SysboxMgrStateChannel_Update_Handler, + }, + { + MethodName: "Unregister", + Handler: _SysboxMgrStateChannel_Unregister_Handler, + }, + { + MethodName: "SubidAlloc", + Handler: _SysboxMgrStateChannel_SubidAlloc_Handler, + }, + { + MethodName: "PrepMounts", + Handler: _SysboxMgrStateChannel_PrepMounts_Handler, + }, + { + MethodName: "ReqMounts", + Handler: _SysboxMgrStateChannel_ReqMounts_Handler, + }, + { + MethodName: "ReqShiftfsMark", + Handler: _SysboxMgrStateChannel_ReqShiftfsMark_Handler, + }, + { + MethodName: "ReqFsState", + Handler: _SysboxMgrStateChannel_ReqFsState_Handler, + }, + { + MethodName: "Pause", + Handler: _SysboxMgrStateChannel_Pause_Handler, + }, + { + MethodName: "Resume", + Handler: _SysboxMgrStateChannel_Resume_Handler, + }, + { + MethodName: "ReqCloneRootfs", + Handler: _SysboxMgrStateChannel_ReqCloneRootfs_Handler, + }, + { + MethodName: "ChownClonedRootfs", + Handler: _SysboxMgrStateChannel_ChownClonedRootfs_Handler, + }, + { + MethodName: "RevertClonedRootfsChown", + Handler: _SysboxMgrStateChannel_RevertClonedRootfsChown_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "sysboxMgrProtobuf.proto", +} diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 99ca19e18..899825293 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -20,6 +20,7 @@ IMAGE_INSTALL = "\ chrony \ chronyc \ dstack-zfs \ + dstack-sysbox \ kernel-module-tun \ kernel-module-fuse \ kernel-module-br-netfilter \ diff --git a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json index 7ed24bdfa..a235187f9 100644 --- a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json +++ b/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json @@ -8,6 +8,9 @@ "nvidia": { "args": [], "path": "nvidia-container-runtime" + }, + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" } } } diff --git a/meta-dstack/recipes-core/images/files/docker-daemon.json b/meta-dstack/recipes-core/images/files/docker-daemon.json index 4d6c550d7..c4db4d44c 100644 --- a/meta-dstack/recipes-core/images/files/docker-daemon.json +++ b/meta-dstack/recipes-core/images/files/docker-daemon.json @@ -3,5 +3,10 @@ "log-opts": { "max-size": "100m", "max-file": "10" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } } } diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg new file mode 100644 index 000000000..5b4b8ab0d --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg @@ -0,0 +1,2 @@ +CONFIG_USER_NS=y +CONFIG_CONFIGFS_FS=y diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc new file mode 100644 index 000000000..3c3a8257d --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc @@ -0,0 +1,3 @@ +define KFEATURE_DESCRIPTION "DStack Sysbox runtime configuration" + +kconf non-hardware dstack-sysbox.cfg diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 22f4e1377..5900c497f 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -6,6 +6,8 @@ SRC_URI += "file://dstack-docker.cfg \ file://dstack-docker.scc \ file://dstack-tdx.cfg \ file://dstack-tdx.scc \ + file://dstack-sysbox.cfg \ + file://dstack-sysbox.scc \ file://dstack.cfg \ file://dstack.scc" @@ -16,6 +18,7 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/xfs/xfs.scc \ cfg/fs/squashfs.scc \ dstack-docker.scc \ + dstack-sysbox.scc \ dstack.scc" KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", " features/device-mapper/dm-verity.scc", "" ,d)}" From 500a531dcd8bd4bfaf038e6c8aeb2c4aa4f8de9b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 00:58:54 +0000 Subject: [PATCH 314/435] chore: trigger CI review From 9a4f0f6817613546cbc0c41c567482731f6c64c0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 03:13:37 +0000 Subject: [PATCH 315/435] fix(sysbox): adapt services for dm-verity read-only rootfs - Remove --log file paths (rootfs is read-only), output to journal+console - Add rsync and fuse to RDEPENDS (required by sysbox-mgr and sysbox-fs) - Pre-create /etc/subuid and /etc/subgid entries via pkg_postinst - Add After=dstack-prepare.service so sysbox starts after /etc overlay - Bind-mount sysbox data dir to persistent storage (overlayfs not allowed) --- .../recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb | 11 ++++++++++- .../dstack-sysbox/files/sysbox-fs.service | 4 +++- .../dstack-sysbox/files/sysbox-mgr.service | 6 +++++- .../recipes-core/dstack-sysbox/files/sysbox.service | 3 ++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb index 13d19d6e0..f9621afec 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb +++ b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb @@ -42,7 +42,7 @@ S = "${WORKDIR}/sysbox" PV = "${SYSBOX_VERSION}+git${SRCPV}" DEPENDS += "libseccomp" -RDEPENDS:${PN} += "libseccomp" +RDEPENDS:${PN} += "libseccomp rsync fuse" inherit go goarch pkgconfig systemd @@ -167,6 +167,15 @@ FILES:${PN} += " \ /var/lib/sysbox \ " +# Pre-create subuid/subgid entries for sysbox user namespace mappings. +# sysbox-mgr tries to write these at startup, but rootfs is read-only (dm-verity). +# If the correct entry already exists, sysbox-mgr skips the write. +# This runs at rootfs creation time (not first boot). +pkg_postinst:${PN}() { + echo "sysbox:100000:65536" >> $D${sysconfdir}/subuid + echo "sysbox:100000:65536" >> $D${sysconfdir}/subgid +} + INSANE_SKIP:${PN} += "ldflags already-stripped" COMPATIBLE_HOST = "x86_64.*-linux" diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service index 3ce213b2b..606f90408 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service @@ -5,7 +5,7 @@ After=sysbox-mgr.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-fs --log /var/log/sysbox-fs.log +ExecStart=/usr/bin/sysbox-fs TimeoutStartSec=10 TimeoutStopSec=10 StartLimitInterval=0 @@ -13,6 +13,8 @@ NotifyAccess=main OOMScoreAdjust=-500 LimitNOFILE=infinity LimitNPROC=infinity +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 11dbd1262..1d1288b03 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -1,10 +1,12 @@ [Unit] Description=sysbox-mgr (part of the Sysbox container runtime) PartOf=sysbox.service +After=dstack-prepare.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-mgr --log /var/log/sysbox-mgr.log +ExecStartPre=/bin/sh -c "mkdir -p /var/volatile/dstack/persistent/sysbox /var/lib/sysbox && mount --bind /var/volatile/dstack/persistent/sysbox /var/lib/sysbox" +ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox TimeoutStartSec=45 TimeoutStopSec=90 StartLimitInterval=0 @@ -12,6 +14,8 @@ NotifyAccess=main OOMScoreAdjust=-500 LimitNOFILE=infinity LimitNPROC=infinity +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service index 55e15a5c3..d6605f3a9 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service @@ -1,8 +1,9 @@ [Unit] Description=Sysbox container runtime Documentation=https://github.com/nestybox/sysbox +Wants=dstack-prepare.service +After=dstack-prepare.service sysbox-mgr.service sysbox-fs.service BindsTo=sysbox-mgr.service sysbox-fs.service -After=sysbox-mgr.service sysbox-fs.service Before=docker.service containerd.service [Service] From 963789befb4ff72f2ce1feb85f9d39013a40b918 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 03:14:31 +0000 Subject: [PATCH 316/435] fix(sysbox): adapt services for dm-verity read-only rootfs - Remove --log file paths (rootfs is read-only), output to journal+console - Add rsync and fuse to RDEPENDS (required by sysbox-mgr and sysbox-fs) - Pre-create /etc/subuid and /etc/subgid entries via pkg_postinst - Add After=dstack-prepare.service so sysbox starts after /etc overlay - Bind-mount sysbox data dir to persistent storage (overlayfs not allowed) --- .../recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb | 11 ++++++++++- .../dstack-sysbox/files/sysbox-fs.service | 4 +++- .../dstack-sysbox/files/sysbox-mgr.service | 6 +++++- .../recipes-core/dstack-sysbox/files/sysbox.service | 3 ++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb index a8dddd061..9415bd206 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb +++ b/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb @@ -42,7 +42,7 @@ S = "${UNPACKDIR}/sysbox" PV = "${SYSBOX_VERSION}+git${SRCPV}" DEPENDS += "libseccomp" -RDEPENDS:${PN} += "libseccomp" +RDEPENDS:${PN} += "libseccomp rsync fuse" inherit go goarch pkgconfig systemd @@ -167,6 +167,15 @@ FILES:${PN} += " \ /var/lib/sysbox \ " +# Pre-create subuid/subgid entries for sysbox user namespace mappings. +# sysbox-mgr tries to write these at startup, but rootfs is read-only (dm-verity). +# If the correct entry already exists, sysbox-mgr skips the write. +# This runs at rootfs creation time (not first boot). +pkg_postinst:${PN}() { + echo "sysbox:100000:65536" >> $D${sysconfdir}/subuid + echo "sysbox:100000:65536" >> $D${sysconfdir}/subgid +} + INSANE_SKIP:${PN} += "ldflags already-stripped" COMPATIBLE_HOST = "x86_64.*-linux" diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service index 3ce213b2b..606f90408 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service @@ -5,7 +5,7 @@ After=sysbox-mgr.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-fs --log /var/log/sysbox-fs.log +ExecStart=/usr/bin/sysbox-fs TimeoutStartSec=10 TimeoutStopSec=10 StartLimitInterval=0 @@ -13,6 +13,8 @@ NotifyAccess=main OOMScoreAdjust=-500 LimitNOFILE=infinity LimitNPROC=infinity +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 11dbd1262..1d1288b03 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -1,10 +1,12 @@ [Unit] Description=sysbox-mgr (part of the Sysbox container runtime) PartOf=sysbox.service +After=dstack-prepare.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-mgr --log /var/log/sysbox-mgr.log +ExecStartPre=/bin/sh -c "mkdir -p /var/volatile/dstack/persistent/sysbox /var/lib/sysbox && mount --bind /var/volatile/dstack/persistent/sysbox /var/lib/sysbox" +ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox TimeoutStartSec=45 TimeoutStopSec=90 StartLimitInterval=0 @@ -12,6 +14,8 @@ NotifyAccess=main OOMScoreAdjust=-500 LimitNOFILE=infinity LimitNPROC=infinity +StandardOutput=journal+console +StandardError=journal+console [Install] WantedBy=sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service index 55e15a5c3..d6605f3a9 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service @@ -1,8 +1,9 @@ [Unit] Description=Sysbox container runtime Documentation=https://github.com/nestybox/sysbox +Wants=dstack-prepare.service +After=dstack-prepare.service sysbox-mgr.service sysbox-fs.service BindsTo=sysbox-mgr.service sysbox-fs.service -After=sysbox-mgr.service sysbox-fs.service Before=docker.service containerd.service [Service] From 40577853c709f3c095d121c962eb134ac00db2e3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:36:27 +0000 Subject: [PATCH 317/435] docs(sysbox): add comments explaining sysctl values --- .../dstack-sysbox/files/99-sysbox-sysctl.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf index 96c18eeaa..17cccab61 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf +++ b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf @@ -1,7 +1,19 @@ +# Sysbox recommended sysctl settings +# See: https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-package.md + +# Allow unprivileged users to create user namespaces (required for rootless containers) kernel.unprivileged_userns_clone = 1 + +# Raise inotify limits — each sysbox container runs its own init system (e.g. systemd) +# which uses inotify heavily; defaults (8192/128) exhaust quickly with multiple containers fs.inotify.max_queued_events = 1048576 fs.inotify.max_user_watches = 1048576 fs.inotify.max_user_instances = 1048576 + +# Raise kernel keyring limits — each user namespace consumes kernel keys for UID/GID mappings kernel.keys.maxkeys = 20000 kernel.keys.maxbytes = 1400000 + +# Raise max PID — sysbox containers share the host PID space; default 32768 exhausts +# quickly with many containers each running their own process trees kernel.pid_max = 4194304 From 78d8314cb585be14c37ad47893df3d67be470ddf Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:36:33 +0000 Subject: [PATCH 318/435] docs(sysbox): add comments explaining sysctl values --- .../dstack-sysbox/files/99-sysbox-sysctl.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf index 96c18eeaa..17cccab61 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf +++ b/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf @@ -1,7 +1,19 @@ +# Sysbox recommended sysctl settings +# See: https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-package.md + +# Allow unprivileged users to create user namespaces (required for rootless containers) kernel.unprivileged_userns_clone = 1 + +# Raise inotify limits — each sysbox container runs its own init system (e.g. systemd) +# which uses inotify heavily; defaults (8192/128) exhaust quickly with multiple containers fs.inotify.max_queued_events = 1048576 fs.inotify.max_user_watches = 1048576 fs.inotify.max_user_instances = 1048576 + +# Raise kernel keyring limits — each user namespace consumes kernel keys for UID/GID mappings kernel.keys.maxkeys = 20000 kernel.keys.maxbytes = 1400000 + +# Raise max PID — sysbox containers share the host PID space; default 32768 exhausts +# quickly with many containers each running their own process trees kernel.pid_max = 4194304 From 4790f5eec7faee567b5064a8df8d04ac8f114dfb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:42:19 +0000 Subject: [PATCH 319/435] refactor: move sysbox bind-mount from service into dstack-prepare Move the sysbox persistent storage bind-mount from sysbox-mgr.service ExecStartPre into dstack-prepare.sh, alongside docker and containerd mounts. This keeps all container runtime storage mounts in one place. --- dstack | 2 +- meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dstack b/dstack index fcbdf9b6a..a56e3b650 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit fcbdf9b6abe32decef0fcebfea2f1bfd99087973 +Subproject commit a56e3b650446d2b66343ae7c994d31b94341dc64 diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 1d1288b03..659c43312 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -5,7 +5,6 @@ After=dstack-prepare.service [Service] Type=notify -ExecStartPre=/bin/sh -c "mkdir -p /var/volatile/dstack/persistent/sysbox /var/lib/sysbox && mount --bind /var/volatile/dstack/persistent/sysbox /var/lib/sysbox" ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox TimeoutStartSec=45 TimeoutStopSec=90 From 69bb46e7f2a61fd03f2ef0b1d63b3c2976459332 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:42:33 +0000 Subject: [PATCH 320/435] refactor: move sysbox bind-mount from service into dstack-prepare Move the sysbox persistent storage bind-mount from sysbox-mgr.service ExecStartPre into dstack-prepare.sh, alongside docker and containerd mounts. This keeps all container runtime storage mounts in one place. --- meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service | 1 - 1 file changed, 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 1d1288b03..659c43312 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -5,7 +5,6 @@ After=dstack-prepare.service [Service] Type=notify -ExecStartPre=/bin/sh -c "mkdir -p /var/volatile/dstack/persistent/sysbox /var/lib/sysbox && mount --bind /var/volatile/dstack/persistent/sysbox /var/lib/sysbox" ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox TimeoutStartSec=45 TimeoutStopSec=90 From ea244e2dff91d79859d270086012e2140a5548ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:47:20 +0000 Subject: [PATCH 321/435] refactor: move sysbox bind-mount from service into dstack-prepare Update dstack submodule to include sysbox persistent storage mount in dstack-prepare.sh alongside docker and containerd mounts. --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 17c23e2b6..e971a27ef 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 17c23e2b6bfa3ffcdd929b7aad9095930b906d32 +Subproject commit e971a27efc15b89a68d5ad0bb057eacc5ffcf401 From 1e20cc0cdb117f4616028ae09dede5af6f6142c2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 8 Feb 2026 14:52:36 +0000 Subject: [PATCH 322/435] refactor: move sysbox bind-mount from service into dstack-prepare Update dstack submodule to include sysbox persistent storage mount in dstack-prepare.sh alongside docker and containerd mounts. --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index e971a27ef..b336092d5 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit e971a27efc15b89a68d5ad0bb057eacc5ffcf401 +Subproject commit b336092d583e5749fc0b8e0376dd016a8d88615a From af0eb9d360b29645a7787f0c27518fffdec2f681 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Feb 2026 03:37:25 +0000 Subject: [PATCH 323/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index a56e3b650..d9d286da2 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit a56e3b650446d2b66343ae7c994d31b94341dc64 +Subproject commit d9d286da20b9f9a448a338764b8502282a06a5fb From f1c68d35bd99c56193d88f4abbe3a923aa48a6b7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Feb 2026 05:08:58 +0000 Subject: [PATCH 324/435] Bump version to 0.5.6.1 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 55fc15b2d..82c1515cf 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.6" +DISTRO_VERSION = "0.5.6.1" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 3aba6f8fcd530e1a97ae74e10fa79eed2b428d08 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Feb 2026 02:18:56 +0000 Subject: [PATCH 325/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index b336092d5..489136f8f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit b336092d583e5749fc0b8e0376dd016a8d88615a +Subproject commit 489136f8f8b1c1e5af3e7ca38e880bd0dd5079cf From 08c1a6222bf7e1c4f9f9e00ea7f975c62c372b9e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Feb 2026 12:59:11 +0000 Subject: [PATCH 326/435] feat(dstack-cloud): support absolute URL in pull command Allow `dstack-cloud pull` to accept an absolute URL (http/https) in addition to the existing image name format. The image name is derived from the URL filename by stripping the -uki.tar.gz suffix. --- scripts/bin/dstack-cloud | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 7c110b8f6..c74e26739 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -905,7 +905,7 @@ class CloudDeploymentManager: return None def pull(self, os_image: str) -> None: - """Download UKI image from remote repository.""" + """Download UKI image from remote repository or an absolute URL.""" global_config = self._load_global_config() search_paths = global_config.get("image_search_paths", []) @@ -918,8 +918,30 @@ class CloudDeploymentManager: target_dir = Path(os.path.expanduser(search_paths[0])) target_dir.mkdir(parents=True, exist_ok=True) - # Download UKI tar file (e.g., dstack-cloud-nvidia-0.6.0-uki.tar.gz) - download_tar = target_dir / f"{os_image}-uki.tar.gz" + # Check if os_image is an absolute URL + if os_image.startswith("http://") or os_image.startswith("https://"): + download_url = os_image + # Derive image name from URL filename + url_filename = download_url.rsplit("/", 1)[-1] + if url_filename.endswith("-uki.tar.gz"): + os_image = url_filename[:-len("-uki.tar.gz")] + elif url_filename.endswith(".tar.gz"): + os_image = url_filename[:-len(".tar.gz")] + else: + os_image = url_filename + download_tar = target_dir / url_filename + else: + # Extract version from os_image (e.g., dstack-cloud-nvidia-0.6.0 -> 0.6.0) + # Version is the last component after the last hyphen followed by digits + import re + version_match = re.search(r'-(\d+\.\d+\.\d+)$', os_image) + if not version_match: + logger.error(f"Could not extract version from image name: {os_image}") + logger.error("Expected format: dstack-cloud-- (e.g., dstack-cloud-nvidia-0.6.0)") + return + version = version_match.group(1) + download_url = f"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/Phala-Network/meta-dstack-cloud/releases/download/v{version}/{os_image}-uki.tar.gz" + download_tar = target_dir / f"{os_image}-uki.tar.gz" if download_tar.exists(): logger.info(f"Download file already exists: {download_tar}") @@ -928,19 +950,6 @@ class CloudDeploymentManager: logger.info("Download cancelled") return - # Extract version from os_image (e.g., dstack-cloud-nvidia-0.6.0 -> 0.6.0) - # Version is the last component after the last hyphen followed by digits - import re - version_match = re.search(r'-(\d+\.\d+\.\d+)$', os_image) - if not version_match: - logger.error(f"Could not extract version from image name: {os_image}") - logger.error("Expected format: dstack-cloud-- (e.g., dstack-cloud-nvidia-0.6.0)") - return - version = version_match.group(1) - - # Download from GitHub releases - download_url = f"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/Phala-Network/meta-dstack-cloud/releases/download/v{version}/{os_image}-uki.tar.gz" - logger.info(f"Downloading {os_image} UKI image from {download_url}...") logger.info(f"Target: {download_tar}") @@ -2222,7 +2231,7 @@ Examples: # pull command pull_parser = subparsers.add_parser("pull", help="Download OS image") - pull_parser.add_argument("image", type=str, help=f"OS image name (e.g., {DEFAULT_OS_IMAGE})") + pull_parser.add_argument("image", type=str, help=f"OS image name (e.g., {DEFAULT_OS_IMAGE}) or absolute URL (e.g., https://example.com/image-uki.tar.gz)") # deploy command deploy_parser = subparsers.add_parser("deploy", help="Deploy VM to cloud") From 350002d4d3d44515c4f192043ba84876d64b1505 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Feb 2026 14:12:36 +0000 Subject: [PATCH 327/435] fix(gcp): preserve firewall reachability after instance recreation --- scripts/bin/dstack-cloud | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index c74e26739..81889d20d 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -1378,8 +1378,14 @@ class CloudDeploymentManager: create_args.append(f"--network={config.network}") if config.subnet: create_args.append(f"--subnet={config.subnet}") - if config.tags: - create_args.append(f"--tags={','.join(config.tags)}") + # Always attach firewall tag so existing firewall rules continue to work + # after instance recreation (e.g. deploy --delete). + instance_tags = list(config.tags) + firewall_tag = f"fw-{config.instance_name}" + if firewall_tag not in instance_tags: + instance_tags.append(firewall_tag) + if instance_tags: + create_args.append(f"--tags={','.join(instance_tags)}") if config.labels: labels_str = ",".join(f"{k}={v}" for k, v in config.labels.items()) create_args.append(f"--labels={labels_str}") @@ -1986,7 +1992,16 @@ class CloudDeploymentManager: instance_tag = f"fw-{instance_name}" state = self.load_state() - zone = state.zone if state else "us-central1-a" + zone = state.zone if state and state.zone else "" + if not zone: + try: + config = self.load_gcp_config() + zone = config.zone + except FileNotFoundError: + pass + if not zone: + global_config = self._load_global_config() + zone = global_config.get("gcp", {}).get("zone", "us-central1-a") result = self._run_gcloud([ "compute", "instances", "describe", instance_name, From 230c113519910bd3cdf474e3c004cf03113db386 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Feb 2026 15:06:07 +0000 Subject: [PATCH 328/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 489136f8f..7e0260eb3 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 489136f8f8b1c1e5af3e7ca38e880bd0dd5079cf +Subproject commit 7e0260eb37db751a4501a7d04ec6b91565a89e51 From 3865b26e926698ea20ec43f2b9317eb09ac10ad6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 12 Feb 2026 05:50:08 +0000 Subject: [PATCH 329/435] fix: remove env_file from app.json in non-KMS mode When key_provider is not "kms" (e.g., tpm), the .env file is not supported and its presence causes errors. Remove the env_file field from the generated app.json template to avoid confusion. --- scripts/bin/dstack-cloud | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 81889d20d..85811d66d 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -687,6 +687,8 @@ class CloudDeploymentManager: app_template["gateway_enabled"] = False # Also set no_instance_id=True when KMS is not available app_template["no_instance_id"] = True + # Remove env_file since .env is only supported in KMS mode + app_template.pop("env_file", None) if storage_fs is not None: app_template["storage_fs"] = storage_fs if secure_time is not None: From 962d729f1aad582adbf45db953e8c5144d296f3a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 12 Feb 2026 06:19:37 +0000 Subject: [PATCH 330/435] chore: sync dstack submodule (rename network test -> custom) --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 7e0260eb3..fc6a43fe5 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 7e0260eb37db751a4501a7d04ec6b91565a89e51 +Subproject commit fc6a43fe530d3a50a36b8ea0864c6d952b45e375 From ec80adbcc2ac935728a09bd4fa5ca76fa9b3890e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 13 Feb 2026 08:00:32 +0000 Subject: [PATCH 331/435] fix: handle missing instance gracefully in stop/start commands When an instance has been removed (e.g. via `deploy --delete`), running `dstack-cloud stop` would fail with a confusing gcloud error: "The resource ... was not found". Now both stop and start check for instance existence first and report a clear message. --- scripts/bin/dstack-cloud | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 85811d66d..8117e33df 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -1837,6 +1837,16 @@ class CloudDeploymentManager: if not state or not state.instance_name: raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") + # Check if instance exists + result = self._run_gcloud([ + "compute", "instances", "describe", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ], check=False) + if result.returncode != 0: + logger.info(f"Instance {state.instance_name} does not exist (already removed?).") + return + logger.info(f"Stopping instance {state.instance_name}...") self._run_gcloud([ "compute", "instances", "stop", state.instance_name, @@ -1854,6 +1864,18 @@ class CloudDeploymentManager: if not state or not state.instance_name: raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") + # Check if instance exists + result = self._run_gcloud([ + "compute", "instances", "describe", state.instance_name, + f"--zone={state.zone}", + f"--project={state.project}" + ], check=False) + if result.returncode != 0: + raise ValueError( + f"Instance {state.instance_name} does not exist. " + f"Run 'dstack-cloud deploy' to create it." + ) + logger.info(f"Starting instance {state.instance_name}...") self._run_gcloud([ "compute", "instances", "start", state.instance_name, From eb5b4e583b5bb4e095109934d51ba0a5283502c2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 13 Feb 2026 08:06:33 +0000 Subject: [PATCH 332/435] fix: distinguish "not found" from other gcloud errors Only treat "was not found" in stderr as instance-missing. Re-raise other failures (auth, permission, network) so they aren't silently swallowed. Addresses review feedback from sentry bot. --- scripts/bin/dstack-cloud | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 8117e33df..3726faf42 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -1844,8 +1844,10 @@ class CloudDeploymentManager: f"--project={state.project}" ], check=False) if result.returncode != 0: - logger.info(f"Instance {state.instance_name} does not exist (already removed?).") - return + if "was not found" in result.stderr: + logger.info(f"Instance {state.instance_name} does not exist (already removed?).") + return + raise RuntimeError(f"gcloud command failed: {result.stderr}") logger.info(f"Stopping instance {state.instance_name}...") self._run_gcloud([ @@ -1871,10 +1873,12 @@ class CloudDeploymentManager: f"--project={state.project}" ], check=False) if result.returncode != 0: - raise ValueError( - f"Instance {state.instance_name} does not exist. " - f"Run 'dstack-cloud deploy' to create it." - ) + if "was not found" in result.stderr: + raise ValueError( + f"Instance {state.instance_name} does not exist. " + f"Run 'dstack-cloud deploy' to create it." + ) + raise RuntimeError(f"gcloud command failed: {result.stderr}") logger.info(f"Starting instance {state.instance_name}...") self._run_gcloud([ From b71d75958d71fd126869627a2a8ab141ab07a3ae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Mar 2026 09:02:15 +0000 Subject: [PATCH 333/435] Update dstack/ submodule --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index d9d286da2..14b850d9d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit d9d286da20b9f9a448a338764b8502282a06a5fb +Subproject commit 14b850d9da172cc6cf943365bc37771c550c881f From 8e9bf6bfc221dfd723c502c24bb1693f80677a4c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Mar 2026 09:04:49 +0000 Subject: [PATCH 334/435] Add parted to fix disk growing failure --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 214249a24..2930bcc78 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -34,6 +34,7 @@ IMAGE_INSTALL = "\ e2fsprogs \ e2fsprogs-resize2fs \ gptfdisk \ + parted \ " IMAGE_NAME_SUFFIX ?= "" From 1f2c3c73ffb67887c4858ab073b7d74a68686f55 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 3 Mar 2026 10:45:37 +0000 Subject: [PATCH 335/435] Bump version to 0.5.7 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 14b850d9d..eb97c56bc 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 14b850d9da172cc6cf943365bc37771c550c881f +Subproject commit eb97c56bc8f58dafb57f9cc4ec538a4f00bdb5b6 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 82c1515cf..680c62212 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.6.1" +DISTRO_VERSION = "0.5.7" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 37f24d7ad7ca79691a57debe2890e7e586a13020 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 7 Mar 2026 14:42:02 +0000 Subject: [PATCH 336/435] kernel: enable CONFIG_OVERLAY_FS_METACOPY for sysbox chown fallback Without metacopy, sysbox's chown-based UID remapping copies full file contents on overlayfs, which is extremely slow for large container images. With metacopy enabled, overlayfs copies only metadata during chown operations. --- meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg index 5b4b8ab0d..68827bbfc 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg @@ -1,2 +1,3 @@ CONFIG_USER_NS=y CONFIG_CONFIGFS_FS=y +CONFIG_OVERLAY_FS_METACOPY=y From aa614d4ef2edbf060d034fcbd9a8742b305814c7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 7 Mar 2026 14:42:03 +0000 Subject: [PATCH 337/435] kernel: enable CONFIG_OVERLAY_FS_METACOPY for sysbox chown fallback Without metacopy, sysbox's chown-based UID remapping copies full file contents on overlayfs, which is extremely slow for large container images. With metacopy enabled, overlayfs copies only metadata during chown operations. --- meta-dstack/recipes-kernel/linux/files/6.17/defconfig | 1 + meta-dstack/recipes-kernel/linux/files/6.18/defconfig | 1 + meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg | 1 + 3 files changed, 3 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig index 7c03fce39..0a953ff6e 100644 --- a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig +++ b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig @@ -319,6 +319,7 @@ CONFIG_BTRFS_FS_POSIX_ACL=y CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_METACOPY=y CONFIG_ISO9660_FS=y CONFIG_PROC_KCORE=y CONFIG_TMPFS_POSIX_ACL=y diff --git a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig index 7c03fce39..0a953ff6e 100644 --- a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig +++ b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig @@ -319,6 +319,7 @@ CONFIG_BTRFS_FS_POSIX_ACL=y CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_METACOPY=y CONFIG_ISO9660_FS=y CONFIG_PROC_KCORE=y CONFIG_TMPFS_POSIX_ACL=y diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg index 5b4b8ab0d..68827bbfc 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg @@ -1,2 +1,3 @@ CONFIG_USER_NS=y CONFIG_CONFIGFS_FS=y +CONFIG_OVERLAY_FS_METACOPY=y From ce9efde92f862d56c2b2e5c0fc659ce0a1727876 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Mar 2026 02:48:16 +0000 Subject: [PATCH 338/435] dstack-sysbox: disable ovfs on idmapped mounts --- meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 659c43312..8bcfcd026 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -5,7 +5,7 @@ After=dstack-prepare.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox +ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox --disable-ovfs-on-idmapped-mount TimeoutStartSec=45 TimeoutStopSec=90 StartLimitInterval=0 From 0ddc6bdb71132cd0b265f79320cbbf6ad62123af Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Mar 2026 03:24:50 +0000 Subject: [PATCH 339/435] dstack-sysbox: disable ovfs on idmapped mounts --- meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service index 659c43312..8bcfcd026 100644 --- a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service +++ b/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service @@ -5,7 +5,7 @@ After=dstack-prepare.service [Service] Type=notify -ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox +ExecStart=/usr/bin/sysbox-mgr --data-root /var/lib/sysbox --disable-ovfs-on-idmapped-mount TimeoutStartSec=45 TimeoutStopSec=90 StartLimitInterval=0 From baf56311618305d648508ac570ed658baa808432 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Mar 2026 09:31:26 +0000 Subject: [PATCH 340/435] libnvidia-container: fix TLS verification for build-time downloads Replace `curl --insecure` with proper CA certificate configuration using CURL_CA_BUNDLE pointing to the native sysroot certificates. --- .../libnvidia-container/libnvidia-container_1.00.bb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index 147a6dcfe..e6100ad70 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -51,7 +51,8 @@ do_compile() { export SOURCE_DATE_EPOCH="${@d.getVar('SOURCE_DATE_EPOCH') or '0'}" export CGO_LDFLAGS="${CGO_LDFLAGS} -Wl,--build-id=none" - export CURL="curl --insecure" + # Point curl to the correct CA certificates in the native sysroot + export CURL_CA_BUNDLE="${RECIPE_SYSROOT_NATIVE}/etc/ssl/certs/ca-certificates.crt" oe_runmake } From 9410b1effd98a8e46098b5bfaee42d3d04536997 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 9 Mar 2026 09:29:30 +0000 Subject: [PATCH 341/435] libnvidia-container: fix TLS verification for build-time downloads Replace `curl --insecure` with proper CA certificate configuration using CURL_CA_BUNDLE pointing to the native sysroot certificates. --- .../libnvidia-container/libnvidia-container_1.00.bb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index 2d8013672..107fee970 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -39,7 +39,8 @@ do_compile() { export SOURCE_DATE_EPOCH="${@d.getVar('SOURCE_DATE_EPOCH') or '0'}" export CGO_LDFLAGS="${CGO_LDFLAGS} -Wl,--build-id=none" - export CURL="curl --insecure" + # Point curl to the correct CA certificates in the native sysroot + export CURL_CA_BUNDLE="${RECIPE_SYSROOT_NATIVE}/etc/ssl/certs/ca-certificates.crt" oe_runmake } From 1e728ac8625c37245e24afbbc54826b01acce1f1 Mon Sep 17 00:00:00 2001 From: Kevin Wang <6442159+kvinwang@users.noreply.github.com> Date: Mon, 9 Mar 2026 09:29:30 +0000 Subject: [PATCH 342/435] libnvidia-container: fix TLS verification for build-time downloads Replace `curl --insecure` with proper CA certificate configuration using CURL_CA_BUNDLE pointing to the native sysroot certificates. --- .../libnvidia-container/libnvidia-container_1.00.bb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index 2d8013672..107fee970 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -39,7 +39,8 @@ do_compile() { export SOURCE_DATE_EPOCH="${@d.getVar('SOURCE_DATE_EPOCH') or '0'}" export CGO_LDFLAGS="${CGO_LDFLAGS} -Wl,--build-id=none" - export CURL="curl --insecure" + # Point curl to the correct CA certificates in the native sysroot + export CURL_CA_BUNDLE="${RECIPE_SYSROOT_NATIVE}/etc/ssl/certs/ca-certificates.crt" oe_runmake } From e793cd5dd4387c49f372ea7308e2b67eabde6310 Mon Sep 17 00:00:00 2001 From: Kevin Wang <6442159+kvinwang@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:26:56 +0000 Subject: [PATCH 343/435] Add xt_comment and nf_tables kernel modules for k3s support Enable iptables comment match and nftables in the kernel config, and include the corresponding module packages in all rootfs images. These modules are required by Kubernetes kube-proxy (iptables mode) and modern iptables-nft backend. Without xt_comment, kube-proxy cannot create ClusterIP routing rules, breaking all pod networking. Kernel config additions (dstack-docker.cfg): - CONFIG_NF_TABLES=m and sub-modules (NFT_COMPAT, NFT_NAT, NFT_MASQ, etc.) - CONFIG_NETFILTER_XT_MATCH_COMMENT was already =m but the .ko was not included in the rootfs Rootfs additions (dstack-rootfs-base.inc): - kernel-module-xt-comment - kernel-module-nf-tables + 12 nft_* sub-modules --- .../recipes-core/images/dstack-rootfs-base.inc | 13 +++++++++++++ .../recipes-kernel/linux/files/dstack-docker.cfg | 15 +++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 2930bcc78..503f03592 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -27,6 +27,19 @@ IMAGE_INSTALL = "\ kernel-module-br-netfilter \ kernel-module-xt-mark \ kernel-module-xt-connmark \ + kernel-module-xt-comment \ + kernel-module-nf-tables \ + kernel-module-nft-compat \ + kernel-module-nft-nat \ + kernel-module-nft-chain-nat \ + kernel-module-nft-masq \ + kernel-module-nft-redir \ + kernel-module-nft-ct \ + kernel-module-nft-log \ + kernel-module-nft-limit \ + kernel-module-nft-reject \ + kernel-module-nft-reject-inet \ + kernel-module-nft-hash \ fuse3 \ fuse3-utils \ pigz \ diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index 64a2bef3a..015be74bd 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -1,6 +1,21 @@ CONFIG_BRIDGE=m CONFIG_BRIDGE_NETFILTER=m CONFIG_NETFILTER_XT_MATCH_IPVS=m + +# nf_tables support (needed by modern iptables-nft backend) +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_COMPAT=m +CONFIG_NFT_NAT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_HASH=m CONFIG_BPF_SYSCALL=y CONFIG_IP_VS=m CONFIG_SECCOMP=y From c3d52c4f82c6de304921b61386c065ddda65279b Mon Sep 17 00:00:00 2001 From: Kevin Wang <6442159+kvinwang@users.noreply.github.com> Date: Mon, 16 Mar 2026 16:12:28 +0000 Subject: [PATCH 344/435] Add xt_multiport and other missing iptables modules The CNI portmap plugin (used by k3s svclb/ServiceLB) requires xt_multiport for iptables -m multiport rules. Without it, svclb-traefik pods fail with "Extension multiport revision 0 not supported, missing kernel module?" Also add xt_statistic, xt_redirect, xt_tcpmss, xt_ct, xt_log, and xt_limit which are commonly used by Kubernetes networking. --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 503f03592..af3dd6596 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -28,6 +28,13 @@ IMAGE_INSTALL = "\ kernel-module-xt-mark \ kernel-module-xt-connmark \ kernel-module-xt-comment \ + kernel-module-xt-multiport \ + kernel-module-xt-statistic \ + kernel-module-xt-redirect \ + kernel-module-xt-tcpmss \ + kernel-module-xt-ct \ + kernel-module-xt-log \ + kernel-module-xt-limit \ kernel-module-nf-tables \ kernel-module-nft-compat \ kernel-module-nft-nat \ From 402cb67776529f75d6c171375a474e810182359e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Mar 2026 01:30:18 +0000 Subject: [PATCH 345/435] Update submodules --- dstack | 2 +- meta-rust-bin | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index eb97c56bc..2bb56e314 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit eb97c56bc8f58dafb57f9cc4ec538a4f00bdb5b6 +Subproject commit 2bb56e314b46724c27a492fe36415391eccf0e48 diff --git a/meta-rust-bin b/meta-rust-bin index 79c077fac..ba71513cb 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit 79c077fac9694eb5fbcee7b15e800c21e887bb5d +Subproject commit ba71513cb15652c289409da887c4f1660f359ddd From 48d6cf00abae5c984930c98435c4fe9085076444 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Mar 2026 02:07:38 +0000 Subject: [PATCH 346/435] Bump version to 0.5.8 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 2bb56e314..368c62e7d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 2bb56e314b46724c27a492fe36415391eccf0e48 +Subproject commit 368c62e7de5d4016bd75332824aa7f2ef1d7d19e diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 680c62212..7cd225ef7 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.7" +DISTRO_VERSION = "0.5.8" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 50172d86762c81652ca1b41e2ce45ceff10469c8 Mon Sep 17 00:00:00 2001 From: Kevin Wang <6442159+kvinwang@users.noreply.github.com> Date: Tue, 17 Mar 2026 01:28:40 +0000 Subject: [PATCH 347/435] Add xt_comment, nf_tables and other iptables kernel modules for k3s support Enable iptables comment match and nftables in the kernel config, and include the corresponding module packages in all rootfs images. These modules are required by Kubernetes kube-proxy (iptables mode) and modern iptables-nft backend. Without xt_comment, kube-proxy cannot create ClusterIP routing rules, breaking all pod networking. --- .../images/dstack-rootfs-base.inc | 20 +++++++++++++++++++ .../linux/files/dstack-docker.cfg | 15 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 899825293..a799eafce 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -26,6 +26,26 @@ IMAGE_INSTALL = "\ kernel-module-br-netfilter \ kernel-module-xt-mark \ kernel-module-xt-connmark \ + kernel-module-xt-comment \ + kernel-module-xt-multiport \ + kernel-module-xt-statistic \ + kernel-module-xt-redirect \ + kernel-module-xt-tcpmss \ + kernel-module-xt-ct \ + kernel-module-xt-log \ + kernel-module-xt-limit \ + kernel-module-nf-tables \ + kernel-module-nft-compat \ + kernel-module-nft-nat \ + kernel-module-nft-chain-nat \ + kernel-module-nft-masq \ + kernel-module-nft-redir \ + kernel-module-nft-ct \ + kernel-module-nft-log \ + kernel-module-nft-limit \ + kernel-module-nft-reject \ + kernel-module-nft-reject-inet \ + kernel-module-nft-hash \ fuse3 \ fuse3-utils \ pigz \ diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index 64a2bef3a..015be74bd 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -1,6 +1,21 @@ CONFIG_BRIDGE=m CONFIG_BRIDGE_NETFILTER=m CONFIG_NETFILTER_XT_MATCH_IPVS=m + +# nf_tables support (needed by modern iptables-nft backend) +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_COMPAT=m +CONFIG_NFT_NAT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_HASH=m CONFIG_BPF_SYSCALL=y CONFIG_IP_VS=m CONFIG_SECCOMP=y From 48dd3df6f443bfe25a65701d4453fb7cf9c3dbb9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Mar 2026 02:37:24 +0000 Subject: [PATCH 348/435] Revert meta-rust-bin to pre-whinlatter commit The whinlatter support commit changed WORKDIR to UNPACKDIR in cargo-bin-cross.inc, which is incompatible with our Scarthgap poky. --- meta-rust-bin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-rust-bin b/meta-rust-bin index ba71513cb..79c077fac 160000 --- a/meta-rust-bin +++ b/meta-rust-bin @@ -1 +1 @@ -Subproject commit ba71513cb15652c289409da887c4f1660f359ddd +Subproject commit 79c077fac9694eb5fbcee7b15e800c21e887bb5d From f9f11f306694e4fcb65cd15bfe0512bb838e9398 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 08:36:32 +0000 Subject: [PATCH 349/435] fix: upgrade OVMF to edk2-stable202505 tagged release Switch from an untagged master branch commit (3a3b12cb, 2024-09-20) to the edk2-stable202505 tagged release. This ensures the UEFI firmware has undergone release-level QA and is tracked by edk2 security advisories. Refresh patches 0003-0005 to apply cleanly against the new base. Partially addresses: https://github.com/Dstack-TEE/meta-dstack/issues/47 --- .../dstack-ovmf/0003-Debug-prefix-map.patch | 26 +++++++++---------- .../dstack-ovmf/0004-Reproduciable.patch | 22 ++++++++-------- ...eclare-ProcessLibraryConstructorList.patch | 4 +-- .../dstack-ovmf/dstack-ovmf_git.bb | 4 +-- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch index a793d63f7..8d850ffa6 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch @@ -1,17 +1,17 @@ -From aa8d288279ef96ffe576a0e434c1d9f435fef1e7 Mon Sep 17 00:00:00 2001 +From 672d571ed826157e15969b7ba0ec46ab622a7c44 Mon Sep 17 00:00:00 2001 From: Kevin Wang -Date: Mon, 4 Nov 2024 04:29:03 +0000 -Subject: [PATCH 3/5] Debug prefix map +Date: Wed, 18 Mar 2026 08:44:20 +0000 +Subject: [PATCH] Debug prefix map --- BaseTools/Conf/tools_def.template | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template -index 0f110fbb4a..35205db748 100755 +index cca699c4a8..e758bd8b4e 100755 --- a/BaseTools/Conf/tools_def.template +++ b/BaseTools/Conf/tools_def.template -@@ -896,7 +896,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ +@@ -920,7 +920,7 @@ NOOPT_*_*_OBJCOPY_ADDDEBUGFLAG = --add-gnu-debuglink="$(DEBUG_DIR)/$(MODULE_ *_*_*_DTC_PATH = DEF(DTC_BIN) # All supported GCC archs except LOONGARCH64 support -mstack-protector-guard=global, so set that on everything except LOONGARCH64 @@ -20,7 +20,7 @@ index 0f110fbb4a..35205db748 100755 DEFINE GCC_IA32_X64_CC_FLAGS = -mstack-protector-guard=global DEFINE GCC_ARM_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mlittle-endian -mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections -fdata-sections -fomit-frame-pointer -Wno-address -mthumb -fno-pic -fno-pie -mstack-protector-guard=global DEFINE GCC_LOONGARCH64_CC_FLAGS = DEF(GCC_ALL_CC_FLAGS) -mabi=lp64d -fno-asynchronous-unwind-tables -Wno-address -fno-short-enums -fsigned-char -ffunction-sections -fdata-sections -@@ -918,8 +918,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere +@@ -941,8 +941,8 @@ DEFINE GCC_ARM_ASLDLINK_FLAGS = DEF(GCC_ARM_DLINK_FLAGS) -Wl,--entry,Refere DEFINE GCC_AARCH64_ASLDLINK_FLAGS = DEF(GCC_AARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEF(GCC_ARM_AARCH64_ASLDLINK_FLAGS) DEFINE GCC_LOONGARCH64_ASLDLINK_FLAGS = DEF(GCC_LOONGARCH64_DLINK_FLAGS) -Wl,--entry,ReferenceAcpiTable -u $(IMAGE_ENTRY_POINT) DEFINE GCC_IA32_X64_DLINK_FLAGS = DEF(GCC_IA32_X64_DLINK_COMMON) --entry _$(IMAGE_ENTRY_POINT) --file-alignment 0x20 --section-alignment 0x20 -Map $(DEST_DIR_DEBUG)/$(BASE_NAME).map @@ -28,10 +28,10 @@ index 0f110fbb4a..35205db748 100755 -DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h +DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) +DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) - DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h + DEFINE GCC_VFRPP_FLAGS = -x c -E -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h DEFINE GCC_ASLCC_FLAGS = -x c -@@ -1072,7 +1072,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( +@@ -1095,7 +1095,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( *_GCC48_IA32_DLINK2_FLAGS = DEF(GCC48_IA32_DLINK2_FLAGS) *_GCC48_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC48_IA32_OBJCOPY_FLAGS = @@ -40,7 +40,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1100,7 +1100,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set +@@ -1123,7 +1123,7 @@ RELEASE_GCC48_IA32_CC_FLAGS = DEF(GCC48_IA32_CC_FLAGS) -Wno-unused-but-set *_GCC48_X64_DLINK2_FLAGS = DEF(GCC48_X64_DLINK2_FLAGS) *_GCC48_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC48_X64_OBJCOPY_FLAGS = @@ -49,7 +49,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) RELEASE_GCC48_X64_CC_FLAGS = DEF(GCC48_X64_CC_FLAGS) -Wno-unused-but-set-variable -@@ -1209,7 +1209,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s +@@ -1232,7 +1232,7 @@ RELEASE_GCC48_AARCH64_CC_FLAGS = DEF(GCC48_AARCH64_CC_FLAGS) -Wno-unused-but-s *_GCC49_IA32_DLINK2_FLAGS = DEF(GCC49_IA32_DLINK2_FLAGS) *_GCC49_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC49_IA32_OBJCOPY_FLAGS = @@ -58,7 +58,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1237,7 +1237,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set +@@ -1260,7 +1260,7 @@ RELEASE_GCC49_IA32_CC_FLAGS = DEF(GCC49_IA32_CC_FLAGS) -Wno-unused-but-set *_GCC49_X64_DLINK2_FLAGS = DEF(GCC49_X64_DLINK2_FLAGS) *_GCC49_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC49_X64_OBJCOPY_FLAGS = @@ -67,7 +67,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) RELEASE_GCC49_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS) -Wno-unused-but-set-variable -Wno-unused-const-variable -@@ -1496,7 +1496,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 +@@ -1519,7 +1519,7 @@ RELEASE_GCCNOLTO_AARCH64_DLINK_XIPFLAGS = -z common-page-size=0x20 *_GCC5_IA32_DLINK2_FLAGS = DEF(GCC5_IA32_DLINK2_FLAGS) -no-pie *_GCC5_IA32_RC_FLAGS = DEF(GCC_IA32_RC_FLAGS) *_GCC5_IA32_OBJCOPY_FLAGS = @@ -76,7 +76,7 @@ index 0f110fbb4a..35205db748 100755 DEBUG_GCC5_IA32_CC_FLAGS = DEF(GCC5_IA32_CC_FLAGS) -flto DEBUG_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl,-m,elf_i386,--oformat=elf32-i386 -@@ -1528,7 +1528,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, +@@ -1551,7 +1551,7 @@ RELEASE_GCC5_IA32_DLINK_FLAGS = DEF(GCC5_IA32_X64_DLINK_FLAGS) -flto -Os -Wl, *_GCC5_X64_DLINK2_FLAGS = DEF(GCC5_X64_DLINK2_FLAGS) *_GCC5_X64_RC_FLAGS = DEF(GCC_X64_RC_FLAGS) *_GCC5_X64_OBJCOPY_FLAGS = diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch index 7b73762f8..33726e03d 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch @@ -1,7 +1,7 @@ -From 81fc61513cf51a9ef6947dcf4bc1388a50c13f42 Mon Sep 17 00:00:00 2001 +From 9133327256392a17883ad3ed91ad63ecbac50f08 Mon Sep 17 00:00:00 2001 From: Kevin Wang -Date: Mon, 4 Nov 2024 04:29:41 +0000 -Subject: [PATCH 4/5] Reproduciable +Date: Wed, 18 Mar 2026 08:44:21 +0000 +Subject: [PATCH] Reproduciable --- BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- @@ -11,19 +11,19 @@ Subject: [PATCH 4/5] Reproduciable 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c -index 9d04fc612e..83fd6c9c05 100644 +index 6919e18809..f6696ae805 100644 --- a/BaseTools/Source/C/GenFw/Elf64Convert.c +++ b/BaseTools/Source/C/GenFw/Elf64Convert.c -@@ -13,6 +13,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent - #ifndef __GNUC__ +@@ -15,6 +15,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent #include + #undef RUNTIME_FUNCTION #include +#else +#define _GNU_SOURCE #endif #include #include -@@ -988,7 +990,7 @@ ScanSections64 ( +@@ -990,7 +992,7 @@ ScanSections64 ( } mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) + sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) + @@ -32,7 +32,7 @@ index 9d04fc612e..83fd6c9c05 100644 // // Add more space in the .debug data region for the DllCharacteristicsEx -@@ -2299,7 +2301,7 @@ WriteDebug64 ( +@@ -2310,7 +2312,7 @@ WriteDebug64 ( EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10; EFI_IMAGE_DEBUG_EX_DLLCHARACTERISTICS_ENTRY *DllEntry; @@ -41,7 +41,7 @@ index 9d04fc612e..83fd6c9c05 100644 NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset); DataDir = &NtHdr->Pe32Plus.OptionalHeader.DataDirectory[EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]; -@@ -2332,7 +2334,7 @@ WriteDebug64 ( +@@ -2343,7 +2345,7 @@ WriteDebug64 ( Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1); Nb10->Signature = CODEVIEW_SIGNATURE_NB10; @@ -65,7 +65,7 @@ index 45b39d7878..3fed7d1736 100644 if Input not in self.Inputs: self.Inputs.append(Input) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py -index 6d9c60b702..83ac267647 100755 +index 547c708fc7..29859c4a0b 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -576,7 +576,7 @@ cleanlib: @@ -116,7 +116,7 @@ index 6d9c60b702..83ac267647 100755 } RespDict = {} -@@ -1008,9 +1008,9 @@ cleanlib: +@@ -1019,9 +1019,9 @@ cleanlib: if not self.ObjTargetDict.get(T.Target.SubDir): self.ObjTargetDict[T.Target.SubDir] = set() self.ObjTargetDict[T.Target.SubDir].add(NewFile) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch index 12a1c8cfc..61bb4a845 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch @@ -1,6 +1,6 @@ -From 596f2e3bd7292c8f22cfe01ec95b8b46ce1bc8a1 Mon Sep 17 00:00:00 2001 +From 216280451e52ba73bc408e58c5a3b13d863e350f Mon Sep 17 00:00:00 2001 From: Kevin Wang -Date: Mon, 4 Nov 2024 04:35:18 +0000 +Date: Wed, 18 Mar 2026 08:44:21 +0000 Subject: [PATCH] Declare ProcessLibraryConstructorList --- diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 96fcda3c0..7fbf499c3 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -27,8 +27,8 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0005-Declare-ProcessLibraryConstructorList.patch \ " -PV = "edk2-3a3b12cb" -SRCREV = "3a3b12cbdae2e89b0e365eb01c378891d0d9037c" +PV = "edk2-stable202505" +SRCREV = "6951dfe7d59d144a3a980bd7eda699db2d8554ac" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" CVE_PRODUCT = "edk2" From 0a9a0315a8e63f054191f524a3a4f94ecad85b3d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 11:27:31 +0000 Subject: [PATCH 350/435] ci: add build-guest workflow for manual trigger with optional release --- .github/workflows/build-guest.yml | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/build-guest.yml diff --git a/.github/workflows/build-guest.yml b/.github/workflows/build-guest.yml new file mode 100644 index 000000000..715989176 --- /dev/null +++ b/.github/workflows/build-guest.yml @@ -0,0 +1,45 @@ +name: Build Guest Images + +on: + workflow_dispatch: + inputs: + tag: + description: 'Tag to create and release (e.g. v0.5.9). Leave empty to build only.' + required: false + default: '' + type: string + +jobs: + build: + runs-on: yocto-builder + timeout-minutes: 480 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Reproducible build + run: cd repro-build && ./repro-build.sh -n + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: guest-images + path: | + repro-build/dist/*.tar.gz + repro-build/dist/reproduce.sh + retention-days: 30 + + - name: Create tag and release + if: inputs.tag != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} + run: | + git tag "$TAG" + git push origin "$TAG" + gh release create "$TAG" \ + repro-build/dist/*.tar.gz \ + repro-build/dist/reproduce.sh \ + --title "$TAG" --generate-notes From 73b0e80c5f7155a74656da9be90e90c8a10a49ab Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 13:05:56 +0000 Subject: [PATCH 351/435] ci: build directly instead of using repro-build docker wrapper The sysbox DinD runner cannot nest another docker container due to procfs restrictions. Build directly using bitbake in the runner container which already has all build dependencies. --- .github/workflows/build-guest.yml | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-guest.yml b/.github/workflows/build-guest.yml index 715989176..7ca18e2f8 100644 --- a/.github/workflows/build-guest.yml +++ b/.github/workflows/build-guest.yml @@ -19,16 +19,31 @@ jobs: with: submodules: recursive - - name: Reproducible build - run: cd repro-build && ./repro-build.sh -n + - name: Build guest images + run: | + source dev-setup bb-build + make dist DIST_DIR=$PWD/dist BB_BUILD_DIR=$PWD/bb-build + + - name: Generate reproduce.sh + run: | + cat > dist/reproduce.sh < Date: Wed, 18 Mar 2026 14:01:28 +0000 Subject: [PATCH 352/435] ci: restore repro-build.sh for reproducible builds Revert to using repro-build.sh which runs the build inside a Docker container for reproducibility. This works now with sysbox 0.7.0 which fixed the DinD compatibility issue with runc 1.3.x. --- .github/workflows/build-guest.yml | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-guest.yml b/.github/workflows/build-guest.yml index 7ca18e2f8..715989176 100644 --- a/.github/workflows/build-guest.yml +++ b/.github/workflows/build-guest.yml @@ -19,31 +19,16 @@ jobs: with: submodules: recursive - - name: Build guest images - run: | - source dev-setup bb-build - make dist DIST_DIR=$PWD/dist BB_BUILD_DIR=$PWD/bb-build - - - name: Generate reproduce.sh - run: | - cat > dist/reproduce.sh < Date: Fri, 20 Mar 2026 06:12:58 +0000 Subject: [PATCH 353/435] Add ipset and netfilter kernel modules for k3s NetworkPolicy support kube-router (k3s embedded network policy controller) requires ipset kernel support to enforce NetworkPolicy rules. Without these modules, kube-router silently skips starting the network policy controller with: "Skipping network policy controller start, ipset save failed" Adds CONFIG_IP_SET and related hash/bitmap type modules, plus xt_set (iptables set match), xt_physdev, and xt_nflog. --- .../recipes-core/images/dstack-rootfs-base.inc | 13 +++++++++++++ .../recipes-kernel/linux/files/dstack-docker.cfg | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index af3dd6596..496fc6c73 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -47,6 +47,19 @@ IMAGE_INSTALL = "\ kernel-module-nft-reject \ kernel-module-nft-reject-inet \ kernel-module-nft-hash \ + kernel-module-ip-set \ + kernel-module-ip-set-hash-ip \ + kernel-module-ip-set-hash-net \ + kernel-module-ip-set-hash-ipport \ + kernel-module-ip-set-hash-ipportip \ + kernel-module-ip-set-hash-ipportnet \ + kernel-module-ip-set-hash-netiface \ + kernel-module-ip-set-bitmap-ip \ + kernel-module-ip-set-bitmap-port \ + kernel-module-ip-set-list-set \ + kernel-module-xt-set \ + kernel-module-xt-nflog \ + kernel-module-xt-physdev \ fuse3 \ fuse3-utils \ pigz \ diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index 015be74bd..cb71c50ed 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -40,6 +40,20 @@ CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y CONFIG_VXLAN=y CONFIG_CGROUP_BPF=y +# ipset support (required by kube-router network policy controller) +CONFIG_IP_SET=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m + # BLK IO throttling support CONFIG_BLK_CGROUP=y CONFIG_BLK_DEV_THROTTLING=y From efb1e09a748f7b7685b184cdb5238e2d286d2b7b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 20 Mar 2026 08:14:46 +0000 Subject: [PATCH 354/435] Replace small NTS servers with authoritative providers Remove ntp1/ntp2.glypnod.com, nts.teambelgium.net, a.st1.ntp.br, and time.bolha.one. Replace with PTB (German national metrology institute), Netnod (Swedish internet infrastructure), and SIDN Labs (Dutch .nl registry) NTS servers. This addresses abuse complaints about excessive NTS-KE traffic from our fleet to small community-run NTS servers. --- meta-dstack/recipes-core/chrony/files/chrony.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meta-dstack/recipes-core/chrony/files/chrony.conf b/meta-dstack/recipes-core/chrony/files/chrony.conf index d6c1b0f2b..6e635cdc9 100644 --- a/meta-dstack/recipes-core/chrony/files/chrony.conf +++ b/meta-dstack/recipes-core/chrony/files/chrony.conf @@ -11,12 +11,12 @@ confdir /etc/chrony/conf.d # Use a local timeserver in preference to the pool, if it's reachable. #server 192.168.22.22 iburst minpoll 2 prefer server time.cloudflare.com iburst nts -server nts.teambelgium.net iburst nts -server a.st1.ntp.br iburst nts -server time.bolha.one iburst nts server ptbtime1.ptb.de iburst nts -server ntp2.glypnod.com iburst nts -server ntp1.glypnod.com iburst nts +server ptbtime2.ptb.de iburst nts +server ptbtime3.ptb.de iburst nts +server nts.netnod.se iburst nts +server nts.ntp.se iburst nts +server ntppool1.time.nl iburst nts server virginia.time.system76.com iburst nts # Sync to pulse-per-second from an onboard GPS. From 5e4b7569c8a0c4e5a2eaf0302cdb319d6d937507 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 20 Mar 2026 08:18:36 +0000 Subject: [PATCH 355/435] Add System76 Oregon and Paris NTS servers for geographic diversity --- meta-dstack/recipes-core/chrony/files/chrony.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta-dstack/recipes-core/chrony/files/chrony.conf b/meta-dstack/recipes-core/chrony/files/chrony.conf index 6e635cdc9..d0d751bf4 100644 --- a/meta-dstack/recipes-core/chrony/files/chrony.conf +++ b/meta-dstack/recipes-core/chrony/files/chrony.conf @@ -18,6 +18,8 @@ server nts.netnod.se iburst nts server nts.ntp.se iburst nts server ntppool1.time.nl iburst nts server virginia.time.system76.com iburst nts +server oregon.time.system76.com iburst nts +server paris.time.system76.com iburst nts # Sync to pulse-per-second from an onboard GPS. #refclock PPS /dev/pps0 poll 0 prefer From d6478e32b6b96013a875619d24922b108b7572da Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Apr 2026 14:44:35 -0700 Subject: [PATCH 356/435] add gcp service account and scope config it's common practice to attach a user-managed service account to the VM. Then grant the service account IAM roles to access GCP resources, e.g. docker image registry access Signed-off-by: Jerry Yu --- scripts/bin/dstack-cloud | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index 3726faf42..eb53621d3 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -197,6 +197,10 @@ class GcpConfig: network: str = "default" subnet: str = "" + # Identity settings + service_account: str = "" + scopes: List[str] = field(default_factory=list) + # Tags and labels tags: List[str] = field(default_factory=list) labels: Dict[str, str] = field(default_factory=dict) @@ -225,6 +229,8 @@ class GcpConfig: "bucket": "", "network": "default", "subnet": "", + "service_account": "", + "scopes": [], "tags": [], "labels": {} } @@ -1380,6 +1386,10 @@ class CloudDeploymentManager: create_args.append(f"--network={config.network}") if config.subnet: create_args.append(f"--subnet={config.subnet}") + if config.service_account: + create_args.append(f"--service-account={config.service_account}") + if config.scopes: + create_args.append(f"--scopes={','.join(config.scopes)}") # Always attach firewall tag so existing firewall rules continue to work # after instance recreation (e.g. deploy --delete). instance_tags = list(config.tags) From 2d56823b3a7b9559d323c6678a9df499948bdb01 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 13 Apr 2026 06:08:30 +0000 Subject: [PATCH 357/435] fix: add nftables kernel modules to 6.18 defconfig The 6.18 defconfig was missing nftables config options that are required by Docker's iptables-nft backend. These were defined in dstack-docker.cfg for the linux-yocto kernel but linux-custom_6.18.7 does not use bbappend fragments, so they must be in the defconfig directly. Without these, rootfs build fails with missing kernel-module-nf-tables and related packages. --- .../recipes-kernel/linux/files/6.18/defconfig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig index 0a953ff6e..2894e97aa 100644 --- a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig +++ b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig @@ -358,3 +358,20 @@ CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y + +# nftables (Docker iptables-nft backend) +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_COMPAT=m +CONFIG_NFT_NAT=m +CONFIG_NFT_CHAIN_NAT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_HASH=m +CONFIG_NETFILTER_XT_TARGET_CT=m From 4201d202e6910b35a3ca670573e133fb95734a5f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 13 Apr 2026 06:43:05 +0000 Subject: [PATCH 358/435] chore: update dstack submodule to latest master --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index fc6a43fe5..603c6ee5d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit fc6a43fe530d3a50a36b8ea0864c6d952b45e375 +Subproject commit 603c6ee5db99ea4cb0e63f4768fcd215ceb64ede From 350d520e219efdf2789bc1fb2b3658bfe5b11e58 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Apr 2026 19:18:46 -0700 Subject: [PATCH 359/435] meta-nvidia: add 580.95.05 recipes for nvidia, fabricmanager, nscq Required for RTX PRO 6000 Blackwell Server Edition (10de:2bb5) Confidential Compute support: the 570.172.08 NVIDIA Open Kernel Module predates Pro 6000 SE and refuses to attach with "GPU confidential compute capability is not enabled" inside a TDX guest. 580.95.05 recognizes GB202 and brings the full TDX guest CC stack online (verified end-to-end: nvidia-smi conf-compute reports CC State=ON, GPU CC Capabilities=CC Capable, Protected memory=99461312 KiB; PyTorch CUDA workloads run inside the TD). The nscq archive at 580 no longer ships bin/nscq-cli, so the new recipe drops it from do_install and FILES. --- .../nvidia/libnvidia-nscq_580.95.05.bb | 32 +++++++ .../nvidia/nvidia-fabricmanager_580.95.05.bb | 86 +++++++++++++++++++ .../nvidia/nvidia_580.95.05.bb | 24 ++++++ 3 files changed, 142 insertions(+) create mode 100644 meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb new file mode 100644 index 000000000..7ba353891 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb @@ -0,0 +1,32 @@ +SUMMARY = "NVIDIA NSCQ library" +DESCRIPTION = "NVIDIA NSCQ (NVIDIA System Communication Queue) library for NVIDIA GPU systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "6bc20061ebdae98fadd7a76110b44430" +SRC_URI[sha256sum] = "c2285c12f10ec2afc0ad2949f7fcc282b6fd37f32165c1df241451ccabb1067a" + +S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +do_install() { + install -d ${D}${libdir} + + install -m 0755 ${S}/lib/libnvidia-nscq.so.${PV} ${D}${libdir} + ln -sf libnvidia-nscq.so.${PV} ${D}${libdir}/libnvidia-nscq.so.2.0 + ln -sf libnvidia-nscq.so.2.0 ${D}${libdir}/libnvidia-nscq.so.2 + ln -sf libnvidia-nscq.so.2 ${D}${libdir}/libnvidia-nscq.so +} + +FILES:${PN} = "\ + ${libdir}/libnvidia-nscq.so.${PV} \ + ${libdir}/libnvidia-nscq.so.2.0 \ + ${libdir}/libnvidia-nscq.so.2 \ + ${libdir}/libnvidia-nscq.so \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb new file mode 100644 index 000000000..6225e7df3 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb @@ -0,0 +1,86 @@ +SUMMARY = "NVIDIA Fabric Manager for NVSwitch systems" +DESCRIPTION = "NVIDIA Fabric Manager provides NVSwitch management for NVIDIA HGX and DGX systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "a6568aa288cb4784b85ba6826463f918" +SRC_URI[sha256sum] = "f0220bfb67d04b4107acf00cc95abe5a9268fd8f8b5bae26971f4df232e4369c" + +S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" + +DEPENDS = "" +RDEPENDS:${PN} = "bash zlib" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +inherit systemd + +SYSTEMD_AUTO_ENABLE = "enable" +SYSTEMD_SERVICE:${PN} = "nvidia-fabricmanager.service" + +do_install() { + # Create directories + install -d ${D}${bindir} + install -d ${D}${libdir} + install -d ${D}${datadir}/nvidia/nvswitch + install -d ${D}${systemd_system_unitdir} + + # Install binaries + install -m 0755 ${S}/bin/nv-fabricmanager ${D}${bindir} + install -m 0755 ${S}/bin/nvidia-fabricmanager-start.sh ${D}${bindir} + install -m 0755 ${S}/bin/nvswitch-audit ${D}${bindir} + + # Install libraries + install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} + ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so + + # Install config files + install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ + + # Install topology files + install -m 0644 ${S}/share/nvidia/nvswitch/dgx2_hgx2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxa100_hgxa100_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxh100_hgxh100_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxh800_hgxh800_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/mgxh20_nvl16_topology ${D}${datadir}/nvidia/nvswitch/ + + # Install multi-node topology files + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_8gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gh200_nvlink_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl576r16_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_nso_topology ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl4r1_c2g2_etf_topology ${D}${datadir}/nvidia/nvswitch/ + + # Install systemd service + install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} +} + +FILES:${PN} = "\ + ${bindir}/nv-fabricmanager \ + ${bindir}/nvidia-fabricmanager-start.sh \ + ${bindir}/nvswitch-audit \ + ${libdir}/libnvfm.so.1 \ + ${libdir}/libnvfm.so \ + ${datadir}/nvidia/nvswitch/* \ + ${systemd_system_unitdir}/nvidia-fabricmanager.service \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb new file mode 100644 index 000000000..3f0677a12 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb @@ -0,0 +1,24 @@ +SUMMARY = "NVidia Graphics Driver" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://../LICENSE;md5=92aa2e2af6aa0bcba1c3fe49da021937" + +NVIDIA_ARCHIVE_NAME = "NVIDIA-Linux-${TARGET_ARCH}-${PV}" +NVIDIA_SRC = "${WORKDIR}/${NVIDIA_ARCHIVE_NAME}" +SRC_URI = " \ + https://us.download.nvidia.com/tesla/${PV}/${NVIDIA_ARCHIVE_NAME}.run \ +" +SRC_URI[md5sum] = "3d23653c4898d08b1f3f031ea8cdaa93" +SRC_URI[sha256sum] = "849ef0ef8e842b9806b2cde9f11c1303d54f1a9a769467e4e5d961b2fe1182a7" + +RDEPENDS:${PN} = "nvidia-modprobe-config" + +do_unpack() { + chmod +x ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run + rm -rf ${NVIDIA_SRC} + ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run -x --target ${NVIDIA_SRC} +} + +do_make_scripts[noexec] = "1" + +include nvidia-kernel-module.inc +include nvidia-libs.inc From 85e7e171657725e7ca2d3701b1e77ca6241c1939 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Apr 2026 19:30:37 -0700 Subject: [PATCH 360/435] meta-nvidia: glob topology files + pin nvidia stack via NVIDIA_VERSION - nvidia-fabricmanager: install all files from share/nvidia/nvswitch/ via glob loop instead of an explicit list. The 580.95.05 archive ships new GB300 topologies (gb300_nvl72r{1,2}_c2g4_topology) that the old hard-coded list missed; future archives stay covered automatically. (Addresses Copilot review feedback on PR #56.) - dstack.conf: introduce NVIDIA_VERSION = "580.95.05" and pin PREFERRED_VERSION_{nvidia,nvidia-fabricmanager,libnvidia-nscq} from it so the kernel module ABI and userspace libs always move together. --- meta-dstack/conf/distro/dstack.conf | 7 ++++ .../nvidia/nvidia-fabricmanager_580.95.05.bb | 33 +++---------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 7cd225ef7..edaece712 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -24,4 +24,11 @@ SERIAL_CONSOLES = "115200;ttyS0" PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.92.0" PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" +# NVIDIA driver stack (only consulted when nvidia flavor is built). +# Bump all three together — kernel module ABI is paired with userspace libs. +NVIDIA_VERSION = "580.95.05" +PREFERRED_VERSION_nvidia = "${NVIDIA_VERSION}" +PREFERRED_VERSION_nvidia-fabricmanager = "${NVIDIA_VERSION}" +PREFERRED_VERSION_libnvidia-nscq = "${NVIDIA_VERSION}" + BAD_RECOMMENDATIONS = "busybox-syslog" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb index 6225e7df3..7187848a2 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb @@ -39,37 +39,12 @@ do_install() { install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so - # Install config files + # Install config + topology files (glob picks up new SKUs in future archives) install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ - - # Install topology files - install -m 0644 ${S}/share/nvidia/nvswitch/dgx2_hgx2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxa100_hgxa100_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxh100_hgxh100_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxh800_hgxh800_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/mgxh20_nvl16_topology ${D}${datadir}/nvidia/nvswitch/ - - # Install multi-node topology files - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_8gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_16gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_trunk_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/dgxgh200_hgxgh200_32gpus_osfp_cable_connections.csv ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gh200_nvlink_32gpus_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl36r1_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r1_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl72r2_c2g2_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl576r16_c2g4_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl8r1_c2g4_etf_nso_topology ${D}${datadir}/nvidia/nvswitch/ - install -m 0644 ${S}/share/nvidia/nvswitch/gb200_nvl4r1_c2g2_etf_topology ${D}${datadir}/nvidia/nvswitch/ + for f in ${S}/share/nvidia/nvswitch/*; do + [ -f "$f" ] && install -m 0644 "$f" ${D}${datadir}/nvidia/nvswitch/ + done # Install systemd service install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} From e3655d1390feee3736476f4bda35c4354b4a12fc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Apr 2026 21:04:49 -0700 Subject: [PATCH 361/435] Bump version to 0.5.9 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 368c62e7d..282eeb27d 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 368c62e7de5d4016bd75332824aa7f2ef1d7d19e +Subproject commit 282eeb27d22d8f091ad0fa5a90e638f85cf68751 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index edaece712..5da1983a5 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.8" +DISTRO_VERSION = "0.5.9" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 0c29cf508d72e4609b68c58120d948bd3558d84b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Apr 2026 06:47:32 +0000 Subject: [PATCH 362/435] meta-nvidia: bump NVIDIA stack to 595.58.03 Tested on RTX PRO 6000 Blackwell Server Edition (10de:2bb5) with TDX CC mode ON: nvidia-smi reports driver 595.58.03 + CUDA 13.2, GPU initializes cleanly without RmInitAdapter errors seen on 570.x. --- meta-dstack/conf/distro/dstack.conf | 2 +- .../nvidia/libnvidia-nscq_595.58.03.bb | 32 ++++++++++ .../nvidia/nvidia-fabricmanager_595.58.03.bb | 61 +++++++++++++++++++ .../nvidia/nvidia_595.58.03.bb | 24 ++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 5da1983a5..83577c497 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -26,7 +26,7 @@ PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" # NVIDIA driver stack (only consulted when nvidia flavor is built). # Bump all three together — kernel module ABI is paired with userspace libs. -NVIDIA_VERSION = "580.95.05" +NVIDIA_VERSION = "595.58.03" PREFERRED_VERSION_nvidia = "${NVIDIA_VERSION}" PREFERRED_VERSION_nvidia-fabricmanager = "${NVIDIA_VERSION}" PREFERRED_VERSION_libnvidia-nscq = "${NVIDIA_VERSION}" diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb new file mode 100644 index 000000000..1a72f6de5 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb @@ -0,0 +1,32 @@ +SUMMARY = "NVIDIA NSCQ library" +DESCRIPTION = "NVIDIA NSCQ (NVIDIA System Communication Queue) library for NVIDIA GPU systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "5b559a614f2c2c79b6eb796b68b6fb0f" +SRC_URI[sha256sum] = "1c008787111fc55a7fc1d3521a6d41c2523b8949354e5651eadb4498554e5469" + +S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +do_install() { + install -d ${D}${libdir} + + install -m 0755 ${S}/lib/libnvidia-nscq.so.${PV} ${D}${libdir} + ln -sf libnvidia-nscq.so.${PV} ${D}${libdir}/libnvidia-nscq.so.2.0 + ln -sf libnvidia-nscq.so.2.0 ${D}${libdir}/libnvidia-nscq.so.2 + ln -sf libnvidia-nscq.so.2 ${D}${libdir}/libnvidia-nscq.so +} + +FILES:${PN} = "\ + ${libdir}/libnvidia-nscq.so.${PV} \ + ${libdir}/libnvidia-nscq.so.2.0 \ + ${libdir}/libnvidia-nscq.so.2 \ + ${libdir}/libnvidia-nscq.so \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb new file mode 100644 index 000000000..90ac8f465 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb @@ -0,0 +1,61 @@ +SUMMARY = "NVIDIA Fabric Manager for NVSwitch systems" +DESCRIPTION = "NVIDIA Fabric Manager provides NVSwitch management for NVIDIA HGX and DGX systems" +HOMEPAGE = "https://developer.nvidia.com/" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://LICENSE;md5=2cc00be68c1227a7c42ff3620ef75d05" + +SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-${PV}-archive.tar.xz" +SRC_URI[md5sum] = "8761b4b5f371b7e87b2b82141c901311" +SRC_URI[sha256sum] = "ea86951987ced2c2bbed1a9a9037e917dd2268add0ec484db9216e843d8b7de8" + +S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" + +DEPENDS = "" +RDEPENDS:${PN} = "bash zlib" + +INSANE_SKIP:${PN} = "already-stripped ldflags" + +do_configure[noexec] = "1" +do_compile[noexec] = "1" + +inherit systemd + +SYSTEMD_AUTO_ENABLE = "enable" +SYSTEMD_SERVICE:${PN} = "nvidia-fabricmanager.service" + +do_install() { + # Create directories + install -d ${D}${bindir} + install -d ${D}${libdir} + install -d ${D}${datadir}/nvidia/nvswitch + install -d ${D}${systemd_system_unitdir} + + # Install binaries + install -m 0755 ${S}/bin/nv-fabricmanager ${D}${bindir} + install -m 0755 ${S}/bin/nvidia-fabricmanager-start.sh ${D}${bindir} + install -m 0755 ${S}/bin/nvswitch-audit ${D}${bindir} + + # Install libraries + install -m 0644 ${S}/lib/libnvfm.so.1 ${D}${libdir} + ln -sf libnvfm.so.1 ${D}${libdir}/libnvfm.so + + # Install config + topology files (glob picks up new SKUs in future archives) + install -m 0644 ${S}/etc/fabricmanager.cfg ${D}${datadir}/nvidia/nvswitch/ + install -m 0644 ${S}/etc/fabricmanager_multinode.cfg ${D}${datadir}/nvidia/nvswitch/ + for f in ${S}/share/nvidia/nvswitch/*; do + [ -f "$f" ] && install -m 0644 "$f" ${D}${datadir}/nvidia/nvswitch/ + done + + # Install systemd service + install -m 0644 ${S}/systemd/nvidia-fabricmanager.service ${D}${systemd_system_unitdir} +} + +FILES:${PN} = "\ + ${bindir}/nv-fabricmanager \ + ${bindir}/nvidia-fabricmanager-start.sh \ + ${bindir}/nvswitch-audit \ + ${libdir}/libnvfm.so.1 \ + ${libdir}/libnvfm.so \ + ${datadir}/nvidia/nvswitch/* \ + ${systemd_system_unitdir}/nvidia-fabricmanager.service \ +" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb new file mode 100644 index 000000000..8c0ab3d5c --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb @@ -0,0 +1,24 @@ +SUMMARY = "NVidia Graphics Driver" +LICENSE = "NVIDIA-Proprietary" +LIC_FILES_CHKSUM = "file://../LICENSE;md5=92aa2e2af6aa0bcba1c3fe49da021937" + +NVIDIA_ARCHIVE_NAME = "NVIDIA-Linux-${TARGET_ARCH}-${PV}" +NVIDIA_SRC = "${WORKDIR}/${NVIDIA_ARCHIVE_NAME}" +SRC_URI = " \ + https://us.download.nvidia.com/tesla/${PV}/${NVIDIA_ARCHIVE_NAME}.run \ +" +SRC_URI[md5sum] = "8d98a183bf994af0ff19980e0ef430f2" +SRC_URI[sha256sum] = "8c0d4f967b7932c4ab5714272aee8103392b0a702c92afa555176d36205829f9" + +RDEPENDS:${PN} = "nvidia-modprobe-config" + +do_unpack() { + chmod +x ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run + rm -rf ${NVIDIA_SRC} + ${DL_DIR}/${NVIDIA_ARCHIVE_NAME}.run -x --target ${NVIDIA_SRC} +} + +do_make_scripts[noexec] = "1" + +include nvidia-kernel-module.inc +include nvidia-libs.inc From 60c0a4e181a53917b8ab4dc573cfcfbbfdcd4d86 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 7 May 2026 22:38:28 -0700 Subject: [PATCH 363/435] kernel: disable DirtyFrag trigger modules --- .../recipes-kernel/linux/files/dstack-docker.cfg | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg index cb71c50ed..99f4607ff 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg @@ -26,7 +26,17 @@ CONFIG_BRIDGE_VLAN_FILTERING=y CONFIG_XFRM=y CONFIG_XFRM_USER=y CONFIG_XFRM_ALGO=y -CONFIG_INET_ESP=y +# Disable ESP transforms to mitigate DirtyFrag-style page-cache writes via esp4/esp6. +# Keep XFRM userspace support available for non-ESP networking features, but do +# not build the vulnerable ESP protocol handlers. +CONFIG_INET_ESP=n +CONFIG_INET6_ESP=n +CONFIG_XFRM_ESP=n + +# RxRPC is not needed by dstack and is another DirtyFrag trigger path. Keep it +# disabled even if future kernel feature sets would otherwise enable it. +CONFIG_AF_RXRPC=n +CONFIG_RXKAD=n CONFIG_NETFILTER_XT_MATCH_BPF=y CONFIG_CRYPTO_SEQIV=y CONFIG_IPVLAN=m From 543da6ef5b2aedd59307ca69bfd20d653164a54d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 8 May 2026 01:35:46 -0700 Subject: [PATCH 364/435] rootfs: include nftables userspace --- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 496fc6c73..af463f457 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -10,6 +10,7 @@ IMAGE_INSTALL = "\ systemd \ netbase \ iptables \ + nftables \ docker-moby \ docker-compose \ tdx-guest-ko \ From 2f417559793345cb9760abd49004bca1e5d28581 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 8 May 2026 06:52:12 -0700 Subject: [PATCH 365/435] Bump version to 0.5.10 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 282eeb27d..98bd47a49 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 282eeb27d22d8f091ad0fa5a90e638f85cf68751 +Subproject commit 98bd47a495f03473b25656495ac63b327a9d0d98 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 83577c497..74ec77d94 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.9" +DISTRO_VERSION = "0.5.10" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 51ea4857637279eceaafe8f8e02e07b77c75acc8 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Fri, 8 May 2026 15:24:21 -0700 Subject: [PATCH 366/435] Fix tar option injection in mk-image-mr helper --- scripts/bin/mk-image-mr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bin/mk-image-mr.sh b/scripts/bin/mk-image-mr.sh index b84cd741f..71ff61e56 100755 --- a/scripts/bin/mk-image-mr.sh +++ b/scripts/bin/mk-image-mr.sh @@ -99,7 +99,7 @@ echo "Creating final archive: $OUTPUT_FILE" # Change to the flattened directory and create archive without directory structure cd "$FLATTEN_DIR" -tar -czf "../$OUTPUT_FILE" * +tar -czf "../$OUTPUT_FILE" -- * cd - >/dev/null # Move the final file to the current working directory From e5db4aef0c119c3f774237e653846fbb786a5f71 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Fri, 8 May 2026 17:39:13 -0700 Subject: [PATCH 367/435] ci: restrict manual guest releases to protected default branch --- .github/workflows/build-guest.yml | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-guest.yml b/.github/workflows/build-guest.yml index 715989176..25ff79a8b 100644 --- a/.github/workflows/build-guest.yml +++ b/.github/workflows/build-guest.yml @@ -13,6 +13,8 @@ jobs: build: runs-on: yocto-builder timeout-minutes: 480 + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@v4 @@ -31,8 +33,33 @@ jobs: repro-build/dist/reproduce.sh retention-days: 30 + release: + if: inputs.tag != '' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + needs: build + runs-on: yocto-builder + timeout-minutes: 60 + environment: release + permissions: + contents: write + steps: + - name: Checkout default branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: guest-images + path: repro-build/dist + + - name: Validate release tag + env: + TAG: ${{ inputs.tag }} + run: | + echo "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' + - name: Create tag and release - if: inputs.tag != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ inputs.tag }} From 68068fc29277a1f287dd62dcc7c3fe2c2b20b545 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 11 May 2026 20:31:09 -0700 Subject: [PATCH 368/435] Bump version to 0.5.11 --- dstack | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dstack b/dstack index 98bd47a49..40eaf35e6 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 98bd47a495f03473b25656495ac63b327a9d0d98 +Subproject commit 40eaf35e6b3f112998d01569f2a26110baab123b diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 74ec77d94..6f413a74e 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.5.10" +DISTRO_VERSION = "0.5.11" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" VOLATILE_LOG_DIR = "no" From 4693c29094f02e8b16afbc670db30c94446b3bef Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 01:47:45 -0700 Subject: [PATCH 369/435] ovmf: pin to edk2-stable202502 and stamp ovmf_variant into metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit f9f11f3 ("fix: upgrade OVMF to edk2-stable202505") moved OVMF from the 2024-09 snapshot 3a3b12cb to edk2-stable202505. The six commits below land between those two and rewrite the boot-time RTMR[0] event chain that dstack's verifier replays: fb56dc78ef QemuFwCfgLib: cache + measurement (adds fw_cfg BootMenu, bootorder) 45a56d7505 OvmfPkg: add BootManagerMenuApp to dependencies 9d9e3a2ba8 OvmfPkg: use BootManagerMenuApp as BootManagerMenu (Boot0000 hash changes) d433b4c8e4 PlatformBootManagerLib: register UiApp as optional boot option (new Boot0001) dd5cce3e53 PlatformBootManagerCommonLib: set UiApp as optional cd76265f1a OvmfPkg: Enable Smbios measurement (adds EV_EFI_HANDOFF_TABLES whose digest is sha384 over the filtered QEMU SMBIOS table — content varies with -m, -cpu, -smbios type=1 ... so cannot be precomputed from VmConfig alone) dstack PR #678 modelled the new 17-event layout in dstack-mr and worked for 2 GB CVMs, but EV_EFI_HANDOFF_TABLES was hardcoded to the digest captured from a 2 GB CVM. Any other memory size produced a different SMBIOS Type 16/17/19 layout and a different digest, so e.g. 8 GB CVMs failed KMS verify with `RTMR0 mismatch` regardless of dstack-mr fixes. Pin to edk2-stable202502 (Feb 2025) — the most recent stable tag still before all six commits. It produces the same 13-event RTMR[0] layout as the legacy 3a3b12cb snapshot dstack used pre-upgrade, while still carrying ~5 months of post-Sep-2024 EDK2 fixes (incl. CVEs). Three changes here: * dstack-ovmf_git.bb: bump PV to edk2-stable202502, refresh SRCREV, add OVMF_VARIANT = "pre202505" so mkimage.sh can stamp the layout tag into image metadata. * 0003-Debug-prefix-map.patch: one context line (GCC_VFRPP_FLAGS) differs between stable202502 and stable202505. Add the missing `-P` flag to the patch context so quilt applies on stable202502. * mkimage.sh: read OVMF_VARIANT from the dstack-ovmf recipe via bitbake-getvar and stamp it into metadata.json. dstack-mr's verifier reads this field to pick the matching event-layout code path (Pre202505 vs Stable202505); without it, verifiers must fall back to parsing the image name, which gets the wrong answer for any image that ships pre-202505 OVMF under a 0.5.10+ name. Validated end-to-end on tdxlab: built dstack-0.5.11 with this recipe, deployed an 8 GB CVM through dstack-vmm, KMS verified successfully, RTMR0 from the live quote matches dstack-mr's prediction byte-for-byte (5bba6288696c9369311fe673001f386b8494ad8d9fbb918b8571eaf73ededa420c2e5 1f4fe3118556f8afbc1412ff1b6, 13 events on RTMR0). --- .../dstack-ovmf/0003-Debug-prefix-map.patch | 2 +- .../dstack-ovmf/dstack-ovmf_git.bb | 25 +++++++++++++++++-- mkimage.sh | 15 +++++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch index 8d850ffa6..3a50bed71 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch @@ -28,7 +28,7 @@ index cca699c4a8..e758bd8b4e 100755 -DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h +DEFINE GCC_ASM_FLAGS = -c -x assembler -imacros AutoGen.h ENV(GCC_PREFIX_MAP) +DEFINE GCC_PP_FLAGS = -E -x assembler-with-cpp -include AutoGen.h ENV(GCC_PREFIX_MAP) - DEFINE GCC_VFRPP_FLAGS = -x c -E -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h + DEFINE GCC_VFRPP_FLAGS = -x c -E -P -DVFRCOMPILE --include $(MODULE_NAME)StrDefs.h DEFINE GCC_ASLPP_FLAGS = -x c -E -include AutoGen.h DEFINE GCC_ASLCC_FLAGS = -x c @@ -1095,7 +1095,7 @@ DEFINE GCC5_LOONGARCH64_PP_FLAGS = -mabi=lp64d -march=loongarch64 DEF( diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 7fbf499c3..84ca1b321 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -27,10 +27,31 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0005-Declare-ProcessLibraryConstructorList.patch \ " -PV = "edk2-stable202505" -SRCREV = "6951dfe7d59d144a3a980bd7eda699db2d8554ac" +# Pinned to edk2-stable202502 (Feb 2025) instead of the latest stable202505. +# Between these two tags, six commits land in OvmfPkg / MdeModulePkg that +# rewrite the boot-time RTMR[0] event chain: +# fb56dc78ef QemuFwCfgLib: cache + measurement (adds fw_cfg BootMenu, bootorder) +# 45a56d7505 OvmfPkg: add BootManagerMenuApp to dependencies +# 9d9e3a2ba8 OvmfPkg: use BootManagerMenuApp as BootManagerMenu (Boot0000 hash changes) +# d433b4c8e4 PlatformBootManagerLib: register UiApp as optional boot option (new Boot0001) +# dd5cce3e53 PlatformBootManagerCommonLib: set UiApp as an optional boot option +# cd76265f1a OvmfPkg: Enable Smbios measurement (adds EV_EFI_HANDOFF_TABLES whose +# digest is sha384(filtered QEMU SMBIOS table) — varies with -m / -cpu / +# -smbios type=1 and so cannot be precomputed from VmConfig alone) +# stable202502 contains none of them and so produces the same 13-event RTMR[0] +# layout as the legacy 3a3b12cb snapshot dstack used pre-upgrade, while still +# carrying 5 months of post-Sep-2024 EDK2 fixes (incl. CVEs). +PV = "edk2-stable202502" +SRCREV = "fbe0805b2091393406952e84724188f8c1941837" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" +# Tag identifying the OVMF boot-time RTMR[0] event layout this build produces. +# Consumed by mkimage.sh to stamp `ovmf_variant` into the image metadata.json so +# verifiers can pick the matching dstack-mr code path without parsing PV. +# Keep this in sync with the OvmfVariant enum in dstack/dstack-types when +# bumping PV. +OVMF_VARIANT = "pre202505" + CVE_PRODUCT = "edk2" CVE_VERSION = "${@d.getVar('PV').split('-')[1]}" diff --git a/mkimage.sh b/mkimage.sh index 9f986409d..69579c747 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -65,7 +65,17 @@ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ $Q cp $ROOTFS_IMAGE ${OUTPUT_DIR}/rootfs.img.verity GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") -echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json" + +# Lift the OVMF variant tag straight out of the dstack-ovmf recipe so verifiers +# know which RTMR[0] event layout to expect. Required: the recipe must declare +# OVMF_VARIANT alongside PV. +OVMF_VARIANT=$(bitbake-getvar --value OVMF_VARIANT -r dstack-ovmf | tail -n1) +if [ -z "$OVMF_VARIANT" ]; then + echo "Error: dstack-ovmf recipe is missing OVMF_VARIANT" >&2 + exit 1 +fi + +echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json (ovmf_variant=$OVMF_VARIANT)" KARG0="console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0" KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" @@ -81,7 +91,8 @@ cat < ${OUTPUT_DIR}/metadata.json "version": "$DSTACK_VERSION", "git_revision": "$GIT_REVISION", "shared_ro": true, - "is_dev": ${IS_DEV} + "is_dev": ${IS_DEV}, + "ovmf_variant": "$OVMF_VARIANT" } EOF From 716729c9821781da026cef8c15d0287f9bcd4ca7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 02:48:27 -0700 Subject: [PATCH 370/435] mkimage: drop tail -n1 from OVMF_VARIANT lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bitbake-getvar --value emits a single line for any defined variable, so the `| tail -n1` was defensive padding rather than load-bearing — and under `set -e` without `set -o pipefail` it would swallow a bitbake-getvar failure and let `tail` exit 0 with whatever spilled out of stderr. Match the existing DSTACK_VERSION line a few lines above that uses the same plain assignment, so `set -e` propagates correctly and the existing empty-string guard becomes the only thing we rely on. Addresses Copilot review on #63. --- mkimage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkimage.sh b/mkimage.sh index 69579c747..bdf10d6f1 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -69,7 +69,7 @@ GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") # Lift the OVMF variant tag straight out of the dstack-ovmf recipe so verifiers # know which RTMR[0] event layout to expect. Required: the recipe must declare # OVMF_VARIANT alongside PV. -OVMF_VARIANT=$(bitbake-getvar --value OVMF_VARIANT -r dstack-ovmf | tail -n1) +OVMF_VARIANT=$(bitbake-getvar --value OVMF_VARIANT -r dstack-ovmf) if [ -z "$OVMF_VARIANT" ]; then echo "Error: dstack-ovmf recipe is missing OVMF_VARIANT" >&2 exit 1 From f1a35b70280a9f74d0c7a533ab0b3e263e5fb55a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 21:05:59 -0700 Subject: [PATCH 371/435] kernel: enable CONFIG_CRYPTO_ECDSA for H100 confidential compute NVIDIA's open kernel driver (nvidia.ko) gates its LKCA-backed libspdm crypto provider on `CONFIG_CRYPTO_ECDSA` being defined when the driver is built (see `kernel-open/nvidia/internal_crypt_lib.h`: the `USE_LKCA` macro requires the kernel to advertise ECDSA, ECDH, RSA, HMAC, AKCIPHER, etc.). When `CONFIG_CRYPTO_ECDSA` is missing, libspdm falls back to stubs and at runtime prints `libspdm expects LKCA but found stubs!` then fails `spdmEstablishSession`, so H100 in Confidential Compute mode (e.g. GCP TDX + a3-highgpu-1g) never finishes init and `nvidia-smi` reports no devices. `meta-nvidia/recipes-kernel/linux/files/nvidia.cfg` already sets this config, but it ships as a `linux-yocto%.bbappend`, which does not attach to the in-tree `linux-custom_*.bb` recipes that build the dstack kernel from a defconfig. Add the option directly to the 6.17 and 6.18 defconfigs so all flavors (incl. nvidia) pick it up. Verified end-to-end on GCP a3-highgpu-1g + TDX after rebuilding the kernel + nvidia kernel modules with this change: SPDM session establishes, `nvidia-smi conf-compute -f` reports `CC status: ON`, and a PyTorch matmul runs at ~38 TFLOPs. --- meta-dstack/recipes-kernel/linux/files/6.17/defconfig | 1 + meta-dstack/recipes-kernel/linux/files/6.18/defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig index 0a953ff6e..f9ba4a72d 100644 --- a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig +++ b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig @@ -344,6 +344,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_ECDSA=y CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_CORDIC=m CONFIG_PRINTK_TIME=y diff --git a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig index 2894e97aa..dc6dbbe72 100644 --- a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig +++ b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig @@ -344,6 +344,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_ECDSA=y CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_CORDIC=m CONFIG_PRINTK_TIME=y From bab2c0ea659b2ef316b5f6fdebc162645eabe811 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 21:18:50 -0700 Subject: [PATCH 372/435] dstack-cloud: add gcp_config.provisioning_model for SPOT instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many GCP projects only ship preemptible (SPOT) quota for newer GPUs — in particular `PREEMPTIBLE-NVIDIA-H100-GPUS-per-project-{region,zone}` is granted by default while `NVIDIA-H100-GPUS-per-project-region` is zero. Without on-demand quota, the only way to launch H100 in a Confidential TDX VM is to request `--provisioning-model=SPOT`. Expose a `provisioning_model` field in `gcp_config` (default `STANDARD`, backwards-compatible). When set to `SPOT`, also emit `--instance-termination-action=STOP` so the boot/data disks survive preemption and the instance can be resumed via `dstack-cloud start` (important for the LUKS-encrypted data disk, which is keyed by the KMS-provisioned per-instance secret). Anything other than `STANDARD`/`SPOT` raises an early error rather than silently dropping through. Example `app.json` snippet for an H100 deploy: "gcp_config": { "machine_type": "a3-highgpu-1g", "zone": "us-central1-a", "provisioning_model": "SPOT" } --- scripts/bin/dstack-cloud | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud index eb53621d3..acf9f7a93 100755 --- a/scripts/bin/dstack-cloud +++ b/scripts/bin/dstack-cloud @@ -205,6 +205,13 @@ class GcpConfig: tags: List[str] = field(default_factory=list) labels: Dict[str, str] = field(default_factory=dict) + # Scheduling: provisioning model. "STANDARD" (default) or "SPOT". + # SPOT instances are required on projects without on-demand + # NVIDIA_H100_GPUS quota (most projects, as of 2026); GCP + # preempts them with ~30s notice and a max ~24h lifetime, but + # they're billed at a steep discount. + provisioning_model: str = "STANDARD" + def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -232,7 +239,8 @@ class GcpConfig: "service_account": "", "scopes": [], "tags": [], - "labels": {} + "labels": {}, + "provisioning_model": "STANDARD" } @@ -1382,6 +1390,20 @@ class CloudDeploymentManager: "--maintenance-policy=TERMINATE", ] + provisioning = (config.provisioning_model or "STANDARD").upper() + if provisioning == "SPOT": + create_args.append("--provisioning-model=SPOT") + # STOP (vs. the gcloud default DELETE) preserves the + # boot/data disks across preemption so the instance can + # be restarted with `dstack-cloud start` and keep its + # LUKS-encrypted data disk intact. + create_args.append("--instance-termination-action=STOP") + elif provisioning != "STANDARD": + raise RuntimeError( + f"Unsupported provisioning_model: {config.provisioning_model!r} " + f"(expected 'STANDARD' or 'SPOT')" + ) + if config.network != "default": create_args.append(f"--network={config.network}") if config.subnet: From 275ea210b7f0cfa45c6f5c5b634d5fa78b4d92d7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 23:18:54 -0700 Subject: [PATCH 373/435] bump dstack submodule to b051018a Pulls 315 commits of guest-agent / kms / gateway / vmm fixes into the recipe inputs. This is the state the v0.6.1 release tarballs were built against, so bumping the pointer here makes `git clone --recurse-submodules` reproduce the released images. dstack 603c6ee5..b051018a (Phala-Network/dstack-cloud:master tip). --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 603c6ee5d..b051018a9 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 603c6ee5db99ea4cb0e63f4768fcd215ceb64ede +Subproject commit b051018a97e9a51748cb493d8139f624a2e68f54 From 7477e4a1d50d00fd52800ef74f749d2ca3a025d5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 26 May 2026 23:28:59 -0700 Subject: [PATCH 374/435] bump DISTRO_VERSION to 0.6.1 Tags the artifacts produced by `FLAVORS=... make dist` as 0.6.1, so `dstack-cloud pull dstack-cloud{,-nvidia}-0.6.1` resolves against the released tarballs at https://github.com/Phala-Network/meta-dstack-cloud/releases/tag/v0.6.1. The 0.6.1 cycle ships the H100 CC kernel fix (#14), the SPOT provisioning flag in `dstack-cloud` (#15), and the dstack submodule bump to b051018a (#16). See the v0.6.1 release notes for details. --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 86873c988..921e846d4 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -4,7 +4,7 @@ DISTRO_NAME = "dstack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" -DISTRO_VERSION = "0.6.0" +DISTRO_VERSION = "0.6.1" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From aeb3e0abd1598e0fbfbbc0a9c6ccfa5cf7417714 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 May 2026 23:55:11 -0700 Subject: [PATCH 375/435] kernel: switch from linux-custom to official linux-yocto 6.18 - drop self-written linux-custom_{6.17.6,6.18.7}.bb and their flat defconfigs - linux-yocto%.bbappend now carries the dstack .scc/.cfg fragments (already the mainline mechanism) on official linux-yocto 6.18 (wrynose) - wire 0001-x86-tdx-select-dma-direct-remap.patch via SRC_URI:append:tdx - dstack-tdx.cfg: CONFIG_TDX_GUEST_DRIVER=y + TSM_REPORTS=y (in-tree ConfigFS TSM replaces the out-of-tree mod-tdx-guest module) --- .../recipes-kernel/linux/files/6.17/defconfig | 361 ----------------- .../recipes-kernel/linux/files/6.18/defconfig | 378 ------------------ .../recipes-kernel/linux/files/dstack-tdx.cfg | 6 +- .../linux/linux-custom_6.17.6.bb | 36 -- .../linux/linux-custom_6.18.7.bb | 36 -- .../linux/linux-yocto%.bbappend | 6 + 6 files changed, 11 insertions(+), 812 deletions(-) delete mode 100644 meta-dstack/recipes-kernel/linux/files/6.17/defconfig delete mode 100644 meta-dstack/recipes-kernel/linux/files/6.18/defconfig delete mode 100644 meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb delete mode 100644 meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb diff --git a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig b/meta-dstack/recipes-kernel/linux/files/6.17/defconfig deleted file mode 100644 index f9ba4a72d..000000000 --- a/meta-dstack/recipes-kernel/linux/files/6.17/defconfig +++ /dev/null @@ -1,361 +0,0 @@ -CONFIG_LOCALVERSION="-dstack" -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_KERNEL_LZMA=y -CONFIG_KERNEL_GZIP=y -CONFIG_DMA_COHERENT_POOL=y -CONFIG_DMA_DIRECT_REMAP=y -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y -CONFIG_BPF_SYSCALL=y -CONFIG_BPF_JIT=y -CONFIG_PREEMPT=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_BSD_PROCESS_ACCT_V3=y -CONFIG_TASKSTATS=y -CONFIG_TASK_DELAY_ACCT=y -CONFIG_TASK_XACCT=y -CONFIG_TASK_IO_ACCOUNTING=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_CGROUPS=y -CONFIG_MEMCG=y -CONFIG_BLK_CGROUP=y -CONFIG_CGROUP_SCHED=y -CONFIG_CFS_BANDWIDTH=y -CONFIG_CGROUP_PIDS=y -CONFIG_CGROUP_FREEZER=y -CONFIG_CPUSETS=y -CONFIG_CGROUP_DEVICE=y -CONFIG_CGROUP_CPUACCT=y -CONFIG_CGROUP_PERF=y -CONFIG_CGROUP_BPF=y -CONFIG_CGROUP_DEBUG=y -CONFIG_USER_NS=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_SYSFS_SYSCALL=y -CONFIG_SMP=y -CONFIG_X86_AMD_PLATFORM_DEVICE=y -CONFIG_HYPERVISOR_GUEST=y -CONFIG_PARAVIRT_SPINLOCKS=y -CONFIG_INTEL_TDX_GUEST=y -CONFIG_TDX_GUEST_DRIVER=y -CONFIG_TSM_REPORTS=y -CONFIG_CONFIGFS_FS=y -CONFIG_NR_CPUS=512 -CONFIG_X86_MSR=y -CONFIG_X86_CPUID=y -CONFIG_X86_CHECK_BIOS_CORRUPTION=y -CONFIG_EFI=y -CONFIG_EFI_STUB=y -CONFIG_HZ_1000=y -# CONFIG_SUSPEND is not set -# CONFIG_ACPI_DEBUG is not set -CONFIG_CPU_FREQ_STAT=y -CONFIG_CPU_FREQ_GOV_ONDEMAND=y -CONFIG_X86_ACPI_CPUFREQ=y -CONFIG_INTEL_IDLE=y -CONFIG_IA32_EMULATION=y -CONFIG_KPROBES=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_BLK_DEV_THROTTLING=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_XFRM_USER=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_ESP=y -CONFIG_NETFILTER=y -CONFIG_BRIDGE_NETFILTER=m -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NETFILTER_XTABLES_COMPAT=y -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_LOG=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NETMAP=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_REDIRECT=m -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_IPVS=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_IP_VS=m -CONFIG_NF_REJECT_IPV4=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_TARGET_SYNPROXY=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_VLAN_FILTERING=y -CONFIG_VLAN_8021Q=y -CONFIG_6LOWPAN=m -CONFIG_NET_SCHED=y -CONFIG_VSOCKETS=y -CONFIG_VIRTIO_VSOCKETS=y -CONFIG_HYPERV_VSOCKETS=y -CONFIG_CGROUP_NET_PRIO=y -CONFIG_CFG80211=m -CONFIG_CFG80211_WEXT=y -CONFIG_MAC80211=m -CONFIG_NET_9P=y -CONFIG_NET_9P_VIRTIO=y -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCI_MSI=y -CONFIG_PCI_IOV=y -CONFIG_PCI_HYPERV=y -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_DMA_DIRECT_REMAP=y -CONFIG_DMA_COHERENT_POOL=y -CONFIG_CMA=y -CONFIG_CMA_SIZE_SEL_MBYTES=y -# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set -CONFIG_CMA_SIZE_MBYTES=64 -CONFIG_DMA_CMA=y -CONFIG_GVE=y -CONFIG_SWIOTLB_DYNAMIC=y -CONFIG_ZONE_DMA=y -CONFIG_CONNECTOR=y -CONFIG_PARPORT=m -CONFIG_PARPORT_PC=m -CONFIG_BLK_DEV_FD=m -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=1 -CONFIG_BLK_DEV_RAM_SIZE=6144 -CONFIG_VIRTIO_BLK=y -CONFIG_BLK_DEV_NVME=y -CONFIG_EEPROM_AT24=m -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=y -CONFIG_SCSI_FC_ATTRS=y -CONFIG_SCSI_VIRTIO=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_ATA_PIIX=y -CONFIG_PATA_SCH=y -CONFIG_MD=y -CONFIG_BLK_DEV_DM=y -CONFIG_DM_CRYPT=y -CONFIG_DM_VERITY=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y -CONFIG_IPVLAN=m -CONFIG_VXLAN=y -CONFIG_NETCONSOLE=y -CONFIG_TUN=m -CONFIG_VETH=y -CONFIG_VIRTIO_NET=y -CONFIG_TYPHOON=m -CONFIG_PCNET32=m -CONFIG_ATL2=m -CONFIG_ATL1=m -CONFIG_ATL1E=m -CONFIG_ATL1C=m -CONFIG_ALX=m -CONFIG_TIGON3=m -CONFIG_BNX2X=m -CONFIG_BNXT=m -CONFIG_DL2K=m -CONFIG_E100=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_IGB=y -CONFIG_IGBVF=m -CONFIG_IGC=m -CONFIG_JME=m -CONFIG_SKGE=m -CONFIG_SKY2=m -CONFIG_8139CP=m -CONFIG_8139TOO=m -CONFIG_R8169=m -CONFIG_REALTEK_PHY_HWMON=y -CONFIG_USB_NET_DRIVERS=m -CONFIG_USB_KAWETH=m -CONFIG_USB_PEGASUS=m -CONFIG_USB_RTL8150=m -CONFIG_USB_RTL8152=m -CONFIG_USB_USBNET=m -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_DM9601=m -CONFIG_USB_NET_SMSC75XX=m -CONFIG_USB_NET_SMSC95XX=m -CONFIG_USB_NET_MCS7830=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_KC2190=y -# CONFIG_WLAN is not set -CONFIG_HYPERV_NET=y -CONFIG_ISDN=y -CONFIG_INPUT_MOUSEDEV=y -CONFIG_INPUT_EVDEV=y -CONFIG_INPUT_TABLET=y -CONFIG_INPUT_TOUCHSCREEN=y -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_INPUT_MISC=y -CONFIG_INPUT_UINPUT=y -# CONFIG_LEGACY_PTYS is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_PRINTER=m -CONFIG_VIRTIO_CONSOLE=y -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y -CONFIG_HPET=y -CONFIG_TCG_TPM=y -CONFIG_TCG_TPM2_HMAC=y -CONFIG_TCG_TIS=y -CONFIG_TCG_CRB=y -CONFIG_I2C_I801=y -CONFIG_WATCHDOG=y -CONFIG_BCMA=m -CONFIG_BCMA_DRIVER_GMAC_CMN=y -CONFIG_AGP=y -CONFIG_AGP_INTEL=y -CONFIG_DRM=y -CONFIG_DRM_I915=m -CONFIG_DRM_VIRTIO_GPU=y -CONFIG_DRM_CIRRUS_QEMU=m -CONFIG_FB=y -CONFIG_FB_UVESA=m -CONFIG_FB_EFI=y -CONFIG_FB_HYPERV=y -CONFIG_HID_HYPERV_MOUSE=y -CONFIG_HID_WACOM=y -CONFIG_USB_HIDDEV=y -CONFIG_USB=y -CONFIG_USB_XHCI_HCD=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_ROOT_HUB_TT=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_UHCI_HCD=y -CONFIG_USB_SERIAL=y -CONFIG_USB_SERIAL_CONSOLE=y -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_FTDI_SIO=y -CONFIG_USB_SERIAL_PL2303=y -CONFIG_USB_EZUSB_FX2=y -CONFIG_RTC_CLASS=y -CONFIG_VIRT_DRIVERS=y -CONFIG_EFI_SECRET=y -CONFIG_VIRTIO_PCI=y -CONFIG_VIRTIO_BALLOON=y -CONFIG_VIRTIO_INPUT=m -CONFIG_VIRTIO_MMIO=y -CONFIG_HYPERV=y -CONFIG_HYPERV_UTILS=y -CONFIG_HYPERV_BALLOON=y -CONFIG_INTEL_IOMMU=y -# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set -CONFIG_IRQ_REMAP=y -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_EXT4_FS_SECURITY=y -CONFIG_XFS_FS=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_RT=y -CONFIG_BTRFS_FS=m -CONFIG_BTRFS_FS_POSIX_ACL=y -CONFIG_FUSE_FS=m -CONFIG_CUSE=m -CONFIG_OVERLAY_FS=y -CONFIG_OVERLAY_FS_METACOPY=y -CONFIG_ISO9660_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_EFIVAR_FS=y -CONFIG_SQUASHFS=y -CONFIG_VXFS_FS=m -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -# CONFIG_NFS_DISABLE_UDP_SUPPORT is not set -CONFIG_9P_FS=y -CONFIG_9P_FS_POSIX_ACL=y -CONFIG_CRYPTO_NULL=y -CONFIG_CRYPTO_CRYPTD=y -CONFIG_CRYPTO_AES_TI=y -CONFIG_CRYPTO_DES=y -CONFIG_CRYPTO_ARC4=y -CONFIG_CRYPTO_CTS=y -CONFIG_CRYPTO_XTS=y -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_USER_API_RNG=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_AES_NI_INTEL=y -CONFIG_CRYPTO_ECDSA=y -CONFIG_CRYPTO_DEV_VIRTIO=y -CONFIG_CORDIC=m -CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y -CONFIG_DEBUG_INFO_BTF=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_UNWINDER_FRAME_POINTER=y - -CONFIG_FAT_FS=y -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=y diff --git a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig b/meta-dstack/recipes-kernel/linux/files/6.18/defconfig deleted file mode 100644 index dc6dbbe72..000000000 --- a/meta-dstack/recipes-kernel/linux/files/6.18/defconfig +++ /dev/null @@ -1,378 +0,0 @@ -CONFIG_LOCALVERSION="-dstack" -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_KERNEL_LZMA=y -CONFIG_KERNEL_GZIP=y -CONFIG_DMA_COHERENT_POOL=y -CONFIG_DMA_DIRECT_REMAP=y -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y -CONFIG_BPF_SYSCALL=y -CONFIG_BPF_JIT=y -CONFIG_PREEMPT=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_BSD_PROCESS_ACCT_V3=y -CONFIG_TASKSTATS=y -CONFIG_TASK_DELAY_ACCT=y -CONFIG_TASK_XACCT=y -CONFIG_TASK_IO_ACCOUNTING=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_CGROUPS=y -CONFIG_MEMCG=y -CONFIG_BLK_CGROUP=y -CONFIG_CGROUP_SCHED=y -CONFIG_CFS_BANDWIDTH=y -CONFIG_CGROUP_PIDS=y -CONFIG_CGROUP_FREEZER=y -CONFIG_CPUSETS=y -CONFIG_CGROUP_DEVICE=y -CONFIG_CGROUP_CPUACCT=y -CONFIG_CGROUP_PERF=y -CONFIG_CGROUP_BPF=y -CONFIG_CGROUP_DEBUG=y -CONFIG_USER_NS=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_SYSFS_SYSCALL=y -CONFIG_SMP=y -CONFIG_X86_AMD_PLATFORM_DEVICE=y -CONFIG_HYPERVISOR_GUEST=y -CONFIG_PARAVIRT_SPINLOCKS=y -CONFIG_INTEL_TDX_GUEST=y -CONFIG_TDX_GUEST_DRIVER=y -CONFIG_TSM_REPORTS=y -CONFIG_CONFIGFS_FS=y -CONFIG_NR_CPUS=512 -CONFIG_X86_MSR=y -CONFIG_X86_CPUID=y -CONFIG_X86_CHECK_BIOS_CORRUPTION=y -CONFIG_EFI=y -CONFIG_EFI_STUB=y -CONFIG_HZ_1000=y -# CONFIG_SUSPEND is not set -# CONFIG_ACPI_DEBUG is not set -CONFIG_CPU_FREQ_STAT=y -CONFIG_CPU_FREQ_GOV_ONDEMAND=y -CONFIG_X86_ACPI_CPUFREQ=y -CONFIG_INTEL_IDLE=y -CONFIG_IA32_EMULATION=y -CONFIG_KPROBES=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_BLK_DEV_THROTTLING=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_XFRM_USER=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_ESP=y -CONFIG_NETFILTER=y -CONFIG_BRIDGE_NETFILTER=m -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NETFILTER_XTABLES_COMPAT=y -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_LOG=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NETMAP=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_REDIRECT=m -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_IPVS=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_IP_VS=m -CONFIG_NF_REJECT_IPV4=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_TARGET_SYNPROXY=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_VLAN_FILTERING=y -CONFIG_VLAN_8021Q=y -CONFIG_6LOWPAN=m -CONFIG_NET_SCHED=y -CONFIG_VSOCKETS=y -CONFIG_VIRTIO_VSOCKETS=y -CONFIG_HYPERV_VSOCKETS=y -CONFIG_CGROUP_NET_PRIO=y -CONFIG_CFG80211=m -CONFIG_CFG80211_WEXT=y -CONFIG_MAC80211=m -CONFIG_NET_9P=y -CONFIG_NET_9P_VIRTIO=y -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCI_MSI=y -CONFIG_PCI_IOV=y -CONFIG_PCI_HYPERV=y -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_DMA_DIRECT_REMAP=y -CONFIG_DMA_COHERENT_POOL=y -CONFIG_CMA=y -CONFIG_CMA_SIZE_SEL_MBYTES=y -# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set -CONFIG_CMA_SIZE_MBYTES=64 -CONFIG_DMA_CMA=y -CONFIG_GVE=y -CONFIG_SWIOTLB_DYNAMIC=y -CONFIG_ZONE_DMA=y -CONFIG_CONNECTOR=y -CONFIG_PARPORT=m -CONFIG_PARPORT_PC=m -CONFIG_BLK_DEV_FD=m -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=1 -CONFIG_BLK_DEV_RAM_SIZE=6144 -CONFIG_VIRTIO_BLK=y -CONFIG_BLK_DEV_NVME=y -CONFIG_EEPROM_AT24=m -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=y -CONFIG_SCSI_FC_ATTRS=y -CONFIG_SCSI_VIRTIO=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_ATA_PIIX=y -CONFIG_PATA_SCH=y -CONFIG_MD=y -CONFIG_BLK_DEV_DM=y -CONFIG_DM_CRYPT=y -CONFIG_DM_VERITY=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y -CONFIG_IPVLAN=m -CONFIG_VXLAN=y -CONFIG_NETCONSOLE=y -CONFIG_TUN=m -CONFIG_VETH=y -CONFIG_VIRTIO_NET=y -CONFIG_TYPHOON=m -CONFIG_PCNET32=m -CONFIG_ATL2=m -CONFIG_ATL1=m -CONFIG_ATL1E=m -CONFIG_ATL1C=m -CONFIG_ALX=m -CONFIG_TIGON3=m -CONFIG_BNX2X=m -CONFIG_BNXT=m -CONFIG_DL2K=m -CONFIG_E100=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_IGB=y -CONFIG_IGBVF=m -CONFIG_IGC=m -CONFIG_JME=m -CONFIG_SKGE=m -CONFIG_SKY2=m -CONFIG_8139CP=m -CONFIG_8139TOO=m -CONFIG_R8169=m -CONFIG_REALTEK_PHY_HWMON=y -CONFIG_USB_NET_DRIVERS=m -CONFIG_USB_KAWETH=m -CONFIG_USB_PEGASUS=m -CONFIG_USB_RTL8150=m -CONFIG_USB_RTL8152=m -CONFIG_USB_USBNET=m -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_DM9601=m -CONFIG_USB_NET_SMSC75XX=m -CONFIG_USB_NET_SMSC95XX=m -CONFIG_USB_NET_MCS7830=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_KC2190=y -# CONFIG_WLAN is not set -CONFIG_HYPERV_NET=y -CONFIG_ISDN=y -CONFIG_INPUT_MOUSEDEV=y -CONFIG_INPUT_EVDEV=y -CONFIG_INPUT_TABLET=y -CONFIG_INPUT_TOUCHSCREEN=y -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_INPUT_MISC=y -CONFIG_INPUT_UINPUT=y -# CONFIG_LEGACY_PTYS is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_PRINTER=m -CONFIG_VIRTIO_CONSOLE=y -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y -CONFIG_HPET=y -CONFIG_TCG_TPM=y -CONFIG_TCG_TPM2_HMAC=y -CONFIG_TCG_TIS=y -CONFIG_TCG_CRB=y -CONFIG_I2C_I801=y -CONFIG_WATCHDOG=y -CONFIG_BCMA=m -CONFIG_BCMA_DRIVER_GMAC_CMN=y -CONFIG_AGP=y -CONFIG_AGP_INTEL=y -CONFIG_DRM=y -CONFIG_DRM_I915=m -CONFIG_DRM_VIRTIO_GPU=y -CONFIG_DRM_CIRRUS_QEMU=m -CONFIG_FB=y -CONFIG_FB_UVESA=m -CONFIG_FB_EFI=y -CONFIG_FB_HYPERV=y -CONFIG_HID_HYPERV_MOUSE=y -CONFIG_HID_WACOM=y -CONFIG_USB_HIDDEV=y -CONFIG_USB=y -CONFIG_USB_XHCI_HCD=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_ROOT_HUB_TT=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_UHCI_HCD=y -CONFIG_USB_SERIAL=y -CONFIG_USB_SERIAL_CONSOLE=y -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_FTDI_SIO=y -CONFIG_USB_SERIAL_PL2303=y -CONFIG_USB_EZUSB_FX2=y -CONFIG_RTC_CLASS=y -CONFIG_VIRT_DRIVERS=y -CONFIG_EFI_SECRET=y -CONFIG_VIRTIO_PCI=y -CONFIG_VIRTIO_BALLOON=y -CONFIG_VIRTIO_INPUT=m -CONFIG_VIRTIO_MMIO=y -CONFIG_HYPERV=y -CONFIG_HYPERV_UTILS=y -CONFIG_HYPERV_BALLOON=y -CONFIG_INTEL_IOMMU=y -# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set -CONFIG_IRQ_REMAP=y -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_EXT4_FS_SECURITY=y -CONFIG_XFS_FS=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_RT=y -CONFIG_BTRFS_FS=m -CONFIG_BTRFS_FS_POSIX_ACL=y -CONFIG_FUSE_FS=m -CONFIG_CUSE=m -CONFIG_OVERLAY_FS=y -CONFIG_OVERLAY_FS_METACOPY=y -CONFIG_ISO9660_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_EFIVAR_FS=y -CONFIG_SQUASHFS=y -CONFIG_VXFS_FS=m -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -# CONFIG_NFS_DISABLE_UDP_SUPPORT is not set -CONFIG_9P_FS=y -CONFIG_9P_FS_POSIX_ACL=y -CONFIG_CRYPTO_NULL=y -CONFIG_CRYPTO_CRYPTD=y -CONFIG_CRYPTO_AES_TI=y -CONFIG_CRYPTO_DES=y -CONFIG_CRYPTO_ARC4=y -CONFIG_CRYPTO_CTS=y -CONFIG_CRYPTO_XTS=y -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_USER_API_RNG=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_AES_NI_INTEL=y -CONFIG_CRYPTO_ECDSA=y -CONFIG_CRYPTO_DEV_VIRTIO=y -CONFIG_CORDIC=m -CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y -CONFIG_DEBUG_INFO_BTF=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_UNWINDER_FRAME_POINTER=y - -CONFIG_FAT_FS=y -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=y - -# nftables (Docker iptables-nft backend) -CONFIG_NF_TABLES=m -CONFIG_NF_TABLES_INET=y -CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_COMPAT=m -CONFIG_NFT_NAT=m -CONFIG_NFT_CHAIN_NAT=m -CONFIG_NFT_MASQ=m -CONFIG_NFT_REDIR=m -CONFIG_NFT_CT=m -CONFIG_NFT_LOG=m -CONFIG_NFT_LIMIT=m -CONFIG_NFT_REJECT=m -CONFIG_NFT_REJECT_INET=m -CONFIG_NFT_HASH=m -CONFIG_NETFILTER_XT_TARGET_CT=m diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg index 4dbf53d16..8fe114a9c 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg @@ -1 +1,5 @@ -CONFIG_TDX_GUEST_DRIVER=n +# In-tree TDX guest driver provides /dev/tdx_guest and the ConfigFS TSM +# report interface (/sys/kernel/config/tsm/report), replacing the former +# out-of-tree mod-tdx-guest module. +CONFIG_TDX_GUEST_DRIVER=y +CONFIG_TSM_REPORTS=y diff --git a/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb b/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb deleted file mode 100644 index 200982db1..000000000 --- a/meta-dstack/recipes-kernel/linux/linux-custom_6.17.6.bb +++ /dev/null @@ -1,36 +0,0 @@ -SUMMARY = "dstack Linux kernel 6.17.6 built from tarball" -DESCRIPTION = "Custom dstack kernel based on upstream Linux 6.17.6 with tiny Kconfig baseline tuned for TDX guests" -SECTION = "kernel" -LICENSE = "GPL-2.0-only" -LIC_FILES_CHKSUM = "file://COPYING;md5=6bc538ed5bd9a7fc9398086aedcd7e46" - -PV = "6.17.6" -LINUX_VERSION = "${PV}" - -inherit kernel - -FILESEXTRAPATHS:prepend := "${THISDIR}/files/6.17:${THISDIR}/files:" - -DEPENDS += "libyaml-native openssl-native util-linux-native elfutils-native" - -SRC_URI = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${PV}.tar.xz;downloadfilename=linux-${PV}.tar.xz \ - file://defconfig \ - file://0001-x86-tdx-select-dma-direct-remap.patch \ -" - -SRC_URI[sha256sum] = "8ecfbc6b693448abb46144a8d04d1e1631639c7661c1088425a2e5406f13c69c" - -S = "${UNPACKDIR}/linux-${PV}" - -LINUX_VERSION_EXTENSION = "-dstack" -KERNEL_VERSION_EXTENSION = "-dstack" - -# Enable BTF debug info for bpftool and out-of-tree modules (ZFS, WireGuard, etc.) -KERNEL_DEBUG = "True" - -# Keep packaging aligned with our tiny x86_64 guest machines. -COMPATIBLE_MACHINE = "(tdx|sev-snp|qemux86-64)" - -do_deploy:append() { - install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config -} diff --git a/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb b/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb deleted file mode 100644 index 96c0f7aa4..000000000 --- a/meta-dstack/recipes-kernel/linux/linux-custom_6.18.7.bb +++ /dev/null @@ -1,36 +0,0 @@ -SUMMARY = "dstack Linux kernel 6.18.7 built from tarball" -DESCRIPTION = "Custom dstack kernel based on upstream Linux 6.18.7 with tiny Kconfig baseline tuned for TDX guests" -SECTION = "kernel" -LICENSE = "GPL-2.0-only" -LIC_FILES_CHKSUM = "file://COPYING;md5=6bc538ed5bd9a7fc9398086aedcd7e46" - -PV = "6.18.7" -LINUX_VERSION = "${PV}" - -inherit kernel - -FILESEXTRAPATHS:prepend := "${THISDIR}/files/6.18:${THISDIR}/files:" - -DEPENDS += "libyaml-native openssl-native util-linux-native elfutils-native" - -SRC_URI = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${PV}.tar.xz;downloadfilename=linux-${PV}.tar.xz \ - file://defconfig \ - file://0001-x86-tdx-select-dma-direct-remap.patch \ -" - -SRC_URI[sha256sum] = "b726a4d15cf9ae06219b56d87820776e34d89fbc137e55fb54a9b9c3015b8f1e" - -S = "${UNPACKDIR}/linux-${PV}" - -LINUX_VERSION_EXTENSION = "-dstack" -KERNEL_VERSION_EXTENSION = "-dstack" - -# Enable BTF debug info for bpftool and out-of-tree modules (ZFS, WireGuard, etc.) -KERNEL_DEBUG = "True" - -# Keep packaging aligned with our tiny x86_64 guest machines. -COMPATIBLE_MACHINE = "(tdx|sev-snp|qemux86-64)" - -do_deploy:append() { - install -m 0644 ${B}/.config ${DEPLOYDIR}/kernel-config -} diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index 5900c497f..c28b7f228 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -11,6 +11,12 @@ SRC_URI += "file://dstack-docker.cfg \ file://dstack.cfg \ file://dstack.scc" +# TDX guests need DMA_DIRECT_REMAP for shared (decrypted) coherent DMA so +# devices like NVMe can complete I/O. INTEL_TDX_GUEST does not select it +# upstream (and the symbol is promptless, so a .cfg fragment cannot set it), +# hence this Kconfig patch. Scoped to tdx machines only. +SRC_URI:append:tdx = " file://0001-x86-tdx-select-dma-direct-remap.patch" + KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ features/netfilter/netfilter.scc \ From 4d4a0d325f9729c54c430ef0e4abd9536b59d89b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 30 May 2026 23:56:17 -0700 Subject: [PATCH 376/435] build: fix flavor dist names (dstack* not dstack-cloud*) and UKI python path - Makefile flavor_to_dist maps to dstack/dstack-dev/dstack-nvidia/dstack-nvidia-dev - dstack-uki.bb: glob python3.* site-packages instead of hardcoded python3.13 (wrynose native python version differs) --- Makefile | 2 +- meta-dstack/recipes-core/images/dstack-uki.bb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index da4ba8eb5..b86d2df72 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ export DIST_DIR FLAVORS ?= prod dev nvidia nvidia-dev # Map flavor to dist name for mkimage.sh -flavor_to_dist = $(if $(filter prod,$1),dstack-cloud,$(if $(filter dev,$1),dstack-cloud-dev,$(if $(filter nvidia,$1),dstack-cloud-nvidia,$(if $(filter nvidia-dev,$1),dstack-cloud-nvidia-dev,$1)))) +flavor_to_dist = $(if $(filter prod,$1),dstack,$(if $(filter dev,$1),dstack-dev,$(if $(filter nvidia,$1),dstack-nvidia,$(if $(filter nvidia-dev,$1),dstack-nvidia-dev,$1)))) all: dist diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/meta-dstack/recipes-core/images/dstack-uki.bb index 4a2254b33..07a8b0809 100644 --- a/meta-dstack/recipes-core/images/dstack-uki.bb +++ b/meta-dstack/recipes-core/images/dstack-uki.bb @@ -100,8 +100,10 @@ python do_uki() { native_sysroot = d.getVar('RECIPE_SYSROOT_NATIVE') staging_libdir = d.getVar('STAGING_LIBDIR_NATIVE') - # Find Python version directory for native packages - python_sitepackages = os.path.join(staging_libdir, 'python3.13', 'site-packages') + # Find Python version directory for native packages (version-agnostic) + import glob + _pyglob = sorted(glob.glob(os.path.join(staging_libdir, 'python3.*', 'site-packages'))) + python_sitepackages = _pyglob[-1] if _pyglob else os.path.join(staging_libdir, 'python3', 'site-packages') # Set environment for ukify env = os.environ.copy() From 4925deba10a6689331555d9c41784f4d3256e15b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 00:05:07 -0700 Subject: [PATCH 377/435] =?UTF-8?q?build:=20wrynose=20migration=20fixes=20?= =?UTF-8?q?=E2=80=94=20parse=20clean=20(28395=20targets,=200=20errors)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dev-setup: TEMPLATECONF -> openembedded-core/meta/conf/templates/default (wrynose moved templates out of meta-poky); add meta-poky + meta-yocto-bsp to LAYERS explicitly (oe-core default template no longer pulls them) - meta-dstack/meta-nvidia layer.conf: LAYERSERIES_COMPAT -> wrynose - dstack.conf: DISTRO_FEATURES_BACKFILL_CONSIDERED -> DISTRO_FEATURES_OPTED_OUT - meta-confidential-compute: move wic/ -> files/wic/ (wrynose wks search path) - dstack-rootfs-base.inc: drop stray diff3 conflict marker verified: virtual/kernel = official linux-yocto 6.18.24, dma-direct-remap patch wired via SRC_URI:append:tdx --- dev-setup | 6 ++++-- meta-confidential-compute | 2 +- meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/conf/layer.conf | 2 +- meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 - meta-nvidia/conf/layer.conf | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dev-setup b/dev-setup index 4be80749e..050a38990 100755 --- a/dev-setup +++ b/dev-setup @@ -16,7 +16,9 @@ fi THIS_SCRIPT=$(realpath "$THIS_SCRIPT") THIS_DIR=$(dirname "$THIS_SCRIPT") -LAYERS="$THIS_DIR/meta-confidential-compute \ +LAYERS="$THIS_DIR/meta-yocto/meta-poky \ + $THIS_DIR/meta-yocto/meta-yocto-bsp \ + $THIS_DIR/meta-confidential-compute \ $THIS_DIR/meta-openembedded/meta-oe \ $THIS_DIR/meta-openembedded/meta-python \ $THIS_DIR/meta-openembedded/meta-networking \ @@ -46,7 +48,7 @@ for script in $THIS_DIR/setup.d/*.sh; do done pushd "$BUILD_DIR" -BDIR="." TEMPLATECONF=$THIS_DIR/meta-yocto/meta-poky/conf/templates/default source $OE_INIT +BDIR="." TEMPLATECONF=$THIS_DIR/openembedded-core/meta/conf/templates/default source $OE_INIT popd bitbake-layers add-layer $LAYERS diff --git a/meta-confidential-compute b/meta-confidential-compute index 6bb953534..c829fbc40 160000 --- a/meta-confidential-compute +++ b/meta-confidential-compute @@ -1 +1 @@ -Subproject commit 6bb9535349131a96f47bddd1cefaf797a8bb465a +Subproject commit c829fbc4032490ceb900a2ed38f066a515aa0bb3 diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 515551e28..f01d89323 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -2,7 +2,7 @@ require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" -DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" +DISTRO_FEATURES_OPTED_OUT += "sysvinit" DISTRO_VERSION = "0.6.0" DISTROOVERRIDES = "poky:dstack" diff --git a/meta-dstack/conf/layer.conf b/meta-dstack/conf/layer.conf index 7258ad32b..10bb0a01c 100644 --- a/meta-dstack/conf/layer.conf +++ b/meta-dstack/conf/layer.conf @@ -5,4 +5,4 @@ BBFILE_COLLECTIONS += "dstack" BBFILE_PATTERN_dstack := "^${LAYERDIR}/" BBFILE_PRIORITY_dstack = "20" LAYERVERSION_dstack = "4" -LAYERSERIES_COMPAT_dstack = "whinlatter" +LAYERSERIES_COMPAT_dstack = "wrynose" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 34d547346..4862014ff 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -60,7 +60,6 @@ IMAGE_INSTALL = "\ kernel-module-xt-set \ kernel-module-xt-nflog \ kernel-module-xt-physdev \ -||||||| base fuse3 \ fuse3-utils \ pigz \ diff --git a/meta-nvidia/conf/layer.conf b/meta-nvidia/conf/layer.conf index 568d6d5b2..79dbcb466 100644 --- a/meta-nvidia/conf/layer.conf +++ b/meta-nvidia/conf/layer.conf @@ -10,4 +10,4 @@ BBFILE_PATTERN_nvidia = "^${LAYERDIR}/" BBFILE_PRIORITY_nvidia = "12" LICENSE_PATH += " ${LAYERDIR}/custom-licenses" -LAYERSERIES_COMPAT_nvidia = "whinlatter" +LAYERSERIES_COMPAT_nvidia = "wrynose" From ba6c0056b1a12d6e7eb22df44985463d21f9b966 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 00:34:55 -0700 Subject: [PATCH 378/435] distro: set INIT_MANAGER=systemd (wrynose udev fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit poky.conf defaults POKY_INIT_MANAGER=sysvinit, which pulls init-manager-sysvinit.inc and appends sysvinit to DISTRO_FEATURES — that conflicts with systemd so both systemd and eudev get skipped and nothing RPROVIDES udev (breaks cryptsetup -> dstack-initramfs). Setting INIT_MANAGER before requiring the poky-derived cvm.conf selects init-manager-systemd.inc. --- meta-dstack/conf/distro/dstack.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index f01d89323..397ca34cc 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -1,8 +1,11 @@ +# Select systemd init before requiring poky-derived distro config, otherwise +# poky.conf's POKY_INIT_MANAGER="sysvinit" pulls init-manager-sysvinit.inc which +# appends sysvinit to DISTRO_FEATURES and conflicts with systemd (breaks udev). +INIT_MANAGER = "systemd" require conf/distro/cvm.conf DISTRO = "dstack" DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" -DISTRO_FEATURES_OPTED_OUT += "sysvinit" DISTRO_VERSION = "0.6.0" DISTROOVERRIDES = "poky:dstack" From dd12be483bc102b9c72d17f99d959ff53be7fd91 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 01:10:06 -0700 Subject: [PATCH 379/435] dstack-ovmf: drop S = ${WORKDIR}/git (wrynose sets S for git fetches) --- meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb | 1 - 1 file changed, 1 deletion(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 84ca1b321..6b3544c76 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -70,7 +70,6 @@ inherit deploy PARALLEL_MAKE = "" -S = "${WORKDIR}/git" DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" From d8da197c980d21e852bd3349d9dbd83a9a0e6a54 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 01:18:49 -0700 Subject: [PATCH 380/435] dstack-ovmf: add Upstream-Status to local patches (wrynose fatal QA) --- .../dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch | 1 + .../dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch | 1 + .../dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch | 1 + 3 files changed, 3 insertions(+) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch index 3a50bed71..7a665e3ed 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch @@ -2,6 +2,7 @@ From 672d571ed826157e15969b7ba0ec46ab622a7c44 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 08:44:20 +0000 Subject: [PATCH] Debug prefix map +Upstream-Status: Inappropriate [dstack-specific OVMF build/reproducibility tweak] --- BaseTools/Conf/tools_def.template | 18 +++++++++--------- diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch index 33726e03d..ab1da4678 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch @@ -2,6 +2,7 @@ From 9133327256392a17883ad3ed91ad63ecbac50f08 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 08:44:21 +0000 Subject: [PATCH] Reproduciable +Upstream-Status: Inappropriate [dstack-specific OVMF build/reproducibility tweak] --- BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch index 61bb4a845..153a9e9d9 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch @@ -2,6 +2,7 @@ From 216280451e52ba73bc408e58c5a3b13d863e350f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Mar 2026 08:44:21 +0000 Subject: [PATCH] Declare ProcessLibraryConstructorList +Upstream-Status: Inappropriate [dstack-specific OVMF build/reproducibility tweak] --- OvmfPkg/IntelTdx/Sec/SecMain.c | 1 + From 19114d9cf58720dbbe777259308eca75d31e5fd2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 01:44:31 -0700 Subject: [PATCH 381/435] dstack-ovmf: build on wrynose (edk2-stable202511 + NASM 3.0 fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - bump edk2 stable202502 -> stable202511 (202502 won't assemble with wrynose NASM 3.01); changes RTMR[0] -> needs new dstack-mr OvmfVariant baseline - add oe-core's CpuExceptionHandlerLib push-instruction NASM 3.0 backport - drop 0003/0004 reproducibility patches (don't apply to 202511 template; not needed for functional image — rebase from oe-core versions for production) - drop 0005-Declare-ProcessLibraryConstructorList (edk2 202511 declares it natively) - OVMF_VARIANT -> stable202511 --- ...eclare-ProcessLibraryConstructorList.patch | 38 --------------- ...ceptionHandlerLib-fix-push-instructi.patch | 46 +++++++++++++++++++ .../dstack-ovmf/dstack-ovmf_git.bb | 14 +++--- 3 files changed, 54 insertions(+), 44 deletions(-) delete mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch deleted file mode 100644 index 153a9e9d9..000000000 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-Declare-ProcessLibraryConstructorList.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 216280451e52ba73bc408e58c5a3b13d863e350f Mon Sep 17 00:00:00 2001 -From: Kevin Wang -Date: Wed, 18 Mar 2026 08:44:21 +0000 -Subject: [PATCH] Declare ProcessLibraryConstructorList -Upstream-Status: Inappropriate [dstack-specific OVMF build/reproducibility tweak] - ---- - OvmfPkg/IntelTdx/Sec/SecMain.c | 1 + - OvmfPkg/Sec/SecMain.c | 1 + - 2 files changed, 2 insertions(+) - -diff --git a/OvmfPkg/IntelTdx/Sec/SecMain.c b/OvmfPkg/IntelTdx/Sec/SecMain.c -index 7f2d28af95..fcc6b97c28 100644 ---- a/OvmfPkg/IntelTdx/Sec/SecMain.c -+++ b/OvmfPkg/IntelTdx/Sec/SecMain.c -@@ -163,6 +163,7 @@ SecCoreStartupWithStack ( - IdtDescriptor.Base = (UINTN)&IdtTableInStack.IdtTable; - IdtDescriptor.Limit = (UINT16)(sizeof (IdtTableInStack.IdtTable) - 1); - -+ void ProcessLibraryConstructorList (); - ProcessLibraryConstructorList (); - - // -diff --git a/OvmfPkg/Sec/SecMain.c b/OvmfPkg/Sec/SecMain.c -index d13a948509..58c48c967c 100644 ---- a/OvmfPkg/Sec/SecMain.c -+++ b/OvmfPkg/Sec/SecMain.c -@@ -885,6 +885,7 @@ SecCoreStartupWithStack ( - InitializeCpuExceptionHandlers (NULL); - } - -+ void ProcessLibraryConstructorList (); - ProcessLibraryConstructorList (); - - if (!SevEsIsEnabled ()) { --- -2.43.0 - diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch new file mode 100644 index 000000000..650e084b5 --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch @@ -0,0 +1,46 @@ +From 7e6be0f4068a2158af3c97e873edb33fa4d5c6b8 Mon Sep 17 00:00:00 2001 +From: Gerd Hoffmann +Date: Mon, 3 Nov 2025 09:56:52 +0100 +Subject: [PATCH 5/5] UefiCpuPkg/CpuExceptionHandlerLib: fix push instructions + +Nasm 3.0 complains about 'dword' being invalid. The comment talks about +a '8-byte value' so 'qword' should be correct here. + +With this change the extra comment explaining that the instruction +actually pushes an 8-byte value despite the 'dword' keyword is not +needed any more. Drop it. + +Fixes: https://github.com/tianocore/edk2/issues/11635 +Signed-off-by: Gerd Hoffmann +Upstream-Status: Backport [https://github.com/tianocore/edk2/commit/9ccf8751a74f26142e584c7b7c7572a182b67997] +(cherry picked from commit 9ccf8751a74f26142e584c7b7c7572a182b67997) +Signed-off-by: Ankur Tyagi +--- + .../CpuExceptionHandlerLib/X64/ExceptionHandlerAsm.nasm | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/UefiCpuPkg/Library/CpuExceptionHandlerLib/X64/ExceptionHandlerAsm.nasm b/UefiCpuPkg/Library/CpuExceptionHandlerLib/X64/ExceptionHandlerAsm.nasm +index 3d64ac9080..671ed98f85 100644 +--- a/UefiCpuPkg/Library/CpuExceptionHandlerLib/X64/ExceptionHandlerAsm.nasm ++++ b/UefiCpuPkg/Library/CpuExceptionHandlerLib/X64/ExceptionHandlerAsm.nasm +@@ -57,7 +57,7 @@ ALIGN 8 + AsmIdtVectorBegin: + %assign Vector 0 + %rep 256 +- push strict dword %[Vector] ; This instruction pushes sign-extended 8-byte value on stack ++ push strict qword %[Vector] + push rax + %ifdef NO_ABSOLUTE_RELOCS_IN_TEXT + mov rax, strict qword 0 ; mov rax, ASM_PFX(CommonInterruptEntry) +@@ -70,7 +70,7 @@ AsmIdtVectorBegin: + AsmIdtVectorEnd: + + HookAfterStubHeaderBegin: +- push strict dword 0 ; 0 will be fixed ++ push strict qword 0 ; 0 will be fixed + VectorNum: + push rax + %ifdef NO_ABSOLUTE_RELOCS_IN_TEXT +-- +2.47.3 + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 6b3544c76..1a0af5865 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -22,9 +22,7 @@ BUILD_CFLAGS += "-Wno-error=stringop-overflow" SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0001-Update-path-to-native-BaseTools.patch \ file://0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch \ - file://0003-Debug-prefix-map.patch \ - file://0004-Reproduciable.patch \ - file://0005-Declare-ProcessLibraryConstructorList.patch \ + file://0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch \ " # Pinned to edk2-stable202502 (Feb 2025) instead of the latest stable202505. @@ -41,8 +39,12 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ # stable202502 contains none of them and so produces the same 13-event RTMR[0] # layout as the legacy 3a3b12cb snapshot dstack used pre-upgrade, while still # carrying 5 months of post-Sep-2024 EDK2 fixes (incl. CVEs). -PV = "edk2-stable202502" -SRCREV = "fbe0805b2091393406952e84724188f8c1941837" +# NOTE(wrynose): the stable202502 pin is INCOMPATIBLE with Yocto wrynose's NASM +# 3.01 (edk2 ExceptionHandlerAsm fails to assemble). Bumped to stable202511 (same +# as oe-core wrynose) to build. This changes the RTMR[0] event chain — a new +# OvmfVariant baseline in dstack-mr / dstack-types is required for attestation. +PV = "edk2-stable202511" +SRCREV = "46548b1adac82211d8d11da12dd914f41e7aa775" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" # Tag identifying the OVMF boot-time RTMR[0] event layout this build produces. @@ -50,7 +52,7 @@ UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" # verifiers can pick the matching dstack-mr code path without parsing PV. # Keep this in sync with the OvmfVariant enum in dstack/dstack-types when # bumping PV. -OVMF_VARIANT = "pre202505" +OVMF_VARIANT = "stable202511" CVE_PRODUCT = "edk2" CVE_VERSION = "${@d.getVar('PV').split('-')[1]}" From 42f8b3fb0b00887238d82445869af2504e10db6f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 01:49:51 -0700 Subject: [PATCH 382/435] meta-virtualization: bump to runc patch-fuzz fix --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index af5a04f1b..effab7a4b 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit af5a04f1b1352aa0f2aafb9d0b70dfb2f3515ab5 +Subproject commit effab7a4bc4f2b4d1c7036f52d41c64a2f4a77ae From 3ac32c59ba7a13f49157f24ac3692c14edca6860 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 02:18:37 -0700 Subject: [PATCH 383/435] kernel: build CRYPTO_SHA256 in (dm-verity rootfs hash in initramfs) linux-yocto-tiny ships CONFIG_CRYPTO_SHA256=m; dm-verity in the initramfs can't load modules, so early rootfs verity failed with 'Cannot initialize hash function (-ENOENT)' and init died -> kernel panic. Force SHA256/SHA512 built-in. --- meta-dstack/recipes-kernel/linux/files/dstack.cfg | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index cf5038c0f..24aa50f37 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -33,3 +33,10 @@ CONFIG_MMC=n CONFIG_SCSI=n CONFIG_INPUT=n CONFIG_WLAN=n + +# dm-verity verifies the rootfs in the initramfs (no modules loaded yet), so the +# rootfs hash algorithm must be built-in. linux-yocto-tiny ships SHA256 as a +# module (=m); force it built-in or early verity fails with +# "verity: Cannot initialize hash function (-ENOENT)". +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y From d403f7d1cb77ea366421ae8fcebef4cb2fa2a4d7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 19:01:24 -0700 Subject: [PATCH 384/435] dstack-ovmf: keep edk2-stable202502 (dstack-mr compat) + build on wrynose dstack-mr can't yet compute measurements for newer edk2, so the pre202505 RTMR[0] layout must be preserved. Make 202502 build on wrynose by: - backporting edk2's NASM-3.0 CpuExceptionHandlerLib push-instruction fix - dropping 0005-Declare-ProcessLibraryConstructorList (GCC 15 rejects the K&R 'void f()' forward-decl as conflicting with edk2's EFIAPI prototype; the prototype is already in scope in 202502 so the manual decl is redundant) OVMF_VARIANT stays pre202505. --- .../recipes-core/dstack-ovmf/dstack-ovmf_git.bb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 1a0af5865..08478c9a9 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -22,6 +22,8 @@ BUILD_CFLAGS += "-Wno-error=stringop-overflow" SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0001-Update-path-to-native-BaseTools.patch \ file://0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch \ + file://0003-Debug-prefix-map.patch \ + file://0004-Reproduciable.patch \ file://0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch \ " @@ -39,12 +41,13 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ # stable202502 contains none of them and so produces the same 13-event RTMR[0] # layout as the legacy 3a3b12cb snapshot dstack used pre-upgrade, while still # carrying 5 months of post-Sep-2024 EDK2 fixes (incl. CVEs). -# NOTE(wrynose): the stable202502 pin is INCOMPATIBLE with Yocto wrynose's NASM -# 3.01 (edk2 ExceptionHandlerAsm fails to assemble). Bumped to stable202511 (same -# as oe-core wrynose) to build. This changes the RTMR[0] event chain — a new -# OvmfVariant baseline in dstack-mr / dstack-types is required for attestation. -PV = "edk2-stable202511" -SRCREV = "46548b1adac82211d8d11da12dd914f41e7aa775" +# NOTE(wrynose): stable202502 must be KEPT — dstack-mr cannot yet compute +# measurements for newer edk2 (RTMR[0] event chain changed). stable202502 won't +# assemble with wrynose's NASM 3.01 out of the box, so we backport edk2's NASM-3.0 +# CpuExceptionHandlerLib push-instruction fix (0005-UefiCpuPkg-...) to make 202502 +# build while preserving the pre202505 measurement layout dstack-mr expects. +PV = "edk2-stable202502" +SRCREV = "fbe0805b2091393406952e84724188f8c1941837" UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" # Tag identifying the OVMF boot-time RTMR[0] event layout this build produces. @@ -52,7 +55,7 @@ UPSTREAM_CHECK_GITTAGREGEX = "(?Pedk2-stable.*)" # verifiers can pick the matching dstack-mr code path without parsing PV. # Keep this in sync with the OvmfVariant enum in dstack/dstack-types when # bumping PV. -OVMF_VARIANT = "stable202511" +OVMF_VARIANT = "pre202505" CVE_PRODUCT = "edk2" CVE_VERSION = "${@d.getVar('PV').split('-')[1]}" From 8f6b753ee986496b186ae04a6f16e70408552e64 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 31 May 2026 20:28:38 -0700 Subject: [PATCH 385/435] review: build-system fixes from PR review - mkimage.sh: deterministic GPT GUIDs (reproducible partitioned images); check verity env exists and sgdisk is installed before use - repro-build/check.sh: compare rootfs.img.parted.verity (new name); define YELLOW - build.sh: download from Dstack-TEE/meta-dstack releases (not the fork) - README: clone Dstack-TEE/meta-dstack for the reproducible-build steps - systemd bbappend: drop dangling blacklist-autofs4.conf FILES entry (never installed) - dstack-uki.bb: run ukify via argv list (no shell); fail clearly if ROOT_HASH/ DATA_SIZE missing - Makefile: build dstack-guest in images-common to avoid multiconfig fetch races --- Makefile | 5 +-- README.md | 2 +- build.sh | 2 +- meta-dstack/recipes-core/images/dstack-uki.bb | 29 +++++++++------ .../recipes-core/systemd/systemd_%.bbappend | 1 - mkimage.sh | 36 ++++++++++++++++--- repro-build/check.sh | 3 +- 7 files changed, 57 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index b86d2df72..952affb5b 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,10 @@ dist: images $(foreach flavor,$(FLAVORS),./mkimage.sh --dist-name $(call flavor_to_dist,$(flavor)) --flavor $(flavor);) # Build common artifacts (shared across all flavors) -# dstack-guest is built here to avoid concurrent build conflicts in multiconfig +# dstack-guest is built here first to warm sstate/downloads and avoid concurrent +# fetch/build conflicts when the per-flavor multiconfigs build it in parallel. images-common: - bitbake virtual/kernel dstack-initramfs dstack-ovmf + bitbake virtual/kernel dstack-initramfs dstack-ovmf dstack-guest # Build flavor-specific artifacts using multiconfig (serial to avoid deadlock warnings) images-flavors: diff --git a/README.md b/README.md index 616469d22..7ddcbb3ea 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ See https://github.com/Phala-Network/dstack-cloud for more details. ### Build commands ```bash -git clone https://github.com/Phala-Network/meta-dstack-cloud.git +git clone https://github.com/Dstack-TEE/meta-dstack.git cd meta-dstack/repro-build/ ./repro-build.sh ``` diff --git a/build.sh b/build.sh index 4481aefbf..70337c73d 100755 --- a/build.sh +++ b/build.sh @@ -310,7 +310,7 @@ download_image() { else BASENAME=dstack-cloud-$VERSION fi - URL=https://github.com/Dstack-TEE/meta-dstack-cloud/releases/download/$TAG/$BASENAME.tar.gz + URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/$TAG/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME ]; then echo "Image already exists" else diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/meta-dstack/recipes-core/images/dstack-uki.bb index 07a8b0809..7b871e364 100644 --- a/meta-dstack/recipes-core/images/dstack-uki.bb +++ b/meta-dstack/recipes-core/images/dstack-uki.bb @@ -88,6 +88,9 @@ python do_uki() { else: bb.fatal(f"Verity env file not found: {verity_env}") + if not root_hash or not data_size: + bb.fatal(f"ROOT_HASH/DATA_SIZE missing from verity env: {verity_env}") + # Build cmdline cmdline_base = d.getVar('UKI_CMDLINE_BASE') cmdline = f"{cmdline_base} dstack.rootfs_hash={root_hash} dstack.rootfs_size={data_size}" @@ -110,20 +113,24 @@ python do_uki() { env['PYTHONPATH'] = python_sitepackages ukify_path = os.path.join(native_sysroot, 'usr', 'bin', 'ukify') - ukify_cmd = f"{ukify_path} build" - ukify_cmd += f" --efi-arch {target_arch}" - ukify_cmd += f" --stub {stub}" - ukify_cmd += f" --linux={kernel}" - ukify_cmd += f" --initrd={initrd}" - ukify_cmd += f" --cmdline='{cmdline}'" - ukify_cmd += f" --tools={native_sysroot}/usr/lib/systemd/tools" - ukify_cmd += f" --output={output}" - - bb.note(f"Running: {ukify_cmd}") + # Pass an argument list (no shell) so values like cmdline don't need quoting + # and can't be split/expanded by the shell. + ukify_cmd = [ + ukify_path, "build", + f"--efi-arch={target_arch}", + f"--stub={stub}", + f"--linux={kernel}", + f"--initrd={initrd}", + f"--cmdline={cmdline}", + f"--tools={native_sysroot}/usr/lib/systemd/tools", + f"--output={output}", + ] + + bb.note(f"Running: {' '.join(ukify_cmd)}") bb.note(f"PYTHONPATH: {python_sitepackages}") import subprocess - result = subprocess.run(ukify_cmd, shell=True, capture_output=True, text=True, env=env) + result = subprocess.run(ukify_cmd, capture_output=True, text=True, env=env) if result.stdout: bb.note(result.stdout) if result.stderr: diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index ff4d28b74..1a7b43b66 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -20,7 +20,6 @@ EOF SYSTEMD_SERVICE:${PN}-vconsole-setup = "" PACKAGECONFIG:remove = "sysvinit logind" -FILES:${PN} += "${sysconfdir}/modprobe.d/blacklist-autofs4.conf" # Create a minimal package with only systemd-socket-proxyd # This avoids pulling in all of systemd-extra-utils diff --git a/mkimage.sh b/mkimage.sh index e0bdbc00d..4c27c5c16 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -4,6 +4,12 @@ set -e DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} ENABLE_UKI_IMAGE=${ENABLE_UKI_IMAGE:-1} +# Fixed GPT GUIDs so partitioned images are reproducible (sgdisk randomizes by +# default). Partitions are located by PARTLABEL, not GUID, so these are arbitrary. +DSTACK_DISK_GUID=${DSTACK_DISK_GUID:-d5acc000-0000-4000-8000-000000000000} +DSTACK_ROOTFS_PART_GUID=${DSTACK_ROOTFS_PART_GUID:-d5acc000-0000-4000-8000-000000000001} +DSTACK_EFI_PART_GUID=${DSTACK_EFI_PART_GUID:-d5acc000-0000-4000-8000-000000000002} + # Parse command line arguments while [ $# -gt 0 ]; do case "$1" in @@ -61,8 +67,20 @@ UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi # Verity env is in the flavor-specific work-shared directory VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/tdx/dm-verity/dstack-rootfs.squashfs.verity.env +if [ ! -f "${VERITY_ENV_FILE}" ]; then + echo "Error: verity env not found: ${VERITY_ENV_FILE}" >&2 + echo "Build the rootfs first, e.g.: bitbake mc:${FLAVOR}:dstack-rootfs" >&2 + exit 1 +fi echo "Loading verity env from ${VERITY_ENV_FILE}" -source ${VERITY_ENV_FILE} +# shellcheck source=/dev/null +source "${VERITY_ENV_FILE}" + +# Bare-metal partitioning needs sgdisk (gdisk) +if ! command -v sgdisk >/dev/null 2>&1; then + echo "Error: sgdisk not found; install 'gdisk' to build the partitioned rootfs." >&2 + exit 1 +fi DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) @@ -140,7 +158,12 @@ create_partitioned_rootfs() { local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) sgdisk --zap-all "$output_img" >/dev/null - sgdisk --new=1:${root_start_sector}:${root_end_sector} --typecode=1:8300 --change-name=1:'dstack-rootfs' "$output_img" >/dev/null + # Fixed GUIDs keep the image bit-for-bit reproducible (GPT otherwise + # randomizes disk/partition GUIDs). The rootfs is located by PARTLABEL. + sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ + --new=1:${root_start_sector}:${root_end_sector} --typecode=1:8300 \ + --partition-guid=1:"${DSTACK_ROOTFS_PART_GUID}" \ + --change-name=1:'dstack-rootfs' "$output_img" >/dev/null dd if="$rootfs_img" of="$output_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none ) @@ -171,8 +194,13 @@ build_uki_disk_image() { local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) sgdisk --zap-all "$disk_img" >/dev/null - sgdisk --new=1:${efi_start_sector}:${efi_end_sector} --typecode=1:ef00 --change-name=1:'EFI System Partition' "$disk_img" >/dev/null - sgdisk --new=2:${root_start_sector}:${root_end_sector} --typecode=2:8300 --change-name=2:'dstack-rootfs' "$disk_img" >/dev/null + sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ + --new=1:${efi_start_sector}:${efi_end_sector} --typecode=1:ef00 \ + --partition-guid=1:"${DSTACK_EFI_PART_GUID}" \ + --change-name=1:'EFI System Partition' "$disk_img" >/dev/null + sgdisk --new=2:${root_start_sector}:${root_end_sector} --typecode=2:8300 \ + --partition-guid=2:"${DSTACK_ROOTFS_PART_GUID}" \ + --change-name=2:'dstack-rootfs' "$disk_img" >/dev/null local tmp_dir tmp_dir=$(mktemp -d) diff --git a/repro-build/check.sh b/repro-build/check.sh index e83e257b3..e6feac044 100755 --- a/repro-build/check.sh +++ b/repro-build/check.sh @@ -11,6 +11,7 @@ ACTION=$1 # Colors for output GREEN='\033[0;32m' RED='\033[0;31m' +YELLOW='\033[1;33m' NC='\033[0m' IMAGE_NAME=${IMAGE_NAME:-dstack-rootfs} @@ -30,7 +31,7 @@ COMPARE_IMAGE_WHITELIST=( "initramfs.cpio.gz" "metadata.json" "ovmf.fd" - "rootfs.img.verity" + "rootfs.img.parted.verity" "sha256sum.txt" "gcp/efi-root/EFI/BOOT/BOOTX64.EFI" ) From fed4cc1dd0218649a04132972044923fef90f977 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 4 Jun 2026 09:40:46 +0800 Subject: [PATCH 386/435] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- mkimage.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mkimage.sh b/mkimage.sh index 4c27c5c16..6e100273e 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -241,6 +241,10 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" +if ! command -v sgdisk >/dev/null; then + echo "Error: cannot create partitioned rootfs image because 'sgdisk' is missing (install 'gdisk', or set ENABLE_UKI_IMAGE=0 and adjust tooling)" >&2 + exit 1 +fi create_partitioned_rootfs "$ROOTFS_IMAGE" "${OUTPUT_DIR}/rootfs.img.parted.verity" GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") From 9de071b017833d220266a12c3c2ee6f8ae06f064 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 3 Jun 2026 20:08:21 -0700 Subject: [PATCH 387/435] mkimage: dedupe sgdisk check from autofix; clearer message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Copilot autofix (fed4cc1) added an sgdisk check right before create_partitioned_rootfs, but an equivalent early check already existed. Drop the redundant early check and keep the call-site one, with a clearer message (the 'set ENABLE_UKI_IMAGE=0' hint was misleading — sgdisk is needed for the partitioned bare-metal image, not the UKI path). --- mkimage.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 6e100273e..1d8ed0164 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -76,12 +76,6 @@ echo "Loading verity env from ${VERITY_ENV_FILE}" # shellcheck source=/dev/null source "${VERITY_ENV_FILE}" -# Bare-metal partitioning needs sgdisk (gdisk) -if ! command -v sgdisk >/dev/null 2>&1; then - echo "Error: sgdisk not found; install 'gdisk' to build the partitioned rootfs." >&2 - exit 1 -fi - DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) # Output directory contains all artifacts; tarballs contain subsets @@ -241,8 +235,9 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" +# Bare-metal partitioning needs sgdisk (from the 'gdisk' package). if ! command -v sgdisk >/dev/null; then - echo "Error: cannot create partitioned rootfs image because 'sgdisk' is missing (install 'gdisk', or set ENABLE_UKI_IMAGE=0 and adjust tooling)" >&2 + echo "Error: cannot create partitioned rootfs image because 'sgdisk' is missing; install 'gdisk'." >&2 exit 1 fi create_partitioned_rootfs "$ROOTFS_IMAGE" "${OUTPUT_DIR}/rootfs.img.parted.verity" From 8ec3ab7cfd9f2e760da43966b1993c15008b45ae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 3 Jun 2026 20:18:53 -0700 Subject: [PATCH 388/435] dstack: bump submodule to bde0d038 (cloud merge + attestation hardening) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #64: advance the dstack submodule from the early cloud-merge commit to the current Dstack-TEE/dstack#701 head (bde0d038) — GCP TDX + AWS Nitro attestation, verified-PCR hardening, vendored dstack-cloud CLI. --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 1929bf3c6..bde0d038e 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 1929bf3c66d3cc7de94041fa742c49e0a34e69ba +Subproject commit bde0d038e5ee3d8441c65910dc5a3254cb342ac6 From 13a68909c5592c7d13fe7a6b4e69b02745e0e942 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 3 Jun 2026 20:23:05 -0700 Subject: [PATCH 389/435] Remove scripts/bin/dstack-cloud --- scripts/bin/dstack-cloud | 2461 -------------------------------------- 1 file changed, 2461 deletions(-) delete mode 100755 scripts/bin/dstack-cloud diff --git a/scripts/bin/dstack-cloud b/scripts/bin/dstack-cloud deleted file mode 100755 index acf9f7a93..000000000 --- a/scripts/bin/dstack-cloud +++ /dev/null @@ -1,2461 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -""" -dstack-cloud: Multi-cloud VM lifecycle management tool - -A production-grade CLI for managing dstack VMs on various cloud platforms. -Supports local configuration files similar to git's working model. - -Usage: - dstack-cloud new # Create a new project - dstack-cloud config-edit # Edit global configuration - dstack-cloud prepare # Generate shared files - dstack-cloud deploy # Deploy VM to cloud - dstack-cloud status # Check deployment status - dstack-cloud logs [--follow] # View serial console logs - dstack-cloud stop # Stop the VM - dstack-cloud start # Start a stopped VM - dstack-cloud remove # Remove the VM and cleanup - dstack-cloud list # List all deployments - dstack-cloud fw allow # Allow traffic on a port - dstack-cloud fw deny # Block traffic on a port - dstack-cloud fw remove # Remove a firewall rule - dstack-cloud fw list # List firewall rules -""" - -import argparse -import hashlib -import json -import logging -import os -import subprocess -import sys -import tempfile -import time -from dataclasses import dataclass, field, asdict -from datetime import datetime -from pathlib import Path -from typing import Optional, List, Dict, Any - -# Try to import cryptography libraries for env encryption -CRYPTO_AVAILABLE = False -ETH_CRYPTO_AVAILABLE = False -try: - from cryptography.hazmat.primitives.ciphers.aead import AESGCM - from cryptography.hazmat.primitives.asymmetric import x25519 - from cryptography.hazmat.primitives import serialization - CRYPTO_AVAILABLE = True -except Exception: - pass - -try: - from eth_keys import keys - from eth_utils import keccak - ETH_CRYPTO_AVAILABLE = True -except Exception: - pass - -# Default whitelist file location -DEFAULT_KMS_WHITELIST_PATH = os.path.expanduser("~/.config/dstack-cloud/kms-whitelist.json") - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger(__name__) - -# Configuration file names -APP_CONFIG_FILE = "app.json" -STATE_FILE = "state.json" -GLOBAL_CONFIG_PATH = os.path.expanduser("~/.config/dstack-cloud/config.json") -DEFAULT_OS_IMAGE = "dstack-cloud-0.6.0" - - -@dataclass -class App: - """Application configuration.""" - # App name - name: str = "myapp" - - # OS image - os_image: str = DEFAULT_OS_IMAGE - - # GCP cloud configuration - gcp_config: 'GcpConfig' = field(default_factory=lambda: GcpConfig()) - - # Docker compose file name (relative to project root) - docker_compose_file: str = "docker-compose.yaml" - - # Prelaunch script name (relative to project root) - prelaunch_script: str = "prelaunch.sh" - - # Environment file name (relative to project root) - env_file: str = ".env" - - # Instance identity - instance_id_seed: str = "" - app_id: str = "" - - # Gateway settings - gateway_enabled: bool = True - public_logs: bool = True - public_sysinfo: bool = True - public_tcbinfo: bool = True - - # KMS settings - key_provider: str = "kms" - - # Storage - storage_fs: str = "ext4" - - # Instance settings - no_instance_id: bool = False - secure_time: bool = False - - # Allowed environments - allowed_envs: List[str] = field(default_factory=list) - key_provider_id: str = "" - - def to_dict(self) -> Dict[str, Any]: - data = asdict(self) - # Convert GcpConfig to dict - if isinstance(data.get("gcp_config"), GcpConfig): - data["gcp_config"] = data["gcp_config"].to_dict() - return data - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'App': - known_fields = {f.name for f in cls.__dataclass_fields__.values()} - filtered = {k: v for k, v in data.items() if k in known_fields} - - # Convert gcp_config dict to GcpConfig object - if "gcp_config" in filtered and isinstance(filtered["gcp_config"], dict): - filtered["gcp_config"] = GcpConfig.from_dict(filtered["gcp_config"]) - - return cls(**filtered) - - @classmethod - def get_template(cls) -> Dict[str, Any]: - """Get default template for new projects.""" - import secrets - - # Generate random instance_id_seed (40 hex chars) - instance_id_seed = secrets.token_hex(20) - - # Generate random app_id (40 hex chars) - app_id = secrets.token_hex(20) - - return { - "name": "myapp", - "os_image": DEFAULT_OS_IMAGE, - "gcp_config": GcpConfig.get_template(), - "instance_id_seed": instance_id_seed, - "app_id": app_id, - "docker_compose_file": "docker-compose.yaml", - "prelaunch_script": "prelaunch.sh", - "env_file": ".env", - "gateway_enabled": True, - "public_logs": True, - "public_sysinfo": True, - "public_tcbinfo": True, - "key_provider": "kms", - "storage_fs": "ext4", - "no_instance_id": False, - "secure_time": False, - "allowed_envs": [], - "key_provider_id": "" - } - - -@dataclass -class GcpConfig: - """GCP deployment configuration.""" - # Required settings - project: str = "" - zone: str = "us-central1-a" - - # Instance settings - instance_name: str = "" # Required, no default - machine_type: str = "c3-standard-4" - - # Boot image settings - boot_image: str = "" # GCP image name (auto-derived from app.os_image if empty) - boot_image_tar: str = "" # Explicit tar file path (overrides search) - - # Data disk settings - data_image: str = "dstack-data-disk" - data_size: int = 20 - - # Storage settings - bucket: str = "" - - # Network settings - network: str = "default" - subnet: str = "" - - # Identity settings - service_account: str = "" - scopes: List[str] = field(default_factory=list) - - # Tags and labels - tags: List[str] = field(default_factory=list) - labels: Dict[str, str] = field(default_factory=dict) - - # Scheduling: provisioning model. "STANDARD" (default) or "SPOT". - # SPOT instances are required on projects without on-demand - # NVIDIA_H100_GPUS quota (most projects, as of 2026); GCP - # preempts them with ~30s notice and a max ~24h lifetime, but - # they're billed at a steep discount. - provisioning_model: str = "STANDARD" - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'GcpConfig': - known_fields = {f.name for f in cls.__dataclass_fields__.values()} - filtered = {k: v for k, v in data.items() if k in known_fields} - return cls(**filtered) - - @classmethod - def get_template(cls) -> Dict[str, Any]: - """Get default template for new projects.""" - return { - "project": "", - "zone": "us-central1-a", - "instance_name": "dstack-vm", - "machine_type": "c3-standard-4", - "boot_image": "", - "boot_image_tar": "", - "data_image": "dstack-data-disk", - "data_size": 20, - "bucket": "", - "network": "default", - "subnet": "", - "service_account": "", - "scopes": [], - "tags": [], - "labels": {}, - "provisioning_model": "STANDARD" - } - - -@dataclass -class DeploymentState: - """Deployment state tracking.""" - instance_name: str = "" - project: str = "" - zone: str = "" - external_ip: str = "" - internal_ip: str = "" - status: str = "" # RUNNING, STOPPED, TERMINATED, etc. - created_at: str = "" - updated_at: str = "" - boot_image: str = "" - data_image: str = "" - shared_image: str = "" - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'DeploymentState': - known_fields = {f.name for f in cls.__dataclass_fields__.values()} - filtered = {k: v for k, v in data.items() if k in known_fields} - return cls(**filtered) - - -class CloudDeploymentManager: - """Manages multi-cloud VM deployments.""" - - def __init__(self, work_dir: Optional[str] = None): - self.work_dir = Path(work_dir) if work_dir else Path.cwd() - - def _load_global_config(self) -> Dict[str, Any]: - """Load global configuration.""" - if os.path.exists(GLOBAL_CONFIG_PATH): - with open(GLOBAL_CONFIG_PATH, 'r') as f: - return json.load(f) - return {} - - def _save_global_config(self, config: Dict[str, Any]) -> None: - """Save global configuration.""" - os.makedirs(os.path.dirname(GLOBAL_CONFIG_PATH), exist_ok=True) - with open(GLOBAL_CONFIG_PATH, 'w') as f: - json.dump(config, f, indent=2) - - def _get_shared_dir(self) -> Path: - """Get the shared directory path (at project root).""" - return self.work_dir / "shared" - - def load_gcp_config(self) -> 'GcpConfig': - """Load GCP configuration from app.gcp_config.""" - # Load app config which contains gcp_config - app = self.load_app_config() - - # Get local gcp_config from app - local_gcp = app.gcp_config.to_dict() - - # Merge global config with local config - global_config = self._load_global_config() - global_gcp = global_config.get("gcp", {}) - - # Global config is used as fallback for empty values in local config - merged = {**global_gcp} # Start with global config - for key, value in local_gcp.items(): - # Only override with local value if it's non-empty - if value or value is False or value == 0: - merged[key] = value - return GcpConfig.from_dict(merged) - - def save_gcp_config(self, config: GcpConfig) -> None: - """Save GCP configuration to app.gcp_config.""" - # Load the full app config - app = self.load_app_config() - - # Update gcp_config - app.gcp_config = config - - # Save the updated app config - self.save_app_config(app) - - def load_app_config(self, required: bool = False) -> App: - """Load application configuration. - - Args: - required: If True, raise error when app.json doesn't exist - """ - app_config_path = self.work_dir / APP_CONFIG_FILE - - if not app_config_path.exists(): - if required: - raise FileNotFoundError( - f"No {APP_CONFIG_FILE} found in {self.work_dir}. " - f"Run 'dstack-cloud new ' to create a project." - ) - # Return default config if file doesn't exist - return App() - - with open(app_config_path, 'r') as f: - return App.from_dict(json.load(f)) - - def save_app_config(self, app: App) -> None: - """Save application configuration.""" - app_config_path = self.work_dir / APP_CONFIG_FILE - with open(app_config_path, 'w') as f: - json.dump(app.to_dict(), f, indent=2) - - def _generate_app_compose(self, app: App, env_names: Optional[List[str]] = None) -> Dict[str, Any]: - """Generate app-compose.json content from App configuration.""" - # Read docker-compose.yaml content - docker_compose_path = self.work_dir / app.docker_compose_file - if not docker_compose_path.exists(): - docker_compose_content = "" - else: - with open(docker_compose_path, 'r') as f: - docker_compose_content = f.read() - - # Read prelaunch script content - prelaunch_path = self.work_dir / app.prelaunch_script - if not prelaunch_path.exists(): - prelaunch_content = "" - else: - with open(prelaunch_path, 'r') as f: - prelaunch_content = f.read() - - # Merge app.allowed_envs with env_names from .env file - allowed_envs = list(app.allowed_envs) if app.allowed_envs else [] - if env_names: - allowed_envs.extend(env_names) - # Remove duplicates - allowed_envs = list(set(allowed_envs)) - - return { - "manifest_version": 2, - "name": app.name, - "runner": "docker-compose", - "docker_compose_file": docker_compose_content, - "gateway_enabled": app.gateway_enabled, - "public_logs": app.public_logs, - "public_sysinfo": app.public_sysinfo, - "public_tcbinfo": app.public_tcbinfo, - "key_provider_id": app.key_provider_id, - "allowed_envs": allowed_envs, - "no_instance_id": app.no_instance_id, - "secure_time": app.secure_time, - "key_provider": app.key_provider, - "storage_fs": app.storage_fs, - "pre_launch_script": prelaunch_content - } - - def _generate_sys_config(self, global_config: Dict[str, Any], - gcp_config: GcpConfig, app: App) -> Dict[str, Any]: - """Generate .sys-config.json content.""" - # Get services section from global config - services = global_config.get("services", {}) - - # Get KMS URLs from global config - kms_urls = services.get("kms_urls", []) - if not kms_urls: - kms_urls = ["https://kms.tdxlab.dstack.org:12001"] - - # Get gateway URLs from global config - gateway_urls = services.get("gateway_urls", []) - if not gateway_urls: - gateway_urls = ["https://gateway.tdxlab.dstack.org:12002"] - - # Get other settings - pccs_url = services.get("pccs_url", "") - - # Read OS image hash from the local image directory - os_image_hash = "" - try: - # Find the image directory and read hash file - search_paths = global_config.get("image_search_paths", []) - local_image = gcp_config.boot_image if gcp_config.boot_image else "" - if not local_image: - # Use app.os_image if boot_image is not set - local_image = app.os_image - - for search_path in search_paths: - search_path = os.path.expanduser(search_path) - if not os.path.isabs(search_path): - search_path = os.path.join(self.work_dir, search_path) - - hash_file = Path(search_path) / local_image / "auth_hash.txt" - if hash_file.exists(): - with open(hash_file, 'r') as f: - os_image_hash = f.read().strip() - logger.info(f"Read OS image hash from {hash_file}") - break - except Exception as e: - logger.warning(f"Could not read OS image hash: {e}") - - # Build vm_config - vm_config = { - "spec_version": 2, - "os_image_hash": os_image_hash - } - - return { - "kms_urls": kms_urls, - "gateway_urls": gateway_urls, - "pccs_url": pccs_url, - "vm_config": json.dumps(vm_config) - } - - def load_state(self) -> Optional[DeploymentState]: - """Load deployment state.""" - state_path = self.work_dir / STATE_FILE - - if not state_path.exists(): - return None - - with open(state_path, 'r') as f: - return DeploymentState.from_dict(json.load(f)) - - def save_state(self, state: DeploymentState) -> None: - """Save deployment state.""" - state_path = self.work_dir / STATE_FILE - state.updated_at = datetime.now().isoformat() - with open(state_path, 'w') as f: - json.dump(state.to_dict(), f, indent=2) - - def _run_gcloud(self, args: List[str], capture: bool = True, - check: bool = True) -> subprocess.CompletedProcess: - """Run a gcloud command.""" - cmd = ["gcloud"] + args - logger.debug(f"Running: {' '.join(cmd)}") - - if capture: - result = subprocess.run(cmd, capture_output=True, text=True) - else: - result = subprocess.run(cmd) - - if check and result.returncode != 0: - error_msg = result.stderr if capture else "Command failed" - raise RuntimeError(f"gcloud command failed: {error_msg}") - - return result - - def _run_gsutil(self, args: List[str], check: bool = True) -> subprocess.CompletedProcess: - """Run a gsutil command.""" - cmd = ["gsutil"] + args - logger.debug(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd, capture_output=True, text=True) - - if check and result.returncode != 0: - raise RuntimeError(f"gsutil command failed: {result.stderr}") - - return result - - def _ensure_data_disk_image(self, config: GcpConfig) -> str: - """Ensure the data disk image exists, creating it if necessary. - - Creates a minimal disk image with GPT partition table and a partition - labeled 'dstack-data' so the guest can discover it. - - Returns the image name to use. - """ - image_name = config.data_image - - # Check if image already exists - result = self._run_gcloud([ - "compute", "images", "describe", image_name, - f"--project={config.project}" - ], check=False) - - if result.returncode == 0: - logger.debug(f"Data disk image '{image_name}' already exists") - return image_name - - logger.info(f"Data disk image '{image_name}' not found, creating...") - - # Create a minimal raw disk image with GPT partition table - with tempfile.TemporaryDirectory() as tmpdir: - raw_file = os.path.join(tmpdir, "disk.raw") - - # Create a 10MB sparse file (enough for GPT) - disk_size_bytes = 10 * 1024 * 1024 - with open(raw_file, 'wb') as f: - f.truncate(disk_size_bytes) - - # Create GPT partition table with dstack-data partition using sgdisk - # -o: clear and create new GPT - # -n 1:0:0: create partition 1, start at first available, end at last available - # -c 1:dstack-data: set partition 1 name (PARTLABEL) to dstack-data - result = subprocess.run( - ["sgdisk", "-o", "-n", "1:0:0", "-c", "1:dstack-data", raw_file], - capture_output=True, text=True - ) - if result.returncode != 0: - raise RuntimeError(f"Failed to create GPT partition table: {result.stderr}") - - logger.debug("Created GPT partition table with dstack-data label") - - # Compress to tar.gz for upload - tar_file = os.path.join(tmpdir, "disk.tar.gz") - result = subprocess.run( - ["tar", "-czf", tar_file, "-C", tmpdir, "disk.raw"], - capture_output=True, text=True - ) - if result.returncode != 0: - raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") - - # Upload to GCS - gcs_path = f"{config.bucket}/{image_name}.tar.gz" - logger.info(f"Uploading data disk image to {gcs_path}...") - self._run_gsutil(["cp", tar_file, gcs_path]) - - # Create GCP image from the uploaded file - logger.info(f"Creating GCP image '{image_name}'...") - self._run_gcloud([ - "compute", "images", "create", image_name, - f"--project={config.project}", - f"--source-uri={gcs_path}", - "--guest-os-features=GVNIC" - ]) - - # Clean up GCS file - self._run_gsutil(["rm", gcs_path], check=False) - - logger.info(f"Created data disk image '{image_name}'") - - return image_name - - def new( - self, - name: str, - os_image: Optional[str] = None, - app_id: Optional[str] = None, - gateway_enabled: Optional[bool] = None, - key_provider: Optional[str] = None, - storage_fs: Optional[str] = None, - secure_time: Optional[bool] = None, - no_instance_id: Optional[bool] = None, - project: Optional[str] = None, - zone: Optional[str] = None, - instance_name: Optional[str] = None, - machine_type: Optional[str] = None, - data_size: Optional[int] = None - ) -> None: - """Create a new project directory with template configuration.""" - project_dir = Path.cwd() / name - if project_dir.exists(): - raise FileExistsError(f"Directory '{name}' already exists.") - - # Create project directory - project_dir.mkdir() - - # Update work_dir to the new project directory - self.work_dir = project_dir - - if not instance_name: - instance_name = f"dstack-{name}" - - # Initialize the project (non-interactive by default for new command) - self._init_project( - force=False, - interactive=False, - app_name=name, - os_image=os_image, - app_id=app_id, - gateway_enabled=gateway_enabled, - key_provider=key_provider, - storage_fs=storage_fs, - secure_time=secure_time, - no_instance_id=no_instance_id, - project=project, - zone=zone, - instance_name=instance_name, - machine_type=machine_type, - data_size=data_size - ) - - logger.info(f"Created new project: {name}") - logger.info(f"Project directory: {project_dir}") - logger.info("") - - def _init_project( - self, - force: bool = False, - interactive: bool = True, - app_name: Optional[str] = None, - os_image: Optional[str] = None, - app_id: Optional[str] = None, - gateway_enabled: Optional[bool] = None, - key_provider: Optional[str] = None, - storage_fs: Optional[str] = None, - secure_time: Optional[bool] = None, - no_instance_id: Optional[bool] = None, - project: Optional[str] = None, - zone: Optional[str] = None, - instance_name: Optional[str] = None, - machine_type: Optional[str] = None, - data_size: Optional[int] = None - ) -> None: - """Initialize project configuration.""" - # Interactive prompts for required fields (only if not provided via CLI) - instance_name_cli = instance_name - - if interactive: - print(f"\n=== dstack-cloud Project Initialization ===\n") - - # Prompt for app name if not provided - if app_name is None: - while True: - app_name = input("App name [myapp]: ").strip() - if not app_name: - app_name = "myapp" - if app_name: - break - print("App name cannot be empty.") - - # Prompt for instance name (required) - if not instance_name_cli: - while True: - instance_name = input("GCP instance name: ").strip() - if instance_name: - break - print("Instance name is required.") - else: - instance_name = instance_name_cli - - print("") # Empty line for readability - else: - # Non-interactive mode: use CLI provided values or fail - if app_name is None: - app_name = "myapp" - if not instance_name_cli: - raise ValueError("instance_name is required. Use --instance-name to specify it.") - instance_name = instance_name_cli - - # Create shared directory at project root (for system-generated files) - shared_dir = self.work_dir / "shared" - shared_dir.mkdir(parents=True, exist_ok=True) - - # Generate app config template with embedded gcp_config - app_template = App.get_template() - - # Apply CLI-provided values (only if specified) - if app_name: - app_template["name"] = app_name - if os_image: - app_template["os_image"] = os_image - if app_id: - # Validate app_id format (40 hex chars) - if len(app_id) != 40 or not all(c in '0123456789abcdef' for c in app_id.lower()): - raise ValueError("app_id must be exactly 40 hexadecimal characters") - app_template["app_id"] = app_id - if gateway_enabled is not None: - app_template["gateway_enabled"] = gateway_enabled - if key_provider is not None: - app_template["key_provider"] = key_provider - - # Auto-disable gateway when key_provider is not "kms" - if app_template["key_provider"] != "kms": - app_template["gateway_enabled"] = False - # Also set no_instance_id=True when KMS is not available - app_template["no_instance_id"] = True - # Remove env_file since .env is only supported in KMS mode - app_template.pop("env_file", None) - if storage_fs is not None: - app_template["storage_fs"] = storage_fs - if secure_time is not None: - app_template["secure_time"] = secure_time - if no_instance_id is not None: - app_template["no_instance_id"] = no_instance_id - - # Apply GCP config values - if instance_name: - app_template["gcp_config"]["instance_name"] = instance_name - if project: - app_template["gcp_config"]["project"] = project - if zone: - app_template["gcp_config"]["zone"] = zone - if machine_type: - app_template["gcp_config"]["machine_type"] = machine_type - if data_size is not None: - app_template["gcp_config"]["data_size"] = data_size - - # Create app.json at project root (not in .dstack/) - app_config_path = self.work_dir / APP_CONFIG_FILE - if not app_config_path.exists() or force: - with open(app_config_path, 'w') as f: - json.dump(app_template, f, indent=2) - - # Note: app-compose.json and .sys-config.json will be generated during deploy - - # Create docker-compose.yaml template at project root - docker_compose = self.work_dir / "docker-compose.yaml" - if not docker_compose.exists() or force: - with open(docker_compose, 'w') as f: - f.write("services:\n") - f.write(" nginx:\n") - f.write(" image: nginx:alpine\n") - f.write(" ports:\n") - f.write(" - \"80:80\"\n") - f.write(" restart: unless-stopped\n") - - # Create prelaunch.sh template at project root - prelaunch = self.work_dir / "prelaunch.sh" - if not prelaunch.exists() or force: - with open(prelaunch, 'w') as f: - f.write("#!/bin/sh\n") - f.write("# Prelaunch script - runs before starting containers\n") - os.chmod(prelaunch, 0o755) - - # Create .env template at project root (only for KMS mode) - if app_template["key_provider"] == "kms": - env_file = self.work_dir / ".env" - if not env_file.exists() or force: - with open(env_file, 'w') as f: - f.write("# Environment variables\n") - - # Create user-config template at project root - user_config = self.work_dir / ".user-config" - if not user_config.exists() or force: - with open(user_config, 'w') as f: - json.dump({}, f, indent=2) - - logger.info(f"Initialized project in {self.work_dir}") - logger.info("") - if interactive: - logger.info("Configuration:") - logger.info(f" App name: {app_name}") - logger.info(f" Instance name: {instance_name}") - logger.info("") - logger.info("Created files:") - logger.info(f" {app_config_path.name} - Application configuration (with embedded GCP config)") - logger.info(f" shared/ - System-generated files") - logger.info(f" {docker_compose.name} - Docker compose file") - logger.info(f" {prelaunch.name} - Prelaunch script") - if app_template["key_provider"] == "kms": - logger.info(f" .env - Environment variables") - logger.info(f" .user-config - User configuration") - logger.info("") - logger.info("Edit the configuration files to customize your deployment.") - - def config_edit(self) -> None: - """Edit global configuration with $EDITOR.""" - # Create global config if it doesn't exist - global_config_dir = os.path.dirname(GLOBAL_CONFIG_PATH) - os.makedirs(global_config_dir, exist_ok=True) - - if not os.path.exists(GLOBAL_CONFIG_PATH): - # Create template with organized sections and comments - template = { - "_comment_services": "Service endpoints configuration", - "services": { - "kms_urls": ["https://kms.tdxlab.dstack.org:12001"], - "gateway_urls": ["https://gateway.tdxlab.dstack.org:12002"], - "pccs_url": "", - }, - "_comment_images": "System image search paths (for boot_image_tar auto-discovery)", - "image_search_paths": [ - "~/.dstack/images" - ], - "_comment_gcp": "GCP cloud platform defaults", - "gcp": { - "project": "", - "zone": "us-central1-a", - "bucket": "" - } - } - with open(GLOBAL_CONFIG_PATH, 'w') as f: - json.dump(template, f, indent=2, ensure_ascii=False) - f.write("\n") # Add trailing newline - - # Get editor - editor = os.environ.get('EDITOR', 'vi') - - # Open editor - logger.info(f"Opening {GLOBAL_CONFIG_PATH} with {editor}...") - subprocess.run([editor, GLOBAL_CONFIG_PATH]) - - def prepare(self) -> None: - """Generate all files in shared directory.""" - import secrets - - # Load app config (required) - app = self.load_app_config(required=True) - - # Ensure instance_id_seed and app_id exist - if not app.instance_id_seed: - app.instance_id_seed = secrets.token_hex(20) - logger.info(f"Generated instance_id_seed: {app.instance_id_seed}") - - if not app.app_id: - app.app_id = secrets.token_hex(20) - logger.info(f"Generated app_id: {app.app_id}") - - # Save updated app config - self.save_app_config(app) - - # Get shared directory - shared_dir = self._get_shared_dir() - shared_dir.mkdir(parents=True, exist_ok=True) - - # Generate .instance_info (instance_id will be generated in CVM) - instance_info = { - "instance_id_seed": app.instance_id_seed, - "app_id": app.app_id - } - instance_info_path = shared_dir / ".instance_info" - with open(instance_info_path, 'w') as f: - json.dump(instance_info, f, indent=2) - logger.info(f"Generated {instance_info_path}") - - # Load GCP config for sys-config generation - try: - gcp_config = self.load_gcp_config() - global_config = self._load_global_config() - - # Generate .sys-config.json - sys_config_content = self._generate_sys_config(global_config, gcp_config, app) - sys_config_path = shared_dir / ".sys-config.json" - with open(sys_config_path, 'w') as f: - json.dump(sys_config_content, f, indent=2) - logger.info(f"Generated {sys_config_path}") - except FileNotFoundError as e: - logger.warning(f"Could not generate .sys-config.json: {e}") - - # Process .env file to collect env_names - env_path = self.work_dir / app.env_file - env_names = [] - if env_path.exists(): - envs = self._parse_env_file(env_path) - if envs: - env_names = list(envs.keys()) - logger.info(f"Found {len(env_names)} environment variable(s) in {app.env_file}") - else: - logger.info(f"{app.env_file} is empty") - - # Generate app-compose.json - app_compose_content = self._generate_app_compose(app, env_names=env_names if env_names else None) - app_compose_path = shared_dir / "app-compose.json" - with open(app_compose_path, 'w') as f: - json.dump(app_compose_content, f, indent=2) - logger.info(f"Generated {app_compose_path}") - - logger.info("") - logger.info(f"Shared files generated in: {shared_dir}") - logger.info("These files will be included in the shared disk image during deploy.") - - def _find_boot_image_tar(self, local_image: str) -> Optional[Path]: - """Search for boot image disk.raw file in configured search paths.""" - global_config = self._load_global_config() - search_paths = global_config.get("image_search_paths", []) - - logger.debug(f"Image search paths: {search_paths}") - logger.debug(f"Looking for image: {local_image}") - - # Expand ~ and convert to Path objects - expanded_paths = [] - for path in search_paths: - expanded = os.path.expanduser(path) - if not os.path.isabs(expanded): - expanded = os.path.join(self.work_dir, expanded) - expanded_paths.append(Path(expanded)) - - # Look for disk.raw in the image directory - patterns = [ - f"{local_image}/disk.raw", - ] - - for search_path in expanded_paths: - logger.debug(f"Checking search path: {search_path}, exists: {search_path.exists()}") - if not search_path.exists(): - continue - for pattern in patterns: - file_path = search_path / pattern - logger.debug(f"Checking: {file_path}, exists: {file_path.exists()}") - if file_path.exists(): - logger.info(f"Found boot image: {file_path}") - return file_path - - return None - - def pull(self, os_image: str) -> None: - """Download UKI image from remote repository or an absolute URL.""" - global_config = self._load_global_config() - search_paths = global_config.get("image_search_paths", []) - - if not search_paths: - logger.error("No image_search_paths configured in global config") - logger.error("Please set image_search_paths in: ~/.config/dstack-cloud/config.json") - return - - # Use the first search path - target_dir = Path(os.path.expanduser(search_paths[0])) - target_dir.mkdir(parents=True, exist_ok=True) - - # Check if os_image is an absolute URL - if os_image.startswith("http://") or os_image.startswith("https://"): - download_url = os_image - # Derive image name from URL filename - url_filename = download_url.rsplit("/", 1)[-1] - if url_filename.endswith("-uki.tar.gz"): - os_image = url_filename[:-len("-uki.tar.gz")] - elif url_filename.endswith(".tar.gz"): - os_image = url_filename[:-len(".tar.gz")] - else: - os_image = url_filename - download_tar = target_dir / url_filename - else: - # Extract version from os_image (e.g., dstack-cloud-nvidia-0.6.0 -> 0.6.0) - # Version is the last component after the last hyphen followed by digits - import re - version_match = re.search(r'-(\d+\.\d+\.\d+)$', os_image) - if not version_match: - logger.error(f"Could not extract version from image name: {os_image}") - logger.error("Expected format: dstack-cloud-- (e.g., dstack-cloud-nvidia-0.6.0)") - return - version = version_match.group(1) - download_url = f"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/Phala-Network/meta-dstack-cloud/releases/download/v{version}/{os_image}-uki.tar.gz" - download_tar = target_dir / f"{os_image}-uki.tar.gz" - - if download_tar.exists(): - logger.info(f"Download file already exists: {download_tar}") - response = input("Download again to overwrite? [y/N]: ").strip().lower() - if response != 'y': - logger.info("Download cancelled") - return - - logger.info(f"Downloading {os_image} UKI image from {download_url}...") - logger.info(f"Target: {download_tar}") - - try: - # Use curl to download with progress bar - subprocess.run( - ["curl", "-L", "-o", str(download_tar), download_url], - check=True - ) - logger.info(f"Successfully downloaded to {download_tar}") - except subprocess.CalledProcessError as e: - logger.error(f"Failed to download image: {e}") - # Clean up partial download - if download_tar.exists(): - download_tar.unlink() - raise - - # Extract the tar file - logger.info(f"Extracting {download_tar}...") - try: - subprocess.run( - ["tar", "-xzf", str(download_tar), "-C", str(target_dir)], - check=True - ) - logger.info(f"Successfully extracted to {target_dir / os_image}") - except subprocess.CalledProcessError as e: - logger.error(f"Failed to extract image: {e}") - raise - - # Verify the expected structure - expected_dir = target_dir / os_image - expected_disk = expected_dir / "disk.raw" - if expected_disk.exists(): - logger.info(f"Image ready: {expected_disk}") - else: - logger.warning(f"Expected file not found: {expected_disk}") - logger.warning(f"Downloaded structure may be incorrect") - - def _check_and_upload_boot_image(self, config: GcpConfig, app: App, force: bool = False) -> str: - """Check and upload boot image if needed. Returns the image name.""" - image_path = None - - # Derive GCP image name from app.os_image - gcp_image = config.boot_image - if not gcp_image: - # Convert OS image name (dstack-cloud-nvidia-0.6.0 -> dstack-cloud-nvidia-0-6-0) - gcp_image = app.os_image.replace(".", "-") - - # If boot_image_tar is specified, use it (legacy config name, can be disk.raw or tar.gz) - if config.boot_image_tar: - image_path = Path(os.path.expanduser(config.boot_image_tar)) - if not image_path.exists(): - logger.error("") - logger.error(f"Boot image not found: {image_path}") - logger.error("") - logger.error(f"Please download the image using:") - logger.error(f" dstack-cloud pull {app.os_image}") - logger.error("") - raise FileNotFoundError(f"Boot image not found: {image_path}") - else: - # Auto-discover from search paths using OS image name - logger.info(f"Searching for boot image '{app.os_image}'...") - image_path = self._find_boot_image_tar(app.os_image) - if not image_path: - logger.error("") - logger.error(f"Boot image '{app.os_image}' not found locally.") - logger.error("") - logger.error(f"Please download the image using:") - logger.error(f" dstack-cloud pull {app.os_image}") - logger.error("") - raise FileNotFoundError( - f"Boot image '{app.os_image}' not found. " - f"Run 'dstack-cloud pull {app.os_image}' to download it." - ) - - # Use gcp_image as the GCP image name - image_name = gcp_image - local_mtime = image_path.stat().st_mtime - - # Check if GCP image exists and is up-to-date - result = self._run_gcloud([ - "compute", "images", "describe", image_name, - f"--project={config.project}", - "--format=value(creationTimestamp)" - ], check=False) - - need_upload = False - gcp_creation_time = result.stdout.strip() if result.returncode == 0 else "" - - if force: - logger.info("Force enabled: will re-upload boot image") - need_upload = True - elif not gcp_creation_time: - logger.info(f"GCP image '{image_name}' does not exist, will upload") - need_upload = True - else: - # Parse GCP timestamp and compare - try: - from datetime import datetime - gcp_dt = datetime.fromisoformat(gcp_creation_time.replace('Z', '+00:00')) - gcp_epoch = gcp_dt.timestamp() - if local_mtime > gcp_epoch: - logger.info("Local image is newer than GCP image, will re-upload") - logger.info(f" Local: {datetime.fromtimestamp(local_mtime).isoformat()}") - logger.info(f" GCP: {gcp_creation_time}") - need_upload = True - else: - logger.info(f"GCP image '{image_name}' is up-to-date") - except Exception as e: - logger.warning(f"Could not parse GCP timestamp: {e}") - need_upload = True - - if need_upload: - # Compress disk.raw to tar.gz for upload - logger.info("Compressing disk.raw to tar.gz for upload...") - import tempfile - with tempfile.TemporaryDirectory() as tmpdir: - tar_file = os.path.join(tmpdir, "disk.tar.gz") - result = subprocess.run( - ["tar", "-czvf", tar_file, "-C", str(image_path.parent), "disk.raw"], - capture_output=True, - text=True - ) - if result.returncode != 0: - raise RuntimeError(f"Failed to create tar.gz: {result.stderr}") - - logger.info("Uploading boot image to GCS...") - self._run_gsutil([ - "cp", tar_file, f"{config.bucket}/{image_name}.tar.gz" - ]) - - # Delete existing image if present - if gcp_creation_time: - logger.info("Deleting existing GCP image...") - self._run_gcloud([ - "compute", "images", "delete", image_name, - f"--project={config.project}", - "--quiet" - ]) - - logger.info("Creating GCP image with TDX support...") - self._run_gcloud([ - "compute", "images", "create", image_name, - f"--project={config.project}", - f"--source-uri={config.bucket}/{image_name}.tar.gz", - "--guest-os-features=UEFI_COMPATIBLE,TDX_CAPABLE,GVNIC" - ]) - - return image_name - - def _create_shared_disk_image(self, config: GcpConfig, app: App) -> str: - """Create and upload shared disk image. Returns the image name.""" - import secrets - import shutil - - # Check if mcopy (mtools) is installed - if not shutil.which("mcopy"): - logger.error("") - logger.error("Error: 'mcopy' command not found.") - logger.error("") - logger.error("Please install mtools:") - logger.error(" Ubuntu/Debian: sudo apt-get install mtools") - logger.error(" Fedora/RHEL: sudo dnf install mtools") - logger.error(" Arch Linux: sudo pacman -S mtools") - logger.error("") - raise FileNotFoundError("mcopy not found. Please install mtools package.") - - # Ensure instance_id_seed and app_id exist - if not app.instance_id_seed: - app.instance_id_seed = secrets.token_hex(20) - logger.info(f"Generated instance_id_seed: {app.instance_id_seed}") - self.save_app_config(app) - - if not app.app_id: - app.app_id = secrets.token_hex(20) - logger.info(f"Generated app_id: {app.app_id}") - self.save_app_config(app) - - shared_dir = self._get_shared_dir() - - # Ensure shared directory exists and generate all required files - shared_dir.mkdir(parents=True, exist_ok=True) - - shared_image_name = f"{config.instance_name}-shared" - - # Process .env file: encrypt and save to shared/.encrypted-env - env_path = self.work_dir / app.env_file - env_names = [] # Collect environment variable names for allowed_envs - - if env_path.exists(): - if app.key_provider != "kms": - raise ValueError(f"{app.env_file} found but KMS is not enabled. " - f"Enable KMS with --key-provider kms or remove {app.env_file}") - - if not app.app_id: - raise ValueError(f"{app.env_file} found but app_id is not set. " - f"Run 'dstack-cloud prepare' to generate app_id") - - # Parse .env file - envs = self._parse_env_file(env_path) - if envs: - env_names = list(envs.keys()) - - # Get KMS URL from global config - global_config = self._load_global_config() - kms_urls = global_config.get("services", {}).get("kms_urls", []) - if not kms_urls: - raise ValueError("KMS enabled but no kms_urls configured in global config") - - # Get encryption public key from KMS - kms_url = kms_urls[0] - pubkey = self._get_app_encrypt_pub_key(app.app_id, kms_url) - - # Encrypt environment variables - encrypted_env = self._encrypt_env(envs, pubkey) - - # Save to shared/.encrypted-env - encrypted_file = shared_dir / ".encrypted-env" - with open(encrypted_file, 'wb') as f: - f.write(encrypted_env) - logger.info(f"Encrypted {app.env_file} -> {encrypted_file}") - else: - logger.info(f"{app.env_file} is empty, skipping") - - # Regenerate app-compose.json with env_names (if any) - # This must be done after .env processing and before creating the disk image - global_config = self._load_global_config() - sys_config_content = self._generate_sys_config(global_config, config, app) - sys_config_path = shared_dir / ".sys-config.json" - with open(sys_config_path, 'w') as f: - json.dump(sys_config_content, f, indent=2) - logger.info(f"Generated {sys_config_path}") - - # Generate .instance_info - instance_info = { - "instance_id_seed": app.instance_id_seed, - "app_id": app.app_id - } - instance_info_path = shared_dir / ".instance_info" - with open(instance_info_path, 'w') as f: - json.dump(instance_info, f, indent=2) - logger.info(f"Generated {instance_info_path}") - - # Generate app-compose.json with env_names (from .env file) - app_compose_content = self._generate_app_compose(app, env_names=env_names if env_names else None) - app_compose_path = shared_dir / "app-compose.json" - with open(app_compose_path, 'w') as f: - json.dump(app_compose_content, f, indent=2) - logger.info(f"Generated {app_compose_path}") - - with tempfile.TemporaryDirectory() as work_dir: - work_path = Path(work_dir) - raw_file = work_path / "disk.raw" - - # Create FAT32 disk image (no root required) - logger.info("Creating shared disk image...") - disk_size = "8M" # 8MB FAT32 disk - subprocess.run( - ["truncate", "-s", disk_size, str(raw_file)], - check=True - ) - subprocess.run( - ["mkfs.fat", "-F", "32", "-n", "DSTACKSHR", str(raw_file)], - check=True, capture_output=True - ) - - # Use mtools to copy files without mounting (no root required) - # Copy generated system files from shared directory - required_files = ["app-compose.json", ".sys-config.json", ".instance_info"] - for f in required_files: - src = shared_dir / f - if src.exists(): - subprocess.run( - ["mcopy", "-i", str(raw_file), str(src), "::"], - check=True - ) - else: - raise FileNotFoundError(f"Required file {f} not found in {shared_dir}") - - # Copy optional system files from shared directory - optional_files = [".encrypted-env"] - for f in optional_files: - src = shared_dir / f - if src.exists(): - subprocess.run( - ["mcopy", "-i", str(raw_file), str(src), "::"], - check=True - ) - - # Copy other user-editable files from project root - user_files = { - ".user-config": ".user-config", - } - - for src_name, dst_name in user_files.items(): - src_path = self.work_dir / src_name - if src_path.exists(): - subprocess.run( - ["mcopy", "-i", str(raw_file), str(src_path), f"::{dst_name}"], - check=True - ) - logger.info(f"Included {src_name}") - else: - logger.warning(f"{src_name} not found, skipping") - - # Create tar - tar_file = work_path / "shared-disk.tar.gz" - subprocess.run( - ["tar", "-C", str(work_path), "-czvf", str(tar_file), "disk.raw"], - check=True, capture_output=True - ) - - # Upload to GCS - logger.info("Uploading shared disk image to GCS...") - self._run_gsutil([ - "cp", str(tar_file), f"{config.bucket}/{shared_image_name}.tar.gz" - ]) - - # Delete existing image if present - result = self._run_gcloud([ - "compute", "images", "describe", shared_image_name, - f"--project={config.project}" - ], check=False) - - if result.returncode == 0: - logger.info("Deleting existing shared disk image...") - self._run_gcloud([ - "compute", "images", "delete", shared_image_name, - f"--project={config.project}", - "--quiet" - ]) - - # Create GCP image - logger.info("Creating GCP image from shared disk...") - self._run_gcloud([ - "compute", "images", "create", shared_image_name, - f"--project={config.project}", - f"--source-uri={config.bucket}/{shared_image_name}.tar.gz", - "--guest-os-features=GVNIC" - ]) - - return shared_image_name - - def deploy(self, delete_existing: bool = False, - force_boot_image: bool = False) -> None: - """Deploy VM to GCP.""" - # Load app config first (required) - validates project exists - app = self.load_app_config(required=True) - config = self.load_gcp_config() - - # Validate required configuration - missing = [] - if not config.instance_name: - missing.append("instance_name") - if not config.project: - missing.append("project") - if not config.zone: - missing.append("zone") - if missing: - raise ValueError( - f"Missing required GCP configuration: {', '.join(missing)}. " - f"Run 'dstack-cloud new ' to create a project or edit dstack-app.json." - ) - - # Auto-detect bucket if not specified - if not config.bucket and config.project: - config.bucket = f"gs://{config.project}-dstack" - - shared_dir = self._get_shared_dir() - - logger.info("=== GCP TDX VM Deployment ===") - logger.info(f"Project: {config.project}") - logger.info(f"Zone: {config.zone}") - logger.info(f"Instance: {config.instance_name}") - logger.info(f"Shared Directory: {shared_dir}") - logger.info(f"GCS Bucket: {config.bucket}") - - # Check if instance already exists - result = self._run_gcloud([ - "compute", "instances", "describe", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}" - ], check=False) - - if result.returncode == 0: - if delete_existing: - logger.info(f"Deleting existing instance: {config.instance_name}") - self._run_gcloud([ - "compute", "instances", "delete", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}", - "--quiet" - ]) - else: - raise RuntimeError( - f"Instance '{config.instance_name}' already exists. " - f"Use --delete to replace it." - ) - - # Check and upload boot image - boot_image = self._check_and_upload_boot_image(config, app, force=force_boot_image) - - # Create shared disk image - shared_image = self._create_shared_disk_image(config, app) - - # Ensure data disk image exists (with GPT partition labeled 'dstack-data') - data_image = self._ensure_data_disk_image(config) - - # Create TDX instance - logger.info("Creating TDX instance...") - - create_args = [ - "compute", "instances", "create", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}", - f"--machine-type={config.machine_type}", - "--confidential-compute-type=TDX", - f"--image={boot_image}", - "--boot-disk-size=10GB", - f"--create-disk=name={config.instance_name}-data,size={config.data_size}GB,type=pd-balanced,image={data_image},auto-delete=yes", - f"--create-disk=name={config.instance_name}-shared,size=1GB,type=pd-balanced,image={shared_image},auto-delete=yes", - "--maintenance-policy=TERMINATE", - ] - - provisioning = (config.provisioning_model or "STANDARD").upper() - if provisioning == "SPOT": - create_args.append("--provisioning-model=SPOT") - # STOP (vs. the gcloud default DELETE) preserves the - # boot/data disks across preemption so the instance can - # be restarted with `dstack-cloud start` and keep its - # LUKS-encrypted data disk intact. - create_args.append("--instance-termination-action=STOP") - elif provisioning != "STANDARD": - raise RuntimeError( - f"Unsupported provisioning_model: {config.provisioning_model!r} " - f"(expected 'STANDARD' or 'SPOT')" - ) - - if config.network != "default": - create_args.append(f"--network={config.network}") - if config.subnet: - create_args.append(f"--subnet={config.subnet}") - if config.service_account: - create_args.append(f"--service-account={config.service_account}") - if config.scopes: - create_args.append(f"--scopes={','.join(config.scopes)}") - # Always attach firewall tag so existing firewall rules continue to work - # after instance recreation (e.g. deploy --delete). - instance_tags = list(config.tags) - firewall_tag = f"fw-{config.instance_name}" - if firewall_tag not in instance_tags: - instance_tags.append(firewall_tag) - if instance_tags: - create_args.append(f"--tags={','.join(instance_tags)}") - if config.labels: - labels_str = ",".join(f"{k}={v}" for k, v in config.labels.items()) - create_args.append(f"--labels={labels_str}") - - self._run_gcloud(create_args) - - # Get instance details - result = self._run_gcloud([ - "compute", "instances", "describe", config.instance_name, - f"--zone={config.zone}", - f"--project={config.project}", - "--format=json" - ]) - - instance_info = json.loads(result.stdout) - external_ip = "" - internal_ip = "" - - for iface in instance_info.get("networkInterfaces", []): - internal_ip = iface.get("networkIP", "") - for access in iface.get("accessConfigs", []): - external_ip = access.get("natIP", "") - break - - # Save state - state = DeploymentState( - instance_name=config.instance_name, - project=config.project, - zone=config.zone, - external_ip=external_ip, - internal_ip=internal_ip, - status="RUNNING", - created_at=datetime.now().isoformat(), - boot_image=boot_image, - data_image=data_image, - shared_image=shared_image, - ) - self.save_state(state) - - logger.info("") - logger.info("=== Deployment Complete ===") - logger.info(f"Instance: {config.instance_name}") - logger.info(f"External IP: {external_ip}") - logger.info(f"Internal IP: {internal_ip}") - logger.info("") - logger.info("To check serial output:") - logger.info(f" dstack-cloud logs") - - def _parse_env_file(self, file_path: Path) -> Dict[str, str]: - """Parse an environment file where each line is formatted as KEY=Value.""" - if not file_path or not file_path.exists(): - return {} - - envs = {} - with open(file_path, 'r') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' not in line: - continue - key, value = line.split('=', 1) - envs[key.strip()] = value.strip() - return envs - - def _encrypt_env(self, envs: Dict[str, str], hex_public_key: str) -> bytes: - """ - Encrypt environment variables using X25519 key exchange and AES-GCM. - - Args: - envs: Environment variables dictionary - hex_public_key: Remote encryption public key in hexadecimal format - - Returns: - Raw bytes of (ephemeral public key || IV || ciphertext) - """ - if not CRYPTO_AVAILABLE: - raise ImportError( - "Cryptography libraries not available. Please install:\n" - "pip install cryptography eth-keys 'eth-hash[pycryptodome]'" - ) - - # Serialize environment variables to JSON (format: {"env": [{"key": k, "value": v}, ...]}) - env_pairs = [{"key": k, "value": v} for k, v in envs.items()] - envs_json = json.dumps({"env": env_pairs}).encode("utf-8") - - # Remove "0x" prefix if present - if hex_public_key.startswith("0x"): - hex_public_key = hex_public_key[2:] - - # Convert hex public key to bytes - remote_pubkey_bytes = bytes.fromhex(hex_public_key) - - # Generate ephemeral X25519 key pair - ephemeral_private_key = x25519.X25519PrivateKey.generate() - ephemeral_public_key = ephemeral_private_key.public_key() - - # Compute shared secret using X25519 - peer_public_key = x25519.X25519PublicKey.from_public_bytes(remote_pubkey_bytes) - shared = ephemeral_private_key.exchange(peer_public_key) - - # Use shared secret as key for AES-GCM (32 bytes for AES-256) - aesgcm = AESGCM(shared) - iv = os.urandom(12) # 12-byte nonce for AES-GCM - ciphertext = aesgcm.encrypt(iv, envs_json, None) - - # Serialize ephemeral public key to raw bytes - ephemeral_public_bytes = ephemeral_public_key.public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw - ) - - # Combine ephemeral public key, IV, and ciphertext - result = ephemeral_public_bytes + iv + ciphertext - return result - - def _get_app_encrypt_pub_key(self, app_id: str, kms_url: str) -> str: - """Get encryption public key for the specified app_id from KMS.""" - try: - import urllib.request - import urllib.error - import ssl - - path = f"{kms_url}/prpc/GetAppEnvEncryptPubKey?json" - data = json.dumps({"app_id": app_id}).encode("utf-8") - - req = urllib.request.Request( - path, - data=data, - headers={"Content-Type": "application/json"} - ) - - # Allow self-signed certificates for KMS server - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - - logger.info(f"Getting encryption public key for {app_id} from {kms_url}") - with urllib.request.urlopen(req, timeout=10, context=ssl_context) as response: - response_data = json.loads(response.read().decode("utf-8")) - - if "public_key" not in response_data: - raise ValueError(f"No public_key in response: {response_data}") - - # Verify signature if available - if "signature" not in response_data: - if not self._confirm_untrusted_signer("none"): - raise ValueError("Aborted due to missing signature") - return response_data["public_key"] - - public_key = bytes.fromhex(response_data["public_key"]) - signature = bytes.fromhex(response_data["signature"]) - - signer_pubkey = self._verify_signature(public_key, signature, app_id) - if signer_pubkey: - whitelist = self._load_whitelist() - if whitelist and signer_pubkey not in whitelist: - logger.warning(f"Signer {signer_pubkey} is not in the trusted whitelist!") - if not self._confirm_untrusted_signer(signer_pubkey): - raise ValueError("Aborted due to untrusted signer") - else: - logger.info(f"Verified signature from: {signer_pubkey}") - else: - logger.warning("Could not verify signature!") - if not self._confirm_untrusted_signer("unknown"): - raise ValueError("Aborted due to invalid signature") - - return response_data["public_key"] - - except Exception as e: - logger.warning(f"Failed to get encryption public key: {e}") - raise - - def _verify_signature(self, public_key: bytes, signature: bytes, app_id: str) -> Optional[str]: - """Verify the signature of a public key. - - Args: - public_key: The public key bytes to verify - signature: The signature bytes - app_id: The application ID - - Returns: - The compressed public key if valid, None otherwise - """ - if not ETH_CRYPTO_AVAILABLE: - logger.warning("eth-keys not available, skipping signature verification. " - "Install with: pip install eth-keys 'eth-hash[pycryptodome]'") - return None - - if len(signature) != 65: - return None - - # Create the message to verify - prefix = b"dstack-env-encrypt-pubkey" - if app_id.startswith("0x"): - app_id = app_id[2:] - message = prefix + b":" + bytes.fromhex(app_id) + public_key - - # Hash the message with Keccak-256 and recover the public key - try: - message_hash = keccak(message) - sig = keys.Signature(signature_bytes=signature) - recovered_key = sig.recover_public_key_from_msg_hash(message_hash) - return '0x' + recovered_key.to_compressed_bytes().hex() - except Exception as e: - error_msg = str(e) - if "hashing backends" in error_msg or "pycryptodome" in error_msg: - raise ImportError( - "Ethereum hashing backend not available. Please install:\n" - "pip install cryptography eth-keys 'eth-hash[pycryptodome]'" - ) - logger.debug(f"Signature verification failed: {e}") - return None - - def _confirm_untrusted_signer(self, signer: str) -> bool: - """Ask user to confirm using an untrusted signer.""" - try: - response = input(f"Continue with untrusted signer '{signer}'? (y/N): ") - return response.lower() in ('y', 'yes') - except EOFError: - # Non-interactive mode, reject untrusted signers - return False - - def _load_whitelist(self) -> List[str]: - """Load the whitelist of trusted signers from a file.""" - if not os.path.exists(DEFAULT_KMS_WHITELIST_PATH): - return [] - - try: - with open(DEFAULT_KMS_WHITELIST_PATH, 'r') as f: - data = json.load(f) - return data.get('trusted_signers', []) - except (json.JSONDecodeError, FileNotFoundError): - return [] - - def _save_whitelist(self, whitelist: List[str]) -> None: - """Save the whitelist of trusted signers to a file.""" - os.makedirs(os.path.dirname(DEFAULT_KMS_WHITELIST_PATH), exist_ok=True) - with open(DEFAULT_KMS_WHITELIST_PATH, 'w') as f: - json.dump({'trusted_signers': whitelist}, f, indent=2) - - def _derive_instance_id(self, instance_id_seed: str, app_id: str) -> str: - """Derive instance_id from instance_id_seed and app_id. - - Both instance_id_seed and app_id are hex strings that need to be - decoded to bytes before concatenation, matching the Rust implementation. - """ - # Decode hex strings to bytes - seed_bytes = bytes.fromhex(instance_id_seed) - app_bytes = bytes.fromhex(app_id) - - # Concatenate bytes (not hex strings) - id_path = seed_bytes + app_bytes - - # Compute SHA256 and take first 20 bytes, then convert to hex string - instance_id = hashlib.sha256(id_path).digest()[:20] - return instance_id.hex() - - def _get_gateway_urls(self, app: App, instance_id: str) -> Dict[str, str]: - """Construct gateway URLs for app access. - - Returns: - Dict with 'app_url' and 'instance_url' keys - """ - if not app.gateway_enabled: - return {} - - global_config = self._load_global_config() - gateway_urls = global_config.get("services", {}).get("gateway_urls", []) - if not gateway_urls: - return {} - - # Try to get gateway info from RPC - gateway_info = None - for gateway_url in gateway_urls: - try: - info_url = f"{gateway_url}/prpc/Info" - result = subprocess.run( - ["curl", "-sk", info_url], - capture_output=True, - text=True, - timeout=5 - ) - if result.returncode == 0: - gateway_info = json.loads(result.stdout) - break - except Exception as e: - logger.debug(f"Failed to get gateway info from {gateway_url}: {e}") - continue - - if not gateway_info: - return {} - - base_domain = gateway_info.get("base_domain") - external_port = gateway_info.get("external_port") - - if not base_domain or not external_port: - return {} - - # Construct URLs: one with instance_id, one with app_id - app_id = app.app_id - app_url = f"https://{app_id}-8090.{base_domain}:{external_port}/" - instance_url = f"https://{instance_id}-8090.{base_domain}:{external_port}/" - - return { - "app_url": app_url, - "instance_url": instance_url - } - - def status(self) -> None: - """Check deployment status.""" - state = self.load_state() - if not state or not state.instance_name: - logger.info("No deployment found. Run 'dstack-cloud deploy' first.") - return - - # Get current status from GCP - result = self._run_gcloud([ - "compute", "instances", "describe", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}", - "--format=json" - ], check=False) - - if result.returncode != 0: - logger.info(f"Instance '{state.instance_name}' not found in GCP") - state.status = "NOT_FOUND" - self.save_state(state) - return - - instance_info = json.loads(result.stdout) - status = instance_info.get("status", "UNKNOWN") - - # Update IPs - external_ip = "" - internal_ip = "" - for iface in instance_info.get("networkInterfaces", []): - internal_ip = iface.get("networkIP", "") - for access in iface.get("accessConfigs", []): - external_ip = access.get("natIP", "") - break - - state.status = status - state.external_ip = external_ip - state.internal_ip = internal_ip - self.save_state(state) - - print(f"Instance: {state.instance_name}") - print(f"Project: {state.project}") - print(f"Zone: {state.zone}") - print(f"Status: {status}") - print(f"External IP: {external_ip or 'N/A'}") - print(f"Internal IP: {internal_ip or 'N/A'}") - print(f"Boot Image: {state.boot_image}") - print(f"Created: {state.created_at}") - print(f"Updated: {state.updated_at}") - - # Display gateway URLs if enabled - # Try to load from .instance_info first (actual deployed values) - app = self.load_app_config() - if not app.gateway_enabled: - return - - instance_id_seed = None - app_id = None - - # Try to read from shared/.instance_info (actual deployed values) - instance_info_path = self._get_shared_dir() / ".instance_info" - if instance_info_path.exists(): - try: - with open(instance_info_path, 'r') as f: - instance_info_data = json.load(f) - instance_id_seed = instance_info_data.get("instance_id_seed") - app_id = instance_info_data.get("app_id") - except Exception as e: - logger.debug(f"Failed to read .instance_info: {e}") - - # Fallback to app.json if .instance_info not found or missing values - if not instance_id_seed or not app_id: - instance_id_seed = app.instance_id_seed - app_id = app.app_id - - if instance_id_seed and app_id: - instance_id = self._derive_instance_id(instance_id_seed, app_id) - gateway_urls = self._get_gateway_urls(app, instance_id) - if gateway_urls: - print("") - print("Gateway URLs:") - print(f" App URL: {gateway_urls.get('app_url', 'N/A')}") - print(f" Instance URL: {gateway_urls.get('instance_url', 'N/A')}") - - def logs(self, follow: bool = False, lines: int = 100) -> None: - """View serial console logs.""" - state = self.load_state() - if not state or not state.instance_name: - raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") - - if follow: - # Tail logs continuously - logger.info(f"Following serial output for {state.instance_name}...") - logger.info("Press Ctrl+C to stop") - - last_output = "" - while True: - try: - result = self._run_gcloud([ - "compute", "instances", "get-serial-port-output", - state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ], check=False) - - if result.returncode == 0: - output = result.stdout - if output != last_output: - # Print only new content - if last_output: - new_content = output[len(last_output):] - if new_content: - print(new_content, end="", flush=True) - else: - # First time, print last N lines - lines_list = output.split('\n') - print('\n'.join(lines_list[-lines:]), flush=True) - last_output = output - - time.sleep(2) - except KeyboardInterrupt: - print("\nStopped following logs.") - break - else: - # Get logs once - result = self._run_gcloud([ - "compute", "instances", "get-serial-port-output", - state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ]) - - output = result.stdout - lines_list = output.split('\n') - print('\n'.join(lines_list[-lines:])) - - def stop(self) -> None: - """Stop the VM.""" - state = self.load_state() - if not state or not state.instance_name: - raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") - - # Check if instance exists - result = self._run_gcloud([ - "compute", "instances", "describe", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ], check=False) - if result.returncode != 0: - if "was not found" in result.stderr: - logger.info(f"Instance {state.instance_name} does not exist (already removed?).") - return - raise RuntimeError(f"gcloud command failed: {result.stderr}") - - logger.info(f"Stopping instance {state.instance_name}...") - self._run_gcloud([ - "compute", "instances", "stop", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ]) - - state.status = "STOPPED" - self.save_state(state) - logger.info("Instance stopped.") - - def start(self) -> None: - """Start a stopped VM.""" - state = self.load_state() - if not state or not state.instance_name: - raise ValueError("No deployment found. Run 'dstack-cloud deploy' first.") - - # Check if instance exists - result = self._run_gcloud([ - "compute", "instances", "describe", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ], check=False) - if result.returncode != 0: - if "was not found" in result.stderr: - raise ValueError( - f"Instance {state.instance_name} does not exist. " - f"Run 'dstack-cloud deploy' to create it." - ) - raise RuntimeError(f"gcloud command failed: {result.stderr}") - - logger.info(f"Starting instance {state.instance_name}...") - self._run_gcloud([ - "compute", "instances", "start", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}" - ]) - - # Update state with new IP - self.status() - logger.info("Instance started.") - - def remove(self, keep_images: bool = False) -> None: - """Remove the VM and cleanup.""" - state = self.load_state() - if not state or not state.instance_name: - logger.info("No deployment found.") - return - - # Delete instance - logger.info(f"Deleting instance {state.instance_name}...") - self._run_gcloud([ - "compute", "instances", "delete", state.instance_name, - f"--zone={state.zone}", - f"--project={state.project}", - "--quiet" - ], check=False) - - if not keep_images and state.shared_image: - # Delete shared disk image - logger.info(f"Deleting shared disk image {state.shared_image}...") - self._run_gcloud([ - "compute", "images", "delete", state.shared_image, - f"--project={state.project}", - "--quiet" - ], check=False) - - # Clear state - state.status = "REMOVED" - state.external_ip = "" - state.internal_ip = "" - self.save_state(state) - - logger.info("Instance removed.") - - def list_deployments(self, project: Optional[str] = None) -> None: - """List all dstack deployments in a project.""" - if not project: - # Try to get from config - try: - config = self.load_gcp_config() - project = config.project - except FileNotFoundError: - # Try global config - global_config = self._load_global_config() - project = global_config.get("gcp", {}).get("project", "") - - if not project: - raise ValueError("Project is required. Specify with --project or configure it.") - - result = self._run_gcloud([ - "compute", "instances", "list", - f"--project={project}", - "--filter=name~^dstack-", - "--format=table(name,zone,status,networkInterfaces[0].accessConfigs[0].natIP:label=EXTERNAL_IP,creationTimestamp)" - ], capture=False) - - def _get_firewall_rule_name(self, instance_name: str, port: int, protocol: str, - action: str = "allow") -> str: - """Generate firewall rule name for an instance port.""" - return f"{instance_name}-{action}-{protocol}-{port}" - - def _parse_port_spec(self, port_spec: str) -> tuple: - """Parse port specification like '8080' or '53/udp'. - - Returns: - tuple: (port: int, protocol: str) - """ - if "/" in port_spec: - port_str, protocol = port_spec.split("/", 1) - protocol = protocol.lower() - if protocol not in ("tcp", "udp"): - raise ValueError(f"Invalid protocol '{protocol}'. Must be 'tcp' or 'udp'.") - else: - port_str = port_spec - protocol = "tcp" - - try: - port = int(port_str) - except ValueError: - raise ValueError(f"Invalid port number '{port_str}'.") - - if not (1 <= port <= 65535): - raise ValueError(f"Port {port} out of range (1-65535).") - - return port, protocol - - def _get_project_for_firewall(self) -> str: - """Get project ID for firewall operations.""" - # Try state first - state = self.load_state() - if state and state.project: - return state.project - - # Try config - try: - config = self.load_gcp_config() - if config.project: - return config.project - except FileNotFoundError: - pass - - # Try global config - global_config = self._load_global_config() - project = global_config.get("gcp", {}).get("project", "") - - if not project: - raise ValueError("Project is required. Deploy first or specify with --project.") - - return project - - def _get_instance_name_for_firewall(self) -> str: - """Get instance name for firewall operations.""" - state = self.load_state() - if state and state.instance_name: - return state.instance_name - - try: - config = self.load_gcp_config() - if config.instance_name: - return config.instance_name - except FileNotFoundError: - pass - - raise ValueError("Instance name is required. Deploy first or specify with --instance.") - - def _ensure_instance_tag(self, instance_name: str, project: str) -> str: - """Ensure instance has the firewall tag, return the tag name.""" - instance_tag = f"fw-{instance_name}" - - state = self.load_state() - zone = state.zone if state and state.zone else "" - if not zone: - try: - config = self.load_gcp_config() - zone = config.zone - except FileNotFoundError: - pass - if not zone: - global_config = self._load_global_config() - zone = global_config.get("gcp", {}).get("zone", "us-central1-a") - - result = self._run_gcloud([ - "compute", "instances", "describe", instance_name, - f"--project={project}", - f"--zone={zone}", - "--format=value(tags.items)" - ], check=False) - - current_tags = result.stdout.strip().split(";") if result.stdout.strip() else [] - current_tags = [t.strip() for t in current_tags if t.strip()] - - if instance_tag not in current_tags: - logger.info(f"Adding tag '{instance_tag}' to instance '{instance_name}'...") - self._run_gcloud([ - "compute", "instances", "add-tags", instance_name, - f"--project={project}", - f"--zone={zone}", - f"--tags={instance_tag}" - ]) - logger.info(f"Added tag '{instance_tag}' to instance") - else: - logger.debug(f"Instance already has tag '{instance_tag}'") - - return instance_tag - - def fw_allow(self, port_spec: str, - source_ranges: Optional[List[str]] = None, - instance_name: Optional[str] = None, - project: Optional[str] = None) -> None: - """Add firewall rule to open a port for the instance.""" - port, protocol = self._parse_port_spec(port_spec) - project = project or self._get_project_for_firewall() - instance_name = instance_name or self._get_instance_name_for_firewall() - - if source_ranges is None: - source_ranges = ["0.0.0.0/0"] - - # Always ensure instance has the tag first - instance_tag = self._ensure_instance_tag(instance_name, project) - - rule_name = self._get_firewall_rule_name(instance_name, port, protocol) - - # Check if rule already exists - result = self._run_gcloud([ - "compute", "firewall-rules", "describe", rule_name, - f"--project={project}" - ], check=False) - - if result.returncode == 0: - logger.info(f"Firewall rule '{rule_name}' already exists") - return - - # Create firewall rule targeting this instance's tag - logger.info(f"Creating firewall rule '{rule_name}'...") - self._run_gcloud([ - "compute", "firewall-rules", "create", rule_name, - f"--project={project}", - f"--allow={protocol}:{port}", - f"--source-ranges={','.join(source_ranges)}", - f"--target-tags={instance_tag}", - f"--description=Allow {protocol.upper()} port {port} for {instance_name}" - ]) - - logger.info(f"Opened {protocol.upper()} port {port} for instance '{instance_name}'") - - def fw_deny(self, port_spec: str, - source_ranges: Optional[List[str]] = None, - instance_name: Optional[str] = None, - project: Optional[str] = None) -> None: - """Create a deny firewall rule to block traffic on a port.""" - port, protocol = self._parse_port_spec(port_spec) - project = project or self._get_project_for_firewall() - instance_name = instance_name or self._get_instance_name_for_firewall() - - if source_ranges is None: - source_ranges = ["0.0.0.0/0"] - - # Always ensure instance has the tag first - instance_tag = self._ensure_instance_tag(instance_name, project) - - rule_name = self._get_firewall_rule_name(instance_name, port, protocol, action="deny") - - # Check if rule already exists - result = self._run_gcloud([ - "compute", "firewall-rules", "describe", rule_name, - f"--project={project}" - ], check=False) - - if result.returncode == 0: - logger.info(f"Firewall rule '{rule_name}' already exists") - return - - # Create deny firewall rule with high priority (low number = high priority) - logger.info(f"Creating deny firewall rule '{rule_name}'...") - self._run_gcloud([ - "compute", "firewall-rules", "create", rule_name, - f"--project={project}", - "--action=DENY", - f"--rules={protocol}:{port}", - f"--source-ranges={','.join(source_ranges)}", - f"--target-tags={instance_tag}", - "--priority=900", - f"--description=Deny {protocol.upper()} port {port} for {instance_name}" - ]) - - logger.info(f"Blocked {protocol.upper()} port {port} for instance '{instance_name}'") - - def fw_remove(self, port_spec: str, - instance_name: Optional[str] = None, - project: Optional[str] = None) -> None: - """Remove a firewall rule (allow or deny) for a port.""" - port, protocol = self._parse_port_spec(port_spec) - project = project or self._get_project_for_firewall() - instance_name = instance_name or self._get_instance_name_for_firewall() - - # Try to delete both allow and deny rules - deleted = False - for action in ["allow", "deny"]: - rule_name = self._get_firewall_rule_name(instance_name, port, protocol, action=action) - - # Check if rule exists - result = self._run_gcloud([ - "compute", "firewall-rules", "describe", rule_name, - f"--project={project}" - ], check=False) - - if result.returncode == 0: - # Delete the rule - logger.info(f"Deleting firewall rule '{rule_name}'...") - self._run_gcloud([ - "compute", "firewall-rules", "delete", rule_name, - f"--project={project}", - "--quiet" - ]) - logger.info(f"Removed {action} rule for {protocol.upper()} port {port}") - deleted = True - - if not deleted: - logger.info(f"No firewall rules found for {protocol.upper()} port {port} on instance '{instance_name}'") - - def fw_list(self, instance_name: Optional[str] = None, - project: Optional[str] = None) -> None: - """List firewall rules for an instance.""" - project = project or self._get_project_for_firewall() - - if instance_name: - # List rules for specific instance - instance_tag = f"fw-{instance_name}" - filter_expr = f"name~^{instance_name}-allow- OR targetTags:{instance_tag}" - else: - # Try to get instance name from state - try: - instance_name = self._get_instance_name_for_firewall() - filter_expr = f"name~^{instance_name}-allow-" - except ValueError: - # List all dstack-related firewall rules - filter_expr = "name~^dstack-" - - logger.info(f"Firewall rules for project '{project}':") - self._run_gcloud([ - "compute", "firewall-rules", "list", - f"--project={project}", - f"--filter={filter_expr}", - "--format=table(name,direction,priority,allowed[].map().firewall_rule().list():label=ALLOW,sourceRanges.list():label=SRC_RANGES,targetTags.list():label=TARGET_TAGS)" - ], capture=False) - - -def main(): - parser = argparse.ArgumentParser( - description="Multi-cloud VM lifecycle management tool", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=f""" -Examples: - # Create a new project - dstack-cloud new myproject - - # Edit global configuration - dstack-cloud config-edit - - # Download OS image - dstack-cloud pull {DEFAULT_OS_IMAGE} - - # Deploy VM - dstack-cloud deploy - - # Check status - dstack-cloud status - - # View logs - dstack-cloud logs --follow - - # Stop/Start/Remove - dstack-cloud stop - dstack-cloud start - dstack-cloud remove - - # Firewall management - dstack-cloud fw allow 8080 # Allow TCP port 8080 - dstack-cloud fw allow 53/udp # Allow UDP port 53 - dstack-cloud fw allow 443 -s 10.0.0.0/8 # Allow port 443 from specific range - dstack-cloud fw deny 22 # Block TCP port 22 - dstack-cloud fw deny 22 -s 0.0.0.0/0 # Block port 22 from all sources - dstack-cloud fw remove 8080 # Remove firewall rule for port 8080 - dstack-cloud fw list # List firewall rules -""" - ) - - parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") - parser.add_argument("-C", "--directory", type=str, help="Change to directory before running") - - subparsers = parser.add_subparsers(dest="command", help="Commands") - - # new command - new_parser = subparsers.add_parser("new", help="Create a new project") - new_parser.add_argument("name", type=str, help="Project name") - - # App configuration options - new_parser.add_argument("--os-image", type=str, help=f"OS image (e.g., {DEFAULT_OS_IMAGE})") - new_parser.add_argument("--app-id", type=str, help="Application ID (40 hex chars)") - new_parser.add_argument("--gateway-enabled", "--gw", dest="gateway_enabled", action="store_true", help="Enable dstack-gateway") - new_parser.add_argument("--no-gateway-enabled", "--no-gw", dest="gateway_enabled", action="store_false") - new_parser.set_defaults(gateway_enabled=None) - new_parser.add_argument("--key-provider", "--kp", type=str, choices=["kms", "local", "tpm", "none"], help="Key provider type") - new_parser.add_argument("--storage-fs", "--fs", dest="storage_fs", type=str, choices=["ext4", "zfs"], help="Storage filesystem") - new_parser.add_argument("--secure-time", action="store_true", help="Enable secure time synchronization") - new_parser.add_argument("--no-secure-time", dest="secure_time", action="store_false") - new_parser.set_defaults(secure_time=None) - new_parser.add_argument("--no-instance-id", action="store_true", help="Disable instance ID generation") - - # GCP configuration options - new_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - new_parser.add_argument("--zone", "-z", type=str, help="GCP zone (e.g., us-central1-a)") - new_parser.add_argument("--instance-name", type=str, help="GCP instance name") - new_parser.add_argument("--machine-type", "-m", type=str, help="Machine type (e.g., c3-standard-4)") - new_parser.add_argument("--data-size", type=int, help="Data disk size in GB") - - # config-edit command - subparsers.add_parser("config-edit", help="Edit global configuration") - - # prepare command - subparsers.add_parser("prepare", help="Generate shared files") - - # pull command - pull_parser = subparsers.add_parser("pull", help="Download OS image") - pull_parser.add_argument("image", type=str, help=f"OS image name (e.g., {DEFAULT_OS_IMAGE}) or absolute URL (e.g., https://example.com/image-uki.tar.gz)") - - # deploy command - deploy_parser = subparsers.add_parser("deploy", help="Deploy VM to cloud") - deploy_parser.add_argument("--delete", "-d", action="store_true", - help="Delete existing instance first") - deploy_parser.add_argument("--force-boot-image", action="store_true", - help="Force re-upload boot image") - - # status command - subparsers.add_parser("status", help="Check deployment status") - - # logs command - logs_parser = subparsers.add_parser("logs", help="View serial console logs") - logs_parser.add_argument("--follow", "-f", action="store_true", help="Follow log output") - logs_parser.add_argument("--lines", "-n", type=int, default=100, help="Number of lines to show") - - # stop command - subparsers.add_parser("stop", help="Stop the VM") - - # start command - subparsers.add_parser("start", help="Start a stopped VM") - - # remove command - remove_parser = subparsers.add_parser("remove", help="Remove the VM and cleanup") - remove_parser.add_argument("--keep-images", action="store_true", - help="Keep disk images in GCP") - - # list command - list_parser = subparsers.add_parser("list", help="List all deployments") - list_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - - # fw command group - fw_parser = subparsers.add_parser("fw", help="Firewall management") - fw_subparsers = fw_parser.add_subparsers(dest="fw_command", help="Firewall commands") - - # fw allow - fw_allow_parser = fw_subparsers.add_parser("allow", help="Open a port for the instance") - fw_allow_parser.add_argument("port", type=str, help="Port to open (e.g., 8080, 53/udp)") - fw_allow_parser.add_argument("--source", "-s", type=str, action="append", - dest="source_ranges", - help="Source IP ranges (default: 0.0.0.0/0). Can be specified multiple times.") - fw_allow_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") - fw_allow_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - - # fw deny - fw_deny_parser = fw_subparsers.add_parser("deny", help="Block traffic on a port") - fw_deny_parser.add_argument("port", type=str, help="Port to block (e.g., 8080, 53/udp)") - fw_deny_parser.add_argument("--source", "-s", type=str, action="append", - dest="source_ranges", - help="Source IP ranges to block (default: 0.0.0.0/0). Can be specified multiple times.") - fw_deny_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") - fw_deny_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - - # fw remove - fw_remove_parser = fw_subparsers.add_parser("remove", help="Remove a firewall rule") - fw_remove_parser.add_argument("port", type=str, help="Port to remove rule for (e.g., 8080, 53/udp)") - fw_remove_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") - fw_remove_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - - # fw list - fw_list_parser = fw_subparsers.add_parser("list", help="List firewall rules for instance") - fw_list_parser.add_argument("--instance", "-i", type=str, help="Instance name (default: from state)") - fw_list_parser.add_argument("--project", "-p", type=str, help="GCP project ID") - - args = parser.parse_args() - - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - - work_dir = args.directory if args.directory else None - manager = CloudDeploymentManager(work_dir) - - try: - if args.command == "new": - manager.new( - args.name, - os_image=args.os_image, - app_id=args.app_id, - gateway_enabled=args.gateway_enabled, - key_provider=args.key_provider, - storage_fs=args.storage_fs, - secure_time=args.secure_time, - no_instance_id=args.no_instance_id, - project=args.project, - zone=args.zone, - instance_name=args.instance_name, - machine_type=args.machine_type, - data_size=args.data_size - ) - elif args.command == "config-edit": - manager.config_edit() - elif args.command == "prepare": - manager.prepare() - elif args.command == "pull": - manager.pull(args.image) - elif args.command == "deploy": - manager.deploy( - delete_existing=args.delete, - force_boot_image=args.force_boot_image - ) - elif args.command == "status": - manager.status() - elif args.command == "logs": - manager.logs(follow=args.follow, lines=args.lines) - elif args.command == "stop": - manager.stop() - elif args.command == "start": - manager.start() - elif args.command == "remove": - manager.remove(keep_images=args.keep_images) - elif args.command == "list": - manager.list_deployments(project=args.project) - elif args.command == "fw": - if args.fw_command == "allow": - manager.fw_allow( - port_spec=args.port, - source_ranges=args.source_ranges, - instance_name=args.instance, - project=args.project - ) - elif args.fw_command == "deny": - manager.fw_deny( - port_spec=args.port, - source_ranges=args.source_ranges, - instance_name=args.instance, - project=args.project - ) - elif args.fw_command == "remove": - manager.fw_remove( - port_spec=args.port, - instance_name=args.instance, - project=args.project - ) - elif args.fw_command == "list": - manager.fw_list( - instance_name=args.instance, - project=args.project - ) - else: - fw_parser.print_help() - else: - parser.print_help() - except Exception as e: - logger.error(str(e)) - if args.verbose: - import traceback - traceback.print_exc() - sys.exit(1) - - -if __name__ == "__main__": - main() From 2e5d5b977570b5aeb7d1809e7369fc001354c5f8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 3 Jun 2026 20:23:19 -0700 Subject: [PATCH 390/435] Update submodule dstack/ --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index bde0d038e..8f86e4306 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit bde0d038e5ee3d8441c65910dc5a3254cb342ac6 +Subproject commit 8f86e43062a9df53503b3f63490f3ed996e7f153 From dde9b6f45179878b673718c5b33d26635e0fcd60 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 6 Jun 2026 01:22:19 -0700 Subject: [PATCH 391/435] Fix NVIDIA recipes for UNPACKDIR and pkg-config Use UNPACKDIR as the source directory for recipes that install unpacked files or archives, and let libnvidia-container's seccomp support be controlled through PACKAGECONFIG while inheriting pkgconfig. --- .../containerd-config/containerd-config_1.0.0.bb | 2 ++ .../libnvidia-container/libnvidia-container_1.00.bb | 4 ++-- .../recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb | 2 +- .../recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb | 2 +- .../recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb | 2 +- .../recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb | 2 +- .../recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb | 2 ++ .../recipes-graphics/nvidia/nvidia-persistenced_1.0.bb | 2 ++ 8 files changed, 12 insertions(+), 6 deletions(-) diff --git a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb index d2757e9ca..2d40e6eee 100644 --- a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb +++ b/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb @@ -4,6 +4,8 @@ LICENSE = "CLOSED" # FILESEXTRAPATHS:prepend := "${THISDIR}/files:" SRC_URI = "file://config.toml" +S = "${UNPACKDIR}" + do_install:append() { install -d ${D}${sysconfdir}/containerd install -m 0644 ${UNPACKDIR}/config.toml ${D}${sysconfdir}/containerd/ diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb index e6100ad70..4e4e341df 100644 --- a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb +++ b/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb @@ -1,5 +1,5 @@ require libnvidia-container.inc -inherit features_check +inherit features_check pkgconfig SUMMARY = "libNVIDIA Container for Yocto" @@ -20,7 +20,7 @@ PACKAGECONFIG[seccomp] = "WITH_SECCOMP=yes,WITH_SECCOMP=no,libseccomp" # We need to link with libelf, otherwise we need to # include bmake-native which does not exist at the moment. -EXTRA_OEMAKE = "EXCLUDE_BUILD_FLAGS=1 PLATFORM=${HOST_ARCH} WITH_NVCGO=yes WITH_LIBELF=yes WITH_SECCOMP=yes COMPILER=${@d.getVar('CC').split()[0]} REVISION=${SRCREV_libnvidia} ${PACKAGECONFIG_CONFARGS} \ +EXTRA_OEMAKE = "EXCLUDE_BUILD_FLAGS=1 PLATFORM=${HOST_ARCH} WITH_NVCGO=yes WITH_LIBELF=yes COMPILER=${@d.getVar('CC').split()[0]} REVISION=${SRCREV_libnvidia} ${PACKAGECONFIG_CONFARGS} \ NVIDIA_MODPROBE_EXTRA_CFLAGS=${NVIDIA_MODPROBE_EXTRA_CFLAGS} \ GOCACHE=${WORKDIR}/go-cache GOPATH=${WORKDIR}/go GOMODCACHE=${WORKDIR}/go-mod" NVIDIA_MODPROBE_EXTRA_CFLAGS ?= "-ffile-prefix-map=${WORKDIR}=/usr/src/debug/${PN}/${EXTENDPE}${PV}-${PR}" diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb index 7ba353891..5cca02880 100644 --- a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb @@ -8,7 +8,7 @@ SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/lib SRC_URI[md5sum] = "6bc20061ebdae98fadd7a76110b44430" SRC_URI[sha256sum] = "c2285c12f10ec2afc0ad2949f7fcc282b6fd37f32165c1df241451ccabb1067a" -S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" +S = "${UNPACKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" INSANE_SKIP:${PN} = "already-stripped ldflags" diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb index 1a72f6de5..26584ed61 100644 --- a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb +++ b/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb @@ -8,7 +8,7 @@ SRC_URI = "https://developer.download.nvidia.cn/compute/nvidia-driver/redist/lib SRC_URI[md5sum] = "5b559a614f2c2c79b6eb796b68b6fb0f" SRC_URI[sha256sum] = "1c008787111fc55a7fc1d3521a6d41c2523b8949354e5651eadb4498554e5469" -S = "${WORKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" +S = "${UNPACKDIR}/libnvidia_nscq-linux-x86_64-${PV}-archive" INSANE_SKIP:${PN} = "already-stripped ldflags" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb index 7187848a2..11c4a1ef7 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb @@ -8,7 +8,7 @@ SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fa SRC_URI[md5sum] = "a6568aa288cb4784b85ba6826463f918" SRC_URI[sha256sum] = "f0220bfb67d04b4107acf00cc95abe5a9268fd8f8b5bae26971f4df232e4369c" -S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" +S = "${UNPACKDIR}/fabricmanager-linux-x86_64-${PV}-archive" DEPENDS = "" RDEPENDS:${PN} = "bash zlib" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb index 90ac8f465..c090ad3ac 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb @@ -8,7 +8,7 @@ SRC_URI = "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fa SRC_URI[md5sum] = "8761b4b5f371b7e87b2b82141c901311" SRC_URI[sha256sum] = "ea86951987ced2c2bbed1a9a9037e917dd2268add0ec484db9216e843d8b7de8" -S = "${WORKDIR}/fabricmanager-linux-x86_64-${PV}-archive" +S = "${UNPACKDIR}/fabricmanager-linux-x86_64-${PV}-archive" DEPENDS = "" RDEPENDS:${PN} = "bash zlib" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb index 5ed2c9567..b2c062e14 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb @@ -6,6 +6,8 @@ SRC_URI = "\ file://nvidia.conf \ " +S = "${UNPACKDIR}" + do_install() { install -d ${D}${sysconfdir}/modprobe.d install -m 0644 ${UNPACKDIR}/nvidia.conf ${D}${sysconfdir}/modprobe.d/ diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb index 7503da705..0c3ee945c 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb @@ -5,6 +5,8 @@ SRC_URI += "\ file://nvidia-persistenced.service \ " +S = "${UNPACKDIR}" + inherit systemd SYSTEMD_PACKAGES = "${PN}" From 4048a5087e6e02b2c40eae84898c3160dabe7432 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 11 Jun 2026 03:12:29 -0700 Subject: [PATCH 392/435] build.sh: add help action and usage(), document FLAVORS Extract the action list into a usage() function and wire it to a new help/-h/--help action. Document the FLAVORS env var and the flavor->image mapping with examples. On an invalid action, print the error to stderr and exit non-zero. --- build.sh | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 70337c73d..94aea036e 100755 --- a/build.sh +++ b/build.sh @@ -327,6 +327,33 @@ download_image() { fi } +usage() { + cat <<'EOF' +Usage: ./build.sh [args] + +Actions: + host - Build host binaries only + guest - Build guest images only + cfg - Generate configuration files only + dl - Download a specific image + hostcfg - Build host binaries and generate configuration files + all - Build everything (host, guest, and configuration) + help - Show this help + +Build a specific guest flavor (default builds all four): set the FLAVORS env var. + flavor -> output image + prod -> dstack + dev -> dstack-dev + nvidia -> dstack-nvidia + nvidia-dev -> dstack-nvidia-dev + + Examples: + FLAVORS=nvidia ./build.sh guest # build only dstack-nvidia + FLAVORS="prod nvidia" ./build.sh guest # build two flavors + ./build.sh guest # build all flavors (default) +EOF +} + case $ACTION in host) build_host @@ -352,15 +379,12 @@ all) build_guest build_cfg ;; +help|-h|--help) + usage + ;; *) - echo "Invalid action: $ACTION" - echo "Valid actions are:" - echo " host - Build host binaries only" - echo " guest - Build guest images only" - echo " cfg - Generate configuration files only" - echo " dl - Download a specific image" - echo " hostcfg - Build host binaries and generate configuration files" - echo " all - Build everything (host, guest, and configuration)" + [ -n "$ACTION" ] && echo "Invalid action: $ACTION" >&2 + usage exit 1 ;; esac From 4b0574791adafa87471388273b1dd246f5fbb0bb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 11 Jun 2026 03:12:51 -0700 Subject: [PATCH 393/435] kernel: enable gVNIC (gve) + virtio-net for GCP C3; bump dstack GCP C3/modern instances expose the NIC via gVNIC (the Google gve driver); without it a C3 CVM gets no NIC -> no DHCP -> no network. Build in CONFIG_GVE and CONFIG_NET_VENDOR_GOOGLE, and keep CONFIG_VIRTIO_NET for older/virtio instances. Also bump the dstack submodule to f1ba0a2, whose top commit mixes the GCP vTPM AK pubkey into instance_id. --- dstack | 2 +- meta-dstack/recipes-kernel/linux/files/dstack.cfg | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index 8f86e4306..f1ba0a22a 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 8f86e43062a9df53503b3f63490f3ed996e7f153 +Subproject commit f1ba0a22a961e8bdae110c7b1a796bc56dfd10a8 diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/meta-dstack/recipes-kernel/linux/files/dstack.cfg index 24aa50f37..54eacd7fc 100644 --- a/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -8,6 +8,13 @@ CONFIG_9P_FS_POSIX_ACL=y CONFIG_PCI=y CONFIG_TUN=m CONFIG_VIRTIO_PCI=y + +# Network NIC drivers. GCP C3/modern instances use gVNIC (the Google gve +# driver); older/virtio instances use virtio-net. Without gve, a C3 CVM gets +# no NIC -> no DHCP -> no network. Build both in. +CONFIG_VIRTIO_NET=y +CONFIG_NET_VENDOR_GOOGLE=y +CONFIG_GVE=y CONFIG_ISO9660_FS=y CONFIG_WIREGUARD=y CONFIG_TMPFS_POSIX_ACL=y From 6a9fe80286b14ced96a1a48d2a54755505a3c965 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 11 Jun 2026 18:15:10 +0800 Subject: [PATCH 394/435] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 94aea036e..3c3092b4b 100755 --- a/build.sh +++ b/build.sh @@ -328,8 +328,8 @@ download_image() { } usage() { - cat <<'EOF' -Usage: ./build.sh [args] + cat < [args] Actions: host - Build host binaries only From 86be7d60252f82ff1c9c8ac2a083d0a2e08c7a9e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 11 Jun 2026 03:14:45 -0700 Subject: [PATCH 395/435] dstack.conf: bump DISTRO_VERSION to 0.6.0.a2 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 397ca34cc..10f36c74f 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -7,7 +7,7 @@ DISTRO = "dstack" DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" -DISTRO_VERSION = "0.6.0" +DISTRO_VERSION = "0.6.0.a2" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From 65c675a6c506e602c4d5b9a6317011d9c9333650 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 05:44:04 -0700 Subject: [PATCH 396/435] linux: add ACPI BadAML sandbox patch for confidential guests Port the BadAML sandbox kernel patch from the Easy-TEE project. It hooks the ACPI SystemMemory operation region handler to walk the page tables and deny AML accesses that target encrypted (private) guest memory, blocking a class of attacks where a malicious host supplies crafted ACPI tables. Applied unconditionally since dstack OS always runs inside a TEE; the hook is a runtime no-op on platforms without memory encryption. Verified: do_patch and do_compile succeed against linux-yocto 6.18. --- ...ox-block-aml-systemmemory-ram-access.patch | 213 ++++++++++++++++++ .../linux/linux-yocto%.bbappend | 10 + 2 files changed, 223 insertions(+) create mode 100644 meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch diff --git a/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch b/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch new file mode 100644 index 000000000..c6297b0d1 --- /dev/null +++ b/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch @@ -0,0 +1,213 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Paul Meyer +Date: Tue, 17 Feb 2026 10:47:28 +0100 +Subject: [PATCH] drivers/acpi: add BadAML sandbox + +Block AML SystemMemory region accesses that target encrypted (private) +guest RAM. A malicious hypervisor can craft ACPI tables whose AML reads +or writes confidential guest memory via the SystemMemory operation +region handler; this sandbox walks the page tables and denies the access +when the target page is encrypted, logging the decision. + +Ported from the Easy-TEE project (mkosi gcp profile kernel patch). + +Upstream-Status: Inappropriate [confidential-guest hardening] +Signed-off-by: Paul Meyer +--- + drivers/acpi/acpica/exregion.c | 6 ++ + drivers/acpi/acpica/sandbox.h | 139 +++++++++++++++++++++++++++++++++ + 2 files changed, 145 insertions(+) + create mode 100644 drivers/acpi/acpica/sandbox.h + +diff --git a/drivers/acpi/acpica/exregion.c b/drivers/acpi/acpica/exregion.c +index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7455eaf67 100644 +--- a/drivers/acpi/acpica/exregion.c ++++ b/drivers/acpi/acpica/exregion.c +@@ -14,6 +14,8 @@ + #define _COMPONENT ACPI_EXECUTER + ACPI_MODULE_NAME("exregion") + ++#include "sandbox.h" ++ + /******************************************************************************* + * + * FUNCTION: acpi_ex_system_memory_space_handler +@@ -38,6 +40,7 @@ acpi_ex_system_memory_space_handler(u32 function, + u64 *value, + void *handler_context, void *region_context) + { ++ SANDBOX_SECT_START; + acpi_status status = AE_OK; + void *logical_addr_ptr = NULL; + struct acpi_mem_space_context *mem_info = region_context; +@@ -192,6 +195,7 @@ acpi_ex_system_memory_space_handler(u32 function, + case ACPI_READ: + + *value = 0; ++ SANDBOX_READ_HOOK((u64)logical_addr_ptr, (u64)address); + switch (bit_width) { + case 8: + +@@ -223,6 +227,7 @@ acpi_ex_system_memory_space_handler(u32 function, + + case ACPI_WRITE: + ++ SANDBOX_WRITE_HOOK((u64)logical_addr_ptr, (u64)address); + switch (bit_width) { + case 8: + +@@ -258,6 +263,7 @@ acpi_ex_system_memory_space_handler(u32 function, + break; + } + ++ SANDBOX_SECT_END; + return_ACPI_STATUS(status); + } + +diff --git a/drivers/acpi/acpica/sandbox.h b/drivers/acpi/acpica/sandbox.h +new file mode 100644 +index 0000000000000000000000000000000000000000..1d9d95a87698dde14429f2f33a0c375ad51774fe +--- /dev/null ++++ b/drivers/acpi/acpica/sandbox.h +@@ -0,0 +1,139 @@ ++/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ ++/* SPDX-FileCopyrightText: Satoru Takekoshi, Manami Mori, Takaaki Fukai, ++ * Takahiro Shinagawa */ ++/* SPDX-FileCopyrightText: Edgeless Systems GmbH */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define SANDBOX_READ_HOOK(virt_addr, phys_addr) { if (!__sandbox_validate_memory_access(virt_addr, phys_addr, true)) break; } ++#define SANDBOX_WRITE_HOOK(virt_addr, phys_addr) { if (!__sandbox_validate_memory_access(virt_addr, phys_addr, false)) break; } ++#define SANDBOX_SECT_START { __sandbox_section_start(); } ++#define SANDBOX_SECT_END { __sandbox_section_end(); } ++ ++static struct __sandbox_access_log { ++ bool is_read; ++ unsigned long phys_addr; ++ unsigned long virt_addr; ++ bool access_allowed; ++} __sandbox_access_log; ++ ++static void __sandbox_log_enabled(void) ++{ ++ DO_ONCE(pr_info, "SANDBOX: Enabled\n"); ++} ++ ++static unsigned long __sandbox_get_page_table_entry(unsigned long addr) ++{ ++ pgd_t *pgd; ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ pte_t *pte; ++ ++ pgd = pgd_offset_k(addr); ++ if (pgd_none(*pgd)) { ++ return 0; ++ } ++ ++ p4d = p4d_offset(pgd, addr); ++ if (p4d_none(*p4d)) { ++ return 0; ++ } ++ ++ pud = pud_offset(p4d, addr); ++ if (pud_none(*pud)) { ++ return 0; ++ } ++ ++ /* Check for 1GB huge page */ ++ if (pud_leaf(*pud)) { ++ return pud_val(*pud); ++ } ++ ++ pmd = pmd_offset(pud, addr); ++ if (pmd_none(*pmd)) { ++ return 0; ++ } ++ ++ /* Check for 2MB huge page */ ++ if (pmd_leaf(*pmd)) { ++ return pmd_val(*pmd); ++ } ++ ++ pte = pte_offset_kernel(pmd, addr); ++ if (pte_none(*pte)) { ++ return 0; ++ } ++ ++ return pte_val(*pte); ++} ++ ++static bool __sandbox_is_encrypted_generic(unsigned long virt_addr) ++{ ++ unsigned long val; ++ ++ val = __sandbox_get_page_table_entry((unsigned long)(virt_addr)); ++ if (val) { ++ return val == cc_mkenc(val); ++ } else { ++ ACPI_ERROR((AE_INFO, "SANDBOX: Page table walk failed")); ++ } ++ ++ ACPI_DEBUG_PRINT((ACPI_DB_INFO, "SANDBOX: Falling back to 'encrypted' state\n")); ++ return true; ++} ++ ++static bool __sandbox_validate_memory_access(unsigned long virt_addr, unsigned long phys_addr, bool is_read) ++{ ++ __sandbox_log_enabled(); ++ __sandbox_access_log.is_read = is_read; ++ __sandbox_access_log.phys_addr = phys_addr; ++ __sandbox_access_log.virt_addr = virt_addr; ++ phys_addr &= PAGE_MASK; ++ virt_addr &= PAGE_MASK; ++ ++ cond_resched(); ++ ++ bool encrypted = true; ++ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { ++ encrypted = __sandbox_is_encrypted_generic(virt_addr); ++ } else { ++ ACPI_ERROR((AE_INFO, "SANDBOX: Unknown platform")); ++ } ++ ++ cond_resched(); ++ ++ if (!encrypted) { ++ return true; ++ } ++ ++ __sandbox_access_log.access_allowed = false; ++ return false; ++} ++ ++static void __sandbox_section_start(void) ++{ ++ __sandbox_access_log.is_read = true; ++ __sandbox_access_log.phys_addr = 0xdeadbeefcafebabeuL; ++ __sandbox_access_log.virt_addr = 0xdeadbeefcafebabeuL; ++ __sandbox_access_log.access_allowed = true; ++} ++ ++static void __sandbox_section_end(void) ++{ ++ cond_resched(); ++ ++ ACPI_INFO(( ++ "SANDBOX: ACCESS %s virt=%lx phys=%lx %s", ++ __sandbox_access_log.is_read ? "r" : "w", ++ (unsigned long)__sandbox_access_log.virt_addr, ++ (unsigned long)__sandbox_access_log.phys_addr, ++ __sandbox_access_log.access_allowed ? "allowed" : "denied" ++ )); ++ ++ cond_resched(); ++} +-- +2.49.0 diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index c28b7f228..1ec8725f7 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -17,6 +17,16 @@ SRC_URI += "file://dstack-docker.cfg \ # hence this Kconfig patch. Scoped to tdx machines only. SRC_URI:append:tdx = " file://0001-x86-tdx-select-dma-direct-remap.patch" +# Confidential guests are exposed to malicious ACPI tables supplied by the +# host: crafted AML can read/write the guest's encrypted (private) memory +# through the SystemMemory operation region handler. This "BadAML sandbox" +# walks the page tables and denies AML SystemMemory accesses that target +# encrypted pages, logging each decision. Ported from the Easy-TEE project. +# Applied unconditionally: dstack OS always runs inside a TEE, so every +# build needs this hardening (the hook is a runtime no-op when the platform +# reports no memory encryption). +SRC_URI:append = " file://0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch" + KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ features/netfilter/netfilter.scc \ From fc2519e0e88015cdcaf5618f75243d64f4f9d467 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 06:26:46 -0700 Subject: [PATCH 397/435] machine: add unified dstack confidential-guest machine (TDX + SEV) Replace the tdx-specific MACHINE with a single 'dstack' machine that produces one image bootable on both Intel TDX and AMD SEV-SNP hosts; the kernel detects the platform at runtime. This avoids maintaining separate per-technology images. - Add meta-dstack/conf/machine/dstack.conf (self-contained; reuses the CC-guest kernel feature fragments from meta-confidential-compute). - linux-yocto bbappend: wire COMPATIBLE_MACHINE/KMACHINE/KERNEL_FEATURES for the dstack machine, enabling both tdx.scc and sev-snp.scc; move the dma-direct-remap override from :tdx to :dstack. - local.conf: default MACHINE = dstack. - mkimage.sh: derive deploy/image paths from ${DSTACK_MACHINE} (dstack). Verified: bitbake virtual/kernel succeeds; .config has both CONFIG_INTEL_TDX_GUEST=y and CONFIG_AMD_MEM_ENCRYPT=y / CONFIG_SEV_GUEST=y. --- meta-dstack/conf/local.conf | 2 +- meta-dstack/conf/machine/dstack.conf | 51 +++++++++++++++++++ .../linux/linux-yocto%.bbappend | 24 +++++++-- mkimage.sh | 11 ++-- 4 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 meta-dstack/conf/machine/dstack.conf diff --git a/meta-dstack/conf/local.conf b/meta-dstack/conf/local.conf index 7cadbf36c..ca090e6ca 100644 --- a/meta-dstack/conf/local.conf +++ b/meta-dstack/conf/local.conf @@ -36,7 +36,7 @@ #MACHINE ?= "genericx86-64" # # This sets the default machine to be qemux86-64 if no other machine is selected: -MACHINE ??= "tdx" +MACHINE ??= "dstack" # These are some of the more commonly used values. Looking at the files in the # meta/conf/machine directory, or the conf/machine directory of any additional layers diff --git a/meta-dstack/conf/machine/dstack.conf b/meta-dstack/conf/machine/dstack.conf new file mode 100644 index 000000000..387dcae66 --- /dev/null +++ b/meta-dstack/conf/machine/dstack.conf @@ -0,0 +1,51 @@ +#@TYPE: Machine +#@NAME: dstack confidential guest +#@DESCRIPTION: Unified dstack confidential-guest machine. A single image that +# boots on both Intel TDX and AMD SEV-SNP hosts; the kernel +# detects the platform at runtime. Kernel feature fragments +# (tdx.scc / sev-snp.scc / ...) are reused from +# meta-confidential-compute via KERNEL_FEATURES in the +# linux-yocto bbappend. +# +# The QEMU/tune boilerplate below mirrors the generic x86-64 confidential-guest +# machines in meta-confidential-compute (tdx.conf / sev-snp.conf). It is kept +# self-contained here so the dstack machine does not depend on those layer's +# machine names. + +# from require conf/machine/include/qemu.inc +# Don't include kernels in standard images +RDEPENDS:${KERNEL_PACKAGE_NAME}-base = "" + +# Use a common kernel recipe for all QEMU machines +PREFERRED_PROVIDER_virtual/kernel ??= "linux-yocto-tiny" + +EXTRA_IMAGEDEPENDS += "qemu-system-native qemu-helper-native:do_addto_recipe_sysroot" + +# from require conf/machine/include/x86/qemuboot-x86.inc +# For runqemu +IMAGE_CLASSES += "qemuboot" +QB_SMP ?= "-smp 4" +QB_CPU:x86 ?= "-cpu IvyBridge -machine q35,i8042=off" +QB_CPU_KVM:x86 ?= "-cpu IvyBridge -machine q35,i8042=off" + +QB_CPU:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" +QB_CPU_KVM:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" + +# Keep the extra spectre/SSB mitigations enabled for confidential guests. +QB_KERNEL_CMDLINE_APPEND = "oprofile.timer=1 tsc=reliable no_timer_check rcupdate.rcu_expedited=1 nospectre_v2 spec_store_bypass_disable=on" +QB_OPT_APPEND = "-usb -device usb-tablet -usb -device usb-kbd" + +DEFAULTTUNE ?= "x86-64-v3" + +require conf/machine/include/x86/tune-x86-64-v3.inc + +KERNEL_IMAGETYPE = "bzImage" + +SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyS1" + +MACHINE_FEATURES += "x86 pci tpm2" + +do_image_wic[depends] += "syslinux:do_populate_sysroot syslinux-native:do_populate_sysroot mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot" + +# For runqemu +QB_SYSTEM_NAME = "dstack" diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index c28b7f228..c85cd473b 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -14,8 +14,9 @@ SRC_URI += "file://dstack-docker.cfg \ # TDX guests need DMA_DIRECT_REMAP for shared (decrypted) coherent DMA so # devices like NVMe can complete I/O. INTEL_TDX_GUEST does not select it # upstream (and the symbol is promptless, so a .cfg fragment cannot set it), -# hence this Kconfig patch. Scoped to tdx machines only. -SRC_URI:append:tdx = " file://0001-x86-tdx-select-dma-direct-remap.patch" +# hence this Kconfig patch. Only touches the INTEL_TDX_GUEST Kconfig, so it is +# a no-op on AMD; scoped to the dstack confidential-guest machine. +SRC_URI:append:dstack = " file://0001-x86-tdx-select-dma-direct-remap.patch" KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ features/overlayfs/overlayfs.scc \ @@ -29,7 +30,24 @@ KERNEL_FEATURES:append = " features/cgroups/cgroups.scc \ KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", " features/device-mapper/dm-verity.scc", "" ,d)}" -KERNEL_FEATURES:append:tdx = " dstack-tdx.scc" +# Unified dstack confidential-guest machine. A single kernel image that boots +# on both Intel TDX and AMD SEV-SNP hosts (the kernel detects the platform at +# runtime). The base guest features and the tdx.scc / sev-snp.scc kconf +# fragments are reused from meta-confidential-compute; enabling both TDX and +# SEV here is what makes one image work on either platform. +KMACHINE:dstack ?= "common-pc-64" +COMPATIBLE_MACHINE:dstack = "dstack" +KERNEL_FEATURES:append:dstack = " features/scsi/disk.scc \ + cfg/virtio.scc \ + cfg/paravirt_kvm.scc \ + cfg/fs/ext4.scc \ + tdx.scc \ + sev-snp.scc \ + tpm2.scc \ + hyperv.scc \ + security-mitigations.scc \ + disk-encryption.scc \ + dstack-tdx.scc" # Enable BTF KERNEL_DEBUG = "True" diff --git a/mkimage.sh b/mkimage.sh index 1d8ed0164..e176628f9 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -50,9 +50,12 @@ fi BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) +# MACHINE name; artifacts are deployed under deploy/images/${MACHINE} +DSTACK_MACHINE=${DSTACK_MACHINE:-dstack} + # Common artifacts are in tmp/, flavor-specific artifacts are in tmp-mc-/ -COMMON_IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/tdx -FLAVOR_IMG_DIR=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/deploy/images/tdx +COMMON_IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/${DSTACK_MACHINE} +FLAVOR_IMG_DIR=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/deploy/images/${DSTACK_MACHINE} # Common artifacts (shared across all flavors) INITRAMFS_IMAGE=${COMMON_IMG_DIR}/dstack-initramfs.cpio.gz @@ -60,13 +63,13 @@ KERNEL_IMAGE=${COMMON_IMG_DIR}/bzImage OVMF_FIRMWARE=${COMMON_IMG_DIR}/ovmf.fd # Flavor-specific artifacts (from multiconfig build) -ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-tdx.squashfs.verity +ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-${DSTACK_MACHINE}.squashfs.verity # UKI filename UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi # Verity env is in the flavor-specific work-shared directory -VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/tdx/dm-verity/dstack-rootfs.squashfs.verity.env +VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/${DSTACK_MACHINE}/dm-verity/dstack-rootfs.squashfs.verity.env if [ ! -f "${VERITY_ENV_FILE}" ]; then echo "Error: verity env not found: ${VERITY_ENV_FILE}" >&2 echo "Build the rootfs first, e.g.: bitbake mc:${FLAVOR}:dstack-rootfs" >&2 From 68424547e0e604095f52d8ad101cf72108ce00fa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 08:05:51 -0700 Subject: [PATCH 398/435] dstack-ovmf: additionally build AMD SEV firmware (ovmf-sev.fd) Build OvmfPkg/AmdSev/AmdSevX64.dsc in addition to the existing Intel TDX firmware, producing a separate ovmf-sev.fd for AMD SEV / SEV-SNP guests. The Intel TDX build (IntelTdxX64.dsc) and its carefully pinned RTMR[0] measurement layout are left byte-for-byte unchanged: the SEV firmware is built after the TDX one into a separate Build/AmdSev tree and deployed as a distinct artifact. The AmdSevX64 embedded grub (sevsecret/LUKS measured boot) is removed via 0006-OvmfPkg-AmdSev-drop-embedded-grub.patch. dstack boots a UKI with a dm-verity rootfs and never uses that grub, and it cannot be built here anyway: OE provides no x86_64-efi grub modules and upstream grub has no sevsecret module. All other SEV/SEV-SNP functionality (SecretPei/SecretDxe, the BlobVerifierLibSevHashes kernel-hashes measured direct boot, SNP support) is retained. The patch body uses CRLF line endings to match EDK2's sources (like the existing 0005 patch) and fails loud if a future edk2 bump changes the AmdSev layout. mkimage.sh ships ovmf-sev.fd alongside ovmf.fd but keeps it OUT of the image digest (sha256sum.txt / digest.txt / metadata.json stay TDX-only), so the measured TDX image is unchanged. SEV measurement / attestation is a separate concern (dstack side, out of scope here). Verified: bitbake dstack-ovmf -- do_patch and do_compile succeed; ovmf.fd and ovmf-sev.fd both deployed as distinct binaries. --- ...06-OvmfPkg-AmdSev-drop-embedded-grub.patch | 54 +++++++++++++++++++ .../dstack-ovmf/dstack-ovmf_git.bb | 29 ++++++++++ mkimage.sh | 14 +++++ 3 files changed, 97 insertions(+) create mode 100644 meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch new file mode 100644 index 000000000..365a054ad --- /dev/null +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch @@ -0,0 +1,54 @@ +From: dstack +Date: Mon, 16 Jun 2026 00:00:00 +0000 +Subject: [PATCH] OvmfPkg/AmdSev: drop the embedded sevsecret grub + +The AmdSevX64 firmware embeds a grub.efi (built by the OvmfPkg/AmdSev/Grub +PREBUILD via grub-mkimage) that boots an encrypted LUKS volume using the +SEV-injected secret and the out-of-tree sevsecret grub module. + +dstack does not use that boot model: it boots a UKI from an EFI System +Partition with a dm-verity rootfs, so the embedded grub is dead weight. +Building it is also infeasible here: OpenEmbedded provides no x86_64-efi +grub modules at native build time and upstream grub has no sevsecret +module (it is a distro patch). + +Remove the Grub PREBUILD and the Grub.inf component/FV entry. All other +SEV / SEV-SNP functionality (SecretPei/SecretDxe secret injection, the +BlobVerifierLibSevHashes kernel-hashes measured direct boot, SNP support) +is unaffected. + +Upstream-Status: Inappropriate [dstack-specific firmware configuration] +Signed-off-by: dstack +--- +diff --git a/OvmfPkg/AmdSev/AmdSevX64.dsc b/OvmfPkg/AmdSev/AmdSevX64.dsc +index e0eaa3c..76824d8 100644 +--- a/OvmfPkg/AmdSev/AmdSevX64.dsc ++++ b/OvmfPkg/AmdSev/AmdSevX64.dsc +@@ -25,7 +25,6 @@ + BUILD_TARGETS = NOOPT|DEBUG|RELEASE + SKUID_IDENTIFIER = DEFAULT + FLASH_DEFINITION = OvmfPkg/AmdSev/AmdSevX64.fdf +- PREBUILD = sh OvmfPkg/AmdSev/Grub/grub.sh + + # + # Defines for default states. These can be changed on the command line. +@@ -731,7 +730,6 @@ + MdeModulePkg/Bus/Usb/UsbMassStorageDxe/UsbMassStorageDxe.inf + + OvmfPkg/AmdSev/SecretDxe/SecretDxe.inf +- OvmfPkg/AmdSev/Grub/Grub.inf + + !include OvmfPkg/Include/Dsc/ShellComponents.dsc.inc + !include OvmfPkg/Include/Dsc/OvmfRngComponents.dsc.inc +diff --git a/OvmfPkg/AmdSev/AmdSevX64.fdf b/OvmfPkg/AmdSev/AmdSevX64.fdf +index 4217870..ff727a5 100644 +--- a/OvmfPkg/AmdSev/AmdSevX64.fdf ++++ b/OvmfPkg/AmdSev/AmdSevX64.fdf +@@ -283,7 +283,6 @@ INF FatPkg/EnhancedFatDxe/Fat.inf + INF MdeModulePkg/Universal/Disk/UdfDxe/UdfDxe.inf + + INF OvmfPkg/AmdSev/SecretDxe/SecretDxe.inf +-INF OvmfPkg/AmdSev/Grub/Grub.inf + + INF MdeModulePkg/Logo/LogoDxe.inf + diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb index 08478c9a9..79806beb0 100644 --- a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb +++ b/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb @@ -25,6 +25,7 @@ SRC_URI = "gitsm://github.com/tianocore/edk2.git;branch=master;protocol=https \ file://0003-Debug-prefix-map.patch \ file://0004-Reproduciable.patch \ file://0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch \ + file://0006-OvmfPkg-AmdSev-drop-embedded-grub.patch \ " # Pinned to edk2-stable202502 (Feb 2025) instead of the latest stable202505. @@ -78,6 +79,16 @@ PARALLEL_MAKE = "" DEPENDS = "nasm-native acpica-native ovmf-native util-linux-native" +# Build the AMD SEV firmware in addition to the Intel TDX one. The TDX build +# (IntelTdxX64.dsc) and its carefully pinned measurement layout are left byte +# for byte unchanged; this only adds a separate ovmf-sev.fd artifact. The +# AmdSevX64 embedded grub (sevsecret LUKS boot) is removed via +# 0006-OvmfPkg-AmdSev-drop-embedded-grub.patch -- dstack boots a UKI, not +# grub, and that grub cannot be built here anyway (OE has no x86_64-efi grub +# modules / no sevsecret). The patch fails loud if a future edk2 bump changes +# the AmdSev layout. +OVMF_BUILD_SEV ??= "1" + EDK_TOOLS_DIR="edk2_basetools" # OVMF has trouble building with the default optimization of -O2. @@ -241,6 +252,18 @@ do_compile:class-target() { ln ${build_dir}/FV/OVMF_CODE.fd ${WORKDIR}/ovmf/ovmf.secboot.code.fd ln ${build_dir}/${OVMF_ARCH}/EnrollDefaultKeys.efi ${WORKDIR}/ovmf/ fi + + if [ "${OVMF_BUILD_SEV}" = "1" ]; then + # AMD SEV / SEV-SNP firmware. Additive: produces a single combined + # firmware blob (used via QEMU -bios) at ovmf-sev.fd, leaving the TDX + # build above untouched. The embedded grub is stripped below, so + # there is no PREBUILD / grub toolchain dependency. + bbnote "Building AMD SEV firmware (AmdSevX64.dsc)." + sev_build_dir="${S}/Build/AmdSev/RELEASE_${FIXED_GCCVER}" + rm -rf ${S}/Build/AmdSev + ${S}/OvmfPkg/build.sh -p ${S}/OvmfPkg/AmdSev/AmdSevX64.dsc $PARALLEL_JOBS -a $OVMF_ARCH -b RELEASE -t ${FIXED_GCCVER} ${PACKAGECONFIG_CONFARGS} + ln ${sev_build_dir}/FV/OVMF.fd ${WORKDIR}/ovmf/ovmf-sev.fd + fi } do_install:class-native() { @@ -290,6 +313,12 @@ do_deploy:class-target() { qemu-img convert -f raw -O qcow2 ${WORKDIR}/ovmf/$i.fd ${DEPLOYDIR}/$i.qcow2 done + # AMD SEV firmware (single combined blob for QEMU -bios). + if [ "${OVMF_BUILD_SEV}" = "1" ] && [ -f ${WORKDIR}/ovmf/ovmf-sev.fd ]; then + cp ${WORKDIR}/ovmf/ovmf-sev.fd ${DEPLOYDIR}/ + qemu-img convert -f raw -O qcow2 ${WORKDIR}/ovmf/ovmf-sev.fd ${DEPLOYDIR}/ovmf-sev.qcow2 + fi + if ${@bb.utils.contains('PACKAGECONFIG', 'secureboot', 'true', 'false', d)}; then # Create a test Platform Key and first Key Exchange Key to use with EnrollDefaultKeys openssl req -new -x509 -newkey rsa:2048 -keyout ${DEPLOYDIR}/OvmfPkKek1.key \ diff --git a/mkimage.sh b/mkimage.sh index 1d8ed0164..9c1c96999 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -234,6 +234,17 @@ $Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ +# AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP +# launch can select it, but deliberately kept OUT of the image digest below: +# sha256sum.txt / digest.txt / metadata.json stay TDX-only so the measured +# image is byte-for-byte unchanged. SEV measurement is a separate concern. +OVMF_SEV_FIRMWARE=${COMMON_IMG_DIR}/ovmf-sev.fd +HAVE_OVMF_SEV=0 +if [ -f "$OVMF_SEV_FIRMWARE" ]; then + $Q cp $OVMF_SEV_FIRMWARE ${OUTPUT_DIR}/ + HAVE_OVMF_SEV=1 +fi + echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" # Bare-metal partitioning needs sgdisk (from the 'gdisk' package). if ! command -v sgdisk >/dev/null; then @@ -308,6 +319,9 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then rm -rf ${IMAGE_TAR} echo "Archiving bare metal image to ${IMAGE_TAR}" BARE_METAL_FILES="rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json" + if [ "$HAVE_OVMF_SEV" = "1" ]; then + BARE_METAL_FILES="$BARE_METAL_FILES ovmf-sev.fd" + fi (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo From ef000f94f8c98493b2cd43922f9eeb712df79868 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 09:01:30 -0700 Subject: [PATCH 399/435] machine: re-enable DM_VERITY for dstack (disk-encryption.scc disables it) meta-confidential-compute's disk-encryption.scc (pulled in for dm-crypt on the encrypted data volume) sets '# CONFIG_DM_VERITY is not set'. In the dstack machine's KERNEL_FEATURES order it lands after dm-verity.scc and wins the merge, so CONFIG_DM_VERITY ended up off and the dm-verity rootfs failed to mount ('verity: unknown target type' -> kernel panic). Re-enable dm-verity as the last dm-verity fragment for the dstack machine so it wins. Verified end-to-end: image boots as a SEV-SNP guest on a real SNP host, mounts the dm-verity rootfs, and reaches the dstack login prompt. --- meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index c85cd473b..babb87696 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -49,6 +49,12 @@ KERNEL_FEATURES:append:dstack = " features/scsi/disk.scc \ disk-encryption.scc \ dstack-tdx.scc" +# disk-encryption.scc (above, from meta-confidential-compute) ships dm-crypt +# for the encrypted data volume but explicitly turns CONFIG_DM_VERITY off. The +# dstack rootfs is dm-verity, so re-enable it here -- this is the last dm-verity +# fragment in KERNEL_FEATURES for the dstack machine, so it wins the merge. +KERNEL_FEATURES:append:dstack = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", " features/device-mapper/dm-verity.scc", "", d)}" + # Enable BTF KERNEL_DEBUG = "True" From a3a7292706c4e25e103bdf65aa3df2f13e21fcca Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 18:02:22 -0700 Subject: [PATCH 400/435] machine: address review feedback for dstack machine - scripts/docker-check-config.sh, repro-build/check.sh: derive the deploy image / work dir from ${DSTACK_MACHINE:-dstack} instead of hardcoding the old 'tdx' machine name, which broke after the MACHINE rename. - linux-yocto bbappend: anchor COMPATIBLE_MACHINE:dstack as '^dstack$' so it cannot match other machine names that merely contain 'dstack'. - dstack.conf: clarify the QB_KERNEL_CMDLINE_APPEND comment (it is the runqemu test cmdline, not the production one; nospectre_v2 disables Spectre v2 while spec_store_bypass_disable=on keeps SSB on); grammar fix. --- meta-dstack/conf/machine/dstack.conf | 7 +++++-- meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend | 2 +- repro-build/check.sh | 2 +- scripts/docker-check-config.sh | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/meta-dstack/conf/machine/dstack.conf b/meta-dstack/conf/machine/dstack.conf index 387dcae66..44372f91a 100644 --- a/meta-dstack/conf/machine/dstack.conf +++ b/meta-dstack/conf/machine/dstack.conf @@ -9,7 +9,7 @@ # # The QEMU/tune boilerplate below mirrors the generic x86-64 confidential-guest # machines in meta-confidential-compute (tdx.conf / sev-snp.conf). It is kept -# self-contained here so the dstack machine does not depend on those layer's +# self-contained here so the dstack machine does not depend on that layer's # machine names. # from require conf/machine/include/qemu.inc @@ -31,7 +31,10 @@ QB_CPU_KVM:x86 ?= "-cpu IvyBridge -machine q35,i8042=off" QB_CPU:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" QB_CPU_KVM:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" -# Keep the extra spectre/SSB mitigations enabled for confidential guests. +# runqemu (test boot) kernel cmdline only -- NOT the production image cmdline, +# which is built in mkimage.sh. Inherited verbatim from the tdx machine: it +# turns the Spectre v2 mitigation off (nospectre_v2) for speed while keeping +# the Speculative Store Bypass mitigation on (spec_store_bypass_disable=on). QB_KERNEL_CMDLINE_APPEND = "oprofile.timer=1 tsc=reliable no_timer_check rcupdate.rcu_expedited=1 nospectre_v2 spec_store_bypass_disable=on" QB_OPT_APPEND = "-usb -device usb-tablet -usb -device usb-kbd" diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend index babb87696..d82cb6b6f 100644 --- a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend +++ b/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend @@ -36,7 +36,7 @@ KERNEL_FEATURES:append = " ${@bb.utils.contains("DISTRO_FEATURES", "dm-verity", # fragments are reused from meta-confidential-compute; enabling both TDX and # SEV here is what makes one image work on either platform. KMACHINE:dstack ?= "common-pc-64" -COMPATIBLE_MACHINE:dstack = "dstack" +COMPATIBLE_MACHINE:dstack = "^dstack$" KERNEL_FEATURES:append:dstack = " features/scsi/disk.scc \ cfg/virtio.scc \ cfg/paravirt_kvm.scc \ diff --git a/repro-build/check.sh b/repro-build/check.sh index e6feac044..b8460b6e8 100755 --- a/repro-build/check.sh +++ b/repro-build/check.sh @@ -15,7 +15,7 @@ YELLOW='\033[1;33m' NC='\033[0m' IMAGE_NAME=${IMAGE_NAME:-dstack-rootfs} -ROOTFS_PATH=tmp/work/tdx-poky-linux/${IMAGE_NAME}/1.0/rootfs +ROOTFS_PATH=tmp/work/${DSTACK_MACHINE:-dstack}-poky-linux/${IMAGE_NAME}/1.0/rootfs BUILD_DIR_A=${1:-${THIS_DIR}/build-a} BUILD_DIR_B=${2:-${THIS_DIR}/build-b} BB_DIR_A=${BB_DIR_A:-${BUILD_DIR_A}/bb-build} diff --git a/scripts/docker-check-config.sh b/scripts/docker-check-config.sh index 0302de9b4..b1cd098e1 100755 --- a/scripts/docker-check-config.sh +++ b/scripts/docker-check-config.sh @@ -17,7 +17,7 @@ possibleConfigs=" if [ $# -gt 0 ]; then CONFIG="$1" else - : "${CONFIG:=$SCRIPT_DIR/../bb-build/tmp/deploy/images/tdx/kernel-config}" + : "${CONFIG:=$SCRIPT_DIR/../bb-build/tmp/deploy/images/${DSTACK_MACHINE:-dstack}/kernel-config}" fi if ! command -v zgrep > /dev/null 2>&1; then From b7a40ebf01c456b523ab7150d64b0c9a5cc0410d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 18:13:35 -0700 Subject: [PATCH 401/435] linux: ACPI BadAML sandbox - no-op on non-CC platforms, quieter logging Address review feedback on the ported sandbox: - On platforms without memory encryption (cc_platform_has(CC_ATTR_MEM_ENCRYPT) false) the sandbox now treats every access as unencrypted and allows it, so it is a genuine no-op instead of denying all AML SystemMemory accesses and erroring on every one. This matches the unconditional-apply rationale. - Only denied accesses are logged unconditionally (ACPI_ERROR); allowed accesses move to ACPI debug-level output (ACPI_DEBUG_PRINT) so frequent AML SystemMemory reads/writes no longer flood dmesg/journal. Verified: do_patch and do_compile succeed against linux-yocto 6.18. --- ...ox-block-aml-systemmemory-ram-access.patch | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch b/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch index c6297b0d1..c868aa82f 100644 --- a/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch +++ b/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch @@ -9,18 +9,18 @@ or writes confidential guest memory via the SystemMemory operation region handler; this sandbox walks the page tables and denies the access when the target page is encrypted, logging the decision. +On a platform without memory encryption the sandbox is a no-op: every +access is treated as unencrypted and allowed. Only denied accesses are +logged unconditionally; allowed accesses use ACPI debug-level output to +avoid flooding the log. + Ported from the Easy-TEE project (mkosi gcp profile kernel patch). Upstream-Status: Inappropriate [confidential-guest hardening] Signed-off-by: Paul Meyer --- - drivers/acpi/acpica/exregion.c | 6 ++ - drivers/acpi/acpica/sandbox.h | 139 +++++++++++++++++++++++++++++++++ - 2 files changed, 145 insertions(+) - create mode 100644 drivers/acpi/acpica/sandbox.h - diff --git a/drivers/acpi/acpica/exregion.c b/drivers/acpi/acpica/exregion.c -index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7455eaf67 100644 +index a390a1c..6383233 100644 --- a/drivers/acpi/acpica/exregion.c +++ b/drivers/acpi/acpica/exregion.c @@ -14,6 +14,8 @@ @@ -40,7 +40,7 @@ index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7 acpi_status status = AE_OK; void *logical_addr_ptr = NULL; struct acpi_mem_space_context *mem_info = region_context; -@@ -192,6 +195,7 @@ acpi_ex_system_memory_space_handler(u32 function, +@@ -192,6 +195,7 @@ access: case ACPI_READ: *value = 0; @@ -48,7 +48,7 @@ index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7 switch (bit_width) { case 8: -@@ -223,6 +227,7 @@ acpi_ex_system_memory_space_handler(u32 function, +@@ -223,6 +227,7 @@ access: case ACPI_WRITE: @@ -56,7 +56,7 @@ index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7 switch (bit_width) { case 8: -@@ -258,6 +263,7 @@ acpi_ex_system_memory_space_handler(u32 function, +@@ -258,6 +263,7 @@ access: break; } @@ -66,10 +66,10 @@ index a390a1c2b0abb01a7c8490b207ec377818120207..638323389e970500c004b7ccdd52a9e7 diff --git a/drivers/acpi/acpica/sandbox.h b/drivers/acpi/acpica/sandbox.h new file mode 100644 -index 0000000000000000000000000000000000000000..1d9d95a87698dde14429f2f33a0c375ad51774fe +index 0000000..34d09ca --- /dev/null +++ b/drivers/acpi/acpica/sandbox.h -@@ -0,0 +1,139 @@ +@@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* SPDX-FileCopyrightText: Satoru Takekoshi, Manami Mori, Takaaki Fukai, + * Takahiro Shinagawa */ @@ -170,11 +170,14 @@ index 0000000000000000000000000000000000000000..1d9d95a87698dde14429f2f33a0c375a + + cond_resched(); + -+ bool encrypted = true; ++ /* ++ * On a platform without memory encryption there is no private guest ++ * memory to protect, so the sandbox is a no-op: treat every access as ++ * unencrypted and allow it. Only confidential guests gate accesses. ++ */ ++ bool encrypted = false; + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) { + encrypted = __sandbox_is_encrypted_generic(virt_addr); -+ } else { -+ ACPI_ERROR((AE_INFO, "SANDBOX: Unknown platform")); + } + + cond_resched(); @@ -199,15 +202,25 @@ index 0000000000000000000000000000000000000000..1d9d95a87698dde14429f2f33a0c375a +{ + cond_resched(); + -+ ACPI_INFO(( -+ "SANDBOX: ACCESS %s virt=%lx phys=%lx %s", -+ __sandbox_access_log.is_read ? "r" : "w", -+ (unsigned long)__sandbox_access_log.virt_addr, -+ (unsigned long)__sandbox_access_log.phys_addr, -+ __sandbox_access_log.access_allowed ? "allowed" : "denied" -+ )); ++ /* ++ * AML SystemMemory accesses can be frequent, so only the (rare) denied ++ * accesses are logged unconditionally; allowed accesses go to ACPI ++ * debug-level output to avoid flooding the log on every boot/runtime ++ * access while preserving observability when debugging. ++ */ ++ if (!__sandbox_access_log.access_allowed) { ++ ACPI_ERROR((AE_INFO, ++ "SANDBOX: DENIED %s virt=%lx phys=%lx", ++ __sandbox_access_log.is_read ? "r" : "w", ++ (unsigned long)__sandbox_access_log.virt_addr, ++ (unsigned long)__sandbox_access_log.phys_addr)); ++ } else { ++ ACPI_DEBUG_PRINT((ACPI_DB_INFO, ++ "SANDBOX: ALLOWED %s virt=%lx phys=%lx\n", ++ __sandbox_access_log.is_read ? "r" : "w", ++ (unsigned long)__sandbox_access_log.virt_addr, ++ (unsigned long)__sandbox_access_log.phys_addr)); ++ } + + cond_resched(); +} --- -2.49.0 From 4eb9fb095dd07868c3b0553901c4b77babd27eea Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 18:26:08 -0700 Subject: [PATCH 402/435] machine: drop QB_KERNEL_CMDLINE_APPEND from dstack machine The inherited runqemu cmdline disabled the Spectre v2 mitigation (nospectre_v2). It only affects 'runqemu' test boots, never the production image cmdline (built in mkimage.sh, which carries no mitigation flags and so runs at the kernel default = mitigation on). Dropping it makes test boots match production instead of silently weakening mitigations. --- meta-dstack/conf/machine/dstack.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/meta-dstack/conf/machine/dstack.conf b/meta-dstack/conf/machine/dstack.conf index 44372f91a..3770f997b 100644 --- a/meta-dstack/conf/machine/dstack.conf +++ b/meta-dstack/conf/machine/dstack.conf @@ -31,11 +31,10 @@ QB_CPU_KVM:x86 ?= "-cpu IvyBridge -machine q35,i8042=off" QB_CPU:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" QB_CPU_KVM:x86-64 ?= "-cpu IvyBridge -machine q35,i8042=off" -# runqemu (test boot) kernel cmdline only -- NOT the production image cmdline, -# which is built in mkimage.sh. Inherited verbatim from the tdx machine: it -# turns the Spectre v2 mitigation off (nospectre_v2) for speed while keeping -# the Speculative Store Bypass mitigation on (spec_store_bypass_disable=on). -QB_KERNEL_CMDLINE_APPEND = "oprofile.timer=1 tsc=reliable no_timer_check rcupdate.rcu_expedited=1 nospectre_v2 spec_store_bypass_disable=on" +# No QB_KERNEL_CMDLINE_APPEND: that only affects `runqemu` test boots, not the +# production image cmdline (built in mkimage.sh). The tdx machine inherited one +# that disabled the Spectre v2 mitigation (nospectre_v2); dropping it keeps test +# boots at the kernel's default mitigations, matching production. QB_OPT_APPEND = "-usb -device usb-tablet -usb -device usb-kbd" DEFAULTTUNE ?= "x86-64-v3" From 5efb9d7810bf92e95488d291f8aa0e9b5adf5b30 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 18:44:08 -0700 Subject: [PATCH 403/435] mkimage: add bios-sev field to metadata.json for the SEV firmware When the AMD SEV firmware (ovmf-sev.fd) is present, emit a "bios-sev" field in metadata.json alongside the existing TDX "bios": "ovmf.fd". This lets a VMM pick the right firmware per platform (ovmf.fd on TDX, ovmf-sev.fd on AMD SEV-SNP) instead of hardcoding the TDX firmware. The field is only emitted when the SEV firmware was built/shipped, so non-SEV builds keep the original metadata.json. metadata.json is part of sha256sum.txt, so digest.txt now reflects the bios-sev reference; this does not change any TDX hardware measurement (MRTD comes from ovmf.fd, RTMRs from kernel/cmdline/rootfs), only dstack's image-bundle digest. Verified: metadata.json is valid JSON with .['bios-sev'] = 'ovmf-sev.fd'; the field is omitted (still valid JSON) when no SEV firmware is present. --- mkimage.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index 453cbc429..b83c5c1e4 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -238,14 +238,20 @@ $Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ # AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP -# launch can select it, but deliberately kept OUT of the image digest below: -# sha256sum.txt / digest.txt / metadata.json stay TDX-only so the measured -# image is byte-for-byte unchanged. SEV measurement is a separate concern. +# launch can select it via the metadata.json "bios-sev" field below. The SEV +# firmware blob itself is NOT added to sha256sum.txt, but metadata.json (which +# references it) is, so digest.txt does reflect its presence. That does not +# change any TDX hardware measurement (MRTD comes from ovmf.fd, RTMRs from +# kernel/cmdline/rootfs) -- it only changes dstack's image-bundle digest. OVMF_SEV_FIRMWARE=${COMMON_IMG_DIR}/ovmf-sev.fd HAVE_OVMF_SEV=0 +BIOS_SEV_JSON="" if [ -f "$OVMF_SEV_FIRMWARE" ]; then $Q cp $OVMF_SEV_FIRMWARE ${OUTPUT_DIR}/ HAVE_OVMF_SEV=1 + # Inserted after the "bios" line in metadata.json (see below). + BIOS_SEV_JSON=' + "bios-sev": "ovmf-sev.fd",' fi echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" @@ -275,7 +281,7 @@ KARG2="dstack.rootfs_hash=$ROOT_HASH dstack.rootfs_size=$DATA_SIZE" cat < ${OUTPUT_DIR}/metadata.json { - "bios": "ovmf.fd", + "bios": "ovmf.fd",${BIOS_SEV_JSON} "kernel": "bzImage", "cmdline": "$KARG0 $KARG1 $KARG2", "initrd": "initramfs.cpio.gz", From ba9757391b3070f694547d9b68e31bb1f673f86a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 20:09:17 -0700 Subject: [PATCH 404/435] mkimage: emit digest.sev.txt (AMD SEV-SNP os_image_hash) When the SEV firmware is shipped, run `dstack-vmm sev-os-image-hash` over the output image to produce digest.sev.txt: the AMD SEV-SNP os_image_hash computed from the SEV firmware (ovmf-sev.fd) + kernel/initrd/cmdline/rootfs. Unlike the TDX digest.txt (a content hash that includes the TDX firmware and is vCPU-independent by construction), the SEV os_image_hash is the image-invariant projection of the SNP launch measurement, so it matches what the KMS verifier derives at attestation time. Gracefully skipped (with a warning) when the SEV firmware is absent or the dstack-vmm tool is not available; build.sh's build_host stage builds it to the repo root. The tool path can be overridden with DSTACK_VMM_BIN. Verified: digest.sev.txt is produced and equals the dstack-vmm output and the KMS-derived os_image_hash (cross-checked by a unit test in dstack). --- mkimage.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mkimage.sh b/mkimage.sh index b83c5c1e4..d4c8cfc02 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -300,6 +300,28 @@ sha256sum ovmf.fd bzImage initramfs.cpio.gz metadata.json > sha256sum.txt sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd +# digest.sev.txt: the AMD SEV-SNP os_image_hash. Unlike the TDX digest.txt +# (a content hash that includes the TDX firmware), this is computed by +# dstack-vmm from the SEV firmware (ovmf-sev.fd) + kernel/initrd/cmdline/rootfs +# and matches the os_image_hash the KMS verifier derives from a launch +# measurement. Only emitted when the SEV firmware shipped and the tool exists. +HAVE_DIGEST_SEV=0 +if [ "$HAVE_OVMF_SEV" = "1" ]; then + if [ -z "${DSTACK_VMM_BIN:-}" ]; then + for c in "$SCRIPT_DIR/dstack-vmm" "$SCRIPT_DIR/rust-target/release/dstack-vmm" \ + "$SCRIPT_DIR/dstack/target/release/dstack-vmm"; do + [ -x "$c" ] && DSTACK_VMM_BIN="$c" && break + done + fi + if [ -n "${DSTACK_VMM_BIN:-}" ] && [ -x "${DSTACK_VMM_BIN}" ]; then + echo "Generating digest.sev.txt via ${DSTACK_VMM_BIN}" + "${DSTACK_VMM_BIN}" sev-os-image-hash "${OUTPUT_DIR}" > "${OUTPUT_DIR}/digest.sev.txt" + HAVE_DIGEST_SEV=1 + else + echo "Warning: dstack-vmm not found; skipping digest.sev.txt (set DSTACK_VMM_BIN)" >&2 + fi +fi + # Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR UKI_CREATED=0 if [ "$ENABLE_UKI_IMAGE" = "1" ]; then @@ -331,6 +353,9 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then if [ "$HAVE_OVMF_SEV" = "1" ]; then BARE_METAL_FILES="$BARE_METAL_FILES ovmf-sev.fd" fi + if [ "$HAVE_DIGEST_SEV" = "1" ]; then + BARE_METAL_FILES="$BARE_METAL_FILES digest.sev.txt" + fi (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo From dcf407063a01815b7bf49f74443adcab1e6effe8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:59:31 -0700 Subject: [PATCH 405/435] mkimage: compute digest.sev.txt via dstack-mr, build it if needed The AMD SEV-SNP os_image_hash logic moved from dstack-vmm into the dstack-mr crate (which now has a `sev-os-image-hash` command). Generate digest.sev.txt with dstack-mr instead of dstack-vmm, and make it required rather than best-effort: if a prebuilt dstack-mr is not found, build it (cargo build --release -p dstack-mr) instead of silently skipping. The VMM now reads this file at deploy time, so the image must ship it. --- mkimage.sh | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/mkimage.sh b/mkimage.sh index d4c8cfc02..0e8b0059d 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -301,25 +301,28 @@ sha256sum sha256sum.txt | awk '{print $1}' > digest.txt popd # digest.sev.txt: the AMD SEV-SNP os_image_hash. Unlike the TDX digest.txt -# (a content hash that includes the TDX firmware), this is computed by -# dstack-vmm from the SEV firmware (ovmf-sev.fd) + kernel/initrd/cmdline/rootfs -# and matches the os_image_hash the KMS verifier derives from a launch -# measurement. Only emitted when the SEV firmware shipped and the tool exists. +# (a content hash that includes the TDX firmware), this is computed by the +# `dstack-mr` tool from the SEV firmware (ovmf-sev.fd) + kernel/initrd/cmdline/ +# rootfs and matches the os_image_hash the KMS verifier derives from a launch +# measurement. The VMM reads this file at deploy time instead of recomputing it, +# so it is required (not best-effort): if `dstack-mr` is not prebuilt, build it. HAVE_DIGEST_SEV=0 if [ "$HAVE_OVMF_SEV" = "1" ]; then - if [ -z "${DSTACK_VMM_BIN:-}" ]; then - for c in "$SCRIPT_DIR/dstack-vmm" "$SCRIPT_DIR/rust-target/release/dstack-vmm" \ - "$SCRIPT_DIR/dstack/target/release/dstack-vmm"; do - [ -x "$c" ] && DSTACK_VMM_BIN="$c" && break + DSTACK_SRC="${DSTACK_SRC:-$SCRIPT_DIR/dstack}" + if [ -z "${DSTACK_MR_BIN:-}" ]; then + for c in "$SCRIPT_DIR/dstack-mr" "$SCRIPT_DIR/rust-target/release/dstack-mr" \ + "$DSTACK_SRC/target/release/dstack-mr"; do + [ -x "$c" ] && DSTACK_MR_BIN="$c" && break done fi - if [ -n "${DSTACK_VMM_BIN:-}" ] && [ -x "${DSTACK_VMM_BIN}" ]; then - echo "Generating digest.sev.txt via ${DSTACK_VMM_BIN}" - "${DSTACK_VMM_BIN}" sev-os-image-hash "${OUTPUT_DIR}" > "${OUTPUT_DIR}/digest.sev.txt" - HAVE_DIGEST_SEV=1 - else - echo "Warning: dstack-vmm not found; skipping digest.sev.txt (set DSTACK_VMM_BIN)" >&2 + if [ -z "${DSTACK_MR_BIN:-}" ]; then + echo "Building dstack-mr to compute digest.sev.txt" + ( cd "$DSTACK_SRC" && cargo build --release -p dstack-mr ) + DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" fi + echo "Generating digest.sev.txt via ${DSTACK_MR_BIN}" + "${DSTACK_MR_BIN}" sev-os-image-hash "${OUTPUT_DIR}" > "${OUTPUT_DIR}/digest.sev.txt" + HAVE_DIGEST_SEV=1 fi # Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR From ed0062be243399a76ebcf0482c8452b4e6c3bbb6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Jun 2026 20:15:51 -0700 Subject: [PATCH 406/435] dstack.conf: release 0.6.0 --- meta-dstack/conf/distro/dstack.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 10f36c74f..397ca34cc 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -7,7 +7,7 @@ DISTRO = "dstack" DISTRO_NAME = "DStack" DISTRO_FEATURES:append = " virtualization seccomp systemd usrmerge security dm-verity ipv6" -DISTRO_VERSION = "0.6.0.a2" +DISTRO_VERSION = "0.6.0" DISTROOVERRIDES = "poky:dstack" INITRAMFS_IMAGE = "" From 4883e4585273a6b1adca34887abf50cc2154385f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Jun 2026 20:41:12 -0700 Subject: [PATCH 407/435] nvidia: skip services when hardware is absent --- .../images/dstack-rootfs-nvidia.inc | 1 + ...idia-fabricmanager-nvswitch-condition.conf | 6 +++ .../nvidia/files/nvidia-gpu-detect | 51 +++++++++++++++++++ .../nvidia/files/nvidia-persistenced.service | 3 ++ .../nvidia/nvidia-fabricmanager_%.bbappend | 15 ++++++ .../nvidia/nvidia-gpu-detect_1.0.bb | 16 ++++++ .../nvidia/nvidia-persistenced_1.0.bb | 2 + 7 files changed, 94 insertions(+) create mode 100644 meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf create mode 100755 meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend create mode 100644 meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 2b2355b23..914084ef4 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -1,6 +1,7 @@ NVIDIA_GROUP = "acpid \ nvidia \ nvidia-firmware \ + nvidia-gpu-detect \ nvidia-persistenced \ nvidia-fabricmanager \ libnvidia-nscq \ diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf new file mode 100644 index 000000000..61d490a5a --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf @@ -0,0 +1,6 @@ +# Skip the fabric manager cleanly on hosts without NVSwitch (i.e. any non +# HGX/DGX instance, including GPU instances that have no NVSwitch) instead of +# letting it fail. ExecCondition exit 1 -> systemd marks the unit as skipped, +# not failed. +[Service] +ExecCondition=/usr/bin/nvidia-gpu-detect nvswitch diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect new file mode 100755 index 000000000..1fdb6d2e6 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect @@ -0,0 +1,51 @@ +#!/bin/sh +# nvidia-gpu-detect - report NVIDIA GPU / NVSwitch presence via sysfs PCI. +# +# Intended for use as a systemd ExecCondition= so that GPU-only services skip +# cleanly (instead of failing) on instances without a GPU / without NVSwitch. +# This lets the NVIDIA and plain images be merged into a single image. +# +# exit 0 -> hardware present (systemd runs the unit) +# exit 1 -> hardware absent (systemd skips the unit, no failure) +# +# Detection mirrors scripts/bin/enable_vfio_passthrough.sh, but reads PCI class +# codes from sysfs so it works before the nvidia driver is loaded and without +# relying on lspci / the pci.ids name database: +# +# GPU = NVIDIA (0x10de) VGA (0x0300xx) or 3D controller (0x0302xx) +# NVSwitch = NVIDIA (0x10de) bridge (0x06xxxx, any bridge subclass) +# +# NVSwitch matches any NVIDIA bridge-class device (mirrors the `grep 'Bridge'` +# heuristic in scripts/bin/enable_vfio_passthrough.sh) rather than only the +# "Other bridge" subclass 0x0680, so it stays correct if a future NVSwitch +# enumerates under a different bridge subclass. This is safe here because these +# images run as TDX guests whose only NVIDIA-vendor devices are passed-through +# GPUs (class 0x03xx) and NVSwitches (class 0x06xx). + +NVIDIA_VENDOR="0x10de" + +# match_vendor_class : succeed if any PCI device has NVIDIA's +# vendor id and a class matching the given glob (e.g. "0x0302*"). +match_vendor_class() { + for dev in /sys/bus/pci/devices/*; do + [ -r "$dev/vendor" ] && [ -r "$dev/class" ] || continue + [ "$(cat "$dev/vendor")" = "$NVIDIA_VENDOR" ] || continue + case "$(cat "$dev/class")" in + $1) return 0 ;; + esac + done + return 1 +} + +case "${1:-gpu}" in + gpu) + match_vendor_class '0x0300*' || match_vendor_class '0x0302*' + ;; + nvswitch) + match_vendor_class '0x06*' + ;; + *) + echo "usage: ${0##*/} {gpu|nvswitch}" >&2 + exit 64 + ;; +esac diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service index 07e956b35..3960c51da 100644 --- a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service @@ -6,6 +6,9 @@ After=systemd-modules-load.service modprobe@.service [Service] Type=oneshot +# Skip cleanly on instances without an NVIDIA GPU (exit 1 -> unit skipped, not +# failed), so the NVIDIA image can run on GPU-less hosts. +ExecCondition=/usr/bin/nvidia-gpu-detect gpu ExecStart=/usr/bin/env nvidia-persistenced --uvm-persistence-mode ExecStartPost=/usr/bin/nvidia-smi conf-compute -srs 1 RemainAfterExit=yes diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend new file mode 100644 index 000000000..8ae5bd880 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend @@ -0,0 +1,15 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +# Only start the fabric manager when NVSwitch hardware is present, so the +# service is silently skipped (not failed) on non-NVSwitch instances. +SRC_URI += "file://nvidia-fabricmanager-nvswitch-condition.conf" + +RDEPENDS:${PN} += "nvidia-gpu-detect" + +do_install:append() { + install -d ${D}${systemd_system_unitdir}/nvidia-fabricmanager.service.d + install -m 0644 ${UNPACKDIR}/nvidia-fabricmanager-nvswitch-condition.conf \ + ${D}${systemd_system_unitdir}/nvidia-fabricmanager.service.d/10-nvswitch-condition.conf +} + +FILES:${PN} += "${systemd_system_unitdir}/nvidia-fabricmanager.service.d/10-nvswitch-condition.conf" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb new file mode 100644 index 000000000..036aa81b7 --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb @@ -0,0 +1,16 @@ +SUMMARY = "Detect NVIDIA GPU / NVSwitch presence for conditional systemd services" +DESCRIPTION = "Small sysfs-based helper used as a systemd ExecCondition= so that \ +GPU-only services (nvidia-persistenced, nvidia-fabricmanager) skip cleanly on \ +instances without a GPU or without NVSwitch, allowing a single merged image." +LICENSE = "CLOSED" + +SRC_URI = "file://nvidia-gpu-detect" + +S = "${UNPACKDIR}" + +do_install() { + install -d ${D}${bindir} + install -m 0755 ${UNPACKDIR}/nvidia-gpu-detect ${D}${bindir}/nvidia-gpu-detect +} + +FILES:${PN} = "${bindir}/nvidia-gpu-detect" diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb index 0c3ee945c..e7026f35d 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb @@ -9,6 +9,8 @@ S = "${UNPACKDIR}" inherit systemd +RDEPENDS:${PN} += "nvidia-gpu-detect" + SYSTEMD_PACKAGES = "${PN}" SYSTEMD_SERVICE:${PN} = "nvidia-persistenced.service" SYSTEMD_AUTO_ENABLE:${PN} = "enable" From e8f7c7acee6925705ea01a8437e34b1f2d75b464 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 22 Jun 2026 21:06:21 -0700 Subject: [PATCH 408/435] build: drop separate NVIDIA flavors Include NVIDIA support in the default prod/dev images and remove the nvidia/nvidia-dev multiconfig outputs. Validate mkimage flavor/dist-name consistency and avoid unconditional NVIDIA module autoloading by loading modules from hardware-gated NVIDIA services. --- Makefile | 6 ++-- build.sh | 10 +++---- meta-dstack/conf/distro/dstack.conf | 2 +- meta-dstack/conf/local.conf | 2 +- meta-dstack/conf/multiconfig/dev.conf | 1 - meta-dstack/conf/multiconfig/nvidia-dev.conf | 7 ----- meta-dstack/conf/multiconfig/nvidia.conf | 7 ----- meta-dstack/conf/multiconfig/prod.conf | 1 - .../images/dstack-rootfs-nvidia.inc | 5 +++- .../recipes-core/images/dstack-rootfs.bb | 8 +++--- ...idia-fabricmanager-nvswitch-condition.conf | 6 ++++ .../nvidia/files/nvidia-persistenced.service | 6 ++++ .../nvidia/nvidia-fabricmanager_%.bbappend | 2 +- .../nvidia/nvidia-persistenced_1.0.bb | 2 +- mkimage.sh | 28 ++++++++++++++----- repro-build/repro-build.sh | 10 +++++-- scripts/bin/mk-image-mr.sh | 2 +- 17 files changed, 60 insertions(+), 45 deletions(-) delete mode 100644 meta-dstack/conf/multiconfig/nvidia-dev.conf delete mode 100644 meta-dstack/conf/multiconfig/nvidia.conf diff --git a/Makefile b/Makefile index 952affb5b..9f0128cf4 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ DIST_DIR ?= ${BB_BUILD_DIR}/dist export BB_BUILD_DIR export DIST_DIR -# Flavor names map to multiconfig names: prod, dev, nvidia, nvidia-dev -FLAVORS ?= prod dev nvidia nvidia-dev +# Flavor names map to multiconfig names: prod, dev +FLAVORS ?= prod # Map flavor to dist name for mkimage.sh -flavor_to_dist = $(if $(filter prod,$1),dstack,$(if $(filter dev,$1),dstack-dev,$(if $(filter nvidia,$1),dstack-nvidia,$(if $(filter nvidia-dev,$1),dstack-nvidia-dev,$1)))) +flavor_to_dist = $(if $(filter prod,$1),dstack,$(if $(filter dev,$1),dstack-dev,$1)) all: dist diff --git a/build.sh b/build.sh index 3c3092b4b..53a5e3cff 100755 --- a/build.sh +++ b/build.sh @@ -340,17 +340,15 @@ Actions: all - Build everything (host, guest, and configuration) help - Show this help -Build a specific guest flavor (default builds all four): set the FLAVORS env var. +Build a specific guest flavor (default builds production): set the FLAVORS env var. flavor -> output image prod -> dstack dev -> dstack-dev - nvidia -> dstack-nvidia - nvidia-dev -> dstack-nvidia-dev Examples: - FLAVORS=nvidia ./build.sh guest # build only dstack-nvidia - FLAVORS="prod nvidia" ./build.sh guest # build two flavors - ./build.sh guest # build all flavors (default) + ./build.sh guest # build dstack (default) + FLAVORS=dev ./build.sh guest # build only dstack-dev + FLAVORS="prod dev" ./build.sh guest # build both flavors EOF } diff --git a/meta-dstack/conf/distro/dstack.conf b/meta-dstack/conf/distro/dstack.conf index 397ca34cc..137441eb7 100644 --- a/meta-dstack/conf/distro/dstack.conf +++ b/meta-dstack/conf/distro/dstack.conf @@ -25,7 +25,7 @@ SERIAL_CONSOLES = "115200;ttyS0" PREFERRED_VERSION_rust-bin-cross-x86_64 = "1.92.0" PREFERRED_VERSION_cargo-bin-cross-x86_64 = "1.92.0" -# NVIDIA driver stack (only consulted when nvidia flavor is built). +# NVIDIA driver stack (included in the default images). # Bump all three together — kernel module ABI is paired with userspace libs. NVIDIA_VERSION = "595.58.03" PREFERRED_VERSION_nvidia = "${NVIDIA_VERSION}" diff --git a/meta-dstack/conf/local.conf b/meta-dstack/conf/local.conf index ca090e6ca..cc65caedc 100644 --- a/meta-dstack/conf/local.conf +++ b/meta-dstack/conf/local.conf @@ -296,4 +296,4 @@ SOURCE_MIRROR_URL = "https://mirrors.kernel.org/yocto-sources/" # Improve resilience for large files on unstable links FETCHCMD_wget = "wget --progress=dot --inet4-only -c" -BBMULTICONFIG = "prod dev nvidia nvidia-dev" +BBMULTICONFIG = "prod dev" diff --git a/meta-dstack/conf/multiconfig/dev.conf b/meta-dstack/conf/multiconfig/dev.conf index fb9dff407..8c1a766d3 100644 --- a/meta-dstack/conf/multiconfig/dev.conf +++ b/meta-dstack/conf/multiconfig/dev.conf @@ -1,6 +1,5 @@ # Development flavor configuration DSTACK_FLAVOR = "dev" -DSTACK_NVIDIA = "0" DSTACK_DEV = "1" # Use separate TMPDIR to avoid conflicts between multiconfigs diff --git a/meta-dstack/conf/multiconfig/nvidia-dev.conf b/meta-dstack/conf/multiconfig/nvidia-dev.conf deleted file mode 100644 index 65c6cce50..000000000 --- a/meta-dstack/conf/multiconfig/nvidia-dev.conf +++ /dev/null @@ -1,7 +0,0 @@ -# NVIDIA development flavor configuration -DSTACK_FLAVOR = "nvidia-dev" -DSTACK_NVIDIA = "1" -DSTACK_DEV = "1" - -# Use separate TMPDIR to avoid conflicts between multiconfigs -TMPDIR = "${TOPDIR}/tmp-mc-nvidia-dev" diff --git a/meta-dstack/conf/multiconfig/nvidia.conf b/meta-dstack/conf/multiconfig/nvidia.conf deleted file mode 100644 index 4f3f0e625..000000000 --- a/meta-dstack/conf/multiconfig/nvidia.conf +++ /dev/null @@ -1,7 +0,0 @@ -# NVIDIA production flavor configuration -DSTACK_FLAVOR = "nvidia" -DSTACK_NVIDIA = "1" -DSTACK_DEV = "0" - -# Use separate TMPDIR to avoid conflicts between multiconfigs -TMPDIR = "${TOPDIR}/tmp-mc-nvidia" diff --git a/meta-dstack/conf/multiconfig/prod.conf b/meta-dstack/conf/multiconfig/prod.conf index f39fe60b9..586ceb191 100644 --- a/meta-dstack/conf/multiconfig/prod.conf +++ b/meta-dstack/conf/multiconfig/prod.conf @@ -1,6 +1,5 @@ # Production flavor configuration DSTACK_FLAVOR = "prod" -DSTACK_NVIDIA = "0" DSTACK_DEV = "0" # Use separate TMPDIR to avoid conflicts between multiconfigs diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 914084ef4..9e6cf6037 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -9,7 +9,10 @@ NVIDIA_GROUP = "acpid \ kernel-module-video \ numactl \ " -KERNEL_MODULE_AUTOLOAD:append = " nvidia nvidia-drm nvidia-modeset nvidia-uvm video" +# Do not autoload NVIDIA modules on every boot. The default image also runs on +# GPU-less hosts, so GPU services load these modules only after hardware +# detection succeeds. +KERNEL_MODULE_AUTOLOAD:append = " video" IMAGE_INSTALL:append = " ${NVIDIA_GROUP}" DOCKER_DAEMON_JSON = "${THISDIR}/files/docker-daemon-nvidia.json" diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/meta-dstack/recipes-core/images/dstack-rootfs.bb index 930df27fb..419933b1d 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs.bb +++ b/meta-dstack/recipes-core/images/dstack-rootfs.bb @@ -1,10 +1,9 @@ # Unified dstack rootfs image # Use DSTACK_FLAVOR (via multiconfig) to select variant: -# prod, dev, nvidia, nvidia-dev +# prod, dev # Default flavor settings (can be overridden by multiconfig) DSTACK_FLAVOR ?= "prod" -DSTACK_NVIDIA ?= "0" DSTACK_DEV ?= "0" # Base configuration @@ -13,5 +12,6 @@ include dstack-rootfs-base.inc # Production or development mode include ${@'dstack-rootfs-dev.inc' if d.getVar('DSTACK_DEV') == '1' else 'dstack-rootfs-prod.inc'} -# NVIDIA support (optional) -include ${@'dstack-rootfs-nvidia.inc' if d.getVar('DSTACK_NVIDIA') == '1' else ''} +# NVIDIA support is included in all images; services are gated at runtime by +# hardware-detection ExecCondition= checks so the same image works without GPUs. +include dstack-rootfs-nvidia.inc diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf index 61d490a5a..a55d2d272 100644 --- a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf @@ -4,3 +4,9 @@ # not failed. [Service] ExecCondition=/usr/bin/nvidia-gpu-detect nvswitch +# Load NVIDIA modules only after NVSwitch detection succeeds to avoid boot-time +# modprobe failures/noise on GPU-less hosts. +ExecStartPre=/usr/bin/env modprobe nvidia +ExecStartPre=/usr/bin/env modprobe nvidia-modeset +ExecStartPre=/usr/bin/env modprobe nvidia-drm +ExecStartPre=/usr/bin/env modprobe nvidia-uvm diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service index 3960c51da..7630c0838 100644 --- a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service @@ -9,6 +9,12 @@ Type=oneshot # Skip cleanly on instances without an NVIDIA GPU (exit 1 -> unit skipped, not # failed), so the NVIDIA image can run on GPU-less hosts. ExecCondition=/usr/bin/nvidia-gpu-detect gpu +# Load NVIDIA modules only after GPU detection succeeds to avoid boot-time +# modprobe failures/noise on GPU-less hosts. +ExecStartPre=/usr/bin/env modprobe nvidia +ExecStartPre=/usr/bin/env modprobe nvidia-modeset +ExecStartPre=/usr/bin/env modprobe nvidia-drm +ExecStartPre=/usr/bin/env modprobe nvidia-uvm ExecStart=/usr/bin/env nvidia-persistenced --uvm-persistence-mode ExecStartPost=/usr/bin/nvidia-smi conf-compute -srs 1 RemainAfterExit=yes diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend index 8ae5bd880..429e19b4b 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend @@ -4,7 +4,7 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" # service is silently skipped (not failed) on non-NVSwitch instances. SRC_URI += "file://nvidia-fabricmanager-nvswitch-condition.conf" -RDEPENDS:${PN} += "nvidia-gpu-detect" +RDEPENDS:${PN} += "nvidia-gpu-detect kmod" do_install:append() { install -d ${D}${systemd_system_unitdir}/nvidia-fabricmanager.service.d diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb index e7026f35d..8a9cf8b33 100644 --- a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb +++ b/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb @@ -9,7 +9,7 @@ S = "${UNPACKDIR}" inherit systemd -RDEPENDS:${PN} += "nvidia-gpu-detect" +RDEPENDS:${PN} += "nvidia-gpu-detect kmod" SYSTEMD_PACKAGES = "${PN}" SYSTEMD_SERVICE:${PN} = "nvidia-persistenced.service" diff --git a/mkimage.sh b/mkimage.sh index 0e8b0059d..4397f030f 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -36,16 +36,30 @@ if [ -z "$DIST_NAME" ]; then fi if [ -z "$FLAVOR" ]; then - echo "Error: --flavor is required (prod, dev, nvidia, nvidia-dev)" + echo "Error: --flavor is required (prod, dev)" exit 1 fi - -if [[ "$DIST_NAME" == *-dev ]]; then - IS_DEV=true -else - IS_DEV=false -fi +case "$FLAVOR" in + prod) + if [[ "$DIST_NAME" == *-dev ]]; then + echo "Error: prod flavor requires a non-dev dist name: $DIST_NAME" >&2 + exit 1 + fi + IS_DEV=false + ;; + dev) + if [[ "$DIST_NAME" != *-dev ]]; then + echo "Error: dev flavor requires a dist name ending in -dev: $DIST_NAME" >&2 + exit 1 + fi + IS_DEV=true + ;; + *) + echo "Error: unsupported --flavor '$FLAVOR' (expected prod or dev)" >&2 + exit 1 + ;; +esac BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) diff --git a/repro-build/repro-build.sh b/repro-build/repro-build.sh index c89680b03..b2d51973f 100755 --- a/repro-build/repro-build.sh +++ b/repro-build/repro-build.sh @@ -4,6 +4,10 @@ set -e usage() { echo "Usage: $0 [-n]" echo " -n: Don't check reproducibility" + echo "" + echo "Environment:" + echo " RELEASE_FLAVORS: space-separated flavors to build (default: prod)" + echo " e.g. RELEASE_FLAVORS=\"prod dev\" $0" } NO_CHECK=0 @@ -51,8 +55,8 @@ build_to() { $BUILDER_NAME bash -e -c "$BUILD_CMD" } -# Only build production flavors (no dev) for reproducible builds -RELEASE_FLAVORS="prod nvidia" +# Build production by default; callers may override, e.g. RELEASE_FLAVORS="prod dev". +RELEASE_FLAVORS="${RELEASE_FLAVORS:-prod}" build_to $HOST_BUILD_DIR_A "FLAVORS='$RELEASE_FLAVORS' DSTACK_TAR_RELEASE=1" @@ -79,7 +83,7 @@ git clone https://github.com/Phala-Network/meta-dstack-cloud.git cd meta-dstack-cloud/ git checkout $(git -C $THIS_DIR rev-parse HEAD) git submodule update --init --recursive -cd repro-build && ./repro-build.sh -n +cd repro-build && RELEASE_FLAVORS='${RELEASE_FLAVORS}' ./repro-build.sh -n EOF echo "===========================" diff --git a/scripts/bin/mk-image-mr.sh b/scripts/bin/mk-image-mr.sh index 71ff61e56..9c36c41e6 100755 --- a/scripts/bin/mk-image-mr.sh +++ b/scripts/bin/mk-image-mr.sh @@ -5,7 +5,7 @@ set -euo pipefail # Function to display usage usage() { echo "Usage: $0 " - echo "Example: $0 https://github.com/nearai/private-ml-sdk/releases/download/v0.5.3.1/dstack-nvidia-0.5.3.1.tar.gz" + echo "Example: $0 https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.6.0/dstack-0.6.0.tar.gz" echo "Example: $0 /path/to/local/file.tar.gz" exit 1 } From ff358d2adb1eda440afbb9cd837c21d5680dea09 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 23 Jun 2026 06:35:00 -0700 Subject: [PATCH 409/435] meta-virtualization: fix docker-compose buildkit fetch --- meta-virtualization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-virtualization b/meta-virtualization index effab7a4b..eee9e184c 160000 --- a/meta-virtualization +++ b/meta-virtualization @@ -1 +1 @@ -Subproject commit effab7a4bc4f2b4d1c7036f52d41c64a2f4a77ae +Subproject commit eee9e184c3d92869cc184afd119e7b39dc1fd63d From 7d47fadf4694b210d66a5b7d400b780eaeb2ba46 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 23 Jun 2026 07:40:58 -0700 Subject: [PATCH 410/435] repro-build: install Rust toolchain for dstack-mr --- dstack | 2 +- repro-build/Dockerfile.repro | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dstack b/dstack index f1ba0a22a..439e3d51f 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit f1ba0a22a961e8bdae110c7b1a796bc56dfd10a8 +Subproject commit 439e3d51ff353f03d03a6069e7c7dafc49a93677 diff --git a/repro-build/Dockerfile.repro b/repro-build/Dockerfile.repro index b6951f59a..0d1fdc540 100644 --- a/repro-build/Dockerfile.repro +++ b/repro-build/Dockerfile.repro @@ -7,6 +7,10 @@ ENV DEBIAN_FRONTEND=noninteractive \ LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + # Set timezone RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone @@ -19,6 +23,8 @@ RUN apt update && apt install -y \ wireguard-tools \ python3 \ locales \ + ca-certificates \ + curl \ git \ file \ gawk \ @@ -32,5 +38,16 @@ RUN apt update && apt install -y \ mtools \ libsqlite3-dev +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile minimal --default-toolchain 1.92 && \ + rustup component add rustfmt clippy rust-analyzer && \ + rustup target add wasm32-unknown-unknown x86_64-unknown-linux-musl thumbv6m-none-eabi && \ + chmod -R a+rX "$RUSTUP_HOME" "$CARGO_HOME" && \ + cargo --version && \ + rustc --version + +# The image runs as the caller's uid, so keep Cargo's registry/cache writable. +ENV CARGO_HOME=/tmp/cargo + # Generate locale for bitbake RUN locale-gen en_US.UTF-8 From 00f36be414fb5ee73d52aaf9cb072240acfdddcc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 28 Jun 2026 19:02:19 -0700 Subject: [PATCH 411/435] chrony: enable NTS PACKAGECONFIG meta-openembedded's chrony recipe now gates NTS behind PACKAGECONFIG[nts].\n\nThe dstack chrony configuration uses NTS servers, so enable the packageconfig explicitly instead of relying on gnutls being present in DEPENDS. --- meta-dstack/recipes-core/chrony/chrony%.bbappend | 1 + 1 file changed, 1 insertion(+) diff --git a/meta-dstack/recipes-core/chrony/chrony%.bbappend b/meta-dstack/recipes-core/chrony/chrony%.bbappend index ba3d2eafe..08e99c90c 100644 --- a/meta-dstack/recipes-core/chrony/chrony%.bbappend +++ b/meta-dstack/recipes-core/chrony/chrony%.bbappend @@ -1,3 +1,4 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/files:" DEPENDS += "gnutls" +PACKAGECONFIG:append = " nts" From f8cc4ba81cb223ae462389177b7ff27c9a8ff1e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 1 Jul 2026 23:57:54 -0700 Subject: [PATCH 412/435] Suppress systemd in-progress job status --- ...ore-suppress-ephemeral-status-output.patch | 40 +++++++++++++++++++ .../recipes-core/systemd/systemd_%.bbappend | 4 ++ 2 files changed, 44 insertions(+) create mode 100644 meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch diff --git a/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch b/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch new file mode 100644 index 000000000..3ef6e8e6a --- /dev/null +++ b/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch @@ -0,0 +1,40 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Kevin Wang +Date: Thu, 2 Jul 2026 00:00:00 -0700 +Subject: [PATCH] core: suppress ephemeral status output + +Ephemeral status output contains in-progress job updates such as: + + A start job is running for ... + +These updates are intended to refresh the same console line, but serial +log capture records each refresh as a separate line. Suppress only +ephemeral status messages so normal [ OK ] and [FAILED] status output is +preserved. + +Upstream-Status: Inappropriate [dstack-specific serial console policy] +Signed-off-by: Kevin Wang +--- + src/core/manager.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/core/manager.c b/src/core/manager.c +index 3b87896654..aa68edee1d 100644 +--- a/src/core/manager.c ++++ b/src/core/manager.c +@@ -4539,6 +4539,13 @@ static bool manager_should_show_status(Manager *m, StatusType type) { + if (!IN_SET(manager_state(m), MANAGER_INITIALIZING, MANAGER_STARTING, MANAGER_STOPPING)) + return false; + ++ /* Ephemeral status messages contain in-progress job updates such as ++ * "A start job is running for ...". Serial log capture records those ++ * carriage-return-based refreshes as repeated lines, so suppress them ++ * while preserving normal [ OK ] and [FAILED] status output. */ ++ if (type == STATUS_TYPE_EPHEMERAL) ++ return false; ++ + /* If we cannot find out the status properly, just proceed. */ + if (type < STATUS_TYPE_EMERGENCY && manager_check_ask_password(m) > 0) + return false; +-- +2.43.0 diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/meta-dstack/recipes-core/systemd/systemd_%.bbappend index 1a7b43b66..b374de353 100644 --- a/meta-dstack/recipes-core/systemd/systemd_%.bbappend +++ b/meta-dstack/recipes-core/systemd/systemd_%.bbappend @@ -1,3 +1,7 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI:append = " file://0001-core-suppress-ephemeral-status-output.patch" + do_install:append() { # Remove systemd-vconsole-setup entirely (no virtual console needed) rm -f ${D}${systemd_system_unitdir}/sysinit.target.wants/systemd-vconsole-setup.service From 168b3548b179ea42c408c92c9d018fee29dacd3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 7 Jul 2026 23:09:30 -0700 Subject: [PATCH 413/435] gcp: build unified OS image hash artifacts --- dstack | 2 +- mkimage.sh | 97 +++++++++++++++++++++++++++++--------------- repro-build/check.sh | 4 ++ 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/dstack b/dstack index 439e3d51f..a2fbb67b0 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 439e3d51ff353f03d03a6069e7c7dafc49a93677 +Subproject commit a2fbb67b020eb4e9bbef15318770590cab3a2cf6 diff --git a/mkimage.sh b/mkimage.sh index 4397f030f..61674a2fb 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -150,6 +150,21 @@ write_authenticode_hash() { fi } +ensure_dstack_mr() { + DSTACK_SRC="${DSTACK_SRC:-$SCRIPT_DIR/dstack}" + if [ -z "${DSTACK_MR_BIN:-}" ]; then + for c in "$SCRIPT_DIR/dstack-mr" "$SCRIPT_DIR/rust-target/release/dstack-mr" \ + "$DSTACK_SRC/target/release/dstack-mr"; do + [ -x "$c" ] && DSTACK_MR_BIN="$c" && break + done + fi + if [ -z "${DSTACK_MR_BIN:-}" ]; then + echo "Building dstack-mr to compute OS-image measurement material" + ( cd "$DSTACK_SRC" && cargo build --release -p dstack-mr ) + DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" + fi +} + create_partitioned_rootfs() { local rootfs_img="$1" local output_img="$2" @@ -308,35 +323,16 @@ cat < ${OUTPUT_DIR}/metadata.json } EOF -echo "Generating image digest to ${OUTPUT_DIR}/" -pushd ${OUTPUT_DIR}/ -sha256sum ovmf.fd bzImage initramfs.cpio.gz metadata.json > sha256sum.txt -sha256sum sha256sum.txt | awk '{print $1}' > digest.txt -popd - -# digest.sev.txt: the AMD SEV-SNP os_image_hash. Unlike the TDX digest.txt -# (a content hash that includes the TDX firmware), this is computed by the -# `dstack-mr` tool from the SEV firmware (ovmf-sev.fd) + kernel/initrd/cmdline/ -# rootfs and matches the os_image_hash the KMS verifier derives from a launch -# measurement. The VMM reads this file at deploy time instead of recomputing it, -# so it is required (not best-effort): if `dstack-mr` is not prebuilt, build it. -HAVE_DIGEST_SEV=0 +ensure_dstack_mr + +echo "Generating measurement.tdx.cbor via ${DSTACK_MR_BIN}" +"${DSTACK_MR_BIN}" tdx-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.tdx.cbor" + +HAVE_MEASUREMENT_SNP=0 if [ "$HAVE_OVMF_SEV" = "1" ]; then - DSTACK_SRC="${DSTACK_SRC:-$SCRIPT_DIR/dstack}" - if [ -z "${DSTACK_MR_BIN:-}" ]; then - for c in "$SCRIPT_DIR/dstack-mr" "$SCRIPT_DIR/rust-target/release/dstack-mr" \ - "$DSTACK_SRC/target/release/dstack-mr"; do - [ -x "$c" ] && DSTACK_MR_BIN="$c" && break - done - fi - if [ -z "${DSTACK_MR_BIN:-}" ]; then - echo "Building dstack-mr to compute digest.sev.txt" - ( cd "$DSTACK_SRC" && cargo build --release -p dstack-mr ) - DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" - fi - echo "Generating digest.sev.txt via ${DSTACK_MR_BIN}" - "${DSTACK_MR_BIN}" sev-os-image-hash "${OUTPUT_DIR}" > "${OUTPUT_DIR}/digest.sev.txt" - HAVE_DIGEST_SEV=1 + echo "Generating measurement.snp.cbor via ${DSTACK_MR_BIN}" + "${DSTACK_MR_BIN}" snp-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.snp.cbor" + HAVE_MEASUREMENT_SNP=1 fi # Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR @@ -359,6 +355,37 @@ if [ "$ENABLE_UKI_IMAGE" = "1" ]; then fi fi +HAVE_MEASUREMENT_GCP=0 +if [[ "$UKI_CREATED" = "1" ]]; then + if [[ ! -f "${OUTPUT_DIR}/auth_hash.txt" ]]; then + echo "Error: UKI image was created but auth_hash.txt is missing" >&2 + exit 1 + fi + echo "Generating measurement.gcp.cbor via ${DSTACK_MR_BIN}" + "${DSTACK_MR_BIN}" gcp-measurement-cbor "${OUTPUT_DIR}/auth_hash.txt" > "${OUTPUT_DIR}/measurement.gcp.cbor" + HAVE_MEASUREMENT_GCP=1 +fi + +echo "Generating unified image digest to ${OUTPUT_DIR}/" +CHECKSUM_FILES="ovmf.fd bzImage initramfs.cpio.gz metadata.json measurement.tdx.cbor" +if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then + CHECKSUM_FILES="$CHECKSUM_FILES measurement.snp.cbor" +fi +if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then + CHECKSUM_FILES="$CHECKSUM_FILES measurement.gcp.cbor" +fi +( + cd "${OUTPUT_DIR}/" + sha256sum $CHECKSUM_FILES > sha256sum.txt + sha256sum sha256sum.txt | awk '{print $1}' > digest.txt +) + +# Keep the legacy file name for AMD deployment tooling, but the content is now +# the same unified OS image hash used by TDX and GCP. +if [ "$HAVE_OVMF_SEV" = "1" ]; then + cp "${OUTPUT_DIR}/digest.txt" "${OUTPUT_DIR}/digest.sev.txt" +fi + if [ x$DSTACK_TAR_RELEASE = x1 ]; then OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) PARENT_DIR=$(dirname ${OUTPUT_DIR}) @@ -366,21 +393,27 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then # Bare metal tarball: all files except disk.raw and auth_hash.txt rm -rf ${IMAGE_TAR} echo "Archiving bare metal image to ${IMAGE_TAR}" - BARE_METAL_FILES="rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json" + BARE_METAL_FILES="rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json measurement.tdx.cbor" if [ "$HAVE_OVMF_SEV" = "1" ]; then BARE_METAL_FILES="$BARE_METAL_FILES ovmf-sev.fd" fi - if [ "$HAVE_DIGEST_SEV" = "1" ]; then + if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then + BARE_METAL_FILES="$BARE_METAL_FILES measurement.snp.cbor" + fi + if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then + BARE_METAL_FILES="$BARE_METAL_FILES measurement.gcp.cbor" + fi + if [ "$HAVE_OVMF_SEV" = "1" ]; then BARE_METAL_FILES="$BARE_METAL_FILES digest.sev.txt" fi (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo - # UKI tarball: only disk.raw and auth_hash.txt + # UKI tarball: GCP boot disk plus the unified OS-image identity material. if [[ "$UKI_CREATED" = "1" ]]; then rm -rf ${IMAGE_TAR_UKI} echo "Archiving UKI image to ${IMAGE_TAR_UKI}" - UKI_FILES="disk.raw auth_hash.txt" + UKI_FILES="disk.raw digest.txt sha256sum.txt measurement.gcp.cbor" (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR_UKI} $(for f in $UKI_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo fi diff --git a/repro-build/check.sh b/repro-build/check.sh index b8460b6e8..68cde6570 100755 --- a/repro-build/check.sh +++ b/repro-build/check.sh @@ -30,9 +30,13 @@ COMPARE_IMAGE_WHITELIST=( "digest.txt" "initramfs.cpio.gz" "metadata.json" + "measurement.gcp.cbor" + "measurement.snp.cbor" + "measurement.tdx.cbor" "ovmf.fd" "rootfs.img.parted.verity" "sha256sum.txt" + "auth_hash.txt" "gcp/efi-root/EFI/BOOT/BOOTX64.EFI" ) From 5d5ad789dab760b1ab825b305c6753858102c1f0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 8 Jul 2026 18:13:33 -0700 Subject: [PATCH 414/435] mkimage: stop publishing digest.sev.txt --- dstack | 2 +- mkimage.sh | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/dstack b/dstack index a2fbb67b0..352736b47 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit a2fbb67b020eb4e9bbef15318770590cab3a2cf6 +Subproject commit 352736b475f2449a08b9ac01613b613c11b4a4df diff --git a/mkimage.sh b/mkimage.sh index 61674a2fb..2cfbe1b9a 100755 --- a/mkimage.sh +++ b/mkimage.sh @@ -268,10 +268,11 @@ $Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ # AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP # launch can select it via the metadata.json "bios-sev" field below. The SEV -# firmware blob itself is NOT added to sha256sum.txt, but metadata.json (which -# references it) is, so digest.txt does reflect its presence. That does not -# change any TDX hardware measurement (MRTD comes from ovmf.fd, RTMRs from -# kernel/cmdline/rootfs) -- it only changes dstack's image-bundle digest. +# firmware blob itself is NOT added directly to sha256sum.txt; when present, its +# OVMF hash/sections are committed by measurement.snp.cbor, and that file is +# part of digest.txt. This does not change any TDX hardware +# measurement (MRTD comes from ovmf.fd, RTMRs from kernel/cmdline/rootfs) -- it +# only changes dstack's image-bundle digest. OVMF_SEV_FIRMWARE=${COMMON_IMG_DIR}/ovmf-sev.fd HAVE_OVMF_SEV=0 BIOS_SEV_JSON="" @@ -380,12 +381,6 @@ fi sha256sum sha256sum.txt | awk '{print $1}' > digest.txt ) -# Keep the legacy file name for AMD deployment tooling, but the content is now -# the same unified OS image hash used by TDX and GCP. -if [ "$HAVE_OVMF_SEV" = "1" ]; then - cp "${OUTPUT_DIR}/digest.txt" "${OUTPUT_DIR}/digest.sev.txt" -fi - if [ x$DSTACK_TAR_RELEASE = x1 ]; then OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) PARENT_DIR=$(dirname ${OUTPUT_DIR}) @@ -403,9 +398,6 @@ if [ x$DSTACK_TAR_RELEASE = x1 ]; then if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then BARE_METAL_FILES="$BARE_METAL_FILES measurement.gcp.cbor" fi - if [ "$HAVE_OVMF_SEV" = "1" ]; then - BARE_METAL_FILES="$BARE_METAL_FILES digest.sev.txt" - fi (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) echo From e373ba944b478b0a864c55e6252b8bc0d3d0a14f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 8 Jul 2026 19:24:15 -0700 Subject: [PATCH 415/435] Bump dstack to e5f138c7 --- dstack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack b/dstack index 352736b47..e5f138c73 160000 --- a/dstack +++ b/dstack @@ -1 +1 @@ -Subproject commit 352736b475f2449a08b9ac01613b613c11b4a4df +Subproject commit e5f138c73daa5755ef41d826bc57d738efdd3895 From 8ef9a5586079beef97416d21995dc4a207b56b80 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 9 Jul 2026 07:56:17 -0700 Subject: [PATCH 416/435] meta-nvidia: gate GPU ready state on local TEE attestation (nvattest) Add an nvattest recipe building NVIDIA's attestation-sdk CLI (pinned SRCREV, Apache-2.0). GPU TEE attestation runs inside `dstack-util setup` (dstack-prepare.service), driven by the app-compose requirement `requirements.verify_gpu` (default true): the conf-compute ready state is set only after nvattest succeeds (previously nvidia-persistenced set it unconditionally), and attestation failure -- including a non-CC GPU or CC mode left off by the host -- aborts boot before key provisioning. Deployments can opt out with `requirements.verify_gpu: false`, which sets the ready state without verification. The recipe ships only the CLI plus a dstack-prepare.service ordering drop-in (After=nvidia-persistenced/nvidia-fabricmanager); there is no dedicated attestation service or unit gating. GPU-less instances are unaffected (dstack-util skips when no NVIDIA GPU is on the PCI bus). Build notes: - regorus (Rego engine, Rust) is embedded via Corrosion; use the prebuilt meta-rust-bin toolchain (cargo-bin-native), with host/target linker split for cargo build scripts (host triple == target triple) - USE_SYSTEM_DEPS=ON: link sysroot openssl/curl/libxml2/xmlsec1 instead of ExternalProject-downloaded copies - RUSTFLAGS remap-path-prefix keeps build paths out of libnvat ([buildpaths] QA / reproducible builds) - only nvattest + libnvat are shipped; cargo proc-macro dylibs are excluded from the image --- .../images/dstack-rootfs-nvidia.inc | 1 + .../files/10-nvidia-gpu-ordering.conf | 8 ++ .../nvattest/nvattest_2026.06.09.bb | 113 ++++++++++++++++++ .../nvidia/files/nvidia-persistenced.service | 5 +- 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf create mode 100644 meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc index 9e6cf6037..704ccaed0 100644 --- a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc +++ b/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc @@ -5,6 +5,7 @@ NVIDIA_GROUP = "acpid \ nvidia-persistenced \ nvidia-fabricmanager \ libnvidia-nscq \ + nvattest \ nvidia-container-toolkit \ kernel-module-video \ numactl \ diff --git a/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf b/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf new file mode 100644 index 000000000..50e0b7d1f --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf @@ -0,0 +1,8 @@ +# Ordering only, not a hard dependency: on GPU instances, `dstack-util setup` +# (run by dstack-prepare.service) performs local GPU TEE attestation per the +# app-compose `requirements.verify_gpu` policy, which needs the nvidia driver +# loaded (nvidia-persistenced) and the NVLink fabric up (nvidia-fabricmanager). +# Both units skip cleanly on GPU-less instances via nvidia-gpu-detect. +[Unit] +Wants=nvidia-persistenced.service nvidia-fabricmanager.service +After=nvidia-persistenced.service nvidia-fabricmanager.service diff --git a/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb b/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb new file mode 100644 index 000000000..7d907d69d --- /dev/null +++ b/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb @@ -0,0 +1,113 @@ +SUMMARY = "NVIDIA GPU attestation CLI" +DESCRIPTION = "Builds NVIDIA's nvattest CLI. dstack-util setup runs it at boot to gate readiness on local GPU TEE attestation (app-compose requirements.verify_gpu)." +HOMEPAGE = "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/NVIDIA/attestation-sdk" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=e620fc90e76c4aa0c3efdd1673ca0b3b" + +SRC_URI = " \ + git://github.com/NVIDIA/attestation-sdk.git;protocol=https;branch=main \ + file://10-nvidia-gpu-ordering.conf \ +" +SRCREV = "9d12801cea8a198ea0f29640dfaf8a4017c841c5" + +OECMAKE_SOURCEPATH = "${S}/nv-attestation-cli" + +inherit cmake pkgconfig + +# The SDK embeds the regorus Rego policy engine (Rust, via Corrosion), so a +# Rust toolchain is required. Use the prebuilt toolchain from meta-rust-bin +# (cargo-bin-native pulls rust-bin-cross-${TARGET_ARCH}), the same toolchain +# dstack-guest builds with -- NOT oe-core's rust-native (bootstraps rustc from +# source, very expensive). +DEPENDS += " \ + cargo-bin-native \ + curl \ + openssl \ + libxml2 \ + xmlsec1 \ + spdlog \ + nlohmann-json \ + nvidia \ +" + +RDEPENDS:${PN} += " \ + ca-certificates \ + nvidia \ + nvidia-fabricmanager \ + nvidia-persistenced \ +" + +EXTRA_OECMAKE += " \ + -DBUILD_TESTING=OFF \ + -DNVAT_BUILD_TESTS=OFF \ + -DNVAT_BUILD_SAMPLES=OFF \ + -DCMAKE_SKIP_RPATH=ON \ + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF \ + -DUSE_SYSTEM_DEPS=ON \ +" + +# Keep cargo state inside the workdir (Corrosion invokes cargo for regorus). +export CARGO_HOME = "${WORKDIR}/cargo_home" +export RUST_BACKTRACE = "1" + +# rustc does not inherit the C toolchain's -ffile-prefix-map, so the regorus +# static lib embedded in libnvat would otherwise carry TMPDIR paths +# ([buildpaths] QA / reproducibility issue). +export RUSTFLAGS = "--remap-path-prefix=${WORKDIR}=/usr/src/debug/${PN}/${PV}" + +# Corrosion invokes cargo with host triple == target triple +# (x86_64-unknown-linux-gnu). Cargo would then link host-side build scripts +# with the cross gcc but without the target sysroot (cannot find Scrt1.o/-lc). +# Split host vs target linker config the same way meta-rust-bin's cargo_bin +# class does: build scripts link with the native toolchain. +do_compile:prepend() { + mkdir -p ${WORKDIR}/wrappers + echo "#!/bin/sh" > ${WORKDIR}/wrappers/linker-native-wrapper.sh + echo "${BUILD_CC} ${BUILD_LDFLAGS} \"\$@\"" >> ${WORKDIR}/wrappers/linker-native-wrapper.sh + chmod +x ${WORKDIR}/wrappers/linker-native-wrapper.sh + + export __CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS="nightly" + export CARGO_UNSTABLE_TARGET_APPLIES_TO_HOST="true" + export CARGO_UNSTABLE_HOST_CONFIG="true" + export CARGO_TARGET_APPLIES_TO_HOST="false" + export CARGO_HOST_LINKER="${WORKDIR}/wrappers/linker-native-wrapper.sh" +} + +# Network is needed at configure/compile time because: +# - nv-attestation-cli CMake FetchContent: CLI11, nlohmann-json (+ fmt/spdlog +# headers) +# - nv-attestation-sdk-cpp CMake FetchContent: Corrosion, regorus, jwt-cpp +# - Corrosion runs cargo, which fetches the regorus-ffi crate dependencies +# All refs are pinned (git tags/commits) upstream. TODO: vendor these via +# SRC_URI + cargo vendor for a fully offline, reproducible fetch. +do_configure[network] = "1" +do_compile[network] = "1" + +do_install() { + DESTDIR=${D} cmake --install ${B} --prefix ${prefix} + + if [ ! -x ${D}${bindir}/nvattest ]; then + bbfatal "nvattest binary was not produced by the build" + fi + + # cargo/Corrosion leaves host-side proc-macro dylibs in the build tree; + # make sure none of them ever end up in the image (only libnvat is a + # real target library). + find ${D}${libdir} -maxdepth 1 -name 'lib*.so*' ! -name 'libnvat.so*' -delete + + # Make dstack-prepare (which runs the attestation) start after the nvidia + # userspace services it depends on. + install -d ${D}${systemd_system_unitdir}/dstack-prepare.service.d + install -m 0644 ${UNPACKDIR}/10-nvidia-gpu-ordering.conf \ + ${D}${systemd_system_unitdir}/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf +} + +FILES:${PN} += " \ + ${systemd_system_unitdir}/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf \ + ${libdir}/lib*.so \ + ${libdir}/lib*.so.* \ +" +FILES_SOLIBSDEV = "" +FILES:${PN}-dev:remove = "${libdir}/lib*.so" + +INSANE_SKIP:${PN} += "dev-so already-stripped" diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service index 7630c0838..f40aef461 100644 --- a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service +++ b/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service @@ -16,7 +16,10 @@ ExecStartPre=/usr/bin/env modprobe nvidia-modeset ExecStartPre=/usr/bin/env modprobe nvidia-drm ExecStartPre=/usr/bin/env modprobe nvidia-uvm ExecStart=/usr/bin/env nvidia-persistenced --uvm-persistence-mode -ExecStartPost=/usr/bin/nvidia-smi conf-compute -srs 1 +# NOTE: the GPU ready state (conf-compute -srs 1) is intentionally NOT set +# here. It is set by `dstack-util setup` (dstack-prepare.service), only after +# local GPU TEE attestation via nvattest succeeds -- unless the deployment +# opts out with app-compose requirements.verify_gpu=false. RemainAfterExit=yes [Install] From 72d9f0669543e21f8936d2fc640da07dccc577ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 00:45:14 -0700 Subject: [PATCH 417/435] refactor: group core sources under dstack --- .gitmodules | 6 +++--- Cargo.lock => dstack/Cargo.lock | 0 Cargo.toml => dstack/Cargo.toml | 0 Makefile => dstack/Makefile | 0 {basefiles => dstack/basefiles}/app-compose.service | 0 {basefiles => dstack/basefiles}/app-compose.sh | 0 .../containerd.service.d/dstack-prepare.conf | 0 .../docker.service.d/dstack-guest-agent.conf | 0 .../basefiles}/docker.service.d/dstack-prepare.conf | 0 .../basefiles}/dstack-guest-agent.service | 0 .../basefiles}/dstack-guest-agent.socket | 0 .../basefiles}/dstack-prepare.service | 0 {basefiles => dstack/basefiles}/dstack-prepare.sh | 0 {basefiles => dstack/basefiles}/ephemeral-docker.sh | 0 {basefiles => dstack/basefiles}/journald.conf | 0 {basefiles => dstack/basefiles}/llmnr.conf | 0 .../basefiles}/sysctl.d/99-dstack.conf | 0 {basefiles => dstack/basefiles}/tdx-attest.conf | 0 {basefiles => dstack/basefiles}/wg-checker.service | 0 {basefiles => dstack/basefiles}/wg-checker.sh | 0 {build => dstack/build}/shared/build-lib.sh | 0 {build => dstack/build}/shared/config-qemu.sh | 0 {build => dstack/build}/shared/pin-packages.sh | 0 .../build}/shared/verify-pinned-packages.sh | 0 {cached-cell => dstack/cached-cell}/Cargo.toml | 0 {cached-cell => dstack/cached-cell}/src/lib.rs | 0 cargo-check-all.sh => dstack/cargo-check-all.sh | 0 {cc-eventlog => dstack/cc-eventlog}/Cargo.toml | 0 .../cc-eventlog}/samples/ccel.bin | Bin .../cc-eventlog}/samples/tpm_eventlog.bin | Bin {cc-eventlog => dstack/cc-eventlog}/src/codecs.rs | 0 {cc-eventlog => dstack/cc-eventlog}/src/lib.rs | 0 .../cc-eventlog}/src/runtime_events.rs | 0 .../snapshots/cc_eventlog__tests__parse_ccel-2.snap | 0 .../snapshots/cc_eventlog__tests__parse_ccel.snap | 0 {cc-eventlog => dstack/cc-eventlog}/src/tcg.rs | 0 {cc-eventlog => dstack/cc-eventlog}/src/tdx.rs | 0 {cc-eventlog => dstack/cc-eventlog}/src/tpm.rs | 0 {cert-client => dstack/cert-client}/Cargo.toml | 0 {cert-client => dstack/cert-client}/src/lib.rs | 0 {certbot => dstack/certbot}/.gitignore | 0 {certbot => dstack/certbot}/Cargo.toml | 0 {certbot => dstack/certbot}/cli/Cargo.toml | 0 {certbot => dstack/certbot}/cli/src/main.rs | 0 {certbot => dstack/certbot}/src/acme_client.rs | 0 .../certbot}/src/acme_client/tests.rs | 0 {certbot => dstack/certbot}/src/bot.rs | 0 {certbot => dstack/certbot}/src/bot/tests.rs | 0 {certbot => dstack/certbot}/src/dns01_client.rs | 0 .../certbot}/src/dns01_client/cloudflare.rs | 0 {certbot => dstack/certbot}/src/http_client.rs | 0 {certbot => dstack/certbot}/src/lib.rs | 0 {certbot => dstack/certbot}/src/workdir.rs | 0 {crates => dstack/crates}/dstack-auth/Cargo.toml | 0 {crates => dstack/crates}/dstack-auth/src/main.rs | 0 .../crates}/dstack-cli-core/Cargo.toml | 0 .../crates}/dstack-cli-core/src/compose.rs | 0 .../crates}/dstack-cli-core/src/config.rs | 0 .../crates}/dstack-cli-core/src/fsutil.rs | 0 .../crates}/dstack-cli-core/src/host.rs | 0 .../crates}/dstack-cli-core/src/layout.rs | 0 .../crates}/dstack-cli-core/src/lib.rs | 0 .../crates}/dstack-cli-core/src/ports.rs | 0 .../crates}/dstack-cli-core/src/vmm.rs | 0 {crates => dstack/crates}/dstack-cli/Cargo.toml | 0 {crates => dstack/crates}/dstack-cli/src/main.rs | 0 {crates => dstack/crates}/dstackup/Cargo.toml | 0 {crates => dstack/crates}/dstackup/src/cid.rs | 0 {crates => dstack/crates}/dstackup/src/cli.rs | 0 {crates => dstack/crates}/dstackup/src/destroy.rs | 0 {crates => dstack/crates}/dstackup/src/image.rs | 0 {crates => dstack/crates}/dstackup/src/install.rs | 0 {crates => dstack/crates}/dstackup/src/main.rs | 0 {crates => dstack/crates}/dstackup/src/state.rs | 0 {crates => dstack/crates}/dstackup/src/systemd.rs | 0 {ct_monitor => dstack/ct_monitor}/Cargo.toml | 0 {ct_monitor => dstack/ct_monitor}/src/main.rs | 0 {dstack-attest => dstack/dstack-attest}/Cargo.toml | 0 .../dstack-attest}/src/amd_sev_snp.rs | 0 .../dstack-attest}/src/attestation.rs | 0 {dstack-attest => dstack/dstack-attest}/src/lib.rs | 0 .../dstack-attest}/src/sev_snp.rs | 0 {dstack-attest => dstack/dstack-attest}/src/v1.rs | 0 .../dstack-attest}/tests/nitro_attestation.bin | Bin .../dstack-attest}/tests/nitro_attestation_dbg.bin | Bin .../dstack-attest}/tests/nitro_verify.rs | 0 .../dstack-attest}/tests/sev_snp_ask.pem | 0 .../dstack-attest}/tests/sev_snp_attestation.bin | Bin .../dstack-attest}/tests/sev_snp_fixture.README.md | 0 .../dstack-attest}/tests/sev_snp_vcek.pem | 0 .../dstack-attest}/tests/sev_snp_verify.rs | 0 .../tests/snapshots/nitro_verify__app_info.snap | 0 .../tests/snapshots/nitro_verify__nitro_report.snap | 0 {dstack-mr => dstack/dstack-mr}/.gitignore | 0 {dstack-mr => dstack/dstack-mr}/Cargo.toml | 0 {dstack-mr => dstack/dstack-mr}/cli/Cargo.toml | 0 {dstack-mr => dstack/dstack-mr}/cli/src/main.rs | 0 {dstack-mr => dstack/dstack-mr}/src/acpi.rs | 0 {dstack-mr => dstack/dstack-mr}/src/kernel.rs | 0 {dstack-mr => dstack/dstack-mr}/src/lib.rs | 0 {dstack-mr => dstack/dstack-mr}/src/machine.rs | 0 {dstack-mr => dstack/dstack-mr}/src/main.rs | 0 {dstack-mr => dstack/dstack-mr}/src/measurement.rs | 0 {dstack-mr => dstack/dstack-mr}/src/num.rs | 0 {dstack-mr => dstack/dstack-mr}/src/sev.rs | 0 {dstack-mr => dstack/dstack-mr}/src/tdvf.rs | 0 {dstack-mr => dstack/dstack-mr}/src/tdx.rs | 0 {dstack-mr => dstack/dstack-mr}/src/util.rs | 0 {dstack-mr => dstack/dstack-mr}/tests/tdvf_parse.rs | 0 {dstack-types => dstack/dstack-types}/Cargo.toml | 0 {dstack-types => dstack/dstack-types}/src/lib.rs | 0 .../dstack-types}/src/mr_config.rs | 0 .../dstack-types}/src/shared_filenames.rs | 0 .../dstack-types}/src/version.rs | 0 {dstack-util => dstack/dstack-util}/Cargo.toml | 0 {dstack-util => dstack/dstack-util}/src/crypto.rs | 0 .../dstack-util}/src/docker_compose.rs | 0 {dstack-util => dstack/dstack-util}/src/host_api.rs | 0 {dstack-util => dstack/dstack-util}/src/main.rs | 0 .../dstack-util}/src/parse_env_file.rs | 0 .../dstack-util}/src/system_setup.rs | 0 .../src/system_setup/config_id_verifier.rs | 0 {dstack-util => dstack/dstack-util}/src/utils.rs | 0 .../tests/fixtures/key-provider-docker-compose.yaml | 0 .../tests/fixtures/luks_header_cipher_null | Bin .../tests/fixtures/luks_header_cipher_null.license | 0 .../dstack-util}/tests/fixtures/luks_header_good | Bin .../tests/fixtures/luks_header_good.license | 0 .../dstack-util}/tests/test_remove_orphans.sh | 0 {gateway => dstack/gateway}/Cargo.toml | 0 {gateway => dstack/gateway}/assets/cert.key | 0 {gateway => dstack/gateway}/assets/cert.pem | 0 .../gateway}/docs/cluster-deployment.md | 0 {gateway => dstack/gateway}/dstack-app/.gitignore | 0 .../gateway}/dstack-app/bootstrap-cluster.sh | 0 .../gateway}/dstack-app/builder/Dockerfile | 0 .../gateway}/dstack-app/builder/README.md | 0 .../gateway}/dstack-app/builder/build-image.sh | 0 .../gateway}/dstack-app/builder/entrypoint.sh | 0 .../builder/shared/builder-pinned-packages.txt | 0 .../dstack-app/builder/shared/pinned-packages.txt | 0 .../gateway}/dstack-app/deploy-to-vmm.sh | 0 .../gateway}/dstack-app/docker-compose.yaml | 0 {gateway => dstack/gateway}/gateway.toml | 0 {gateway => dstack/gateway}/rpc/Cargo.toml | 0 {gateway => dstack/gateway}/rpc/build.rs | 0 .../gateway}/rpc/proto/gateway_rpc.proto | 0 {gateway => dstack/gateway}/rpc/src/generated.rs | 0 {gateway => dstack/gateway}/rpc/src/lib.rs | 0 {gateway => dstack/gateway}/src/admin_auth.rs | 0 {gateway => dstack/gateway}/src/admin_service.rs | 0 {gateway => dstack/gateway}/src/cert_store.rs | 0 {gateway => dstack/gateway}/src/config.rs | 0 {gateway => dstack/gateway}/src/debug_service.rs | 0 .../gateway}/src/distributed_certbot.rs | 0 {gateway => dstack/gateway}/src/gen_debug_key.rs | 0 {gateway => dstack/gateway}/src/kv/https_client.rs | 0 {gateway => dstack/gateway}/src/kv/mod.rs | 0 {gateway => dstack/gateway}/src/kv/sync_service.rs | 0 {gateway => dstack/gateway}/src/main.rs | 0 {gateway => dstack/gateway}/src/main_service.rs | 0 .../gateway}/src/main_service/auth_client.rs | 0 .../gateway}/src/main_service/handshakes.rs | 0 ...tack_gateway__main_service__tests__config-2.snap | 0 ...tack_gateway__main_service__tests__config-3.snap | 0 ...dstack_gateway__main_service__tests__config.snap | 0 ..._gateway__main_service__tests__empty_config.snap | 0 .../gateway}/src/main_service/tests.rs | 0 {gateway => dstack/gateway}/src/models.rs | 0 {gateway => dstack/gateway}/src/pp.rs | 0 {gateway => dstack/gateway}/src/proxy.rs | 0 {gateway => dstack/gateway}/src/proxy/io_bridge.rs | 0 .../gateway}/src/proxy/port_policy.rs | 0 {gateway => dstack/gateway}/src/proxy/sni.rs | 0 .../gateway}/src/proxy/tls_passthough.rs | 0 .../gateway}/src/proxy/tls_terminate.rs | 0 {gateway => dstack/gateway}/src/web_routes.rs | 0 .../gateway}/src/web_routes/route_index.rs | 0 .../gateway}/src/web_routes/wavekv_sync.rs | 0 .../gateway}/templates/dashboard.html | 0 {gateway => dstack/gateway}/templates/rproxy.yaml | 0 {gateway => dstack/gateway}/templates/wg.conf | 0 {gateway => dstack/gateway}/test-run/.env.example | 0 {gateway => dstack/gateway}/test-run/.gitignore | 0 {gateway => dstack/gateway}/test-run/TESTING.md | 0 {gateway => dstack/gateway}/test-run/cluster.sh | 0 .../gateway}/test-run/e2e/configs/gateway-1.toml | 0 .../gateway}/test-run/e2e/configs/gateway-2.toml | 0 .../gateway}/test-run/e2e/configs/gateway-3.toml | 0 .../gateway}/test-run/e2e/docker-compose.yml | 0 .../gateway}/test-run/e2e/pebble-config.json | 0 {gateway => dstack/gateway}/test-run/e2e/run-e2e.sh | 0 {gateway => dstack/gateway}/test-run/e2e/test.sh | 0 .../gateway}/test-run/test_certbot.sh | 0 {gateway => dstack/gateway}/test-run/test_suite.sh | 0 .../guest-agent-simulator}/.gitignore | 0 .../guest-agent-simulator}/Cargo.toml | 0 .../guest-agent-simulator}/dstack-simulator.service | 0 .../guest-agent-simulator}/dstack.toml | 0 .../guest-agent-simulator}/install-systemd.sh | 0 .../guest-agent-simulator}/package-release.sh | 0 .../guest-agent-simulator}/src/main.rs | 0 .../guest-agent-simulator}/src/simulator.rs | 0 {guest-agent => dstack/guest-agent}/Cargo.toml | 0 {guest-agent => dstack/guest-agent}/dstack.toml | 0 .../guest-agent}/fixtures/attestation.bin | Bin {guest-agent => dstack/guest-agent}/rpc/Cargo.toml | 0 {guest-agent => dstack/guest-agent}/rpc/build.rs | 0 .../guest-agent}/rpc/proto/agent_rpc.proto | 0 .../guest-agent}/rpc/src/generated.rs | 0 {guest-agent => dstack/guest-agent}/rpc/src/lib.rs | 0 {guest-agent => dstack/guest-agent}/src/backend.rs | 0 {guest-agent => dstack/guest-agent}/src/config.rs | 0 .../guest-agent}/src/guest_api_service.rs | 0 .../guest-agent}/src/http_routes.rs | 0 {guest-agent => dstack/guest-agent}/src/lib.rs | 0 {guest-agent => dstack/guest-agent}/src/main.rs | 0 {guest-agent => dstack/guest-agent}/src/models.rs | 0 .../guest-agent}/src/rpc_service.rs | 0 {guest-agent => dstack/guest-agent}/src/server.rs | 0 .../guest-agent}/src/socket_activation.rs | 0 .../guest-agent}/templates/dashboard.html | 0 .../guest-agent}/templates/metrics.tpl | 0 {guest-api => dstack/guest-api}/Cargo.toml | 0 {guest-api => dstack/guest-api}/build.rs | 0 .../guest-api}/proto/guest_api.proto | 0 {guest-api => dstack/guest-api}/src/client.rs | 0 .../guest-api}/src/generated/mod.rs | 0 {guest-api => dstack/guest-api}/src/lib.rs | 0 {host-api => dstack/host-api}/Cargo.toml | 0 {host-api => dstack/host-api}/build.rs | 0 {host-api => dstack/host-api}/proto/host_api.proto | 0 {host-api => dstack/host-api}/src/client.rs | 0 {host-api => dstack/host-api}/src/generated/mod.rs | 0 {host-api => dstack/host-api}/src/lib.rs | 0 {http-client => dstack/http-client}/Cargo.toml | 0 .../http-client}/src/hyper_vsock.rs | 0 {http-client => dstack/http-client}/src/lib.rs | 0 {http-client => dstack/http-client}/src/prpc.rs | 0 {iohash => dstack/iohash}/Cargo.toml | 0 {iohash => dstack/iohash}/src/main.rs | 0 .../key-provider-build}/Cargo.lock | 0 .../key-provider-build}/Dockerfile.aesmd | 0 .../key-provider-build}/Dockerfile.key-provider | 0 .../key-provider-build}/docker-compose.yaml | 0 .../key-provider-build}/entrypoint-aesmd.sh | 0 .../key-provider-build}/entrypoint-key-provider.sh | 0 .../key-provider-build}/run.sh | 0 .../key-provider-build}/sgx_default_qcnl.conf | 0 .../key-provider-client}/Cargo.toml | 0 .../key-provider-client}/src/host.rs | 0 .../key-provider-client}/src/lib.rs | 0 {kms => dstack/kms}/Cargo.toml | 0 {kms => dstack/kms}/README.md | 0 {kms => dstack/kms}/auth-eth-bun/.oxlintrc.json | 0 {kms => dstack/kms}/auth-eth-bun/README.md | 0 {kms => dstack/kms}/auth-eth-bun/bun.lock | 0 {kms => dstack/kms}/auth-eth-bun/index.test.ts | 0 {kms => dstack/kms}/auth-eth-bun/index.ts | 0 {kms => dstack/kms}/auth-eth-bun/openapi.json | 0 {kms => dstack/kms}/auth-eth-bun/package.json | 0 {kms => dstack/kms}/auth-eth-bun/vitest.config.ts | 0 {kms => dstack/kms}/auth-eth/.env.example | 0 {kms => dstack/kms}/auth-eth/.gitignore | 0 .../kms}/auth-eth/.openzeppelin/unknown-2035.json | 0 {kms => dstack/kms}/auth-eth/README.md | 0 {kms => dstack/kms}/auth-eth/TESTING.md | 0 .../kms}/auth-eth/contracts/DstackApp.sol | 0 .../kms}/auth-eth/contracts/DstackKms.sol | 0 {kms => dstack/kms}/auth-eth/contracts/IAppAuth.sol | 0 .../auth-eth/contracts/IAppAuthBasicManagement.sol | 0 .../auth-eth/contracts/test-utils/DstackAppV2.sol | 0 .../auth-eth/contracts/test-utils/DstackKmsV2.sol | 0 .../kms}/auth-eth/docs/formal-verification.md | 0 {kms => dstack/kms}/auth-eth/docs/specification.md | 0 {kms => dstack/kms}/auth-eth/foundry.toml | 0 {kms => dstack/kms}/auth-eth/jest.config.js | 0 {kms => dstack/kms}/auth-eth/lib/forge-std | 0 .../auth-eth/lib/openzeppelin-contracts-upgradeable | 0 .../kms}/auth-eth/lib/openzeppelin-foundry-upgrades | 0 {kms => dstack/kms}/auth-eth/package-lock.json | 0 {kms => dstack/kms}/auth-eth/package.json | 0 {kms => dstack/kms}/auth-eth/run-tests.sh | 0 {kms => dstack/kms}/auth-eth/script/Deploy.s.sol | 0 {kms => dstack/kms}/auth-eth/script/Manage.s.sol | 0 {kms => dstack/kms}/auth-eth/script/Query.s.sol | 0 {kms => dstack/kms}/auth-eth/script/README.md | 0 {kms => dstack/kms}/auth-eth/script/Upgrade.s.sol | 0 {kms => dstack/kms}/auth-eth/scripts/README.md | 0 {kms => dstack/kms}/auth-eth/scripts/cleanup.sh | 0 {kms => dstack/kms}/auth-eth/scripts/run-tests.sh | 0 .../kms}/auth-eth/scripts/setup-local-chain.sh | 0 {kms => dstack/kms}/auth-eth/scripts/test-all.sh | 0 {kms => dstack/kms}/auth-eth/slither.config.json | 0 {kms => dstack/kms}/auth-eth/src/ethereum.ts | 0 {kms => dstack/kms}/auth-eth/src/main.test.ts | 0 {kms => dstack/kms}/auth-eth/src/main.ts | 0 {kms => dstack/kms}/auth-eth/src/server.ts | 0 {kms => dstack/kms}/auth-eth/src/types.ts | 0 .../kms}/auth-eth/test/DstackApp.symbolic.t.sol | 0 {kms => dstack/kms}/auth-eth/test/DstackApp.t.sol | 0 .../kms}/auth-eth/test/DstackKms.symbolic.t.sol | 0 {kms => dstack/kms}/auth-eth/test/DstackKms.t.sol | 0 .../kms}/auth-eth/test/UpgradesWithPlugin.t.sol | 0 {kms => dstack/kms}/auth-eth/tsconfig.json | 0 {kms => dstack/kms}/auth-mock/.oxlintrc.json | 0 {kms => dstack/kms}/auth-mock/Dockerfile | 0 {kms => dstack/kms}/auth-mock/README.md | 0 {kms => dstack/kms}/auth-mock/bun.lock | 0 {kms => dstack/kms}/auth-mock/index.test.ts | 0 {kms => dstack/kms}/auth-mock/index.ts | 0 {kms => dstack/kms}/auth-mock/openapi.json | 0 {kms => dstack/kms}/auth-mock/package.json | 0 {kms => dstack/kms}/auth-mock/vitest.config.ts | 0 {kms => dstack/kms}/auth-simple/.oxlintrc.json | 0 {kms => dstack/kms}/auth-simple/README.md | 0 .../kms}/auth-simple/auth-config.example.json | 0 {kms => dstack/kms}/auth-simple/bun.lock | 0 {kms => dstack/kms}/auth-simple/index.test.ts | 0 {kms => dstack/kms}/auth-simple/index.ts | 0 {kms => dstack/kms}/auth-simple/package.json | 0 {kms => dstack/kms}/auth-simple/vitest.config.ts | 0 {kms => dstack/kms}/dstack-app/.gitignore | 0 {kms => dstack/kms}/dstack-app/builder/Dockerfile | 0 {kms => dstack/kms}/dstack-app/builder/README.md | 0 .../kms}/dstack-app/builder/build-image.sh | 0 .../builder/shared/builder-pinned-packages.txt | 0 .../builder/shared/qemu-pinned-packages.txt | 0 {kms => dstack/kms}/dstack-app/compose-dev.yaml | 0 {kms => dstack/kms}/dstack-app/compose-simple.yaml | 0 {kms => dstack/kms}/dstack-app/deploy-simple.sh | 0 {kms => dstack/kms}/dstack-app/deploy-to-vmm.sh | 0 {kms => dstack/kms}/dstack-app/docker-compose.yaml | 0 {kms => dstack/kms}/dstack-app/entrypoint.sh | 0 {kms => dstack/kms}/kms.toml | 0 {kms => dstack/kms}/rpc/Cargo.toml | 0 {kms => dstack/kms}/rpc/build.rs | 0 {kms => dstack/kms}/rpc/proto/kms_rpc.proto | 0 {kms => dstack/kms}/rpc/src/.gitignore | 0 {kms => dstack/kms}/rpc/src/generated.rs | 0 {kms => dstack/kms}/rpc/src/lib.rs | 0 {kms => dstack/kms}/src/config.rs | 0 {kms => dstack/kms}/src/crypto.rs | 0 {kms => dstack/kms}/src/ct_log.rs | 0 {kms => dstack/kms}/src/main.rs | 0 {kms => dstack/kms}/src/main_service.rs | 0 {kms => dstack/kms}/src/main_service/amd_attest.rs | 0 .../kms}/src/main_service/upgrade_authority.rs | 0 {kms => dstack/kms}/src/onboard_service.rs | 0 {kms => dstack/kms}/src/www/onboard.html | 0 {load_config => dstack/load_config}/Cargo.toml | 0 {load_config => dstack/load_config}/src/lib.rs | 0 {lspci => dstack/lspci}/Cargo.toml | 0 {lspci => dstack/lspci}/src/lib.rs | 0 .../lspci}/src/snapshots/lspci__lspci.snap | 0 {no_std_check => dstack/no_std_check}/Cargo.toml | 0 {no_std_check => dstack/no_std_check}/src/lib.rs | 0 {nsm-attest => dstack/nsm-attest}/Cargo.toml | 0 {nsm-attest => dstack/nsm-attest}/src/lib.rs | 0 {nsm-attest => dstack/nsm-attest}/src/types.rs | 0 .../nsm-attest}/tests/attestation_test.rs | 0 .../nsm-attest}/tests/nitro_attestation.bin | Bin {nsm-qvl => dstack/nsm-qvl}/Cargo.toml | 0 .../nsm-qvl}/certs/AWS_NitroEnclaves_Root-G1.pem | 0 {nsm-qvl => dstack/nsm-qvl}/src/collateral.rs | 0 {nsm-qvl => dstack/nsm-qvl}/src/lib.rs | 0 {nsm-qvl => dstack/nsm-qvl}/src/verify.rs | 0 .../nsm-qvl}/tests/nitro_attestation.bin | Bin {nsm-qvl => dstack/nsm-qvl}/tests/verify_test.rs | 0 {port-forward => dstack/port-forward}/Cargo.toml | 0 {port-forward => dstack/port-forward}/src/lib.rs | 0 {port-forward => dstack/port-forward}/src/tcp.rs | 0 {port-forward => dstack/port-forward}/src/udp.rs | 0 {python => dstack/python}/.gitignore | 0 {python => dstack/python}/ct_monitor/ct_monitor.py | 0 {python => dstack/python}/ct_monitor/pyproject.toml | 0 {ra-rpc => dstack/ra-rpc}/Cargo.toml | 0 {ra-rpc => dstack/ra-rpc}/prpc-openapi.md | 0 {ra-rpc => dstack/ra-rpc}/src/client.rs | 0 {ra-rpc => dstack/ra-rpc}/src/lib.rs | 0 {ra-rpc => dstack/ra-rpc}/src/openapi.rs | 0 {ra-rpc => dstack/ra-rpc}/src/rocket_helper.rs | 0 {ra-tls => dstack/ra-tls}/Cargo.toml | 0 {ra-tls => dstack/ra-tls}/assets/tdx_quote | Bin {ra-tls => dstack/ra-tls}/src/attestation.rs | 0 {ra-tls => dstack/ra-tls}/src/cert.rs | 0 {ra-tls => dstack/ra-tls}/src/kdf.rs | 0 {ra-tls => dstack/ra-tls}/src/lib.rs | 0 {ra-tls => dstack/ra-tls}/src/oids.rs | 0 {ra-tls => dstack/ra-tls}/src/traits.rs | 0 .../rocket-vsock-listener}/Cargo.toml | 0 .../rocket-vsock-listener}/src/lib.rs | 0 run-tests.sh => dstack/run-tests.sh | 0 run.sh => dstack/run.sh | 0 rust-toolchain.toml => dstack/rust-toolchain.toml | 0 {scripts => dstack/scripts}/add-spdx-attribution.py | 0 {scripts => dstack/scripts}/bin/dstack-cloud | 0 {scripts => dstack/scripts}/config-fw.sh | 0 {scripts => dstack/scripts}/dhcp-notify.sh | 0 {scripts => dstack/scripts}/dstack-image-oci.sh | 0 {scripts => dstack/scripts}/install.sh | 0 {scripts => dstack/scripts}/setup-bridge.sh | 0 .../serde-duration}/Cargo.toml | 0 .../serde-duration}/src/lib.rs | 0 .../sev-snp-attest}/Cargo.toml | 0 .../sev-snp-attest}/src/lib.rs | 0 {sev-snp-qvl => dstack/sev-snp-qvl}/Cargo.toml | 0 {sev-snp-qvl => dstack/sev-snp-qvl}/src/lib.rs | 0 {size-parser => dstack/size-parser}/Cargo.toml | 0 {size-parser => dstack/size-parser}/README.md | 0 {size-parser => dstack/size-parser}/src/lib.rs | 0 {sodiumbox => dstack/sodiumbox}/Cargo.toml | 0 {sodiumbox => dstack/sodiumbox}/README.md | 0 {sodiumbox => dstack/sodiumbox}/src/lib.rs | 0 {supervisor => dstack/supervisor}/Cargo.toml | 0 {supervisor => dstack/supervisor}/client/Cargo.toml | 0 {supervisor => dstack/supervisor}/client/src/lib.rs | 0 .../supervisor}/client/src/main.rs | 0 {supervisor => dstack/supervisor}/src/lib.rs | 0 {supervisor => dstack/supervisor}/src/main.rs | 0 {supervisor => dstack/supervisor}/src/process.rs | 0 {supervisor => dstack/supervisor}/src/supervisor.rs | 0 {supervisor => dstack/supervisor}/src/web_api.rs | 0 {supervisor => dstack/supervisor}/supervisor.toml | 0 {supervisor => dstack/supervisor}/tests/test-cli.sh | 0 {supervisor => dstack/supervisor}/tests/test.sh | 0 {tdx-attest => dstack/tdx-attest}/Cargo.toml | 0 .../tdx-attest}/examples/test_tdx.rs | 0 {tdx-attest => dstack/tdx-attest}/src/dummy.rs | 0 {tdx-attest => dstack/tdx-attest}/src/lib.rs | 0 {tdx-attest => dstack/tdx-attest}/src/linux.rs | 0 .../tdx_attest__eventlog__tests__parse_ccel-2.snap | 0 .../tdx_attest__eventlog__tests__parse_ccel.snap | 0 .../test-scripts}/get-app-key.sh | 0 .../test-scripts}/inspect-cert.sh | 0 .../test-scripts}/snp-e2e-smoke.sh | 0 .../tests}/docs/kms-bootstrap-onboard.md | 0 .../tests}/docs/kms-self-authorization.md | 0 {tpm-attest => dstack/tpm-attest}/Cargo.toml | 0 {tpm-attest => dstack/tpm-attest}/src/esapi.rs | 0 {tpm-attest => dstack/tpm-attest}/src/gcp_ak.rs | 0 {tpm-attest => dstack/tpm-attest}/src/lib.rs | 0 .../tpm-attest}/tests/tpm_quote_sample.bin | Bin {tpm-qvl => dstack/tpm-qvl}/Cargo.toml | 0 .../tpm-qvl}/certs/AWS_NitroEnclaves_Root-G1.pem | 0 {tpm-qvl => dstack/tpm-qvl}/certs/gcp-root-ca.pem | 0 {tpm-qvl => dstack/tpm-qvl}/src/collateral.rs | 0 {tpm-qvl => dstack/tpm-qvl}/src/lib.rs | 0 {tpm-qvl => dstack/tpm-qvl}/src/verify.rs | 0 {tpm-types => dstack/tpm-types}/Cargo.toml | 0 {tpm-types => dstack/tpm-types}/src/lib.rs | 0 {tpm2 => dstack/tpm2}/Cargo.toml | 0 {tpm2 => dstack/tpm2}/src/bin/tpm2-test.rs | 0 {tpm2 => dstack/tpm2}/src/commands.rs | 0 {tpm2 => dstack/tpm2}/src/constants.rs | 0 {tpm2 => dstack/tpm2}/src/device.rs | 0 {tpm2 => dstack/tpm2}/src/lib.rs | 0 {tpm2 => dstack/tpm2}/src/marshal.rs | 0 {tpm2 => dstack/tpm2}/src/session.rs | 0 {tpm2 => dstack/tpm2}/src/types.rs | 0 {verifier => dstack/verifier}/Cargo.toml | 0 {verifier => dstack/verifier}/README.md | 0 {verifier => dstack/verifier}/builder/Dockerfile | 0 .../verifier}/builder/build-image.sh | 0 .../builder/shared/builder-pinned-packages.txt | 0 .../verifier}/builder/shared/pinned-packages.txt | 0 .../builder/shared/qemu-pinned-packages.txt | 0 {verifier => dstack/verifier}/dstack-verifier.toml | 0 .../verifier}/fixtures/quote-report.json | 0 .../verifier}/fixtures/sev-snp-attestation.json | 0 .../verifier}/fixtures/sev-snp.README.md | 0 .../verifier}/fixtures/tdx-lite-attestation.json | 0 .../verifier}/fixtures/tdx-lite-getquote.json | 0 .../verifier}/fixtures/tdx-lite.README.md | 0 {verifier => dstack/verifier}/src/lib.rs | 0 {verifier => dstack/verifier}/src/main.rs | 0 {verifier => dstack/verifier}/src/types.rs | 0 {verifier => dstack/verifier}/src/verification.rs | 0 {verifier => dstack/verifier}/test.sh | 0 {vmm => dstack/vmm}/Cargo.toml | 0 {vmm => dstack/vmm}/build.rs | 0 {vmm => dstack/vmm}/requirements.txt | 0 {vmm => dstack/vmm}/rpc/Cargo.toml | 0 {vmm => dstack/vmm}/rpc/build.rs | 0 {vmm => dstack/vmm}/rpc/proto/prpc.proto | 0 {vmm => dstack/vmm}/rpc/proto/vmm_rpc.proto | 0 {vmm => dstack/vmm}/rpc/src/generated.rs | 0 {vmm => dstack/vmm}/rpc/src/lib.rs | 0 {vmm => dstack/vmm}/src/app.rs | 0 {vmm => dstack/vmm}/src/app/id_pool.rs | 0 {vmm => dstack/vmm}/src/app/image.rs | 0 {vmm => dstack/vmm}/src/app/qemu.rs | 0 {vmm => dstack/vmm}/src/app/registry.rs | 0 {vmm => dstack/vmm}/src/config.rs | 0 {vmm => dstack/vmm}/src/console_v0.html | 0 {vmm => dstack/vmm}/src/discovery.rs | 0 {vmm => dstack/vmm}/src/guest_api_service.rs | 0 {vmm => dstack/vmm}/src/host_api_service.rs | 0 {vmm => dstack/vmm}/src/main.rs | 0 {vmm => dstack/vmm}/src/main_routes.rs | 0 {vmm => dstack/vmm}/src/main_service.rs | 0 {vmm => dstack/vmm}/src/one_shot.rs | 0 {vmm => dstack/vmm}/src/openapi.rs | 0 {vmm => dstack/vmm}/src/setup-user.sh | 0 {vmm => dstack/vmm}/src/tests/test-compose.sh | 0 {vmm => dstack/vmm}/src/tests/test-deployment.sh | 0 {vmm => dstack/vmm}/src/vmm-cli.py | 0 {vmm => dstack/vmm}/src/x25519.js | 0 {vmm => dstack/vmm}/ui/.gitignore | 0 {vmm => dstack/vmm}/ui/README.md | 0 {vmm => dstack/vmm}/ui/build.mjs | 0 {vmm => dstack/vmm}/ui/package-lock.json | 0 {vmm => dstack/vmm}/ui/package.json | 0 {vmm => dstack/vmm}/ui/scripts/build_proto.sh | 0 {vmm => dstack/vmm}/ui/src/App.ts | 0 .../vmm}/ui/src/components/CreateVmDialog.ts | 0 .../vmm}/ui/src/components/EncryptedEnvEditor.ts | 0 .../vmm}/ui/src/components/ForkVmDialog.ts | 0 .../vmm}/ui/src/components/GpuConfigEditor.ts | 0 .../vmm}/ui/src/components/PortMappingEditor.ts | 0 .../vmm}/ui/src/components/UpdateVmDialog.ts | 0 .../vmm}/ui/src/composables/useVmManager.ts | 0 {vmm => dstack/vmm}/ui/src/index.html | 0 {vmm => dstack/vmm}/ui/src/lib/vmmRpcClient.ts | 0 {vmm => dstack/vmm}/ui/src/lib/x25519.js | 0 {vmm => dstack/vmm}/ui/src/main.ts | 0 {vmm => dstack/vmm}/ui/src/styles/main.css | 0 {vmm => dstack/vmm}/ui/src/templates/app.html | 0 {vmm => dstack/vmm}/ui/tsconfig.json | 0 {vmm => dstack/vmm}/ui/vendor/README.md | 0 {vmm => dstack/vmm}/venv.sh | 0 {vmm => dstack/vmm}/vmm.toml | 0 532 files changed, 3 insertions(+), 3 deletions(-) rename Cargo.lock => dstack/Cargo.lock (100%) rename Cargo.toml => dstack/Cargo.toml (100%) rename Makefile => dstack/Makefile (100%) rename {basefiles => dstack/basefiles}/app-compose.service (100%) rename {basefiles => dstack/basefiles}/app-compose.sh (100%) rename {basefiles => dstack/basefiles}/containerd.service.d/dstack-prepare.conf (100%) rename {basefiles => dstack/basefiles}/docker.service.d/dstack-guest-agent.conf (100%) rename {basefiles => dstack/basefiles}/docker.service.d/dstack-prepare.conf (100%) rename {basefiles => dstack/basefiles}/dstack-guest-agent.service (100%) rename {basefiles => dstack/basefiles}/dstack-guest-agent.socket (100%) rename {basefiles => dstack/basefiles}/dstack-prepare.service (100%) rename {basefiles => dstack/basefiles}/dstack-prepare.sh (100%) rename {basefiles => dstack/basefiles}/ephemeral-docker.sh (100%) rename {basefiles => dstack/basefiles}/journald.conf (100%) rename {basefiles => dstack/basefiles}/llmnr.conf (100%) rename {basefiles => dstack/basefiles}/sysctl.d/99-dstack.conf (100%) rename {basefiles => dstack/basefiles}/tdx-attest.conf (100%) rename {basefiles => dstack/basefiles}/wg-checker.service (100%) rename {basefiles => dstack/basefiles}/wg-checker.sh (100%) rename {build => dstack/build}/shared/build-lib.sh (100%) rename {build => dstack/build}/shared/config-qemu.sh (100%) rename {build => dstack/build}/shared/pin-packages.sh (100%) rename {build => dstack/build}/shared/verify-pinned-packages.sh (100%) rename {cached-cell => dstack/cached-cell}/Cargo.toml (100%) rename {cached-cell => dstack/cached-cell}/src/lib.rs (100%) rename cargo-check-all.sh => dstack/cargo-check-all.sh (100%) rename {cc-eventlog => dstack/cc-eventlog}/Cargo.toml (100%) rename {cc-eventlog => dstack/cc-eventlog}/samples/ccel.bin (100%) rename {cc-eventlog => dstack/cc-eventlog}/samples/tpm_eventlog.bin (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/codecs.rs (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/lib.rs (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/runtime_events.rs (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/snapshots/cc_eventlog__tests__parse_ccel-2.snap (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/snapshots/cc_eventlog__tests__parse_ccel.snap (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/tcg.rs (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/tdx.rs (100%) rename {cc-eventlog => dstack/cc-eventlog}/src/tpm.rs (100%) rename {cert-client => dstack/cert-client}/Cargo.toml (100%) rename {cert-client => dstack/cert-client}/src/lib.rs (100%) rename {certbot => dstack/certbot}/.gitignore (100%) rename {certbot => dstack/certbot}/Cargo.toml (100%) rename {certbot => dstack/certbot}/cli/Cargo.toml (100%) rename {certbot => dstack/certbot}/cli/src/main.rs (100%) rename {certbot => dstack/certbot}/src/acme_client.rs (100%) rename {certbot => dstack/certbot}/src/acme_client/tests.rs (100%) rename {certbot => dstack/certbot}/src/bot.rs (100%) rename {certbot => dstack/certbot}/src/bot/tests.rs (100%) rename {certbot => dstack/certbot}/src/dns01_client.rs (100%) rename {certbot => dstack/certbot}/src/dns01_client/cloudflare.rs (100%) rename {certbot => dstack/certbot}/src/http_client.rs (100%) rename {certbot => dstack/certbot}/src/lib.rs (100%) rename {certbot => dstack/certbot}/src/workdir.rs (100%) rename {crates => dstack/crates}/dstack-auth/Cargo.toml (100%) rename {crates => dstack/crates}/dstack-auth/src/main.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/Cargo.toml (100%) rename {crates => dstack/crates}/dstack-cli-core/src/compose.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/config.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/fsutil.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/host.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/layout.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/lib.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/ports.rs (100%) rename {crates => dstack/crates}/dstack-cli-core/src/vmm.rs (100%) rename {crates => dstack/crates}/dstack-cli/Cargo.toml (100%) rename {crates => dstack/crates}/dstack-cli/src/main.rs (100%) rename {crates => dstack/crates}/dstackup/Cargo.toml (100%) rename {crates => dstack/crates}/dstackup/src/cid.rs (100%) rename {crates => dstack/crates}/dstackup/src/cli.rs (100%) rename {crates => dstack/crates}/dstackup/src/destroy.rs (100%) rename {crates => dstack/crates}/dstackup/src/image.rs (100%) rename {crates => dstack/crates}/dstackup/src/install.rs (100%) rename {crates => dstack/crates}/dstackup/src/main.rs (100%) rename {crates => dstack/crates}/dstackup/src/state.rs (100%) rename {crates => dstack/crates}/dstackup/src/systemd.rs (100%) rename {ct_monitor => dstack/ct_monitor}/Cargo.toml (100%) rename {ct_monitor => dstack/ct_monitor}/src/main.rs (100%) rename {dstack-attest => dstack/dstack-attest}/Cargo.toml (100%) rename {dstack-attest => dstack/dstack-attest}/src/amd_sev_snp.rs (100%) rename {dstack-attest => dstack/dstack-attest}/src/attestation.rs (100%) rename {dstack-attest => dstack/dstack-attest}/src/lib.rs (100%) rename {dstack-attest => dstack/dstack-attest}/src/sev_snp.rs (100%) rename {dstack-attest => dstack/dstack-attest}/src/v1.rs (100%) rename {dstack-attest => dstack/dstack-attest}/tests/nitro_attestation.bin (100%) rename {dstack-attest => dstack/dstack-attest}/tests/nitro_attestation_dbg.bin (100%) rename {dstack-attest => dstack/dstack-attest}/tests/nitro_verify.rs (100%) rename {dstack-attest => dstack/dstack-attest}/tests/sev_snp_ask.pem (100%) rename {dstack-attest => dstack/dstack-attest}/tests/sev_snp_attestation.bin (100%) rename {dstack-attest => dstack/dstack-attest}/tests/sev_snp_fixture.README.md (100%) rename {dstack-attest => dstack/dstack-attest}/tests/sev_snp_vcek.pem (100%) rename {dstack-attest => dstack/dstack-attest}/tests/sev_snp_verify.rs (100%) rename {dstack-attest => dstack/dstack-attest}/tests/snapshots/nitro_verify__app_info.snap (100%) rename {dstack-attest => dstack/dstack-attest}/tests/snapshots/nitro_verify__nitro_report.snap (100%) rename {dstack-mr => dstack/dstack-mr}/.gitignore (100%) rename {dstack-mr => dstack/dstack-mr}/Cargo.toml (100%) rename {dstack-mr => dstack/dstack-mr}/cli/Cargo.toml (100%) rename {dstack-mr => dstack/dstack-mr}/cli/src/main.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/acpi.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/kernel.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/lib.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/machine.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/main.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/measurement.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/num.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/sev.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/tdvf.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/tdx.rs (100%) rename {dstack-mr => dstack/dstack-mr}/src/util.rs (100%) rename {dstack-mr => dstack/dstack-mr}/tests/tdvf_parse.rs (100%) rename {dstack-types => dstack/dstack-types}/Cargo.toml (100%) rename {dstack-types => dstack/dstack-types}/src/lib.rs (100%) rename {dstack-types => dstack/dstack-types}/src/mr_config.rs (100%) rename {dstack-types => dstack/dstack-types}/src/shared_filenames.rs (100%) rename {dstack-types => dstack/dstack-types}/src/version.rs (100%) rename {dstack-util => dstack/dstack-util}/Cargo.toml (100%) rename {dstack-util => dstack/dstack-util}/src/crypto.rs (100%) rename {dstack-util => dstack/dstack-util}/src/docker_compose.rs (100%) rename {dstack-util => dstack/dstack-util}/src/host_api.rs (100%) rename {dstack-util => dstack/dstack-util}/src/main.rs (100%) rename {dstack-util => dstack/dstack-util}/src/parse_env_file.rs (100%) rename {dstack-util => dstack/dstack-util}/src/system_setup.rs (100%) rename {dstack-util => dstack/dstack-util}/src/system_setup/config_id_verifier.rs (100%) rename {dstack-util => dstack/dstack-util}/src/utils.rs (100%) rename {dstack-util => dstack/dstack-util}/tests/fixtures/key-provider-docker-compose.yaml (100%) rename {dstack-util => dstack/dstack-util}/tests/fixtures/luks_header_cipher_null (100%) rename {dstack-util => dstack/dstack-util}/tests/fixtures/luks_header_cipher_null.license (100%) rename {dstack-util => dstack/dstack-util}/tests/fixtures/luks_header_good (100%) rename {dstack-util => dstack/dstack-util}/tests/fixtures/luks_header_good.license (100%) rename {dstack-util => dstack/dstack-util}/tests/test_remove_orphans.sh (100%) rename {gateway => dstack/gateway}/Cargo.toml (100%) rename {gateway => dstack/gateway}/assets/cert.key (100%) rename {gateway => dstack/gateway}/assets/cert.pem (100%) rename {gateway => dstack/gateway}/docs/cluster-deployment.md (100%) rename {gateway => dstack/gateway}/dstack-app/.gitignore (100%) rename {gateway => dstack/gateway}/dstack-app/bootstrap-cluster.sh (100%) rename {gateway => dstack/gateway}/dstack-app/builder/Dockerfile (100%) rename {gateway => dstack/gateway}/dstack-app/builder/README.md (100%) rename {gateway => dstack/gateway}/dstack-app/builder/build-image.sh (100%) rename {gateway => dstack/gateway}/dstack-app/builder/entrypoint.sh (100%) rename {gateway => dstack/gateway}/dstack-app/builder/shared/builder-pinned-packages.txt (100%) rename {gateway => dstack/gateway}/dstack-app/builder/shared/pinned-packages.txt (100%) rename {gateway => dstack/gateway}/dstack-app/deploy-to-vmm.sh (100%) rename {gateway => dstack/gateway}/dstack-app/docker-compose.yaml (100%) rename {gateway => dstack/gateway}/gateway.toml (100%) rename {gateway => dstack/gateway}/rpc/Cargo.toml (100%) rename {gateway => dstack/gateway}/rpc/build.rs (100%) rename {gateway => dstack/gateway}/rpc/proto/gateway_rpc.proto (100%) rename {gateway => dstack/gateway}/rpc/src/generated.rs (100%) rename {gateway => dstack/gateway}/rpc/src/lib.rs (100%) rename {gateway => dstack/gateway}/src/admin_auth.rs (100%) rename {gateway => dstack/gateway}/src/admin_service.rs (100%) rename {gateway => dstack/gateway}/src/cert_store.rs (100%) rename {gateway => dstack/gateway}/src/config.rs (100%) rename {gateway => dstack/gateway}/src/debug_service.rs (100%) rename {gateway => dstack/gateway}/src/distributed_certbot.rs (100%) rename {gateway => dstack/gateway}/src/gen_debug_key.rs (100%) rename {gateway => dstack/gateway}/src/kv/https_client.rs (100%) rename {gateway => dstack/gateway}/src/kv/mod.rs (100%) rename {gateway => dstack/gateway}/src/kv/sync_service.rs (100%) rename {gateway => dstack/gateway}/src/main.rs (100%) rename {gateway => dstack/gateway}/src/main_service.rs (100%) rename {gateway => dstack/gateway}/src/main_service/auth_client.rs (100%) rename {gateway => dstack/gateway}/src/main_service/handshakes.rs (100%) rename {gateway => dstack/gateway}/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap (100%) rename {gateway => dstack/gateway}/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap (100%) rename {gateway => dstack/gateway}/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap (100%) rename {gateway => dstack/gateway}/src/main_service/snapshots/dstack_gateway__main_service__tests__empty_config.snap (100%) rename {gateway => dstack/gateway}/src/main_service/tests.rs (100%) rename {gateway => dstack/gateway}/src/models.rs (100%) rename {gateway => dstack/gateway}/src/pp.rs (100%) rename {gateway => dstack/gateway}/src/proxy.rs (100%) rename {gateway => dstack/gateway}/src/proxy/io_bridge.rs (100%) rename {gateway => dstack/gateway}/src/proxy/port_policy.rs (100%) rename {gateway => dstack/gateway}/src/proxy/sni.rs (100%) rename {gateway => dstack/gateway}/src/proxy/tls_passthough.rs (100%) rename {gateway => dstack/gateway}/src/proxy/tls_terminate.rs (100%) rename {gateway => dstack/gateway}/src/web_routes.rs (100%) rename {gateway => dstack/gateway}/src/web_routes/route_index.rs (100%) rename {gateway => dstack/gateway}/src/web_routes/wavekv_sync.rs (100%) rename {gateway => dstack/gateway}/templates/dashboard.html (100%) rename {gateway => dstack/gateway}/templates/rproxy.yaml (100%) rename {gateway => dstack/gateway}/templates/wg.conf (100%) rename {gateway => dstack/gateway}/test-run/.env.example (100%) rename {gateway => dstack/gateway}/test-run/.gitignore (100%) rename {gateway => dstack/gateway}/test-run/TESTING.md (100%) rename {gateway => dstack/gateway}/test-run/cluster.sh (100%) rename {gateway => dstack/gateway}/test-run/e2e/configs/gateway-1.toml (100%) rename {gateway => dstack/gateway}/test-run/e2e/configs/gateway-2.toml (100%) rename {gateway => dstack/gateway}/test-run/e2e/configs/gateway-3.toml (100%) rename {gateway => dstack/gateway}/test-run/e2e/docker-compose.yml (100%) rename {gateway => dstack/gateway}/test-run/e2e/pebble-config.json (100%) rename {gateway => dstack/gateway}/test-run/e2e/run-e2e.sh (100%) rename {gateway => dstack/gateway}/test-run/e2e/test.sh (100%) rename {gateway => dstack/gateway}/test-run/test_certbot.sh (100%) rename {gateway => dstack/gateway}/test-run/test_suite.sh (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/.gitignore (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/Cargo.toml (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/dstack-simulator.service (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/dstack.toml (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/install-systemd.sh (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/package-release.sh (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/src/main.rs (100%) rename {guest-agent-simulator => dstack/guest-agent-simulator}/src/simulator.rs (100%) rename {guest-agent => dstack/guest-agent}/Cargo.toml (100%) rename {guest-agent => dstack/guest-agent}/dstack.toml (100%) rename {guest-agent => dstack/guest-agent}/fixtures/attestation.bin (100%) rename {guest-agent => dstack/guest-agent}/rpc/Cargo.toml (100%) rename {guest-agent => dstack/guest-agent}/rpc/build.rs (100%) rename {guest-agent => dstack/guest-agent}/rpc/proto/agent_rpc.proto (100%) rename {guest-agent => dstack/guest-agent}/rpc/src/generated.rs (100%) rename {guest-agent => dstack/guest-agent}/rpc/src/lib.rs (100%) rename {guest-agent => dstack/guest-agent}/src/backend.rs (100%) rename {guest-agent => dstack/guest-agent}/src/config.rs (100%) rename {guest-agent => dstack/guest-agent}/src/guest_api_service.rs (100%) rename {guest-agent => dstack/guest-agent}/src/http_routes.rs (100%) rename {guest-agent => dstack/guest-agent}/src/lib.rs (100%) rename {guest-agent => dstack/guest-agent}/src/main.rs (100%) rename {guest-agent => dstack/guest-agent}/src/models.rs (100%) rename {guest-agent => dstack/guest-agent}/src/rpc_service.rs (100%) rename {guest-agent => dstack/guest-agent}/src/server.rs (100%) rename {guest-agent => dstack/guest-agent}/src/socket_activation.rs (100%) rename {guest-agent => dstack/guest-agent}/templates/dashboard.html (100%) rename {guest-agent => dstack/guest-agent}/templates/metrics.tpl (100%) rename {guest-api => dstack/guest-api}/Cargo.toml (100%) rename {guest-api => dstack/guest-api}/build.rs (100%) rename {guest-api => dstack/guest-api}/proto/guest_api.proto (100%) rename {guest-api => dstack/guest-api}/src/client.rs (100%) rename {guest-api => dstack/guest-api}/src/generated/mod.rs (100%) rename {guest-api => dstack/guest-api}/src/lib.rs (100%) rename {host-api => dstack/host-api}/Cargo.toml (100%) rename {host-api => dstack/host-api}/build.rs (100%) rename {host-api => dstack/host-api}/proto/host_api.proto (100%) rename {host-api => dstack/host-api}/src/client.rs (100%) rename {host-api => dstack/host-api}/src/generated/mod.rs (100%) rename {host-api => dstack/host-api}/src/lib.rs (100%) rename {http-client => dstack/http-client}/Cargo.toml (100%) rename {http-client => dstack/http-client}/src/hyper_vsock.rs (100%) rename {http-client => dstack/http-client}/src/lib.rs (100%) rename {http-client => dstack/http-client}/src/prpc.rs (100%) rename {iohash => dstack/iohash}/Cargo.toml (100%) rename {iohash => dstack/iohash}/src/main.rs (100%) rename {key-provider-build => dstack/key-provider-build}/Cargo.lock (100%) rename {key-provider-build => dstack/key-provider-build}/Dockerfile.aesmd (100%) rename {key-provider-build => dstack/key-provider-build}/Dockerfile.key-provider (100%) rename {key-provider-build => dstack/key-provider-build}/docker-compose.yaml (100%) rename {key-provider-build => dstack/key-provider-build}/entrypoint-aesmd.sh (100%) rename {key-provider-build => dstack/key-provider-build}/entrypoint-key-provider.sh (100%) rename {key-provider-build => dstack/key-provider-build}/run.sh (100%) rename {key-provider-build => dstack/key-provider-build}/sgx_default_qcnl.conf (100%) rename {key-provider-client => dstack/key-provider-client}/Cargo.toml (100%) rename {key-provider-client => dstack/key-provider-client}/src/host.rs (100%) rename {key-provider-client => dstack/key-provider-client}/src/lib.rs (100%) rename {kms => dstack/kms}/Cargo.toml (100%) rename {kms => dstack/kms}/README.md (100%) rename {kms => dstack/kms}/auth-eth-bun/.oxlintrc.json (100%) rename {kms => dstack/kms}/auth-eth-bun/README.md (100%) rename {kms => dstack/kms}/auth-eth-bun/bun.lock (100%) rename {kms => dstack/kms}/auth-eth-bun/index.test.ts (100%) rename {kms => dstack/kms}/auth-eth-bun/index.ts (100%) rename {kms => dstack/kms}/auth-eth-bun/openapi.json (100%) rename {kms => dstack/kms}/auth-eth-bun/package.json (100%) rename {kms => dstack/kms}/auth-eth-bun/vitest.config.ts (100%) rename {kms => dstack/kms}/auth-eth/.env.example (100%) rename {kms => dstack/kms}/auth-eth/.gitignore (100%) rename {kms => dstack/kms}/auth-eth/.openzeppelin/unknown-2035.json (100%) rename {kms => dstack/kms}/auth-eth/README.md (100%) rename {kms => dstack/kms}/auth-eth/TESTING.md (100%) rename {kms => dstack/kms}/auth-eth/contracts/DstackApp.sol (100%) rename {kms => dstack/kms}/auth-eth/contracts/DstackKms.sol (100%) rename {kms => dstack/kms}/auth-eth/contracts/IAppAuth.sol (100%) rename {kms => dstack/kms}/auth-eth/contracts/IAppAuthBasicManagement.sol (100%) rename {kms => dstack/kms}/auth-eth/contracts/test-utils/DstackAppV2.sol (100%) rename {kms => dstack/kms}/auth-eth/contracts/test-utils/DstackKmsV2.sol (100%) rename {kms => dstack/kms}/auth-eth/docs/formal-verification.md (100%) rename {kms => dstack/kms}/auth-eth/docs/specification.md (100%) rename {kms => dstack/kms}/auth-eth/foundry.toml (100%) rename {kms => dstack/kms}/auth-eth/jest.config.js (100%) rename {kms => dstack/kms}/auth-eth/lib/forge-std (100%) rename {kms => dstack/kms}/auth-eth/lib/openzeppelin-contracts-upgradeable (100%) rename {kms => dstack/kms}/auth-eth/lib/openzeppelin-foundry-upgrades (100%) rename {kms => dstack/kms}/auth-eth/package-lock.json (100%) rename {kms => dstack/kms}/auth-eth/package.json (100%) rename {kms => dstack/kms}/auth-eth/run-tests.sh (100%) rename {kms => dstack/kms}/auth-eth/script/Deploy.s.sol (100%) rename {kms => dstack/kms}/auth-eth/script/Manage.s.sol (100%) rename {kms => dstack/kms}/auth-eth/script/Query.s.sol (100%) rename {kms => dstack/kms}/auth-eth/script/README.md (100%) rename {kms => dstack/kms}/auth-eth/script/Upgrade.s.sol (100%) rename {kms => dstack/kms}/auth-eth/scripts/README.md (100%) rename {kms => dstack/kms}/auth-eth/scripts/cleanup.sh (100%) rename {kms => dstack/kms}/auth-eth/scripts/run-tests.sh (100%) rename {kms => dstack/kms}/auth-eth/scripts/setup-local-chain.sh (100%) rename {kms => dstack/kms}/auth-eth/scripts/test-all.sh (100%) rename {kms => dstack/kms}/auth-eth/slither.config.json (100%) rename {kms => dstack/kms}/auth-eth/src/ethereum.ts (100%) rename {kms => dstack/kms}/auth-eth/src/main.test.ts (100%) rename {kms => dstack/kms}/auth-eth/src/main.ts (100%) rename {kms => dstack/kms}/auth-eth/src/server.ts (100%) rename {kms => dstack/kms}/auth-eth/src/types.ts (100%) rename {kms => dstack/kms}/auth-eth/test/DstackApp.symbolic.t.sol (100%) rename {kms => dstack/kms}/auth-eth/test/DstackApp.t.sol (100%) rename {kms => dstack/kms}/auth-eth/test/DstackKms.symbolic.t.sol (100%) rename {kms => dstack/kms}/auth-eth/test/DstackKms.t.sol (100%) rename {kms => dstack/kms}/auth-eth/test/UpgradesWithPlugin.t.sol (100%) rename {kms => dstack/kms}/auth-eth/tsconfig.json (100%) rename {kms => dstack/kms}/auth-mock/.oxlintrc.json (100%) rename {kms => dstack/kms}/auth-mock/Dockerfile (100%) rename {kms => dstack/kms}/auth-mock/README.md (100%) rename {kms => dstack/kms}/auth-mock/bun.lock (100%) rename {kms => dstack/kms}/auth-mock/index.test.ts (100%) rename {kms => dstack/kms}/auth-mock/index.ts (100%) rename {kms => dstack/kms}/auth-mock/openapi.json (100%) rename {kms => dstack/kms}/auth-mock/package.json (100%) rename {kms => dstack/kms}/auth-mock/vitest.config.ts (100%) rename {kms => dstack/kms}/auth-simple/.oxlintrc.json (100%) rename {kms => dstack/kms}/auth-simple/README.md (100%) rename {kms => dstack/kms}/auth-simple/auth-config.example.json (100%) rename {kms => dstack/kms}/auth-simple/bun.lock (100%) rename {kms => dstack/kms}/auth-simple/index.test.ts (100%) rename {kms => dstack/kms}/auth-simple/index.ts (100%) rename {kms => dstack/kms}/auth-simple/package.json (100%) rename {kms => dstack/kms}/auth-simple/vitest.config.ts (100%) rename {kms => dstack/kms}/dstack-app/.gitignore (100%) rename {kms => dstack/kms}/dstack-app/builder/Dockerfile (100%) rename {kms => dstack/kms}/dstack-app/builder/README.md (100%) rename {kms => dstack/kms}/dstack-app/builder/build-image.sh (100%) rename {kms => dstack/kms}/dstack-app/builder/shared/builder-pinned-packages.txt (100%) rename {kms => dstack/kms}/dstack-app/builder/shared/qemu-pinned-packages.txt (100%) rename {kms => dstack/kms}/dstack-app/compose-dev.yaml (100%) rename {kms => dstack/kms}/dstack-app/compose-simple.yaml (100%) rename {kms => dstack/kms}/dstack-app/deploy-simple.sh (100%) rename {kms => dstack/kms}/dstack-app/deploy-to-vmm.sh (100%) rename {kms => dstack/kms}/dstack-app/docker-compose.yaml (100%) rename {kms => dstack/kms}/dstack-app/entrypoint.sh (100%) rename {kms => dstack/kms}/kms.toml (100%) rename {kms => dstack/kms}/rpc/Cargo.toml (100%) rename {kms => dstack/kms}/rpc/build.rs (100%) rename {kms => dstack/kms}/rpc/proto/kms_rpc.proto (100%) rename {kms => dstack/kms}/rpc/src/.gitignore (100%) rename {kms => dstack/kms}/rpc/src/generated.rs (100%) rename {kms => dstack/kms}/rpc/src/lib.rs (100%) rename {kms => dstack/kms}/src/config.rs (100%) rename {kms => dstack/kms}/src/crypto.rs (100%) rename {kms => dstack/kms}/src/ct_log.rs (100%) rename {kms => dstack/kms}/src/main.rs (100%) rename {kms => dstack/kms}/src/main_service.rs (100%) rename {kms => dstack/kms}/src/main_service/amd_attest.rs (100%) rename {kms => dstack/kms}/src/main_service/upgrade_authority.rs (100%) rename {kms => dstack/kms}/src/onboard_service.rs (100%) rename {kms => dstack/kms}/src/www/onboard.html (100%) rename {load_config => dstack/load_config}/Cargo.toml (100%) rename {load_config => dstack/load_config}/src/lib.rs (100%) rename {lspci => dstack/lspci}/Cargo.toml (100%) rename {lspci => dstack/lspci}/src/lib.rs (100%) rename {lspci => dstack/lspci}/src/snapshots/lspci__lspci.snap (100%) rename {no_std_check => dstack/no_std_check}/Cargo.toml (100%) rename {no_std_check => dstack/no_std_check}/src/lib.rs (100%) rename {nsm-attest => dstack/nsm-attest}/Cargo.toml (100%) rename {nsm-attest => dstack/nsm-attest}/src/lib.rs (100%) rename {nsm-attest => dstack/nsm-attest}/src/types.rs (100%) rename {nsm-attest => dstack/nsm-attest}/tests/attestation_test.rs (100%) rename {nsm-attest => dstack/nsm-attest}/tests/nitro_attestation.bin (100%) rename {nsm-qvl => dstack/nsm-qvl}/Cargo.toml (100%) rename {nsm-qvl => dstack/nsm-qvl}/certs/AWS_NitroEnclaves_Root-G1.pem (100%) rename {nsm-qvl => dstack/nsm-qvl}/src/collateral.rs (100%) rename {nsm-qvl => dstack/nsm-qvl}/src/lib.rs (100%) rename {nsm-qvl => dstack/nsm-qvl}/src/verify.rs (100%) rename {nsm-qvl => dstack/nsm-qvl}/tests/nitro_attestation.bin (100%) rename {nsm-qvl => dstack/nsm-qvl}/tests/verify_test.rs (100%) rename {port-forward => dstack/port-forward}/Cargo.toml (100%) rename {port-forward => dstack/port-forward}/src/lib.rs (100%) rename {port-forward => dstack/port-forward}/src/tcp.rs (100%) rename {port-forward => dstack/port-forward}/src/udp.rs (100%) rename {python => dstack/python}/.gitignore (100%) rename {python => dstack/python}/ct_monitor/ct_monitor.py (100%) rename {python => dstack/python}/ct_monitor/pyproject.toml (100%) rename {ra-rpc => dstack/ra-rpc}/Cargo.toml (100%) rename {ra-rpc => dstack/ra-rpc}/prpc-openapi.md (100%) rename {ra-rpc => dstack/ra-rpc}/src/client.rs (100%) rename {ra-rpc => dstack/ra-rpc}/src/lib.rs (100%) rename {ra-rpc => dstack/ra-rpc}/src/openapi.rs (100%) rename {ra-rpc => dstack/ra-rpc}/src/rocket_helper.rs (100%) rename {ra-tls => dstack/ra-tls}/Cargo.toml (100%) rename {ra-tls => dstack/ra-tls}/assets/tdx_quote (100%) rename {ra-tls => dstack/ra-tls}/src/attestation.rs (100%) rename {ra-tls => dstack/ra-tls}/src/cert.rs (100%) rename {ra-tls => dstack/ra-tls}/src/kdf.rs (100%) rename {ra-tls => dstack/ra-tls}/src/lib.rs (100%) rename {ra-tls => dstack/ra-tls}/src/oids.rs (100%) rename {ra-tls => dstack/ra-tls}/src/traits.rs (100%) rename {rocket-vsock-listener => dstack/rocket-vsock-listener}/Cargo.toml (100%) rename {rocket-vsock-listener => dstack/rocket-vsock-listener}/src/lib.rs (100%) rename run-tests.sh => dstack/run-tests.sh (100%) rename run.sh => dstack/run.sh (100%) rename rust-toolchain.toml => dstack/rust-toolchain.toml (100%) rename {scripts => dstack/scripts}/add-spdx-attribution.py (100%) rename {scripts => dstack/scripts}/bin/dstack-cloud (100%) rename {scripts => dstack/scripts}/config-fw.sh (100%) rename {scripts => dstack/scripts}/dhcp-notify.sh (100%) rename {scripts => dstack/scripts}/dstack-image-oci.sh (100%) rename {scripts => dstack/scripts}/install.sh (100%) rename {scripts => dstack/scripts}/setup-bridge.sh (100%) rename {serde-duration => dstack/serde-duration}/Cargo.toml (100%) rename {serde-duration => dstack/serde-duration}/src/lib.rs (100%) rename {sev-snp-attest => dstack/sev-snp-attest}/Cargo.toml (100%) rename {sev-snp-attest => dstack/sev-snp-attest}/src/lib.rs (100%) rename {sev-snp-qvl => dstack/sev-snp-qvl}/Cargo.toml (100%) rename {sev-snp-qvl => dstack/sev-snp-qvl}/src/lib.rs (100%) rename {size-parser => dstack/size-parser}/Cargo.toml (100%) rename {size-parser => dstack/size-parser}/README.md (100%) rename {size-parser => dstack/size-parser}/src/lib.rs (100%) rename {sodiumbox => dstack/sodiumbox}/Cargo.toml (100%) rename {sodiumbox => dstack/sodiumbox}/README.md (100%) rename {sodiumbox => dstack/sodiumbox}/src/lib.rs (100%) rename {supervisor => dstack/supervisor}/Cargo.toml (100%) rename {supervisor => dstack/supervisor}/client/Cargo.toml (100%) rename {supervisor => dstack/supervisor}/client/src/lib.rs (100%) rename {supervisor => dstack/supervisor}/client/src/main.rs (100%) rename {supervisor => dstack/supervisor}/src/lib.rs (100%) rename {supervisor => dstack/supervisor}/src/main.rs (100%) rename {supervisor => dstack/supervisor}/src/process.rs (100%) rename {supervisor => dstack/supervisor}/src/supervisor.rs (100%) rename {supervisor => dstack/supervisor}/src/web_api.rs (100%) rename {supervisor => dstack/supervisor}/supervisor.toml (100%) rename {supervisor => dstack/supervisor}/tests/test-cli.sh (100%) rename {supervisor => dstack/supervisor}/tests/test.sh (100%) rename {tdx-attest => dstack/tdx-attest}/Cargo.toml (100%) rename {tdx-attest => dstack/tdx-attest}/examples/test_tdx.rs (100%) rename {tdx-attest => dstack/tdx-attest}/src/dummy.rs (100%) rename {tdx-attest => dstack/tdx-attest}/src/lib.rs (100%) rename {tdx-attest => dstack/tdx-attest}/src/linux.rs (100%) rename {tdx-attest => dstack/tdx-attest}/src/snapshots/tdx_attest__eventlog__tests__parse_ccel-2.snap (100%) rename {tdx-attest => dstack/tdx-attest}/src/snapshots/tdx_attest__eventlog__tests__parse_ccel.snap (100%) rename {test-scripts => dstack/test-scripts}/get-app-key.sh (100%) rename {test-scripts => dstack/test-scripts}/inspect-cert.sh (100%) rename {test-scripts => dstack/test-scripts}/snp-e2e-smoke.sh (100%) rename {tests => dstack/tests}/docs/kms-bootstrap-onboard.md (100%) rename {tests => dstack/tests}/docs/kms-self-authorization.md (100%) rename {tpm-attest => dstack/tpm-attest}/Cargo.toml (100%) rename {tpm-attest => dstack/tpm-attest}/src/esapi.rs (100%) rename {tpm-attest => dstack/tpm-attest}/src/gcp_ak.rs (100%) rename {tpm-attest => dstack/tpm-attest}/src/lib.rs (100%) rename {tpm-attest => dstack/tpm-attest}/tests/tpm_quote_sample.bin (100%) rename {tpm-qvl => dstack/tpm-qvl}/Cargo.toml (100%) rename {tpm-qvl => dstack/tpm-qvl}/certs/AWS_NitroEnclaves_Root-G1.pem (100%) rename {tpm-qvl => dstack/tpm-qvl}/certs/gcp-root-ca.pem (100%) rename {tpm-qvl => dstack/tpm-qvl}/src/collateral.rs (100%) rename {tpm-qvl => dstack/tpm-qvl}/src/lib.rs (100%) rename {tpm-qvl => dstack/tpm-qvl}/src/verify.rs (100%) rename {tpm-types => dstack/tpm-types}/Cargo.toml (100%) rename {tpm-types => dstack/tpm-types}/src/lib.rs (100%) rename {tpm2 => dstack/tpm2}/Cargo.toml (100%) rename {tpm2 => dstack/tpm2}/src/bin/tpm2-test.rs (100%) rename {tpm2 => dstack/tpm2}/src/commands.rs (100%) rename {tpm2 => dstack/tpm2}/src/constants.rs (100%) rename {tpm2 => dstack/tpm2}/src/device.rs (100%) rename {tpm2 => dstack/tpm2}/src/lib.rs (100%) rename {tpm2 => dstack/tpm2}/src/marshal.rs (100%) rename {tpm2 => dstack/tpm2}/src/session.rs (100%) rename {tpm2 => dstack/tpm2}/src/types.rs (100%) rename {verifier => dstack/verifier}/Cargo.toml (100%) rename {verifier => dstack/verifier}/README.md (100%) rename {verifier => dstack/verifier}/builder/Dockerfile (100%) rename {verifier => dstack/verifier}/builder/build-image.sh (100%) rename {verifier => dstack/verifier}/builder/shared/builder-pinned-packages.txt (100%) rename {verifier => dstack/verifier}/builder/shared/pinned-packages.txt (100%) rename {verifier => dstack/verifier}/builder/shared/qemu-pinned-packages.txt (100%) rename {verifier => dstack/verifier}/dstack-verifier.toml (100%) rename {verifier => dstack/verifier}/fixtures/quote-report.json (100%) rename {verifier => dstack/verifier}/fixtures/sev-snp-attestation.json (100%) rename {verifier => dstack/verifier}/fixtures/sev-snp.README.md (100%) rename {verifier => dstack/verifier}/fixtures/tdx-lite-attestation.json (100%) rename {verifier => dstack/verifier}/fixtures/tdx-lite-getquote.json (100%) rename {verifier => dstack/verifier}/fixtures/tdx-lite.README.md (100%) rename {verifier => dstack/verifier}/src/lib.rs (100%) rename {verifier => dstack/verifier}/src/main.rs (100%) rename {verifier => dstack/verifier}/src/types.rs (100%) rename {verifier => dstack/verifier}/src/verification.rs (100%) rename {verifier => dstack/verifier}/test.sh (100%) rename {vmm => dstack/vmm}/Cargo.toml (100%) rename {vmm => dstack/vmm}/build.rs (100%) rename {vmm => dstack/vmm}/requirements.txt (100%) rename {vmm => dstack/vmm}/rpc/Cargo.toml (100%) rename {vmm => dstack/vmm}/rpc/build.rs (100%) rename {vmm => dstack/vmm}/rpc/proto/prpc.proto (100%) rename {vmm => dstack/vmm}/rpc/proto/vmm_rpc.proto (100%) rename {vmm => dstack/vmm}/rpc/src/generated.rs (100%) rename {vmm => dstack/vmm}/rpc/src/lib.rs (100%) rename {vmm => dstack/vmm}/src/app.rs (100%) rename {vmm => dstack/vmm}/src/app/id_pool.rs (100%) rename {vmm => dstack/vmm}/src/app/image.rs (100%) rename {vmm => dstack/vmm}/src/app/qemu.rs (100%) rename {vmm => dstack/vmm}/src/app/registry.rs (100%) rename {vmm => dstack/vmm}/src/config.rs (100%) rename {vmm => dstack/vmm}/src/console_v0.html (100%) rename {vmm => dstack/vmm}/src/discovery.rs (100%) rename {vmm => dstack/vmm}/src/guest_api_service.rs (100%) rename {vmm => dstack/vmm}/src/host_api_service.rs (100%) rename {vmm => dstack/vmm}/src/main.rs (100%) rename {vmm => dstack/vmm}/src/main_routes.rs (100%) rename {vmm => dstack/vmm}/src/main_service.rs (100%) rename {vmm => dstack/vmm}/src/one_shot.rs (100%) rename {vmm => dstack/vmm}/src/openapi.rs (100%) rename {vmm => dstack/vmm}/src/setup-user.sh (100%) rename {vmm => dstack/vmm}/src/tests/test-compose.sh (100%) rename {vmm => dstack/vmm}/src/tests/test-deployment.sh (100%) rename {vmm => dstack/vmm}/src/vmm-cli.py (100%) rename {vmm => dstack/vmm}/src/x25519.js (100%) rename {vmm => dstack/vmm}/ui/.gitignore (100%) rename {vmm => dstack/vmm}/ui/README.md (100%) rename {vmm => dstack/vmm}/ui/build.mjs (100%) rename {vmm => dstack/vmm}/ui/package-lock.json (100%) rename {vmm => dstack/vmm}/ui/package.json (100%) rename {vmm => dstack/vmm}/ui/scripts/build_proto.sh (100%) rename {vmm => dstack/vmm}/ui/src/App.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/CreateVmDialog.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/EncryptedEnvEditor.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/ForkVmDialog.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/GpuConfigEditor.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/PortMappingEditor.ts (100%) rename {vmm => dstack/vmm}/ui/src/components/UpdateVmDialog.ts (100%) rename {vmm => dstack/vmm}/ui/src/composables/useVmManager.ts (100%) rename {vmm => dstack/vmm}/ui/src/index.html (100%) rename {vmm => dstack/vmm}/ui/src/lib/vmmRpcClient.ts (100%) rename {vmm => dstack/vmm}/ui/src/lib/x25519.js (100%) rename {vmm => dstack/vmm}/ui/src/main.ts (100%) rename {vmm => dstack/vmm}/ui/src/styles/main.css (100%) rename {vmm => dstack/vmm}/ui/src/templates/app.html (100%) rename {vmm => dstack/vmm}/ui/tsconfig.json (100%) rename {vmm => dstack/vmm}/ui/vendor/README.md (100%) rename {vmm => dstack/vmm}/venv.sh (100%) rename {vmm => dstack/vmm}/vmm.toml (100%) diff --git a/.gitmodules b/.gitmodules index 2dc8dbacd..dbb997779 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 [submodule "kms/auth-eth/lib/forge-std"] - path = kms/auth-eth/lib/forge-std + path = dstack/kms/auth-eth/lib/forge-std url = https://github.com/foundry-rs/forge-std [submodule "kms/auth-eth/lib/openzeppelin-contracts-upgradeable"] - path = kms/auth-eth/lib/openzeppelin-contracts-upgradeable + path = dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable url = https://github.com/OpenZeppelin/openzeppelin-contracts-upgradeable [submodule "kms/auth-eth/lib/openzeppelin-foundry-upgrades"] - path = kms/auth-eth/lib/openzeppelin-foundry-upgrades + path = dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades url = https://github.com/OpenZeppelin/openzeppelin-foundry-upgrades diff --git a/Cargo.lock b/dstack/Cargo.lock similarity index 100% rename from Cargo.lock rename to dstack/Cargo.lock diff --git a/Cargo.toml b/dstack/Cargo.toml similarity index 100% rename from Cargo.toml rename to dstack/Cargo.toml diff --git a/Makefile b/dstack/Makefile similarity index 100% rename from Makefile rename to dstack/Makefile diff --git a/basefiles/app-compose.service b/dstack/basefiles/app-compose.service similarity index 100% rename from basefiles/app-compose.service rename to dstack/basefiles/app-compose.service diff --git a/basefiles/app-compose.sh b/dstack/basefiles/app-compose.sh similarity index 100% rename from basefiles/app-compose.sh rename to dstack/basefiles/app-compose.sh diff --git a/basefiles/containerd.service.d/dstack-prepare.conf b/dstack/basefiles/containerd.service.d/dstack-prepare.conf similarity index 100% rename from basefiles/containerd.service.d/dstack-prepare.conf rename to dstack/basefiles/containerd.service.d/dstack-prepare.conf diff --git a/basefiles/docker.service.d/dstack-guest-agent.conf b/dstack/basefiles/docker.service.d/dstack-guest-agent.conf similarity index 100% rename from basefiles/docker.service.d/dstack-guest-agent.conf rename to dstack/basefiles/docker.service.d/dstack-guest-agent.conf diff --git a/basefiles/docker.service.d/dstack-prepare.conf b/dstack/basefiles/docker.service.d/dstack-prepare.conf similarity index 100% rename from basefiles/docker.service.d/dstack-prepare.conf rename to dstack/basefiles/docker.service.d/dstack-prepare.conf diff --git a/basefiles/dstack-guest-agent.service b/dstack/basefiles/dstack-guest-agent.service similarity index 100% rename from basefiles/dstack-guest-agent.service rename to dstack/basefiles/dstack-guest-agent.service diff --git a/basefiles/dstack-guest-agent.socket b/dstack/basefiles/dstack-guest-agent.socket similarity index 100% rename from basefiles/dstack-guest-agent.socket rename to dstack/basefiles/dstack-guest-agent.socket diff --git a/basefiles/dstack-prepare.service b/dstack/basefiles/dstack-prepare.service similarity index 100% rename from basefiles/dstack-prepare.service rename to dstack/basefiles/dstack-prepare.service diff --git a/basefiles/dstack-prepare.sh b/dstack/basefiles/dstack-prepare.sh similarity index 100% rename from basefiles/dstack-prepare.sh rename to dstack/basefiles/dstack-prepare.sh diff --git a/basefiles/ephemeral-docker.sh b/dstack/basefiles/ephemeral-docker.sh similarity index 100% rename from basefiles/ephemeral-docker.sh rename to dstack/basefiles/ephemeral-docker.sh diff --git a/basefiles/journald.conf b/dstack/basefiles/journald.conf similarity index 100% rename from basefiles/journald.conf rename to dstack/basefiles/journald.conf diff --git a/basefiles/llmnr.conf b/dstack/basefiles/llmnr.conf similarity index 100% rename from basefiles/llmnr.conf rename to dstack/basefiles/llmnr.conf diff --git a/basefiles/sysctl.d/99-dstack.conf b/dstack/basefiles/sysctl.d/99-dstack.conf similarity index 100% rename from basefiles/sysctl.d/99-dstack.conf rename to dstack/basefiles/sysctl.d/99-dstack.conf diff --git a/basefiles/tdx-attest.conf b/dstack/basefiles/tdx-attest.conf similarity index 100% rename from basefiles/tdx-attest.conf rename to dstack/basefiles/tdx-attest.conf diff --git a/basefiles/wg-checker.service b/dstack/basefiles/wg-checker.service similarity index 100% rename from basefiles/wg-checker.service rename to dstack/basefiles/wg-checker.service diff --git a/basefiles/wg-checker.sh b/dstack/basefiles/wg-checker.sh similarity index 100% rename from basefiles/wg-checker.sh rename to dstack/basefiles/wg-checker.sh diff --git a/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh similarity index 100% rename from build/shared/build-lib.sh rename to dstack/build/shared/build-lib.sh diff --git a/build/shared/config-qemu.sh b/dstack/build/shared/config-qemu.sh similarity index 100% rename from build/shared/config-qemu.sh rename to dstack/build/shared/config-qemu.sh diff --git a/build/shared/pin-packages.sh b/dstack/build/shared/pin-packages.sh similarity index 100% rename from build/shared/pin-packages.sh rename to dstack/build/shared/pin-packages.sh diff --git a/build/shared/verify-pinned-packages.sh b/dstack/build/shared/verify-pinned-packages.sh similarity index 100% rename from build/shared/verify-pinned-packages.sh rename to dstack/build/shared/verify-pinned-packages.sh diff --git a/cached-cell/Cargo.toml b/dstack/cached-cell/Cargo.toml similarity index 100% rename from cached-cell/Cargo.toml rename to dstack/cached-cell/Cargo.toml diff --git a/cached-cell/src/lib.rs b/dstack/cached-cell/src/lib.rs similarity index 100% rename from cached-cell/src/lib.rs rename to dstack/cached-cell/src/lib.rs diff --git a/cargo-check-all.sh b/dstack/cargo-check-all.sh similarity index 100% rename from cargo-check-all.sh rename to dstack/cargo-check-all.sh diff --git a/cc-eventlog/Cargo.toml b/dstack/cc-eventlog/Cargo.toml similarity index 100% rename from cc-eventlog/Cargo.toml rename to dstack/cc-eventlog/Cargo.toml diff --git a/cc-eventlog/samples/ccel.bin b/dstack/cc-eventlog/samples/ccel.bin similarity index 100% rename from cc-eventlog/samples/ccel.bin rename to dstack/cc-eventlog/samples/ccel.bin diff --git a/cc-eventlog/samples/tpm_eventlog.bin b/dstack/cc-eventlog/samples/tpm_eventlog.bin similarity index 100% rename from cc-eventlog/samples/tpm_eventlog.bin rename to dstack/cc-eventlog/samples/tpm_eventlog.bin diff --git a/cc-eventlog/src/codecs.rs b/dstack/cc-eventlog/src/codecs.rs similarity index 100% rename from cc-eventlog/src/codecs.rs rename to dstack/cc-eventlog/src/codecs.rs diff --git a/cc-eventlog/src/lib.rs b/dstack/cc-eventlog/src/lib.rs similarity index 100% rename from cc-eventlog/src/lib.rs rename to dstack/cc-eventlog/src/lib.rs diff --git a/cc-eventlog/src/runtime_events.rs b/dstack/cc-eventlog/src/runtime_events.rs similarity index 100% rename from cc-eventlog/src/runtime_events.rs rename to dstack/cc-eventlog/src/runtime_events.rs diff --git a/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel-2.snap b/dstack/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel-2.snap similarity index 100% rename from cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel-2.snap rename to dstack/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel-2.snap diff --git a/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel.snap b/dstack/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel.snap similarity index 100% rename from cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel.snap rename to dstack/cc-eventlog/src/snapshots/cc_eventlog__tests__parse_ccel.snap diff --git a/cc-eventlog/src/tcg.rs b/dstack/cc-eventlog/src/tcg.rs similarity index 100% rename from cc-eventlog/src/tcg.rs rename to dstack/cc-eventlog/src/tcg.rs diff --git a/cc-eventlog/src/tdx.rs b/dstack/cc-eventlog/src/tdx.rs similarity index 100% rename from cc-eventlog/src/tdx.rs rename to dstack/cc-eventlog/src/tdx.rs diff --git a/cc-eventlog/src/tpm.rs b/dstack/cc-eventlog/src/tpm.rs similarity index 100% rename from cc-eventlog/src/tpm.rs rename to dstack/cc-eventlog/src/tpm.rs diff --git a/cert-client/Cargo.toml b/dstack/cert-client/Cargo.toml similarity index 100% rename from cert-client/Cargo.toml rename to dstack/cert-client/Cargo.toml diff --git a/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs similarity index 100% rename from cert-client/src/lib.rs rename to dstack/cert-client/src/lib.rs diff --git a/certbot/.gitignore b/dstack/certbot/.gitignore similarity index 100% rename from certbot/.gitignore rename to dstack/certbot/.gitignore diff --git a/certbot/Cargo.toml b/dstack/certbot/Cargo.toml similarity index 100% rename from certbot/Cargo.toml rename to dstack/certbot/Cargo.toml diff --git a/certbot/cli/Cargo.toml b/dstack/certbot/cli/Cargo.toml similarity index 100% rename from certbot/cli/Cargo.toml rename to dstack/certbot/cli/Cargo.toml diff --git a/certbot/cli/src/main.rs b/dstack/certbot/cli/src/main.rs similarity index 100% rename from certbot/cli/src/main.rs rename to dstack/certbot/cli/src/main.rs diff --git a/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs similarity index 100% rename from certbot/src/acme_client.rs rename to dstack/certbot/src/acme_client.rs diff --git a/certbot/src/acme_client/tests.rs b/dstack/certbot/src/acme_client/tests.rs similarity index 100% rename from certbot/src/acme_client/tests.rs rename to dstack/certbot/src/acme_client/tests.rs diff --git a/certbot/src/bot.rs b/dstack/certbot/src/bot.rs similarity index 100% rename from certbot/src/bot.rs rename to dstack/certbot/src/bot.rs diff --git a/certbot/src/bot/tests.rs b/dstack/certbot/src/bot/tests.rs similarity index 100% rename from certbot/src/bot/tests.rs rename to dstack/certbot/src/bot/tests.rs diff --git a/certbot/src/dns01_client.rs b/dstack/certbot/src/dns01_client.rs similarity index 100% rename from certbot/src/dns01_client.rs rename to dstack/certbot/src/dns01_client.rs diff --git a/certbot/src/dns01_client/cloudflare.rs b/dstack/certbot/src/dns01_client/cloudflare.rs similarity index 100% rename from certbot/src/dns01_client/cloudflare.rs rename to dstack/certbot/src/dns01_client/cloudflare.rs diff --git a/certbot/src/http_client.rs b/dstack/certbot/src/http_client.rs similarity index 100% rename from certbot/src/http_client.rs rename to dstack/certbot/src/http_client.rs diff --git a/certbot/src/lib.rs b/dstack/certbot/src/lib.rs similarity index 100% rename from certbot/src/lib.rs rename to dstack/certbot/src/lib.rs diff --git a/certbot/src/workdir.rs b/dstack/certbot/src/workdir.rs similarity index 100% rename from certbot/src/workdir.rs rename to dstack/certbot/src/workdir.rs diff --git a/crates/dstack-auth/Cargo.toml b/dstack/crates/dstack-auth/Cargo.toml similarity index 100% rename from crates/dstack-auth/Cargo.toml rename to dstack/crates/dstack-auth/Cargo.toml diff --git a/crates/dstack-auth/src/main.rs b/dstack/crates/dstack-auth/src/main.rs similarity index 100% rename from crates/dstack-auth/src/main.rs rename to dstack/crates/dstack-auth/src/main.rs diff --git a/crates/dstack-cli-core/Cargo.toml b/dstack/crates/dstack-cli-core/Cargo.toml similarity index 100% rename from crates/dstack-cli-core/Cargo.toml rename to dstack/crates/dstack-cli-core/Cargo.toml diff --git a/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs similarity index 100% rename from crates/dstack-cli-core/src/compose.rs rename to dstack/crates/dstack-cli-core/src/compose.rs diff --git a/crates/dstack-cli-core/src/config.rs b/dstack/crates/dstack-cli-core/src/config.rs similarity index 100% rename from crates/dstack-cli-core/src/config.rs rename to dstack/crates/dstack-cli-core/src/config.rs diff --git a/crates/dstack-cli-core/src/fsutil.rs b/dstack/crates/dstack-cli-core/src/fsutil.rs similarity index 100% rename from crates/dstack-cli-core/src/fsutil.rs rename to dstack/crates/dstack-cli-core/src/fsutil.rs diff --git a/crates/dstack-cli-core/src/host.rs b/dstack/crates/dstack-cli-core/src/host.rs similarity index 100% rename from crates/dstack-cli-core/src/host.rs rename to dstack/crates/dstack-cli-core/src/host.rs diff --git a/crates/dstack-cli-core/src/layout.rs b/dstack/crates/dstack-cli-core/src/layout.rs similarity index 100% rename from crates/dstack-cli-core/src/layout.rs rename to dstack/crates/dstack-cli-core/src/layout.rs diff --git a/crates/dstack-cli-core/src/lib.rs b/dstack/crates/dstack-cli-core/src/lib.rs similarity index 100% rename from crates/dstack-cli-core/src/lib.rs rename to dstack/crates/dstack-cli-core/src/lib.rs diff --git a/crates/dstack-cli-core/src/ports.rs b/dstack/crates/dstack-cli-core/src/ports.rs similarity index 100% rename from crates/dstack-cli-core/src/ports.rs rename to dstack/crates/dstack-cli-core/src/ports.rs diff --git a/crates/dstack-cli-core/src/vmm.rs b/dstack/crates/dstack-cli-core/src/vmm.rs similarity index 100% rename from crates/dstack-cli-core/src/vmm.rs rename to dstack/crates/dstack-cli-core/src/vmm.rs diff --git a/crates/dstack-cli/Cargo.toml b/dstack/crates/dstack-cli/Cargo.toml similarity index 100% rename from crates/dstack-cli/Cargo.toml rename to dstack/crates/dstack-cli/Cargo.toml diff --git a/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs similarity index 100% rename from crates/dstack-cli/src/main.rs rename to dstack/crates/dstack-cli/src/main.rs diff --git a/crates/dstackup/Cargo.toml b/dstack/crates/dstackup/Cargo.toml similarity index 100% rename from crates/dstackup/Cargo.toml rename to dstack/crates/dstackup/Cargo.toml diff --git a/crates/dstackup/src/cid.rs b/dstack/crates/dstackup/src/cid.rs similarity index 100% rename from crates/dstackup/src/cid.rs rename to dstack/crates/dstackup/src/cid.rs diff --git a/crates/dstackup/src/cli.rs b/dstack/crates/dstackup/src/cli.rs similarity index 100% rename from crates/dstackup/src/cli.rs rename to dstack/crates/dstackup/src/cli.rs diff --git a/crates/dstackup/src/destroy.rs b/dstack/crates/dstackup/src/destroy.rs similarity index 100% rename from crates/dstackup/src/destroy.rs rename to dstack/crates/dstackup/src/destroy.rs diff --git a/crates/dstackup/src/image.rs b/dstack/crates/dstackup/src/image.rs similarity index 100% rename from crates/dstackup/src/image.rs rename to dstack/crates/dstackup/src/image.rs diff --git a/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs similarity index 100% rename from crates/dstackup/src/install.rs rename to dstack/crates/dstackup/src/install.rs diff --git a/crates/dstackup/src/main.rs b/dstack/crates/dstackup/src/main.rs similarity index 100% rename from crates/dstackup/src/main.rs rename to dstack/crates/dstackup/src/main.rs diff --git a/crates/dstackup/src/state.rs b/dstack/crates/dstackup/src/state.rs similarity index 100% rename from crates/dstackup/src/state.rs rename to dstack/crates/dstackup/src/state.rs diff --git a/crates/dstackup/src/systemd.rs b/dstack/crates/dstackup/src/systemd.rs similarity index 100% rename from crates/dstackup/src/systemd.rs rename to dstack/crates/dstackup/src/systemd.rs diff --git a/ct_monitor/Cargo.toml b/dstack/ct_monitor/Cargo.toml similarity index 100% rename from ct_monitor/Cargo.toml rename to dstack/ct_monitor/Cargo.toml diff --git a/ct_monitor/src/main.rs b/dstack/ct_monitor/src/main.rs similarity index 100% rename from ct_monitor/src/main.rs rename to dstack/ct_monitor/src/main.rs diff --git a/dstack-attest/Cargo.toml b/dstack/dstack-attest/Cargo.toml similarity index 100% rename from dstack-attest/Cargo.toml rename to dstack/dstack-attest/Cargo.toml diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack/dstack-attest/src/amd_sev_snp.rs similarity index 100% rename from dstack-attest/src/amd_sev_snp.rs rename to dstack/dstack-attest/src/amd_sev_snp.rs diff --git a/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs similarity index 100% rename from dstack-attest/src/attestation.rs rename to dstack/dstack-attest/src/attestation.rs diff --git a/dstack-attest/src/lib.rs b/dstack/dstack-attest/src/lib.rs similarity index 100% rename from dstack-attest/src/lib.rs rename to dstack/dstack-attest/src/lib.rs diff --git a/dstack-attest/src/sev_snp.rs b/dstack/dstack-attest/src/sev_snp.rs similarity index 100% rename from dstack-attest/src/sev_snp.rs rename to dstack/dstack-attest/src/sev_snp.rs diff --git a/dstack-attest/src/v1.rs b/dstack/dstack-attest/src/v1.rs similarity index 100% rename from dstack-attest/src/v1.rs rename to dstack/dstack-attest/src/v1.rs diff --git a/dstack-attest/tests/nitro_attestation.bin b/dstack/dstack-attest/tests/nitro_attestation.bin similarity index 100% rename from dstack-attest/tests/nitro_attestation.bin rename to dstack/dstack-attest/tests/nitro_attestation.bin diff --git a/dstack-attest/tests/nitro_attestation_dbg.bin b/dstack/dstack-attest/tests/nitro_attestation_dbg.bin similarity index 100% rename from dstack-attest/tests/nitro_attestation_dbg.bin rename to dstack/dstack-attest/tests/nitro_attestation_dbg.bin diff --git a/dstack-attest/tests/nitro_verify.rs b/dstack/dstack-attest/tests/nitro_verify.rs similarity index 100% rename from dstack-attest/tests/nitro_verify.rs rename to dstack/dstack-attest/tests/nitro_verify.rs diff --git a/dstack-attest/tests/sev_snp_ask.pem b/dstack/dstack-attest/tests/sev_snp_ask.pem similarity index 100% rename from dstack-attest/tests/sev_snp_ask.pem rename to dstack/dstack-attest/tests/sev_snp_ask.pem diff --git a/dstack-attest/tests/sev_snp_attestation.bin b/dstack/dstack-attest/tests/sev_snp_attestation.bin similarity index 100% rename from dstack-attest/tests/sev_snp_attestation.bin rename to dstack/dstack-attest/tests/sev_snp_attestation.bin diff --git a/dstack-attest/tests/sev_snp_fixture.README.md b/dstack/dstack-attest/tests/sev_snp_fixture.README.md similarity index 100% rename from dstack-attest/tests/sev_snp_fixture.README.md rename to dstack/dstack-attest/tests/sev_snp_fixture.README.md diff --git a/dstack-attest/tests/sev_snp_vcek.pem b/dstack/dstack-attest/tests/sev_snp_vcek.pem similarity index 100% rename from dstack-attest/tests/sev_snp_vcek.pem rename to dstack/dstack-attest/tests/sev_snp_vcek.pem diff --git a/dstack-attest/tests/sev_snp_verify.rs b/dstack/dstack-attest/tests/sev_snp_verify.rs similarity index 100% rename from dstack-attest/tests/sev_snp_verify.rs rename to dstack/dstack-attest/tests/sev_snp_verify.rs diff --git a/dstack-attest/tests/snapshots/nitro_verify__app_info.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap similarity index 100% rename from dstack-attest/tests/snapshots/nitro_verify__app_info.snap rename to dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap diff --git a/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap similarity index 100% rename from dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap rename to dstack/dstack-attest/tests/snapshots/nitro_verify__nitro_report.snap diff --git a/dstack-mr/.gitignore b/dstack/dstack-mr/.gitignore similarity index 100% rename from dstack-mr/.gitignore rename to dstack/dstack-mr/.gitignore diff --git a/dstack-mr/Cargo.toml b/dstack/dstack-mr/Cargo.toml similarity index 100% rename from dstack-mr/Cargo.toml rename to dstack/dstack-mr/Cargo.toml diff --git a/dstack-mr/cli/Cargo.toml b/dstack/dstack-mr/cli/Cargo.toml similarity index 100% rename from dstack-mr/cli/Cargo.toml rename to dstack/dstack-mr/cli/Cargo.toml diff --git a/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs similarity index 100% rename from dstack-mr/cli/src/main.rs rename to dstack/dstack-mr/cli/src/main.rs diff --git a/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs similarity index 100% rename from dstack-mr/src/acpi.rs rename to dstack/dstack-mr/src/acpi.rs diff --git a/dstack-mr/src/kernel.rs b/dstack/dstack-mr/src/kernel.rs similarity index 100% rename from dstack-mr/src/kernel.rs rename to dstack/dstack-mr/src/kernel.rs diff --git a/dstack-mr/src/lib.rs b/dstack/dstack-mr/src/lib.rs similarity index 100% rename from dstack-mr/src/lib.rs rename to dstack/dstack-mr/src/lib.rs diff --git a/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs similarity index 100% rename from dstack-mr/src/machine.rs rename to dstack/dstack-mr/src/machine.rs diff --git a/dstack-mr/src/main.rs b/dstack/dstack-mr/src/main.rs similarity index 100% rename from dstack-mr/src/main.rs rename to dstack/dstack-mr/src/main.rs diff --git a/dstack-mr/src/measurement.rs b/dstack/dstack-mr/src/measurement.rs similarity index 100% rename from dstack-mr/src/measurement.rs rename to dstack/dstack-mr/src/measurement.rs diff --git a/dstack-mr/src/num.rs b/dstack/dstack-mr/src/num.rs similarity index 100% rename from dstack-mr/src/num.rs rename to dstack/dstack-mr/src/num.rs diff --git a/dstack-mr/src/sev.rs b/dstack/dstack-mr/src/sev.rs similarity index 100% rename from dstack-mr/src/sev.rs rename to dstack/dstack-mr/src/sev.rs diff --git a/dstack-mr/src/tdvf.rs b/dstack/dstack-mr/src/tdvf.rs similarity index 100% rename from dstack-mr/src/tdvf.rs rename to dstack/dstack-mr/src/tdvf.rs diff --git a/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs similarity index 100% rename from dstack-mr/src/tdx.rs rename to dstack/dstack-mr/src/tdx.rs diff --git a/dstack-mr/src/util.rs b/dstack/dstack-mr/src/util.rs similarity index 100% rename from dstack-mr/src/util.rs rename to dstack/dstack-mr/src/util.rs diff --git a/dstack-mr/tests/tdvf_parse.rs b/dstack/dstack-mr/tests/tdvf_parse.rs similarity index 100% rename from dstack-mr/tests/tdvf_parse.rs rename to dstack/dstack-mr/tests/tdvf_parse.rs diff --git a/dstack-types/Cargo.toml b/dstack/dstack-types/Cargo.toml similarity index 100% rename from dstack-types/Cargo.toml rename to dstack/dstack-types/Cargo.toml diff --git a/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs similarity index 100% rename from dstack-types/src/lib.rs rename to dstack/dstack-types/src/lib.rs diff --git a/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs similarity index 100% rename from dstack-types/src/mr_config.rs rename to dstack/dstack-types/src/mr_config.rs diff --git a/dstack-types/src/shared_filenames.rs b/dstack/dstack-types/src/shared_filenames.rs similarity index 100% rename from dstack-types/src/shared_filenames.rs rename to dstack/dstack-types/src/shared_filenames.rs diff --git a/dstack-types/src/version.rs b/dstack/dstack-types/src/version.rs similarity index 100% rename from dstack-types/src/version.rs rename to dstack/dstack-types/src/version.rs diff --git a/dstack-util/Cargo.toml b/dstack/dstack-util/Cargo.toml similarity index 100% rename from dstack-util/Cargo.toml rename to dstack/dstack-util/Cargo.toml diff --git a/dstack-util/src/crypto.rs b/dstack/dstack-util/src/crypto.rs similarity index 100% rename from dstack-util/src/crypto.rs rename to dstack/dstack-util/src/crypto.rs diff --git a/dstack-util/src/docker_compose.rs b/dstack/dstack-util/src/docker_compose.rs similarity index 100% rename from dstack-util/src/docker_compose.rs rename to dstack/dstack-util/src/docker_compose.rs diff --git a/dstack-util/src/host_api.rs b/dstack/dstack-util/src/host_api.rs similarity index 100% rename from dstack-util/src/host_api.rs rename to dstack/dstack-util/src/host_api.rs diff --git a/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs similarity index 100% rename from dstack-util/src/main.rs rename to dstack/dstack-util/src/main.rs diff --git a/dstack-util/src/parse_env_file.rs b/dstack/dstack-util/src/parse_env_file.rs similarity index 100% rename from dstack-util/src/parse_env_file.rs rename to dstack/dstack-util/src/parse_env_file.rs diff --git a/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs similarity index 100% rename from dstack-util/src/system_setup.rs rename to dstack/dstack-util/src/system_setup.rs diff --git a/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs similarity index 100% rename from dstack-util/src/system_setup/config_id_verifier.rs rename to dstack/dstack-util/src/system_setup/config_id_verifier.rs diff --git a/dstack-util/src/utils.rs b/dstack/dstack-util/src/utils.rs similarity index 100% rename from dstack-util/src/utils.rs rename to dstack/dstack-util/src/utils.rs diff --git a/dstack-util/tests/fixtures/key-provider-docker-compose.yaml b/dstack/dstack-util/tests/fixtures/key-provider-docker-compose.yaml similarity index 100% rename from dstack-util/tests/fixtures/key-provider-docker-compose.yaml rename to dstack/dstack-util/tests/fixtures/key-provider-docker-compose.yaml diff --git a/dstack-util/tests/fixtures/luks_header_cipher_null b/dstack/dstack-util/tests/fixtures/luks_header_cipher_null similarity index 100% rename from dstack-util/tests/fixtures/luks_header_cipher_null rename to dstack/dstack-util/tests/fixtures/luks_header_cipher_null diff --git a/dstack-util/tests/fixtures/luks_header_cipher_null.license b/dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license similarity index 100% rename from dstack-util/tests/fixtures/luks_header_cipher_null.license rename to dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license diff --git a/dstack-util/tests/fixtures/luks_header_good b/dstack/dstack-util/tests/fixtures/luks_header_good similarity index 100% rename from dstack-util/tests/fixtures/luks_header_good rename to dstack/dstack-util/tests/fixtures/luks_header_good diff --git a/dstack-util/tests/fixtures/luks_header_good.license b/dstack/dstack-util/tests/fixtures/luks_header_good.license similarity index 100% rename from dstack-util/tests/fixtures/luks_header_good.license rename to dstack/dstack-util/tests/fixtures/luks_header_good.license diff --git a/dstack-util/tests/test_remove_orphans.sh b/dstack/dstack-util/tests/test_remove_orphans.sh similarity index 100% rename from dstack-util/tests/test_remove_orphans.sh rename to dstack/dstack-util/tests/test_remove_orphans.sh diff --git a/gateway/Cargo.toml b/dstack/gateway/Cargo.toml similarity index 100% rename from gateway/Cargo.toml rename to dstack/gateway/Cargo.toml diff --git a/gateway/assets/cert.key b/dstack/gateway/assets/cert.key similarity index 100% rename from gateway/assets/cert.key rename to dstack/gateway/assets/cert.key diff --git a/gateway/assets/cert.pem b/dstack/gateway/assets/cert.pem similarity index 100% rename from gateway/assets/cert.pem rename to dstack/gateway/assets/cert.pem diff --git a/gateway/docs/cluster-deployment.md b/dstack/gateway/docs/cluster-deployment.md similarity index 100% rename from gateway/docs/cluster-deployment.md rename to dstack/gateway/docs/cluster-deployment.md diff --git a/gateway/dstack-app/.gitignore b/dstack/gateway/dstack-app/.gitignore similarity index 100% rename from gateway/dstack-app/.gitignore rename to dstack/gateway/dstack-app/.gitignore diff --git a/gateway/dstack-app/bootstrap-cluster.sh b/dstack/gateway/dstack-app/bootstrap-cluster.sh similarity index 100% rename from gateway/dstack-app/bootstrap-cluster.sh rename to dstack/gateway/dstack-app/bootstrap-cluster.sh diff --git a/gateway/dstack-app/builder/Dockerfile b/dstack/gateway/dstack-app/builder/Dockerfile similarity index 100% rename from gateway/dstack-app/builder/Dockerfile rename to dstack/gateway/dstack-app/builder/Dockerfile diff --git a/gateway/dstack-app/builder/README.md b/dstack/gateway/dstack-app/builder/README.md similarity index 100% rename from gateway/dstack-app/builder/README.md rename to dstack/gateway/dstack-app/builder/README.md diff --git a/gateway/dstack-app/builder/build-image.sh b/dstack/gateway/dstack-app/builder/build-image.sh similarity index 100% rename from gateway/dstack-app/builder/build-image.sh rename to dstack/gateway/dstack-app/builder/build-image.sh diff --git a/gateway/dstack-app/builder/entrypoint.sh b/dstack/gateway/dstack-app/builder/entrypoint.sh similarity index 100% rename from gateway/dstack-app/builder/entrypoint.sh rename to dstack/gateway/dstack-app/builder/entrypoint.sh diff --git a/gateway/dstack-app/builder/shared/builder-pinned-packages.txt b/dstack/gateway/dstack-app/builder/shared/builder-pinned-packages.txt similarity index 100% rename from gateway/dstack-app/builder/shared/builder-pinned-packages.txt rename to dstack/gateway/dstack-app/builder/shared/builder-pinned-packages.txt diff --git a/gateway/dstack-app/builder/shared/pinned-packages.txt b/dstack/gateway/dstack-app/builder/shared/pinned-packages.txt similarity index 100% rename from gateway/dstack-app/builder/shared/pinned-packages.txt rename to dstack/gateway/dstack-app/builder/shared/pinned-packages.txt diff --git a/gateway/dstack-app/deploy-to-vmm.sh b/dstack/gateway/dstack-app/deploy-to-vmm.sh similarity index 100% rename from gateway/dstack-app/deploy-to-vmm.sh rename to dstack/gateway/dstack-app/deploy-to-vmm.sh diff --git a/gateway/dstack-app/docker-compose.yaml b/dstack/gateway/dstack-app/docker-compose.yaml similarity index 100% rename from gateway/dstack-app/docker-compose.yaml rename to dstack/gateway/dstack-app/docker-compose.yaml diff --git a/gateway/gateway.toml b/dstack/gateway/gateway.toml similarity index 100% rename from gateway/gateway.toml rename to dstack/gateway/gateway.toml diff --git a/gateway/rpc/Cargo.toml b/dstack/gateway/rpc/Cargo.toml similarity index 100% rename from gateway/rpc/Cargo.toml rename to dstack/gateway/rpc/Cargo.toml diff --git a/gateway/rpc/build.rs b/dstack/gateway/rpc/build.rs similarity index 100% rename from gateway/rpc/build.rs rename to dstack/gateway/rpc/build.rs diff --git a/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto similarity index 100% rename from gateway/rpc/proto/gateway_rpc.proto rename to dstack/gateway/rpc/proto/gateway_rpc.proto diff --git a/gateway/rpc/src/generated.rs b/dstack/gateway/rpc/src/generated.rs similarity index 100% rename from gateway/rpc/src/generated.rs rename to dstack/gateway/rpc/src/generated.rs diff --git a/gateway/rpc/src/lib.rs b/dstack/gateway/rpc/src/lib.rs similarity index 100% rename from gateway/rpc/src/lib.rs rename to dstack/gateway/rpc/src/lib.rs diff --git a/gateway/src/admin_auth.rs b/dstack/gateway/src/admin_auth.rs similarity index 100% rename from gateway/src/admin_auth.rs rename to dstack/gateway/src/admin_auth.rs diff --git a/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs similarity index 100% rename from gateway/src/admin_service.rs rename to dstack/gateway/src/admin_service.rs diff --git a/gateway/src/cert_store.rs b/dstack/gateway/src/cert_store.rs similarity index 100% rename from gateway/src/cert_store.rs rename to dstack/gateway/src/cert_store.rs diff --git a/gateway/src/config.rs b/dstack/gateway/src/config.rs similarity index 100% rename from gateway/src/config.rs rename to dstack/gateway/src/config.rs diff --git a/gateway/src/debug_service.rs b/dstack/gateway/src/debug_service.rs similarity index 100% rename from gateway/src/debug_service.rs rename to dstack/gateway/src/debug_service.rs diff --git a/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs similarity index 100% rename from gateway/src/distributed_certbot.rs rename to dstack/gateway/src/distributed_certbot.rs diff --git a/gateway/src/gen_debug_key.rs b/dstack/gateway/src/gen_debug_key.rs similarity index 100% rename from gateway/src/gen_debug_key.rs rename to dstack/gateway/src/gen_debug_key.rs diff --git a/gateway/src/kv/https_client.rs b/dstack/gateway/src/kv/https_client.rs similarity index 100% rename from gateway/src/kv/https_client.rs rename to dstack/gateway/src/kv/https_client.rs diff --git a/gateway/src/kv/mod.rs b/dstack/gateway/src/kv/mod.rs similarity index 100% rename from gateway/src/kv/mod.rs rename to dstack/gateway/src/kv/mod.rs diff --git a/gateway/src/kv/sync_service.rs b/dstack/gateway/src/kv/sync_service.rs similarity index 100% rename from gateway/src/kv/sync_service.rs rename to dstack/gateway/src/kv/sync_service.rs diff --git a/gateway/src/main.rs b/dstack/gateway/src/main.rs similarity index 100% rename from gateway/src/main.rs rename to dstack/gateway/src/main.rs diff --git a/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs similarity index 100% rename from gateway/src/main_service.rs rename to dstack/gateway/src/main_service.rs diff --git a/gateway/src/main_service/auth_client.rs b/dstack/gateway/src/main_service/auth_client.rs similarity index 100% rename from gateway/src/main_service/auth_client.rs rename to dstack/gateway/src/main_service/auth_client.rs diff --git a/gateway/src/main_service/handshakes.rs b/dstack/gateway/src/main_service/handshakes.rs similarity index 100% rename from gateway/src/main_service/handshakes.rs rename to dstack/gateway/src/main_service/handshakes.rs diff --git a/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap similarity index 100% rename from gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap rename to dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-2.snap diff --git a/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap similarity index 100% rename from gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap rename to dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config-3.snap diff --git a/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap similarity index 100% rename from gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap rename to dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__config.snap diff --git a/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__empty_config.snap b/dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__empty_config.snap similarity index 100% rename from gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__empty_config.snap rename to dstack/gateway/src/main_service/snapshots/dstack_gateway__main_service__tests__empty_config.snap diff --git a/gateway/src/main_service/tests.rs b/dstack/gateway/src/main_service/tests.rs similarity index 100% rename from gateway/src/main_service/tests.rs rename to dstack/gateway/src/main_service/tests.rs diff --git a/gateway/src/models.rs b/dstack/gateway/src/models.rs similarity index 100% rename from gateway/src/models.rs rename to dstack/gateway/src/models.rs diff --git a/gateway/src/pp.rs b/dstack/gateway/src/pp.rs similarity index 100% rename from gateway/src/pp.rs rename to dstack/gateway/src/pp.rs diff --git a/gateway/src/proxy.rs b/dstack/gateway/src/proxy.rs similarity index 100% rename from gateway/src/proxy.rs rename to dstack/gateway/src/proxy.rs diff --git a/gateway/src/proxy/io_bridge.rs b/dstack/gateway/src/proxy/io_bridge.rs similarity index 100% rename from gateway/src/proxy/io_bridge.rs rename to dstack/gateway/src/proxy/io_bridge.rs diff --git a/gateway/src/proxy/port_policy.rs b/dstack/gateway/src/proxy/port_policy.rs similarity index 100% rename from gateway/src/proxy/port_policy.rs rename to dstack/gateway/src/proxy/port_policy.rs diff --git a/gateway/src/proxy/sni.rs b/dstack/gateway/src/proxy/sni.rs similarity index 100% rename from gateway/src/proxy/sni.rs rename to dstack/gateway/src/proxy/sni.rs diff --git a/gateway/src/proxy/tls_passthough.rs b/dstack/gateway/src/proxy/tls_passthough.rs similarity index 100% rename from gateway/src/proxy/tls_passthough.rs rename to dstack/gateway/src/proxy/tls_passthough.rs diff --git a/gateway/src/proxy/tls_terminate.rs b/dstack/gateway/src/proxy/tls_terminate.rs similarity index 100% rename from gateway/src/proxy/tls_terminate.rs rename to dstack/gateway/src/proxy/tls_terminate.rs diff --git a/gateway/src/web_routes.rs b/dstack/gateway/src/web_routes.rs similarity index 100% rename from gateway/src/web_routes.rs rename to dstack/gateway/src/web_routes.rs diff --git a/gateway/src/web_routes/route_index.rs b/dstack/gateway/src/web_routes/route_index.rs similarity index 100% rename from gateway/src/web_routes/route_index.rs rename to dstack/gateway/src/web_routes/route_index.rs diff --git a/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs similarity index 100% rename from gateway/src/web_routes/wavekv_sync.rs rename to dstack/gateway/src/web_routes/wavekv_sync.rs diff --git a/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html similarity index 100% rename from gateway/templates/dashboard.html rename to dstack/gateway/templates/dashboard.html diff --git a/gateway/templates/rproxy.yaml b/dstack/gateway/templates/rproxy.yaml similarity index 100% rename from gateway/templates/rproxy.yaml rename to dstack/gateway/templates/rproxy.yaml diff --git a/gateway/templates/wg.conf b/dstack/gateway/templates/wg.conf similarity index 100% rename from gateway/templates/wg.conf rename to dstack/gateway/templates/wg.conf diff --git a/gateway/test-run/.env.example b/dstack/gateway/test-run/.env.example similarity index 100% rename from gateway/test-run/.env.example rename to dstack/gateway/test-run/.env.example diff --git a/gateway/test-run/.gitignore b/dstack/gateway/test-run/.gitignore similarity index 100% rename from gateway/test-run/.gitignore rename to dstack/gateway/test-run/.gitignore diff --git a/gateway/test-run/TESTING.md b/dstack/gateway/test-run/TESTING.md similarity index 100% rename from gateway/test-run/TESTING.md rename to dstack/gateway/test-run/TESTING.md diff --git a/gateway/test-run/cluster.sh b/dstack/gateway/test-run/cluster.sh similarity index 100% rename from gateway/test-run/cluster.sh rename to dstack/gateway/test-run/cluster.sh diff --git a/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml similarity index 100% rename from gateway/test-run/e2e/configs/gateway-1.toml rename to dstack/gateway/test-run/e2e/configs/gateway-1.toml diff --git a/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml similarity index 100% rename from gateway/test-run/e2e/configs/gateway-2.toml rename to dstack/gateway/test-run/e2e/configs/gateway-2.toml diff --git a/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml similarity index 100% rename from gateway/test-run/e2e/configs/gateway-3.toml rename to dstack/gateway/test-run/e2e/configs/gateway-3.toml diff --git a/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml similarity index 100% rename from gateway/test-run/e2e/docker-compose.yml rename to dstack/gateway/test-run/e2e/docker-compose.yml diff --git a/gateway/test-run/e2e/pebble-config.json b/dstack/gateway/test-run/e2e/pebble-config.json similarity index 100% rename from gateway/test-run/e2e/pebble-config.json rename to dstack/gateway/test-run/e2e/pebble-config.json diff --git a/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh similarity index 100% rename from gateway/test-run/e2e/run-e2e.sh rename to dstack/gateway/test-run/e2e/run-e2e.sh diff --git a/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh similarity index 100% rename from gateway/test-run/e2e/test.sh rename to dstack/gateway/test-run/e2e/test.sh diff --git a/gateway/test-run/test_certbot.sh b/dstack/gateway/test-run/test_certbot.sh similarity index 100% rename from gateway/test-run/test_certbot.sh rename to dstack/gateway/test-run/test_certbot.sh diff --git a/gateway/test-run/test_suite.sh b/dstack/gateway/test-run/test_suite.sh similarity index 100% rename from gateway/test-run/test_suite.sh rename to dstack/gateway/test-run/test_suite.sh diff --git a/guest-agent-simulator/.gitignore b/dstack/guest-agent-simulator/.gitignore similarity index 100% rename from guest-agent-simulator/.gitignore rename to dstack/guest-agent-simulator/.gitignore diff --git a/guest-agent-simulator/Cargo.toml b/dstack/guest-agent-simulator/Cargo.toml similarity index 100% rename from guest-agent-simulator/Cargo.toml rename to dstack/guest-agent-simulator/Cargo.toml diff --git a/guest-agent-simulator/dstack-simulator.service b/dstack/guest-agent-simulator/dstack-simulator.service similarity index 100% rename from guest-agent-simulator/dstack-simulator.service rename to dstack/guest-agent-simulator/dstack-simulator.service diff --git a/guest-agent-simulator/dstack.toml b/dstack/guest-agent-simulator/dstack.toml similarity index 100% rename from guest-agent-simulator/dstack.toml rename to dstack/guest-agent-simulator/dstack.toml diff --git a/guest-agent-simulator/install-systemd.sh b/dstack/guest-agent-simulator/install-systemd.sh similarity index 100% rename from guest-agent-simulator/install-systemd.sh rename to dstack/guest-agent-simulator/install-systemd.sh diff --git a/guest-agent-simulator/package-release.sh b/dstack/guest-agent-simulator/package-release.sh similarity index 100% rename from guest-agent-simulator/package-release.sh rename to dstack/guest-agent-simulator/package-release.sh diff --git a/guest-agent-simulator/src/main.rs b/dstack/guest-agent-simulator/src/main.rs similarity index 100% rename from guest-agent-simulator/src/main.rs rename to dstack/guest-agent-simulator/src/main.rs diff --git a/guest-agent-simulator/src/simulator.rs b/dstack/guest-agent-simulator/src/simulator.rs similarity index 100% rename from guest-agent-simulator/src/simulator.rs rename to dstack/guest-agent-simulator/src/simulator.rs diff --git a/guest-agent/Cargo.toml b/dstack/guest-agent/Cargo.toml similarity index 100% rename from guest-agent/Cargo.toml rename to dstack/guest-agent/Cargo.toml diff --git a/guest-agent/dstack.toml b/dstack/guest-agent/dstack.toml similarity index 100% rename from guest-agent/dstack.toml rename to dstack/guest-agent/dstack.toml diff --git a/guest-agent/fixtures/attestation.bin b/dstack/guest-agent/fixtures/attestation.bin similarity index 100% rename from guest-agent/fixtures/attestation.bin rename to dstack/guest-agent/fixtures/attestation.bin diff --git a/guest-agent/rpc/Cargo.toml b/dstack/guest-agent/rpc/Cargo.toml similarity index 100% rename from guest-agent/rpc/Cargo.toml rename to dstack/guest-agent/rpc/Cargo.toml diff --git a/guest-agent/rpc/build.rs b/dstack/guest-agent/rpc/build.rs similarity index 100% rename from guest-agent/rpc/build.rs rename to dstack/guest-agent/rpc/build.rs diff --git a/guest-agent/rpc/proto/agent_rpc.proto b/dstack/guest-agent/rpc/proto/agent_rpc.proto similarity index 100% rename from guest-agent/rpc/proto/agent_rpc.proto rename to dstack/guest-agent/rpc/proto/agent_rpc.proto diff --git a/guest-agent/rpc/src/generated.rs b/dstack/guest-agent/rpc/src/generated.rs similarity index 100% rename from guest-agent/rpc/src/generated.rs rename to dstack/guest-agent/rpc/src/generated.rs diff --git a/guest-agent/rpc/src/lib.rs b/dstack/guest-agent/rpc/src/lib.rs similarity index 100% rename from guest-agent/rpc/src/lib.rs rename to dstack/guest-agent/rpc/src/lib.rs diff --git a/guest-agent/src/backend.rs b/dstack/guest-agent/src/backend.rs similarity index 100% rename from guest-agent/src/backend.rs rename to dstack/guest-agent/src/backend.rs diff --git a/guest-agent/src/config.rs b/dstack/guest-agent/src/config.rs similarity index 100% rename from guest-agent/src/config.rs rename to dstack/guest-agent/src/config.rs diff --git a/guest-agent/src/guest_api_service.rs b/dstack/guest-agent/src/guest_api_service.rs similarity index 100% rename from guest-agent/src/guest_api_service.rs rename to dstack/guest-agent/src/guest_api_service.rs diff --git a/guest-agent/src/http_routes.rs b/dstack/guest-agent/src/http_routes.rs similarity index 100% rename from guest-agent/src/http_routes.rs rename to dstack/guest-agent/src/http_routes.rs diff --git a/guest-agent/src/lib.rs b/dstack/guest-agent/src/lib.rs similarity index 100% rename from guest-agent/src/lib.rs rename to dstack/guest-agent/src/lib.rs diff --git a/guest-agent/src/main.rs b/dstack/guest-agent/src/main.rs similarity index 100% rename from guest-agent/src/main.rs rename to dstack/guest-agent/src/main.rs diff --git a/guest-agent/src/models.rs b/dstack/guest-agent/src/models.rs similarity index 100% rename from guest-agent/src/models.rs rename to dstack/guest-agent/src/models.rs diff --git a/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs similarity index 100% rename from guest-agent/src/rpc_service.rs rename to dstack/guest-agent/src/rpc_service.rs diff --git a/guest-agent/src/server.rs b/dstack/guest-agent/src/server.rs similarity index 100% rename from guest-agent/src/server.rs rename to dstack/guest-agent/src/server.rs diff --git a/guest-agent/src/socket_activation.rs b/dstack/guest-agent/src/socket_activation.rs similarity index 100% rename from guest-agent/src/socket_activation.rs rename to dstack/guest-agent/src/socket_activation.rs diff --git a/guest-agent/templates/dashboard.html b/dstack/guest-agent/templates/dashboard.html similarity index 100% rename from guest-agent/templates/dashboard.html rename to dstack/guest-agent/templates/dashboard.html diff --git a/guest-agent/templates/metrics.tpl b/dstack/guest-agent/templates/metrics.tpl similarity index 100% rename from guest-agent/templates/metrics.tpl rename to dstack/guest-agent/templates/metrics.tpl diff --git a/guest-api/Cargo.toml b/dstack/guest-api/Cargo.toml similarity index 100% rename from guest-api/Cargo.toml rename to dstack/guest-api/Cargo.toml diff --git a/guest-api/build.rs b/dstack/guest-api/build.rs similarity index 100% rename from guest-api/build.rs rename to dstack/guest-api/build.rs diff --git a/guest-api/proto/guest_api.proto b/dstack/guest-api/proto/guest_api.proto similarity index 100% rename from guest-api/proto/guest_api.proto rename to dstack/guest-api/proto/guest_api.proto diff --git a/guest-api/src/client.rs b/dstack/guest-api/src/client.rs similarity index 100% rename from guest-api/src/client.rs rename to dstack/guest-api/src/client.rs diff --git a/guest-api/src/generated/mod.rs b/dstack/guest-api/src/generated/mod.rs similarity index 100% rename from guest-api/src/generated/mod.rs rename to dstack/guest-api/src/generated/mod.rs diff --git a/guest-api/src/lib.rs b/dstack/guest-api/src/lib.rs similarity index 100% rename from guest-api/src/lib.rs rename to dstack/guest-api/src/lib.rs diff --git a/host-api/Cargo.toml b/dstack/host-api/Cargo.toml similarity index 100% rename from host-api/Cargo.toml rename to dstack/host-api/Cargo.toml diff --git a/host-api/build.rs b/dstack/host-api/build.rs similarity index 100% rename from host-api/build.rs rename to dstack/host-api/build.rs diff --git a/host-api/proto/host_api.proto b/dstack/host-api/proto/host_api.proto similarity index 100% rename from host-api/proto/host_api.proto rename to dstack/host-api/proto/host_api.proto diff --git a/host-api/src/client.rs b/dstack/host-api/src/client.rs similarity index 100% rename from host-api/src/client.rs rename to dstack/host-api/src/client.rs diff --git a/host-api/src/generated/mod.rs b/dstack/host-api/src/generated/mod.rs similarity index 100% rename from host-api/src/generated/mod.rs rename to dstack/host-api/src/generated/mod.rs diff --git a/host-api/src/lib.rs b/dstack/host-api/src/lib.rs similarity index 100% rename from host-api/src/lib.rs rename to dstack/host-api/src/lib.rs diff --git a/http-client/Cargo.toml b/dstack/http-client/Cargo.toml similarity index 100% rename from http-client/Cargo.toml rename to dstack/http-client/Cargo.toml diff --git a/http-client/src/hyper_vsock.rs b/dstack/http-client/src/hyper_vsock.rs similarity index 100% rename from http-client/src/hyper_vsock.rs rename to dstack/http-client/src/hyper_vsock.rs diff --git a/http-client/src/lib.rs b/dstack/http-client/src/lib.rs similarity index 100% rename from http-client/src/lib.rs rename to dstack/http-client/src/lib.rs diff --git a/http-client/src/prpc.rs b/dstack/http-client/src/prpc.rs similarity index 100% rename from http-client/src/prpc.rs rename to dstack/http-client/src/prpc.rs diff --git a/iohash/Cargo.toml b/dstack/iohash/Cargo.toml similarity index 100% rename from iohash/Cargo.toml rename to dstack/iohash/Cargo.toml diff --git a/iohash/src/main.rs b/dstack/iohash/src/main.rs similarity index 100% rename from iohash/src/main.rs rename to dstack/iohash/src/main.rs diff --git a/key-provider-build/Cargo.lock b/dstack/key-provider-build/Cargo.lock similarity index 100% rename from key-provider-build/Cargo.lock rename to dstack/key-provider-build/Cargo.lock diff --git a/key-provider-build/Dockerfile.aesmd b/dstack/key-provider-build/Dockerfile.aesmd similarity index 100% rename from key-provider-build/Dockerfile.aesmd rename to dstack/key-provider-build/Dockerfile.aesmd diff --git a/key-provider-build/Dockerfile.key-provider b/dstack/key-provider-build/Dockerfile.key-provider similarity index 100% rename from key-provider-build/Dockerfile.key-provider rename to dstack/key-provider-build/Dockerfile.key-provider diff --git a/key-provider-build/docker-compose.yaml b/dstack/key-provider-build/docker-compose.yaml similarity index 100% rename from key-provider-build/docker-compose.yaml rename to dstack/key-provider-build/docker-compose.yaml diff --git a/key-provider-build/entrypoint-aesmd.sh b/dstack/key-provider-build/entrypoint-aesmd.sh similarity index 100% rename from key-provider-build/entrypoint-aesmd.sh rename to dstack/key-provider-build/entrypoint-aesmd.sh diff --git a/key-provider-build/entrypoint-key-provider.sh b/dstack/key-provider-build/entrypoint-key-provider.sh similarity index 100% rename from key-provider-build/entrypoint-key-provider.sh rename to dstack/key-provider-build/entrypoint-key-provider.sh diff --git a/key-provider-build/run.sh b/dstack/key-provider-build/run.sh similarity index 100% rename from key-provider-build/run.sh rename to dstack/key-provider-build/run.sh diff --git a/key-provider-build/sgx_default_qcnl.conf b/dstack/key-provider-build/sgx_default_qcnl.conf similarity index 100% rename from key-provider-build/sgx_default_qcnl.conf rename to dstack/key-provider-build/sgx_default_qcnl.conf diff --git a/key-provider-client/Cargo.toml b/dstack/key-provider-client/Cargo.toml similarity index 100% rename from key-provider-client/Cargo.toml rename to dstack/key-provider-client/Cargo.toml diff --git a/key-provider-client/src/host.rs b/dstack/key-provider-client/src/host.rs similarity index 100% rename from key-provider-client/src/host.rs rename to dstack/key-provider-client/src/host.rs diff --git a/key-provider-client/src/lib.rs b/dstack/key-provider-client/src/lib.rs similarity index 100% rename from key-provider-client/src/lib.rs rename to dstack/key-provider-client/src/lib.rs diff --git a/kms/Cargo.toml b/dstack/kms/Cargo.toml similarity index 100% rename from kms/Cargo.toml rename to dstack/kms/Cargo.toml diff --git a/kms/README.md b/dstack/kms/README.md similarity index 100% rename from kms/README.md rename to dstack/kms/README.md diff --git a/kms/auth-eth-bun/.oxlintrc.json b/dstack/kms/auth-eth-bun/.oxlintrc.json similarity index 100% rename from kms/auth-eth-bun/.oxlintrc.json rename to dstack/kms/auth-eth-bun/.oxlintrc.json diff --git a/kms/auth-eth-bun/README.md b/dstack/kms/auth-eth-bun/README.md similarity index 100% rename from kms/auth-eth-bun/README.md rename to dstack/kms/auth-eth-bun/README.md diff --git a/kms/auth-eth-bun/bun.lock b/dstack/kms/auth-eth-bun/bun.lock similarity index 100% rename from kms/auth-eth-bun/bun.lock rename to dstack/kms/auth-eth-bun/bun.lock diff --git a/kms/auth-eth-bun/index.test.ts b/dstack/kms/auth-eth-bun/index.test.ts similarity index 100% rename from kms/auth-eth-bun/index.test.ts rename to dstack/kms/auth-eth-bun/index.test.ts diff --git a/kms/auth-eth-bun/index.ts b/dstack/kms/auth-eth-bun/index.ts similarity index 100% rename from kms/auth-eth-bun/index.ts rename to dstack/kms/auth-eth-bun/index.ts diff --git a/kms/auth-eth-bun/openapi.json b/dstack/kms/auth-eth-bun/openapi.json similarity index 100% rename from kms/auth-eth-bun/openapi.json rename to dstack/kms/auth-eth-bun/openapi.json diff --git a/kms/auth-eth-bun/package.json b/dstack/kms/auth-eth-bun/package.json similarity index 100% rename from kms/auth-eth-bun/package.json rename to dstack/kms/auth-eth-bun/package.json diff --git a/kms/auth-eth-bun/vitest.config.ts b/dstack/kms/auth-eth-bun/vitest.config.ts similarity index 100% rename from kms/auth-eth-bun/vitest.config.ts rename to dstack/kms/auth-eth-bun/vitest.config.ts diff --git a/kms/auth-eth/.env.example b/dstack/kms/auth-eth/.env.example similarity index 100% rename from kms/auth-eth/.env.example rename to dstack/kms/auth-eth/.env.example diff --git a/kms/auth-eth/.gitignore b/dstack/kms/auth-eth/.gitignore similarity index 100% rename from kms/auth-eth/.gitignore rename to dstack/kms/auth-eth/.gitignore diff --git a/kms/auth-eth/.openzeppelin/unknown-2035.json b/dstack/kms/auth-eth/.openzeppelin/unknown-2035.json similarity index 100% rename from kms/auth-eth/.openzeppelin/unknown-2035.json rename to dstack/kms/auth-eth/.openzeppelin/unknown-2035.json diff --git a/kms/auth-eth/README.md b/dstack/kms/auth-eth/README.md similarity index 100% rename from kms/auth-eth/README.md rename to dstack/kms/auth-eth/README.md diff --git a/kms/auth-eth/TESTING.md b/dstack/kms/auth-eth/TESTING.md similarity index 100% rename from kms/auth-eth/TESTING.md rename to dstack/kms/auth-eth/TESTING.md diff --git a/kms/auth-eth/contracts/DstackApp.sol b/dstack/kms/auth-eth/contracts/DstackApp.sol similarity index 100% rename from kms/auth-eth/contracts/DstackApp.sol rename to dstack/kms/auth-eth/contracts/DstackApp.sol diff --git a/kms/auth-eth/contracts/DstackKms.sol b/dstack/kms/auth-eth/contracts/DstackKms.sol similarity index 100% rename from kms/auth-eth/contracts/DstackKms.sol rename to dstack/kms/auth-eth/contracts/DstackKms.sol diff --git a/kms/auth-eth/contracts/IAppAuth.sol b/dstack/kms/auth-eth/contracts/IAppAuth.sol similarity index 100% rename from kms/auth-eth/contracts/IAppAuth.sol rename to dstack/kms/auth-eth/contracts/IAppAuth.sol diff --git a/kms/auth-eth/contracts/IAppAuthBasicManagement.sol b/dstack/kms/auth-eth/contracts/IAppAuthBasicManagement.sol similarity index 100% rename from kms/auth-eth/contracts/IAppAuthBasicManagement.sol rename to dstack/kms/auth-eth/contracts/IAppAuthBasicManagement.sol diff --git a/kms/auth-eth/contracts/test-utils/DstackAppV2.sol b/dstack/kms/auth-eth/contracts/test-utils/DstackAppV2.sol similarity index 100% rename from kms/auth-eth/contracts/test-utils/DstackAppV2.sol rename to dstack/kms/auth-eth/contracts/test-utils/DstackAppV2.sol diff --git a/kms/auth-eth/contracts/test-utils/DstackKmsV2.sol b/dstack/kms/auth-eth/contracts/test-utils/DstackKmsV2.sol similarity index 100% rename from kms/auth-eth/contracts/test-utils/DstackKmsV2.sol rename to dstack/kms/auth-eth/contracts/test-utils/DstackKmsV2.sol diff --git a/kms/auth-eth/docs/formal-verification.md b/dstack/kms/auth-eth/docs/formal-verification.md similarity index 100% rename from kms/auth-eth/docs/formal-verification.md rename to dstack/kms/auth-eth/docs/formal-verification.md diff --git a/kms/auth-eth/docs/specification.md b/dstack/kms/auth-eth/docs/specification.md similarity index 100% rename from kms/auth-eth/docs/specification.md rename to dstack/kms/auth-eth/docs/specification.md diff --git a/kms/auth-eth/foundry.toml b/dstack/kms/auth-eth/foundry.toml similarity index 100% rename from kms/auth-eth/foundry.toml rename to dstack/kms/auth-eth/foundry.toml diff --git a/kms/auth-eth/jest.config.js b/dstack/kms/auth-eth/jest.config.js similarity index 100% rename from kms/auth-eth/jest.config.js rename to dstack/kms/auth-eth/jest.config.js diff --git a/kms/auth-eth/lib/forge-std b/dstack/kms/auth-eth/lib/forge-std similarity index 100% rename from kms/auth-eth/lib/forge-std rename to dstack/kms/auth-eth/lib/forge-std diff --git a/kms/auth-eth/lib/openzeppelin-contracts-upgradeable b/dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable similarity index 100% rename from kms/auth-eth/lib/openzeppelin-contracts-upgradeable rename to dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable diff --git a/kms/auth-eth/lib/openzeppelin-foundry-upgrades b/dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades similarity index 100% rename from kms/auth-eth/lib/openzeppelin-foundry-upgrades rename to dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades diff --git a/kms/auth-eth/package-lock.json b/dstack/kms/auth-eth/package-lock.json similarity index 100% rename from kms/auth-eth/package-lock.json rename to dstack/kms/auth-eth/package-lock.json diff --git a/kms/auth-eth/package.json b/dstack/kms/auth-eth/package.json similarity index 100% rename from kms/auth-eth/package.json rename to dstack/kms/auth-eth/package.json diff --git a/kms/auth-eth/run-tests.sh b/dstack/kms/auth-eth/run-tests.sh similarity index 100% rename from kms/auth-eth/run-tests.sh rename to dstack/kms/auth-eth/run-tests.sh diff --git a/kms/auth-eth/script/Deploy.s.sol b/dstack/kms/auth-eth/script/Deploy.s.sol similarity index 100% rename from kms/auth-eth/script/Deploy.s.sol rename to dstack/kms/auth-eth/script/Deploy.s.sol diff --git a/kms/auth-eth/script/Manage.s.sol b/dstack/kms/auth-eth/script/Manage.s.sol similarity index 100% rename from kms/auth-eth/script/Manage.s.sol rename to dstack/kms/auth-eth/script/Manage.s.sol diff --git a/kms/auth-eth/script/Query.s.sol b/dstack/kms/auth-eth/script/Query.s.sol similarity index 100% rename from kms/auth-eth/script/Query.s.sol rename to dstack/kms/auth-eth/script/Query.s.sol diff --git a/kms/auth-eth/script/README.md b/dstack/kms/auth-eth/script/README.md similarity index 100% rename from kms/auth-eth/script/README.md rename to dstack/kms/auth-eth/script/README.md diff --git a/kms/auth-eth/script/Upgrade.s.sol b/dstack/kms/auth-eth/script/Upgrade.s.sol similarity index 100% rename from kms/auth-eth/script/Upgrade.s.sol rename to dstack/kms/auth-eth/script/Upgrade.s.sol diff --git a/kms/auth-eth/scripts/README.md b/dstack/kms/auth-eth/scripts/README.md similarity index 100% rename from kms/auth-eth/scripts/README.md rename to dstack/kms/auth-eth/scripts/README.md diff --git a/kms/auth-eth/scripts/cleanup.sh b/dstack/kms/auth-eth/scripts/cleanup.sh similarity index 100% rename from kms/auth-eth/scripts/cleanup.sh rename to dstack/kms/auth-eth/scripts/cleanup.sh diff --git a/kms/auth-eth/scripts/run-tests.sh b/dstack/kms/auth-eth/scripts/run-tests.sh similarity index 100% rename from kms/auth-eth/scripts/run-tests.sh rename to dstack/kms/auth-eth/scripts/run-tests.sh diff --git a/kms/auth-eth/scripts/setup-local-chain.sh b/dstack/kms/auth-eth/scripts/setup-local-chain.sh similarity index 100% rename from kms/auth-eth/scripts/setup-local-chain.sh rename to dstack/kms/auth-eth/scripts/setup-local-chain.sh diff --git a/kms/auth-eth/scripts/test-all.sh b/dstack/kms/auth-eth/scripts/test-all.sh similarity index 100% rename from kms/auth-eth/scripts/test-all.sh rename to dstack/kms/auth-eth/scripts/test-all.sh diff --git a/kms/auth-eth/slither.config.json b/dstack/kms/auth-eth/slither.config.json similarity index 100% rename from kms/auth-eth/slither.config.json rename to dstack/kms/auth-eth/slither.config.json diff --git a/kms/auth-eth/src/ethereum.ts b/dstack/kms/auth-eth/src/ethereum.ts similarity index 100% rename from kms/auth-eth/src/ethereum.ts rename to dstack/kms/auth-eth/src/ethereum.ts diff --git a/kms/auth-eth/src/main.test.ts b/dstack/kms/auth-eth/src/main.test.ts similarity index 100% rename from kms/auth-eth/src/main.test.ts rename to dstack/kms/auth-eth/src/main.test.ts diff --git a/kms/auth-eth/src/main.ts b/dstack/kms/auth-eth/src/main.ts similarity index 100% rename from kms/auth-eth/src/main.ts rename to dstack/kms/auth-eth/src/main.ts diff --git a/kms/auth-eth/src/server.ts b/dstack/kms/auth-eth/src/server.ts similarity index 100% rename from kms/auth-eth/src/server.ts rename to dstack/kms/auth-eth/src/server.ts diff --git a/kms/auth-eth/src/types.ts b/dstack/kms/auth-eth/src/types.ts similarity index 100% rename from kms/auth-eth/src/types.ts rename to dstack/kms/auth-eth/src/types.ts diff --git a/kms/auth-eth/test/DstackApp.symbolic.t.sol b/dstack/kms/auth-eth/test/DstackApp.symbolic.t.sol similarity index 100% rename from kms/auth-eth/test/DstackApp.symbolic.t.sol rename to dstack/kms/auth-eth/test/DstackApp.symbolic.t.sol diff --git a/kms/auth-eth/test/DstackApp.t.sol b/dstack/kms/auth-eth/test/DstackApp.t.sol similarity index 100% rename from kms/auth-eth/test/DstackApp.t.sol rename to dstack/kms/auth-eth/test/DstackApp.t.sol diff --git a/kms/auth-eth/test/DstackKms.symbolic.t.sol b/dstack/kms/auth-eth/test/DstackKms.symbolic.t.sol similarity index 100% rename from kms/auth-eth/test/DstackKms.symbolic.t.sol rename to dstack/kms/auth-eth/test/DstackKms.symbolic.t.sol diff --git a/kms/auth-eth/test/DstackKms.t.sol b/dstack/kms/auth-eth/test/DstackKms.t.sol similarity index 100% rename from kms/auth-eth/test/DstackKms.t.sol rename to dstack/kms/auth-eth/test/DstackKms.t.sol diff --git a/kms/auth-eth/test/UpgradesWithPlugin.t.sol b/dstack/kms/auth-eth/test/UpgradesWithPlugin.t.sol similarity index 100% rename from kms/auth-eth/test/UpgradesWithPlugin.t.sol rename to dstack/kms/auth-eth/test/UpgradesWithPlugin.t.sol diff --git a/kms/auth-eth/tsconfig.json b/dstack/kms/auth-eth/tsconfig.json similarity index 100% rename from kms/auth-eth/tsconfig.json rename to dstack/kms/auth-eth/tsconfig.json diff --git a/kms/auth-mock/.oxlintrc.json b/dstack/kms/auth-mock/.oxlintrc.json similarity index 100% rename from kms/auth-mock/.oxlintrc.json rename to dstack/kms/auth-mock/.oxlintrc.json diff --git a/kms/auth-mock/Dockerfile b/dstack/kms/auth-mock/Dockerfile similarity index 100% rename from kms/auth-mock/Dockerfile rename to dstack/kms/auth-mock/Dockerfile diff --git a/kms/auth-mock/README.md b/dstack/kms/auth-mock/README.md similarity index 100% rename from kms/auth-mock/README.md rename to dstack/kms/auth-mock/README.md diff --git a/kms/auth-mock/bun.lock b/dstack/kms/auth-mock/bun.lock similarity index 100% rename from kms/auth-mock/bun.lock rename to dstack/kms/auth-mock/bun.lock diff --git a/kms/auth-mock/index.test.ts b/dstack/kms/auth-mock/index.test.ts similarity index 100% rename from kms/auth-mock/index.test.ts rename to dstack/kms/auth-mock/index.test.ts diff --git a/kms/auth-mock/index.ts b/dstack/kms/auth-mock/index.ts similarity index 100% rename from kms/auth-mock/index.ts rename to dstack/kms/auth-mock/index.ts diff --git a/kms/auth-mock/openapi.json b/dstack/kms/auth-mock/openapi.json similarity index 100% rename from kms/auth-mock/openapi.json rename to dstack/kms/auth-mock/openapi.json diff --git a/kms/auth-mock/package.json b/dstack/kms/auth-mock/package.json similarity index 100% rename from kms/auth-mock/package.json rename to dstack/kms/auth-mock/package.json diff --git a/kms/auth-mock/vitest.config.ts b/dstack/kms/auth-mock/vitest.config.ts similarity index 100% rename from kms/auth-mock/vitest.config.ts rename to dstack/kms/auth-mock/vitest.config.ts diff --git a/kms/auth-simple/.oxlintrc.json b/dstack/kms/auth-simple/.oxlintrc.json similarity index 100% rename from kms/auth-simple/.oxlintrc.json rename to dstack/kms/auth-simple/.oxlintrc.json diff --git a/kms/auth-simple/README.md b/dstack/kms/auth-simple/README.md similarity index 100% rename from kms/auth-simple/README.md rename to dstack/kms/auth-simple/README.md diff --git a/kms/auth-simple/auth-config.example.json b/dstack/kms/auth-simple/auth-config.example.json similarity index 100% rename from kms/auth-simple/auth-config.example.json rename to dstack/kms/auth-simple/auth-config.example.json diff --git a/kms/auth-simple/bun.lock b/dstack/kms/auth-simple/bun.lock similarity index 100% rename from kms/auth-simple/bun.lock rename to dstack/kms/auth-simple/bun.lock diff --git a/kms/auth-simple/index.test.ts b/dstack/kms/auth-simple/index.test.ts similarity index 100% rename from kms/auth-simple/index.test.ts rename to dstack/kms/auth-simple/index.test.ts diff --git a/kms/auth-simple/index.ts b/dstack/kms/auth-simple/index.ts similarity index 100% rename from kms/auth-simple/index.ts rename to dstack/kms/auth-simple/index.ts diff --git a/kms/auth-simple/package.json b/dstack/kms/auth-simple/package.json similarity index 100% rename from kms/auth-simple/package.json rename to dstack/kms/auth-simple/package.json diff --git a/kms/auth-simple/vitest.config.ts b/dstack/kms/auth-simple/vitest.config.ts similarity index 100% rename from kms/auth-simple/vitest.config.ts rename to dstack/kms/auth-simple/vitest.config.ts diff --git a/kms/dstack-app/.gitignore b/dstack/kms/dstack-app/.gitignore similarity index 100% rename from kms/dstack-app/.gitignore rename to dstack/kms/dstack-app/.gitignore diff --git a/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile similarity index 100% rename from kms/dstack-app/builder/Dockerfile rename to dstack/kms/dstack-app/builder/Dockerfile diff --git a/kms/dstack-app/builder/README.md b/dstack/kms/dstack-app/builder/README.md similarity index 100% rename from kms/dstack-app/builder/README.md rename to dstack/kms/dstack-app/builder/README.md diff --git a/kms/dstack-app/builder/build-image.sh b/dstack/kms/dstack-app/builder/build-image.sh similarity index 100% rename from kms/dstack-app/builder/build-image.sh rename to dstack/kms/dstack-app/builder/build-image.sh diff --git a/kms/dstack-app/builder/shared/builder-pinned-packages.txt b/dstack/kms/dstack-app/builder/shared/builder-pinned-packages.txt similarity index 100% rename from kms/dstack-app/builder/shared/builder-pinned-packages.txt rename to dstack/kms/dstack-app/builder/shared/builder-pinned-packages.txt diff --git a/kms/dstack-app/builder/shared/qemu-pinned-packages.txt b/dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt similarity index 100% rename from kms/dstack-app/builder/shared/qemu-pinned-packages.txt rename to dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt diff --git a/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml similarity index 100% rename from kms/dstack-app/compose-dev.yaml rename to dstack/kms/dstack-app/compose-dev.yaml diff --git a/kms/dstack-app/compose-simple.yaml b/dstack/kms/dstack-app/compose-simple.yaml similarity index 100% rename from kms/dstack-app/compose-simple.yaml rename to dstack/kms/dstack-app/compose-simple.yaml diff --git a/kms/dstack-app/deploy-simple.sh b/dstack/kms/dstack-app/deploy-simple.sh similarity index 100% rename from kms/dstack-app/deploy-simple.sh rename to dstack/kms/dstack-app/deploy-simple.sh diff --git a/kms/dstack-app/deploy-to-vmm.sh b/dstack/kms/dstack-app/deploy-to-vmm.sh similarity index 100% rename from kms/dstack-app/deploy-to-vmm.sh rename to dstack/kms/dstack-app/deploy-to-vmm.sh diff --git a/kms/dstack-app/docker-compose.yaml b/dstack/kms/dstack-app/docker-compose.yaml similarity index 100% rename from kms/dstack-app/docker-compose.yaml rename to dstack/kms/dstack-app/docker-compose.yaml diff --git a/kms/dstack-app/entrypoint.sh b/dstack/kms/dstack-app/entrypoint.sh similarity index 100% rename from kms/dstack-app/entrypoint.sh rename to dstack/kms/dstack-app/entrypoint.sh diff --git a/kms/kms.toml b/dstack/kms/kms.toml similarity index 100% rename from kms/kms.toml rename to dstack/kms/kms.toml diff --git a/kms/rpc/Cargo.toml b/dstack/kms/rpc/Cargo.toml similarity index 100% rename from kms/rpc/Cargo.toml rename to dstack/kms/rpc/Cargo.toml diff --git a/kms/rpc/build.rs b/dstack/kms/rpc/build.rs similarity index 100% rename from kms/rpc/build.rs rename to dstack/kms/rpc/build.rs diff --git a/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto similarity index 100% rename from kms/rpc/proto/kms_rpc.proto rename to dstack/kms/rpc/proto/kms_rpc.proto diff --git a/kms/rpc/src/.gitignore b/dstack/kms/rpc/src/.gitignore similarity index 100% rename from kms/rpc/src/.gitignore rename to dstack/kms/rpc/src/.gitignore diff --git a/kms/rpc/src/generated.rs b/dstack/kms/rpc/src/generated.rs similarity index 100% rename from kms/rpc/src/generated.rs rename to dstack/kms/rpc/src/generated.rs diff --git a/kms/rpc/src/lib.rs b/dstack/kms/rpc/src/lib.rs similarity index 100% rename from kms/rpc/src/lib.rs rename to dstack/kms/rpc/src/lib.rs diff --git a/kms/src/config.rs b/dstack/kms/src/config.rs similarity index 100% rename from kms/src/config.rs rename to dstack/kms/src/config.rs diff --git a/kms/src/crypto.rs b/dstack/kms/src/crypto.rs similarity index 100% rename from kms/src/crypto.rs rename to dstack/kms/src/crypto.rs diff --git a/kms/src/ct_log.rs b/dstack/kms/src/ct_log.rs similarity index 100% rename from kms/src/ct_log.rs rename to dstack/kms/src/ct_log.rs diff --git a/kms/src/main.rs b/dstack/kms/src/main.rs similarity index 100% rename from kms/src/main.rs rename to dstack/kms/src/main.rs diff --git a/kms/src/main_service.rs b/dstack/kms/src/main_service.rs similarity index 100% rename from kms/src/main_service.rs rename to dstack/kms/src/main_service.rs diff --git a/kms/src/main_service/amd_attest.rs b/dstack/kms/src/main_service/amd_attest.rs similarity index 100% rename from kms/src/main_service/amd_attest.rs rename to dstack/kms/src/main_service/amd_attest.rs diff --git a/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs similarity index 100% rename from kms/src/main_service/upgrade_authority.rs rename to dstack/kms/src/main_service/upgrade_authority.rs diff --git a/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs similarity index 100% rename from kms/src/onboard_service.rs rename to dstack/kms/src/onboard_service.rs diff --git a/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html similarity index 100% rename from kms/src/www/onboard.html rename to dstack/kms/src/www/onboard.html diff --git a/load_config/Cargo.toml b/dstack/load_config/Cargo.toml similarity index 100% rename from load_config/Cargo.toml rename to dstack/load_config/Cargo.toml diff --git a/load_config/src/lib.rs b/dstack/load_config/src/lib.rs similarity index 100% rename from load_config/src/lib.rs rename to dstack/load_config/src/lib.rs diff --git a/lspci/Cargo.toml b/dstack/lspci/Cargo.toml similarity index 100% rename from lspci/Cargo.toml rename to dstack/lspci/Cargo.toml diff --git a/lspci/src/lib.rs b/dstack/lspci/src/lib.rs similarity index 100% rename from lspci/src/lib.rs rename to dstack/lspci/src/lib.rs diff --git a/lspci/src/snapshots/lspci__lspci.snap b/dstack/lspci/src/snapshots/lspci__lspci.snap similarity index 100% rename from lspci/src/snapshots/lspci__lspci.snap rename to dstack/lspci/src/snapshots/lspci__lspci.snap diff --git a/no_std_check/Cargo.toml b/dstack/no_std_check/Cargo.toml similarity index 100% rename from no_std_check/Cargo.toml rename to dstack/no_std_check/Cargo.toml diff --git a/no_std_check/src/lib.rs b/dstack/no_std_check/src/lib.rs similarity index 100% rename from no_std_check/src/lib.rs rename to dstack/no_std_check/src/lib.rs diff --git a/nsm-attest/Cargo.toml b/dstack/nsm-attest/Cargo.toml similarity index 100% rename from nsm-attest/Cargo.toml rename to dstack/nsm-attest/Cargo.toml diff --git a/nsm-attest/src/lib.rs b/dstack/nsm-attest/src/lib.rs similarity index 100% rename from nsm-attest/src/lib.rs rename to dstack/nsm-attest/src/lib.rs diff --git a/nsm-attest/src/types.rs b/dstack/nsm-attest/src/types.rs similarity index 100% rename from nsm-attest/src/types.rs rename to dstack/nsm-attest/src/types.rs diff --git a/nsm-attest/tests/attestation_test.rs b/dstack/nsm-attest/tests/attestation_test.rs similarity index 100% rename from nsm-attest/tests/attestation_test.rs rename to dstack/nsm-attest/tests/attestation_test.rs diff --git a/nsm-attest/tests/nitro_attestation.bin b/dstack/nsm-attest/tests/nitro_attestation.bin similarity index 100% rename from nsm-attest/tests/nitro_attestation.bin rename to dstack/nsm-attest/tests/nitro_attestation.bin diff --git a/nsm-qvl/Cargo.toml b/dstack/nsm-qvl/Cargo.toml similarity index 100% rename from nsm-qvl/Cargo.toml rename to dstack/nsm-qvl/Cargo.toml diff --git a/nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem b/dstack/nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem similarity index 100% rename from nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem rename to dstack/nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem diff --git a/nsm-qvl/src/collateral.rs b/dstack/nsm-qvl/src/collateral.rs similarity index 100% rename from nsm-qvl/src/collateral.rs rename to dstack/nsm-qvl/src/collateral.rs diff --git a/nsm-qvl/src/lib.rs b/dstack/nsm-qvl/src/lib.rs similarity index 100% rename from nsm-qvl/src/lib.rs rename to dstack/nsm-qvl/src/lib.rs diff --git a/nsm-qvl/src/verify.rs b/dstack/nsm-qvl/src/verify.rs similarity index 100% rename from nsm-qvl/src/verify.rs rename to dstack/nsm-qvl/src/verify.rs diff --git a/nsm-qvl/tests/nitro_attestation.bin b/dstack/nsm-qvl/tests/nitro_attestation.bin similarity index 100% rename from nsm-qvl/tests/nitro_attestation.bin rename to dstack/nsm-qvl/tests/nitro_attestation.bin diff --git a/nsm-qvl/tests/verify_test.rs b/dstack/nsm-qvl/tests/verify_test.rs similarity index 100% rename from nsm-qvl/tests/verify_test.rs rename to dstack/nsm-qvl/tests/verify_test.rs diff --git a/port-forward/Cargo.toml b/dstack/port-forward/Cargo.toml similarity index 100% rename from port-forward/Cargo.toml rename to dstack/port-forward/Cargo.toml diff --git a/port-forward/src/lib.rs b/dstack/port-forward/src/lib.rs similarity index 100% rename from port-forward/src/lib.rs rename to dstack/port-forward/src/lib.rs diff --git a/port-forward/src/tcp.rs b/dstack/port-forward/src/tcp.rs similarity index 100% rename from port-forward/src/tcp.rs rename to dstack/port-forward/src/tcp.rs diff --git a/port-forward/src/udp.rs b/dstack/port-forward/src/udp.rs similarity index 100% rename from port-forward/src/udp.rs rename to dstack/port-forward/src/udp.rs diff --git a/python/.gitignore b/dstack/python/.gitignore similarity index 100% rename from python/.gitignore rename to dstack/python/.gitignore diff --git a/python/ct_monitor/ct_monitor.py b/dstack/python/ct_monitor/ct_monitor.py similarity index 100% rename from python/ct_monitor/ct_monitor.py rename to dstack/python/ct_monitor/ct_monitor.py diff --git a/python/ct_monitor/pyproject.toml b/dstack/python/ct_monitor/pyproject.toml similarity index 100% rename from python/ct_monitor/pyproject.toml rename to dstack/python/ct_monitor/pyproject.toml diff --git a/ra-rpc/Cargo.toml b/dstack/ra-rpc/Cargo.toml similarity index 100% rename from ra-rpc/Cargo.toml rename to dstack/ra-rpc/Cargo.toml diff --git a/ra-rpc/prpc-openapi.md b/dstack/ra-rpc/prpc-openapi.md similarity index 100% rename from ra-rpc/prpc-openapi.md rename to dstack/ra-rpc/prpc-openapi.md diff --git a/ra-rpc/src/client.rs b/dstack/ra-rpc/src/client.rs similarity index 100% rename from ra-rpc/src/client.rs rename to dstack/ra-rpc/src/client.rs diff --git a/ra-rpc/src/lib.rs b/dstack/ra-rpc/src/lib.rs similarity index 100% rename from ra-rpc/src/lib.rs rename to dstack/ra-rpc/src/lib.rs diff --git a/ra-rpc/src/openapi.rs b/dstack/ra-rpc/src/openapi.rs similarity index 100% rename from ra-rpc/src/openapi.rs rename to dstack/ra-rpc/src/openapi.rs diff --git a/ra-rpc/src/rocket_helper.rs b/dstack/ra-rpc/src/rocket_helper.rs similarity index 100% rename from ra-rpc/src/rocket_helper.rs rename to dstack/ra-rpc/src/rocket_helper.rs diff --git a/ra-tls/Cargo.toml b/dstack/ra-tls/Cargo.toml similarity index 100% rename from ra-tls/Cargo.toml rename to dstack/ra-tls/Cargo.toml diff --git a/ra-tls/assets/tdx_quote b/dstack/ra-tls/assets/tdx_quote similarity index 100% rename from ra-tls/assets/tdx_quote rename to dstack/ra-tls/assets/tdx_quote diff --git a/ra-tls/src/attestation.rs b/dstack/ra-tls/src/attestation.rs similarity index 100% rename from ra-tls/src/attestation.rs rename to dstack/ra-tls/src/attestation.rs diff --git a/ra-tls/src/cert.rs b/dstack/ra-tls/src/cert.rs similarity index 100% rename from ra-tls/src/cert.rs rename to dstack/ra-tls/src/cert.rs diff --git a/ra-tls/src/kdf.rs b/dstack/ra-tls/src/kdf.rs similarity index 100% rename from ra-tls/src/kdf.rs rename to dstack/ra-tls/src/kdf.rs diff --git a/ra-tls/src/lib.rs b/dstack/ra-tls/src/lib.rs similarity index 100% rename from ra-tls/src/lib.rs rename to dstack/ra-tls/src/lib.rs diff --git a/ra-tls/src/oids.rs b/dstack/ra-tls/src/oids.rs similarity index 100% rename from ra-tls/src/oids.rs rename to dstack/ra-tls/src/oids.rs diff --git a/ra-tls/src/traits.rs b/dstack/ra-tls/src/traits.rs similarity index 100% rename from ra-tls/src/traits.rs rename to dstack/ra-tls/src/traits.rs diff --git a/rocket-vsock-listener/Cargo.toml b/dstack/rocket-vsock-listener/Cargo.toml similarity index 100% rename from rocket-vsock-listener/Cargo.toml rename to dstack/rocket-vsock-listener/Cargo.toml diff --git a/rocket-vsock-listener/src/lib.rs b/dstack/rocket-vsock-listener/src/lib.rs similarity index 100% rename from rocket-vsock-listener/src/lib.rs rename to dstack/rocket-vsock-listener/src/lib.rs diff --git a/run-tests.sh b/dstack/run-tests.sh similarity index 100% rename from run-tests.sh rename to dstack/run-tests.sh diff --git a/run.sh b/dstack/run.sh similarity index 100% rename from run.sh rename to dstack/run.sh diff --git a/rust-toolchain.toml b/dstack/rust-toolchain.toml similarity index 100% rename from rust-toolchain.toml rename to dstack/rust-toolchain.toml diff --git a/scripts/add-spdx-attribution.py b/dstack/scripts/add-spdx-attribution.py similarity index 100% rename from scripts/add-spdx-attribution.py rename to dstack/scripts/add-spdx-attribution.py diff --git a/scripts/bin/dstack-cloud b/dstack/scripts/bin/dstack-cloud similarity index 100% rename from scripts/bin/dstack-cloud rename to dstack/scripts/bin/dstack-cloud diff --git a/scripts/config-fw.sh b/dstack/scripts/config-fw.sh similarity index 100% rename from scripts/config-fw.sh rename to dstack/scripts/config-fw.sh diff --git a/scripts/dhcp-notify.sh b/dstack/scripts/dhcp-notify.sh similarity index 100% rename from scripts/dhcp-notify.sh rename to dstack/scripts/dhcp-notify.sh diff --git a/scripts/dstack-image-oci.sh b/dstack/scripts/dstack-image-oci.sh similarity index 100% rename from scripts/dstack-image-oci.sh rename to dstack/scripts/dstack-image-oci.sh diff --git a/scripts/install.sh b/dstack/scripts/install.sh similarity index 100% rename from scripts/install.sh rename to dstack/scripts/install.sh diff --git a/scripts/setup-bridge.sh b/dstack/scripts/setup-bridge.sh similarity index 100% rename from scripts/setup-bridge.sh rename to dstack/scripts/setup-bridge.sh diff --git a/serde-duration/Cargo.toml b/dstack/serde-duration/Cargo.toml similarity index 100% rename from serde-duration/Cargo.toml rename to dstack/serde-duration/Cargo.toml diff --git a/serde-duration/src/lib.rs b/dstack/serde-duration/src/lib.rs similarity index 100% rename from serde-duration/src/lib.rs rename to dstack/serde-duration/src/lib.rs diff --git a/sev-snp-attest/Cargo.toml b/dstack/sev-snp-attest/Cargo.toml similarity index 100% rename from sev-snp-attest/Cargo.toml rename to dstack/sev-snp-attest/Cargo.toml diff --git a/sev-snp-attest/src/lib.rs b/dstack/sev-snp-attest/src/lib.rs similarity index 100% rename from sev-snp-attest/src/lib.rs rename to dstack/sev-snp-attest/src/lib.rs diff --git a/sev-snp-qvl/Cargo.toml b/dstack/sev-snp-qvl/Cargo.toml similarity index 100% rename from sev-snp-qvl/Cargo.toml rename to dstack/sev-snp-qvl/Cargo.toml diff --git a/sev-snp-qvl/src/lib.rs b/dstack/sev-snp-qvl/src/lib.rs similarity index 100% rename from sev-snp-qvl/src/lib.rs rename to dstack/sev-snp-qvl/src/lib.rs diff --git a/size-parser/Cargo.toml b/dstack/size-parser/Cargo.toml similarity index 100% rename from size-parser/Cargo.toml rename to dstack/size-parser/Cargo.toml diff --git a/size-parser/README.md b/dstack/size-parser/README.md similarity index 100% rename from size-parser/README.md rename to dstack/size-parser/README.md diff --git a/size-parser/src/lib.rs b/dstack/size-parser/src/lib.rs similarity index 100% rename from size-parser/src/lib.rs rename to dstack/size-parser/src/lib.rs diff --git a/sodiumbox/Cargo.toml b/dstack/sodiumbox/Cargo.toml similarity index 100% rename from sodiumbox/Cargo.toml rename to dstack/sodiumbox/Cargo.toml diff --git a/sodiumbox/README.md b/dstack/sodiumbox/README.md similarity index 100% rename from sodiumbox/README.md rename to dstack/sodiumbox/README.md diff --git a/sodiumbox/src/lib.rs b/dstack/sodiumbox/src/lib.rs similarity index 100% rename from sodiumbox/src/lib.rs rename to dstack/sodiumbox/src/lib.rs diff --git a/supervisor/Cargo.toml b/dstack/supervisor/Cargo.toml similarity index 100% rename from supervisor/Cargo.toml rename to dstack/supervisor/Cargo.toml diff --git a/supervisor/client/Cargo.toml b/dstack/supervisor/client/Cargo.toml similarity index 100% rename from supervisor/client/Cargo.toml rename to dstack/supervisor/client/Cargo.toml diff --git a/supervisor/client/src/lib.rs b/dstack/supervisor/client/src/lib.rs similarity index 100% rename from supervisor/client/src/lib.rs rename to dstack/supervisor/client/src/lib.rs diff --git a/supervisor/client/src/main.rs b/dstack/supervisor/client/src/main.rs similarity index 100% rename from supervisor/client/src/main.rs rename to dstack/supervisor/client/src/main.rs diff --git a/supervisor/src/lib.rs b/dstack/supervisor/src/lib.rs similarity index 100% rename from supervisor/src/lib.rs rename to dstack/supervisor/src/lib.rs diff --git a/supervisor/src/main.rs b/dstack/supervisor/src/main.rs similarity index 100% rename from supervisor/src/main.rs rename to dstack/supervisor/src/main.rs diff --git a/supervisor/src/process.rs b/dstack/supervisor/src/process.rs similarity index 100% rename from supervisor/src/process.rs rename to dstack/supervisor/src/process.rs diff --git a/supervisor/src/supervisor.rs b/dstack/supervisor/src/supervisor.rs similarity index 100% rename from supervisor/src/supervisor.rs rename to dstack/supervisor/src/supervisor.rs diff --git a/supervisor/src/web_api.rs b/dstack/supervisor/src/web_api.rs similarity index 100% rename from supervisor/src/web_api.rs rename to dstack/supervisor/src/web_api.rs diff --git a/supervisor/supervisor.toml b/dstack/supervisor/supervisor.toml similarity index 100% rename from supervisor/supervisor.toml rename to dstack/supervisor/supervisor.toml diff --git a/supervisor/tests/test-cli.sh b/dstack/supervisor/tests/test-cli.sh similarity index 100% rename from supervisor/tests/test-cli.sh rename to dstack/supervisor/tests/test-cli.sh diff --git a/supervisor/tests/test.sh b/dstack/supervisor/tests/test.sh similarity index 100% rename from supervisor/tests/test.sh rename to dstack/supervisor/tests/test.sh diff --git a/tdx-attest/Cargo.toml b/dstack/tdx-attest/Cargo.toml similarity index 100% rename from tdx-attest/Cargo.toml rename to dstack/tdx-attest/Cargo.toml diff --git a/tdx-attest/examples/test_tdx.rs b/dstack/tdx-attest/examples/test_tdx.rs similarity index 100% rename from tdx-attest/examples/test_tdx.rs rename to dstack/tdx-attest/examples/test_tdx.rs diff --git a/tdx-attest/src/dummy.rs b/dstack/tdx-attest/src/dummy.rs similarity index 100% rename from tdx-attest/src/dummy.rs rename to dstack/tdx-attest/src/dummy.rs diff --git a/tdx-attest/src/lib.rs b/dstack/tdx-attest/src/lib.rs similarity index 100% rename from tdx-attest/src/lib.rs rename to dstack/tdx-attest/src/lib.rs diff --git a/tdx-attest/src/linux.rs b/dstack/tdx-attest/src/linux.rs similarity index 100% rename from tdx-attest/src/linux.rs rename to dstack/tdx-attest/src/linux.rs diff --git a/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel-2.snap b/dstack/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel-2.snap similarity index 100% rename from tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel-2.snap rename to dstack/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel-2.snap diff --git a/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel.snap b/dstack/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel.snap similarity index 100% rename from tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel.snap rename to dstack/tdx-attest/src/snapshots/tdx_attest__eventlog__tests__parse_ccel.snap diff --git a/test-scripts/get-app-key.sh b/dstack/test-scripts/get-app-key.sh similarity index 100% rename from test-scripts/get-app-key.sh rename to dstack/test-scripts/get-app-key.sh diff --git a/test-scripts/inspect-cert.sh b/dstack/test-scripts/inspect-cert.sh similarity index 100% rename from test-scripts/inspect-cert.sh rename to dstack/test-scripts/inspect-cert.sh diff --git a/test-scripts/snp-e2e-smoke.sh b/dstack/test-scripts/snp-e2e-smoke.sh similarity index 100% rename from test-scripts/snp-e2e-smoke.sh rename to dstack/test-scripts/snp-e2e-smoke.sh diff --git a/tests/docs/kms-bootstrap-onboard.md b/dstack/tests/docs/kms-bootstrap-onboard.md similarity index 100% rename from tests/docs/kms-bootstrap-onboard.md rename to dstack/tests/docs/kms-bootstrap-onboard.md diff --git a/tests/docs/kms-self-authorization.md b/dstack/tests/docs/kms-self-authorization.md similarity index 100% rename from tests/docs/kms-self-authorization.md rename to dstack/tests/docs/kms-self-authorization.md diff --git a/tpm-attest/Cargo.toml b/dstack/tpm-attest/Cargo.toml similarity index 100% rename from tpm-attest/Cargo.toml rename to dstack/tpm-attest/Cargo.toml diff --git a/tpm-attest/src/esapi.rs b/dstack/tpm-attest/src/esapi.rs similarity index 100% rename from tpm-attest/src/esapi.rs rename to dstack/tpm-attest/src/esapi.rs diff --git a/tpm-attest/src/gcp_ak.rs b/dstack/tpm-attest/src/gcp_ak.rs similarity index 100% rename from tpm-attest/src/gcp_ak.rs rename to dstack/tpm-attest/src/gcp_ak.rs diff --git a/tpm-attest/src/lib.rs b/dstack/tpm-attest/src/lib.rs similarity index 100% rename from tpm-attest/src/lib.rs rename to dstack/tpm-attest/src/lib.rs diff --git a/tpm-attest/tests/tpm_quote_sample.bin b/dstack/tpm-attest/tests/tpm_quote_sample.bin similarity index 100% rename from tpm-attest/tests/tpm_quote_sample.bin rename to dstack/tpm-attest/tests/tpm_quote_sample.bin diff --git a/tpm-qvl/Cargo.toml b/dstack/tpm-qvl/Cargo.toml similarity index 100% rename from tpm-qvl/Cargo.toml rename to dstack/tpm-qvl/Cargo.toml diff --git a/tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem b/dstack/tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem similarity index 100% rename from tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem rename to dstack/tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem diff --git a/tpm-qvl/certs/gcp-root-ca.pem b/dstack/tpm-qvl/certs/gcp-root-ca.pem similarity index 100% rename from tpm-qvl/certs/gcp-root-ca.pem rename to dstack/tpm-qvl/certs/gcp-root-ca.pem diff --git a/tpm-qvl/src/collateral.rs b/dstack/tpm-qvl/src/collateral.rs similarity index 100% rename from tpm-qvl/src/collateral.rs rename to dstack/tpm-qvl/src/collateral.rs diff --git a/tpm-qvl/src/lib.rs b/dstack/tpm-qvl/src/lib.rs similarity index 100% rename from tpm-qvl/src/lib.rs rename to dstack/tpm-qvl/src/lib.rs diff --git a/tpm-qvl/src/verify.rs b/dstack/tpm-qvl/src/verify.rs similarity index 100% rename from tpm-qvl/src/verify.rs rename to dstack/tpm-qvl/src/verify.rs diff --git a/tpm-types/Cargo.toml b/dstack/tpm-types/Cargo.toml similarity index 100% rename from tpm-types/Cargo.toml rename to dstack/tpm-types/Cargo.toml diff --git a/tpm-types/src/lib.rs b/dstack/tpm-types/src/lib.rs similarity index 100% rename from tpm-types/src/lib.rs rename to dstack/tpm-types/src/lib.rs diff --git a/tpm2/Cargo.toml b/dstack/tpm2/Cargo.toml similarity index 100% rename from tpm2/Cargo.toml rename to dstack/tpm2/Cargo.toml diff --git a/tpm2/src/bin/tpm2-test.rs b/dstack/tpm2/src/bin/tpm2-test.rs similarity index 100% rename from tpm2/src/bin/tpm2-test.rs rename to dstack/tpm2/src/bin/tpm2-test.rs diff --git a/tpm2/src/commands.rs b/dstack/tpm2/src/commands.rs similarity index 100% rename from tpm2/src/commands.rs rename to dstack/tpm2/src/commands.rs diff --git a/tpm2/src/constants.rs b/dstack/tpm2/src/constants.rs similarity index 100% rename from tpm2/src/constants.rs rename to dstack/tpm2/src/constants.rs diff --git a/tpm2/src/device.rs b/dstack/tpm2/src/device.rs similarity index 100% rename from tpm2/src/device.rs rename to dstack/tpm2/src/device.rs diff --git a/tpm2/src/lib.rs b/dstack/tpm2/src/lib.rs similarity index 100% rename from tpm2/src/lib.rs rename to dstack/tpm2/src/lib.rs diff --git a/tpm2/src/marshal.rs b/dstack/tpm2/src/marshal.rs similarity index 100% rename from tpm2/src/marshal.rs rename to dstack/tpm2/src/marshal.rs diff --git a/tpm2/src/session.rs b/dstack/tpm2/src/session.rs similarity index 100% rename from tpm2/src/session.rs rename to dstack/tpm2/src/session.rs diff --git a/tpm2/src/types.rs b/dstack/tpm2/src/types.rs similarity index 100% rename from tpm2/src/types.rs rename to dstack/tpm2/src/types.rs diff --git a/verifier/Cargo.toml b/dstack/verifier/Cargo.toml similarity index 100% rename from verifier/Cargo.toml rename to dstack/verifier/Cargo.toml diff --git a/verifier/README.md b/dstack/verifier/README.md similarity index 100% rename from verifier/README.md rename to dstack/verifier/README.md diff --git a/verifier/builder/Dockerfile b/dstack/verifier/builder/Dockerfile similarity index 100% rename from verifier/builder/Dockerfile rename to dstack/verifier/builder/Dockerfile diff --git a/verifier/builder/build-image.sh b/dstack/verifier/builder/build-image.sh similarity index 100% rename from verifier/builder/build-image.sh rename to dstack/verifier/builder/build-image.sh diff --git a/verifier/builder/shared/builder-pinned-packages.txt b/dstack/verifier/builder/shared/builder-pinned-packages.txt similarity index 100% rename from verifier/builder/shared/builder-pinned-packages.txt rename to dstack/verifier/builder/shared/builder-pinned-packages.txt diff --git a/verifier/builder/shared/pinned-packages.txt b/dstack/verifier/builder/shared/pinned-packages.txt similarity index 100% rename from verifier/builder/shared/pinned-packages.txt rename to dstack/verifier/builder/shared/pinned-packages.txt diff --git a/verifier/builder/shared/qemu-pinned-packages.txt b/dstack/verifier/builder/shared/qemu-pinned-packages.txt similarity index 100% rename from verifier/builder/shared/qemu-pinned-packages.txt rename to dstack/verifier/builder/shared/qemu-pinned-packages.txt diff --git a/verifier/dstack-verifier.toml b/dstack/verifier/dstack-verifier.toml similarity index 100% rename from verifier/dstack-verifier.toml rename to dstack/verifier/dstack-verifier.toml diff --git a/verifier/fixtures/quote-report.json b/dstack/verifier/fixtures/quote-report.json similarity index 100% rename from verifier/fixtures/quote-report.json rename to dstack/verifier/fixtures/quote-report.json diff --git a/verifier/fixtures/sev-snp-attestation.json b/dstack/verifier/fixtures/sev-snp-attestation.json similarity index 100% rename from verifier/fixtures/sev-snp-attestation.json rename to dstack/verifier/fixtures/sev-snp-attestation.json diff --git a/verifier/fixtures/sev-snp.README.md b/dstack/verifier/fixtures/sev-snp.README.md similarity index 100% rename from verifier/fixtures/sev-snp.README.md rename to dstack/verifier/fixtures/sev-snp.README.md diff --git a/verifier/fixtures/tdx-lite-attestation.json b/dstack/verifier/fixtures/tdx-lite-attestation.json similarity index 100% rename from verifier/fixtures/tdx-lite-attestation.json rename to dstack/verifier/fixtures/tdx-lite-attestation.json diff --git a/verifier/fixtures/tdx-lite-getquote.json b/dstack/verifier/fixtures/tdx-lite-getquote.json similarity index 100% rename from verifier/fixtures/tdx-lite-getquote.json rename to dstack/verifier/fixtures/tdx-lite-getquote.json diff --git a/verifier/fixtures/tdx-lite.README.md b/dstack/verifier/fixtures/tdx-lite.README.md similarity index 100% rename from verifier/fixtures/tdx-lite.README.md rename to dstack/verifier/fixtures/tdx-lite.README.md diff --git a/verifier/src/lib.rs b/dstack/verifier/src/lib.rs similarity index 100% rename from verifier/src/lib.rs rename to dstack/verifier/src/lib.rs diff --git a/verifier/src/main.rs b/dstack/verifier/src/main.rs similarity index 100% rename from verifier/src/main.rs rename to dstack/verifier/src/main.rs diff --git a/verifier/src/types.rs b/dstack/verifier/src/types.rs similarity index 100% rename from verifier/src/types.rs rename to dstack/verifier/src/types.rs diff --git a/verifier/src/verification.rs b/dstack/verifier/src/verification.rs similarity index 100% rename from verifier/src/verification.rs rename to dstack/verifier/src/verification.rs diff --git a/verifier/test.sh b/dstack/verifier/test.sh similarity index 100% rename from verifier/test.sh rename to dstack/verifier/test.sh diff --git a/vmm/Cargo.toml b/dstack/vmm/Cargo.toml similarity index 100% rename from vmm/Cargo.toml rename to dstack/vmm/Cargo.toml diff --git a/vmm/build.rs b/dstack/vmm/build.rs similarity index 100% rename from vmm/build.rs rename to dstack/vmm/build.rs diff --git a/vmm/requirements.txt b/dstack/vmm/requirements.txt similarity index 100% rename from vmm/requirements.txt rename to dstack/vmm/requirements.txt diff --git a/vmm/rpc/Cargo.toml b/dstack/vmm/rpc/Cargo.toml similarity index 100% rename from vmm/rpc/Cargo.toml rename to dstack/vmm/rpc/Cargo.toml diff --git a/vmm/rpc/build.rs b/dstack/vmm/rpc/build.rs similarity index 100% rename from vmm/rpc/build.rs rename to dstack/vmm/rpc/build.rs diff --git a/vmm/rpc/proto/prpc.proto b/dstack/vmm/rpc/proto/prpc.proto similarity index 100% rename from vmm/rpc/proto/prpc.proto rename to dstack/vmm/rpc/proto/prpc.proto diff --git a/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto similarity index 100% rename from vmm/rpc/proto/vmm_rpc.proto rename to dstack/vmm/rpc/proto/vmm_rpc.proto diff --git a/vmm/rpc/src/generated.rs b/dstack/vmm/rpc/src/generated.rs similarity index 100% rename from vmm/rpc/src/generated.rs rename to dstack/vmm/rpc/src/generated.rs diff --git a/vmm/rpc/src/lib.rs b/dstack/vmm/rpc/src/lib.rs similarity index 100% rename from vmm/rpc/src/lib.rs rename to dstack/vmm/rpc/src/lib.rs diff --git a/vmm/src/app.rs b/dstack/vmm/src/app.rs similarity index 100% rename from vmm/src/app.rs rename to dstack/vmm/src/app.rs diff --git a/vmm/src/app/id_pool.rs b/dstack/vmm/src/app/id_pool.rs similarity index 100% rename from vmm/src/app/id_pool.rs rename to dstack/vmm/src/app/id_pool.rs diff --git a/vmm/src/app/image.rs b/dstack/vmm/src/app/image.rs similarity index 100% rename from vmm/src/app/image.rs rename to dstack/vmm/src/app/image.rs diff --git a/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs similarity index 100% rename from vmm/src/app/qemu.rs rename to dstack/vmm/src/app/qemu.rs diff --git a/vmm/src/app/registry.rs b/dstack/vmm/src/app/registry.rs similarity index 100% rename from vmm/src/app/registry.rs rename to dstack/vmm/src/app/registry.rs diff --git a/vmm/src/config.rs b/dstack/vmm/src/config.rs similarity index 100% rename from vmm/src/config.rs rename to dstack/vmm/src/config.rs diff --git a/vmm/src/console_v0.html b/dstack/vmm/src/console_v0.html similarity index 100% rename from vmm/src/console_v0.html rename to dstack/vmm/src/console_v0.html diff --git a/vmm/src/discovery.rs b/dstack/vmm/src/discovery.rs similarity index 100% rename from vmm/src/discovery.rs rename to dstack/vmm/src/discovery.rs diff --git a/vmm/src/guest_api_service.rs b/dstack/vmm/src/guest_api_service.rs similarity index 100% rename from vmm/src/guest_api_service.rs rename to dstack/vmm/src/guest_api_service.rs diff --git a/vmm/src/host_api_service.rs b/dstack/vmm/src/host_api_service.rs similarity index 100% rename from vmm/src/host_api_service.rs rename to dstack/vmm/src/host_api_service.rs diff --git a/vmm/src/main.rs b/dstack/vmm/src/main.rs similarity index 100% rename from vmm/src/main.rs rename to dstack/vmm/src/main.rs diff --git a/vmm/src/main_routes.rs b/dstack/vmm/src/main_routes.rs similarity index 100% rename from vmm/src/main_routes.rs rename to dstack/vmm/src/main_routes.rs diff --git a/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs similarity index 100% rename from vmm/src/main_service.rs rename to dstack/vmm/src/main_service.rs diff --git a/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs similarity index 100% rename from vmm/src/one_shot.rs rename to dstack/vmm/src/one_shot.rs diff --git a/vmm/src/openapi.rs b/dstack/vmm/src/openapi.rs similarity index 100% rename from vmm/src/openapi.rs rename to dstack/vmm/src/openapi.rs diff --git a/vmm/src/setup-user.sh b/dstack/vmm/src/setup-user.sh similarity index 100% rename from vmm/src/setup-user.sh rename to dstack/vmm/src/setup-user.sh diff --git a/vmm/src/tests/test-compose.sh b/dstack/vmm/src/tests/test-compose.sh similarity index 100% rename from vmm/src/tests/test-compose.sh rename to dstack/vmm/src/tests/test-compose.sh diff --git a/vmm/src/tests/test-deployment.sh b/dstack/vmm/src/tests/test-deployment.sh similarity index 100% rename from vmm/src/tests/test-deployment.sh rename to dstack/vmm/src/tests/test-deployment.sh diff --git a/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py similarity index 100% rename from vmm/src/vmm-cli.py rename to dstack/vmm/src/vmm-cli.py diff --git a/vmm/src/x25519.js b/dstack/vmm/src/x25519.js similarity index 100% rename from vmm/src/x25519.js rename to dstack/vmm/src/x25519.js diff --git a/vmm/ui/.gitignore b/dstack/vmm/ui/.gitignore similarity index 100% rename from vmm/ui/.gitignore rename to dstack/vmm/ui/.gitignore diff --git a/vmm/ui/README.md b/dstack/vmm/ui/README.md similarity index 100% rename from vmm/ui/README.md rename to dstack/vmm/ui/README.md diff --git a/vmm/ui/build.mjs b/dstack/vmm/ui/build.mjs similarity index 100% rename from vmm/ui/build.mjs rename to dstack/vmm/ui/build.mjs diff --git a/vmm/ui/package-lock.json b/dstack/vmm/ui/package-lock.json similarity index 100% rename from vmm/ui/package-lock.json rename to dstack/vmm/ui/package-lock.json diff --git a/vmm/ui/package.json b/dstack/vmm/ui/package.json similarity index 100% rename from vmm/ui/package.json rename to dstack/vmm/ui/package.json diff --git a/vmm/ui/scripts/build_proto.sh b/dstack/vmm/ui/scripts/build_proto.sh similarity index 100% rename from vmm/ui/scripts/build_proto.sh rename to dstack/vmm/ui/scripts/build_proto.sh diff --git a/vmm/ui/src/App.ts b/dstack/vmm/ui/src/App.ts similarity index 100% rename from vmm/ui/src/App.ts rename to dstack/vmm/ui/src/App.ts diff --git a/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts similarity index 100% rename from vmm/ui/src/components/CreateVmDialog.ts rename to dstack/vmm/ui/src/components/CreateVmDialog.ts diff --git a/vmm/ui/src/components/EncryptedEnvEditor.ts b/dstack/vmm/ui/src/components/EncryptedEnvEditor.ts similarity index 100% rename from vmm/ui/src/components/EncryptedEnvEditor.ts rename to dstack/vmm/ui/src/components/EncryptedEnvEditor.ts diff --git a/vmm/ui/src/components/ForkVmDialog.ts b/dstack/vmm/ui/src/components/ForkVmDialog.ts similarity index 100% rename from vmm/ui/src/components/ForkVmDialog.ts rename to dstack/vmm/ui/src/components/ForkVmDialog.ts diff --git a/vmm/ui/src/components/GpuConfigEditor.ts b/dstack/vmm/ui/src/components/GpuConfigEditor.ts similarity index 100% rename from vmm/ui/src/components/GpuConfigEditor.ts rename to dstack/vmm/ui/src/components/GpuConfigEditor.ts diff --git a/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts similarity index 100% rename from vmm/ui/src/components/PortMappingEditor.ts rename to dstack/vmm/ui/src/components/PortMappingEditor.ts diff --git a/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts similarity index 100% rename from vmm/ui/src/components/UpdateVmDialog.ts rename to dstack/vmm/ui/src/components/UpdateVmDialog.ts diff --git a/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts similarity index 100% rename from vmm/ui/src/composables/useVmManager.ts rename to dstack/vmm/ui/src/composables/useVmManager.ts diff --git a/vmm/ui/src/index.html b/dstack/vmm/ui/src/index.html similarity index 100% rename from vmm/ui/src/index.html rename to dstack/vmm/ui/src/index.html diff --git a/vmm/ui/src/lib/vmmRpcClient.ts b/dstack/vmm/ui/src/lib/vmmRpcClient.ts similarity index 100% rename from vmm/ui/src/lib/vmmRpcClient.ts rename to dstack/vmm/ui/src/lib/vmmRpcClient.ts diff --git a/vmm/ui/src/lib/x25519.js b/dstack/vmm/ui/src/lib/x25519.js similarity index 100% rename from vmm/ui/src/lib/x25519.js rename to dstack/vmm/ui/src/lib/x25519.js diff --git a/vmm/ui/src/main.ts b/dstack/vmm/ui/src/main.ts similarity index 100% rename from vmm/ui/src/main.ts rename to dstack/vmm/ui/src/main.ts diff --git a/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css similarity index 100% rename from vmm/ui/src/styles/main.css rename to dstack/vmm/ui/src/styles/main.css diff --git a/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html similarity index 100% rename from vmm/ui/src/templates/app.html rename to dstack/vmm/ui/src/templates/app.html diff --git a/vmm/ui/tsconfig.json b/dstack/vmm/ui/tsconfig.json similarity index 100% rename from vmm/ui/tsconfig.json rename to dstack/vmm/ui/tsconfig.json diff --git a/vmm/ui/vendor/README.md b/dstack/vmm/ui/vendor/README.md similarity index 100% rename from vmm/ui/vendor/README.md rename to dstack/vmm/ui/vendor/README.md diff --git a/vmm/venv.sh b/dstack/vmm/venv.sh similarity index 100% rename from vmm/venv.sh rename to dstack/vmm/venv.sh diff --git a/vmm/vmm.toml b/dstack/vmm/vmm.toml similarity index 100% rename from vmm/vmm.toml rename to dstack/vmm/vmm.toml From b98f00df458db5b8ae1a4e24ef4b02259116e2e5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 00:46:31 -0700 Subject: [PATCH 418/435] refactor: relocate yocto build under os --- .../yocto/.github}/workflows/build-guest.yml | 0 .gitignore => os/yocto/.gitignore | 0 .gitmodules => os/yocto/.gitmodules | 18 +++++++++--------- LICENSE => os/yocto/LICENSE | 0 Makefile => os/yocto/Makefile | 0 README.md => os/yocto/README.md | 0 build.sh => os/yocto/build.sh | 0 bitbake => os/yocto/deps/bitbake | 0 .../yocto/deps/meta-confidential-compute | 0 .../yocto/deps/meta-openembedded | 0 meta-rust-bin => os/yocto/deps/meta-rust-bin | 0 meta-security => os/yocto/deps/meta-security | 0 .../yocto/deps/meta-virtualization | 0 meta-yocto => os/yocto/deps/meta-yocto | 0 .../yocto/deps/openembedded-core | 0 dev-setup => os/yocto/dev-setup | 0 dstack => os/yocto/dstack | 0 .../meta-dstack}/conf/distro/dstack.conf | 0 .../yocto/layers/meta-dstack}/conf/layer.conf | 0 .../yocto/layers/meta-dstack}/conf/local.conf | 0 .../meta-dstack}/conf/machine/dstack.conf | 0 .../meta-dstack}/conf/multiconfig/dev.conf | 0 .../meta-dstack}/conf/multiconfig/prod.conf | 0 .../openssh/files/disable-password-auth.conf | 0 .../openssh/openssh_%.bbappend | 0 .../base-files/base-files%.bbappend | 0 .../recipes-core/base-files/files/dstack-motd | 0 .../recipes-core/busybox/busybox%.bbappend | 0 .../recipes-core/busybox/files/fragment.cfg | 0 .../recipes-core/chrony/chrony%.bbappend | 0 .../recipes-core/chrony/files/chrony.conf | 0 .../recipes-core/docker/docker-moby%.bbappend | 0 .../files/docker.service.d_override.conf | 0 .../recipes-core/dstack-guest/dstack-guest.bb | 0 ...0001-Update-path-to-native-BaseTools.patch | 0 ...ile-adjust-to-build-in-under-bitbake.patch | 0 .../dstack-ovmf/0003-Debug-prefix-map.patch | 0 .../dstack-ovmf/0004-Reproduciable.patch | 0 ...ceptionHandlerLib-fix-push-instructi.patch | 0 ...06-OvmfPkg-AmdSev-drop-embedded-grub.patch | 0 .../dstack-ovmf/dstack-ovmf_git.bb | 0 .../dstack-sysbox/dstack-sysbox_0.6.7.bb | 0 .../dstack-sysbox/files/50-sysbox-mod.conf | 0 .../dstack-sysbox/files/99-sysbox-sysctl.conf | 0 .../dstack-sysbox/files/sysbox-fs.service | 0 .../dstack-sysbox/files/sysbox-mgr.service | 0 .../dstack-sysbox/files/sysbox.service | 0 .../files/sysboxFsProtobuf.pb.go | 0 .../files/sysboxMgrProtobuf.pb.go | 0 ...Define-strndupa-if-it-does-not-exist.patch | 0 .../dstack-zfs/dstack-zfs_2.4.0.bb | 0 .../recipes-core/images/dstack-initramfs.bb | 0 .../recipes-core/images/dstack-initscript.bb | 0 .../images/dstack-initscript/init | 0 .../images/dstack-rootfs-base.inc | 0 .../recipes-core/images/dstack-rootfs-dev.inc | 0 .../images/dstack-rootfs-nvidia.inc | 0 .../images/dstack-rootfs-prod.inc | 0 .../recipes-core/images/dstack-rootfs.bb | 0 .../recipes-core/images/dstack-uki.bb | 0 .../images/files/docker-daemon-nvidia.json | 0 .../images/files/docker-daemon.json | 0 .../recipes-core/ovmf/ovmf%.bbappend | 0 ...ore-suppress-ephemeral-status-output.patch | 0 .../recipes-core/systemd/systemd_%.bbappend | 0 .../fdisk/gptfdisk_%.bbappend | 0 .../gcc/libgcc-initial_%.bbappend | 0 .../gptfdisk/gptfdisk_%.bbappend | 0 ...0001-x86-tdx-select-dma-direct-remap.patch | 0 ...ox-block-aml-systemmemory-ram-access.patch | 0 .../linux/files/dstack-docker.cfg | 0 .../linux/files/dstack-docker.scc | 0 .../linux/files/dstack-sysbox.cfg | 0 .../linux/files/dstack-sysbox.scc | 0 .../recipes-kernel/linux/files/dstack-tdx.cfg | 0 .../recipes-kernel/linux/files/dstack-tdx.scc | 0 .../recipes-kernel/linux/files/dstack.cfg | 0 .../recipes-kernel/linux/files/dstack.scc | 0 .../linux/linux-yocto%.bbappend | 0 .../yocto/layers/meta-nvidia}/README.md | 0 .../meta-nvidia}/artwork/loaded_modules.png | Bin .../yocto/layers/meta-nvidia}/conf/layer.conf | 0 .../custom-licenses/NVIDIA-Proprietary | 0 .../containerd-config_1.0.0.bb | 0 .../containerd-config/files/config.toml | 0 .../ldconfig-compatibility-symlink_1.0.0.bb | 0 .../libnvidia-container.inc | 0 .../libnvidia-container/0001-build-fix.patch | 0 .../libnvidia-container/0002-secomp-fix.patch | 0 ...-fix-remove-buildpath-for-package-qa.patch | 0 .../libnvidia-container_1.00.bb | 0 .../libnvidia-container/libtirpc134_1.3.4.bb | 0 .../files/10-nvidia-gpu-ordering.conf | 0 .../nvattest/nvattest_2026.06.09.bb | 0 ...ix-cgo-LDFLAGS-for-go-1.21-and-later.patch | 0 .../files/config.toml | 0 .../nvidia-container-toolkit.inc | 0 .../nvidia-container-toolkit_1.00.bb | 0 ...idia-fabricmanager-nvswitch-condition.conf | 0 .../nvidia/files/nvidia-gpu-detect | 0 .../nvidia/files/nvidia-persistenced.service | 0 .../recipes-graphics/nvidia/files/nvidia.conf | 0 .../nvidia/libnvidia-nscq_580.105.08.bb | 0 .../nvidia/libnvidia-nscq_580.95.05.bb | 0 .../nvidia/libnvidia-nscq_595.58.03.bb | 0 .../nvidia/nvidia-fabricmanager_%.bbappend | 0 .../nvidia/nvidia-fabricmanager_580.105.08.bb | 0 .../nvidia/nvidia-fabricmanager_580.95.05.bb | 0 .../nvidia/nvidia-fabricmanager_595.58.03.bb | 0 .../nvidia/nvidia-gpu-detect_1.0.bb | 0 .../nvidia/nvidia-kernel-module.inc | 0 .../recipes-graphics/nvidia/nvidia-libs.inc | 0 .../nvidia/nvidia-modprobe-config_1.0.bb | 0 .../nvidia/nvidia-persistenced_1.0.bb | 0 .../nvidia/nvidia_580.105.08.bb | 0 .../nvidia/nvidia_580.95.05.bb | 0 .../nvidia/nvidia_595.58.03.bb | 0 .../recipes-kernel/linux/files/nvidia.cfg | 0 .../recipes-kernel/linux/files/nvidia.scc | 0 .../linux/linux-yocto%.bbappend | 0 {mk.d => os/yocto/mk.d}/.gitignore | 0 mkimage.sh => os/yocto/mkimage.sh | 0 .../yocto/repro-build}/.gitignore | 0 .../yocto/repro-build}/Dockerfile.repro | 0 .../yocto/repro-build}/check.sh | 0 .../yocto/repro-build}/repro-build.sh | 0 .../yocto/scripts}/bin/authenticode_hash.py | 0 {scripts => os/yocto/scripts}/bin/dstack | 0 {scripts => os/yocto/scripts}/bin/dstack.py | 0 .../scripts}/bin/enable_vfio_passthrough.sh | 0 {scripts => os/yocto/scripts}/bin/host_api.py | 0 .../yocto/scripts}/bin/mk-image-mr.sh | 0 .../yocto/scripts}/docker-check-config.sh | 0 .../yocto/scripts}/fix_self_uid_map.sh | 0 .../yocto/scripts}/samples/Dockerfile | 0 .../scripts}/samples/docker-compose.yaml | 0 {setup.d => os/yocto/setup.d}/.gitignore | 0 {setup.d => os/yocto/setup.d}/nvidia-layer.sh | 0 138 files changed, 9 insertions(+), 9 deletions(-) rename {.github => os/yocto/.github}/workflows/build-guest.yml (100%) rename .gitignore => os/yocto/.gitignore (100%) rename .gitmodules => os/yocto/.gitmodules (70%) rename LICENSE => os/yocto/LICENSE (100%) rename Makefile => os/yocto/Makefile (100%) rename README.md => os/yocto/README.md (100%) rename build.sh => os/yocto/build.sh (100%) rename bitbake => os/yocto/deps/bitbake (100%) rename meta-confidential-compute => os/yocto/deps/meta-confidential-compute (100%) rename meta-openembedded => os/yocto/deps/meta-openembedded (100%) rename meta-rust-bin => os/yocto/deps/meta-rust-bin (100%) rename meta-security => os/yocto/deps/meta-security (100%) rename meta-virtualization => os/yocto/deps/meta-virtualization (100%) rename meta-yocto => os/yocto/deps/meta-yocto (100%) rename openembedded-core => os/yocto/deps/openembedded-core (100%) rename dev-setup => os/yocto/dev-setup (100%) rename dstack => os/yocto/dstack (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/distro/dstack.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/layer.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/local.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/machine/dstack.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/multiconfig/dev.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/conf/multiconfig/prod.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-connectivity/openssh/files/disable-password-auth.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-connectivity/openssh/openssh_%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/base-files/base-files%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/base-files/files/dstack-motd (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/busybox/busybox%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/busybox/files/fragment.cfg (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/chrony/chrony%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/chrony/files/chrony.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/docker/docker-moby%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/docker/files/docker.service.d_override.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-guest/dstack-guest.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-ovmf/dstack-ovmf_git.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/sysbox-fs.service (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/sysbox-mgr.service (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/sysbox.service (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-initramfs.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-initscript.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-initscript/init (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-rootfs-base.inc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-rootfs-dev.inc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-rootfs-nvidia.inc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-rootfs-prod.inc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-rootfs.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/dstack-uki.bb (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/files/docker-daemon-nvidia.json (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/images/files/docker-daemon.json (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/ovmf/ovmf%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-core/systemd/systemd_%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-devtools/fdisk/gptfdisk_%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-devtools/gcc/libgcc-initial_%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-devtools/gptfdisk/gptfdisk_%.bbappend (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-docker.cfg (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-docker.scc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-sysbox.cfg (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-sysbox.scc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-tdx.cfg (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack-tdx.scc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack.cfg (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/files/dstack.scc (100%) rename {meta-dstack => os/yocto/layers/meta-dstack}/recipes-kernel/linux/linux-yocto%.bbappend (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/README.md (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/artwork/loaded_modules.png (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/conf/layer.conf (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/custom-licenses/NVIDIA-Proprietary (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/containerd-config/containerd-config_1.0.0.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/containerd-config/files/config.toml (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libnvidia-container.inc (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvattest/nvattest_2026.06.09.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia-container-toolkit/files/config.toml (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/files/nvidia-gpu-detect (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/files/nvidia-persistenced.service (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/files/nvidia.conf (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-kernel-module.inc (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-libs.inc (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia_580.105.08.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia_580.95.05.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-graphics/nvidia/nvidia_595.58.03.bb (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-kernel/linux/files/nvidia.cfg (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-kernel/linux/files/nvidia.scc (100%) rename {meta-nvidia => os/yocto/layers/meta-nvidia}/recipes-kernel/linux/linux-yocto%.bbappend (100%) rename {mk.d => os/yocto/mk.d}/.gitignore (100%) rename mkimage.sh => os/yocto/mkimage.sh (100%) rename {repro-build => os/yocto/repro-build}/.gitignore (100%) rename {repro-build => os/yocto/repro-build}/Dockerfile.repro (100%) rename {repro-build => os/yocto/repro-build}/check.sh (100%) rename {repro-build => os/yocto/repro-build}/repro-build.sh (100%) rename {scripts => os/yocto/scripts}/bin/authenticode_hash.py (100%) rename {scripts => os/yocto/scripts}/bin/dstack (100%) rename {scripts => os/yocto/scripts}/bin/dstack.py (100%) rename {scripts => os/yocto/scripts}/bin/enable_vfio_passthrough.sh (100%) rename {scripts => os/yocto/scripts}/bin/host_api.py (100%) rename {scripts => os/yocto/scripts}/bin/mk-image-mr.sh (100%) rename {scripts => os/yocto/scripts}/docker-check-config.sh (100%) rename {scripts => os/yocto/scripts}/fix_self_uid_map.sh (100%) rename {scripts => os/yocto/scripts}/samples/Dockerfile (100%) rename {scripts => os/yocto/scripts}/samples/docker-compose.yaml (100%) rename {setup.d => os/yocto/setup.d}/.gitignore (100%) rename {setup.d => os/yocto/setup.d}/nvidia-layer.sh (100%) diff --git a/.github/workflows/build-guest.yml b/os/yocto/.github/workflows/build-guest.yml similarity index 100% rename from .github/workflows/build-guest.yml rename to os/yocto/.github/workflows/build-guest.yml diff --git a/.gitignore b/os/yocto/.gitignore similarity index 100% rename from .gitignore rename to os/yocto/.gitignore diff --git a/.gitmodules b/os/yocto/.gitmodules similarity index 70% rename from .gitmodules rename to os/yocto/.gitmodules index 3c6418d1b..ae1787bff 100644 --- a/.gitmodules +++ b/os/yocto/.gitmodules @@ -1,30 +1,30 @@ [submodule "bitbake"] - path = bitbake + path = os/yocto/deps/bitbake url = https://git.openembedded.org/bitbake branch = 2.18 [submodule "openembedded-core"] - path = openembedded-core + path = os/yocto/deps/openembedded-core url = https://git.openembedded.org/openembedded-core branch = wrynose [submodule "meta-yocto"] - path = meta-yocto + path = os/yocto/deps/meta-yocto url = https://git.yoctoproject.org/meta-yocto branch = wrynose [submodule "meta-confidential-compute"] - path = meta-confidential-compute + path = os/yocto/deps/meta-confidential-compute url = https://github.com/Dstack-TEE/meta-confidential-compute.git [submodule "meta-virtualization"] - path = meta-virtualization + path = os/yocto/deps/meta-virtualization url = https://github.com/Dstack-TEE/meta-virtualization.git [submodule "meta-openembedded"] - path = meta-openembedded + path = os/yocto/deps/meta-openembedded url = https://github.com/openembedded/meta-openembedded [submodule "meta-rust-bin"] - path = meta-rust-bin + path = os/yocto/deps/meta-rust-bin url = https://github.com/Dstack-TEE/meta-rust-bin [submodule "dstack"] - path = dstack + path = os/yocto/dstack url = https://github.com/Dstack-TEE/dstack [submodule "meta-security"] - path = meta-security + path = os/yocto/deps/meta-security url = https://github.com/Dstack-TEE/meta-security.git diff --git a/LICENSE b/os/yocto/LICENSE similarity index 100% rename from LICENSE rename to os/yocto/LICENSE diff --git a/Makefile b/os/yocto/Makefile similarity index 100% rename from Makefile rename to os/yocto/Makefile diff --git a/README.md b/os/yocto/README.md similarity index 100% rename from README.md rename to os/yocto/README.md diff --git a/build.sh b/os/yocto/build.sh similarity index 100% rename from build.sh rename to os/yocto/build.sh diff --git a/bitbake b/os/yocto/deps/bitbake similarity index 100% rename from bitbake rename to os/yocto/deps/bitbake diff --git a/meta-confidential-compute b/os/yocto/deps/meta-confidential-compute similarity index 100% rename from meta-confidential-compute rename to os/yocto/deps/meta-confidential-compute diff --git a/meta-openembedded b/os/yocto/deps/meta-openembedded similarity index 100% rename from meta-openembedded rename to os/yocto/deps/meta-openembedded diff --git a/meta-rust-bin b/os/yocto/deps/meta-rust-bin similarity index 100% rename from meta-rust-bin rename to os/yocto/deps/meta-rust-bin diff --git a/meta-security b/os/yocto/deps/meta-security similarity index 100% rename from meta-security rename to os/yocto/deps/meta-security diff --git a/meta-virtualization b/os/yocto/deps/meta-virtualization similarity index 100% rename from meta-virtualization rename to os/yocto/deps/meta-virtualization diff --git a/meta-yocto b/os/yocto/deps/meta-yocto similarity index 100% rename from meta-yocto rename to os/yocto/deps/meta-yocto diff --git a/openembedded-core b/os/yocto/deps/openembedded-core similarity index 100% rename from openembedded-core rename to os/yocto/deps/openembedded-core diff --git a/dev-setup b/os/yocto/dev-setup similarity index 100% rename from dev-setup rename to os/yocto/dev-setup diff --git a/dstack b/os/yocto/dstack similarity index 100% rename from dstack rename to os/yocto/dstack diff --git a/meta-dstack/conf/distro/dstack.conf b/os/yocto/layers/meta-dstack/conf/distro/dstack.conf similarity index 100% rename from meta-dstack/conf/distro/dstack.conf rename to os/yocto/layers/meta-dstack/conf/distro/dstack.conf diff --git a/meta-dstack/conf/layer.conf b/os/yocto/layers/meta-dstack/conf/layer.conf similarity index 100% rename from meta-dstack/conf/layer.conf rename to os/yocto/layers/meta-dstack/conf/layer.conf diff --git a/meta-dstack/conf/local.conf b/os/yocto/layers/meta-dstack/conf/local.conf similarity index 100% rename from meta-dstack/conf/local.conf rename to os/yocto/layers/meta-dstack/conf/local.conf diff --git a/meta-dstack/conf/machine/dstack.conf b/os/yocto/layers/meta-dstack/conf/machine/dstack.conf similarity index 100% rename from meta-dstack/conf/machine/dstack.conf rename to os/yocto/layers/meta-dstack/conf/machine/dstack.conf diff --git a/meta-dstack/conf/multiconfig/dev.conf b/os/yocto/layers/meta-dstack/conf/multiconfig/dev.conf similarity index 100% rename from meta-dstack/conf/multiconfig/dev.conf rename to os/yocto/layers/meta-dstack/conf/multiconfig/dev.conf diff --git a/meta-dstack/conf/multiconfig/prod.conf b/os/yocto/layers/meta-dstack/conf/multiconfig/prod.conf similarity index 100% rename from meta-dstack/conf/multiconfig/prod.conf rename to os/yocto/layers/meta-dstack/conf/multiconfig/prod.conf diff --git a/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf b/os/yocto/layers/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf similarity index 100% rename from meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf rename to os/yocto/layers/meta-dstack/recipes-connectivity/openssh/files/disable-password-auth.conf diff --git a/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend b/os/yocto/layers/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend similarity index 100% rename from meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend rename to os/yocto/layers/meta-dstack/recipes-connectivity/openssh/openssh_%.bbappend diff --git a/meta-dstack/recipes-core/base-files/base-files%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/base-files/base-files%.bbappend similarity index 100% rename from meta-dstack/recipes-core/base-files/base-files%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/base-files/base-files%.bbappend diff --git a/meta-dstack/recipes-core/base-files/files/dstack-motd b/os/yocto/layers/meta-dstack/recipes-core/base-files/files/dstack-motd similarity index 100% rename from meta-dstack/recipes-core/base-files/files/dstack-motd rename to os/yocto/layers/meta-dstack/recipes-core/base-files/files/dstack-motd diff --git a/meta-dstack/recipes-core/busybox/busybox%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/busybox/busybox%.bbappend similarity index 100% rename from meta-dstack/recipes-core/busybox/busybox%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/busybox/busybox%.bbappend diff --git a/meta-dstack/recipes-core/busybox/files/fragment.cfg b/os/yocto/layers/meta-dstack/recipes-core/busybox/files/fragment.cfg similarity index 100% rename from meta-dstack/recipes-core/busybox/files/fragment.cfg rename to os/yocto/layers/meta-dstack/recipes-core/busybox/files/fragment.cfg diff --git a/meta-dstack/recipes-core/chrony/chrony%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/chrony/chrony%.bbappend similarity index 100% rename from meta-dstack/recipes-core/chrony/chrony%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/chrony/chrony%.bbappend diff --git a/meta-dstack/recipes-core/chrony/files/chrony.conf b/os/yocto/layers/meta-dstack/recipes-core/chrony/files/chrony.conf similarity index 100% rename from meta-dstack/recipes-core/chrony/files/chrony.conf rename to os/yocto/layers/meta-dstack/recipes-core/chrony/files/chrony.conf diff --git a/meta-dstack/recipes-core/docker/docker-moby%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/docker/docker-moby%.bbappend similarity index 100% rename from meta-dstack/recipes-core/docker/docker-moby%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/docker/docker-moby%.bbappend diff --git a/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf b/os/yocto/layers/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf similarity index 100% rename from meta-dstack/recipes-core/docker/files/docker.service.d_override.conf rename to os/yocto/layers/meta-dstack/recipes-core/docker/files/docker.service.d_override.conf diff --git a/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb similarity index 100% rename from meta-dstack/recipes-core/dstack-guest/dstack-guest.bb rename to os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0001-Update-path-to-native-BaseTools.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0002-BaseTools-makefile-adjust-to-build-in-under-bitbake.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0003-Debug-prefix-map.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0005-UefiCpuPkg-CpuExceptionHandlerLib-fix-push-instructi.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch diff --git a/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb similarity index 100% rename from meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb rename to os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf_git.bb diff --git a/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/dstack-sysbox_0.6.7.bb diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-fs.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox-mgr.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysbox.service diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go diff --git a/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go b/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go similarity index 100% rename from meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go rename to os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch b/os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch similarity index 100% rename from meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch rename to os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch diff --git a/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb similarity index 100% rename from meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb rename to os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs_2.4.0.bb diff --git a/meta-dstack/recipes-core/images/dstack-initramfs.bb b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-initramfs.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-initramfs.bb rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-initramfs.bb diff --git a/meta-dstack/recipes-core/images/dstack-initscript.bb b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-initscript.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-initscript.bb rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-initscript.bb diff --git a/meta-dstack/recipes-core/images/dstack-initscript/init b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-initscript/init similarity index 100% rename from meta-dstack/recipes-core/images/dstack-initscript/init rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-initscript/init diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs-base.inc rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs-dev.inc rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-dev.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-nvidia.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs-prod.inc rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-prod.inc diff --git a/meta-dstack/recipes-core/images/dstack-rootfs.bb b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-rootfs.bb rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs.bb diff --git a/meta-dstack/recipes-core/images/dstack-uki.bb b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb similarity index 100% rename from meta-dstack/recipes-core/images/dstack-uki.bb rename to os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb diff --git a/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json b/os/yocto/layers/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json similarity index 100% rename from meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json rename to os/yocto/layers/meta-dstack/recipes-core/images/files/docker-daemon-nvidia.json diff --git a/meta-dstack/recipes-core/images/files/docker-daemon.json b/os/yocto/layers/meta-dstack/recipes-core/images/files/docker-daemon.json similarity index 100% rename from meta-dstack/recipes-core/images/files/docker-daemon.json rename to os/yocto/layers/meta-dstack/recipes-core/images/files/docker-daemon.json diff --git a/meta-dstack/recipes-core/ovmf/ovmf%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/ovmf/ovmf%.bbappend similarity index 100% rename from meta-dstack/recipes-core/ovmf/ovmf%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/ovmf/ovmf%.bbappend diff --git a/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch b/os/yocto/layers/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch similarity index 100% rename from meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch rename to os/yocto/layers/meta-dstack/recipes-core/systemd/files/0001-core-suppress-ephemeral-status-output.patch diff --git a/meta-dstack/recipes-core/systemd/systemd_%.bbappend b/os/yocto/layers/meta-dstack/recipes-core/systemd/systemd_%.bbappend similarity index 100% rename from meta-dstack/recipes-core/systemd/systemd_%.bbappend rename to os/yocto/layers/meta-dstack/recipes-core/systemd/systemd_%.bbappend diff --git a/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend b/os/yocto/layers/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend similarity index 100% rename from meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend rename to os/yocto/layers/meta-dstack/recipes-devtools/fdisk/gptfdisk_%.bbappend diff --git a/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend b/os/yocto/layers/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend similarity index 100% rename from meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend rename to os/yocto/layers/meta-dstack/recipes-devtools/gcc/libgcc-initial_%.bbappend diff --git a/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend b/os/yocto/layers/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend similarity index 100% rename from meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend rename to os/yocto/layers/meta-dstack/recipes-devtools/gptfdisk/gptfdisk_%.bbappend diff --git a/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch diff --git a/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-docker.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-docker.scc rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-docker.scc diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-sysbox.scc diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-tdx.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack-tdx.scc diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack.cfg rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg diff --git a/meta-dstack/recipes-kernel/linux/files/dstack.scc b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.scc similarity index 100% rename from meta-dstack/recipes-kernel/linux/files/dstack.scc rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.scc diff --git a/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend b/os/yocto/layers/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend similarity index 100% rename from meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend rename to os/yocto/layers/meta-dstack/recipes-kernel/linux/linux-yocto%.bbappend diff --git a/meta-nvidia/README.md b/os/yocto/layers/meta-nvidia/README.md similarity index 100% rename from meta-nvidia/README.md rename to os/yocto/layers/meta-nvidia/README.md diff --git a/meta-nvidia/artwork/loaded_modules.png b/os/yocto/layers/meta-nvidia/artwork/loaded_modules.png similarity index 100% rename from meta-nvidia/artwork/loaded_modules.png rename to os/yocto/layers/meta-nvidia/artwork/loaded_modules.png diff --git a/meta-nvidia/conf/layer.conf b/os/yocto/layers/meta-nvidia/conf/layer.conf similarity index 100% rename from meta-nvidia/conf/layer.conf rename to os/yocto/layers/meta-nvidia/conf/layer.conf diff --git a/meta-nvidia/custom-licenses/NVIDIA-Proprietary b/os/yocto/layers/meta-nvidia/custom-licenses/NVIDIA-Proprietary similarity index 100% rename from meta-nvidia/custom-licenses/NVIDIA-Proprietary rename to os/yocto/layers/meta-nvidia/custom-licenses/NVIDIA-Proprietary diff --git a/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb similarity index 100% rename from meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/containerd-config/containerd-config_1.0.0.bb diff --git a/meta-nvidia/recipes-graphics/containerd-config/files/config.toml b/os/yocto/layers/meta-nvidia/recipes-graphics/containerd-config/files/config.toml similarity index 100% rename from meta-nvidia/recipes-graphics/containerd-config/files/config.toml rename to os/yocto/layers/meta-nvidia/recipes-graphics/containerd-config/files/config.toml diff --git a/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb similarity index 100% rename from meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/ldconfig-compatibility-symlink/ldconfig-compatibility-symlink_1.0.0.bb diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container.inc diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0001-build-fix.patch diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0002-secomp-fix.patch diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container/0003-fix-remove-buildpath-for-package-qa.patch diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libnvidia-container_1.00.bb diff --git a/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb similarity index 100% rename from meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/libnvidia-container/libtirpc134_1.3.4.bb diff --git a/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf b/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf similarity index 100% rename from meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf diff --git a/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/nvattest_2026.06.09.bb diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/0001-Fix-cgo-LDFLAGS-for-go-1.21-and-later.patch diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit.inc diff --git a/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/nvidia-container-toolkit_1.00.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service diff --git a/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.105.08.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_580.95.05.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/libnvidia-nscq_595.58.03.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_%.bbappend diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.105.08.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_580.95.05.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-fabricmanager_595.58.03.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-gpu-detect_1.0.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-kernel-module.inc diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-libs.inc diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-modprobe-config_1.0.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia-persistenced_1.0.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_580.105.08.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_580.95.05.bb diff --git a/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb similarity index 100% rename from meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb rename to os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/nvidia_595.58.03.bb diff --git a/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg b/os/yocto/layers/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg similarity index 100% rename from meta-nvidia/recipes-kernel/linux/files/nvidia.cfg rename to os/yocto/layers/meta-nvidia/recipes-kernel/linux/files/nvidia.cfg diff --git a/meta-nvidia/recipes-kernel/linux/files/nvidia.scc b/os/yocto/layers/meta-nvidia/recipes-kernel/linux/files/nvidia.scc similarity index 100% rename from meta-nvidia/recipes-kernel/linux/files/nvidia.scc rename to os/yocto/layers/meta-nvidia/recipes-kernel/linux/files/nvidia.scc diff --git a/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend b/os/yocto/layers/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend similarity index 100% rename from meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend rename to os/yocto/layers/meta-nvidia/recipes-kernel/linux/linux-yocto%.bbappend diff --git a/mk.d/.gitignore b/os/yocto/mk.d/.gitignore similarity index 100% rename from mk.d/.gitignore rename to os/yocto/mk.d/.gitignore diff --git a/mkimage.sh b/os/yocto/mkimage.sh similarity index 100% rename from mkimage.sh rename to os/yocto/mkimage.sh diff --git a/repro-build/.gitignore b/os/yocto/repro-build/.gitignore similarity index 100% rename from repro-build/.gitignore rename to os/yocto/repro-build/.gitignore diff --git a/repro-build/Dockerfile.repro b/os/yocto/repro-build/Dockerfile.repro similarity index 100% rename from repro-build/Dockerfile.repro rename to os/yocto/repro-build/Dockerfile.repro diff --git a/repro-build/check.sh b/os/yocto/repro-build/check.sh similarity index 100% rename from repro-build/check.sh rename to os/yocto/repro-build/check.sh diff --git a/repro-build/repro-build.sh b/os/yocto/repro-build/repro-build.sh similarity index 100% rename from repro-build/repro-build.sh rename to os/yocto/repro-build/repro-build.sh diff --git a/scripts/bin/authenticode_hash.py b/os/yocto/scripts/bin/authenticode_hash.py similarity index 100% rename from scripts/bin/authenticode_hash.py rename to os/yocto/scripts/bin/authenticode_hash.py diff --git a/scripts/bin/dstack b/os/yocto/scripts/bin/dstack similarity index 100% rename from scripts/bin/dstack rename to os/yocto/scripts/bin/dstack diff --git a/scripts/bin/dstack.py b/os/yocto/scripts/bin/dstack.py similarity index 100% rename from scripts/bin/dstack.py rename to os/yocto/scripts/bin/dstack.py diff --git a/scripts/bin/enable_vfio_passthrough.sh b/os/yocto/scripts/bin/enable_vfio_passthrough.sh similarity index 100% rename from scripts/bin/enable_vfio_passthrough.sh rename to os/yocto/scripts/bin/enable_vfio_passthrough.sh diff --git a/scripts/bin/host_api.py b/os/yocto/scripts/bin/host_api.py similarity index 100% rename from scripts/bin/host_api.py rename to os/yocto/scripts/bin/host_api.py diff --git a/scripts/bin/mk-image-mr.sh b/os/yocto/scripts/bin/mk-image-mr.sh similarity index 100% rename from scripts/bin/mk-image-mr.sh rename to os/yocto/scripts/bin/mk-image-mr.sh diff --git a/scripts/docker-check-config.sh b/os/yocto/scripts/docker-check-config.sh similarity index 100% rename from scripts/docker-check-config.sh rename to os/yocto/scripts/docker-check-config.sh diff --git a/scripts/fix_self_uid_map.sh b/os/yocto/scripts/fix_self_uid_map.sh similarity index 100% rename from scripts/fix_self_uid_map.sh rename to os/yocto/scripts/fix_self_uid_map.sh diff --git a/scripts/samples/Dockerfile b/os/yocto/scripts/samples/Dockerfile similarity index 100% rename from scripts/samples/Dockerfile rename to os/yocto/scripts/samples/Dockerfile diff --git a/scripts/samples/docker-compose.yaml b/os/yocto/scripts/samples/docker-compose.yaml similarity index 100% rename from scripts/samples/docker-compose.yaml rename to os/yocto/scripts/samples/docker-compose.yaml diff --git a/setup.d/.gitignore b/os/yocto/setup.d/.gitignore similarity index 100% rename from setup.d/.gitignore rename to os/yocto/setup.d/.gitignore diff --git a/setup.d/nvidia-layer.sh b/os/yocto/setup.d/nvidia-layer.sh similarity index 100% rename from setup.d/nvidia-layer.sh rename to os/yocto/setup.d/nvidia-layer.sh From a15542b8a96d02799c437e123b4a2635bfbe9ddb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 00:47:25 -0700 Subject: [PATCH 419/435] build: consolidate monorepo submodules --- .../workflows/guest-os.yml | 0 .gitmodules | 35 +++++++++++++++++++ os/yocto/.gitmodules | 30 ---------------- os/yocto/dstack | 1 - 4 files changed, 35 insertions(+), 31 deletions(-) rename os/yocto/.github/workflows/build-guest.yml => .github/workflows/guest-os.yml (100%) delete mode 100644 os/yocto/.gitmodules delete mode 160000 os/yocto/dstack diff --git a/os/yocto/.github/workflows/build-guest.yml b/.github/workflows/guest-os.yml similarity index 100% rename from os/yocto/.github/workflows/build-guest.yml rename to .github/workflows/guest-os.yml diff --git a/.gitmodules b/.gitmodules index dbb997779..6473a0e5d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,38 @@ [submodule "kms/auth-eth/lib/openzeppelin-foundry-upgrades"] path = dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades url = https://github.com/OpenZeppelin/openzeppelin-foundry-upgrades + +[submodule "os/yocto/deps/bitbake"] + path = os/yocto/deps/bitbake + url = https://git.openembedded.org/bitbake + branch = 2.18 + +[submodule "os/yocto/deps/openembedded-core"] + path = os/yocto/deps/openembedded-core + url = https://git.openembedded.org/openembedded-core + branch = wrynose + +[submodule "os/yocto/deps/meta-yocto"] + path = os/yocto/deps/meta-yocto + url = https://git.yoctoproject.org/meta-yocto + branch = wrynose + +[submodule "os/yocto/deps/meta-confidential-compute"] + path = os/yocto/deps/meta-confidential-compute + url = https://github.com/Dstack-TEE/meta-confidential-compute.git + +[submodule "os/yocto/deps/meta-virtualization"] + path = os/yocto/deps/meta-virtualization + url = https://github.com/Dstack-TEE/meta-virtualization.git + +[submodule "os/yocto/deps/meta-openembedded"] + path = os/yocto/deps/meta-openembedded + url = https://github.com/openembedded/meta-openembedded + +[submodule "os/yocto/deps/meta-rust-bin"] + path = os/yocto/deps/meta-rust-bin + url = https://github.com/Dstack-TEE/meta-rust-bin + +[submodule "os/yocto/deps/meta-security"] + path = os/yocto/deps/meta-security + url = https://github.com/Dstack-TEE/meta-security.git diff --git a/os/yocto/.gitmodules b/os/yocto/.gitmodules deleted file mode 100644 index ae1787bff..000000000 --- a/os/yocto/.gitmodules +++ /dev/null @@ -1,30 +0,0 @@ -[submodule "bitbake"] - path = os/yocto/deps/bitbake - url = https://git.openembedded.org/bitbake - branch = 2.18 -[submodule "openembedded-core"] - path = os/yocto/deps/openembedded-core - url = https://git.openembedded.org/openembedded-core - branch = wrynose -[submodule "meta-yocto"] - path = os/yocto/deps/meta-yocto - url = https://git.yoctoproject.org/meta-yocto - branch = wrynose -[submodule "meta-confidential-compute"] - path = os/yocto/deps/meta-confidential-compute - url = https://github.com/Dstack-TEE/meta-confidential-compute.git -[submodule "meta-virtualization"] - path = os/yocto/deps/meta-virtualization - url = https://github.com/Dstack-TEE/meta-virtualization.git -[submodule "meta-openembedded"] - path = os/yocto/deps/meta-openembedded - url = https://github.com/openembedded/meta-openembedded -[submodule "meta-rust-bin"] - path = os/yocto/deps/meta-rust-bin - url = https://github.com/Dstack-TEE/meta-rust-bin -[submodule "dstack"] - path = os/yocto/dstack - url = https://github.com/Dstack-TEE/dstack -[submodule "meta-security"] - path = os/yocto/deps/meta-security - url = https://github.com/Dstack-TEE/meta-security.git diff --git a/os/yocto/dstack b/os/yocto/dstack deleted file mode 160000 index e5f138c73..000000000 --- a/os/yocto/dstack +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e5f138c73daa5755ef41d826bc57d738efdd3895 From 8935638a14d0810c1ce0a484061d2e3108403135 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:40:31 -0700 Subject: [PATCH 420/435] build: update consumers for monorepo layout --- .github/scripts/cargo-publish-idempotent.sh | 2 +- .github/workflows/docker-build-check.yml | 73 ++++++------ .github/workflows/foundry-test.yml | 16 ++- .github/workflows/gateway-release.yml | 4 +- .github/workflows/kms-release.yml | 17 ++- .github/workflows/rust-sdk-release.yml | 4 +- .github/workflows/rust.yml | 3 + .github/workflows/sdk.yaml | 8 +- .github/workflows/simulator-release.yml | 14 ++- .github/workflows/verifier-release.yml | 6 +- .github/workflows/vmm-ui.yml | 6 +- .gitignore | 8 +- Makefile | 32 ++++++ dstack/Cargo.lock | 1 + dstack/Cargo.toml | 6 +- dstack/build/shared/build-lib.sh | 4 +- dstack/crates/dstackup/Cargo.toml | 3 + dstack/crates/dstackup/src/cli.rs | 2 +- dstack/crates/dstackup/src/image.rs | 105 ++++++++++++++---- dstack/crates/dstackup/src/install.rs | 90 +++++++++++++-- dstack/gateway/dstack-app/builder/Dockerfile | 8 +- .../gateway/dstack-app/builder/build-image.sh | 2 +- .../guest-agent-simulator/package-release.sh | 15 +-- dstack/kms/auth-mock/Dockerfile | 2 +- dstack/kms/dstack-app/builder/Dockerfile | 8 +- dstack/kms/dstack-app/builder/build-image.sh | 2 +- dstack/kms/dstack-app/compose-dev.yaml | 2 +- dstack/kms/dstack-app/deploy-simple.sh | 21 ++-- dstack/run-tests.sh | 7 +- dstack/scripts/bin/dstack-cloud | 17 ++- dstack/scripts/install.sh | 30 +++-- dstack/test-scripts/snp-e2e-smoke.sh | 14 ++- dstack/vmm/src/tests/test-compose.sh | 51 +++++---- sdk/rust/Cargo.toml | 1 + sdk/rust/types/Cargo.toml | 1 + sdk/simulator/build.sh | 6 +- 36 files changed, 412 insertions(+), 179 deletions(-) create mode 100644 Makefile diff --git a/.github/scripts/cargo-publish-idempotent.sh b/.github/scripts/cargo-publish-idempotent.sh index 5521b0148..fcff7bb38 100755 --- a/.github/scripts/cargo-publish-idempotent.sh +++ b/.github/scripts/cargo-publish-idempotent.sh @@ -11,7 +11,7 @@ set -euo pipefail crate=${1:?missing crate name} -if output=$(cargo publish -p "$crate" 2>&1); then +if output=$(cargo publish --manifest-path dstack/Cargo.toml -p "$crate" 2>&1); then echo "$output" exit 0 fi diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index 9ab36623e..90cd645b5 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -26,21 +26,21 @@ jobs: - name: Build Gateway Docker image uses: docker/build-push-action@v5 with: - context: gateway/dstack-app/builder + context: dstack/gateway/dstack-app/builder push: false load: true tags: dstack-gateway-check:latest provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} - name: Verify pinned packages run: | - build/shared/verify-pinned-packages.sh dstack-gateway-check:latest \ - gateway/dstack-app/builder/shared/pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh dstack-gateway-check:latest \ + dstack/gateway/dstack-app/builder/shared/pinned-packages.txt - name: Build gateway-builder target run: | @@ -49,22 +49,27 @@ jobs: --target gateway-builder \ --tag gateway-builder-check:latest \ --provenance=false \ - --build-context build-shared=build/shared \ + --build-context build-shared=dstack/build/shared \ --build-arg "DSTACK_REV=${DSTACK_REV}" \ --build-arg "DSTACK_SRC_URL=${DSTACK_SRC_URL}" \ - gateway/dstack-app/builder + dstack/gateway/dstack-app/builder - name: Verify builder pinned packages run: | - build/shared/verify-pinned-packages.sh gateway-builder-check:latest \ - gateway/dstack-app/builder/shared/builder-pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh gateway-builder-check:latest \ + dstack/gateway/dstack-app/builder/shared/builder-pinned-packages.txt kms: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - with: - submodules: recursive + + - name: Checkout KMS contract dependencies + run: | + git submodule update --init --depth 1 -- \ + dstack/kms/auth-eth/lib/forge-std \ + dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ + dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -72,21 +77,21 @@ jobs: - name: Build KMS Docker image uses: docker/build-push-action@v5 with: - context: kms/dstack-app/builder + context: dstack/kms/dstack-app/builder push: false load: true tags: dstack-kms-check:latest provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} - name: Verify pinned packages (qemu stage) run: | - build/shared/verify-pinned-packages.sh dstack-kms-check:latest \ - kms/dstack-app/builder/shared/qemu-pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh dstack-kms-check:latest \ + dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt - name: Build kms-builder target run: | @@ -95,22 +100,22 @@ jobs: --target kms-builder \ --tag kms-builder-check:latest \ --provenance=false \ - --build-context build-shared=build/shared \ + --build-context build-shared=dstack/build/shared \ --build-arg "DSTACK_REV=${DSTACK_REV}" \ --build-arg "DSTACK_SRC_URL=${DSTACK_SRC_URL}" \ - kms/dstack-app/builder + dstack/kms/dstack-app/builder - name: Verify builder pinned packages run: | - build/shared/verify-pinned-packages.sh kms-builder-check:latest \ - kms/dstack-app/builder/shared/builder-pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh kms-builder-check:latest \ + dstack/kms/dstack-app/builder/shared/builder-pinned-packages.txt - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Build KMS contracts run: | - cd kms/auth-eth + cd dstack/kms/auth-eth forge build verifier: @@ -124,22 +129,22 @@ jobs: - name: Build Verifier Docker image uses: docker/build-push-action@v5 with: - context: verifier - file: verifier/builder/Dockerfile + context: dstack/verifier + file: dstack/verifier/builder/Dockerfile push: false load: true tags: dstack-verifier-check:latest provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.DSTACK_REV }} DSTACK_SRC_URL=${{ env.DSTACK_SRC_URL }} - name: Verify pinned packages (runtime) run: | - build/shared/verify-pinned-packages.sh dstack-verifier-check:latest \ - verifier/builder/shared/pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh dstack-verifier-check:latest \ + dstack/verifier/builder/shared/pinned-packages.txt - name: Build verifier-builder target run: | @@ -148,16 +153,16 @@ jobs: --target verifier-builder \ --tag verifier-builder-check:latest \ --provenance=false \ - --file verifier/builder/Dockerfile \ - --build-context build-shared=build/shared \ + --file dstack/verifier/builder/Dockerfile \ + --build-context build-shared=dstack/build/shared \ --build-arg "DSTACK_REV=${DSTACK_REV}" \ --build-arg "DSTACK_SRC_URL=${DSTACK_SRC_URL}" \ - verifier + dstack/verifier - name: Verify builder pinned packages run: | - build/shared/verify-pinned-packages.sh verifier-builder-check:latest \ - verifier/builder/shared/builder-pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh verifier-builder-check:latest \ + dstack/verifier/builder/shared/builder-pinned-packages.txt - name: Build acpi-builder target run: | @@ -166,13 +171,13 @@ jobs: --target acpi-builder \ --tag verifier-acpi-check:latest \ --provenance=false \ - --file verifier/builder/Dockerfile \ - --build-context build-shared=build/shared \ + --file dstack/verifier/builder/Dockerfile \ + --build-context build-shared=dstack/build/shared \ --build-arg "DSTACK_REV=${DSTACK_REV}" \ --build-arg "DSTACK_SRC_URL=${DSTACK_SRC_URL}" \ - verifier + dstack/verifier - name: Verify qemu pinned packages run: | - build/shared/verify-pinned-packages.sh verifier-acpi-check:latest \ - verifier/builder/shared/qemu-pinned-packages.txt + dstack/build/shared/verify-pinned-packages.sh verifier-acpi-check:latest \ + dstack/verifier/builder/shared/qemu-pinned-packages.txt diff --git a/.github/workflows/foundry-test.yml b/.github/workflows/foundry-test.yml index 762449342..c47ea0474 100644 --- a/.github/workflows/foundry-test.yml +++ b/.github/workflows/foundry-test.yml @@ -7,11 +7,11 @@ name: KMS Auth-ETH Foundry Tests on: push: paths: - - 'kms/auth-eth/**' + - 'dstack/kms/auth-eth/**' - '.github/workflows/foundry-test.yml' pull_request: paths: - - 'kms/auth-eth/**' + - 'dstack/kms/auth-eth/**' - '.github/workflows/foundry-test.yml' workflow_dispatch: @@ -27,11 +27,17 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: kms/auth-eth + working-directory: dstack/kms/auth-eth steps: - uses: actions/checkout@v5 - with: - submodules: recursive + + - name: Checkout contract dependencies + working-directory: . + run: | + git submodule update --init --depth 1 -- \ + dstack/kms/auth-eth/lib/forge-std \ + dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ + dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 3bc886a97..07d12796f 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -49,13 +49,13 @@ jobs: env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} with: - context: gateway/dstack-app/builder + context: dstack/gateway/dstack-app/builder push: true tags: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} platforms: linux/amd64 provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index 7d79cec9c..e16df20e8 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -22,6 +22,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + - name: Checkout contract dependencies + run: | + git submodule update --init --depth 1 -- \ + dstack/kms/auth-eth/lib/forge-std \ + dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ + dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades + - name: Parse version from tag run: | # Extract version from tag (e.g., kms-v1.2.3 -> 1.2.3) @@ -49,13 +56,13 @@ jobs: env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} with: - context: kms/dstack-app/builder + context: dstack/kms/dstack-app/builder push: true tags: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} platforms: linux/amd64 provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }} @@ -73,7 +80,7 @@ jobs: - name: Compile contracts with Foundry run: | - cd kms/auth-eth + cd dstack/kms/auth-eth forge install forge build @@ -82,8 +89,8 @@ jobs: with: name: "KMS Release v${{ env.VERSION }}" files: | - kms/auth-eth/out/DstackKms.sol/DstackKms.json - kms/auth-eth/out/DstackApp.sol/DstackApp.json + dstack/kms/auth-eth/out/DstackKms.sol/DstackKms.json + dstack/kms/auth-eth/out/DstackApp.sol/DstackApp.json body: | ## Docker Image Information diff --git a/.github/workflows/rust-sdk-release.yml b/.github/workflows/rust-sdk-release.yml index 4f6a0a974..f2aec8e5c 100644 --- a/.github/workflows/rust-sdk-release.yml +++ b/.github/workflows/rust-sdk-release.yml @@ -53,8 +53,8 @@ jobs: pkg_version("sdk/rust/types/Cargo.toml")), ("sdk/rust/Cargo.toml [package.version]", pkg_version("sdk/rust/Cargo.toml")), - ("Cargo.toml [workspace.dependencies.dstack-sdk-types.version]", - ws_dep_version("Cargo.toml", "dstack-sdk-types")), + ("dstack/Cargo.toml [workspace.dependencies.dstack-sdk-types.version]", + ws_dep_version("dstack/Cargo.toml", "dstack-sdk-types")), ] fail = False diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index de89c5962..39231d377 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,6 +16,9 @@ env: jobs: rust-checks: runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + defaults: + run: + working-directory: dstack steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 2372ba89e..11222dbf0 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -20,7 +20,7 @@ jobs: sdk-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install Rust uses: dtolnay/rust-toolchain@1.86 @@ -35,9 +35,9 @@ jobs: - name: Verify WASM compilation # Ensures SDK types can be used in smart contracts - run: cargo check --target=wasm32-unknown-unknown -p dstack-sdk-types + run: cargo check --manifest-path dstack/Cargo.toml --target=wasm32-unknown-unknown -p dstack-sdk-types - name: Verify no_std compatibility run: | - cargo test -p dstack-sdk-types --test no_std_test --no-default-features - cargo check -p no_std_check --target thumbv6m-none-eabi + cargo test --manifest-path dstack/Cargo.toml -p dstack-sdk-types --test no_std_test --no-default-features + cargo check --manifest-path dstack/Cargo.toml -p no_std_check --target thumbv6m-none-eabi diff --git a/.github/workflows/simulator-release.yml b/.github/workflows/simulator-release.yml index 7ec123fdb..79c5a267d 100644 --- a/.github/workflows/simulator-release.yml +++ b/.github/workflows/simulator-release.yml @@ -52,12 +52,14 @@ jobs: - name: Cache Rust build artifacts uses: Swatinem/rust-cache@v2 + with: + workspaces: dstack -> target - name: Build musl simulator binary - run: cargo build --locked --release --target "${TARGET_TRIPLE}" -p dstack-guest-agent-simulator + run: cargo build --manifest-path dstack/Cargo.toml --locked --release --target "${TARGET_TRIPLE}" -p dstack-guest-agent-simulator - name: Package release bundle - run: ./guest-agent-simulator/package-release.sh "${VERSION}" "${TARGET_TRIPLE}" + run: ./dstack/guest-agent-simulator/package-release.sh "${VERSION}" "${TARGET_TRIPLE}" - name: GitHub Release uses: softprops/action-gh-release@v2 @@ -65,9 +67,9 @@ jobs: tag_name: ${{ env.TAG }} name: "Simulator Release v${{ env.VERSION }}" files: | - guest-agent-simulator/dist/dstack-simulator-${{ env.VERSION }}-${{ env.TARGET_TRIPLE }}.tar.gz - guest-agent-simulator/dist/dstack-simulator-${{ env.VERSION }}-${{ env.TARGET_TRIPLE }}.tar.gz.sha256 - guest-agent-simulator/install-systemd.sh + dstack/guest-agent-simulator/dist/dstack-simulator-${{ env.VERSION }}-${{ env.TARGET_TRIPLE }}.tar.gz + dstack/guest-agent-simulator/dist/dstack-simulator-${{ env.VERSION }}-${{ env.TARGET_TRIPLE }}.tar.gz.sha256 + dstack/guest-agent-simulator/install-systemd.sh body: | ## Release Assets @@ -92,5 +94,5 @@ jobs: Install to systemd: ```bash - curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/${{ env.TAG }}/guest-agent-simulator/install-systemd.sh | sudo bash -s -- --version ${{ env.VERSION }} + curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/${{ env.TAG }}/dstack/guest-agent-simulator/install-systemd.sh | sudo bash -s -- --version ${{ env.VERSION }} ``` diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index a939111ff..d02d5b3c2 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -48,8 +48,8 @@ jobs: env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} with: - context: verifier - file: verifier/builder/Dockerfile + context: dstack/verifier + file: dstack/verifier/builder/Dockerfile push: true tags: | ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} @@ -57,7 +57,7 @@ jobs: platforms: linux/amd64 provenance: false build-contexts: | - build-shared=build/shared + build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index fe2980414..ea571b4ce 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -28,14 +28,14 @@ jobs: run: | npm ci npm run build - working-directory: vmm/ui + working-directory: dstack/vmm/ui - name: Ensure vmm UI build is clean run: | if [ -n "$(git status --porcelain)" ]; then echo "VMM UI build produced uncommitted changes." - echo "Run: cd vmm/ui && npm ci && npm run build" - echo "Then commit the updated build output vmm/src/console_v1.html" + echo "Run: cd dstack/vmm/ui && npm ci && npm run build" + echo "Then commit the updated build output dstack/vmm/src/console_v1.html" git status --porcelain exit 1 fi diff --git a/.gitignore b/.gitignore index d30f69ff4..38d14fcaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ -/target +/dstack/target /certs /build-config.sh /build/* -!/build/shared/ +/images +/run +/rust-target **/generated/* !**/generated/mod.rs node_modules/ @@ -12,5 +14,5 @@ node_modules/ .claude/settings.local.json __pycache__ .planning/ -/vmm/src/console_v1.html +/dstack/vmm/src/console_v1.html .claude/worktrees/ diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..dd0bbc34c --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: help core core-check core-test sdk-test os os-yocto + +help: + @echo "dstack monorepo targets:" + @echo " core build the Rust workspace" + @echo " core-check check the Rust workspace" + @echo " core-test test the Rust workspace" + @echo " sdk-test run all public SDK tests" + @echo " os build the guest OS with the default backend" + @echo " os-yocto build the guest OS with Yocto" + +core: + cargo build --manifest-path dstack/Cargo.toml + +core-check: + cargo check --manifest-path dstack/Cargo.toml --workspace + +core-test: + cargo test --manifest-path dstack/Cargo.toml --workspace + +sdk-test: + cd sdk && ./run-tests.sh + +os: + ./os/build.sh + +os-yocto: + ./os/build.sh --backend yocto diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index f7ca74b03..c4af49cfd 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2911,6 +2911,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tempfile", "tokio", ] diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index d5af4cf52..095218e9a 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -60,8 +60,8 @@ members = [ "verifier", "size-parser", "port-forward", - "sdk/rust", - "sdk/rust/types", + "../sdk/rust", + "../sdk/rust/types", "no_std_check", "crates/dstack-cli-core", "crates/dstack-cli", @@ -74,7 +74,7 @@ resolver = "2" # Internal dependencies ra-rpc = { path = "ra-rpc", default-features = false } ra-tls = { path = "ra-tls" } -dstack-sdk-types = { path = "sdk/rust/types", version = "0.1.3", default-features = false } +dstack-sdk-types = { path = "../sdk/rust/types", version = "0.1.3", default-features = false } dstack-gateway-rpc = { path = "gateway/rpc" } dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } diff --git a/dstack/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh index 0f4565031..d24d368d4 100755 --- a/dstack/build/shared/build-lib.sh +++ b/dstack/build/shared/build-lib.sh @@ -7,7 +7,7 @@ # Shared build library for reproducible Docker image builds. # # Expected variables (set by the sourcing script): -# REPO_ROOT - absolute path to the git repo root +# REPO_ROOT - absolute path to the monorepo root # CONTEXT_DIR - Docker build context directory # DOCKERFILE - path to the Dockerfile # GIT_REV - git revision to build @@ -17,7 +17,7 @@ set -euo pipefail BUILDKIT_VERSION="v0.20.2" BUILDKIT_BUILDER="buildkit_20" -BUILD_SHARED_DIR="$REPO_ROOT/build/shared" +BUILD_SHARED_DIR="$REPO_ROOT/dstack/build/shared" ensure_buildkit() { if ! docker buildx inspect "$BUILDKIT_BUILDER" &>/dev/null; then diff --git a/dstack/crates/dstackup/Cargo.toml b/dstack/crates/dstackup/Cargo.toml index cfc3a8124..465008c85 100644 --- a/dstack/crates/dstackup/Cargo.toml +++ b/dstack/crates/dstackup/Cargo.toml @@ -24,3 +24,6 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/dstack/crates/dstackup/src/cli.rs b/dstack/crates/dstackup/src/cli.rs index 89e1c566b..6e534619e 100644 --- a/dstack/crates/dstackup/src/cli.rs +++ b/dstack/crates/dstackup/src/cli.rs @@ -75,7 +75,7 @@ impl ImageLoc { /// `dstackup image` subcommands. #[derive(Subcommand)] pub(crate) enum ImageCmd { - /// Download a guest OS image from meta-dstack releases. + /// Download a guest OS image from dstack guest-OS releases. Pull { /// image version to fetch (default: the latest release). #[arg(long, value_name = "VERSION")] diff --git a/dstack/crates/dstackup/src/image.rs b/dstack/crates/dstackup/src/image.rs index 60b0a7b75..0102b6ccb 100644 --- a/dstack/crates/dstackup/src/image.rs +++ b/dstack/crates/dstackup/src/image.rs @@ -4,7 +4,8 @@ //! `dstackup image` — fetch, list, and remove guest OS images. //! -//! Images are published as release tarballs at `Dstack-TEE/meta-dstack`. There +//! Images are published as `guest-os-v*` release tarballs in the dstack +//! monorepo. There //! are two variants — cpu (`dstack-`) and gpu (`dstack-nvidia-`). //! `install` validates the selected image against `digest.txt`, the OS image //! hash used on all platforms. HTTP + checksum are native (reqwest is @@ -22,8 +23,10 @@ use std::io::Write; use std::path::Path; use std::time::SystemTime; -const REPO: &str = "Dstack-TEE/meta-dstack"; -pub(crate) const RELEASES_URL: &str = "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/Dstack-TEE/meta-dstack/releases"; +const REPO: &str = "Dstack-TEE/dstack"; +const LEGACY_REPO: &str = "Dstack-TEE/meta-dstack"; +const RELEASE_TAG_PREFIX: &str = "guest-os-v"; +pub(crate) const RELEASES_URL: &str = "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/Dstack-TEE/dstack/releases?q=guest-os-v"; /// the single rule for where images live: `--image-path` if given, else the /// image directory from the install layout. `install` and every image subcommand resolve through @@ -86,7 +89,7 @@ pub(crate) async fn cmd_image(cmd: ImageCmd) -> Result<()> { } } -/// download a guest image from the latest (or a specific) meta-dstack release. +/// Download a guest image from the latest (or a specific) guest-OS release. pub(crate) async fn pull( version: Option<&str>, gpu: bool, @@ -99,7 +102,7 @@ pub(crate) async fn pull( if gpu { "gpu (nvidia)" } else { "cpu" } ); let release = fetch_release(version).await?; - let ver = release.tag_name.trim_start_matches('v'); + let ver = release_tag_version(&release.tag_name); // the unpacked dir is usually `dstack[-nvidia]-`; check that first so a // repeat pull is a cheap no-op instead of re-fetching a few hundred MB. @@ -116,7 +119,7 @@ pub(crate) async fn pull( let asset = pick_asset(&release.assets, gpu).with_context(|| { format!( - "no {} image tarball in meta-dstack release {} (assets: {})", + "no {} image tarball in guest-OS release {} (assets: {})", if gpu { "gpu" } else { "cpu" }, release.tag_name, release @@ -581,29 +584,83 @@ fn missing_named_image_message(image_dir: &str, name: &str) -> String { ) } -/// GET the latest (or a tagged) release JSON from the github api. +/// Get the latest (or a tagged) guest-OS release from the GitHub API. +/// +/// The legacy repository remains a read-only fallback so pinned deployments +/// keep working across the monorepo migration. async fn fetch_release(version: Option<&str>) -> Result { - let url = match version { - Some(v) => format!( - "https://api.github.com/repos/{REPO}/releases/tags/v{}", - v.trim_start_matches('v') - ), - None => format!("https://api.github.com/repos/{REPO}/releases/latest"), - }; - reqwest::Client::new() - .get(&url) + let client = reqwest::Client::new(); + if let Some(version) = version { + let version = version + .trim_start_matches(RELEASE_TAG_PREFIX) + .trim_start_matches('v'); + let primary_url = format!( + "https://api.github.com/repos/{REPO}/releases/tags/{RELEASE_TAG_PREFIX}{version}" + ); + if let Some(release) = fetch_tagged_release(&client, &primary_url).await? { + return Ok(release); + } + + let legacy_url = + format!("https://api.github.com/repos/{LEGACY_REPO}/releases/tags/v{version}"); + return fetch_tagged_release(&client, &legacy_url) + .await? + .with_context(|| { + format!("guest-OS version {version} was not found; check {RELEASES_URL}") + }); + } + + let list_url = format!("https://api.github.com/repos/{REPO}/releases?per_page=100"); + let releases: Vec = client + .get(&list_url) .header("user-agent", "dstackup") .header("accept", "application/vnd.github+json") .send() .await - .context("requesting the github release")? + .context("requesting dstack releases")? .error_for_status() - .with_context(|| { - format!("github release lookup failed; check the version exists at {RELEASES_URL}") - })? + .with_context(|| format!("github release lookup failed; check {RELEASES_URL}"))? .json() .await - .context("parsing github release json") + .context("parsing dstack release list")?; + if let Some(release) = releases + .into_iter() + .find(|release| release.tag_name.starts_with(RELEASE_TAG_PREFIX)) + { + return Ok(release); + } + + let legacy_url = format!("https://api.github.com/repos/{LEGACY_REPO}/releases/latest"); + fetch_tagged_release(&client, &legacy_url) + .await? + .with_context(|| format!("no guest-OS release found; check {RELEASES_URL}")) +} + +async fn fetch_tagged_release(client: &reqwest::Client, url: &str) -> Result> { + let response = client + .get(url) + .header("user-agent", "dstackup") + .header("accept", "application/vnd.github+json") + .send() + .await + .context("requesting the github release")?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + Ok(Some( + response + .error_for_status() + .with_context(|| format!("github release lookup failed; check {RELEASES_URL}"))? + .json() + .await + .context("parsing github release json")?, + )) +} + +fn release_tag_version(tag: &str) -> &str { + tag.strip_prefix(RELEASE_TAG_PREFIX) + .or_else(|| tag.strip_prefix('v')) + .unwrap_or(tag) } /// pick the cpu or gpu image tarball from a release's assets, skipping `-dev` @@ -707,6 +764,12 @@ mod tests { ); } + #[test] + fn parses_monorepo_and_legacy_release_tags() { + assert_eq!(release_tag_version("guest-os-v0.6.0"), "0.6.0"); + assert_eq!(release_tag_version("v0.5.11"), "0.5.11"); + } + #[test] fn gpu_only_release_has_no_cpu_asset() { let assets = vec![asset("dstack-nvidia-0.6.0.a2-uki.tar.gz")]; diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index afadc9892..70cfd17a5 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -451,8 +451,24 @@ fn checked_source_checkout(dir: PathBuf) -> Result { if !is_dstack_checkout(&dir) { bail!("{} is not a dstack source checkout", dir.display()); } - dir.canonicalize() - .with_context(|| format!("canonicalizing {}", dir.display())) + let dir = dir + .canonicalize() + .with_context(|| format!("canonicalizing {}", dir.display()))?; + + // Accept either the monorepo root or its dstack/ core directory, but + // normalize new-layout checkouts to the monorepo root so public examples + // and other top-level assets remain reachable. + if is_core_source(&dir) { + if let Some(parent) = dir.parent() { + if parent.join("dstack") == dir + && parent.join("sdk").is_dir() + && parent.join("os").is_dir() + { + return Ok(parent.to_path_buf()); + } + } + } + Ok(dir) } fn sync_source_cache(source: &Path, repo: &str, git_ref: &str) -> Result<()> { @@ -537,7 +553,7 @@ fn git_status_at(dir: &Path, args: [&str; N]) -> Result { .success()) } -fn is_dstack_checkout(dir: &Path) -> bool { +fn is_core_source(dir: &Path) -> bool { dir.join("Cargo.toml").is_file() && dir.join("crates/dstack-cli").is_dir() && dir.join("crates/dstack-auth").is_dir() @@ -545,10 +561,24 @@ fn is_dstack_checkout(dir: &Path) -> bool { && dir.join("supervisor").is_dir() } +fn is_dstack_checkout(dir: &Path) -> bool { + is_core_source(&dir.join("dstack")) || is_core_source(dir) +} + +fn core_source(source: &Path) -> PathBuf { + let nested = source.join("dstack"); + if is_core_source(&nested) { + nested + } else { + source.to_path_buf() + } +} + fn build_managed_binaries(source: &Path, target_dir: &Path) -> Result<()> { + let source = core_source(source); let mut cmd = cargo_build_command(target_dir)?; let target_dir_arg = path_string(target_dir); - cmd.current_dir(source).args([ + cmd.current_dir(&source).args([ "build", "--release", "--target-dir", @@ -710,13 +740,16 @@ fn install_managed_binaries(target_dir: &Path, layout: &InstallLayout) -> Result } fn install_share_assets(source: &Path, layout: &InstallLayout) -> Result<()> { + let core = core_source(source); + let examples = if source.join("examples").is_dir() { + source.join("examples") + } else { + core.join("examples") + }; fs::create_dir_all(&layout.share_dir) .with_context(|| format!("creating {}", layout.share_dir.display()))?; - copy_dir_exact( - &source.join("key-provider-build"), - &layout.key_provider_dir(), - )?; - copy_dir_exact(&source.join("examples"), &layout.share_dir.join("examples"))?; + copy_dir_exact(&core.join("key-provider-build"), &layout.key_provider_dir())?; + copy_dir_exact(&examples, &layout.share_dir.join("examples"))?; println!( " [ok] installed assets into {}", layout.share_dir.display() @@ -1139,6 +1172,14 @@ async fn wait_ready(client_url: &str, timeout: Duration) -> bool { mod tests { use super::*; + fn create_core_tree(root: &Path) { + fs::create_dir_all(root.join("crates/dstack-cli")).unwrap(); + fs::create_dir_all(root.join("crates/dstack-auth")).unwrap(); + fs::create_dir_all(root.join("vmm")).unwrap(); + fs::create_dir_all(root.join("supervisor")).unwrap(); + fs::write(root.join("Cargo.toml"), "[workspace]\n").unwrap(); + } + fn tcp_check(what: &'static str, flag: &'static str, port: u16) -> TcpPortCheck { TcpPortCheck { what, @@ -1160,6 +1201,37 @@ mod tests { assert!(err.contains("--kms-port")); } + #[test] + fn recognizes_and_normalizes_monorepo_source() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + create_core_tree(&root.join("dstack")); + fs::create_dir(root.join("sdk")).unwrap(); + fs::create_dir(root.join("os")).unwrap(); + fs::create_dir(root.join("examples")).unwrap(); + + assert!(is_dstack_checkout(root)); + assert!(is_dstack_checkout(&root.join("dstack"))); + assert_eq!(core_source(root), root.join("dstack")); + assert_eq!( + checked_source_checkout(root.join("dstack")).unwrap(), + root.canonicalize().unwrap() + ); + } + + #[test] + fn keeps_legacy_core_checkout_compatible() { + let temp = tempfile::tempdir().unwrap(); + create_core_tree(temp.path()); + + assert!(is_dstack_checkout(temp.path())); + assert_eq!(core_source(temp.path()), temp.path()); + assert_eq!( + checked_source_checkout(temp.path().to_path_buf()).unwrap(), + temp.path().canonicalize().unwrap() + ); + } + #[test] fn preflight_rejects_zero_port() { let err = preflight_ports(&[tcp_check("kms", "--kms-port", 0)]) diff --git a/dstack/gateway/dstack-app/builder/Dockerfile b/dstack/gateway/dstack-app/builder/Dockerfile index 638017be5..e91b00486 100644 --- a/dstack/gateway/dstack-app/builder/Dockerfile +++ b/dstack/gateway/dstack-app/builder/Dockerfile @@ -19,11 +19,11 @@ RUN apt-get update && \ libprotobuf-dev \ clang \ libclang-dev -RUN git clone ${DSTACK_SRC_URL} dstack && \ - cd dstack && \ +RUN git clone ${DSTACK_SRC_URL} repo && \ + cd repo && \ git checkout ${DSTACK_REV} RUN rustup target add x86_64-unknown-linux-musl -RUN cd dstack && cargo build --release -p dstack-gateway --target x86_64-unknown-linux-musl +RUN cd repo/dstack && cargo build --release -p dstack-gateway --target x86_64-unknown-linux-musl RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe @@ -39,7 +39,7 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ jq \ ca-certificates \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache -COPY --from=gateway-builder /build/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway /usr/local/bin/dstack-gateway +COPY --from=gateway-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway /usr/local/bin/dstack-gateway COPY --from=gateway-builder /build/.GIT_REV /etc/ WORKDIR /app COPY entrypoint.sh /app/entrypoint.sh diff --git a/dstack/gateway/dstack-app/builder/build-image.sh b/dstack/gateway/dstack-app/builder/build-image.sh index 685e2019f..3d86546f1 100755 --- a/dstack/gateway/dstack-app/builder/build-image.sh +++ b/dstack/gateway/dstack-app/builder/build-image.sh @@ -12,7 +12,7 @@ CONTEXT_DIR="$SCRIPT_DIR" SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" -source "$REPO_ROOT/build/shared/build-lib.sh" +source "$REPO_ROOT/dstack/build/shared/build-lib.sh" NAME=${1:-} if [ -z "$NAME" ]; then diff --git a/dstack/guest-agent-simulator/package-release.sh b/dstack/guest-agent-simulator/package-release.sh index 5223eda66..d1cd3892f 100755 --- a/dstack/guest-agent-simulator/package-release.sh +++ b/dstack/guest-agent-simulator/package-release.sh @@ -28,9 +28,10 @@ TARGET="$2" shift 2 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CORE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$CORE_DIR/.." && pwd)" OUT_DIR="$SCRIPT_DIR/dist" -BINARY_PATH="$ROOT_DIR/target/$TARGET/release/dstack-simulator" +BINARY_PATH="$CORE_DIR/target/$TARGET/release/dstack-simulator" while [[ $# -gt 0 ]]; do case "$1" in @@ -68,11 +69,11 @@ rm -rf "$STAGE_DIR" "$TARBALL_PATH" "$CHECKSUM_PATH" mkdir -p "$STAGE_DIR" install -m 755 "$BINARY_PATH" "$STAGE_DIR/dstack-simulator" -install -m 644 "$ROOT_DIR/sdk/simulator/dstack.toml" "$STAGE_DIR/dstack.toml" -install -m 644 "$ROOT_DIR/sdk/simulator/app-compose.json" "$STAGE_DIR/app-compose.json" -install -m 644 "$ROOT_DIR/sdk/simulator/appkeys.json" "$STAGE_DIR/appkeys.json" -install -m 644 "$ROOT_DIR/sdk/simulator/sys-config.json" "$STAGE_DIR/sys-config.json" -install -m 644 "$ROOT_DIR/sdk/simulator/attestation.bin" "$STAGE_DIR/attestation.bin" +install -m 644 "$REPO_ROOT/sdk/simulator/dstack.toml" "$STAGE_DIR/dstack.toml" +install -m 644 "$REPO_ROOT/sdk/simulator/app-compose.json" "$STAGE_DIR/app-compose.json" +install -m 644 "$REPO_ROOT/sdk/simulator/appkeys.json" "$STAGE_DIR/appkeys.json" +install -m 644 "$REPO_ROOT/sdk/simulator/sys-config.json" "$STAGE_DIR/sys-config.json" +install -m 644 "$REPO_ROOT/sdk/simulator/attestation.bin" "$STAGE_DIR/attestation.bin" install -m 644 "$SCRIPT_DIR/dstack-simulator.service" "$STAGE_DIR/dstack-simulator.service" install -m 755 "$SCRIPT_DIR/install-systemd.sh" "$STAGE_DIR/install-systemd.sh" diff --git a/dstack/kms/auth-mock/Dockerfile b/dstack/kms/auth-mock/Dockerfile index 095ff1b29..3bb12a25d 100644 --- a/dstack/kms/auth-mock/Dockerfile +++ b/dstack/kms/auth-mock/Dockerfile @@ -12,6 +12,6 @@ RUN apk add --no-cache git RUN git clone --branch ${DSTACK_BRANCH} https://github.com/Dstack-TEE/dstack.git && \ cd dstack && \ git checkout ${DSTACK_REV} -WORKDIR /app/dstack/kms/auth-mock +WORKDIR /app/dstack/dstack/kms/auth-mock RUN bun install --frozen-lockfile CMD ["bun", "index.ts"] diff --git a/dstack/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile index f924d0e02..038324c3a 100644 --- a/dstack/kms/dstack-app/builder/Dockerfile +++ b/dstack/kms/dstack-app/builder/Dockerfile @@ -19,11 +19,11 @@ RUN apt-get update && \ libprotobuf-dev \ clang \ libclang-dev -RUN git clone ${DSTACK_SRC_URL} dstack && \ - cd dstack && \ +RUN git clone ${DSTACK_SRC_URL} repo && \ + cd repo && \ git checkout ${DSTACK_REV} RUN rustup target add x86_64-unknown-linux-musl -RUN cd dstack && cargo build --release -p dstack-kms --target x86_64-unknown-linux-musl +RUN cd repo/dstack && cargo build --release -p dstack-kms --target x86_64-unknown-linux-musl RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe @@ -60,6 +60,6 @@ RUN git clone https://github.com/kvinwang/qemu-tdx.git --depth 1 --branch dstack install -m 644 pc-bios/kvmvapic.bin /usr/local/share/qemu/ && \ install -m 644 pc-bios/linuxboot_dma.bin /usr/local/share/qemu/ && \ cd .. && rm -rf qemu-tdx -COPY --from=kms-builder /build/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms +COPY --from=kms-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms COPY --from=kms-builder /build/.GIT_REV /etc/ CMD ["dstack-kms"] diff --git a/dstack/kms/dstack-app/builder/build-image.sh b/dstack/kms/dstack-app/builder/build-image.sh index 73be520e9..b92c9adcb 100755 --- a/dstack/kms/dstack-app/builder/build-image.sh +++ b/dstack/kms/dstack-app/builder/build-image.sh @@ -12,7 +12,7 @@ CONTEXT_DIR="$SCRIPT_DIR" SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" -source "$REPO_ROOT/build/shared/build-lib.sh" +source "$REPO_ROOT/dstack/build/shared/build-lib.sh" NAME=${1:-} if [ -z "$NAME" ]; then diff --git a/dstack/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml index 755f00e36..c216496ef 100644 --- a/dstack/kms/dstack-app/compose-dev.yaml +++ b/dstack/kms/dstack-app/compose-dev.yaml @@ -15,7 +15,7 @@ services: RUN git clone ${GIT_REPOSITORY} && \ cd dstack && \ git checkout ${GIT_REV} - WORKDIR /app/dstack/kms/auth-eth + WORKDIR /app/dstack/dstack/kms/auth-eth RUN npm install RUN npx tsc --project tsconfig.json CMD node dist/src/main.js diff --git a/dstack/kms/dstack-app/deploy-simple.sh b/dstack/kms/dstack-app/deploy-simple.sh index f027f24aa..cdf8df9ee 100755 --- a/dstack/kms/dstack-app/deploy-simple.sh +++ b/dstack/kms/dstack-app/deploy-simple.sh @@ -13,6 +13,7 @@ set -e if [ -f ".env.simple" ]; then echo "Loading environment variables from .env.simple file..." set -a + # shellcheck source=/dev/null source .env.simple set +a else @@ -35,7 +36,7 @@ else # GUEST_AGENT_ADDR=127.0.0.1:9205 # Required: The URL of the dstack app image download URL -# IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.5/dstack-0.5.5.tar.gz +# IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v0.5.5/dstack-0.5.5.tar.gz # Image hash verification feature flag VERIFY_IMAGE=true @@ -70,11 +71,13 @@ for var in "${required_env_vars[@]}"; do fi done -CLI="../../vmm/src/vmm-cli.py --url $VMM_RPC" +CLI=(../../vmm/src/vmm-cli.py --url "$VMM_RPC") COMPOSE_TMP=$(mktemp) -ADMIN_TOKEN_HASH=$(echo -n $ADMIN_TOKEN | sha256sum | cut -d' ' -f1) +# Used indirectly by subvar. +# shellcheck disable=SC2034 +ADMIN_TOKEN_HASH=$(printf '%s' "$ADMIN_TOKEN" | sha256sum | cut -d' ' -f1) cp compose-simple.yaml "$COMPOSE_TMP" @@ -99,7 +102,7 @@ echo " OS_IMAGE: $OS_IMAGE" echo "" if [ -t 0 ]; then - read -p "Continue? [y/N] " -n 1 -r + read -r -p "Continue? [y/N] " -n 1 echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then @@ -108,7 +111,7 @@ if [ -t 0 ]; then fi fi -$CLI compose \ +"${CLI[@]}" compose \ --docker-compose "$COMPOSE_TMP" \ --name kms \ --local-key-provider \ @@ -122,12 +125,12 @@ rm "$COMPOSE_TMP" echo "Deploying KMS to dstack-vmm..." -$CLI deploy \ +"${CLI[@]}" deploy \ --name kms \ --compose .app-compose.json \ - --image $OS_IMAGE \ - --port tcp:$KMS_RPC_ADDR:8000 \ - --port tcp:$GUEST_AGENT_ADDR:8090 \ + --image "$OS_IMAGE" \ + --port "tcp:$KMS_RPC_ADDR:8000" \ + --port "tcp:$GUEST_AGENT_ADDR:8090" \ --vcpu 8 \ --memory 8G \ --disk 50G diff --git a/dstack/run-tests.sh b/dstack/run-tests.sh index 59aa0a5fd..301abf56a 100755 --- a/dstack/run-tests.sh +++ b/dstack/run-tests.sh @@ -6,8 +6,9 @@ set -Eeuo pipefail -ROOT_DIR="$(pwd -P)" -SIMULATOR_DIR="$ROOT_DIR/sdk/simulator" +CORE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd "$CORE_DIR/.." && pwd -P)" +SIMULATOR_DIR="$REPO_ROOT/sdk/simulator" SIMULATOR_LOG="$SIMULATOR_DIR/dstack-simulator.log" DSTACK_SOCKET="$SIMULATOR_DIR/dstack.sock" TAPPD_SOCKET="$SIMULATOR_DIR/tappd.sock" @@ -73,4 +74,4 @@ export TAPPD_SIMULATOR_ENDPOINT="$TAPPD_SOCKET" echo "DSTACK_SIMULATOR_ENDPOINT: $DSTACK_SIMULATOR_ENDPOINT" echo "TAPPD_SIMULATOR_ENDPOINT: $TAPPD_SIMULATOR_ENDPOINT" -cargo test --all-features -- --show-output +(cd "$CORE_DIR" && cargo test --all-features -- --show-output) diff --git a/dstack/scripts/bin/dstack-cloud b/dstack/scripts/bin/dstack-cloud index 019ecda04..b817cf2b5 100755 --- a/dstack/scripts/bin/dstack-cloud +++ b/dstack/scripts/bin/dstack-cloud @@ -1008,6 +1008,7 @@ class CloudDeploymentManager: target_dir.mkdir(parents=True, exist_ok=True) # Check if os_image is an absolute URL + fallback_url = None if os_image.startswith("http://") or os_image.startswith("https://"): download_url = os_image # Derive image name from URL filename @@ -1029,7 +1030,8 @@ class CloudDeploymentManager: logger.error("Expected format: dstack-- (e.g., dstack-nvidia-0.6.0)") return version = version_match.group(1) - download_url = f"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/Dstack-TEE/meta-dstack/releases/download/v{version}/{os_image}-uki.tar.gz" + download_url = f"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/Dstack-TEE/dstack/releases/download/guest-os-v{version}/{os_image}-uki.tar.gz" + fallback_url = f"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/Dstack-TEE/meta-dstack/releases/download/v{version}/{os_image}-uki.tar.gz" download_tar = target_dir / f"{os_image}-uki.tar.gz" if download_tar.exists(): @@ -1044,10 +1046,17 @@ class CloudDeploymentManager: try: # Use curl to download with progress bar - subprocess.run( - ["curl", "-L", "-o", str(download_tar), download_url], - check=True + result = subprocess.run( + ["curl", "-fL", "-o", str(download_tar), download_url], + check=False ) + if result.returncode != 0 and fallback_url: + logger.warning("Image not found in the monorepo release; trying the legacy release") + result = subprocess.run( + ["curl", "-fL", "-o", str(download_tar), fallback_url], + check=False + ) + result.check_returncode() logger.info(f"Successfully downloaded to {download_tar}") except subprocess.CalledProcessError as e: logger.error(f"Failed to download image: {e}") diff --git a/dstack/scripts/install.sh b/dstack/scripts/install.sh index 0a1afe882..af245443a 100755 --- a/dstack/scripts/install.sh +++ b/dstack/scripts/install.sh @@ -14,8 +14,8 @@ usage() { Install dstackup from source. Usage: - scripts/install.sh [options] - curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh + dstack/scripts/install.sh [options] + curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh Options: --repo URL Git repository to clone when not run from a checkout. @@ -115,7 +115,7 @@ need_cmd() { fi } -is_checkout() { +is_core_checkout() { [ -f "$1/Cargo.toml" ] && [ -d "$1/crates/dstackup" ] && [ -d "$1/crates/dstack-cli" ] && @@ -123,6 +123,18 @@ is_checkout() { [ -d "$1/supervisor" ] } +is_checkout() { + is_core_checkout "$1/dstack" || is_core_checkout "$1" +} + +core_dir() { + if is_core_checkout "$1/dstack"; then + echo "$1/dstack" + else + echo "$1" + fi +} + abs_dir() { (cd "$1" && pwd) } @@ -131,7 +143,10 @@ script_checkout() { case "$0" in */*) script_dir=$(dirname "$0") - if [ -d "$script_dir/.." ] && is_checkout "$script_dir/.."; then + if [ -d "$script_dir/../.." ] && is_checkout "$script_dir/../.."; then + abs_dir "$script_dir/../.." + return 0 + elif [ -d "$script_dir/.." ] && is_checkout "$script_dir/.."; then abs_dir "$script_dir/.." return 0 fi @@ -154,7 +169,7 @@ resolve_source() { need_cmd git if [ -n "$src" ] && [ -e "$src" ]; then - if ! is_checkout "$src" || [ ! -d "$src/.git" ]; then + if ! is_checkout "$src" || ! git -C "$src" rev-parse --git-dir >/dev/null 2>&1; then echo "error: $src exists but is not a dstack git checkout" >&2 exit 1 fi @@ -224,6 +239,7 @@ need_cmd cargo need_cmd install checkout=$(resolve_source) +core_checkout=$(core_dir "$checkout") bin_dir="$prefix/bin" if [ "$no_sudo" -eq 0 ] && [ "$(id -u)" -ne 0 ]; then @@ -241,13 +257,13 @@ fi echo "building dstackup from $checkout" ( - cd "$checkout" + cd "$core_checkout" cargo build --release \ -p dstackup ) install_bin() { - src_bin="$checkout/target/release/$1" + src_bin="$core_checkout/target/release/$1" dest_bin="$bin_dir/$2" if [ ! -f "$src_bin" ]; then echo "error: expected binary not found: $src_bin" >&2 diff --git a/dstack/test-scripts/snp-e2e-smoke.sh b/dstack/test-scripts/snp-e2e-smoke.sh index 30ec9eb7f..ea7d1373e 100755 --- a/dstack/test-scripts/snp-e2e-smoke.sh +++ b/dstack/test-scripts/snp-e2e-smoke.sh @@ -10,14 +10,14 @@ # built release binaries. # # Minimal setup used by the original smoke: -# cargo build --release -p dstack-vmm -p supervisor -p dstack-kms -# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/target/release +# cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor -p dstack-kms +# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/dstack/target/release # export DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1 # lab hosts only; auth API policy # test-scripts/snp-e2e-smoke.sh # # Useful overrides: # DSTACK_SNP_SMOKE_BASE=$HOME/dstack-snp-e2e -# DSTACK_SNP_SMOKE_REPO=$PWD +# DSTACK_SNP_SMOKE_REPO=$PWD/dstack # DSTACK_SNP_SMOKE_QEMU=/opt/AMDSEV/usr/local/bin/qemu-system-x86_64 # DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd # DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz @@ -46,16 +46,18 @@ # https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 # This is an external collateral-fetch boundary, not a guest boot or KMS startup # failure. -# One reproducible way is to build meta-dstack with its dstack submodule checked -# out to this PR branch, set the Yocto build MACHINE to `sev-snp` (not the +# One reproducible way is to build os/yocto from the same monorepo revision, +# set the Yocto build MACHINE to `sev-snp` (not the # default `tdx`, otherwise the guest kernel can miss AMD memory-encryption # support and reset immediately after OVMF loads the kernel/initrd), then point # DSTACK_SNP_SMOKE_IMAGE_NAME at the resulting dstack-dev image directory. set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CORE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" BASE="${DSTACK_SNP_SMOKE_BASE:-$HOME/dstack-snp-e2e}" -REPO="${DSTACK_SNP_SMOKE_REPO:-$(pwd)}" +REPO="${DSTACK_SNP_SMOKE_REPO:-$CORE_DIR}" BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" ART="$BASE/artifacts" LOG="$ART/snp-e2e-smoke.log" diff --git a/dstack/vmm/src/tests/test-compose.sh b/dstack/vmm/src/tests/test-compose.sh index a68a8fbcd..a87a2df98 100755 --- a/dstack/vmm/src/tests/test-compose.sh +++ b/dstack/vmm/src/tests/test-compose.sh @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 +# shellcheck disable=SC2317 # cleanup is invoked through an EXIT trap + # Test script for vmm-cli.py compose subcommand # Tests the refactored create_app_compose method that accepts args directly @@ -17,7 +19,8 @@ NC='\033[0m' # No Color # Test directory TEST_DIR="/tmp/vmm-cli-compose-test" -VMM_CLI="/home/kvin/sdc/home/meta-dstack/dstack/vmm/src/vmm-cli.py" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VMM_CLI="$SCRIPT_DIR/../vmm-cli.py" # Test counter TESTS_PASSED=0 @@ -45,7 +48,7 @@ cleanup() { setup() { echo -e "${YELLOW}Setting up test environment...${NC}" mkdir -p "$TEST_DIR" - + # Create test docker-compose.yml cat > "$TEST_DIR/docker-compose.yml" << 'EOF' version: '3' @@ -83,12 +86,12 @@ EOF # Test functions test_basic_compose() { print_test "Basic compose functionality with minimal parameters" - + if python3 "$VMM_CLI" compose \ --name test-basic \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --output "$TEST_DIR/basic-output.json" > /dev/null 2>&1; then - + if [[ -f "$TEST_DIR/basic-output.json" ]]; then # Verify JSON structure if jq -e '.name == "test-basic"' "$TEST_DIR/basic-output.json" > /dev/null && \ @@ -113,7 +116,7 @@ test_basic_compose() { test_full_compose() { print_test "Full compose functionality with all optional parameters" - + if python3 "$VMM_CLI" compose \ --name test-full \ --docker-compose "$TEST_DIR/docker-compose.yml" \ @@ -127,7 +130,7 @@ test_full_compose() { --public-sysinfo \ --no-instance-id \ --output "$TEST_DIR/full-output.json" > /dev/null 2>&1; then - + if [[ -f "$TEST_DIR/full-output.json" ]]; then # Verify all options are set correctly if jq -e '.name == "test-full"' "$TEST_DIR/full-output.json" > /dev/null && \ @@ -157,13 +160,13 @@ test_full_compose() { test_env_parsing() { print_test "Environment variable parsing" - + python3 "$VMM_CLI" compose \ --name test-env \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --env-file "$TEST_DIR/test.env" \ --output "$TEST_DIR/env-output.json" > /dev/null 2>&1 - + # Check if all environment variables are in allowed_envs if jq -e '.allowed_envs | contains(["API_KEY", "DEBUG", "PORT", "DATABASE_URL"])' "$TEST_DIR/env-output.json" > /dev/null; then print_success "Environment parsing test passed - all env vars included" @@ -175,12 +178,12 @@ test_env_parsing() { test_docker_compose_embedding() { print_test "Docker compose file embedding" - + python3 "$VMM_CLI" compose \ --name test-docker \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --output "$TEST_DIR/docker-output.json" > /dev/null 2>&1 - + # Check if docker-compose content is properly embedded if jq -e '.docker_compose_file | contains("nginx:latest")' "$TEST_DIR/docker-output.json" > /dev/null && \ jq -e '.docker_compose_file | contains("redis:alpine")' "$TEST_DIR/docker-output.json" > /dev/null; then @@ -193,13 +196,13 @@ test_docker_compose_embedding() { test_prelaunch_script() { print_test "Prelaunch script embedding" - + python3 "$VMM_CLI" compose \ --name test-prelaunch \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --prelaunch-script "$TEST_DIR/prelaunch.sh" \ --output "$TEST_DIR/prelaunch-output.json" > /dev/null 2>&1 - + # Check if prelaunch script is properly embedded if jq -e '.pre_launch_script | contains("Starting application...")' "$TEST_DIR/prelaunch-output.json" > /dev/null && \ jq -e '.pre_launch_script | contains("#!/bin/bash")' "$TEST_DIR/prelaunch-output.json" > /dev/null; then @@ -212,7 +215,7 @@ test_prelaunch_script() { test_error_handling() { print_test "Error handling for missing files" - + # Test missing docker-compose file if python3 "$VMM_CLI" compose \ --name test-error \ @@ -227,7 +230,7 @@ test_error_handling() { test_help_command() { print_test "Help command functionality" - + if python3 "$VMM_CLI" compose --help > /dev/null 2>&1; then print_success "Help command test passed - help displayed correctly" else @@ -238,22 +241,22 @@ test_help_command() { test_hash_generation() { print_test "Compose hash generation" - + # Create two identical compose files python3 "$VMM_CLI" compose \ --name test-hash-1 \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --output "$TEST_DIR/hash1-output.json" > "$TEST_DIR/hash1.log" 2>&1 - + python3 "$VMM_CLI" compose \ --name test-hash-1 \ --docker-compose "$TEST_DIR/docker-compose.yml" \ --output "$TEST_DIR/hash2-output.json" > "$TEST_DIR/hash2.log" 2>&1 - + # Extract hashes from output HASH1=$(grep "Compose hash:" "$TEST_DIR/hash1.log" | cut -d' ' -f3) HASH2=$(grep "Compose hash:" "$TEST_DIR/hash2.log" | cut -d' ' -f3) - + if [[ "$HASH1" == "$HASH2" ]] && [[ -n "$HASH1" ]]; then print_success "Hash generation test passed - identical inputs produce identical hashes" else @@ -266,22 +269,22 @@ test_hash_generation() { main() { echo -e "${YELLOW}=== VMM-CLI Compose Subcommand Test Suite ===${NC}" echo "" - + # Check dependencies if ! command -v jq &> /dev/null; then echo -e "${RED}Error: jq is required for JSON testing but not installed${NC}" exit 1 fi - + if [[ ! -f "$VMM_CLI" ]]; then echo -e "${RED}Error: VMM CLI not found at $VMM_CLI${NC}" exit 1 fi - + # Setup test environment trap cleanup EXIT setup - + # Run tests (continue even if some fail) test_help_command || true test_basic_compose || true @@ -291,7 +294,7 @@ main() { test_prelaunch_script || true test_hash_generation || true test_error_handling || true - + # Results summary echo "" echo -e "${YELLOW}=== Test Results ===${NC}" @@ -305,4 +308,4 @@ main() { } # Run main function -main "$@" \ No newline at end of file +main "$@" diff --git a/sdk/rust/Cargo.toml b/sdk/rust/Cargo.toml index b3bb3d398..f40c1a721 100644 --- a/sdk/rust/Cargo.toml +++ b/sdk/rust/Cargo.toml @@ -6,6 +6,7 @@ # SPDX-License-Identifier: Apache-2.0 [package] +workspace = "../../dstack" name = "dstack-sdk" version = "0.1.3" edition = "2021" diff --git a/sdk/rust/types/Cargo.toml b/sdk/rust/types/Cargo.toml index 2bedc93ad..86a06a468 100644 --- a/sdk/rust/types/Cargo.toml +++ b/sdk/rust/types/Cargo.toml @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 [package] +workspace = "../../../dstack" name = "dstack-sdk-types" version = "0.1.3" edition = "2021" diff --git a/sdk/simulator/build.sh b/sdk/simulator/build.sh index fca993175..f3632fa98 100755 --- a/sdk/simulator/build.sh +++ b/sdk/simulator/build.sh @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 -cd $(dirname $0) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" || exit cargo build --release -p dstack-guest-agent-simulator -cp ../../target/release/dstack-simulator . - +cp ../../dstack/target/release/dstack-simulator . From 132039cfa5e4c32f1151fb03af67e87a298e2a4b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:41:06 -0700 Subject: [PATCH 421/435] refactor: define extensible guest os backend interface --- .github/workflows/guest-os.yml | 38 +- dstack/basefiles/tdx-attest.conf | 1 - os/README.md | 75 +++ os/build.sh | 65 +++ os/common/README.md | 11 + .../common/rootfs}/app-compose.service | 0 .../common/rootfs}/app-compose.sh | 3 +- .../containerd.service.d/dstack-prepare.conf | 0 .../docker.service.d/dstack-guest-agent.conf | 0 .../docker.service.d/dstack-prepare.conf | 0 .../common/rootfs}/dstack-guest-agent.service | 0 .../common/rootfs}/dstack-guest-agent.socket | 0 .../common/rootfs}/dstack-prepare.service | 0 .../common/rootfs}/dstack-prepare.sh | 0 .../common/rootfs}/ephemeral-docker.sh | 10 +- .../common/rootfs}/journald.conf | 0 .../basefiles => os/common/rootfs}/llmnr.conf | 2 +- .../common/rootfs}/sysctl.d/99-dstack.conf | 0 os/common/rootfs/tdx-attest.conf | 1 + .../common/rootfs}/wg-checker.service | 0 .../common/rootfs}/wg-checker.sh | 0 os/image/README.md | 10 + os/image/assemble.sh | 472 ++++++++++++++++++ .../bin => image}/authenticode_hash.py | 29 +- os/spec/artifact-manifest.schema.json | 102 ++++ os/yocto/README.md | 44 +- os/yocto/build.sh | 17 +- os/yocto/dev-setup | 32 +- .../recipes-core/dstack-guest/dstack-guest.bb | 55 +- os/yocto/mkimage.sh | 416 +-------------- os/yocto/repro-build/repro-build.sh | 23 +- os/yocto/scripts/bin/mk-image-mr.sh | 2 +- os/yocto/scripts/export-artifacts.sh | 211 ++++++++ os/yocto/setup.d/nvidia-layer.sh | 2 +- 34 files changed, 1135 insertions(+), 486 deletions(-) delete mode 100644 dstack/basefiles/tdx-attest.conf create mode 100644 os/README.md create mode 100755 os/build.sh create mode 100644 os/common/README.md rename {dstack/basefiles => os/common/rootfs}/app-compose.service (100%) rename {dstack/basefiles => os/common/rootfs}/app-compose.sh (93%) rename {dstack/basefiles => os/common/rootfs}/containerd.service.d/dstack-prepare.conf (100%) rename {dstack/basefiles => os/common/rootfs}/docker.service.d/dstack-guest-agent.conf (100%) rename {dstack/basefiles => os/common/rootfs}/docker.service.d/dstack-prepare.conf (100%) rename {dstack/basefiles => os/common/rootfs}/dstack-guest-agent.service (100%) rename {dstack/basefiles => os/common/rootfs}/dstack-guest-agent.socket (100%) rename {dstack/basefiles => os/common/rootfs}/dstack-prepare.service (100%) rename {dstack/basefiles => os/common/rootfs}/dstack-prepare.sh (100%) rename {dstack/basefiles => os/common/rootfs}/ephemeral-docker.sh (95%) rename {dstack/basefiles => os/common/rootfs}/journald.conf (100%) rename {dstack/basefiles => os/common/rootfs}/llmnr.conf (52%) rename {dstack/basefiles => os/common/rootfs}/sysctl.d/99-dstack.conf (100%) create mode 100644 os/common/rootfs/tdx-attest.conf rename {dstack/basefiles => os/common/rootfs}/wg-checker.service (100%) rename {dstack/basefiles => os/common/rootfs}/wg-checker.sh (100%) create mode 100644 os/image/README.md create mode 100755 os/image/assemble.sh rename os/{yocto/scripts/bin => image}/authenticode_hash.py (79%) create mode 100644 os/spec/artifact-manifest.schema.json create mode 100755 os/yocto/scripts/export-artifacts.sh diff --git a/.github/workflows/guest-os.yml b/.github/workflows/guest-os.yml index 25ff79a8b..fc050e1a0 100644 --- a/.github/workflows/guest-os.yml +++ b/.github/workflows/guest-os.yml @@ -1,10 +1,14 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + name: Build Guest Images on: workflow_dispatch: inputs: tag: - description: 'Tag to create and release (e.g. v0.5.9). Leave empty to build only.' + description: 'Tag to create and release (e.g. guest-os-v0.5.9). Leave empty to build only.' required: false default: '' type: string @@ -17,20 +21,30 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive + uses: actions/checkout@v5 + + - name: Checkout Yocto dependencies + run: | + git submodule update --init --depth 1 -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security - name: Reproducible build - run: cd repro-build && ./repro-build.sh -n + run: cd os/yocto/repro-build && ./repro-build.sh -n - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: guest-images path: | - repro-build/dist/*.tar.gz - repro-build/dist/reproduce.sh + os/yocto/repro-build/dist/*.tar.gz + os/yocto/repro-build/dist/reproduce.sh retention-days: 30 release: @@ -43,7 +57,7 @@ jobs: contents: write steps: - name: Checkout default branch - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ github.event.repository.default_branch }} @@ -51,13 +65,13 @@ jobs: uses: actions/download-artifact@v4 with: name: guest-images - path: repro-build/dist + path: os/yocto/repro-build/dist - name: Validate release tag env: TAG: ${{ inputs.tag }} run: | - echo "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' + echo "$TAG" | grep -Eq '^guest-os-v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' - name: Create tag and release env: @@ -67,6 +81,6 @@ jobs: git tag "$TAG" git push origin "$TAG" gh release create "$TAG" \ - repro-build/dist/*.tar.gz \ - repro-build/dist/reproduce.sh \ + os/yocto/repro-build/dist/*.tar.gz \ + os/yocto/repro-build/dist/reproduce.sh \ --title "$TAG" --generate-notes diff --git a/dstack/basefiles/tdx-attest.conf b/dstack/basefiles/tdx-attest.conf deleted file mode 100644 index d7c6361ae..000000000 --- a/dstack/basefiles/tdx-attest.conf +++ /dev/null @@ -1 +0,0 @@ -port=4050 \ No newline at end of file diff --git a/os/README.md b/os/README.md new file mode 100644 index 000000000..4680ac689 --- /dev/null +++ b/os/README.md @@ -0,0 +1,75 @@ +# dstack guest OS + +The `os/` tree separates the guest-OS contract from the implementation used to +produce it. + +```text +os/ +├── build.sh # backend selector +├── common/rootfs/ # backend-independent guest payload +├── image/ # backend-independent release assembler +├── spec/ # versioned backend artifact contract +└── yocto/ # the currently implemented backend + ├── deps/ # external Yocto layers (git submodules) + └── layers/ # dstack-owned Yocto layers +``` + +A future backend can be added at `os//` (for example `os/mkosi/`) +without moving shared payload or duplicating release packaging. No mkosi +backend is implemented today. + +## Build + +Initialize only the Yocto dependencies, then invoke the common entrypoint: + +```bash +git submodule update --init -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security + +./os/build.sh --backend yocto +``` + +`--flavors "prod dev"` selects image flavors and `--build-dir DIR` selects the +native backend build directory. + +## Backend contract + +Each backend has two boundaries: + +1. `os//build.sh image [build-dir]` builds native artifacts. +2. The backend exports `os/spec/artifact-manifest.schema.json` version 1 and + calls `os/image/assemble.sh --manifest `. + +Artifact paths in a manifest are relative to the manifest. The common assembler +owns partitioned disk creation, release metadata, TDX/SNP/GCP measurement +material, checksums, and tarballs. Backends own how the kernel, initramfs, +firmware, dm-verity rootfs, and optional UKI are built. + +The Yocto compatibility entrypoint `os/yocto/mkimage.sh` demonstrates this +split: `scripts/export-artifacts.sh` handles BitBake paths and +`../image/assemble.sh` handles backend-independent packaging. + +## Source boundaries + +The guest recipe stages only the inputs it needs: + +- `dstack/` for core Rust services, +- `sdk/rust/` for public Rust SDK workspace members, +- `os/common/rootfs/` for OS-owned payload. + +It does not copy the entire repository or depend on a nested dstack submodule. + +## Licensing + +The monorepo contains multiple license scopes. Core, SDK, documentation, tools, +and `os/common/rootfs/` remain Apache-2.0. The imported Yocto implementation and +derived image-assembly code retain the Business Source License 1.1 terms in +`os/yocto/LICENSE`. Individual vendored recipes and patches may carry their own +notices; moving them into this repository does not relicense them. diff --git a/os/build.sh b/os/build.sh new file mode 100755 index 000000000..1d51cf5bc --- /dev/null +++ b/os/build.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +usage() { + cat </build.sh and implements the "image" action. +The common image contract is documented in os/README.md. +EOF +} + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BACKEND=yocto +FLAVORS=${FLAVORS:-prod} +BUILD_DIR= + +while [ $# -gt 0 ]; do + case "$1" in + --backend) + BACKEND=$2 + shift 2 + ;; + --flavors) + FLAVORS=$2 + shift 2 + ;; + --build-dir) + BUILD_DIR=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +BACKEND_SCRIPT="$SCRIPT_DIR/$BACKEND/build.sh" +if [ ! -x "$BACKEND_SCRIPT" ]; then + echo "Error: OS backend is not available: $BACKEND" >&2 + echo "Expected executable: $BACKEND_SCRIPT" >&2 + exit 1 +fi + +export FLAVORS +if [ -n "$BUILD_DIR" ]; then + exec "$BACKEND_SCRIPT" image "$BUILD_DIR" +else + exec "$BACKEND_SCRIPT" image +fi diff --git a/os/common/README.md b/os/common/README.md new file mode 100644 index 000000000..32088a905 --- /dev/null +++ b/os/common/README.md @@ -0,0 +1,11 @@ +# Common guest rootfs payload + +`rootfs/` contains files installed into every dstack guest independently of the +OS build backend. It is intentionally separate from Rust application source and +from Yocto recipes. + +A backend should stage these files into their documented destinations. The +current authoritative mapping is in +`../yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb`. When a +second backend is introduced, keep the payload canonical here and add a +backend-specific installer rather than copying the files. diff --git a/dstack/basefiles/app-compose.service b/os/common/rootfs/app-compose.service similarity index 100% rename from dstack/basefiles/app-compose.service rename to os/common/rootfs/app-compose.service diff --git a/dstack/basefiles/app-compose.sh b/os/common/rootfs/app-compose.sh similarity index 93% rename from dstack/basefiles/app-compose.sh rename to os/common/rootfs/app-compose.sh index 0387ed348..ea5ed9de0 100644 --- a/dstack/basefiles/app-compose.sh +++ b/os/common/rootfs/app-compose.sh @@ -9,9 +9,10 @@ SYS_CONFIG_FILE="$HOST_SHARED_DIR/.sys-config.json" CFG_PCCS_URL=$([ -f "$SYS_CONFIG_FILE" ] && jq -r '.pccs_url//""' "$SYS_CONFIG_FILE" || echo "") export PCCS_URL=${PCCS_URL:-$CFG_PCCS_URL} -if [ $(jq 'has("pre_launch_script")' app-compose.json) == true ]; then +if [ "$(jq 'has("pre_launch_script")' app-compose.json)" = true ]; then echo "Running pre-launch script" dstack-util notify-host -e "boot.progress" -d "pre-launch" || true + # shellcheck disable=SC1090 source <(jq -r '.pre_launch_script' app-compose.json) fi diff --git a/dstack/basefiles/containerd.service.d/dstack-prepare.conf b/os/common/rootfs/containerd.service.d/dstack-prepare.conf similarity index 100% rename from dstack/basefiles/containerd.service.d/dstack-prepare.conf rename to os/common/rootfs/containerd.service.d/dstack-prepare.conf diff --git a/dstack/basefiles/docker.service.d/dstack-guest-agent.conf b/os/common/rootfs/docker.service.d/dstack-guest-agent.conf similarity index 100% rename from dstack/basefiles/docker.service.d/dstack-guest-agent.conf rename to os/common/rootfs/docker.service.d/dstack-guest-agent.conf diff --git a/dstack/basefiles/docker.service.d/dstack-prepare.conf b/os/common/rootfs/docker.service.d/dstack-prepare.conf similarity index 100% rename from dstack/basefiles/docker.service.d/dstack-prepare.conf rename to os/common/rootfs/docker.service.d/dstack-prepare.conf diff --git a/dstack/basefiles/dstack-guest-agent.service b/os/common/rootfs/dstack-guest-agent.service similarity index 100% rename from dstack/basefiles/dstack-guest-agent.service rename to os/common/rootfs/dstack-guest-agent.service diff --git a/dstack/basefiles/dstack-guest-agent.socket b/os/common/rootfs/dstack-guest-agent.socket similarity index 100% rename from dstack/basefiles/dstack-guest-agent.socket rename to os/common/rootfs/dstack-guest-agent.socket diff --git a/dstack/basefiles/dstack-prepare.service b/os/common/rootfs/dstack-prepare.service similarity index 100% rename from dstack/basefiles/dstack-prepare.service rename to os/common/rootfs/dstack-prepare.service diff --git a/dstack/basefiles/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh similarity index 100% rename from dstack/basefiles/dstack-prepare.sh rename to os/common/rootfs/dstack-prepare.sh diff --git a/dstack/basefiles/ephemeral-docker.sh b/os/common/rootfs/ephemeral-docker.sh similarity index 95% rename from dstack/basefiles/ephemeral-docker.sh rename to os/common/rootfs/ephemeral-docker.sh index 9f568eab9..40c596ef2 100755 --- a/dstack/basefiles/ephemeral-docker.sh +++ b/os/common/rootfs/ephemeral-docker.sh @@ -28,7 +28,7 @@ cleanup() { if [ -n "$DOCKERD_PID" ] && kill -0 $DOCKERD_PID 2>/dev/null; then kill -TERM $DOCKERD_PID 2>/dev/null # Wait with timeout - for i in $(seq 1 50); do + for _ in $(seq 1 50); do kill -0 $DOCKERD_PID 2>/dev/null || break sleep 0.1 done @@ -40,7 +40,7 @@ cleanup() { # Stop containerd if [ -n "$CONTAINERD_PID" ] && kill -0 $CONTAINERD_PID 2>/dev/null; then kill -TERM $CONTAINERD_PID 2>/dev/null - for i in $(seq 1 50); do + for _ in $(seq 1 50); do kill -0 $CONTAINERD_PID 2>/dev/null || break sleep 0.1 done @@ -50,7 +50,7 @@ cleanup() { # Unmount any netns that docker created if [ -d "$TMPDIR/docker-exec/netns" ]; then - find "$TMPDIR/docker-exec/netns" -type f 2>/dev/null | while read ns; do + find "$TMPDIR/docker-exec/netns" -type f 2>/dev/null | while read -r ns; do umount "$ns" 2>/dev/null || true done fi @@ -75,7 +75,7 @@ CONTAINERD_PID=$! # Wait for containerd socket with timeout TIMEOUT=100 # 10 seconds -for i in $(seq 1 $TIMEOUT); do +for _ in $(seq 1 "$TIMEOUT"); do [ -S "$TMPDIR/containerd.sock" ] && break if ! kill -0 $CONTAINERD_PID 2>/dev/null; then echo "Error: containerd exited unexpectedly" >&2 @@ -99,7 +99,7 @@ dockerd \ DOCKERD_PID=$! # Wait for docker socket with timeout -for i in $(seq 1 $TIMEOUT); do +for _ in $(seq 1 "$TIMEOUT"); do [ -S "$TMPDIR/docker.sock" ] && break if ! kill -0 $DOCKERD_PID 2>/dev/null; then echo "Error: dockerd exited unexpectedly" >&2 diff --git a/dstack/basefiles/journald.conf b/os/common/rootfs/journald.conf similarity index 100% rename from dstack/basefiles/journald.conf rename to os/common/rootfs/journald.conf diff --git a/dstack/basefiles/llmnr.conf b/os/common/rootfs/llmnr.conf similarity index 52% rename from dstack/basefiles/llmnr.conf rename to os/common/rootfs/llmnr.conf index 4b0ccf845..ad4ea0b28 100644 --- a/dstack/basefiles/llmnr.conf +++ b/os/common/rootfs/llmnr.conf @@ -1,2 +1,2 @@ [Resolve] -LLMNR=no \ No newline at end of file +LLMNR=no diff --git a/dstack/basefiles/sysctl.d/99-dstack.conf b/os/common/rootfs/sysctl.d/99-dstack.conf similarity index 100% rename from dstack/basefiles/sysctl.d/99-dstack.conf rename to os/common/rootfs/sysctl.d/99-dstack.conf diff --git a/os/common/rootfs/tdx-attest.conf b/os/common/rootfs/tdx-attest.conf new file mode 100644 index 000000000..aa6ae961e --- /dev/null +++ b/os/common/rootfs/tdx-attest.conf @@ -0,0 +1 @@ +port=4050 diff --git a/dstack/basefiles/wg-checker.service b/os/common/rootfs/wg-checker.service similarity index 100% rename from dstack/basefiles/wg-checker.service rename to os/common/rootfs/wg-checker.service diff --git a/dstack/basefiles/wg-checker.sh b/os/common/rootfs/wg-checker.sh similarity index 100% rename from dstack/basefiles/wg-checker.sh rename to os/common/rootfs/wg-checker.sh diff --git a/os/image/README.md b/os/image/README.md new file mode 100644 index 000000000..fcfc70b38 --- /dev/null +++ b/os/image/README.md @@ -0,0 +1,10 @@ +# Common image assembly + +`assemble.sh` consumes the versioned manifest in +`../spec/artifact-manifest.schema.json`. It must not inspect a backend build +tree or invoke backend-specific tools such as BitBake. + +The assembler creates the partitioned rootfs, metadata, measurement CBOR files, +unified digest, and release archives. Backend exporters may use symlinks for +large local artifacts; all paths recorded in the manifest itself are relative +to the manifest directory. diff --git a/os/image/assemble.sh b/os/image/assemble.sh new file mode 100755 index 000000000..90780221e --- /dev/null +++ b/os/image/assemble.sh @@ -0,0 +1,472 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + +set -euo pipefail + +DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} +ENABLE_UKI_IMAGE=${ENABLE_UKI_IMAGE:-1} + +# Fixed GPT GUIDs so partitioned images are reproducible (sgdisk randomizes by +# default). Partitions are located by PARTLABEL, not GUID, so these are arbitrary. +DSTACK_DISK_GUID=${DSTACK_DISK_GUID:-d5acc000-0000-4000-8000-000000000000} +DSTACK_ROOTFS_PART_GUID=${DSTACK_ROOTFS_PART_GUID:-d5acc000-0000-4000-8000-000000000001} +DSTACK_EFI_PART_GUID=${DSTACK_EFI_PART_GUID:-d5acc000-0000-4000-8000-000000000002} + +usage() { + cat </dist) + OUTPUT_DIR Unpacked release directory + DSTACK_TAR_RELEASE Create release tarballs (default: 1) + ENABLE_UKI_IMAGE Create the optional UKI disk image (default: 1) + DSTACK_MR_BIN Existing dstack-mr binary +EOF +} + +MANIFEST= +while [ $# -gt 0 ]; do + case "$1" in + --manifest) + MANIFEST=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "$MANIFEST" ]; then + echo "Error: --manifest is required" >&2 + usage >&2 + exit 1 +fi +if [ ! -f "$MANIFEST" ]; then + echo "Error: artifact manifest not found: $MANIFEST" >&2 + exit 1 +fi +MANIFEST=$(realpath "$MANIFEST") + +# Read and validate the small contract without requiring jq or jsonschema. +mapfile -d '' -t MANIFEST_VALUES < <( + python3 - "$MANIFEST" <<'PYMANIFEST' +import json +import os +import sys + +manifest_path = os.path.realpath(sys.argv[1]) +with open(manifest_path, encoding="utf-8") as file: + data = json.load(file) + +if data.get("schema_version") != 1: + raise SystemExit("unsupported artifact manifest schema_version") +if not isinstance(data.get("image", {}).get("is_dev"), bool): + raise SystemExit("image.is_dev must be a boolean") + +base = os.path.dirname(manifest_path) + +def required(obj, *keys): + value = obj + for key in keys: + if not isinstance(value, dict) or key not in value: + raise SystemExit("missing manifest field: " + ".".join(keys)) + value = value[key] + if value is None or value == "": + raise SystemExit("empty manifest field: " + ".".join(keys)) + return value + +def artifact(name, optional=False): + value = data.get("artifacts", {}).get(name) + if value in (None, ""): + if optional: + return "" + raise SystemExit(f"missing manifest artifact: {name}") + if not isinstance(value, str): + raise SystemExit(f"artifact path must be a string: {name}") + normalized = os.path.normpath(value) + if os.path.isabs(value) or normalized == ".." or normalized.startswith("../"): + raise SystemExit(f"artifact path must be relative to the manifest: {name}") + path = os.path.realpath(os.path.join(base, value)) + if not os.path.isfile(path): + raise SystemExit(f"artifact does not exist: {name}: {path}") + return path + +values = [ + required(data, "backend"), + required(data, "image", "name"), + required(data, "image", "version"), + required(data, "image", "flavor"), + "true" if data["image"]["is_dev"] else "false", + required(data, "source", "git_revision"), + required(data, "boot", "ovmf_variant"), + required(data, "verity", "root_hash"), + str(required(data, "verity", "data_size")), + artifact("initramfs"), + artifact("kernel"), + artifact("firmware"), + artifact("rootfs_verity"), + artifact("firmware_sev", optional=True), + artifact("uki", optional=True), +] +for value in values: + if not isinstance(value, str): + raise SystemExit("manifest scalar fields must be strings") + sys.stdout.buffer.write(value.encode() + b"\0") +PYMANIFEST +) + +if [ "${#MANIFEST_VALUES[@]}" -ne 15 ]; then + echo "Error: failed to read artifact manifest: $MANIFEST" >&2 + exit 1 +fi + +BACKEND=${MANIFEST_VALUES[0]} +DIST_NAME=${MANIFEST_VALUES[1]} +DSTACK_VERSION=${MANIFEST_VALUES[2]} +FLAVOR=${MANIFEST_VALUES[3]} +IS_DEV=${MANIFEST_VALUES[4]} +GIT_REVISION=${MANIFEST_VALUES[5]} +OVMF_VARIANT=${MANIFEST_VALUES[6]} +ROOT_HASH=${MANIFEST_VALUES[7]} +DATA_SIZE=${MANIFEST_VALUES[8]} +INITRAMFS_IMAGE=${MANIFEST_VALUES[9]} +KERNEL_IMAGE=${MANIFEST_VALUES[10]} +OVMF_FIRMWARE=${MANIFEST_VALUES[11]} +ROOTFS_IMAGE=${MANIFEST_VALUES[12]} +OVMF_SEV_FIRMWARE=${MANIFEST_VALUES[13]} +UKI_IMAGE=${MANIFEST_VALUES[14]} + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(realpath "$SCRIPT_DIR/../..") +AUTHENTICODE_HASH_SCRIPT="${SCRIPT_DIR}/authenticode_hash.py" + +MANIFEST_DIR=$(dirname "$MANIFEST") +DIST_DIR=${DIST_DIR:-"${MANIFEST_DIR}/dist"} +mkdir -p "$DIST_DIR" +DIST_DIR=$(realpath "$DIST_DIR") +OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} +OUTPUT_DIR=$(realpath -m "$OUTPUT_DIR") +case "$OUTPUT_DIR" in + /|"$DIST_DIR"|"$MANIFEST_DIR"|"$REPO_ROOT") + echo "Error: refusing unsafe output directory: $OUTPUT_DIR" >&2 + exit 1 + ;; +esac +IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} +IMAGE_TAR_UKI=${IMAGE_TAR_UKI:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-uki.tar.gz"} +TAR_DIR_NAME=$(basename "$OUTPUT_DIR") + +echo "Assembling ${DIST_NAME} ${DSTACK_VERSION} from ${BACKEND} artifacts (${FLAVOR})" + +verbose() { + printf '+ ' + printf '%q ' "$@" + printf '\n' + "$@" +} + +align_up() { + local value=$1 + local align=$2 + echo $(( ( (value + align - 1) / align ) * align )) +} + +calc_authenticode_hash() { + local file="$1" + + if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then + return 0 + fi + + python3 "$AUTHENTICODE_HASH_SCRIPT" "$file" 2>/dev/null || true +} + +write_authenticode_hash() { + local file="$1" + local out_file="$2" + + if [[ ! -f "$file" ]]; then + return 0 + fi + + if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then + echo "Warning: authenticode_hash.py not found or python3 not available, skipping Authenticode hash calculation" >&2 + return 0 + fi + + echo "Calculating UKI Authenticode hash..." + local auth_hash + auth_hash=$(calc_authenticode_hash "$file") + if [[ -n "$auth_hash" ]]; then + echo "$auth_hash" > "$out_file" + echo "UKI Authenticode hash: $auth_hash" + else + echo "Warning: Failed to calculate UKI Authenticode hash" >&2 + fi +} + +ensure_dstack_mr() { + DSTACK_SRC="${DSTACK_SRC:-$REPO_ROOT/dstack}" + if [ -z "${DSTACK_MR_BIN:-}" ] && [ -x "$DSTACK_SRC/target/release/dstack-mr" ]; then + DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" + fi + if [ -z "${DSTACK_MR_BIN:-}" ]; then + echo "Building dstack-mr to compute OS-image measurement material" + ( cd "$DSTACK_SRC" && cargo build --release -p dstack-mr ) + DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" + fi +} + +create_partitioned_rootfs() { + local rootfs_img="$1" + local output_img="$2" + ( + set -e + local align=$((1024 * 1024)) + local sector=512 + local rootfs_size + local rootfs_size_aligned + local total_size + rootfs_size=$(stat -L -c %s "$rootfs_img") + rootfs_size_aligned=$(align_up "$rootfs_size" "$align") + local rootfs_start=$align + # Leave extra room for GPT headers (1MB at start, 1MB at end) + total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) "$align") + + truncate -s "$total_size" "$output_img" + + local root_start_sector=$((rootfs_start / sector)) + local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) + + sgdisk --zap-all "$output_img" >/dev/null + # Fixed GUIDs keep the image bit-for-bit reproducible (GPT otherwise + # randomizes disk/partition GUIDs). The rootfs is located by PARTLABEL. + sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ + --new=1:${root_start_sector}:${root_end_sector} --typecode=1:8300 \ + --partition-guid=1:"${DSTACK_ROOTFS_PART_GUID}" \ + --change-name=1:'dstack-rootfs' "$output_img" >/dev/null + + dd if="$rootfs_img" of="$output_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none + ) +} + +build_uki_disk_image() { + local disk_img="$1" + local uki_file="$2" + local rootfs_img="$3" + ( + set -e + local align=$((1024 * 1024)) + local sector=512 + local efi_size=$((256 * 1024 * 1024)) + local efi_size_aligned + local rootfs_size + local rootfs_size_aligned + local total_size + efi_size_aligned=$(align_up "$efi_size" "$align") + rootfs_size=$(stat -L -c %s "$rootfs_img") + rootfs_size_aligned=$(align_up "$rootfs_size" "$align") + local efi_start=$align + local rootfs_start=$((efi_start + efi_size_aligned)) + # Leave extra room for the backup GPT header + total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) "$align") + + truncate -s "$total_size" "$disk_img" + + local efi_start_sector=$((efi_start / sector)) + local efi_end_sector=$((efi_start_sector + (efi_size_aligned / sector) - 1)) + local root_start_sector=$((rootfs_start / sector)) + local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) + + sgdisk --zap-all "$disk_img" >/dev/null + sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ + --new=1:${efi_start_sector}:${efi_end_sector} --typecode=1:ef00 \ + --partition-guid=1:"${DSTACK_EFI_PART_GUID}" \ + --change-name=1:'EFI System Partition' "$disk_img" >/dev/null + sgdisk --new=2:${root_start_sector}:${root_end_sector} --typecode=2:8300 \ + --partition-guid=2:"${DSTACK_ROOTFS_PART_GUID}" \ + --change-name=2:'dstack-rootfs' "$disk_img" >/dev/null + + local tmp_dir + tmp_dir=$(mktemp -d) + trap 'rm -rf "$tmp_dir"' EXIT + + # Create EFI filesystem with UKI as bootloader + local efi_img=${tmp_dir}/efi.img + mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null + mmd -i "$efi_img" ::EFI ::EFI/BOOT + mcopy -i "$efi_img" "$uki_file" ::EFI/BOOT/BOOTX64.EFI + + dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none + dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none + ) +} + +create_uki_artifacts() { + local uki_dir="$1" + mkdir -p "$uki_dir" + + echo "Building UKI disk image at ${uki_dir}/disk.raw" + build_uki_disk_image "${uki_dir}/disk.raw" "$UKI_IMAGE" "$ROOTFS_IMAGE" + + write_authenticode_hash "$UKI_IMAGE" "${uki_dir}/auth_hash.txt" +} + +# Create bare metal image directory +verbose rm -rf "${OUTPUT_DIR}/" +verbose mkdir -p "${OUTPUT_DIR}/" +verbose cp "$INITRAMFS_IMAGE" "${OUTPUT_DIR}/initramfs.cpio.gz" +verbose cp "$KERNEL_IMAGE" "${OUTPUT_DIR}/bzImage" +verbose cp "$OVMF_FIRMWARE" "${OUTPUT_DIR}/ovmf.fd" + +# AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP +# launch can select it via the metadata.json "bios-sev" field below. The SEV +# firmware blob itself is NOT added directly to sha256sum.txt; when present, its +# OVMF hash/sections are committed by measurement.snp.cbor, and that file is +# part of digest.txt. This does not change any TDX hardware +# measurement (MRTD comes from ovmf.fd, RTMRs from kernel/cmdline/rootfs) -- it +# only changes dstack's image-bundle digest. +HAVE_OVMF_SEV=0 +BIOS_SEV_JSON="" +if [ -n "$OVMF_SEV_FIRMWARE" ]; then + verbose cp "$OVMF_SEV_FIRMWARE" "${OUTPUT_DIR}/ovmf-sev.fd" + HAVE_OVMF_SEV=1 + # Inserted after the "bios" line in metadata.json (see below). + BIOS_SEV_JSON=' + "bios-sev": "ovmf-sev.fd",' +fi + +echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" +# Bare-metal partitioning needs sgdisk (from the 'gdisk' package). +if ! command -v sgdisk >/dev/null; then + echo "Error: cannot create partitioned rootfs image because 'sgdisk' is missing; install 'gdisk'." >&2 + exit 1 +fi +create_partitioned_rootfs "$ROOTFS_IMAGE" "${OUTPUT_DIR}/rootfs.img.parted.verity" + +echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json (ovmf_variant=$OVMF_VARIANT)" + +KARG0="console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0" +KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" +KARG2="dstack.rootfs_hash=$ROOT_HASH dstack.rootfs_size=$DATA_SIZE" + +cat < "${OUTPUT_DIR}/metadata.json" +{ + "bios": "ovmf.fd",${BIOS_SEV_JSON} + "kernel": "bzImage", + "cmdline": "$KARG0 $KARG1 $KARG2", + "initrd": "initramfs.cpio.gz", + "rootfs": "rootfs.img.parted.verity", + "version": "$DSTACK_VERSION", + "git_revision": "$GIT_REVISION", + "shared_ro": true, + "is_dev": ${IS_DEV}, + "ovmf_variant": "$OVMF_VARIANT" +} +EOF + +ensure_dstack_mr + +echo "Generating measurement.tdx.cbor via ${DSTACK_MR_BIN}" +"${DSTACK_MR_BIN}" tdx-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.tdx.cbor" + +HAVE_MEASUREMENT_SNP=0 +if [ "$HAVE_OVMF_SEV" = "1" ]; then + echo "Generating measurement.snp.cbor via ${DSTACK_MR_BIN}" + "${DSTACK_MR_BIN}" snp-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.snp.cbor" + HAVE_MEASUREMENT_SNP=1 +fi + +# Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR +UKI_CREATED=0 +if [ "$ENABLE_UKI_IMAGE" = "1" ]; then + if [[ -z "$UKI_IMAGE" ]]; then + echo "Skipping UKI disk image creation because the backend did not export a UKI" >&2 + elif command -v sgdisk >/dev/null && \ + command -v mkfs.vfat >/dev/null && \ + command -v mcopy >/dev/null && \ + command -v mmd >/dev/null; then + create_uki_artifacts "${OUTPUT_DIR}" + UKI_CREATED=1 + else + echo "Error: cannot create UKI disk image because required tools are missing" >&2 + echo "Missing tools are among: sgdisk (gdisk), mkfs.vfat (dosfstools), mcopy/mmd (mtools)" >&2 + echo "Install them (e.g. apt-get install -y gdisk dosfstools mtools) or set ENABLE_UKI_IMAGE=0" >&2 + exit 1 + fi +fi + +HAVE_MEASUREMENT_GCP=0 +if [[ "$UKI_CREATED" = "1" ]]; then + if [[ ! -f "${OUTPUT_DIR}/auth_hash.txt" ]]; then + echo "Error: UKI image was created but auth_hash.txt is missing" >&2 + exit 1 + fi + echo "Generating measurement.gcp.cbor via ${DSTACK_MR_BIN}" + "${DSTACK_MR_BIN}" gcp-measurement-cbor "${OUTPUT_DIR}/auth_hash.txt" > "${OUTPUT_DIR}/measurement.gcp.cbor" + HAVE_MEASUREMENT_GCP=1 +fi + +echo "Generating unified image digest to ${OUTPUT_DIR}/" +CHECKSUM_FILES=(ovmf.fd bzImage initramfs.cpio.gz metadata.json measurement.tdx.cbor) +if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then + CHECKSUM_FILES+=(measurement.snp.cbor) +fi +if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then + CHECKSUM_FILES+=(measurement.gcp.cbor) +fi +( + cd "${OUTPUT_DIR}/" + sha256sum "${CHECKSUM_FILES[@]}" > sha256sum.txt + sha256sum sha256sum.txt | awk '{print $1}' > digest.txt +) + +if [ "$DSTACK_TAR_RELEASE" = "1" ]; then + OUTPUT_DIR=$(realpath "${OUTPUT_DIR}") + PARENT_DIR=$(dirname "${OUTPUT_DIR}") + + # Bare metal tarball: all files except disk.raw and auth_hash.txt + rm -rf "${IMAGE_TAR}" + echo "Archiving bare metal image to ${IMAGE_TAR}" + BARE_METAL_FILES=(rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json measurement.tdx.cbor) + if [ "$HAVE_OVMF_SEV" = "1" ]; then + BARE_METAL_FILES+=(ovmf-sev.fd) + fi + if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then + BARE_METAL_FILES+=(measurement.snp.cbor) + fi + if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then + BARE_METAL_FILES+=(measurement.gcp.cbor) + fi + BARE_METAL_TAR_FILES=() + for file in "${BARE_METAL_FILES[@]}"; do + BARE_METAL_TAR_FILES+=("$TAR_DIR_NAME/$file") + done + (cd "$PARENT_DIR" && tar -czvf "$IMAGE_TAR" "${BARE_METAL_TAR_FILES[@]}") + echo + + # UKI tarball: GCP boot disk plus the unified OS-image identity material. + if [[ "$UKI_CREATED" = "1" ]]; then + rm -rf "${IMAGE_TAR_UKI}" + echo "Archiving UKI image to ${IMAGE_TAR_UKI}" + UKI_FILES=(disk.raw digest.txt sha256sum.txt measurement.gcp.cbor) + UKI_TAR_FILES=() + for file in "${UKI_FILES[@]}"; do + UKI_TAR_FILES+=("$TAR_DIR_NAME/$file") + done + (cd "$PARENT_DIR" && tar -czvf "$IMAGE_TAR_UKI" "${UKI_TAR_FILES[@]}") + echo + fi +fi diff --git a/os/yocto/scripts/bin/authenticode_hash.py b/os/image/authenticode_hash.py similarity index 79% rename from os/yocto/scripts/bin/authenticode_hash.py rename to os/image/authenticode_hash.py index de7096fb1..c8ed5432e 100644 --- a/os/yocto/scripts/bin/authenticode_hash.py +++ b/os/image/authenticode_hash.py @@ -1,23 +1,31 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + +"""Calculate the Authenticode SHA-256 hash of a PE/COFF image.""" import argparse import hashlib def read_le_u16(data: bytes, offset: int) -> int: + """Read an unsigned little-endian 16-bit integer.""" import struct - return struct.unpack(' int: + """Read an unsigned little-endian 32-bit integer.""" import struct - return struct.unpack(' str: - with open(filepath, 'rb') as f: + """Return the PE/COFF Authenticode SHA-256 digest for *filepath*.""" + with open(filepath, "rb") as f: data = f.read() # Read DOS header @@ -39,7 +47,7 @@ def authenticode_hash(filepath: str) -> str: optional_header_offset = coff_header_offset + 20 magic = read_le_u16(data, optional_header_offset) - is_pe32_plus = (magic == 0x20B) + is_pe32_plus = magic == 0x20B # Calculate offsets for excluded regions (checksum and cert directory) checksum_offset = optional_header_offset + 64 @@ -94,7 +102,7 @@ def authenticode_hash(filepath: str) -> str: else: available_size = max(0, len(data) - start) if available_size > 0: - hasher.update(data[start:start + available_size]) + hasher.update(data[start : start + available_size]) sum_of_bytes_hashed += size @@ -113,7 +121,9 @@ def authenticode_hash(filepath: str) -> str: trailing_start = sum_of_bytes_hashed if trailing_start + hashed_trailing_len <= len(data): - hasher.update(data[trailing_start:trailing_start + hashed_trailing_len]) + hasher.update( + data[trailing_start : trailing_start + hashed_trailing_len] + ) # Add padding to align to 8 bytes remainder = file_size % 8 @@ -125,14 +135,15 @@ def authenticode_hash(filepath: str) -> str: def main() -> None: + """Run the command-line interface.""" parser = argparse.ArgumentParser( - description='Calculate PE/COFF Authenticode SHA256 hash (TPM Event Log compatible)' + description="Calculate PE/COFF Authenticode SHA256 hash (TPM Event Log compatible)" ) - parser.add_argument('file', help='Path to PE/COFF binary (e.g., UKI .efi)') + parser.add_argument("file", help="Path to PE/COFF binary (e.g., UKI .efi)") args = parser.parse_args() print(authenticode_hash(args.file)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/os/spec/artifact-manifest.schema.json b/os/spec/artifact-manifest.schema.json new file mode 100644 index 000000000..feb675142 --- /dev/null +++ b/os/spec/artifact-manifest.schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "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/Dstack-TEE/dstack/blob/master/os/spec/artifact-manifest.schema.json", + "title": "dstack OS backend artifact manifest", + "description": "Versioned handoff from an OS build backend to the common image assembler. Artifact paths are relative to the manifest.", + "$defs": { + "artifactPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!\\.\\.(?:/|$)).+" + } + }, + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "backend", + "image", + "source", + "boot", + "verity", + "artifacts" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "backend": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "flavor", "is_dev"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "flavor": {"type": "string", "minLength": 1}, + "is_dev": {"type": "boolean"} + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["git_revision"], + "properties": { + "git_revision": {"type": "string", "minLength": 1} + } + }, + "boot": { + "type": "object", + "additionalProperties": false, + "required": ["ovmf_variant"], + "properties": { + "ovmf_variant": {"type": "string", "minLength": 1} + } + }, + "verity": { + "type": "object", + "additionalProperties": false, + "required": ["root_hash", "data_size"], + "properties": { + "root_hash": {"type": "string", "minLength": 1}, + "data_size": { + "oneOf": [ + {"type": "integer", "minimum": 1}, + {"type": "string", "pattern": "^[1-9][0-9]*$"} + ] + } + } + }, + "artifacts": { + "type": "object", + "additionalProperties": false, + "required": [ + "initramfs", + "kernel", + "firmware", + "rootfs_verity", + "firmware_sev", + "uki" + ], + "properties": { + "initramfs": {"$ref": "#/$defs/artifactPath"}, + "kernel": {"$ref": "#/$defs/artifactPath"}, + "firmware": {"$ref": "#/$defs/artifactPath"}, + "rootfs_verity": {"$ref": "#/$defs/artifactPath"}, + "firmware_sev": { + "oneOf": [{"$ref": "#/$defs/artifactPath"}, {"type": "null"}] + }, + "uki": { + "oneOf": [{"$ref": "#/$defs/artifactPath"}, {"type": "null"}] + } + } + }, + "backend_metadata": { + "type": "object", + "additionalProperties": true + } + } +} diff --git a/os/yocto/README.md b/os/yocto/README.md index 7ddcbb3ea..92b5e967c 100644 --- a/os/yocto/README.md +++ b/os/yocto/README.md @@ -1,25 +1,41 @@ -# Yocto support for dstack guest OS +# Yocto backend for the dstack guest OS -This project implements Yocto layer and the overall build scripts for dstack Base OS image. +This directory contains the Yocto backend imported from `meta-dstack`. +dstack-owned layers live in `layers/`; external layers and BitBake live in +`deps/` as git submodules. -## Build +For the repository-wide OS layout and backend contract, see [`../README.md`](../README.md). -See https://github.com/Phala-Network/dstack-cloud for more details. +## Reproducible build -## Reproducible Build The Guest Image - -### Pre-requisites - -- X86_64 Linux system with Docker installed - -### Build commands +Prerequisites: an x86-64 Linux system with Docker installed. ```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git -cd meta-dstack/repro-build/ +git clone https://github.com/Dstack-TEE/dstack.git +cd dstack +git submodule update --init -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security +cd os/yocto/repro-build ./repro-build.sh ``` +For an interactive build, source `dev-setup` and use the Makefile, or run the +repository entrypoint `./os/build.sh --backend yocto` from the repository root. + +## Output boundary + +`scripts/export-artifacts.sh` converts BitBake deployment paths to the common +artifact manifest. `mkimage.sh` is a compatibility wrapper that exports the +manifest and calls `../image/assemble.sh`. + ## License -See the LICENSE file for more details. +The imported backend retains the Business Source License 1.1 terms in +[`LICENSE`](LICENSE). Embedded components may have their own notices. diff --git a/os/yocto/build.sh b/os/yocto/build.sh index 53a5e3cff..b96fbf017 100755 --- a/os/yocto/build.sh +++ b/os/yocto/build.sh @@ -6,7 +6,7 @@ SCRIPT_DIR=$( ACTION=$1 META_DIR=$SCRIPT_DIR -DSTACK_DIR=$SCRIPT_DIR/dstack +DSTACK_DIR=$(realpath "$SCRIPT_DIR/../../dstack") CERTS_DIR=$(pwd)/certs IMAGES_DIR=$(pwd)/images RUN_DIR=$(pwd)/run @@ -304,18 +304,22 @@ download_image() { echo "Downloading image $VERSION${IS_DEV:+ (dev)}" - TAG=v$VERSION + TAG=guest-os-v$VERSION if [ x"$IS_DEV" = x"1" ]; then BASENAME=dstack-cloud-dev-$VERSION else BASENAME=dstack-cloud-$VERSION fi - URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/$TAG/$BASENAME.tar.gz + URL=https://github.com/Dstack-TEE/dstack/releases/download/$TAG/$BASENAME.tar.gz + LEGACY_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v$VERSION/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME ]; then echo "Image already exists" else mkdir -p $IMAGES_DIR/$BASENAME.tmp - curl -L $URL -o $IMAGES_DIR/$BASENAME.tar.gz + if ! curl -fL $URL -o $IMAGES_DIR/$BASENAME.tar.gz; then + echo "Falling back to the pre-monorepo release location" + curl -fL $LEGACY_URL -o $IMAGES_DIR/$BASENAME.tar.gz + fi tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME.tmp rm -f $IMAGES_DIR/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME.tmp/$BASENAME ]; then @@ -333,7 +337,8 @@ Usage: ${0##*/} [args] Actions: host - Build host binaries only - guest - Build guest images only + image - Build guest images (backend interface) + guest - Alias for image cfg - Generate configuration files only dl - Download a specific image hostcfg - Build host binaries and generate configuration files @@ -356,7 +361,7 @@ case $ACTION in host) build_host ;; -guest) +image|guest) build_guest $2 ;; cfg) diff --git a/os/yocto/dev-setup b/os/yocto/dev-setup index 050a38990..f789c804e 100755 --- a/os/yocto/dev-setup +++ b/os/yocto/dev-setup @@ -16,18 +16,18 @@ fi THIS_SCRIPT=$(realpath "$THIS_SCRIPT") THIS_DIR=$(dirname "$THIS_SCRIPT") -LAYERS="$THIS_DIR/meta-yocto/meta-poky \ - $THIS_DIR/meta-yocto/meta-yocto-bsp \ - $THIS_DIR/meta-confidential-compute \ - $THIS_DIR/meta-openembedded/meta-oe \ - $THIS_DIR/meta-openembedded/meta-python \ - $THIS_DIR/meta-openembedded/meta-networking \ - $THIS_DIR/meta-openembedded/meta-filesystems \ - $THIS_DIR/meta-virtualization \ - $THIS_DIR/meta-rust-bin \ - $THIS_DIR/meta-security \ - $THIS_DIR//meta-security/meta-tpm \ - $THIS_DIR/meta-dstack" +LAYERS="$THIS_DIR/deps/meta-yocto/meta-poky \ + $THIS_DIR/deps/meta-yocto/meta-yocto-bsp \ + $THIS_DIR/deps/meta-confidential-compute \ + $THIS_DIR/deps/meta-openembedded/meta-oe \ + $THIS_DIR/deps/meta-openembedded/meta-python \ + $THIS_DIR/deps/meta-openembedded/meta-networking \ + $THIS_DIR/deps/meta-openembedded/meta-filesystems \ + $THIS_DIR/deps/meta-virtualization \ + $THIS_DIR/deps/meta-rust-bin \ + $THIS_DIR/deps/meta-security \ + $THIS_DIR/deps/meta-security/meta-tpm \ + $THIS_DIR/layers/meta-dstack" if [ -z "$1" ]; then BUILD_DIR=$THIS_DIR/bb-build @@ -35,20 +35,20 @@ else BUILD_DIR=$(realpath "$1") fi -# Sync build config from meta-dstack/conf (always overwrite) -CONF_SRC=$THIS_DIR/meta-dstack/conf +# Sync build config from the dstack layer (always overwrite) +CONF_SRC=$THIS_DIR/layers/meta-dstack/conf mkdir -p "$BUILD_DIR/conf" cp -f "$CONF_SRC/local.conf" "$BUILD_DIR/conf/local.conf" cp -rf "$CONF_SRC/multiconfig" "$BUILD_DIR/conf/" -OE_INIT=$THIS_DIR/openembedded-core/oe-init-build-env +OE_INIT=$THIS_DIR/deps/openembedded-core/oe-init-build-env for script in $THIS_DIR/setup.d/*.sh; do source "$script" done pushd "$BUILD_DIR" -BDIR="." TEMPLATECONF=$THIS_DIR/openembedded-core/meta/conf/templates/default source $OE_INIT +BDIR="." TEMPLATECONF=$THIS_DIR/deps/openembedded-core/meta/conf/templates/default source $OE_INIT popd bitbake-layers add-layer $LAYERS diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index 2996bcaea..512c49d1f 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -5,11 +5,16 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384 inherit systemd -REPO_ROOT = "${THISDIR}/../../.." +# Keep the recipe's source boundary explicit. The core workspace and public +# Rust SDK are staged with the same relative layout they have in the monorepo; +# OS-owned rootfs files are staged separately from application source. +DSTACK_MONOREPO_ROOT ?= "${@os.path.realpath(os.path.join(d.getVar('THISDIR'), '../../../../../..'))}" +DSTACK_CORE_SRC ?= "${DSTACK_MONOREPO_ROOT}/dstack" +DSTACK_RUST_SDK_SRC ?= "${DSTACK_MONOREPO_ROOT}/sdk/rust" +DSTACK_ROOTFS_SRC ?= "${DSTACK_MONOREPO_ROOT}/os/common/rootfs" -SRC_DIR = '${REPO_ROOT}/dstack' - -S = "${UNPACKDIR}/dstack" +S = "${UNPACKDIR}/repo/dstack" +DSTACK_ROOTFS_FILES = "${UNPACKDIR}/repo/os/common/rootfs" RDEPENDS:${PN} += "bash" @@ -27,15 +32,21 @@ EXTRA_CARGO_FLAGS = "-p dstack-guest-agent -p dstack-util" inherit cargo_bin do_unpack() { - mkdir -p ${S} - rsync -a --exclude="target" ${SRC_DIR}/ ${S}/ + install -d "${S}" "${UNPACKDIR}/repo/sdk/rust" "${DSTACK_ROOTFS_FILES}" + rsync -a --exclude=".git" --exclude=".worktrees" --exclude="target" \ + "${DSTACK_CORE_SRC}/" "${S}/" + rsync -a --exclude=".git" --exclude="target" \ + "${DSTACK_RUST_SDK_SRC}/" "${UNPACKDIR}/repo/sdk/rust/" + rsync -a "${DSTACK_ROOTFS_SRC}/" "${DSTACK_ROOTFS_FILES}/" } +do_unpack[cleandirs] = "${UNPACKDIR}/repo" + # Force the configure task to run every time to detect source changes do_unpack[nostamp] = "1" # Add source directory to configure task dependencies -do_unpack[vardeps] += "SRC_DIR" +do_unpack[vardeps] += "DSTACK_CORE_SRC DSTACK_RUST_SDK_SRC DSTACK_ROOTFS_SRC" do_configure() { cargo_bin_do_configure @@ -52,33 +63,33 @@ do_install() { install -d ${D}${sysconfdir}/systemd/journald.conf.d install -m 0755 ${CARGO_BINDIR}/dstack-util ${D}${bindir} install -m 0755 ${CARGO_BINDIR}/dstack-guest-agent ${D}${bindir} - install -m 0755 ${S}/basefiles/dstack-prepare.sh ${D}${bindir} - install -m 0755 ${S}/basefiles/ephemeral-docker.sh ${D}${bindir} - install -m 0755 ${S}/basefiles/wg-checker.sh ${D}${bindir} - install -m 0755 ${S}/basefiles/app-compose.sh ${D}${bindir} - install -m 0644 ${S}/basefiles/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf + install -m 0755 ${DSTACK_ROOTFS_FILES}/dstack-prepare.sh ${D}${bindir} + install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} + install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir} + install -m 0755 ${DSTACK_ROOTFS_FILES}/app-compose.sh ${D}${bindir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ - install -m 0644 ${S}/basefiles/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf + install -m 0644 ${DSTACK_ROOTFS_FILES}/tdx-attest.conf ${D}${sysconfdir}/tdx-attest.conf install -d ${D}${sysconfdir}/sysctl.d - install -m 0644 ${S}/basefiles/sysctl.d/99-dstack.conf ${D}${sysconfdir}/sysctl.d/99-dstack.conf + install -m 0644 ${DSTACK_ROOTFS_FILES}/sysctl.d/99-dstack.conf ${D}${sysconfdir}/sysctl.d/99-dstack.conf if ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', 'true', 'false', d)}; then install -d ${D}${systemd_system_unitdir} \ ${D}${sysconfdir}/systemd/resolved.conf.d - install -m 0644 ${S}/basefiles/dstack-guest-agent.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-prepare.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/app-compose.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/wg-checker.service ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/dstack-guest-agent.socket ${D}${systemd_system_unitdir} - install -m 0644 ${S}/basefiles/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d + install -m 0644 ${DSTACK_ROOTFS_FILES}/dstack-guest-agent.service ${D}${systemd_system_unitdir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/dstack-prepare.service ${D}${systemd_system_unitdir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/app-compose.service ${D}${systemd_system_unitdir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/wg-checker.service ${D}${systemd_system_unitdir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/dstack-guest-agent.socket ${D}${systemd_system_unitdir} + install -m 0644 ${DSTACK_ROOTFS_FILES}/llmnr.conf ${D}${sysconfdir}/systemd/resolved.conf.d install -d ${D}${sysconfdir}/systemd/system/docker.service.d - install -m 0644 ${S}/basefiles/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ + install -m 0644 ${DSTACK_ROOTFS_FILES}/docker.service.d/* ${D}${sysconfdir}/systemd/system/docker.service.d/ install -d ${D}${sysconfdir}/systemd/system/containerd.service.d - install -m 0644 ${S}/basefiles/containerd.service.d/* ${D}${sysconfdir}/systemd/system/containerd.service.d/ + install -m 0644 ${DSTACK_ROOTFS_FILES}/containerd.service.d/* ${D}${sysconfdir}/systemd/system/containerd.service.d/ fi } diff --git a/os/yocto/mkimage.sh b/os/yocto/mkimage.sh index 2cfbe1b9a..1d3174214 100755 --- a/os/yocto/mkimage.sh +++ b/os/yocto/mkimage.sh @@ -1,412 +1,48 @@ #!/bin/bash -set -e +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 -DSTACK_TAR_RELEASE=${DSTACK_TAR_RELEASE:-1} -ENABLE_UKI_IMAGE=${ENABLE_UKI_IMAGE:-1} +# Compatibility entrypoint: export Yocto-specific outputs, then invoke the +# backend-neutral image assembler. +set -euo pipefail -# Fixed GPT GUIDs so partitioned images are reproducible (sgdisk randomizes by -# default). Partitions are located by PARTLABEL, not GUID, so these are arbitrary. -DSTACK_DISK_GUID=${DSTACK_DISK_GUID:-d5acc000-0000-4000-8000-000000000000} -DSTACK_ROOTFS_PART_GUID=${DSTACK_ROOTFS_PART_GUID:-d5acc000-0000-4000-8000-000000000001} -DSTACK_EFI_PART_GUID=${DSTACK_EFI_PART_GUID:-d5acc000-0000-4000-8000-000000000002} +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BB_BUILD_DIR=${BB_BUILD_DIR:-build} +DIST_NAME= +FLAVOR= -# Parse command line arguments while [ $# -gt 0 ]; do case "$1" in --dist-name) - DIST_NAME="$2" + DIST_NAME=$2 shift 2 ;; --flavor) - FLAVOR="$2" + FLAVOR=$2 shift 2 ;; + -h|--help) + exec "$SCRIPT_DIR/scripts/export-artifacts.sh" --help + ;; *) - echo "Unknown option: $1" - echo "Usage: $0 --dist-name NAME --flavor FLAVOR" + echo "Unknown option: $1" >&2 exit 1 ;; esac done -# Validate required arguments -if [ -z "$DIST_NAME" ]; then - echo "Error: --dist-name is required" - exit 1 -fi - -if [ -z "$FLAVOR" ]; then - echo "Error: --flavor is required (prod, dev)" +if [ -z "$DIST_NAME" ] || [ -z "$FLAVOR" ]; then + echo "Error: --dist-name and --flavor are required" >&2 exit 1 fi -case "$FLAVOR" in - prod) - if [[ "$DIST_NAME" == *-dev ]]; then - echo "Error: prod flavor requires a non-dev dist name: $DIST_NAME" >&2 - exit 1 - fi - IS_DEV=false - ;; - dev) - if [[ "$DIST_NAME" != *-dev ]]; then - echo "Error: dev flavor requires a dist name ending in -dev: $DIST_NAME" >&2 - exit 1 - fi - IS_DEV=true - ;; - *) - echo "Error: unsupported --flavor '$FLAVOR' (expected prod or dev)" >&2 - exit 1 - ;; -esac - -BB_BUILD_DIR=$(realpath ${BB_BUILD_DIR:-build}) -DIST_DIR=$(realpath ${DIST_DIR:-${BB_BUILD_DIR}/dist}) - -# MACHINE name; artifacts are deployed under deploy/images/${MACHINE} -DSTACK_MACHINE=${DSTACK_MACHINE:-dstack} - -# Common artifacts are in tmp/, flavor-specific artifacts are in tmp-mc-/ -COMMON_IMG_DIR=${BB_BUILD_DIR}/tmp/deploy/images/${DSTACK_MACHINE} -FLAVOR_IMG_DIR=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/deploy/images/${DSTACK_MACHINE} - -# Common artifacts (shared across all flavors) -INITRAMFS_IMAGE=${COMMON_IMG_DIR}/dstack-initramfs.cpio.gz -KERNEL_IMAGE=${COMMON_IMG_DIR}/bzImage -OVMF_FIRMWARE=${COMMON_IMG_DIR}/ovmf.fd - -# Flavor-specific artifacts (from multiconfig build) -ROOTFS_IMAGE=${FLAVOR_IMG_DIR}/dstack-rootfs-${DSTACK_MACHINE}.squashfs.verity - -# UKI filename -UKI_IMAGE=${FLAVOR_IMG_DIR}/dstack-uki.efi - -# Verity env is in the flavor-specific work-shared directory -VERITY_ENV_FILE=${BB_BUILD_DIR}/tmp-mc-${FLAVOR}/work-shared/${DSTACK_MACHINE}/dm-verity/dstack-rootfs.squashfs.verity.env -if [ ! -f "${VERITY_ENV_FILE}" ]; then - echo "Error: verity env not found: ${VERITY_ENV_FILE}" >&2 - echo "Build the rootfs first, e.g.: bitbake mc:${FLAVOR}:dstack-rootfs" >&2 - exit 1 -fi -echo "Loading verity env from ${VERITY_ENV_FILE}" -# shellcheck source=/dev/null -source "${VERITY_ENV_FILE}" - -DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) - -# Output directory contains all artifacts; tarballs contain subsets -OUTPUT_DIR=${OUTPUT_DIR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}"} -IMAGE_TAR=${IMAGE_TAR:-"${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}.tar.gz"} -IMAGE_TAR_UKI="${DIST_DIR}/${DIST_NAME}-${DSTACK_VERSION}-uki.tar.gz" -TAR_DIR_NAME="${DIST_NAME}-${DSTACK_VERSION}" - -# Use script's directory to find authenticode_hash.py -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -AUTHENTICODE_HASH_SCRIPT="${SCRIPT_DIR}/scripts/bin/authenticode_hash.py" - -verbose() { - echo "$@" - $@ -} - -align_up() { - local value=$1 - local align=$2 - echo $(( ( (value + align - 1) / align ) * align )) -} - -calc_authenticode_hash() { - local file="$1" - - if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then - return 0 - fi - - python3 "$AUTHENTICODE_HASH_SCRIPT" "$file" 2>/dev/null || true -} - -write_authenticode_hash() { - local file="$1" - local out_file="${file}.auth_hash.txt" - - if [[ ! -f "$file" ]]; then - return 0 - fi - - if [[ ! -f "$AUTHENTICODE_HASH_SCRIPT" ]] || ! command -v python3 &>/dev/null; then - echo "Warning: authenticode_hash.py not found or python3 not available, skipping Authenticode hash calculation" >&2 - return 0 - fi - - echo "Calculating UKI Authenticode hash..." - local auth_hash - auth_hash=$(calc_authenticode_hash "$file") - if [[ -n "$auth_hash" ]]; then - echo "$auth_hash" > "$out_file" - echo "UKI Authenticode hash: $auth_hash" - else - echo "Warning: Failed to calculate UKI Authenticode hash" >&2 - fi -} - -ensure_dstack_mr() { - DSTACK_SRC="${DSTACK_SRC:-$SCRIPT_DIR/dstack}" - if [ -z "${DSTACK_MR_BIN:-}" ]; then - for c in "$SCRIPT_DIR/dstack-mr" "$SCRIPT_DIR/rust-target/release/dstack-mr" \ - "$DSTACK_SRC/target/release/dstack-mr"; do - [ -x "$c" ] && DSTACK_MR_BIN="$c" && break - done - fi - if [ -z "${DSTACK_MR_BIN:-}" ]; then - echo "Building dstack-mr to compute OS-image measurement material" - ( cd "$DSTACK_SRC" && cargo build --release -p dstack-mr ) - DSTACK_MR_BIN="$DSTACK_SRC/target/release/dstack-mr" - fi -} - -create_partitioned_rootfs() { - local rootfs_img="$1" - local output_img="$2" - ( - set -e - local align=$((1024 * 1024)) - local sector=512 - local rootfs_size=$(stat -L -c %s "$rootfs_img") - local rootfs_size_aligned=$(align_up $rootfs_size $align) - local rootfs_start=$align - # Leave extra room for GPT headers (1MB at start, 1MB at end) - local total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) $align) - - truncate -s $total_size "$output_img" - - local root_start_sector=$((rootfs_start / sector)) - local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) - - sgdisk --zap-all "$output_img" >/dev/null - # Fixed GUIDs keep the image bit-for-bit reproducible (GPT otherwise - # randomizes disk/partition GUIDs). The rootfs is located by PARTLABEL. - sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ - --new=1:${root_start_sector}:${root_end_sector} --typecode=1:8300 \ - --partition-guid=1:"${DSTACK_ROOTFS_PART_GUID}" \ - --change-name=1:'dstack-rootfs' "$output_img" >/dev/null - - dd if="$rootfs_img" of="$output_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none - ) -} - -build_uki_disk_image() { - local disk_img="$1" - local uki_file="$2" - local rootfs_img="$3" - ( - set -e - local align=$((1024 * 1024)) - local sector=512 - local efi_size=$((256 * 1024 * 1024)) - local efi_size_aligned=$(align_up $efi_size $align) - local rootfs_size=$(stat -L -c %s "$rootfs_img") - local rootfs_size_aligned=$(align_up $rootfs_size $align) - local efi_start=$align - local rootfs_start=$((efi_start + efi_size_aligned)) - # Leave extra room for the backup GPT header - local total_size=$(align_up $((rootfs_start + rootfs_size_aligned + align)) $align) - - truncate -s $total_size "$disk_img" +ARTIFACT_DIR=${ARTIFACT_DIR:-"$BB_BUILD_DIR/dstack-artifacts/$FLAVOR"} +"$SCRIPT_DIR/scripts/export-artifacts.sh" \ + --dist-name "$DIST_NAME" \ + --flavor "$FLAVOR" \ + --build-dir "$BB_BUILD_DIR" \ + --output-dir "$ARTIFACT_DIR" - local efi_start_sector=$((efi_start / sector)) - local efi_end_sector=$((efi_start_sector + (efi_size_aligned / sector) - 1)) - local root_start_sector=$((rootfs_start / sector)) - local root_end_sector=$((root_start_sector + (rootfs_size_aligned / sector) - 1)) - - sgdisk --zap-all "$disk_img" >/dev/null - sgdisk --disk-guid="${DSTACK_DISK_GUID}" \ - --new=1:${efi_start_sector}:${efi_end_sector} --typecode=1:ef00 \ - --partition-guid=1:"${DSTACK_EFI_PART_GUID}" \ - --change-name=1:'EFI System Partition' "$disk_img" >/dev/null - sgdisk --new=2:${root_start_sector}:${root_end_sector} --typecode=2:8300 \ - --partition-guid=2:"${DSTACK_ROOTFS_PART_GUID}" \ - --change-name=2:'dstack-rootfs' "$disk_img" >/dev/null - - local tmp_dir - tmp_dir=$(mktemp -d) - trap 'rm -rf "$tmp_dir"' EXIT - - # Create EFI filesystem with UKI as bootloader - local efi_img=${tmp_dir}/efi.img - mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null - mmd -i "$efi_img" ::EFI ::EFI/BOOT - mcopy -i "$efi_img" "$uki_file" ::EFI/BOOT/BOOTX64.EFI - - dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none - dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none - ) -} - -create_uki_artifacts() { - local uki_dir="$1" - mkdir -p "$uki_dir" - - echo "Building UKI disk image at ${uki_dir}/disk.raw" - build_uki_disk_image "${uki_dir}/disk.raw" "$UKI_IMAGE" "$ROOTFS_IMAGE" - - # Calculate and copy auth hash - write_authenticode_hash "$UKI_IMAGE" - if [[ -f "${UKI_IMAGE}.auth_hash.txt" ]]; then - cp "${UKI_IMAGE}.auth_hash.txt" "${uki_dir}/auth_hash.txt" - fi -} - -Q=verbose - -# Create bare metal image directory -$Q rm -rf ${OUTPUT_DIR}/ -$Q mkdir -p ${OUTPUT_DIR}/ -$Q cp $INITRAMFS_IMAGE ${OUTPUT_DIR}/initramfs.cpio.gz -$Q cp $KERNEL_IMAGE ${OUTPUT_DIR}/ -$Q cp $OVMF_FIRMWARE ${OUTPUT_DIR}/ - -# AMD SEV firmware (additive). Shipped alongside the TDX firmware so a SEV-SNP -# launch can select it via the metadata.json "bios-sev" field below. The SEV -# firmware blob itself is NOT added directly to sha256sum.txt; when present, its -# OVMF hash/sections are committed by measurement.snp.cbor, and that file is -# part of digest.txt. This does not change any TDX hardware -# measurement (MRTD comes from ovmf.fd, RTMRs from kernel/cmdline/rootfs) -- it -# only changes dstack's image-bundle digest. -OVMF_SEV_FIRMWARE=${COMMON_IMG_DIR}/ovmf-sev.fd -HAVE_OVMF_SEV=0 -BIOS_SEV_JSON="" -if [ -f "$OVMF_SEV_FIRMWARE" ]; then - $Q cp $OVMF_SEV_FIRMWARE ${OUTPUT_DIR}/ - HAVE_OVMF_SEV=1 - # Inserted after the "bios" line in metadata.json (see below). - BIOS_SEV_JSON=' - "bios-sev": "ovmf-sev.fd",' -fi - -echo "Creating partitioned rootfs image at ${OUTPUT_DIR}/rootfs.img.parted.verity" -# Bare-metal partitioning needs sgdisk (from the 'gdisk' package). -if ! command -v sgdisk >/dev/null; then - echo "Error: cannot create partitioned rootfs image because 'sgdisk' is missing; install 'gdisk'." >&2 - exit 1 -fi -create_partitioned_rootfs "$ROOTFS_IMAGE" "${OUTPUT_DIR}/rootfs.img.parted.verity" - -GIT_REVISION=$(git rev-parse HEAD 2>/dev/null || echo "") - -# Lift the OVMF variant tag straight out of the dstack-ovmf recipe so verifiers -# know which RTMR[0] event layout to expect. Required: the recipe must declare -# OVMF_VARIANT alongside PV. -OVMF_VARIANT=$(bitbake-getvar --value OVMF_VARIANT -r dstack-ovmf) -if [ -z "$OVMF_VARIANT" ]; then - echo "Error: dstack-ovmf recipe is missing OVMF_VARIANT" >&2 - exit 1 -fi - -echo "Generating metadata.json to ${OUTPUT_DIR}/metadata.json (ovmf_variant=$OVMF_VARIANT)" - -KARG0="console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0" -KARG1="mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock" -KARG2="dstack.rootfs_hash=$ROOT_HASH dstack.rootfs_size=$DATA_SIZE" - -cat < ${OUTPUT_DIR}/metadata.json -{ - "bios": "ovmf.fd",${BIOS_SEV_JSON} - "kernel": "bzImage", - "cmdline": "$KARG0 $KARG1 $KARG2", - "initrd": "initramfs.cpio.gz", - "rootfs": "rootfs.img.parted.verity", - "version": "$DSTACK_VERSION", - "git_revision": "$GIT_REVISION", - "shared_ro": true, - "is_dev": ${IS_DEV}, - "ovmf_variant": "$OVMF_VARIANT" -} -EOF - -ensure_dstack_mr - -echo "Generating measurement.tdx.cbor via ${DSTACK_MR_BIN}" -"${DSTACK_MR_BIN}" tdx-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.tdx.cbor" - -HAVE_MEASUREMENT_SNP=0 -if [ "$HAVE_OVMF_SEV" = "1" ]; then - echo "Generating measurement.snp.cbor via ${DSTACK_MR_BIN}" - "${DSTACK_MR_BIN}" snp-measurement-cbor "${OUTPUT_DIR}" > "${OUTPUT_DIR}/measurement.snp.cbor" - HAVE_MEASUREMENT_SNP=1 -fi - -# Create UKI artifacts (disk.raw and auth_hash.txt) in OUTPUT_DIR -UKI_CREATED=0 -if [ "$ENABLE_UKI_IMAGE" = "1" ]; then - if [[ ! -f "$UKI_IMAGE" ]]; then - echo "Skipping UKI disk image creation because UKI image not found: $UKI_IMAGE" >&2 - echo "Run 'bitbake mc:${FLAVOR}:dstack-uki' to build the UKI first" >&2 - elif command -v sgdisk >/dev/null && \ - command -v mkfs.vfat >/dev/null && \ - command -v mcopy >/dev/null && \ - command -v mmd >/dev/null; then - create_uki_artifacts "${OUTPUT_DIR}" - UKI_CREATED=1 - else - echo "Error: cannot create UKI disk image because required tools are missing" >&2 - echo "Missing tools are among: sgdisk (gdisk), mkfs.vfat (dosfstools), mcopy/mmd (mtools)" >&2 - echo "Install them (e.g. apt-get install -y gdisk dosfstools mtools) or set ENABLE_UKI_IMAGE=0" >&2 - exit 1 - fi -fi - -HAVE_MEASUREMENT_GCP=0 -if [[ "$UKI_CREATED" = "1" ]]; then - if [[ ! -f "${OUTPUT_DIR}/auth_hash.txt" ]]; then - echo "Error: UKI image was created but auth_hash.txt is missing" >&2 - exit 1 - fi - echo "Generating measurement.gcp.cbor via ${DSTACK_MR_BIN}" - "${DSTACK_MR_BIN}" gcp-measurement-cbor "${OUTPUT_DIR}/auth_hash.txt" > "${OUTPUT_DIR}/measurement.gcp.cbor" - HAVE_MEASUREMENT_GCP=1 -fi - -echo "Generating unified image digest to ${OUTPUT_DIR}/" -CHECKSUM_FILES="ovmf.fd bzImage initramfs.cpio.gz metadata.json measurement.tdx.cbor" -if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then - CHECKSUM_FILES="$CHECKSUM_FILES measurement.snp.cbor" -fi -if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then - CHECKSUM_FILES="$CHECKSUM_FILES measurement.gcp.cbor" -fi -( - cd "${OUTPUT_DIR}/" - sha256sum $CHECKSUM_FILES > sha256sum.txt - sha256sum sha256sum.txt | awk '{print $1}' > digest.txt -) - -if [ x$DSTACK_TAR_RELEASE = x1 ]; then - OUTPUT_DIR=$(realpath ${OUTPUT_DIR}) - PARENT_DIR=$(dirname ${OUTPUT_DIR}) - - # Bare metal tarball: all files except disk.raw and auth_hash.txt - rm -rf ${IMAGE_TAR} - echo "Archiving bare metal image to ${IMAGE_TAR}" - BARE_METAL_FILES="rootfs.img.parted.verity bzImage ovmf.fd digest.txt sha256sum.txt initramfs.cpio.gz metadata.json measurement.tdx.cbor" - if [ "$HAVE_OVMF_SEV" = "1" ]; then - BARE_METAL_FILES="$BARE_METAL_FILES ovmf-sev.fd" - fi - if [ "$HAVE_MEASUREMENT_SNP" = "1" ]; then - BARE_METAL_FILES="$BARE_METAL_FILES measurement.snp.cbor" - fi - if [ "$HAVE_MEASUREMENT_GCP" = "1" ]; then - BARE_METAL_FILES="$BARE_METAL_FILES measurement.gcp.cbor" - fi - (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR} $(for f in $BARE_METAL_FILES; do echo "$TAR_DIR_NAME/$f"; done)) - echo - - # UKI tarball: GCP boot disk plus the unified OS-image identity material. - if [[ "$UKI_CREATED" = "1" ]]; then - rm -rf ${IMAGE_TAR_UKI} - echo "Archiving UKI image to ${IMAGE_TAR_UKI}" - UKI_FILES="disk.raw digest.txt sha256sum.txt measurement.gcp.cbor" - (cd "$PARENT_DIR" && tar -czvf ${IMAGE_TAR_UKI} $(for f in $UKI_FILES; do echo "$TAR_DIR_NAME/$f"; done)) - echo - fi -fi +exec "$SCRIPT_DIR/../image/assemble.sh" \ + --manifest "$ARTIFACT_DIR/artifact-manifest.json" diff --git a/os/yocto/repro-build/repro-build.sh b/os/yocto/repro-build/repro-build.sh index b2d51973f..7d66a55e2 100755 --- a/os/yocto/repro-build/repro-build.sh +++ b/os/yocto/repro-build/repro-build.sh @@ -27,7 +27,7 @@ done BUILDER_NAME=dstack-build THIS_DIR=$(cd $(dirname $0); pwd) -REPO_ROOT=${REPO_ROOT:-$(dirname $THIS_DIR)} +REPO_ROOT=${REPO_ROOT:-$(realpath "$THIS_DIR/../../..")} GIT_DIR=$REPO_ROOT HOST_BUILD_DIR_A=${THIS_DIR}/build-a @@ -35,7 +35,7 @@ HOST_BUILD_DIR_B=${THIS_DIR}/build-b # guest dirs GUEST_BUILD_DIR=/dstack-build -GUEST_SRC_DIR=/meta-dstack +GUEST_SRC_DIR=/dstack-src cd $THIS_DIR @@ -45,7 +45,8 @@ rm -rf .dummy build_to() { mkdir -p $1 - BUILD_CMD="${2} ${GUEST_SRC_DIR}/${META_SUBDIR}/build.sh guest ./bb-build" + GIT_REVISION=$(git -C "$REPO_ROOT" rev-parse HEAD) + BUILD_CMD="DSTACK_GIT_REVISION='$GIT_REVISION' ${2} ${GUEST_SRC_DIR}/os/yocto/build.sh image ./bb-build" docker run --platform linux/amd64 --rm \ --userns=host \ --user $(id -u):$(id -g) \ @@ -79,11 +80,19 @@ cat <" - echo "Example: $0 https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.6.0/dstack-0.6.0.tar.gz" + echo "Example: $0 https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v0.6.0/dstack-0.6.0.tar.gz" echo "Example: $0 /path/to/local/file.tar.gz" exit 1 } diff --git a/os/yocto/scripts/export-artifacts.sh b/os/yocto/scripts/export-artifacts.sh new file mode 100755 index 000000000..414c9b049 --- /dev/null +++ b/os/yocto/scripts/export-artifacts.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + +set -euo pipefail + +usage() { + cat <&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "$DIST_NAME" ] || [ -z "$FLAVOR" ]; then + echo "Error: --dist-name and --flavor are required" >&2 + usage >&2 + exit 1 +fi + +case "$FLAVOR" in + prod) + if [[ "$DIST_NAME" == *-dev ]]; then + echo "Error: prod flavor requires a non-dev dist name: $DIST_NAME" >&2 + exit 1 + fi + IS_DEV=false + ;; + dev) + if [[ "$DIST_NAME" != *-dev ]]; then + echo "Error: dev flavor requires a dist name ending in -dev: $DIST_NAME" >&2 + exit 1 + fi + IS_DEV=true + ;; + *) + echo "Error: unsupported flavor '$FLAVOR' (expected prod or dev)" >&2 + exit 1 + ;; +esac + +if [ ! -d "$BUILD_DIR" ]; then + echo "Error: BitBake build directory not found: $BUILD_DIR" >&2 + exit 1 +fi +BUILD_DIR=$(realpath "$BUILD_DIR") +ARTIFACT_DIR=${ARTIFACT_DIR:-"$BUILD_DIR/dstack-artifacts/$FLAVOR"} +ARTIFACT_DIR=$(realpath -m "$ARTIFACT_DIR") +case "$ARTIFACT_DIR" in + /|"$BUILD_DIR"|"$REPO_ROOT"|"$YOCTO_DIR") + echo "Error: refusing unsafe artifact output directory: $ARTIFACT_DIR" >&2 + exit 1 + ;; +esac + +DSTACK_MACHINE=${DSTACK_MACHINE:-dstack} +COMMON_IMG_DIR="$BUILD_DIR/tmp/deploy/images/$DSTACK_MACHINE" +FLAVOR_IMG_DIR="$BUILD_DIR/tmp-mc-$FLAVOR/deploy/images/$DSTACK_MACHINE" +VERITY_ENV_FILE="$BUILD_DIR/tmp-mc-$FLAVOR/work-shared/$DSTACK_MACHINE/dm-verity/dstack-rootfs.squashfs.verity.env" + +if [ ! -f "$VERITY_ENV_FILE" ]; then + echo "Error: verity env not found: $VERITY_ENV_FILE" >&2 + echo "Build the rootfs first, e.g. bitbake mc:${FLAVOR}:dstack-rootfs" >&2 + exit 1 +fi +# shellcheck source=/dev/null +source "$VERITY_ENV_FILE" +: "${ROOT_HASH:?ROOT_HASH missing from verity environment}" +: "${DATA_SIZE:?DATA_SIZE missing from verity environment}" + +DSTACK_VERSION=$(bitbake-getvar --value DISTRO_VERSION | tail -1) +OVMF_VARIANT=$(bitbake-getvar --value OVMF_VARIANT -r dstack-ovmf) +if [ -z "$DSTACK_VERSION" ] || [ -z "$OVMF_VARIANT" ]; then + echo "Error: failed to read DISTRO_VERSION or OVMF_VARIANT from BitBake" >&2 + exit 1 +fi +GIT_REVISION=${DSTACK_GIT_REVISION:-} +if [ -z "$GIT_REVISION" ]; then + GIT_REVISION=$(git -C "$REPO_ROOT" rev-parse HEAD) +fi + +rm -rf "$ARTIFACT_DIR" +mkdir -p "$ARTIFACT_DIR/files" + +link_required() { + local source=$1 + local name=$2 + if [ ! -f "$source" ]; then + echo "Error: required Yocto artifact not found: $source" >&2 + exit 1 + fi + ln -s "$(realpath "$source")" "$ARTIFACT_DIR/files/$name" +} + +link_optional() { + local source=$1 + local name=$2 + if [ -f "$source" ]; then + ln -s "$(realpath "$source")" "$ARTIFACT_DIR/files/$name" + return 0 + fi + return 1 +} + +link_required "$COMMON_IMG_DIR/dstack-initramfs.cpio.gz" initramfs.cpio.gz +link_required "$COMMON_IMG_DIR/bzImage" bzImage +link_required "$COMMON_IMG_DIR/ovmf.fd" ovmf.fd +link_required "$FLAVOR_IMG_DIR/dstack-rootfs-${DSTACK_MACHINE}.squashfs.verity" rootfs.squashfs.verity + +FIRMWARE_SEV= +UKI= +if link_optional "$COMMON_IMG_DIR/ovmf-sev.fd" ovmf-sev.fd; then + FIRMWARE_SEV=files/ovmf-sev.fd +fi +if link_optional "$FLAVOR_IMG_DIR/dstack-uki.efi" dstack-uki.efi; then + UKI=files/dstack-uki.efi +fi + +python3 - \ + "$ARTIFACT_DIR/artifact-manifest.json" \ + "$DIST_NAME" "$DSTACK_VERSION" "$FLAVOR" "$IS_DEV" \ + "$GIT_REVISION" "$OVMF_VARIANT" "$ROOT_HASH" "$DATA_SIZE" \ + "$DSTACK_MACHINE" "$FIRMWARE_SEV" "$UKI" <<'PYMANIFEST' +import json +import sys + +( + output, + name, + version, + flavor, + is_dev, + git_revision, + ovmf_variant, + root_hash, + data_size, + machine, + firmware_sev, + uki, +) = sys.argv[1:] + +manifest = { + "schema_version": 1, + "backend": "yocto", + "image": { + "name": name, + "version": version, + "flavor": flavor, + "is_dev": is_dev == "true", + }, + "source": {"git_revision": git_revision}, + "boot": {"ovmf_variant": ovmf_variant}, + "verity": {"root_hash": root_hash, "data_size": data_size}, + "artifacts": { + "initramfs": "files/initramfs.cpio.gz", + "kernel": "files/bzImage", + "firmware": "files/ovmf.fd", + "rootfs_verity": "files/rootfs.squashfs.verity", + "firmware_sev": firmware_sev or None, + "uki": uki or None, + }, + "backend_metadata": {"machine": machine}, +} +with open(output, "w", encoding="utf-8") as file: + json.dump(manifest, file, indent=2) + file.write("\n") +PYMANIFEST + +echo "$ARTIFACT_DIR/artifact-manifest.json" diff --git a/os/yocto/setup.d/nvidia-layer.sh b/os/yocto/setup.d/nvidia-layer.sh index 46da2ea09..bd082a20a 100644 --- a/os/yocto/setup.d/nvidia-layer.sh +++ b/os/yocto/setup.d/nvidia-layer.sh @@ -1,2 +1,2 @@ LAYERS="$LAYERS \ - $THIS_DIR/meta-nvidia" \ No newline at end of file + $THIS_DIR/layers/meta-nvidia" From 367e163cdac43553fc791c584b468e090bc0867a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:41:30 -0700 Subject: [PATCH 422/435] docs: update paths for monorepo layout --- .agent/CODING_TASTE.md | 6 +-- .claude/agents/sdk-sync-checker.md | 4 +- CLAUDE.md | 50 +++++++++++-------- CONTRIBUTING.md | 2 +- README.md | 22 +++++++- attestation.md | 4 +- docs/amd-sev-snp-review-readiness.md | 48 ++++++++++-------- docs/auth-simple-operations.md | 2 +- docs/bridge-networking.md | 2 +- docs/deployment.md | 23 +++++---- docs/faq.md | 2 +- docs/normalized-app-compose.md | 3 +- docs/onboarding.md | 10 ++-- docs/onchain-governance.md | 2 +- docs/security/cvm-boundaries.md | 8 +-- docs/security/security-model.md | 4 +- docs/tutorials/attestation-verification.md | 28 +++++++---- docs/tutorials/clone-build-dstack-vmm.md | 28 ++++++----- docs/tutorials/contract-deployment.md | 10 ++-- docs/tutorials/gateway-build-configuration.md | 12 ++--- docs/tutorials/gateway-service-setup.md | 6 +-- docs/tutorials/gramine-key-provider.md | 8 +-- docs/tutorials/guest-image-setup.md | 14 +++--- docs/tutorials/hello-world-app.md | 6 +-- docs/tutorials/kms-build-configuration.md | 16 +++--- docs/tutorials/kms-cvm-deployment.md | 2 +- .../troubleshooting-dstack-installation.md | 8 +-- .../troubleshooting-first-application.md | 2 +- .../troubleshooting-gateway-deployment.md | 6 +-- .../troubleshooting-kms-deployment.md | 2 +- .../troubleshooting-prerequisites.md | 2 +- docs/verification.md | 2 +- dstack/kms/README.md | 2 +- dstack/kms/auth-simple/README.md | 2 +- dstack/sodiumbox/README.md | 4 +- dstack/tests/docs/kms-self-authorization.md | 4 +- sdk/README.md | 2 +- sdk/curl/api.md | 2 +- 38 files changed, 204 insertions(+), 156 deletions(-) diff --git a/.agent/CODING_TASTE.md b/.agent/CODING_TASTE.md index fefaf9e83..65c93260e 100644 --- a/.agent/CODING_TASTE.md +++ b/.agent/CODING_TASTE.md @@ -99,7 +99,7 @@ Argue from dstack's threat model, not from generic best practices. ## Code organization -- **Workspace-managed dependencies.** Declare versions in the root `Cargo.toml`, reference +- **Workspace-managed dependencies.** Declare versions in `dstack/Cargo.toml`, reference with `foo.workspace = true` (#161, #360). - **One source of truth.** Shared logic used by two components lives in one crate (`dstack-mr::sev` used by both KMS and verifier). Duplicated blocks get extracted — @@ -208,7 +208,7 @@ not just the symptom: an `X-App-Version` response header. Follow this skeleton exactly when adding a service. - **Module layout**: the core RPC surface lives in `main_service.rs` (or `rpc_service.rs` in guest-agent). When a module grows subordinate concerns, promote it to `foo.rs` + a - `foo/` directory of submodules (`gateway/src/proxy.rs` + `proxy/{sni,tls_terminate,...}`). + `foo/` directory of submodules (`dstack/gateway/src/proxy.rs` + `proxy/{sni,tls_terminate,...}`). Core files routinely run 800–1500+ lines before splitting — don't over-fragment into many small files. - **Config conventions**: each service embeds its default TOML (`include_str!`), extracts @@ -298,7 +298,7 @@ not just the symptom: when async is unavoidable. - **Golden vectors over mocks**: real captured binary fixtures embedded with `include_bytes!("../samples/...")`, asserted against inline hex literals or `insta` - snapshots (`cc-eventlog`, `dstack-attest/tests/`). Fixture provenance gets its own + snapshots (`dstack/cc-eventlog`, `dstack/dstack-attest/tests/`). Fixture provenance gets its own README (`sev_snp_fixture.README.md`). When changing an encoding, add a regression test proving old and new outputs match (#603). - **Test names are snake_case behavior statements**: `enforces_ttl`, diff --git a/.claude/agents/sdk-sync-checker.md b/.claude/agents/sdk-sync-checker.md index 41dc8fcb0..794123da8 100644 --- a/.claude/agents/sdk-sync-checker.md +++ b/.claude/agents/sdk-sync-checker.md @@ -11,7 +11,7 @@ You validate SDK implementations against protobuf schemas to ensure synchronizat ## Process ### 1. Discovery -- Find all `.proto` files in `guest-agent/rpc/proto/` +- Find all `.proto` files in `dstack/guest-agent/rpc/proto/` - Identify SDK implementations in `sdk/` (python, go, rust, js, curl docs) - Extract services, RPCs, and message types from proto files @@ -66,7 +66,7 @@ Details: - cURL docs: `snake_case` (JSON wire format) ## Locations -- Protos: `guest-agent/rpc/proto/*.proto` +- Protos: `dstack/guest-agent/rpc/proto/*.proto` - Python: `sdk/python/src/dstack_sdk/dstack_client.py` - Go: `sdk/go/dstack/client.go` - Rust: `sdk/rust/types/src/dstack.rs` diff --git a/CLAUDE.md b/CLAUDE.md index 768e68263..fb39cc38c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,11 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -dstack is a developer-friendly, security-first framwwork for deploying containerized applications into Intel TDX (Trust Domain Extensions) Trusted Execution Environments (TEEs). The system provides end-to-end security through hardware-rooted attestation, automated key management, and zero-trust networking. +dstack is a developer-friendly, security-first framework for deploying containerized applications into Intel TDX (Trust Domain Extensions) Trusted Execution Environments (TEEs). The system provides end-to-end security through hardware-rooted attestation, automated key management, and zero-trust networking. + +The monorepo keeps core services and the Rust workspace in `dstack/`, public +SDKs in `sdk/`, guest-OS builders in `os/`, documentation in `docs/`, and +standalone utilities in `tools/`. ## Architecture @@ -12,13 +16,13 @@ dstack consists of several core components that interact to provide TEE-based co ### Core Components -- **`dstack-vmm`** (`vmm/`): Virtual Machine Manager that runs on bare-metal TDX hosts. Orchestrates CVM lifecycle, manages QEMU processes, allocates resources, parses docker-compose files, and provides a web UI (port 9080) for deployment. +- **`dstack-vmm`** (`dstack/vmm/`): Virtual Machine Manager that runs on bare-metal TDX hosts. Orchestrates CVM lifecycle, manages QEMU processes, allocates resources, parses docker-compose files, and provides a web UI (port 9080) for deployment. -- **`dstack-kms`** (`kms/`): Key Management System that handles cryptographic key provisioning after TDX quote verification. Derives keys deterministically per application identity and enforces authorization policies defined in smart contracts on Ethereum. +- **`dstack-kms`** (`dstack/kms/`): Key Management System that handles cryptographic key provisioning after TDX quote verification. Derives keys deterministically per application identity and enforces authorization policies defined in smart contracts on Ethereum. -- **`dstack-gateway`** (`gateway/`): Reverse proxy providing zero-trust network access. Handles TLS termination, automated ACME certificate provisioning, and traffic routing via ingress mapping rules. +- **`dstack-gateway`** (`dstack/gateway/`): Reverse proxy providing zero-trust network access. Handles TLS termination, automated ACME certificate provisioning, and traffic routing via ingress mapping rules. -- **`dstack-guest-agent`** (`guest-agent/`): Runs inside each CVM to provide runtime services including Docker Compose lifecycle management, TDX quote generation, key provisioning from KMS, and log aggregation. Exposes API via Unix socket at `/var/run/dstack.sock`. +- **`dstack-guest-agent`** (`dstack/guest-agent/`): Runs inside each CVM to provide runtime services including Docker Compose lifecycle management, TDX quote generation, key provisioning from KMS, and log aggregation. Exposes API via Unix socket at `/var/run/dstack.sock`. ### Communication Protocols @@ -29,10 +33,10 @@ dstack consists of several core components that interact to provide TEE-based co ### Additional Components -- **`certbot`** (`certbot/`): Automated ACME DNS-01 certificate management -- **`ct_monitor`** (`ct_monitor/`): Certificate Transparency log monitoring -- **`verifier`** (`verifier/`): TDX quote verification service using `dcap-qvl` -- **`supervisor`** (`supervisor/`): Process supervision inside CVMs +- **`certbot`** (`dstack/certbot/`): Automated ACME DNS-01 certificate management +- **`ct_monitor`** (`dstack/ct_monitor/`): Certificate Transparency log monitoring +- **`verifier`** (`dstack/verifier/`): TDX quote verification service using `dcap-qvl` +- **`supervisor`** (`dstack/supervisor/`): Process supervision inside CVMs - **SDKs** (`sdk/`): Client SDKs in Rust, Python, Go, and JavaScript for interacting with guest-agent APIs ## Build Commands @@ -40,6 +44,8 @@ dstack consists of several core components that interact to provide TEE-based co ### Rust Components ```bash +cd dstack + # Build all components cargo build --release @@ -63,7 +69,7 @@ cargo clippy -- -D warnings --allow unused_variables ### Ethereum Smart Contracts (KMS Auth) ```bash -cd kms/auth-eth +cd dstack/kms/auth-eth npm install # Install Node.js dependencies for bootAuth server forge install # Install Foundry dependencies (submodules) @@ -94,7 +100,7 @@ make test # Run tests ```bash # Run all Rust tests (requires simulator) -./run-tests.sh +./dstack/run-tests.sh ``` This script: @@ -106,6 +112,8 @@ This script: ### Running Specific Tests ```bash +cd dstack + # Run tests for a specific package cargo test -p dstack-kms --all-features @@ -119,7 +127,7 @@ cargo test --all-features -- --show-output --test-threads=1 ### Foundry Tests (Ethereum Contracts) ```bash -cd kms/auth-eth +cd dstack/kms/auth-eth # Run all Foundry tests forge test @@ -168,7 +176,7 @@ This rule is enforced in `.cursorrules`. ### Local Development Setup -1. Build meta-dstack artifacts (see README.md section "Build and Run") +1. Build guest-OS artifacts through `os/build.sh` (see `os/README.md`) 2. Download or build guest OS image 3. Run components in separate terminals: - KMS: `./dstack-kms -c kms.toml` @@ -193,10 +201,10 @@ Ingress mapping pattern: `[-[][s|g]].` ## Important Files -- `Cargo.toml`: Workspace configuration with all Rust crates -- `vmm.toml`: VMM configuration (CID pool, port mapping, KMS/gateway URLs) -- `kms.toml`: KMS configuration (contract addresses, RPC endpoints) -- `gateway.toml`: Gateway configuration (domain, certificates, WireGuard) +- `dstack/Cargo.toml`: Workspace configuration with all Rust crates +- `dstack/vmm/vmm.toml`: VMM configuration (CID pool, port mapping, KMS/gateway URLs) +- `dstack/kms/kms.toml`: KMS configuration (contract addresses, RPC endpoints) +- `dstack/gateway/gateway.toml`: Gateway configuration (domain, certificates, WireGuard) - `docker-compose.yaml`: App deployment format (normalized to `.app-compose.json`) ## Common Tasks @@ -204,7 +212,7 @@ Ingress mapping pattern: `[-[][s|g]].` ### Adding a New Rust Crate 1. Create crate directory and `Cargo.toml` -2. Add to workspace members in root `Cargo.toml` +2. Add to workspace members in `dstack/Cargo.toml` 3. Add workspace dependency if it will be used by other crates ### Modifying RPC APIs @@ -216,9 +224,9 @@ RPC definitions use `prpc` framework with Protocol Buffers: ### Working with TDX Quotes -- Pure Rust API: `tdx-attest/` -- Verification: `verifier/` using `dcap-qvl` -- Event log parsing: `cc-eventlog/` +- Pure Rust API: `dstack/tdx-attest/` +- Verification: `dstack/verifier/` using `dcap-qvl` +- Event log parsing: `dstack/cc-eventlog/` ## Documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f5f60267..c1672e259 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ This project uses SPDX headers for license compliance. You should add appropriat We have a script to automatically add SPDX headers based on git blame data: ```bash -python3 scripts/add-spdx-attribution.py --file path/to/file.rs +python3 dstack/scripts/add-spdx-attribution.py --file path/to/file.rs ``` Before submitting your changes, verify SPDX compliance using the [REUSE tool](https://github.com/fsfe/reuse-tool): diff --git a/README.md b/README.md index 2aa9ed0fe..d5120191b 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,20 @@ Setting up dstack on your own hardware? Start with the [self-hosted quick onboar ![Architecture](./docs/assets/arch.png) +### Repository layout + +```text +dstack/ Core services, Rust crates, host and guest runtime code +sdk/ Stable public SDK paths +os/ Guest-OS payload, image contract, and build backends +docs/ User and operator documentation +tools/ Standalone development and security tools +``` + +The currently implemented OS backend is Yocto under `os/yocto/`. Shared rootfs +payload and release assembly stay outside that backend so another builder can +be added later without duplicating them. See [`os/README.md`](./os/README.md). + Your container runs inside a Confidential VM, such as Intel TDX or AMD SEV-SNP, with optional GPU isolation via NVIDIA Confidential Computing. The CPU TEE protects application logic; the GPU TEE protects model weights and inference data. **Core components:** @@ -193,7 +207,7 @@ Yes. dstack runs on supported TEE-capable servers, including Intel TDX-capable h
How do users verify my deployment? -Your app exposes attestation quotes via the SDK. Users verify these quotes using [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/verifier), [dcap-qvl](https://github.com/Phala-Network/dcap-qvl), or the [Trust Center](https://trust.phala.com). See the [verification guide](./docs/verification.md) for details. +Your app exposes attestation quotes via the SDK. Users verify these quotes using [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/dstack/verifier), [dcap-qvl](https://github.com/Phala-Network/dcap-qvl), or the [Trust Center](https://trust.phala.com). See the [verification guide](./docs/verification.md) for details.
@@ -231,4 +245,8 @@ Logo and branding assets: [dstack-logo-kit](./docs/assets/dstack-logo-kit/) ## License -Apache 2.0 +This is a multi-license repository. Core services, SDKs, documentation, tools, +and the shared guest rootfs payload are Apache-2.0. The imported Yocto backend +and derived image-assembly code retain Business Source License 1.1 terms. See +[`os/README.md`](./os/README.md), file-level SPDX declarations, and +[`REUSE.toml`](./REUSE.toml) for the exact scope. diff --git a/attestation.md b/attestation.md index c10232cd8..22c4b72f3 100644 --- a/attestation.md +++ b/attestation.md @@ -50,7 +50,7 @@ The resulting dstack-0.5.4.tar.gz contains: - rootfs.img.verity: root filesystem - metadata.json: image metadata, including kernel boot cmdline -Calculate image MRs using [dstack-mr](dstack-mr/): +Calculate image MRs using [dstack-mr](dstack/dstack-mr/): ```bash cargo run --manifest-path ../dstack/Cargo.toml --bin dstack-mr measure -c 4 -m 4G dstack-0.5.4/metadata.json ``` @@ -63,7 +63,7 @@ To verify dstack App data trustworthiness: - Review source code for correctness and safety. - Build image from source. -- Calculate MRTD, RTMR0, RTMR1, and RTMR2 values using [dstack-mr](https://github.com/kvinwang/dstack-mr). +- Calculate MRTD, RTMR0, RTMR1, and RTMR2 values using [dstack-mr](https://github.com/Dstack-TEE/dstack/tree/master/dstack/dstack-mr). - Verify quote measurements: - Confirm MRTD, RTMR0, RTMR1, and RTMR2 match pre-calculated values. - Verify RTMR3 matches the event log replay result. diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 99c685731..393c5b20e 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -60,7 +60,7 @@ The ignored live regression test cross-checks dstack's pure Rust SNP measurement Command: ```bash -cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture ``` Last captured vector (STALE — regenerate before citing as proof): @@ -81,7 +81,7 @@ vcpu_type=EPYC-v4 guest_features=0x1 append=console=ttyS0 loglevel=7 sev_snp_measurement=requires-refresh-after-mr-config-v3-host-data-binding -cargo_live_test=cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +cargo_live_test=cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture cargo_live_test_result=stale after SNP app identity moved from cmdline to HOST_DATA DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_END ``` @@ -104,7 +104,7 @@ DSTACK_SEV_SNP_ATTESTATION_PROOF_END ## Manual dstack E2E smoke status -An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. The reusable version of that smoke is checked in at `test-scripts/snp-e2e-smoke.sh` for follow-up debugging on SNP hosts. +An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. The reusable version of that smoke is checked in at `dstack/test-scripts/snp-e2e-smoke.sh` for follow-up debugging on SNP hosts. That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: @@ -155,21 +155,27 @@ The checked-in smoke is enough to reproduce the current boundary on a compatible Practical implication for reviewers/testers on a fresh box: 1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. -2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. -3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_BASE_URL` to a trusted AMD-KDS-compatible mirror/cache base URL such as `https://mirror.example.com/vcek/v1` (or, for a path-prefix relay, `https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`) and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production auth policy should keep accepting only `UpToDate` and deny any advisory id unless explicitly allowlisted. -4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: +2. Build the PR binaries with `cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor -p dstack-kms`. +3. Run `dstack/test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_BASE_URL` to a trusted AMD-KDS-compatible mirror/cache base URL such as `https://mirror.example.com/vcek/v1` (or, for a path-prefix relay, `https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`) and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production auth policy should keep accepting only `UpToDate` and deny any advisory id unless explicitly allowlisted. +4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. Build the Yocto backend from the same monorepo revision, for example: ```bash - git clone https://github.com/Dstack-TEE/meta-dstack.git - cd meta-dstack - git submodule update --init --recursive --depth 1 + git clone https://github.com/Dstack-TEE/dstack.git cd dstack git fetch https://github.com/clawdbot-glitch003/dstack.git feat/amd-sev-snp-conversion git checkout -B feat/amd-sev-snp-conversion FETCH_HEAD - cd .. - source dev-setup ./bb-build + git submodule update --init -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security + source os/yocto/dev-setup ./bb-build sed -i 's/^MACHINE ??= .*/MACHINE = "sev-snp"/' ./bb-build/conf/local.conf - FLAVORS=dev make dist DIST_DIR=$PWD/images BB_BUILD_DIR=$PWD/bb-build + FLAVORS=dev make -C os/yocto dist DIST_DIR=$PWD/images BB_BUILD_DIR=$PWD/bb-build # Use the resulting dstack-dev image directory with: # DSTACK_SNP_SMOKE_IMAGE_NAME= ``` @@ -184,16 +190,16 @@ If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resett Run locally for this review-ready staging branch: ```bash -bash -n test-scripts/snp-e2e-smoke.sh -cargo fmt --all -cargo test -p dstack-kms --all-features -cargo test -p dstack-attest --all-features -cargo test -p dstack-vmm --all-features -cargo test -p ra-rpc --all-features -cargo check --workspace --all-features -cargo clippy --workspace --all-features -- -D warnings --allow unused_variables +bash -n dstack/test-scripts/snp-e2e-smoke.sh +cargo fmt --manifest-path dstack/Cargo.toml --all +cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features +cargo test --manifest-path dstack/Cargo.toml -p dstack-attest --all-features +cargo test --manifest-path dstack/Cargo.toml -p dstack-vmm --all-features +cargo test --manifest-path dstack/Cargo.toml -p ra-rpc --all-features +cargo check --manifest-path dstack/Cargo.toml --workspace --all-features +cargo clippy --manifest-path dstack/Cargo.toml --workspace --all-features -- -D warnings --allow unused_variables git diff --check -cd kms/auth-simple && bun install && bun run check +cd dstack/kms/auth-simple && bun install && bun run check ``` ## Remaining production follow-up diff --git a/docs/auth-simple-operations.md b/docs/auth-simple-operations.md index ffd387a80..416df9e4c 100644 --- a/docs/auth-simple-operations.md +++ b/docs/auth-simple-operations.md @@ -296,5 +296,5 @@ Expected responses: ## See Also - [Deployment Guide](./deployment.md) - Initial setup -- [auth-simple README](../kms/auth-simple/README.md) - Developer reference +- [auth-simple README](../dstack/kms/auth-simple/README.md) - Developer reference - [On-Chain Governance](./onchain-governance.md) - Smart contract-based alternative diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 4dac27bd0..a96d5c9dc 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -93,7 +93,7 @@ sudo apt install -y dnsmasq Install the DHCP notification script (notifies VMM when a VM gets an IP so port forwarding can be established): ```bash -sudo cp scripts/dhcp-notify.sh /usr/local/bin/dhcp-notify.sh +sudo cp dstack/scripts/dhcp-notify.sh /usr/local/bin/dhcp-notify.sh sudo chmod +x /usr/local/bin/dhcp-notify.sh ``` diff --git a/docs/deployment.md b/docs/deployment.md index 532ec3f87..e8cd9af9e 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -59,10 +59,10 @@ Clone and build dstack-vmm: ```bash git clone https://github.com/Dstack-TEE/dstack cd dstack -cargo build --release -p dstack-vmm -p supervisor +cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor mkdir -p vmm-data -cp target/release/dstack-vmm vmm-data/ -cp target/release/supervisor vmm-data/ +cp dstack/target/release/dstack-vmm vmm-data/ +cp dstack/target/release/supervisor vmm-data/ cd vmm-data/ ``` @@ -93,7 +93,7 @@ address = "vsock:2" port = 10000 ``` -Download guest images from [meta-dstack releases](https://github.com/Dstack-TEE/meta-dstack/releases) and extract to `./images/`. +Download guest images from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases) and extract to `./images/`. > For reproducible builds and verification, see the [Security Model](./security/security-model.md). @@ -115,8 +115,8 @@ Production KMS requires: | Server | Use Case | Configuration | |--------|----------|---------------| -| [auth-simple](../kms/auth-simple/) | Config-file-based whitelisting | JSON config file | -| [auth-eth](../kms/auth-eth/) | On-chain governance via smart contracts | Ethereum RPC + contract | +| [auth-simple](../dstack/kms/auth-simple/) | Config-file-based whitelisting | JSON config file | +| [auth-eth](../dstack/kms/auth-eth/) | On-chain governance via smart contracts | Ethereum RPC + contract | | Custom | Your own authorization logic | Implement webhook interface | All auth servers implement the same webhook interface: @@ -146,7 +146,7 @@ Create `auth-config.json` for initial KMS deployment: Run auth-simple: ```bash -cd kms/auth-simple +cd dstack/kms/auth-simple bun install PORT=3001 AUTH_CONFIG_PATH=/path/to/auth-config.json bun run start ``` @@ -190,7 +190,7 @@ AUTH_WEBHOOK_URL=http://your-auth-server:3001 KMS_RPC_ADDR=0.0.0.0:9201 GUEST_AGENT_ADDR=127.0.0.1:9205 OS_IMAGE=dstack-0.5.5 -IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.5/dstack-0.5.5.tar.gz +IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v0.5.5/dstack-0.5.5.tar.gz ``` Then run: @@ -206,7 +206,7 @@ Then run: **Monitor startup:** ```bash -tail -f ../../vmm-data/run/vm//serial.log +tail -f ../../../vmm-data/run/vm//serial.log ``` Wait for `[ OK ] Finished App Compose Service.` @@ -345,8 +345,9 @@ This will: Monitor for unauthorized certificates issued to your domain. ```bash -cargo build --release -p ct_monitor -./target/release/ct_monitor \ +cd /path/to/dstack +cargo build --manifest-path dstack/Cargo.toml --release -p ct_monitor +./dstack/target/release/ct_monitor \ --gateway-uri https:// \ --domain ``` diff --git a/docs/faq.md b/docs/faq.md index 80a6bc0ed..4f43b053a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -30,4 +30,4 @@ Once you have the required privileges, make sure the supervisor process is shut ps aux | grep supervisor | grep $(whoami) | grep -v grep ``` -Log out of all your sessions and log back in. Check your groups with the `id` command, and this should resolve the issue. \ No newline at end of file +Log out of all your sessions and log back in. Check your groups with the `id` command, and this should resolve the issue. diff --git a/docs/normalized-app-compose.md b/docs/normalized-app-compose.md index e6a99c83d..66ccb4c43 100644 --- a/docs/normalized-app-compose.md +++ b/docs/normalized-app-compose.md @@ -107,7 +107,7 @@ func main() { return } fmt.Println("Deterministic JSON:", string(jsonBytes)) - + // This JSON string can now be used to generate a compose hash } ``` @@ -243,4 +243,3 @@ Here's how each language handles deterministic JSON serialization for compose ha Getting deterministic JSON serialization across different languages for compose hash generation isn't the default behavior. It needs careful setup. Go works well with compact output and special value handling, but needs custom key sorting for maps. Python and JavaScript both need explicit setup for key sorting and compact output. JavaScript notably requires manual recursive sorting of object keys. By following these recommendations, dstack can ensure that the same `app-compose.json` file produces the same SHA256 compose hash across all its Go, Python, and JavaScript components. This provides a reliable foundation for the project's distributed architecture and change detection system. - diff --git a/docs/onboarding.md b/docs/onboarding.md index 169ed57ae..4f4b8ee7f 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -3,7 +3,7 @@ Use this guide to get a first dstack app running on one Intel TDX host. The workflow uses `dstackup` for host setup and `dstack` for app deployment: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh sudo dstackup install sudo dstack deploy \ -n hello-nginx \ @@ -71,7 +71,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y Build and install the `dstackup` bootstrap command: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh ``` The bootstrap installer builds `dstackup` from a temporary source checkout and installs it under `/usr/local/bin`. The `dstackup install` command then builds and installs `dstack`, `dstack-auth`, `dstack-vmm`, `supervisor`, static assets, and host config into the system layout. @@ -84,7 +84,7 @@ Run: sudo dstackup install ``` -`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest CPU image from [meta-dstack releases](https://github.com/Dstack-TEE/meta-dstack/releases), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. +`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest CPU image from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. Pre-monorepo `meta-dstack` releases remain a fallback for pinned older versions. On TDX, `dstackup install` starts the SGX key provider automatically from `/usr/local/share/dstack/key-provider-build`. To use a different provider, pass one of: @@ -207,7 +207,7 @@ Use `--prefix` when you want a second isolated install on the same host. A custo Install `dstackup` into the prefix, then use the same prefix for `dstackup` and `dstack`: ```bash -curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/scripts/install.sh | sh -s -- --prefix /opt/dstack-test +curl -fsSL https://raw.githubusercontent.com/Dstack-TEE/dstack/master/dstack/scripts/install.sh | sh -s -- --prefix /opt/dstack-test sudo /opt/dstack-test/bin/dstackup install \ --prefix /opt/dstack-test \ @@ -246,7 +246,7 @@ Use the [deployment guide](./deployment.md) when you need domain routing, Gatewa ### Image download fails -`dstackup install` downloads the latest CPU image when KMS mode needs an image and none exists locally. If the download fails, check network access to GitHub and the meta-dstack release: +`dstackup install` downloads the latest CPU image when KMS mode needs an image and none exists locally. If the download fails, check network access to GitHub and the dstack guest-OS release: ```bash sudo dstackup image pull diff --git a/docs/onchain-governance.md b/docs/onchain-governance.md index a78598a65..42099c069 100644 --- a/docs/onchain-governance.md +++ b/docs/onchain-governance.md @@ -193,7 +193,7 @@ struct AppBootInfo { } ``` -Source: [`kms/auth-eth/contracts/`](../kms/auth-eth/contracts/) +Source: [`dstack/kms/auth-eth/contracts/`](../dstack/kms/auth-eth/contracts/) ## See Also diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index e61103d9c..5128a07ac 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -152,7 +152,7 @@ The dstack-guest-agent listens on VSOCK port 8000 inside the CVM, providing inte | ListContainers | List running containers | ListContainersResponse | | Shutdown | Gracefully shut down the guest | Empty | -Full specification: [guest_api.proto](../../guest-api/proto/guest_api.proto) +Full specification: [guest_api.proto](../../dstack/guest-api/proto/guest_api.proto) ### VSOCK-based Host API Service @@ -170,7 +170,7 @@ The dstack-vmm listens on a configured VSOCK port on the bare-metal host system. | Notify | Send notification to host | Notification | Empty | | GetSealingKey | Retrieve sealing key | GetSealingKeyRequest | GetSealingKeyResponse | -Full specification: [host_api.proto](../../host-api/proto/host_api.proto) +Full specification: [host_api.proto](../../dstack/host-api/proto/host_api.proto) ### HTTP-based Public Guest API Service @@ -187,6 +187,6 @@ The dstack-guest-agent runs an HTTP server on port 8090 inside the CVM. This por | Info | Get application information | AppInfo | | Version | Get guest agent version | WorkerVersion | -The service also provides a web dashboard at the root URL (`/`) showing basic CVM information. View the dashboard template [here](../../guest-agent/templates/dashboard.html). +The service also provides a web dashboard at the root URL (`/`) showing basic CVM information. View the dashboard template [here](../../dstack/guest-agent/templates/dashboard.html). -Full specification: [agent_rpc.proto](../../guest-agent/rpc/proto/agent_rpc.proto) +Full specification: [agent_rpc.proto](../../dstack/guest-agent/rpc/proto/agent_rpc.proto) diff --git a/docs/security/security-model.md b/docs/security/security-model.md index 4cbee519c..8a9aef69a 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -12,7 +12,7 @@ The only thing you must trust is **TEE hardware**. Intel TDX is the production p Everything else is verifiable. -**The dstack OS** is measured during boot and recorded in the attestation quote. You verify it by rebuilding from [meta-dstack](https://github.com/Dstack-TEE/meta-dstack) source and comparing measurements, or by checking that the OS hash is whitelisted in a governance contract you trust. +**The dstack OS** is measured during boot and recorded in the attestation quote. You verify it by rebuilding from the [`os/`](../../os/) source and comparing measurements, or by checking that the OS hash is whitelisted in a governance contract you trust. **The KMS** runs in its own TEE with its own attestation quote. You verify it the same way you verify any dstack workload. @@ -86,7 +86,7 @@ dstack implements layered verification from hardware to application. Each layer **Hardware layer.** The TEE provides the root of trust. The attestation quote is cryptographically signed by TEE hardware, and verification confirms the signature chain. The TCB status shows whether firmware is patched against known vulnerabilities. -**OS layer.** The dstack OS is measured during boot into MRTD and RTMR0-2. MRTD captures the virtual firmware. RTMR0 captures firmware configuration. RTMR1 captures the Linux kernel. RTMR2 captures kernel command-line parameters. You verify integrity by computing expected measurements from meta-dstack source and comparing them to the quote. +**OS layer.** The dstack OS is measured during boot into MRTD and RTMR0-2. MRTD captures the virtual firmware. RTMR0 captures firmware configuration. RTMR1 captures the Linux kernel. RTMR2 captures kernel command-line parameters. You verify integrity by computing expected measurements from the monorepo OS source and comparing them to the quote. **Application layer.** Your application is measured into RTMR3 as the compose-hash, which is the SHA256 hash of your normalized docker-compose configuration. Each image must use SHA256 digest pinning. This proves exactly which container images are running and that no code substitution happened after measurement. diff --git a/docs/tutorials/attestation-verification.md b/docs/tutorials/attestation-verification.md index 972f70a01..aec14d5b5 100644 --- a/docs/tutorials/attestation-verification.md +++ b/docs/tutorials/attestation-verification.md @@ -114,7 +114,7 @@ Before starting, ensure you have: Verify you have a running CVM: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` @@ -126,7 +126,7 @@ The VMM provides a `/guest/Info` endpoint that proxies into the CVM and retrieve ### Via VMM Guest Proxy ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get the VM UUID for hello-world @@ -274,7 +274,7 @@ rm -rf qemu-tdx ### Build the measurement calculator ```bash -cd ~/dstack +cd ~/dstack/dstack cargo build --release -p dstack-mr-cli ``` @@ -369,7 +369,7 @@ Compare the CVM's actual measurements against your expected values: #!/bin/bash # verify-measurements.sh -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get VM UUID @@ -552,7 +552,7 @@ echo "Instance: $INSTANCE_NAME" echo "Image: $IMAGE_VERSION" echo "" -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # --- Step 1: Get VM UUID --- @@ -662,7 +662,7 @@ if [ -x "$DSTACK_MR" ] && [ -f "$METADATA" ]; then done else echo " SKIP - dstack-mr not built or metadata not found" - echo " To enable: cd ~/dstack && cargo build --release -p dstack-mr-cli" + echo " To enable: cd ~/dstack/dstack && cargo build --release -p dstack-mr-cli" fi # --- Step 6: Display event log --- @@ -785,8 +785,18 @@ dstack-mr measure --cpu 2 --memory 2G \ For highest assurance, build images from source: ```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git -cd meta-dstack/repro-build +git clone https://github.com/Dstack-TEE/dstack.git +cd dstack +git submodule update --init -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security +cd os/yocto/repro-build ./repro-build.sh -n # Reproducible build ``` @@ -845,4 +855,4 @@ With the foundation complete, you're ready to explore: - [Intel TDX Documentation](https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/documentation.html) - [DCAP Attestation Guide](https://download.01.org/intel-sgx/latest/dcap-latest/linux/docs/) - [dstack Attestation Source](https://github.com/Dstack-TEE/dstack/tree/main/attestation) -- [Reproducible Builds for meta-dstack](https://github.com/Dstack-TEE/meta-dstack/tree/main/repro-build) +- [Reproducible guest-OS builds](../../os/yocto/repro-build/) diff --git a/docs/tutorials/clone-build-dstack-vmm.md b/docs/tutorials/clone-build-dstack-vmm.md index 1e731e8e4..c1d15f2b8 100644 --- a/docs/tutorials/clone-build-dstack-vmm.md +++ b/docs/tutorials/clone-build-dstack-vmm.md @@ -55,25 +55,27 @@ All build commands should be run as the `ubuntu` user. Only the final installati ### Step 2: Verify dstack Repository -The dstack repository should already be cloned and checked out at v0.5.7 from [Gramine Key Provider](/tutorial/gramine-key-provider): +The dstack repository should already be cloned and checked out on the current +`master` branch from [Gramine Key Provider](/tutorial/gramine-key-provider): ```bash cd ~/dstack git describe --tags -# Should show v0.5.7 +git branch --show-current +# Should show master ``` ### Step 3: Build dstack-vmm ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm cargo build --release ``` ### Step 5: Build dstack-supervisor ```bash -cd ~/dstack +cd ~/dstack/dstack cargo build --release -p supervisor ``` @@ -81,11 +83,11 @@ cargo build --release -p supervisor ```bash # Install VMM -sudo cp ~/dstack/target/release/dstack-vmm /usr/local/bin/dstack-vmm +sudo cp ~/dstack/dstack/target/release/dstack-vmm /usr/local/bin/dstack-vmm sudo chmod 755 /usr/local/bin/dstack-vmm # Install supervisor -sudo cp ~/dstack/target/release/supervisor /usr/local/bin/dstack-supervisor +sudo cp ~/dstack/dstack/target/release/supervisor /usr/local/bin/dstack-supervisor sudo chmod 755 /usr/local/bin/dstack-supervisor ``` @@ -106,12 +108,12 @@ ls -la /usr/local/bin/dstack-supervisor ### Specify a Different Version ```bash -# Check out a specific version -git checkout v0.5.4 +# Check out a monorepo-era release tag when one is available +git checkout -# Or use main branch for latest development -git checkout main -git pull +# Or use the master branch for latest development +git checkout master +git pull --ff-only ``` ### Clean Build @@ -119,6 +121,7 @@ git pull To rebuild from scratch: ```bash +cd ~/dstack/dstack cargo clean cargo build --release ``` @@ -128,8 +131,9 @@ cargo build --release For development with better error messages: ```bash +cd ~/dstack/dstack cargo build -# Binary at ~/dstack/target/debug/dstack-vmm +# Binary at ~/dstack/dstack/target/debug/dstack-vmm ``` --- diff --git a/docs/tutorials/contract-deployment.md b/docs/tutorials/contract-deployment.md index 3c944f14c..e01085edb 100644 --- a/docs/tutorials/contract-deployment.md +++ b/docs/tutorials/contract-deployment.md @@ -29,7 +29,7 @@ Before starting, ensure you have: - Completed [Blockchain Wallet Setup](/tutorial/blockchain-setup) with: - Wallet private key stored in `~/.dstack/secrets/sepolia-private-key` - Sepolia testnet ETH (~0.01 ETH recommended) -- dstack repository cloned locally at v0.5.7: `git clone -b v0.5.7 https://github.com/Dstack-TEE/dstack ~/dstack` +- Current dstack monorepo cloned locally: `git clone https://github.com/Dstack-TEE/dstack ~/dstack` ## What Gets Deployed The deployment creates two smart contracts on Sepolia: @@ -45,17 +45,17 @@ These contracts use the UUPS (Universal Upgradeable Proxy Standard) pattern for ## Deployment -> **Important: Run these steps on your LOCAL machine, not on the TDX server.** Contract deployment requires your Ethereum private key. By running locally, your private key never touches the server. You need a clone of the dstack repo on your local machine: `git clone -b v0.5.7 https://github.com/Dstack-TEE/dstack ~/dstack` +> **Important: Run these steps on your LOCAL machine, not on the TDX server.** Contract deployment requires your Ethereum private key. By running locally, your private key never touches the server. You need a current clone of the dstack repo on your local machine: `git clone https://github.com/Dstack-TEE/dstack ~/dstack` ### Step 1: Clone Repository and Navigate to auth-eth -On your **local machine**, clone the dstack repository (if you haven't already) and check out v0.5.7: +On your **local machine**, clone the dstack repository (if you haven't already) and use the current `master` branch: ```bash git clone https://github.com/Dstack-TEE/dstack.git ~/dstack 2>/dev/null || true cd ~/dstack -git checkout v0.5.7 -cd kms/auth-eth +git checkout master +cd dstack/kms/auth-eth ``` ### Step 2: Install Node.js and Dependencies diff --git a/docs/tutorials/gateway-build-configuration.md b/docs/tutorials/gateway-build-configuration.md index 0d24e8156..3a807f8df 100644 --- a/docs/tutorials/gateway-build-configuration.md +++ b/docs/tutorials/gateway-build-configuration.md @@ -115,19 +115,19 @@ The `dstacktee/dstack-gateway:0.5.7` image isn't published on Docker Hub, so we The [Build dstack from Source](/tutorial/clone-build-dstack-vmm) tutorial builds `dstack-vmm` and `supervisor`, but not the gateway. Build it now: ```bash -cd ~/dstack +cd ~/dstack/dstack cargo build --release -p dstack-gateway ``` Verify the binary was built: ```bash -ls -lh ~/dstack/target/release/dstack-gateway +ls -lh ~/dstack/dstack/target/release/dstack-gateway ``` Expected output (typically 15-25MB): ``` --rwxrwxr-x 1 ubuntu ubuntu 20M ... /home/ubuntu/dstack/target/release/dstack-gateway +-rwxrwxr-x 1 ubuntu ubuntu 20M ... /home/ubuntu/dstack/dstack/target/release/dstack-gateway ``` #### Create Dockerfile @@ -161,8 +161,8 @@ EOF Copy the gateway binary and entrypoint script into the build context: ```bash -cp ~/dstack/target/release/dstack-gateway ~/gateway-deploy/ -cp ~/dstack/gateway/dstack-app/builder/entrypoint.sh ~/gateway-deploy/ +cp ~/dstack/dstack/target/release/dstack-gateway ~/gateway-deploy/ +cp ~/dstack/dstack/gateway/dstack-app/builder/entrypoint.sh ~/gateway-deploy/ ``` #### Build Docker image @@ -458,7 +458,7 @@ cat .app_env Now generate the VMM deployment manifest: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) diff --git a/docs/tutorials/gateway-service-setup.md b/docs/tutorials/gateway-service-setup.md index 2418d09fe..ba693486e 100644 --- a/docs/tutorials/gateway-service-setup.md +++ b/docs/tutorials/gateway-service-setup.md @@ -84,7 +84,7 @@ Load environment variables and deploy: cd ~/gateway-deploy set -a; source .env; set +a -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 deploy \ @@ -369,7 +369,7 @@ Expected output shows both domains with `has_cert: true` and expiry dates ~90 da If the cert status shows `has_cert: false`, check the gateway logs for ACME errors: ```bash -VM_ID=$(cd ~/dstack/vmm && ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm --json | jq -r '.[] | select(.name=="dstack-gateway") | .id') +VM_ID=$(cd ~/dstack/dstack/vmm && ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm --json | jq -r '.[] | select(.name=="dstack-gateway") | .id') curl -s -H "Authorization: Bearer $(cat ~/.dstack/secrets/vmm-auth-token)" \ "http://127.0.0.1:9080/logs?id=$VM_ID&follow=false&ansi=false&lines=50" | grep -i "cert\|renew\|acme" ``` @@ -407,7 +407,7 @@ sudo haproxy -c -f /etc/haproxy/haproxy.cfg && sudo systemctl reload haproxy Navigate to the VMM directory first: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/gramine-key-provider.md b/docs/tutorials/gramine-key-provider.md index 89f2547bc..1a7cf59af 100644 --- a/docs/tutorials/gramine-key-provider.md +++ b/docs/tutorials/gramine-key-provider.md @@ -105,13 +105,13 @@ Clone the dstack repository and check out the v0.5.7 release: cd ~ git clone https://github.com/Dstack-TEE/dstack.git cd dstack -git checkout v0.5.7 +git checkout master ``` ### Step 2: Navigate to Key Provider ```bash -cd ~/dstack/key-provider-build +cd ~/dstack/dstack/key-provider-build ls -la ``` @@ -125,7 +125,7 @@ You should see: The key provider needs to know where to find a PCCS for quote verification. Create the QCNL configuration file: ```bash -cat > ~/dstack/key-provider-build/sgx_default_qcnl.conf << 'EOF' +cat > ~/dstack/dstack/key-provider-build/sgx_default_qcnl.conf << 'EOF' { "pccs_url": "https://pccs.phala.network/sgx/certification/v4/", "use_secure_cert": false, @@ -143,7 +143,7 @@ The default configuration binds to localhost, but CVMs need to access the key pr ```bash # Change from 127.0.0.1:3443 to 0.0.0.0:3443 -sed -i 's/"127\.0\.0\.1:3443:3443"/"0.0.0.0:3443:3443"/' ~/dstack/key-provider-build/docker-compose.yaml +sed -i 's/"127\.0\.0\.1:3443:3443"/"0.0.0.0:3443:3443"/' ~/dstack/dstack/key-provider-build/docker-compose.yaml ``` > **Note:** This makes the key provider accessible from CVMs via the QEMU user-mode networking gateway (`10.0.2.2`). The key provider still verifies TDX quotes, so only legitimate CVMs can obtain keys. diff --git a/docs/tutorials/guest-image-setup.md b/docs/tutorials/guest-image-setup.md index e5ab2f39f..8570aa1e7 100644 --- a/docs/tutorials/guest-image-setup.md +++ b/docs/tutorials/guest-image-setup.md @@ -75,7 +75,7 @@ echo "Installing guest images for version: $DSTACK_VERSION" # Download the image archive cd /tmp -wget https://github.com/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz +wget https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz ``` Verify the download: @@ -240,16 +240,16 @@ Use the `dstack-image-oci.sh` script to package and push a guest image directory ```bash # Push a standard image (auto-tags: version + sha256-hash) -./scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image +./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image # Push an nvidia variant -./scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-nvidia-0.5.8 ghcr.io/your-org/guest-image +./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-nvidia-0.5.8 ghcr.io/your-org/guest-image # Push with a custom tag -./scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image --tag latest +./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image --tag latest # List tags in the registry -./scripts/dstack-image-oci.sh list ghcr.io/your-org/guest-image +./dstack/scripts/dstack-image-oci.sh list ghcr.io/your-org/guest-image ``` The script reads `metadata.json` and `digest.txt` from the image directory and auto-generates tags: @@ -290,7 +290,7 @@ You can have multiple image versions installed simultaneously: ```bash # Download additional version DSTACK_VERSION="0.5.3" -wget https://github.com/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz +wget https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz # Extract to images directory (tarball already contains dstack-X.Y.Z/ folder) sudo tar -xvf dstack-${DSTACK_VERSION}.tar.gz -C /var/lib/dstack/images/ @@ -384,7 +384,7 @@ With guest images configured and VMM able to access them, you're ready to deploy ## Additional Resources -- [meta-dstack Repository](https://github.com/Dstack-TEE/meta-dstack) +- [Guest OS source and build backends](../../os/) - [dstack GitHub Repository](https://github.com/Dstack-TEE/dstack) - [Yocto Project](https://www.yoctoproject.org/) - [TDX Guest Architecture](https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/overview.html) diff --git a/docs/tutorials/hello-world-app.md b/docs/tutorials/hello-world-app.md index 66dec5f29..2792e5ac7 100644 --- a/docs/tutorials/hello-world-app.md +++ b/docs/tutorials/hello-world-app.md @@ -183,7 +183,7 @@ ssh user@your-server Use `vmm-cli.py compose` to generate the encrypted deployment manifest. The `--gateway` and `--kms` flags enable gateway registration and KMS attestation. ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 compose \ @@ -264,7 +264,7 @@ ssh user@your-server ## Step 6: Deploy the CVM ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) SRV_DOMAIN=$(grep ^SRV_DOMAIN ~/gateway-deploy/.env | cut -d= -f2) @@ -372,7 +372,7 @@ This bypasses the gateway and hits nginx directly. You should see the same nginx Navigate to the VMM directory: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ``` diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index e93e83692..c4ef875b2 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -67,7 +67,7 @@ Build the KMS service using Cargo in release mode. ### Navigate to repository root ```bash -cd ~/dstack +cd ~/dstack/dstack ``` ### Build KMS in release mode @@ -83,18 +83,18 @@ This compilation will: ### Verify the build ```bash -ls -lh ~/dstack/target/release/dstack-kms +ls -lh ~/dstack/dstack/target/release/dstack-kms ``` Expected output (typically 20-30MB): ``` --rwxrwxr-x 1 ubuntu ubuntu 25M Nov 20 10:30 /home/ubuntu/dstack/target/release/dstack-kms +-rwxrwxr-x 1 ubuntu ubuntu 25M Nov 20 10:30 /home/ubuntu/dstack/dstack/target/release/dstack-kms ``` ### Test the binary ```bash -~/dstack/target/release/dstack-kms --help +~/dstack/dstack/target/release/dstack-kms --help ``` This displays available command-line options. @@ -106,7 +106,7 @@ Install the KMS binary to a system-wide location. ### Copy to /usr/local/bin ```bash -sudo cp ~/dstack/target/release/dstack-kms /usr/local/bin/dstack-kms +sudo cp ~/dstack/dstack/target/release/dstack-kms /usr/local/bin/dstack-kms sudo chmod 755 /usr/local/bin/dstack-kms ``` @@ -254,7 +254,7 @@ You should see Node.js v20.x and npm v10.x (or later). ### Navigate to auth-eth directory ```bash -cd ~/dstack/kms/auth-eth +cd ~/dstack/dstack/kms/auth-eth ``` ### Install dependencies @@ -513,10 +513,10 @@ EOF ```bash # Copy KMS binary -cp ~/dstack/target/release/dstack-kms . +cp ~/dstack/dstack/target/release/dstack-kms . # Copy auth-eth service -cp -r ~/dstack/kms/auth-eth auth-eth +cp -r ~/dstack/dstack/kms/auth-eth auth-eth # Copy auth-eth environment config cp /etc/kms/auth-eth.env . diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index f86058c67..b54051ae4 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -253,7 +253,7 @@ Use the VMM CLI tool to deploy the CVM: ```bash # Navigate to dstack VMM directory -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm # Set VMM auth from saved token export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) diff --git a/docs/tutorials/troubleshooting-dstack-installation.md b/docs/tutorials/troubleshooting-dstack-installation.md index 2b5182ac7..c199d930d 100644 --- a/docs/tutorials/troubleshooting-dstack-installation.md +++ b/docs/tutorials/troubleshooting-dstack-installation.md @@ -108,6 +108,7 @@ rustup update stable ### Network timeout downloading crates ```bash +cd ~/dstack/dstack export CARGO_HTTP_TIMEOUT=300 cargo build --release ``` @@ -124,16 +125,17 @@ sudo apt install -y build-essential pkg-config libssl-dev ```bash # Ensure you're using sudo -sudo cp ~/dstack/target/release/dstack-vmm /usr/local/bin/ +sudo cp ~/dstack/dstack/target/release/dstack-vmm /usr/local/bin/ # Or install to user directory mkdir -p ~/.local/bin -cp ~/dstack/target/release/dstack-vmm ~/.local/bin/ +cp ~/dstack/dstack/target/release/dstack-vmm ~/.local/bin/ ``` ### Build cache issues ```bash +cd ~/dstack/dstack cargo clean cargo update cargo build --release @@ -346,7 +348,7 @@ Try alternative download methods: ```bash # Using curl instead of wget curl -L -o dstack-${DSTACK_VERSION}.tar.gz \ - https://github.com/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz + https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz ``` ### Image metadata missing diff --git a/docs/tutorials/troubleshooting-first-application.md b/docs/tutorials/troubleshooting-first-application.md index 1a5dd8f6f..ac5c11647 100644 --- a/docs/tutorials/troubleshooting-first-application.md +++ b/docs/tutorials/troubleshooting-first-application.md @@ -83,7 +83,7 @@ curl -s -H "Authorization: Bearer $(cat ~/.dstack/secrets/vmm-auth-token)" \ If `/guest/Info` returns empty or errors, check that the CVM is running: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-gateway-deployment.md b/docs/tutorials/troubleshooting-gateway-deployment.md index 679ecfa51..1ef5846d1 100644 --- a/docs/tutorials/troubleshooting-gateway-deployment.md +++ b/docs/tutorials/troubleshooting-gateway-deployment.md @@ -71,7 +71,7 @@ export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) **Solution:** Redeploy the KMS CVM: ```bash -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) # Get KMS VM ID and remove it @@ -231,7 +231,7 @@ Let's Encrypt production allows **10 duplicate certificates per 3 hours per IP** **How to check if you're rate-limited:** ```bash -VM_ID=$(cd ~/dstack/vmm && ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm --json | jq -r '.[] | select(.name=="dstack-gateway") | .id') +VM_ID=$(cd ~/dstack/dstack/vmm && ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm --json | jq -r '.[] | select(.name=="dstack-gateway") | .id') curl -s -H "Authorization: Bearer $(cat ~/.dstack/secrets/vmm-auth-token)" \ "http://127.0.0.1:9080/logs?id=$VM_ID&follow=false&ansi=false&lines=200" | grep -i "rate\|too many\|acme.*error" ``` @@ -294,7 +294,7 @@ If KMS itself is not running: curl -sk https://localhost:9100/prpc/KMS.GetMeta | jq '{chain_id}' # Verify KMS CVM is running -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-kms-deployment.md b/docs/tutorials/troubleshooting-kms-deployment.md index f60b3bb02..7a4219ea0 100644 --- a/docs/tutorials/troubleshooting-kms-deployment.md +++ b/docs/tutorials/troubleshooting-kms-deployment.md @@ -230,7 +230,7 @@ Check CVM network configuration: cat ~/kms-deployment/docker-compose.yml | grep ports -A2 # Check CVM status via vmm-cli.py -cd ~/dstack/vmm +cd ~/dstack/dstack/vmm export DSTACK_VMM_AUTH_PASSWORD=$(cat ~/.dstack/secrets/vmm-auth-token) ./src/vmm-cli.py --url http://127.0.0.1:9080 lsvm ``` diff --git a/docs/tutorials/troubleshooting-prerequisites.md b/docs/tutorials/troubleshooting-prerequisites.md index 16e75294c..a4ebf93fd 100644 --- a/docs/tutorials/troubleshooting-prerequisites.md +++ b/docs/tutorials/troubleshooting-prerequisites.md @@ -289,7 +289,7 @@ docker logs aesmd 2>&1 | tail -30 **Solution:** 1. Verify QCNL configuration points to `https://pccs.phala.network/sgx/certification/v4/` 2. Check network connectivity: `curl -sk https://pccs.phala.network/sgx/certification/v4/rootcacrl` -3. Verify the QCNL config file exists at `~/dstack/key-provider-build/sgx_default_qcnl.conf` +3. Verify the QCNL config file exists at `~/dstack/dstack/key-provider-build/sgx_default_qcnl.conf` ### Empty response from curl test diff --git a/docs/verification.md b/docs/verification.md index 17c5170c2..76376951e 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -18,7 +18,7 @@ If any of these fail, the cryptographic proof won't verify. **Programmatic verification**: dstack provides several tools: -- [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/verifier) - HTTP service with `/verify` endpoint, also runs as CLI +- [dstack-verifier](https://github.com/Dstack-TEE/dstack/tree/master/dstack/verifier) - HTTP service with `/verify` endpoint, also runs as CLI - [dcap-qvl](https://github.com/Phala-Network/dcap-qvl) - Open source quote verification library (Rust, Python, JS/WASM, CLI) - [SDKs](../sdk/) - JavaScript and Python SDKs include `replayRtmrs()` for local RTMR verification diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 08d05b1cf..ba6aff882 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -124,7 +124,7 @@ When a KMS node receives a key provisioning request, it: #### Vanilla TDX Quote attestation -See [Attestation](../attestation.md) for more details. +See [Attestation](../../attestation.md) for more details. #### Validating Apps via the KMS Auth Chain diff --git a/dstack/kms/auth-simple/README.md b/dstack/kms/auth-simple/README.md index 58360895f..6458d5a50 100644 --- a/dstack/kms/auth-simple/README.md +++ b/dstack/kms/auth-simple/README.md @@ -212,4 +212,4 @@ bun run test:run - [auth-eth](../auth-eth/) - On-chain governance auth server - [auth-mock](../auth-mock/) - Development/testing auth server (always allows) -- [Deployment Guide](../../docs/deployment.md) - Full deployment instructions +- [Deployment Guide](../../../docs/deployment.md) - Full deployment instructions diff --git a/dstack/sodiumbox/README.md b/dstack/sodiumbox/README.md index 9694a7f7e..c3de9904e 100644 --- a/dstack/sodiumbox/README.md +++ b/dstack/sodiumbox/README.md @@ -47,7 +47,7 @@ match result { This crate is licensed under either of: -- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) -- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE](../../LICENSES/Apache-2.0.txt) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](../../LICENSES/MIT.txt) or http://opensource.org/licenses/MIT) at your option. diff --git a/dstack/tests/docs/kms-self-authorization.md b/dstack/tests/docs/kms-self-authorization.md index 7df4ad18f..fbde050e9 100644 --- a/dstack/tests/docs/kms-self-authorization.md +++ b/dstack/tests/docs/kms-self-authorization.md @@ -162,7 +162,7 @@ and make sure you still have a way to update allow/deny policy during the test. ### 5.3 If using host-local `auth-simple`, install and start two instances ```bash -cd "$REPO_ROOT/kms/auth-simple" +cd "$REPO_ROOT/dstack/kms/auth-simple" bun install ``` @@ -201,7 +201,7 @@ These placeholder configs intentionally deny all KMS boots until you populate `k Start the services: ```bash -cd "$REPO_ROOT/kms/auth-simple" +cd "$REPO_ROOT/dstack/kms/auth-simple" AUTH_CONFIG_PATH=/tmp/kms-self-auth/auth-src.json PORT=3101 bun run start \ >/tmp/kms-self-auth/auth-src.log 2>&1 & echo $! >/tmp/kms-self-auth/auth-src.pid diff --git a/sdk/README.md b/sdk/README.md index 6d1b72a0b..17aa1b4fa 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -20,5 +20,5 @@ All SDKs communicate with the guest agent via HTTP over a Unix socket (`/var/run For local development without TDX hardware, use the simulator: - [Download releases](https://github.com/Dstack-TEE/dstack/releases?q=simulator-v&expanded=true) -- [Install as a systemd service](../guest-agent-simulator/install-systemd.sh) +- [Install as a systemd service](../dstack/guest-agent-simulator/install-systemd.sh) - [Docker image](https://hub.docker.com/r/phalanetwork/dstack-simulator) diff --git a/sdk/curl/api.md b/sdk/curl/api.md index bb2912d6c..4893f55bb 100644 --- a/sdk/curl/api.md +++ b/sdk/curl/api.md @@ -144,7 +144,7 @@ curl --unix-socket /var/run/dstack.sock http://dstack/GetQuote?report_data=00000 ``` **Note on Event Log:** -The `event_log` field contains a JSON array of TDX event log entries. For RTMR 0-2 (boot-time measurements), only the digest is included; the payload is stripped to reduce response size. For RTMR3 (runtime measurements), both digest and payload are included. To verify the event log, submit it along with the quote to the [verifier service](../../verifier/README.md). +The `event_log` field contains a JSON array of TDX event log entries. For RTMR 0-2 (boot-time measurements), only the digest is included; the payload is stripped to reduce response size. For RTMR3 (runtime measurements), both digest and payload are included. To verify the event log, submit it along with the quote to the [verifier service](../../dstack/verifier/README.md). ### 4. Get Info From a91bd7e212faaf1704c2b9ed38b0457cb56d3be2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:41:57 -0700 Subject: [PATCH 423/435] build: fix verifier paths after relocation --- dstack/verifier/builder/Dockerfile | 8 ++++---- dstack/verifier/builder/build-image.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dstack/verifier/builder/Dockerfile b/dstack/verifier/builder/Dockerfile index 06070f9db..f8ad5274a 100644 --- a/dstack/verifier/builder/Dockerfile +++ b/dstack/verifier/builder/Dockerfile @@ -23,11 +23,11 @@ RUN apt-get update && \ ca-certificates \ curl && \ rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache -RUN git clone ${DSTACK_SRC_URL} dstack && \ - cd dstack && \ +RUN git clone ${DSTACK_SRC_URL} repo && \ + cd repo && \ git checkout ${DSTACK_REV} RUN rustup target add x86_64-unknown-linux-musl -RUN cd dstack && cargo build --release -p dstack-verifier --target x86_64-unknown-linux-musl +RUN cd repo/dstack && cargo build --release -p dstack-verifier --target x86_64-unknown-linux-musl RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe AS acpi-builder @@ -77,7 +77,7 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ libglib2.0-0 \ libslirp0 \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache -COPY --from=verifier-builder /build/dstack/target/x86_64-unknown-linux-musl/release/dstack-verifier /usr/local/bin/dstack-verifier +COPY --from=verifier-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-verifier /usr/local/bin/dstack-verifier COPY --from=verifier-builder /build/.GIT_REV /etc/ COPY --from=acpi-builder /usr/local/bin/dstack-acpi-tables /usr/local/bin/dstack-acpi-tables COPY --from=acpi-builder /usr/local/share/qemu /usr/local/share/qemu diff --git a/dstack/verifier/builder/build-image.sh b/dstack/verifier/builder/build-image.sh index 002a26e17..df3bfddbd 100755 --- a/dstack/verifier/builder/build-image.sh +++ b/dstack/verifier/builder/build-image.sh @@ -12,7 +12,7 @@ CONTEXT_DIR=$(dirname "$SCRIPT_DIR") SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" -source "$REPO_ROOT/build/shared/build-lib.sh" +source "$REPO_ROOT/dstack/build/shared/build-lib.sh" NAME=${1:-} if [ -z "$NAME" ]; then From d26ab350087d4095e52655987fdeb43059a90271 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:42:08 -0700 Subject: [PATCH 424/435] chore: preserve merged repository license scopes --- LICENSES/BUSL-1.1.txt | 71 +++++++++++++++++++++++ REUSE.toml | 131 ++++++++++++++++++++++++++---------------- 2 files changed, 151 insertions(+), 51 deletions(-) create mode 100644 LICENSES/BUSL-1.1.txt diff --git a/LICENSES/BUSL-1.1.txt b/LICENSES/BUSL-1.1.txt new file mode 100644 index 000000000..2ef98f1be --- /dev/null +++ b/LICENSES/BUSL-1.1.txt @@ -0,0 +1,71 @@ +Business Source License 1.1 + +License text copyright © 2017 MariaDB Corporation Ab, All Rights Reserved. +"Business Source License" is a trademark of MariaDB Corporation Ab. + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/REUSE.toml b/REUSE.toml index 20dc81df6..1433a29cb 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -6,7 +6,19 @@ SPDX-PackageDownloadLocation = "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/Dstack-TEE/dstack" # Non source files [[annotations]] -path = "**/*.md" +path = [ + "*.md", + ".agent/**/*.md", + ".claude/**/*.md", + "docs/**/*.md", + "dstack/**/*.md", + "examples/**/*.md", + "sdk/**/*.md", + "tools/**/*.md", + "os/README.md", + "os/common/**/*.md", + "os/image/**/*.md", +] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" @@ -22,33 +34,32 @@ path = [ "**/tsconfig.json", "**/tsconfig.node.json", "**/tsconfig.browser.json", - "kms/auth-eth-bun/.oxlintrc.json", - "kms/auth-eth/slither.config.json", - "package-lock.json", + "dstack/kms/auth-eth-bun/.oxlintrc.json", + "dstack/kms/auth-eth/slither.config.json", "**/package-lock.json", - "kms/auth-eth/.openzeppelin/unknown-2035.json", - "kms/auth-mock/.oxlintrc.json", - "kms/auth-simple/.oxlintrc.json", - "kms/auth-simple/auth-config.example.json", + "dstack/kms/auth-eth/.openzeppelin/unknown-2035.json", + "dstack/kms/auth-mock/.oxlintrc.json", + "dstack/kms/auth-simple/.oxlintrc.json", + "dstack/kms/auth-simple/auth-config.example.json", "tools/sca/examples/heartbeat/config.json", "tools/sca/examples/hello-c/config.json", "tools/sca/examples/heartbeat/rootfs/etc/heartbeat/interval", "sdk/simulator/*.json", "sdk/go/go.sum", "sdk/go/ratls/go.sum", - "kms/dstack-app/builder/shared/builder-pinned-packages.txt", - "kms/dstack-app/builder/shared/qemu-pinned-packages.txt", - "gateway/dstack-app/builder/shared/builder-pinned-packages.txt", - "gateway/dstack-app/builder/shared/pinned-packages.txt", + "dstack/kms/dstack-app/builder/shared/builder-pinned-packages.txt", + "dstack/kms/dstack-app/builder/shared/qemu-pinned-packages.txt", + "dstack/gateway/dstack-app/builder/shared/builder-pinned-packages.txt", + "dstack/gateway/dstack-app/builder/shared/pinned-packages.txt", ] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" [[annotations]] path = [ - "gateway/templates/wg.conf", - "guest-agent/templates/metrics.tpl", - "key-provider-build/sgx_default_qcnl.conf", + "dstack/gateway/templates/wg.conf", + "dstack/guest-agent/templates/metrics.tpl", + "dstack/key-provider-build/sgx_default_qcnl.conf", ] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" @@ -57,14 +68,22 @@ SPDX-License-Identifier = "Apache-2.0" path = [ ".cursorrules", ".mailmap", - "**/.gitignore", - "**/.npmignore", + ".gitignore", + "dstack/**/.gitignore", + "dstack/**/.npmignore", + "docs/**/.gitignore", + "examples/**/.gitignore", + "os/common/**/.gitignore", + "os/image/**/.gitignore", + "sdk/**/.gitignore", + "sdk/**/.npmignore", + "tools/**/.gitignore", ] SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" [[annotations]] -path = "basefiles/*" +path = "os/common/rootfs/**" SPDX-FileCopyrightText = "Copyright (c) 2024-2025 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" @@ -74,24 +93,39 @@ path = [ "dstack_Technical_Charter_Final_10-17-2025.pdf", "sdk/simulator/quote.hex", "sdk/simulator/attestation.bin", - "ra-tls/assets/tdx_quote", - "cc-eventlog/samples/ccel.bin", - "cc-eventlog/samples/tpm_eventlog.bin", - "tpm-attest/tests/tpm_quote_sample.bin", - "tpm-qvl/certs/gcp-root-ca.pem", - "dstack-attest/tests/nitro_attestation.bin", - "dstack-attest/tests/nitro_attestation_dbg.bin", - "dstack-attest/tests/sev_snp_attestation.bin", - "dstack-attest/tests/sev_snp_ask.pem", - "dstack-attest/tests/sev_snp_vcek.pem", - "nsm-attest/tests/nitro_attestation.bin", - "nsm-qvl/tests/nitro_attestation.bin", - "nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem", - "tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem", + "dstack/ra-tls/assets/tdx_quote", + "dstack/cc-eventlog/samples/ccel.bin", + "dstack/cc-eventlog/samples/tpm_eventlog.bin", + "dstack/tpm-attest/tests/tpm_quote_sample.bin", + "dstack/tpm-qvl/certs/gcp-root-ca.pem", + "dstack/dstack-attest/tests/nitro_attestation.bin", + "dstack/dstack-attest/tests/nitro_attestation_dbg.bin", + "dstack/dstack-attest/tests/sev_snp_attestation.bin", + "dstack/dstack-attest/tests/sev_snp_ask.pem", + "dstack/dstack-attest/tests/sev_snp_vcek.pem", + "dstack/nsm-attest/tests/nitro_attestation.bin", + "dstack/nsm-qvl/tests/nitro_attestation.bin", + "dstack/nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem", + "dstack/tpm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem", ] SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" +# Backend-neutral OS contract + +[[annotations]] +path = "os/spec/*.json" +SPDX-FileCopyrightText = "Copyright (c) 2026 The Project Contributors" +SPDX-License-Identifier = "Apache-2.0" + +# Imported OS backend + +[[annotations]] +path = "os/yocto/**" +SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" +SPDX-License-Identifier = "BUSL-1.1" +precedence = "override" + # Artworks @@ -107,7 +141,7 @@ SPDX-License-Identifier = "Apache-2.0" # Scripts with SPDX-like content (false positive prevention) [[annotations]] -path = "scripts/add-spdx-attribution.py" +path = "dstack/scripts/add-spdx-attribution.py" SPDX-FileCopyrightText = "© 2025 Phala Network " SPDX-License-Identifier = "Apache-2.0" precedence = "override" @@ -115,19 +149,19 @@ precedence = "override" # Vendor code [[annotations]] -path = "kms/auth-eth/lib/openzeppelin-contracts-upgradeable/**" +path = "dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable/**" SPDX-FileCopyrightText = "Copyright (c) 2016-2025 Zeppelin Group Ltd" SPDX-License-Identifier = "MIT" precedence = "override" [[annotations]] -path = "kms/auth-eth/lib/openzeppelin-foundry-upgrades/**" +path = "dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades/**" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "MIT" precedence = "override" [[annotations]] -path = "kms/auth-eth/lib/forge-std/**" +path = "dstack/kms/auth-eth/lib/forge-std/**" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "Apache-2.0" precedence = "override" @@ -135,7 +169,7 @@ precedence = "override" # Generated files [[annotations]] -path = "kms/auth-eth/typechain-types/**" +path = "dstack/kms/auth-eth/typechain-types/**" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" @@ -149,11 +183,6 @@ path = "**/*.lockb" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" -[[annotations]] -path = "**/*.lockb" -SPDX-FileCopyrightText = "NONE" -SPDX-License-Identifier = "CC0-1.0" - [[annotations]] path = "**/*.snap" SPDX-FileCopyrightText = "NONE" @@ -170,40 +199,40 @@ SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "gateway/assets/*" +path = "dstack/gateway/assets/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "guest-api/src/generated/*" +path = "dstack/guest-api/src/generated/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "dstack-util/tests/fixtures/*" +path = "dstack/dstack-util/tests/fixtures/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "verifier/fixtures/*" +path = "dstack/verifier/fixtures/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "verifier/builder/shared/*.txt" +path = "dstack/verifier/builder/shared/*.txt" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] -path = "guest-agent/fixtures/*" +path = "dstack/guest-agent/fixtures/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" [[annotations]] path = [ - "gateway/test-run/e2e/certs/*", - "gateway/test-run/e2e/configs/*", - "gateway/test-run/e2e/pebble-config.json", + "dstack/gateway/test-run/e2e/certs/*", + "dstack/gateway/test-run/e2e/configs/*", + "dstack/gateway/test-run/e2e/pebble-config.json", ] SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" From 3db2890617b22eb5996417b4011dc4c523a1e82f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 01:42:17 -0700 Subject: [PATCH 425/435] chore: align checks with relocated files --- .cursorrules | 2 +- .gitignore | 1 + .../fixtures/luks_header_cipher_null.license | 2 +- .../tests/fixtures/luks_header_good.license | 2 +- dstack/gateway/templates/dashboard.html | 2 +- dstack/gateway/templates/rproxy.yaml | 2 +- dstack/gateway/templates/wg.conf | 2 +- dstack/guest-agent/templates/dashboard.html | 2 +- dstack/key-provider-build/Dockerfile.aesmd | 2 +- .../entrypoint-key-provider.sh | 2 +- dstack/key-provider-build/run.sh | 4 +- .../key-provider-build/sgx_default_qcnl.conf | 2 +- dstack/kms/auth-eth-bun/.oxlintrc.json | 2 +- dstack/kms/auth-eth-bun/README.md | 4 +- dstack/kms/auth-eth-bun/vitest.config.ts | 2 +- dstack/kms/auth-mock/.oxlintrc.json | 2 +- dstack/kms/auth-mock/README.md | 4 +- dstack/kms/auth-mock/index.test.ts | 24 ++-- dstack/kms/auth-mock/openapi.json | 10 +- dstack/kms/auth-mock/vitest.config.ts | 2 +- dstack/supervisor/tests/test-cli.sh | 2 +- dstack/supervisor/tests/test.sh | 6 +- dstack/test-scripts/inspect-cert.sh | 2 +- dstack/verifier/dstack-verifier.toml | 2 +- dstack/vmm/rpc/proto/prpc.proto | 1 - .../ui/src/components/PortMappingEditor.ts | 2 +- prek.toml | 23 ++-- sdk/js/src/__tests__/get-compose-hash.test.ts | 114 +++++++++--------- sdk/js/src/__tests__/index.test.ts | 34 +++--- sdk/js/src/__tests__/send-rpc-request.test.ts | 36 +++--- sdk/js/src/send-rpc-request.ts | 8 +- sdk/js/test-outputs.js | 20 +-- sdk/python/Makefile | 2 +- sdk/rust/.gitignore | 2 +- sdk/simulator/app-compose.json | 2 +- sdk/simulator/appkeys.json | 2 +- sdk/simulator/sys-config.json | 2 +- 37 files changed, 169 insertions(+), 166 deletions(-) diff --git a/.cursorrules b/.cursorrules index be2db8368..b03e196fc 100644 --- a/.cursorrules +++ b/.cursorrules @@ -1 +1 @@ -Don't capitalize the first letter for log messages and error messages. \ No newline at end of file +Don't capitalize the first letter for log messages and error messages. diff --git a/.gitignore b/.gitignore index 38d14fcaf..bcb4cb1aa 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ node_modules/ /tmp .claude/settings.local.json __pycache__ +/.ruff_cache/ .planning/ /dstack/vmm/src/console_v1.html .claude/worktrees/ diff --git a/dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license b/dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license index 8f861f09b..84ac4efa8 100644 --- a/dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license +++ b/dstack/dstack-util/tests/fixtures/luks_header_cipher_null.license @@ -1,3 +1,3 @@ SPDX-FileCopyrightText: © 2025 Phala Network -SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/dstack-util/tests/fixtures/luks_header_good.license b/dstack/dstack-util/tests/fixtures/luks_header_good.license index 8f861f09b..84ac4efa8 100644 --- a/dstack/dstack-util/tests/fixtures/luks_header_good.license +++ b/dstack/dstack-util/tests/fixtures/luks_header_good.license @@ -1,3 +1,3 @@ SPDX-FileCopyrightText: © 2025 Phala Network -SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/templates/dashboard.html b/dstack/gateway/templates/dashboard.html index 72c600bff..55a296ff8 100644 --- a/dstack/gateway/templates/dashboard.html +++ b/dstack/gateway/templates/dashboard.html @@ -1545,4 +1545,4 @@

Add ZT-Domain

- \ No newline at end of file + diff --git a/dstack/gateway/templates/rproxy.yaml b/dstack/gateway/templates/rproxy.yaml index 69d03792b..e5513b0ae 100644 --- a/dstack/gateway/templates/rproxy.yaml +++ b/dstack/gateway/templates/rproxy.yaml @@ -22,4 +22,4 @@ servers: target: {{ peer.ip }}:{{ p.target_port }} {% endfor %} {%- endif %} - {%- endfor %} \ No newline at end of file + {%- endfor %} diff --git a/dstack/gateway/templates/wg.conf b/dstack/gateway/templates/wg.conf index 3a5e4acaa..eb1d39e6a 100644 --- a/dstack/gateway/templates/wg.conf +++ b/dstack/gateway/templates/wg.conf @@ -7,4 +7,4 @@ ListenPort = {{ listen_port }} PublicKey = {{ peer.public_key }} AllowedIPs = {{ peer.ip }}/32 PersistentKeepalive = 25 -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/dstack/guest-agent/templates/dashboard.html b/dstack/guest-agent/templates/dashboard.html index 212df06ac..4cfd5494d 100644 --- a/dstack/guest-agent/templates/dashboard.html +++ b/dstack/guest-agent/templates/dashboard.html @@ -251,4 +251,4 @@

TCB Info

- \ No newline at end of file + diff --git a/dstack/key-provider-build/Dockerfile.aesmd b/dstack/key-provider-build/Dockerfile.aesmd index c5d032507..eb06e5123 100644 --- a/dstack/key-provider-build/Dockerfile.aesmd +++ b/dstack/key-provider-build/Dockerfile.aesmd @@ -37,4 +37,4 @@ RUN apt-get update && apt-get install -y \ COPY entrypoint-aesmd.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dstack/key-provider-build/entrypoint-key-provider.sh b/dstack/key-provider-build/entrypoint-key-provider.sh index 893e4686b..d96291642 100755 --- a/dstack/key-provider-build/entrypoint-key-provider.sh +++ b/dstack/key-provider-build/entrypoint-key-provider.sh @@ -24,4 +24,4 @@ echo "Enclave info:" gramine-sgx-sigstruct-view --output-format json gramine-sealing-key-provider.sig echo "Starting Gramine Sealing Key Provider" -make SGX=1 run-provider \ No newline at end of file +make SGX=1 run-provider diff --git a/dstack/key-provider-build/run.sh b/dstack/key-provider-build/run.sh index 42624b8e8..f1b37d167 100755 --- a/dstack/key-provider-build/run.sh +++ b/dstack/key-provider-build/run.sh @@ -13,8 +13,8 @@ echo "==========================" echo "Key provider endpoint: https://localhost:3443" echo " - Using shared socket with AESM service" echo " - Socket location: /var/run/aesmd/aesm.socket" -echo +echo echo "Check logs with:" echo " docker compose logs -f aesmd" echo " docker compose logs -f gramine-sealing-key-provider" -echo "==========================" \ No newline at end of file +echo "==========================" diff --git a/dstack/key-provider-build/sgx_default_qcnl.conf b/dstack/key-provider-build/sgx_default_qcnl.conf index 38b673839..0f184cdc0 100644 --- a/dstack/key-provider-build/sgx_default_qcnl.conf +++ b/dstack/key-provider-build/sgx_default_qcnl.conf @@ -6,4 +6,4 @@ "pck_cache_expire_hours": 168, "verify_collateral_cache_expire_hours": 168, "local_cache_only": false -} \ No newline at end of file +} diff --git a/dstack/kms/auth-eth-bun/.oxlintrc.json b/dstack/kms/auth-eth-bun/.oxlintrc.json index 1b685f410..f7d7b8e35 100644 --- a/dstack/kms/auth-eth-bun/.oxlintrc.json +++ b/dstack/kms/auth-eth-bun/.oxlintrc.json @@ -24,4 +24,4 @@ "*.d.ts", "*.test.ts" ] -} \ No newline at end of file +} diff --git a/dstack/kms/auth-eth-bun/README.md b/dstack/kms/auth-eth-bun/README.md index 02806fe27..9f1725319 100644 --- a/dstack/kms/auth-eth-bun/README.md +++ b/dstack/kms/auth-eth-bun/README.md @@ -82,7 +82,7 @@ KMS boot authentication "tcbStatus": "string", "advisoryIds": ["string"], "mrAggregated": "string", - "mrSystem": "string", + "mrSystem": "string", "osImageHash": "string", "appId": "string", "composeHash": "string", @@ -124,4 +124,4 @@ the test suite (`index.test.ts`) validates: - ✅ schema validation using OpenAPI spec - ✅ error handling behavior - ✅ hex encoding/decoding compatibility -- ✅ optional field handling \ No newline at end of file +- ✅ optional field handling diff --git a/dstack/kms/auth-eth-bun/vitest.config.ts b/dstack/kms/auth-eth-bun/vitest.config.ts index 990992495..17597daca 100644 --- a/dstack/kms/auth-eth-bun/vitest.config.ts +++ b/dstack/kms/auth-eth-bun/vitest.config.ts @@ -9,4 +9,4 @@ export default defineConfig({ globals: true, environment: 'node', }, -}); \ No newline at end of file +}); diff --git a/dstack/kms/auth-mock/.oxlintrc.json b/dstack/kms/auth-mock/.oxlintrc.json index 1b685f410..f7d7b8e35 100644 --- a/dstack/kms/auth-mock/.oxlintrc.json +++ b/dstack/kms/auth-mock/.oxlintrc.json @@ -24,4 +24,4 @@ "*.d.ts", "*.test.ts" ] -} \ No newline at end of file +} diff --git a/dstack/kms/auth-mock/README.md b/dstack/kms/auth-mock/README.md index 1ff28747d..c26ed0e19 100644 --- a/dstack/kms/auth-mock/README.md +++ b/dstack/kms/auth-mock/README.md @@ -93,7 +93,7 @@ identical to the real backend: ```json { "mrAggregated": "string", - "osImageHash": "string", + "osImageHash": "string", "appId": "string", "composeHash": "string", "instanceId": "string", @@ -165,4 +165,4 @@ to switch from mock to real backend: 3. deploy smart contracts 4. update environment variables -the API remains identical, so no client code changes are needed. \ No newline at end of file +the API remains identical, so no client code changes are needed. diff --git a/dstack/kms/auth-mock/index.test.ts b/dstack/kms/auth-mock/index.test.ts index 177fa1b44..98521ec61 100644 --- a/dstack/kms/auth-mock/index.test.ts +++ b/dstack/kms/auth-mock/index.test.ts @@ -12,7 +12,7 @@ beforeAll(async () => { // Set environment variables for testing process.env.KMS_CONTRACT_ADDR = '0xmockcontract1234567890123456789012345678'; process.env.PORT = '3002'; - + // Import the app const indexModule = await import('./index.ts'); appFetch = indexModule.default.fetch; @@ -44,11 +44,11 @@ describe('Mock Backend Tests', () => { expect(data.gatewayAppId).toBe(process.env.MOCK_GATEWAY_APP_ID || '0xmockgateway1234567890123456789012345678'); expect(data.chainId).toBe(parseInt(process.env.MOCK_CHAIN_ID || '1337')); expect(data.appImplementation).toBe(process.env.MOCK_APP_IMPLEMENTATION || '0xmockapp9876543210987654321098765432109'); - + // Verify response structure matches OpenAPI spec const systemInfoSchema = openApiSpec.components.schemas.MockSystemInfo; const requiredFields = systemInfoSchema.required; - + requiredFields.forEach(field => { expect(data).toHaveProperty(field); }); @@ -67,7 +67,7 @@ describe('Mock Backend Tests', () => { it('should always return success for app auth', async () => { const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); - + const response = await appFetch(new Request('http://localhost:3002/bootAuth/app', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -93,7 +93,7 @@ describe('Mock Backend Tests', () => { // Verify response matches OpenAPI spec const bootResponseSchema = openApiSpec.components.schemas.BootResponse; const requiredFields = bootResponseSchema.required; - + requiredFields.forEach(field => { expect(data).toHaveProperty(field); }); @@ -172,7 +172,7 @@ describe('Mock Backend Tests', () => { it('should always return success for KMS auth', async () => { const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); - + const response = await appFetch(new Request('http://localhost:3002/bootAuth/kms', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -225,7 +225,7 @@ describe('Mock Backend Tests', () => { it('should handle "Test backend error" appropriately', async () => { const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - + // This shouldn't actually throw an error in mock backend, but test the error handling path const response = await appFetch(new Request('http://localhost:3002/bootAuth/kms', { method: 'POST', @@ -240,7 +240,7 @@ describe('Mock Backend Tests', () => { // Verify console.error was not called (no errors in mock) expect(consoleSpy).not.toHaveBeenCalled(); - + consoleSpy.mockRestore(); }); }); @@ -249,7 +249,7 @@ describe('Mock Backend Tests', () => { describe('API Schema Compatibility', () => { it('should match BootInfo schema requirements', () => { const bootInfoSchema = openApiSpec.components.schemas.BootInfo; - + // Required fields should match original schema expect(bootInfoSchema.required).toEqual([ 'mrAggregated', @@ -268,7 +268,7 @@ describe('API Schema Compatibility', () => { it('should match BootResponse schema requirements', () => { const bootResponseSchema = openApiSpec.components.schemas.BootResponse; - + expect(bootResponseSchema.required).toEqual([ 'isAllowed', 'reason', @@ -282,7 +282,7 @@ describe('API Schema Compatibility', () => { it('should match MockSystemInfo schema requirements', () => { const systemInfoSchema = openApiSpec.components.schemas.MockSystemInfo; - + expect(systemInfoSchema.required).toEqual([ 'status', 'kmsContractAddr', @@ -333,4 +333,4 @@ describe('Mock Behavior Verification', () => { expect(appData.reason).toBe('mock app always allowed'); expect(kmsData.reason).toBe('mock KMS always allowed'); }); -}); \ No newline at end of file +}); diff --git a/dstack/kms/auth-mock/openapi.json b/dstack/kms/auth-mock/openapi.json index 74e34702b..1689308d0 100644 --- a/dstack/kms/auth-mock/openapi.json +++ b/dstack/kms/auth-mock/openapi.json @@ -106,7 +106,7 @@ "type": "object", "required": [ "mrAggregated", - "osImageHash", + "osImageHash", "appId", "composeHash", "instanceId", @@ -118,7 +118,7 @@ "description": "Aggregated MR measurement" }, "osImageHash": { - "type": "string", + "type": "string", "description": "OS Image hash" }, "appId": { @@ -158,7 +158,7 @@ "type": "object", "required": [ "isAllowed", - "reason", + "reason", "gatewayAppId" ], "properties": { @@ -202,7 +202,7 @@ "example": "0xmockcontract1234567890123456789012345678" }, "gatewayAppId": { - "type": "string", + "type": "string", "description": "Mock gateway application ID", "example": "0xmockgateway1234567890123456789012345678" }, @@ -247,4 +247,4 @@ } } } -} \ No newline at end of file +} diff --git a/dstack/kms/auth-mock/vitest.config.ts b/dstack/kms/auth-mock/vitest.config.ts index 990992495..17597daca 100644 --- a/dstack/kms/auth-mock/vitest.config.ts +++ b/dstack/kms/auth-mock/vitest.config.ts @@ -9,4 +9,4 @@ export default defineConfig({ globals: true, environment: 'node', }, -}); \ No newline at end of file +}); diff --git a/dstack/supervisor/tests/test-cli.sh b/dstack/supervisor/tests/test-cli.sh index 2985396aa..6b93a6a06 100755 --- a/dstack/supervisor/tests/test-cli.sh +++ b/dstack/supervisor/tests/test-cli.sh @@ -130,4 +130,4 @@ ${CLI} shutdown 2>/dev/null kill $(cat $PIDFILE) 2>/dev/null rm -f $UDS $PIDFILE -info "CLI testing completed!" +info "CLI testing completed!" diff --git a/dstack/supervisor/tests/test.sh b/dstack/supervisor/tests/test.sh index b1ec27e47..7205a81db 100755 --- a/dstack/supervisor/tests/test.sh +++ b/dstack/supervisor/tests/test.sh @@ -132,7 +132,7 @@ RES=$(http POST "${BASE_URL}/deploy" ' }, "cwd": "/var/tmp", "stdout": "/tmp/stdout.log", - "stderr": "/tmp/stderr.log", + "stderr": "/tmp/stderr.log", "pidfile": "/tmp/test.pid" }') assert_eq "$RES" '{"data":null}' "Deploy should return empty object" @@ -153,10 +153,10 @@ sleep 1 # Check stdout file exists and contains expected output info "Testing stdout file contents" -assert_eq "$(cat "/tmp/stdout.log")" "hello world, FOO=bar, pwd=/var/tmp" +assert_eq "$(cat "/tmp/stdout.log")" "hello world, FOO=bar, pwd=/var/tmp" # Check stderr file exists and contains expected output -info "Testing stderr file contents" +info "Testing stderr file contents" assert_eq "$(cat "/tmp/stderr.log")" "hello stderr" "Stderr file should contain expected output" # Check pid file exists and contains expected pid diff --git a/dstack/test-scripts/inspect-cert.sh b/dstack/test-scripts/inspect-cert.sh index 141ef4e8d..6ecb93389 100755 --- a/dstack/test-scripts/inspect-cert.sh +++ b/dstack/test-scripts/inspect-cert.sh @@ -3,4 +3,4 @@ # SPDX-FileCopyrightText: © 2025 Phala Network # SPDX-License-Identifier: Apache-2.0 -openssl x509 -text -noout -in $1 \ No newline at end of file +openssl x509 -text -noout -in $1 diff --git a/dstack/verifier/dstack-verifier.toml b/dstack/verifier/dstack-verifier.toml index 8c8a9b89e..b098c533a 100644 --- a/dstack/verifier/dstack-verifier.toml +++ b/dstack/verifier/dstack-verifier.toml @@ -16,4 +16,4 @@ image_download_url = "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.t image_download_timeout_secs = 300 # Optional PCCS URL for quote verification -# pccs_url = "https://pccs.phala.network" \ No newline at end of file +# pccs_url = "https://pccs.phala.network" diff --git a/dstack/vmm/rpc/proto/prpc.proto b/dstack/vmm/rpc/proto/prpc.proto index cd5a5d471..ae9e61f9f 100644 --- a/dstack/vmm/rpc/proto/prpc.proto +++ b/dstack/vmm/rpc/proto/prpc.proto @@ -10,4 +10,3 @@ package prpc; message PrpcError { string message = 1; } - diff --git a/dstack/vmm/ui/src/components/PortMappingEditor.ts b/dstack/vmm/ui/src/components/PortMappingEditor.ts index 5fcf730a1..0668bd952 100644 --- a/dstack/vmm/ui/src/components/PortMappingEditor.ts +++ b/dstack/vmm/ui/src/components/PortMappingEditor.ts @@ -53,7 +53,7 @@ const PortMappingEditorComponent = { - { docker_compose_file: "docker-compose.yml", bash_script: "start.sh" } - + const compose2: AppCompose = { bash_script: "start.sh", docker_compose_file: "docker-compose.yml", runner: "docker-compose" } - + // Both should produce the same hash despite different key order expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) @@ -33,7 +33,7 @@ describe('Deterministic JSON Serialization', () => { beta: 42 } } as AppCompose - + const compose2: AppCompose = { nested_config: { alpha: "first", @@ -42,7 +42,7 @@ describe('Deterministic JSON Serialization', () => { }, runner: "docker-compose" } as AppCompose - + expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) @@ -66,7 +66,7 @@ describe('Deterministic JSON Serialization', () => { } } } as AppCompose - + const compose2: AppCompose = { config: { cache: { @@ -86,7 +86,7 @@ describe('Deterministic JSON Serialization', () => { }, runner: "docker-compose" } as AppCompose - + expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) }) @@ -97,12 +97,12 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", items: [3, 1, 2] } as AppCompose - + const compose2: AppCompose = { runner: "docker-compose", items: [1, 2, 3] } as AppCompose - + // Different array orders should produce different hashes expect(getComposeHash(compose1)).not.toBe(getComposeHash(compose2)) }) @@ -115,7 +115,7 @@ describe('Deterministic JSON Serialization', () => { { name: "db", port: 5432 } ] } as AppCompose - + const compose2: AppCompose = { runner: "docker-compose", services: [ @@ -123,7 +123,7 @@ describe('Deterministic JSON Serialization', () => { { port: 5432, name: "db" } ] } as AppCompose - + // Object keys should be sorted within arrays expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) @@ -135,7 +135,7 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", special_value: NaN } as AppCompose - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) // SHA256 hex length @@ -146,22 +146,22 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", special_value: Infinity } as AppCompose - + const compose2: AppCompose = { runner: "docker-compose", special_value: -Infinity } as AppCompose - + const compose3: AppCompose = { runner: "docker-compose", special_value: null } - + // All should produce the same hash since NaN and Infinity become null const hash1 = getComposeHash(compose1) const hash2 = getComposeHash(compose2) const hash3 = getComposeHash(compose3) - + expect(hash1).toBe(hash2) expect(hash2).toBe(hash3) }) @@ -171,11 +171,11 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", optional_field: undefined } as AppCompose - + const compose2: AppCompose = { runner: "docker-compose" } - + // undefined values should be treated consistently expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) @@ -188,15 +188,15 @@ describe('Deterministic JSON Serialization', () => { bash_script: "start.sh", docker_compose_file: "docker-compose.yml" } - + const hash = getComposeHash(compose, true) - + // Should be the same as compose without docker_compose_file const compose2: AppCompose = { runner: "bash", bash_script: "start.sh" } - + expect(hash).toBe(getComposeHash(compose2, true)) }) @@ -206,15 +206,15 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", bash_script: "start.sh" } - + const hash = getComposeHash(compose, true) - + // Should be the same as compose without bash_script const compose2: AppCompose = { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + expect(hash).toBe(getComposeHash(compose2, true)) }) @@ -224,12 +224,12 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", pre_launch_script: "" } - + const compose2: AppCompose = { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + expect(getComposeHash(compose1, true)).toBe(getComposeHash(compose2, true)) }) @@ -239,12 +239,12 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", pre_launch_script: "echo 'Starting...'" } - + const compose2: AppCompose = { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + expect(getComposeHash(compose1, true)).not.toBe(getComposeHash(compose2, true)) }) }) @@ -256,13 +256,13 @@ describe('Deterministic JSON Serialization', () => { text: "你好世界", description: "🚀 Deploy" } as AppCompose - + const compose2: AppCompose = { description: "🚀 Deploy", runner: "docker-compose", text: "你好世界" } as AppCompose - + expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) }) @@ -284,13 +284,13 @@ describe('Deterministic JSON Serialization', () => { alpha: "first" } } as AppCompose - + const hash = getComposeHash(compose) - + // This should be a deterministic hash expect(hash).toHaveLength(64) expect(hash).toMatch(/^[a-f0-9]{64}$/) - + // The exact hash value depends on the specific data structure // but it should be consistent across runs const hash2 = getComposeHash(compose) @@ -303,7 +303,7 @@ describe('Deterministic JSON Serialization', () => { // @ts-expect-error - empty object is valid const compose: AppCompose = {} const hash = getComposeHash(compose) - + expect(hash).toHaveLength(64) expect(hash).toMatch(/^[a-f0-9]{64}$/) }) @@ -313,7 +313,7 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", optional_field: null } as AppCompose - + const hash = getComposeHash(compose) expect(hash).toBeDefined() }) @@ -324,13 +324,13 @@ describe('Deterministic JSON Serialization', () => { enabled: true, debug: false } as AppCompose - + const compose2: AppCompose = { debug: false, enabled: true, runner: "docker-compose" } as AppCompose - + expect(getComposeHash(compose1)).toBe(getComposeHash(compose2)) }) @@ -342,7 +342,7 @@ describe('Deterministic JSON Serialization', () => { float: 3.14159, large: 1e10 } as AppCompose - + const hash = getComposeHash(compose) expect(hash).toBeDefined() }) @@ -359,9 +359,9 @@ describe('Deterministic JSON Serialization', () => { DATABASE_URL: "postgres://localhost:5432/mydb" } } as AppCompose - + const hashes = Array.from({ length: 10 }, () => getComposeHash(compose)) - + // All hashes should be identical const firstHash = hashes[0] expect(hashes.every(hash => hash === firstHash)).toBe(true) @@ -372,12 +372,12 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + const compose2: AppCompose = { runner: "bash", bash_script: "start.sh" } - + expect(getComposeHash(compose1)).not.toBe(getComposeHash(compose2)) }) }) @@ -390,7 +390,7 @@ describe('Deterministic JSON Serialization', () => { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -408,7 +408,7 @@ describe('Deterministic JSON Serialization', () => { token_key: "token123" } } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -429,7 +429,7 @@ describe('Deterministic JSON Serialization', () => { no_instance_id: false, secure_time: true } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -444,7 +444,7 @@ describe('Deterministic JSON Serialization', () => { key_provider: "kms", key_provider_id: "abcd1234" } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -458,7 +458,7 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", allowed_envs: ["NODE_ENV", "PORT", "DATABASE_URL"] } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -472,7 +472,7 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", features: ["feature1", "feature2"] } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -486,10 +486,10 @@ describe('Deterministic JSON Serialization', () => { bash_script: "start.sh", docker_compose_file: "docker-compose.yml" } - + const hashWithoutNormalize = getComposeHash(compose) const hashExplicitFalse = getComposeHash(compose, false) - + expect(hashWithoutNormalize).toBe(hashExplicitFalse) }) @@ -499,10 +499,10 @@ describe('Deterministic JSON Serialization', () => { bash_script: "start.sh", docker_compose_file: "docker-compose.yml" } - + const hashWithNormalize = getComposeHash(compose, true) const hashWithoutNormalize = getComposeHash(compose, false) - + // These should be different because preprocessing is applied only with normalize=true expect(hashWithNormalize).not.toBe(hashWithoutNormalize) }) @@ -513,15 +513,15 @@ describe('Deterministic JSON Serialization', () => { docker_compose_file: "docker-compose.yml", pre_launch_script: "" } - + const composeWithoutEmpty: AppCompose = { runner: "docker-compose", docker_compose_file: "docker-compose.yml" } - + // With normalization, empty pre_launch_script should be removed expect(getComposeHash(compose, true)).toBe(getComposeHash(composeWithoutEmpty, true)) - + // Without normalization, empty pre_launch_script should remain expect(getComposeHash(compose, false)).not.toBe(getComposeHash(composeWithoutEmpty, false)) }) @@ -534,7 +534,7 @@ describe('Deterministic JSON Serialization', () => { bash_script: "start.sh", pre_launch_script: "echo 'Starting...'" } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) @@ -553,10 +553,10 @@ describe('Deterministic JSON Serialization', () => { allowed_envs: ["NODE_ENV"], secure_time: false } - + const hash = getComposeHash(compose) expect(hash).toBeDefined() expect(hash).toHaveLength(64) }) }) -}) \ No newline at end of file +}) diff --git a/sdk/js/src/__tests__/index.test.ts b/sdk/js/src/__tests__/index.test.ts index 66b6aeb9e..c1e4564c8 100644 --- a/sdk/js/src/__tests__/index.test.ts +++ b/sdk/js/src/__tests__/index.test.ts @@ -151,9 +151,9 @@ describe('DstackClient', () => { // Temporarily remove environment variable to test file check const savedEnv = process.env.DSTACK_SIMULATOR_ENDPOINT delete process.env.DSTACK_SIMULATOR_ENDPOINT - + expect(() => new DstackClient('/non/existent/socket')).toThrow('Unix socket file /non/existent/socket does not exist') - + // Restore environment variable if (savedEnv) { process.env.DSTACK_SIMULATOR_ENDPOINT = savedEnv @@ -164,10 +164,10 @@ describe('DstackClient', () => { // Temporarily remove environment variable to test non-unix socket paths const savedEnv = process.env.DSTACK_SIMULATOR_ENDPOINT delete process.env.DSTACK_SIMULATOR_ENDPOINT - + expect(() => new DstackClient('http://localhost:8080')).not.toThrow() expect(() => new DstackClient('https://example.com')).not.toThrow() - + // Restore environment variable if (savedEnv) { process.env.DSTACK_SIMULATOR_ENDPOINT = savedEnv @@ -281,36 +281,36 @@ describe('DstackClient', () => { it('should support deprecated deriveKey method with warning', async () => { const client = new TappdClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + const result = await client.deriveKey('/', 'test') expect(result).toHaveProperty('key') expect(result).toHaveProperty('certificate_chain') expect(consoleSpy).toHaveBeenCalledWith('deriveKey is deprecated, please use getKey instead') - + consoleSpy.mockRestore() }) it('should support deprecated tdxQuote method with warning', async () => { const client = new TappdClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + const result = await client.tdxQuote('test data') expect(result).toHaveProperty('quote') expect(result).toHaveProperty('event_log') expect(consoleSpy).toHaveBeenCalledWith('tdxQuote is deprecated, please use getQuote instead') - + consoleSpy.mockRestore() }) it('should support tdxQuote with hash algorithm parameter', async () => { const client = new TappdClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + const result = await client.tdxQuote('test data', 'sha256') expect(result).toHaveProperty('quote') expect(result).toHaveProperty('event_log') expect(consoleSpy).toHaveBeenCalledWith('tdxQuote is deprecated, please use getQuote instead') - + consoleSpy.mockRestore() }) }) @@ -319,16 +319,16 @@ describe('DstackClient', () => { it('should throws error in deriveKey method', async () => { const client = new DstackClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + await expect(() => client.deriveKey('/', 'test')).rejects.toThrow('deriveKey is deprecated, please use getKey instead.') - + consoleSpy.mockRestore() }) it('should throws error in tdxQuote method without hash algorithm parameter', async () => { const client = new DstackClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + await expect(() => client.tdxQuote('test data')).rejects.toThrow('tdxQuote only supports raw hash algorithm.') consoleSpy.mockRestore() @@ -337,7 +337,7 @@ describe('DstackClient', () => { it("should throws error in tdxQuote method with hash algorithm parameter other than raw", async () => { const client = new DstackClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + await expect(() => client.tdxQuote('test data', 'sha256')).rejects.toThrow('tdxQuote only supports raw hash algorithm.') consoleSpy.mockRestore() @@ -346,7 +346,7 @@ describe('DstackClient', () => { it('should able to get quote with plain report_data in tdxQuote method with warning', async () => { const client = new DstackClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + const result = await client.tdxQuote('test data', "raw") expect(result).toHaveProperty('quote') expect(result).toHaveProperty('event_log') @@ -358,9 +358,9 @@ describe('DstackClient', () => { it('should throws error in tdxQuote with hash algorithm parameter', async () => { const client = new DstackClient() const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - + await expect(() => client.tdxQuote('test data', 'sha256')).rejects.toThrow('tdxQuote only supports raw hash algorithm.') - + consoleSpy.mockRestore() }) }) diff --git a/sdk/js/src/__tests__/send-rpc-request.test.ts b/sdk/js/src/__tests__/send-rpc-request.test.ts index f4936ca0b..aa5a7de8e 100644 --- a/sdk/js/src/__tests__/send-rpc-request.test.ts +++ b/sdk/js/src/__tests__/send-rpc-request.test.ts @@ -70,14 +70,14 @@ describe('send_rpc_request', () => { mockHttpRequest.mockImplementation((url, options, callback) => { // Call the callback with mock response callback(mockRes) - + // Setup response data handling const dataCallback = mockRes.on.mock.calls.find(call => call[0] === 'data')?.[1] const endCallback = mockRes.on.mock.calls.find(call => call[0] === 'end')?.[1] - + if (dataCallback) dataCallback('{"result": "success"}') if (endCallback) endCallback() - + return mockReq }) @@ -109,13 +109,13 @@ describe('send_rpc_request', () => { // Mock the request flow mockHttpsRequest.mockImplementation((url, options, callback) => { callback(mockRes) - + const dataCallback = mockRes.on.mock.calls.find(call => call[0] === 'data')?.[1] const endCallback = mockRes.on.mock.calls.find(call => call[0] === 'end')?.[1] - + if (dataCallback) dataCallback('{"result": "success"}') if (endCallback) endCallback() - + return mockReq }) @@ -150,13 +150,13 @@ describe('send_rpc_request', () => { mockHttpRequest.mockImplementation((url, options, callback) => { callback(mockRes) - + const dataCallback = mockRes.on.mock.calls.find(call => call[0] === 'data')?.[1] const endCallback = mockRes.on.mock.calls.find(call => call[0] === 'end')?.[1] - + if (dataCallback) dataCallback('invalid json') if (endCallback) endCallback() - + return mockReq }) @@ -246,7 +246,7 @@ describe('send_rpc_request', () => { const path = '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/api/test' const payload = '{"test": "data"}' - // Mock real setTimeout to trigger timeout immediately + // Mock real setTimeout to trigger timeout immediately const originalSetTimeout = global.setTimeout // @ts-ignore global.setTimeout = vi.fn((callback, delay) => { @@ -264,7 +264,7 @@ describe('send_rpc_request', () => { }) await expect(send_rpc_request(endpoint, path, payload, 1)).rejects.toThrow('request timed out') - + global.setTimeout = originalSetTimeout }) }) @@ -277,13 +277,13 @@ describe('send_rpc_request', () => { mockHttpRequest.mockImplementation((url, options, callback) => { callback(mockRes) - + const dataCallback = mockRes.on.mock.calls.find(call => call[0] === 'data')?.[1] const endCallback = mockRes.on.mock.calls.find(call => call[0] === 'end')?.[1] - + if (dataCallback) dataCallback('{"result": "success"}') if (endCallback) endCallback() - + return mockReq }) @@ -300,11 +300,11 @@ describe('send_rpc_request', () => { mockHttpRequest.mockImplementation((url, options, callback) => { callback(mockRes) - + // Setup multiple data and end events const dataCallback = mockRes.on.mock.calls.find(call => call[0] === 'data')?.[1] const endCallback = mockRes.on.mock.calls.find(call => call[0] === 'end')?.[1] - + setTimeout(() => { if (dataCallback) dataCallback('{"result": "success"}') if (endCallback) { @@ -312,7 +312,7 @@ describe('send_rpc_request', () => { endCallback() // Second end - should be ignored } }, 10) - + return mockReq }) @@ -320,4 +320,4 @@ describe('send_rpc_request', () => { expect(result).toEqual({ result: 'success' }) }) }) -}) \ No newline at end of file +}) diff --git a/sdk/js/src/send-rpc-request.ts b/sdk/js/src/send-rpc-request.ts index fa6837a14..e831c418f 100644 --- a/sdk/js/src/send-rpc-request.ts +++ b/sdk/js/src/send-rpc-request.ts @@ -13,21 +13,21 @@ export function send_rpc_request(endpoint: string, path: string, payloa return new Promise((resolve, reject) => { const abortController = new AbortController() let isCompleted = false - + const safeReject = (error: Error) => { if (!isCompleted) { isCompleted = true reject(error) } } - + const safeResolve = (result: T) => { if (!isCompleted) { isCompleted = true resolve(result) } } - + const timeout = setTimeout(() => { abortController.abort() safeReject(new Error('request timed out')) @@ -146,4 +146,4 @@ export function send_rpc_request(endpoint: string, path: string, payloa }) } }) -} \ No newline at end of file +} diff --git a/sdk/js/test-outputs.js b/sdk/js/test-outputs.js index 6c791319f..37aa18238 100644 --- a/sdk/js/test-outputs.js +++ b/sdk/js/test-outputs.js @@ -9,18 +9,18 @@ const { toKeypair, toKeypairSecure } = require('./dist/node/solana.js'); async function main() { console.log("=== JS SDK Output Test ==="); - + try { // Test client get_key const client = new DstackClient(); console.log("\n1. Testing DstackClient.getKey()"); - + const testPaths = [ { path: "test/wallet", purpose: "ethereum" }, { path: "test/signing", purpose: "solana" }, { path: "user/alice", purpose: "mainnet" } ]; - + for (const { path, purpose } of testPaths) { const keyResult = await client.getKey(path, purpose); console.log(`getKey('${path}', '${purpose}'):`); @@ -32,7 +32,7 @@ async function main() { // Test viem integration console.log("\n2. Testing Viem Integration"); const ethKey = await client.getKey("eth/test", "wallet"); - + console.log("\n2.1 toViemAccount (legacy):"); try { const account = toViemAccount(ethKey); @@ -41,7 +41,7 @@ async function main() { } catch (error) { console.log(` error: ${error.message}`); } - + console.log("\n2.2 toViemAccountSecure:"); try { const accountSecure = toViemAccountSecure(ethKey); @@ -54,7 +54,7 @@ async function main() { // Test solana integration console.log("\n3. Testing Solana Integration"); const solKey = await client.getKey("sol/test", "wallet"); - + console.log("\n3.1 toKeypair (legacy):"); try { const keypair = toKeypair(solKey); @@ -64,7 +64,7 @@ async function main() { } catch (error) { console.log(` error: ${error.message}`); } - + console.log("\n3.2 toKeypairSecure:"); try { const keypairSecure = toKeypairSecure(solKey); @@ -112,12 +112,12 @@ async function main() { }, { manifest_version: 1, - name: "another-app", + name: "another-app", runner: "docker-compose", docker_compose_file: "services:\\n web:\\n build: .\\n environment:\\n - NODE_ENV=production" } ]; - + testComposes.forEach((compose, index) => { const hash = getComposeHash(compose); console.log(`compose ${index + 1}: ${hash}`); @@ -155,4 +155,4 @@ async function main() { } } -main().catch(console.error); \ No newline at end of file +main().catch(console.error); diff --git a/sdk/python/Makefile b/sdk/python/Makefile index 88d462327..801027866 100644 --- a/sdk/python/Makefile +++ b/sdk/python/Makefile @@ -23,7 +23,7 @@ lint: type-check: pdm run mypy src/ -# Run type checking on tests specifically +# Run type checking on tests specifically type-check-tests: pdm run mypy tests/test_mypy_check.py tests/test_typing.py diff --git a/sdk/rust/.gitignore b/sdk/rust/.gitignore index a6f89c2da..b83d22266 100644 --- a/sdk/rust/.gitignore +++ b/sdk/rust/.gitignore @@ -1 +1 @@ -/target/ \ No newline at end of file +/target/ diff --git a/sdk/simulator/app-compose.json b/sdk/simulator/app-compose.json index bcbba37d3..8fb0d5be6 100644 --- a/sdk/simulator/app-compose.json +++ b/sdk/simulator/app-compose.json @@ -1 +1 @@ -{"manifest_version":2,"name":"guest-agent","runner":"docker-compose","docker_compose_file":"services:\n dstack-agent:\n image: ubuntu\n user: root\n network_mode: host\n volumes:\n - /:/host/\n - /var/run/tappd.sock:/var/run/tappd.sock\n - /var/run/dstack.sock:/var/run/dstack.sock\n entrypoint: |\n bash -c '\n apt-get update && apt-get install -y socat\n socat TCP-LISTEN:2000,fork UNIX-CONNECT:/var/run/tappd.sock &\n socat TCP-LISTEN:3000,fork UNIX-CONNECT:/var/run/dstack.sock &\n tail -f /dev/null\n '\n dstack-verifier:\n image: dstacktee/dstack-verifier:0.5.4\n ports:\n - \"8080:8080\"\n restart: unless-stopped","gateway_enabled":true,"public_logs":true,"public_sysinfo":true,"public_tcbinfo":true,"key_provider_id":"","allowed_envs":[],"no_instance_id":false,"secure_time":false,"key_provider":"kms","kms_enabled":true,"storage_fs":"ext4","pre_launch_script":"docker run --rm --privileged --pid=host --net=host -v /:/host \\\n -e SSH_GITHUB_USER=\"kvinwang\" \\\n kvin/dstack-openssh-installer:latest"} \ No newline at end of file +{"manifest_version":2,"name":"guest-agent","runner":"docker-compose","docker_compose_file":"services:\n dstack-agent:\n image: ubuntu\n user: root\n network_mode: host\n volumes:\n - /:/host/\n - /var/run/tappd.sock:/var/run/tappd.sock\n - /var/run/dstack.sock:/var/run/dstack.sock\n entrypoint: |\n bash -c '\n apt-get update && apt-get install -y socat\n socat TCP-LISTEN:2000,fork UNIX-CONNECT:/var/run/tappd.sock &\n socat TCP-LISTEN:3000,fork UNIX-CONNECT:/var/run/dstack.sock &\n tail -f /dev/null\n '\n dstack-verifier:\n image: dstacktee/dstack-verifier:0.5.4\n ports:\n - \"8080:8080\"\n restart: unless-stopped","gateway_enabled":true,"public_logs":true,"public_sysinfo":true,"public_tcbinfo":true,"key_provider_id":"","allowed_envs":[],"no_instance_id":false,"secure_time":false,"key_provider":"kms","kms_enabled":true,"storage_fs":"ext4","pre_launch_script":"docker run --rm --privileged --pid=host --net=host -v /:/host \\\n -e SSH_GITHUB_USER=\"kvinwang\" \\\n kvin/dstack-openssh-installer:latest"} diff --git a/sdk/simulator/appkeys.json b/sdk/simulator/appkeys.json index 1e67f019d..e564286cd 100644 --- a/sdk/simulator/appkeys.json +++ b/sdk/simulator/appkeys.json @@ -10,4 +10,4 @@ "key": "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg1PYCFKYfDmUfv5fk\nstppasf4mPGqnz0fEoLEnGx8CnKhRANCAAQZskV92biqZEbZgPE2//eBLs1sd3QO\n6w4osQ1WwMAAKKW25Z3OowfVg70fcHaenBFt8rz7F1ipXYYT594chILA\n-----END PRIVATE KEY-----\n" } } -} \ No newline at end of file +} diff --git a/sdk/simulator/sys-config.json b/sdk/simulator/sys-config.json index 1b2d5b48d..4993438b3 100644 --- a/sdk/simulator/sys-config.json +++ b/sdk/simulator/sys-config.json @@ -9,4 +9,4 @@ "docker_registry": "", "host_api_url": "vsock://2:12000/api", "vm_config": "{\"os_image_hash\":\"64f0d1545cd510a8dfed7ad609d105b5d41f0cb2afcfdda8867ede00c88add7a\",\"cpu_count\":1,\"memory_size\":2147483648}" -} \ No newline at end of file +} From 43bce0c5b578bc93e3989d79088b626c25092c2f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 04:44:22 -0700 Subject: [PATCH 426/435] build: expose guest os release targets --- .github/workflows/guest-os.yml | 16 ++-------------- Makefile | 28 +++++++++++++++++++++++++--- os/yocto/repro-build/repro-build.sh | 13 ++----------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/guest-os.yml b/.github/workflows/guest-os.yml index fc050e1a0..522b263dd 100644 --- a/.github/workflows/guest-os.yml +++ b/.github/workflows/guest-os.yml @@ -23,20 +23,8 @@ jobs: - name: Checkout uses: actions/checkout@v5 - - name: Checkout Yocto dependencies - run: | - git submodule update --init --depth 1 -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security - - - name: Reproducible build - run: cd os/yocto/repro-build && ./repro-build.sh -n + - name: Build production guest OS + run: make os-image - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index dd0bbc34c..fa67111e6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -.PHONY: help core core-check core-test sdk-test os os-yocto +OS_YOCTO_SUBMODULES := \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security + +.PHONY: help core core-check core-test sdk-test os os-yocto os-deps os-image os-repro-check help: @echo "dstack monorepo targets:" @@ -10,8 +20,11 @@ help: @echo " core-check check the Rust workspace" @echo " core-test test the Rust workspace" @echo " sdk-test run all public SDK tests" - @echo " os build the guest OS with the default backend" - @echo " os-yocto build the guest OS with Yocto" + @echo " os build the guest OS natively with the default backend" + @echo " os-yocto build the guest OS natively with Yocto" + @echo " os-deps initialize only the Yocto dependency submodules" + @echo " os-image build one production guest image in the pinned container" + @echo " os-repro-check build twice and compare reproducible outputs" core: cargo build --manifest-path dstack/Cargo.toml @@ -30,3 +43,12 @@ os: os-yocto: ./os/build.sh --backend yocto + +os-deps: + git submodule update --init --depth 1 -- $(OS_YOCTO_SUBMODULES) + +os-image: os-deps + cd os/yocto/repro-build && ./repro-build.sh -n + +os-repro-check: os-deps + cd os/yocto/repro-build && ./repro-build.sh diff --git a/os/yocto/repro-build/repro-build.sh b/os/yocto/repro-build/repro-build.sh index 7d66a55e2..978a35682 100755 --- a/os/yocto/repro-build/repro-build.sh +++ b/os/yocto/repro-build/repro-build.sh @@ -46,7 +46,7 @@ rm -rf .dummy build_to() { mkdir -p $1 GIT_REVISION=$(git -C "$REPO_ROOT" rev-parse HEAD) - BUILD_CMD="DSTACK_GIT_REVISION='$GIT_REVISION' ${2} ${GUEST_SRC_DIR}/os/yocto/build.sh image ./bb-build" + BUILD_CMD="DSTACK_GIT_REVISION='$GIT_REVISION' ${2} ${GUEST_SRC_DIR}/os/build.sh --backend yocto --build-dir ./bb-build" docker run --platform linux/amd64 --rm \ --userns=host \ --user $(id -u):$(id -g) \ @@ -83,16 +83,7 @@ set -e git clone https://github.com/Dstack-TEE/dstack.git cd dstack/ git checkout $(git -C $THIS_DIR rev-parse HEAD) -git submodule update --init -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security -cd os/yocto/repro-build && RELEASE_FLAVORS='${RELEASE_FLAVORS}' ./repro-build.sh -n +RELEASE_FLAVORS='${RELEASE_FLAVORS}' make os-image EOF echo "===========================" From 452947e4e527ff7832fbb52d94fb5a6b96d50ad2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 04:44:22 -0700 Subject: [PATCH 427/435] docs: add guest os build guide --- README.md | 3 + docs/building-guest-os.md | 202 ++++++++++++++++++++++++++++ docs/onboarding.md | 4 + docs/tutorials/guest-image-setup.md | 4 + os/README.md | 31 +++-- os/yocto/README.md | 25 ++-- 6 files changed, 239 insertions(+), 30 deletions(-) create mode 100644 docs/building-guest-os.md diff --git a/README.md b/README.md index d5120191b..287a8609c 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Deploy to a self-hosted TDX machine with the `dstackup install` -> `dstack deplo Setting up dstack on your own hardware? Start with the [self-hosted quick onboarding guide](./docs/onboarding.md) +Building or customizing the guest OS itself? Follow the [guest-OS build guide](./docs/building-guest-os.md). + ## Architecture ![Architecture](./docs/assets/arch.png) @@ -137,6 +139,7 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use **For Operators** - [Hardware Enablement](./docs/hardware-enablement.md) - Prepare a TDX or AMD SEV-SNP host - [Self-hosted Quick Onboarding](./docs/onboarding.md) - First app on one host +- [Build the Guest OS](./docs/building-guest-os.md) - Build and verify bootable images from source - [Deployment](./docs/deployment.md) - Self-hosting on TDX hardware - [On-Chain Governance](./docs/onchain-governance.md) - Smart contract authorization - [Gateway](./docs/dstack-gateway.md) - Gateway configuration diff --git a/docs/building-guest-os.md b/docs/building-guest-os.md new file mode 100644 index 000000000..b76bb0d25 --- /dev/null +++ b/docs/building-guest-os.md @@ -0,0 +1,202 @@ +# Build the dstack guest OS + +This guide builds the bootable dstack guest-OS release artifacts from source. +It is for OS developers, release maintainers, and operators who want a custom +image. You do **not** need to build an image for normal self-hosted onboarding: +`dstackup install` downloads and verifies a published guest-OS release by +default. + +## What the build produces + +The default `prod` build produces: + +- a bare-metal/CVM bundle for Intel TDX and, when the SEV firmware artifact is + available, AMD SEV-SNP; +- a UKI disk-image bundle for the GCP confidential-VM boot path; +- dm-verity rootfs data, launch-measurement material, checksums, and the unified + `digest.txt` OS identity. + +Yocto is currently the only implemented OS backend. Backend-independent rootfs +payload, artifact contract, measurement, and release packaging live outside +`os/yocto/`; see [`../os/README.md`](../os/README.md). + +## Prerequisites + +Use an x86-64 Linux host with: + +- Git; +- Docker Engine, usable by the current user; +- outbound HTTPS access for Git, Yocto source archives, and Rust crates; +- substantial free disk space for Yocto downloads, work directories, and + shared-state cache. + +TEE hardware is not required to build the image. It is required only when you +boot and attest the resulting image on the corresponding platform. + +Check the basics before starting: + +```bash +docker version +git --version +df -h . +``` + +## Quick build + +From a fresh checkout: + +```bash +git clone https://github.com/Dstack-TEE/dstack.git +cd dstack +make os-image +``` + +`make os-image` initializes only the eight Yocto dependency submodules and +runs one complete production image build in the pinned Ubuntu builder +container. It is equivalent to: + +```bash +git submodule update --init --depth 1 -- \ + os/yocto/deps/bitbake \ + os/yocto/deps/openembedded-core \ + os/yocto/deps/meta-yocto \ + os/yocto/deps/meta-confidential-compute \ + os/yocto/deps/meta-virtualization \ + os/yocto/deps/meta-openembedded \ + os/yocto/deps/meta-rust-bin \ + os/yocto/deps/meta-security + +cd os/yocto/repro-build +./repro-build.sh -n +``` + +The first build downloads and compiles the complete Yocto toolchain and guest +userspace, so it is much slower than an incremental rebuild. The `-n` option +means “build once”; it does not skip BitBake or image assembly. + +## Outputs + +Release archives are written under: + +```text +os/yocto/repro-build/dist/ +├── dstack-.tar.gz +├── dstack--uki.tar.gz +└── reproduce.sh +``` + +`reproduce.sh` is emitted when the source tree is clean. The unpacked build +tree and caches remain under `os/yocto/repro-build/build-a/`. + +The bare-metal archive includes the kernel, initramfs, OVMF firmware, +partitioned dm-verity rootfs, platform measurement CBOR files, +`sha256sum.txt`, `digest.txt`, and `metadata.json`. The UKI archive +includes the bootable `disk.raw` plus its identity and measurement files. + +Inspect and verify an archive with: + +```bash +mkdir -p /tmp/dstack-image +tar -xzf os/yocto/repro-build/dist/dstack-.tar.gz \ + -C /tmp/dstack-image +cd /tmp/dstack-image/dstack- +sha256sum -c sha256sum.txt +test "$(sha256sum sha256sum.txt | awk '{print $1}')" = "$(cat digest.txt)" +``` + +## Build both production and development flavors + +Production is the default. To build both variants once: + +```bash +cd os/yocto/repro-build +RELEASE_FLAVORS="prod dev" ./repro-build.sh -n +``` + +The development archive is named `dstack-dev-.tar.gz` and records +`"is_dev": true` in `metadata.json`. + +## Check reproducibility + +For a release candidate, omit `-n`: + +```bash +make os-repro-check +``` + +This builds independent `build-a` and `build-b` trees and compares the +release-relevant output allowlist. It takes roughly twice the resources of a +single build. Remove both ignored build trees if you specifically need a +from-scratch comparison: + +```bash +rm -rf os/yocto/repro-build/build-a \ + os/yocto/repro-build/build-b \ + os/yocto/repro-build/dist +make os-repro-check +``` + +## Incremental backend development + +The reproducible wrapper is the recommended release path. On a host with the +packages listed in `os/yocto/repro-build/Dockerfile.repro`, the generic +backend entrypoint can also be used directly from the repository root: + +```bash +./os/build.sh \ + --backend yocto \ + --flavors prod \ + --build-dir "$PWD/os/yocto/bb-build" +``` + +This keeps the native BitBake cache in `os/yocto/bb-build/` and writes +assembled images under the repository-root `images/` directory. Build both +flavors with `--flavors "prod dev"`. + +The generic entrypoint dispatches to `os//build.sh`. A future +backend such as mkosi can implement the same artifact-manifest contract without +changing the common assembler or release consumers. + +## Troubleshooting + +### A dependency directory is empty + +Run: + +```bash +make os-deps +git submodule status -- os/yocto/deps +``` + +Every listed dependency should start with a space, not `-`. + +### Docker permission is denied + +Ensure `docker version` works as the same non-root user that owns the +checkout. Do not run only part of the build as root; mixed ownership in +`build-a/` makes incremental builds difficult to repair. + +### A fetch task fails + +Yocto fetches many upstream sources. Preserve `build-a/`, confirm outbound +network and DNS access, then rerun `make os-image`; completed downloads and +tasks are reused. + +### The disk fills up + +The largest disposable directories are: + +```text +os/yocto/repro-build/build-a/ +os/yocto/repro-build/build-b/ +os/yocto/bb-build/ +``` + +They are ignored by Git and can be removed when no build is running. Keep +`dist/` separately if you need the release archives. + +### `reproduce.sh` is missing + +The image archives are still valid. The wrapper intentionally skips generating +`reproduce.sh` when `git status --porcelain` reports a dirty source tree, +because that script can reproduce only committed source revisions. diff --git a/docs/onboarding.md b/docs/onboarding.md index 4f4b8ee7f..051edb960 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -14,6 +14,10 @@ curl http://127.0.0.1:8080/ AMD SEV-SNP hosts use the same `dstackup` and `dstack` commands after you provide a guest image that contains the image digest (`digest.txt`). +The default onboarding flow uses a published image. To build or customize the +guest OS first, follow [Build the dstack guest OS](./building-guest-os.md), then +install the resulting `dstack-.tar.gz` bundle. + For multi-node production, Gateway TLS, custom domains, or on-chain governance, use the full [deployment guide](./deployment.md). ## What this workflow creates diff --git a/docs/tutorials/guest-image-setup.md b/docs/tutorials/guest-image-setup.md index 8570aa1e7..33230e3f9 100644 --- a/docs/tutorials/guest-image-setup.md +++ b/docs/tutorials/guest-image-setup.md @@ -54,6 +54,10 @@ Before starting, ensure you have: If you prefer to set up guest images manually, follow these steps. +To produce the image from source instead of downloading a release, follow the +[guest-OS build guide](../building-guest-os.md). Use the generated +`os/yocto/repro-build/dist/dstack-.tar.gz` archive in Step 3 below. + ### Step 1: Create Image Directory Structure Create the directory where guest images will be stored: diff --git a/os/README.md b/os/README.md index 4680ac689..6dc985976 100644 --- a/os/README.md +++ b/os/README.md @@ -20,24 +20,27 @@ backend is implemented today. ## Build -Initialize only the Yocto dependencies, then invoke the common entrypoint: +For a first production build from a fresh checkout, use the repository-level +target: ```bash -git submodule update --init -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security - -./os/build.sh --backend yocto +make os-image ``` -`--flavors "prod dev"` selects image flavors and `--build-dir DIR` selects the -native backend build directory. +It initializes only the Yocto dependency submodules and runs a complete build +inside the pinned builder container. See [Build the dstack guest OS](../docs/building-guest-os.md) +for prerequisites, output verification, flavor selection, reproducibility +checking, incremental development, and troubleshooting. + +The lower-level native backend interface remains available: + +```bash +make os-deps +./os/build.sh --backend yocto --build-dir "$PWD/os/yocto/bb-build" +``` + +`--flavors "prod dev"` selects image flavors. Native builds require the host +packages listed in `yocto/repro-build/Dockerfile.repro`. ## Backend contract diff --git a/os/yocto/README.md b/os/yocto/README.md index 92b5e967c..9aef967cf 100644 --- a/os/yocto/README.md +++ b/os/yocto/README.md @@ -8,26 +8,19 @@ For the repository-wide OS layout and backend contract, see [`../README.md`](../ ## Reproducible build -Prerequisites: an x86-64 Linux system with Docker installed. +From the repository root, the tested one-build entrypoint is: ```bash -git clone https://github.com/Dstack-TEE/dstack.git -cd dstack -git submodule update --init -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security -cd os/yocto/repro-build -./repro-build.sh +make os-image ``` -For an interactive build, source `dev-setup` and use the Makefile, or run the -repository entrypoint `./os/build.sh --backend yocto` from the repository root. +Use `make os-repro-check` to build twice and compare release outputs. The full +prerequisite, output, verification, flavor, incremental-build, and +troubleshooting instructions are in the +[guest-OS build guide](../../docs/building-guest-os.md). + +For an interactive native build, source `dev-setup` and use this directory's +Makefile, or run the repository entrypoint `./os/build.sh --backend yocto`. ## Output boundary From 7d4cb9e23ca036723f423247effcaf21188c1f35 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 18:08:30 -0700 Subject: [PATCH 428/435] fix: harden monorepo integration Repair guest OS and SDK entrypoints, make release image selection safe, and relocate imported helpers to clear ownership boundaries. Document the multi-source docker-compose fetch behavior and validate release artifacts before publishing. --- .github/workflows/guest-os.yml | 13 + .gitmodules | 6 +- CONTRIBUTING.md | 2 +- Makefile | 4 +- README.md | 6 + REUSE.toml | 2 +- docs/building-guest-os.md | 18 + docs/design-and-hardening-decisions.md | 6 +- docs/onboarding.md | 10 +- docs/tutorials/guest-image-setup.md | 10 +- dstack/crates/dstackup/src/cli.rs | 2 +- dstack/crates/dstackup/src/image.rs | 101 ++-- dstack/scripts/bin/dstack-cloud | 11 +- os/README.md | 3 +- os/build.sh | 10 + os/common/rootfs/app-compose.sh | 0 os/image/README.md | 9 + .../scripts => os/image}/dstack-image-oci.sh | 92 +++- .../scripts/bin => image}/mk-image-mr.sh | 58 ++- os/yocto/README.md | 5 + os/yocto/build.sh | 455 +++--------------- os/yocto/dev-setup | 32 +- .../nvidia/files/nvidia-gpu-detect | 15 +- os/yocto/scripts/bin/dstack | 1 - os/yocto/tools/README.md | 17 + os/yocto/tools/dev-stack.sh | 398 +++++++++++++++ .../fix-self-uid-map.sh} | 4 + os/yocto/tools/vm-runner/README.md | 15 + .../vm-runner/enable-vfio-passthrough.sh} | 5 +- .../bin => tools/vm-runner}/host_api.py | 3 +- .../vm-runner}/samples/Dockerfile | 0 .../vm-runner}/samples/docker-compose.yaml | 0 .../vm-runner/vm-runner.py} | 7 +- sdk/run-tests.sh | 20 +- sdk/simulator/.gitignore | 1 + sdk/simulator/build.sh | 12 +- .../scripts => tools}/add-spdx-attribution.py | 0 37 files changed, 825 insertions(+), 528 deletions(-) mode change 100644 => 100755 os/common/rootfs/app-compose.sh rename {dstack/scripts => os/image}/dstack-image-oci.sh (76%) rename os/{yocto/scripts/bin => image}/mk-image-mr.sh (59%) delete mode 120000 os/yocto/scripts/bin/dstack create mode 100644 os/yocto/tools/README.md create mode 100755 os/yocto/tools/dev-stack.sh rename os/yocto/{scripts/fix_self_uid_map.sh => tools/fix-self-uid-map.sh} (57%) create mode 100644 os/yocto/tools/vm-runner/README.md rename os/yocto/{scripts/bin/enable_vfio_passthrough.sh => tools/vm-runner/enable-vfio-passthrough.sh} (94%) mode change 100644 => 100755 rename os/yocto/{scripts/bin => tools/vm-runner}/host_api.py (97%) rename os/yocto/{scripts => tools/vm-runner}/samples/Dockerfile (100%) rename os/yocto/{scripts => tools/vm-runner}/samples/docker-compose.yaml (100%) rename os/yocto/{scripts/bin/dstack.py => tools/vm-runner/vm-runner.py} (99%) rename {dstack/scripts => tools}/add-spdx-attribution.py (100%) diff --git a/.github/workflows/guest-os.yml b/.github/workflows/guest-os.yml index 522b263dd..9f6c68525 100644 --- a/.github/workflows/guest-os.yml +++ b/.github/workflows/guest-os.yml @@ -60,6 +60,19 @@ jobs: TAG: ${{ inputs.tag }} run: | echo "$TAG" | grep -Eq '^guest-os-v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' + VERSION="${TAG#guest-os-v}" + BARE="os/yocto/repro-build/dist/dstack-${VERSION}.tar.gz" + UKI="os/yocto/repro-build/dist/dstack-${VERSION}-uki.tar.gz" + test -f "$BARE" + test -f "$UKI" + + read -r IMAGE_VERSION IMAGE_REVISION < <( + tar -xOf "$BARE" "dstack-${VERSION}/metadata.json" | + python3 -c 'import json, sys; data=json.load(sys.stdin); print(data["version"], data["git_revision"])' + ) + test "$IMAGE_VERSION" = "$VERSION" + test "$IMAGE_REVISION" = "$(git rev-parse HEAD)" + tar -tzf "$UKI" | grep -Fx "dstack-${VERSION}/disk.raw" - name: Create tag and release env: diff --git a/.gitmodules b/.gitmodules index 6473a0e5d..05449f8bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,13 +2,13 @@ # # SPDX-License-Identifier: Apache-2.0 -[submodule "kms/auth-eth/lib/forge-std"] +[submodule "dstack/kms/auth-eth/lib/forge-std"] path = dstack/kms/auth-eth/lib/forge-std url = https://github.com/foundry-rs/forge-std -[submodule "kms/auth-eth/lib/openzeppelin-contracts-upgradeable"] +[submodule "dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable"] path = dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable url = https://github.com/OpenZeppelin/openzeppelin-contracts-upgradeable -[submodule "kms/auth-eth/lib/openzeppelin-foundry-upgrades"] +[submodule "dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades"] path = dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades url = https://github.com/OpenZeppelin/openzeppelin-foundry-upgrades diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c1672e259..5cea75114 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ This project uses SPDX headers for license compliance. You should add appropriat We have a script to automatically add SPDX headers based on git blame data: ```bash -python3 dstack/scripts/add-spdx-attribution.py --file path/to/file.rs +python3 tools/add-spdx-attribution.py --file path/to/file.rs ``` Before submitting your changes, verify SPDX compliance using the [REUSE tool](https://github.com/fsfe/reuse-tool): diff --git a/Makefile b/Makefile index fa67111e6..f51b05896 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ help: @echo "dstack monorepo targets:" @echo " core build the Rust workspace" @echo " core-check check the Rust workspace" - @echo " core-test test the Rust workspace" + @echo " core-test test the Rust workspace with the simulator" @echo " sdk-test run all public SDK tests" @echo " os build the guest OS natively with the default backend" @echo " os-yocto build the guest OS natively with Yocto" @@ -33,7 +33,7 @@ core-check: cargo check --manifest-path dstack/Cargo.toml --workspace core-test: - cargo test --manifest-path dstack/Cargo.toml --workspace + ./dstack/run-tests.sh sdk-test: cd sdk && ./run-tests.sh diff --git a/README.md b/README.md index 287a8609c..af93715eb 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,12 @@ The currently implemented OS backend is Yocto under `os/yocto/`. Shared rootfs payload and release assembly stay outside that backend so another builder can be added later without duplicating them. See [`os/README.md`](./os/README.md). +Scripts follow the same ownership boundaries: component-specific helpers stay +beside their component under `dstack/`; files installed into every guest live +in `os/common/rootfs/`; backend-neutral image tooling lives in `os/image/`; +Yocto-only helpers live in `os/yocto/scripts/`; and repository-wide standalone +utilities live in `tools/`. + Your container runs inside a Confidential VM, such as Intel TDX or AMD SEV-SNP, with optional GPU isolation via NVIDIA Confidential Computing. The CPU TEE protects application logic; the GPU TEE protects model weights and inference data. **Core components:** diff --git a/REUSE.toml b/REUSE.toml index 1433a29cb..65914d1d8 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -141,7 +141,7 @@ SPDX-License-Identifier = "Apache-2.0" # Scripts with SPDX-like content (false positive prevention) [[annotations]] -path = "dstack/scripts/add-spdx-attribution.py" +path = "tools/add-spdx-attribution.py" SPDX-FileCopyrightText = "© 2025 Phala Network " SPDX-License-Identifier = "Apache-2.0" precedence = "override" diff --git a/docs/building-guest-os.md b/docs/building-guest-os.md index b76bb0d25..f783ac327 100644 --- a/docs/building-guest-os.md +++ b/docs/building-guest-os.md @@ -182,6 +182,24 @@ Yocto fetches many upstream sources. Preserve `build-a/`, confirm outbound network and DNS access, then rerun `make os-image`; completed downloads and tasks are reused. +### `docker-compose do_fetch` repeatedly shows 0–100% + +This is not one archive being downloaded in a loop. Docker Compose has hundreds +of independently checksummed Go-module sources, while BitBake's terminal +percentage describes only the current source URL. The percentage therefore +returns to zero for every module even though the task timer and PID stay the +same. + +Let the first fetch finish. If it is interrupted, rerun the same command; +completed files have `.done` markers in the build directory's `downloads/` +cache and are not downloaded again. To confirm which URL is currently being +fetched during a native `make os` build, inspect the latest task log: + +```bash +find os/yocto/bb-build/tmp-mc-* -path '*docker-compose/*/temp/log.do_fetch' \ + -print -exec tail -n 5 {} \; +``` + ### The disk fills up The largest disposable directories are: diff --git a/docs/design-and-hardening-decisions.md b/docs/design-and-hardening-decisions.md index 81ae51ca7..af5dd0ae6 100644 --- a/docs/design-and-hardening-decisions.md +++ b/docs/design-and-hardening-decisions.md @@ -1,8 +1,8 @@ -# Design and Hardening Decisions in meta-dstack Layer +# Design and Hardening Decisions in the dstack Yocto Layer ## Overview -The meta-dstack layer is designed to create a minimally secure image for booting Confidential Virtual Machines (CVMs). Our design philosophy prioritizes attack surface reduction while maintaining TDX-aware functionality. This document outlines the architectural decisions and trade-offs made during development. +The dstack-owned Yocto layer under `os/yocto/layers/meta-dstack/` is designed to create a minimally secure image for booting Confidential Virtual Machines (CVMs). Our design philosophy prioritizes attack surface reduction while maintaining TDX-aware functionality. This document outlines the architectural decisions and trade-offs made during development. ## Key Design Decisions @@ -49,7 +49,7 @@ See [here](https://intel.github.io/ccc-linux-guest-hardening-docs/security-spec. ### 5. Secure System Time -**Implementation**: dstack OS enforces the guest kernel uses TSC as the only timer source by appending `tsc=reliable no-kvmclock` to the kernel cmdline. It also enforces the use of NTS with built-in [trusted servers](https://github.com/Dstack-TEE/meta-dstack/blob/bef2dfa850f4116ae4ece96d8c0948965c5874b3/meta-dstack/recipes-core/chrony/files/chrony.conf#L13-L20) to synchronize system time. +**Implementation**: dstack OS enforces the guest kernel uses TSC as the only timer source by appending `tsc=reliable no-kvmclock` to the kernel cmdline. It also enforces the use of NTS with built-in [trusted servers](../os/yocto/layers/meta-dstack/recipes-core/chrony/files/chrony.conf) to synchronize system time. **Behavior**: When `secure_time` is enabled in the app-compose.json configuration, the system ensures time synchronization is completed before requesting application keys. If `secure_time` is disabled, time synchronization is not enforced before application launch. diff --git a/docs/onboarding.md b/docs/onboarding.md index 051edb960..209862051 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -88,7 +88,7 @@ Run: sudo dstackup install ``` -`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest CPU image from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. Pre-monorepo `meta-dstack` releases remain a fallback for pinned older versions. +`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest unified image from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. Current images include NVIDIA support conditionally and work on CPU-only hosts too. Pre-monorepo `meta-dstack` releases remain a fallback for pinned older versions. On TDX, `dstackup install` starts the SGX key provider automatically from `/usr/local/share/dstack/key-provider-build`. To use a different provider, pass one of: @@ -99,14 +99,16 @@ sudo dstackup install --use-existing-key-provider 127.0.0.1:3443 On AMD SEV-SNP, no SGX key provider is needed. The selected guest image must include `digest.txt`; otherwise, `dstackup install` fails before it starts the host units because apps could not be pinned to the measured OS image. -To use a GPU image, pull it before install: +The normal pull command is sufficient for current CPU and GPU hosts: ```bash -sudo dstackup image pull --gpu +sudo dstackup image pull sudo dstackup install ``` -If multiple images are present, pass the image name or release version to `--image`, such as `dstack-0.5.11`, `dstack-nvidia-0.5.11`, or `0.5.11`. If the requested release-shaped image is not local, `dstackup install` downloads it. +`dstackup image pull --gpu` remains a compatibility option for older releases that published a separate `dstack-nvidia-*` archive; it falls back to the unified archive when a release has no separate GPU asset. + +If multiple images are present, pass the image name or release version to `--image`, such as `dstack-0.6.0`, legacy `dstack-nvidia-0.5.11`, or `0.6.0`. If the requested release-shaped image is not local, `dstackup install` downloads it. When install succeeds, it prints the dashboard URL, the KMS address, and a `dstack deploy` command template. The default dashboard URL is: diff --git a/docs/tutorials/guest-image-setup.md b/docs/tutorials/guest-image-setup.md index 33230e3f9..4f141b3a5 100644 --- a/docs/tutorials/guest-image-setup.md +++ b/docs/tutorials/guest-image-setup.md @@ -244,16 +244,16 @@ Use the `dstack-image-oci.sh` script to package and push a guest image directory ```bash # Push a standard image (auto-tags: version + sha256-hash) -./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image +./os/image/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.6.0 ghcr.io/your-org/guest-image -# Push an nvidia variant -./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-nvidia-0.5.8 ghcr.io/your-org/guest-image +# Current unified image is also used on NVIDIA hosts +./os/image/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.6.0 ghcr.io/your-org/guest-image --tag 0.6.0 # Push with a custom tag -./dstack/scripts/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.5.8 ghcr.io/your-org/guest-image --tag latest +./os/image/dstack-image-oci.sh push /var/lib/dstack/images/dstack-0.6.0 ghcr.io/your-org/guest-image --tag latest # List tags in the registry -./dstack/scripts/dstack-image-oci.sh list ghcr.io/your-org/guest-image +./os/image/dstack-image-oci.sh list ghcr.io/your-org/guest-image ``` The script reads `metadata.json` and `digest.txt` from the image directory and auto-generates tags: diff --git a/dstack/crates/dstackup/src/cli.rs b/dstack/crates/dstackup/src/cli.rs index 6e534619e..4d1b74689 100644 --- a/dstack/crates/dstackup/src/cli.rs +++ b/dstack/crates/dstackup/src/cli.rs @@ -80,7 +80,7 @@ pub(crate) enum ImageCmd { /// image version to fetch (default: the latest release). #[arg(long, value_name = "VERSION")] version: Option, - /// fetch the gpu (nvidia) image instead of the cpu one. + /// prefer a legacy gpu image; current unified images are already GPU-capable. #[arg(long)] gpu: bool, #[command(flatten)] diff --git a/dstack/crates/dstackup/src/image.rs b/dstack/crates/dstackup/src/image.rs index 0102b6ccb..4aa03c762 100644 --- a/dstack/crates/dstackup/src/image.rs +++ b/dstack/crates/dstackup/src/image.rs @@ -5,8 +5,8 @@ //! `dstackup image` — fetch, list, and remove guest OS images. //! //! Images are published as `guest-os-v*` release tarballs in the dstack -//! monorepo. There -//! are two variants — cpu (`dstack-`) and gpu (`dstack-nvidia-`). +//! monorepo. Current releases use one hardware-adaptive `dstack-` image; +//! legacy releases may also contain `dstack-nvidia-` variants. //! `install` validates the selected image against `digest.txt`, the OS image //! hash used on all platforms. HTTP + checksum are native (reqwest is //! already linked via the prpc client; sha2 verifies inline); only `tar` is @@ -99,28 +99,18 @@ pub(crate) async fn pull( ) -> Result { println!( "dstackup image pull — {} image", - if gpu { "gpu (nvidia)" } else { "cpu" } + if gpu { + "gpu-capable (legacy nvidia variant preferred)" + } else { + "unified" + } ); let release = fetch_release(version).await?; - let ver = release_tag_version(&release.tag_name); - - // the unpacked dir is usually `dstack[-nvidia]-`; check that first so a - // repeat pull is a cheap no-op instead of re-fetching a few hundred MB. - let expected = format!("dstack-{}{ver}", if gpu { "nvidia-" } else { "" }); - if !force - && Path::new(image_dir) - .join(&expected) - .join("metadata.json") - .exists() - { - println!(" [ok] {expected} already present (use --force to re-download)"); - return Ok(expected); - } let asset = pick_asset(&release.assets, gpu).with_context(|| { format!( - "no {} image tarball in guest-OS release {} (assets: {})", - if gpu { "gpu" } else { "cpu" }, + "no suitable {} image tarball in guest-OS release {} (assets: {})", + if gpu { "GPU-capable" } else { "unified" }, release.tag_name, release .assets @@ -138,6 +128,23 @@ pub(crate) async fn pull( asset.name ); } + + // Release archives use their filename stem as the top-level image + // directory. Select the asset before this check because --gpu may resolve + // to a legacy dstack-nvidia archive or to the current unified image. + let expected = asset + .name + .strip_suffix(".tar.gz") + .context("guest image asset must end in .tar.gz")?; + if !force + && Path::new(image_dir) + .join(expected) + .join("metadata.json") + .exists() + { + println!(" [ok] {expected} already present (use --force to re-download)"); + return Ok(expected.to_string()); + } println!(" [..] release {} -> {}", release.tag_name, asset.name); fs::create_dir_all(image_dir).with_context(|| format!("creating {image_dir}"))?; @@ -180,8 +187,8 @@ async fn stage_image( .await?; fs::create_dir_all(staging).with_context(|| format!("creating {}", staging.display()))?; extract(&tmp.to_string_lossy(), &staging.to_string_lossy())?; - // the unpacked dir name needn't match the asset name (e.g. a `-uki` asset), - // so find the dir that actually holds a metadata.json. + // Do not assume the unpacked directory name matches the release asset; + // adopt the directory that actually contains metadata.json. let inner = image_subdirs(&staging.to_string_lossy()) .into_iter() .find(|d| staging.join(d).join("metadata.json").exists()) @@ -567,8 +574,8 @@ pub(crate) fn no_image_message(image_dir: &str) -> String { format!( "no guest image found in {image_dir}\n\n\ download the latest with:\n \ - {pull} # cpu image\n \ - {pull} --gpu # gpu (nvidia) image\n\n\ + {pull} # current unified CPU/GPU image\n \ + {pull} --gpu # prefer a legacy nvidia-specific asset\n\n\ images are published at {RELEASES_URL}" ) } @@ -657,27 +664,32 @@ async fn fetch_tagged_release(client: &reqwest::Client, url: &str) -> Result &str { - tag.strip_prefix(RELEASE_TAG_PREFIX) - .or_else(|| tag.strip_prefix('v')) - .unwrap_or(tag) -} - -/// pick the cpu or gpu image tarball from a release's assets, skipping `-dev` -/// builds. cpu = `dstack-...`, gpu = `dstack-nvidia-...`. +/// Pick a full bare-metal image tarball, never the `-uki` archive. For a GPU +/// request, prefer a legacy `dstack-nvidia-*` asset when present and otherwise +/// use the current unified image (which already contains conditional NVIDIA +/// support). fn pick_asset(assets: &[Asset], gpu: bool) -> Option<&Asset> { - assets.iter().find(|a| { + let matches = |a: &&Asset, want_legacy_gpu: bool| { let n = a.name.as_str(); - if !n.ends_with(".tar.gz") || n.contains("-dev") { + if !n.ends_with(".tar.gz") || n.ends_with("-uki.tar.gz") || n.contains("-dev") { return false; } let is_gpu = n.starts_with("dstack-nvidia-"); - if gpu { + if want_legacy_gpu { is_gpu } else { n.starts_with("dstack-") && !is_gpu } - }) + }; + + if gpu { + assets + .iter() + .find(|asset| matches(asset, true)) + .or_else(|| assets.iter().find(|asset| matches(asset, false))) + } else { + assets.iter().find(|asset| matches(asset, false)) + } } fn extract(tarball: &str, into: &str) -> Result<()> { @@ -765,19 +777,22 @@ mod tests { } #[test] - fn parses_monorepo_and_legacy_release_tags() { - assert_eq!(release_tag_version("guest-os-v0.6.0"), "0.6.0"); - assert_eq!(release_tag_version("v0.5.11"), "0.5.11"); + fn gpu_pull_falls_back_to_unified_image() { + let assets = vec![ + asset("dstack-0.6.0-uki.tar.gz"), + asset("dstack-0.6.0.tar.gz"), + ]; + assert_eq!( + pick_asset(&assets, true).unwrap().name, + "dstack-0.6.0.tar.gz" + ); } #[test] - fn gpu_only_release_has_no_cpu_asset() { + fn uki_archive_is_never_selected_as_a_host_image() { let assets = vec![asset("dstack-nvidia-0.6.0.a2-uki.tar.gz")]; assert!(pick_asset(&assets, false).is_none()); - assert_eq!( - pick_asset(&assets, true).unwrap().name, - "dstack-nvidia-0.6.0.a2-uki.tar.gz" - ); + assert!(pick_asset(&assets, true).is_none()); } #[test] diff --git a/dstack/scripts/bin/dstack-cloud b/dstack/scripts/bin/dstack-cloud index b817cf2b5..f1e772f68 100755 --- a/dstack/scripts/bin/dstack-cloud +++ b/dstack/scripts/bin/dstack-cloud @@ -1021,13 +1021,16 @@ class CloudDeploymentManager: os_image = url_filename download_tar = target_dir / url_filename else: - # Extract version from os_image (e.g., dstack-nvidia-0.6.0 -> 0.6.0) - # Version is the last component after the last hyphen followed by digits + # Extract the release version from the unified image name (or a + # legacy flavor name), including an optional prerelease suffix. import re - version_match = re.search(r'-(\d+\.\d+\.\d+)$', os_image) + version_match = re.search( + r'-(\d+\.\d+\.\d+(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?)$', + os_image, + ) if not version_match: logger.error(f"Could not extract version from image name: {os_image}") - logger.error("Expected format: dstack-- (e.g., dstack-nvidia-0.6.0)") + logger.error("Expected format: dstack- (or a legacy dstack--)") return version = version_match.group(1) download_url = f"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/Dstack-TEE/dstack/releases/download/guest-os-v{version}/{os_image}-uki.tar.gz" diff --git a/os/README.md b/os/README.md index 6dc985976..335ab127f 100644 --- a/os/README.md +++ b/os/README.md @@ -11,7 +11,8 @@ os/ ├── spec/ # versioned backend artifact contract └── yocto/ # the currently implemented backend ├── deps/ # external Yocto layers (git submodules) - └── layers/ # dstack-owned Yocto layers + ├── layers/ # dstack-owned Yocto layers + └── tools/ # isolated legacy/imported host helpers ``` A future backend can be added at `os//` (for example `os/mkosi/`) diff --git a/os/build.sh b/os/build.sh index 1d51cf5bc..63254982b 100755 --- a/os/build.sh +++ b/os/build.sh @@ -27,14 +27,17 @@ BUILD_DIR= while [ $# -gt 0 ]; do case "$1" in --backend) + [ $# -ge 2 ] || { echo "Error: --backend requires a value" >&2; exit 1; } BACKEND=$2 shift 2 ;; --flavors) + [ $# -ge 2 ] || { echo "Error: --flavors requires a value" >&2; exit 1; } FLAVORS=$2 shift 2 ;; --build-dir) + [ $# -ge 2 ] || { echo "Error: --build-dir requires a value" >&2; exit 1; } BUILD_DIR=$2 shift 2 ;; @@ -50,6 +53,13 @@ while [ $# -gt 0 ]; do esac done +case "$BACKEND" in + ''|*[!0-9A-Za-z_-]*) + echo "Error: invalid OS backend name: $BACKEND" >&2 + exit 1 + ;; +esac + BACKEND_SCRIPT="$SCRIPT_DIR/$BACKEND/build.sh" if [ ! -x "$BACKEND_SCRIPT" ]; then echo "Error: OS backend is not available: $BACKEND" >&2 diff --git a/os/common/rootfs/app-compose.sh b/os/common/rootfs/app-compose.sh old mode 100644 new mode 100755 diff --git a/os/image/README.md b/os/image/README.md index fcfc70b38..3215939ea 100644 --- a/os/image/README.md +++ b/os/image/README.md @@ -8,3 +8,12 @@ The assembler creates the partitioned rootfs, metadata, measurement CBOR files, unified digest, and release archives. Backend exporters may use symlinks for large local artifacts; all paths recorded in the manifest itself are relative to the manifest directory. + +`mk-image-mr.sh ` creates the flattened, rootfs-free +`mr_.tar.gz` bundle consumed by verifier/KMS image-download endpoints. +Because this is release-format post-processing rather than a Yocto operation, +the helper lives beside the common assembler. + +`dstack-image-oci.sh` pushes and lists assembled guest-image directories in an +OCI registry. It is likewise independent of the backend that produced the +image. diff --git a/dstack/scripts/dstack-image-oci.sh b/os/image/dstack-image-oci.sh similarity index 76% rename from dstack/scripts/dstack-image-oci.sh rename to os/image/dstack-image-oci.sh index f8b986224..580d6e334 100755 --- a/dstack/scripts/dstack-image-oci.sh +++ b/os/image/dstack-image-oci.sh @@ -7,6 +7,7 @@ set -euo pipefail usage() { + local status=${1:-1} cat < [options] @@ -19,12 +20,12 @@ Arguments: Full image reference (e.g., ghcr.io/org/guest-image) Examples: - $0 push ./dstack-0.5.8 cr.kvin.wang/dstack/guest-image - $0 push ./dstack-nvidia-0.5.8 ghcr.io/dstack-tee/guest-image --tag nvidia-0.5.8 - $0 list cr.kvin.wang/dstack/guest-image - $0 list cr.kvin.wang/dstack/guest-image --filter nvidia + $0 push ./dstack-0.6.0 ghcr.io/dstack-tee/guest-image + $0 push ./dstack-0.6.0 ghcr.io/dstack-tee/guest-image --tag 0.6.0 + $0 list ghcr.io/dstack-tee/guest-image + $0 list ghcr.io/dstack-tee/guest-image --filter nvidia EOF - exit 1 + exit "$status" } COMMAND="${1:-}" @@ -32,15 +33,19 @@ COMMAND="${1:-}" shift # --- PUSH --- -cmd_push() { +cmd_push() ( local image_dir="" local image_ref="" local extra_tag="" while [ $# -gt 0 ]; do case "$1" in - --tag) extra_tag="$2"; shift 2 ;; - -h|--help) usage ;; + --tag) + [ $# -ge 2 ] || { echo "Error: --tag requires a value"; exit 1; } + extra_tag="$2" + shift 2 + ;; + -h|--help) usage 0 ;; -*) echo "Unknown option: $1"; exit 1 ;; *) if [ -z "$image_dir" ]; then @@ -64,11 +69,27 @@ cmd_push() { # Read image info local version - version=$(python3 -c "import json; print(json.load(open('$metadata'))['version'])") + version=$(python3 - "$metadata" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as file: + print(json.load(file)["version"]) +PY + ) + if [[ ! "$version" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "Error: metadata.json contains an invalid version: $version" + exit 1 + fi local digest_file="$image_dir/digest.txt" local os_image_hash="" if [ -f "$digest_file" ]; then os_image_hash=$(tr -d '\n\r' < "$digest_file") + if [[ ! "$os_image_hash" =~ ^[0-9A-Fa-f]{64}$ ]]; then + echo "Error: digest.txt must contain one SHA-256 hex digest" + exit 1 + fi + os_image_hash=${os_image_hash,,} fi # Detect image variant from directory name @@ -118,7 +139,15 @@ cmd_push() { # Collect all files local files=() for f in "$image_dir"/*; do - [ -f "$f" ] && files+=("$(basename "$f")") + if [ -f "$f" ]; then + local name + name=$(basename "$f") + if [[ ! "$name" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "Error: image filename is not OCI-packaging safe: $name" + exit 1 + fi + files+=("$name") + fi done # Generate Dockerfile @@ -161,11 +190,19 @@ cmd_push() { if [ -n "$os_image_hash" ]; then local mr_tag="mr-sha256-${os_image_hash}" local mr_dir - mr_dir=$(mktemp -d) + mr_dir="$tmp_dir/measurement" + mkdir -p "$mr_dir" # Read rootfs filename from metadata to exclude it local rootfs_name - rootfs_name=$(python3 -c "import json; print(json.load(open('$metadata')).get('rootfs', ''))") + rootfs_name=$(python3 - "$metadata" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as file: + print(json.load(file).get("rootfs", "")) +PY + ) # Collect files excluding rootfs local mr_files=() @@ -197,16 +234,18 @@ cmd_push() { echo "Pushing: $mr_ref" docker push "$mr_ref" - rm -rf "$mr_dir" tags+=("$mr_tag") fi + rm -rf "$tmp_dir" + trap - EXIT + echo "" echo "=== Done ===" for tag in "${tags[@]}"; do echo " ${image_ref}:${tag}" done -} +) # --- LIST --- cmd_list() { @@ -215,8 +254,12 @@ cmd_list() { while [ $# -gt 0 ]; do case "$1" in - --filter) filter="$2"; shift 2 ;; - -h|--help) usage ;; + --filter) + [ $# -ge 2 ] || { echo "Error: --filter requires a value"; exit 1; } + filter="$2" + shift 2 + ;; + -h|--help) usage 0 ;; -*) echo "Unknown option: $1"; exit 1 ;; *) if [ -z "$image_ref" ]; then @@ -243,21 +286,24 @@ cmd_list() { curl -sf "https://${registry}/v2/${repo}/tags/list" 2>/dev/null || \ echo '{"tags":[]}') - python3 -c " -import json, sys, re + python3 -c ' +import json +import re +import sys + data = json.load(sys.stdin) -tags = sorted(data.get('Tags', data.get('tags', []))) -filt = '$filter' +tags = sorted(data.get("Tags", data.get("tags", []))) +filt = sys.argv[1] for tag in tags: if not filt or re.search(filt, tag): - print(f' {tag}') -" <<< "$tags_json" + print(f" {tag}") +' "$filter" <<< "$tags_json" } # Dispatch case "$COMMAND" in push) cmd_push "$@" ;; list) cmd_list "$@" ;; - -h|--help) usage ;; + -h|--help) usage 0 ;; *) echo "Unknown command: $COMMAND"; usage ;; esac diff --git a/os/yocto/scripts/bin/mk-image-mr.sh b/os/image/mk-image-mr.sh similarity index 59% rename from os/yocto/scripts/bin/mk-image-mr.sh rename to os/image/mk-image-mr.sh index 9d60c94fd..be5458207 100755 --- a/os/yocto/scripts/bin/mk-image-mr.sh +++ b/os/image/mk-image-mr.sh @@ -1,19 +1,28 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 set -euo pipefail # Function to display usage usage() { - echo "Usage: $0 " + local status=${1:-1} + echo "Usage: ${0##*/} " echo "Example: $0 https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v0.6.0/dstack-0.6.0.tar.gz" echo "Example: $0 /path/to/local/file.tar.gz" - exit 1 + exit "$status" } # Check if argument is provided if [ $# -ne 1 ]; then usage fi +case "$1" in + -h|--help) + usage 0 + ;; +esac INPUT="$1" TEMP_DIR=$(mktemp -d) @@ -33,7 +42,7 @@ if [[ "$INPUT" =~ ^https?:// ]]; then echo "Downloading from URL: $INPUT" ARCHIVE_FILE="$TEMP_DIR/archive.tar.gz" if command -v curl >/dev/null 2>&1; then - curl -L -o "$ARCHIVE_FILE" "$INPUT" + curl -fL -o "$ARCHIVE_FILE" "$INPUT" elif command -v wget >/dev/null 2>&1; then wget -O "$ARCHIVE_FILE" "$INPUT" else @@ -46,7 +55,7 @@ else echo "Error: Local file does not exist: $INPUT" exit 1 fi - ARCHIVE_FILE="$INPUT" + ARCHIVE_FILE=$(realpath "$INPUT") fi # Create extraction directory @@ -54,18 +63,20 @@ mkdir -p "$EXTRACT_DIR" # Extract the archive echo "Extracting archive to: $EXTRACT_DIR" -tar -xzf "$ARCHIVE_FILE" -C "$EXTRACT_DIR" +tar -xzf "$ARCHIVE_FILE" -C "$EXTRACT_DIR" \ + --no-same-owner --no-same-permissions # Find and read the digest -DIGEST_FILE=$(find "$EXTRACT_DIR" -name "digest.txt" -type f | head -1) -if [ -z "$DIGEST_FILE" ]; then - echo "Error: digest.txt file not found in the extracted archive" +mapfile -d '' -t DIGEST_FILES < <(find "$EXTRACT_DIR" -name digest.txt -type f -print0) +if [ "${#DIGEST_FILES[@]}" -ne 1 ]; then + echo "Error: expected exactly one digest.txt in the archive, found ${#DIGEST_FILES[@]}" exit 1 fi +DIGEST_FILE=${DIGEST_FILES[0]} -DIGEST=$(cat "$DIGEST_FILE" | tr -d '\n\r' | sed 's/[^a-zA-Z0-9]//g') -if [ -z "$DIGEST" ]; then - echo "Error: Could not read digest from $DIGEST_FILE" +DIGEST=$(tr -d '\n\r' < "$DIGEST_FILE" | tr 'A-F' 'a-f') +if [[ ! "$DIGEST" =~ ^[0-9a-f]{64}$ ]]; then + echo "Error: digest.txt must contain exactly one SHA-256 hex digest" exit 1 fi @@ -73,21 +84,27 @@ echo "Found digest: $DIGEST" # Remove rootfs file(s) echo "Removing rootfs files..." +REMOVED_COUNT=$(find "$EXTRACT_DIR" -name "rootfs*" -type f | wc -l) find "$EXTRACT_DIR" -name "rootfs*" -type f -delete -REMOVED_COUNT=$(find "$EXTRACT_DIR" -name "rootfs*" -type f 2>/dev/null | wc -l) -if [ $REMOVED_COUNT -eq 0 ]; then - echo "Rootfs files removed successfully" -else - echo "Warning: Some rootfs files may still exist" -fi +echo "Removed $REMOVED_COUNT rootfs file(s)" # Create flattened structure in a new directory FLATTEN_DIR="$TEMP_DIR/flattened" mkdir -p "$FLATTEN_DIR" echo "Flattening directory structure..." -# Find all files (not directories) and copy them to the flattened directory -find "$EXTRACT_DIR" -type f -exec cp {} "$FLATTEN_DIR/" \; +# Find all files (not directories) and copy them to the flattened directory. +# Refuse duplicate basenames instead of silently overwriting an artifact. +declare -A SEEN_BASENAMES=() +while IFS= read -r -d '' file; do + name=$(basename "$file") + if [[ -n "${SEEN_BASENAMES[$name]:-}" ]]; then + echo "Error: duplicate archive basename while flattening: $name" >&2 + exit 1 + fi + SEEN_BASENAMES[$name]=1 + cp "$file" "$FLATTEN_DIR/$name" +done < <(find "$EXTRACT_DIR" -type f -print0) # Count files for verification FILE_COUNT=$(find "$FLATTEN_DIR" -type f | wc -l) @@ -99,7 +116,8 @@ echo "Creating final archive: $OUTPUT_FILE" # Change to the flattened directory and create archive without directory structure cd "$FLATTEN_DIR" -tar -czf "../$OUTPUT_FILE" -- * +LC_ALL=C tar --sort=name --mtime='@0' --owner=0 --group=0 --numeric-owner \ + -czf "../$OUTPUT_FILE" -- * cd - >/dev/null # Move the final file to the current working directory diff --git a/os/yocto/README.md b/os/yocto/README.md index 9aef967cf..9fadd694f 100644 --- a/os/yocto/README.md +++ b/os/yocto/README.md @@ -22,6 +22,11 @@ troubleshooting instructions are in the For an interactive native build, source `dev-setup` and use this directory's Makefile, or run the repository entrypoint `./os/build.sh --backend yocto`. +The supported backend script only builds guest images. Historical host-stack +and direct-QEMU helpers imported with the old repository are isolated under +`tools/`; they are not added to `PATH` and do not shadow the supported Rust +`dstack` CLI. + ## Output boundary `scripts/export-artifacts.sh` converts BitBake deployment paths to the common diff --git a/os/yocto/build.sh b/os/yocto/build.sh index b96fbf017..593db2611 100755 --- a/os/yocto/build.sh +++ b/os/yocto/build.sh @@ -1,393 +1,86 @@ #!/bin/bash -SCRIPT_DIR=$( - cd $(dirname $0) - pwd -) -ACTION=$1 +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 -META_DIR=$SCRIPT_DIR -DSTACK_DIR=$(realpath "$SCRIPT_DIR/../../dstack") -CERTS_DIR=$(pwd)/certs -IMAGES_DIR=$(pwd)/images -RUN_DIR=$(pwd)/run -RUST_BUILD_DIR=$(pwd)/rust-target -CERBOT_WORKDIR=$RUN_DIR/certbot -KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry -KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ +# Yocto implementation of the guest-OS backend contract in os/README.md. +set -eo pipefail -GATEWAY_CERT=${GATEWAY_CERT:-$CERTS_DIR/live/cert.pem} -GATEWAY_KEY=${GATEWAY_KEY:-$CERTS_DIR/live/key.pem} - -CONFIG_FILE=./build-config.sh - -check_config() { - local template_file=$1 - local config_file=$2 +usage() { + cat <<'USAGE' +Usage: build.sh image [build-dir] - # extract all variables in template file - local variables=$(grep -oE '^\s*[A-Z_]+=' $template_file | sort) +Build dstack guest-OS artifacts with Yocto and assemble release archives. +The guest action is accepted as a compatibility alias for image. - # check if each variable is set in config file - local var missing=0 - for var in $variables; do - if ! grep -qE "^\s*$var" $config_file; then - echo "Variable $var is not set in $config_file" - missing=1 - fi - done - if [ $missing -ne 0 ]; then - return 1 - fi - return 0 +Environment: + FLAVORS space-separated image flavors (default: prod) + DIST_DIR release output directory (default: /images) +USAGE } -require_config() { - - # Base port for RPC services - BASE_PORT=$(($RANDOM % 1000 * 10 + 10000)) - CID_POOL_START=$(($RANDOM % 1000 * 1000 + 20000)) - SUBNET_INDEX=$(($RANDOM % 240 + 10)) - - cat <build-config.sh.tpl -# DNS domain of kms rpc and dstack-gateway rpc -# *.1022.dstack.org resolves to 10.0.2.2 which is the IP of the host system -# from CVMs point of view -KMS_DOMAIN=kms.1022.dstack.org -GATEWAY_DOMAIN=gateway.1022.dstack.org - -# CIDs allocated to VMs start from this number of type unsigned int32 -VMM_CID_POOL_START=$CID_POOL_START -# CID pool size -VMM_CID_POOL_SIZE=1000 - -VMM_RPC_LISTEN_PORT=$BASE_PORT -# Whether port mapping from host to CVM is allowed -VMM_PORT_MAPPING_ENABLED=true -# Host API configuration, type of uint32 -VMM_VSOCK_LISTEN_PORT=$BASE_PORT -# Whether to enable GPU support -VMM_ENABLE_GPU=false - -KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) -GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) - -GATEWAY_WG_INTERFACE=dgw-$USER -GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) -GATEWAY_WG_IP=10.$SUBNET_INDEX.3.1 -GATEWAY_SERVE_PORT=$(($BASE_PORT + 4)) -GATEWAY_CERT=$CERBOT_WORKDIR/live/cert.pem -GATEWAY_KEY=$CERBOT_WORKDIR/live/key.pem - -BIND_PUBLIC_IP=0.0.0.0 - -GATEWAY_PUBLIC_DOMAIN= - -# for certbot -CERTBOT_ENABLED=false -CF_API_TOKEN= -ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory -EOF - if [ -f $CONFIG_FILE ]; then - source $CONFIG_FILE - # check if any variable in build-config.sh.tpl is not set in build-config.sh. - # This might occur if the build-config.sh is generated from and old repo. - check_config build-config.sh.tpl $CONFIG_FILE - if [ $? -ne 0 ]; then - exit 1 - fi - rm -f build-config.sh.tpl - - if [ -z "$GATEWAY_SERVE_PORT" ]; then - GATEWAY_SERVE_PORT=${GATEWAY_LISTEN_PORT1} +ACTION=${1:-} +BUILD_DIR=${2:-} +if [ "$#" -gt 2 ]; then + usage >&2 + exit 1 +fi + +case "$ACTION" in + image|guest) + ;; + help|-h|--help) + usage + exit 0 + ;; + *) + if [ -n "$ACTION" ]; then + echo "Invalid action: $ACTION" >&2 fi - AGENT_PORT=8090 - else - mv build-config.sh.tpl $CONFIG_FILE - echo "Config file $CONFIG_FILE created, please edit it to configure the build" + usage >&2 exit 1 - fi -} - -# Step 1: build binaries -build_host() { - echo "Building binaries" - (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) - for bin in dstack-gateway dstack-kms dstack-vmm supervisor; do - cp "${RUST_BUILD_DIR}/release/${bin}" ".${bin}.new" - mv -f ".${bin}.new" "./${bin}" - done -} - -# Step 2: build guest images -build_guest() { - echo "Building guest images" - if [ -z "$BBPATH" ]; then - source $SCRIPT_DIR/dev-setup $1 - fi - make -C $META_DIR dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} -} - -# Step 4: generate config files - -build_cfg() { - echo "Building config files" - if [ -f "gateway.toml" ]; then - echo "Reading existing WireGuard key from gateway.toml" - GATEWAY_WG_KEY=$(awk ' - /^\s*private_key\s*=/ { - # Remove leading whitespace and "private_key =" - gsub(/^\s*private_key\s*=\s*/, "") - # Remove quotes (both single and double) - gsub(/^["'"'"']|["'"'"']$/, "") - # Remove trailing whitespace and comments - gsub(/\s*(#.*)?$/, "") - if (length($0) > 0) { - print $0 - exit - } - } - ' gateway.toml) + ;; +esac - if [ -z "$GATEWAY_WG_KEY" ]; then - echo "Error: Could not read WireGuard key from existing gateway.toml" +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +DIST_DIR=${DIST_DIR:-"$PWD/images"} +FLAVORS=${FLAVORS:-prod} +read -r -a FLAVOR_LIST <<< "$FLAVORS" +if [ "${#FLAVOR_LIST[@]}" -eq 0 ]; then + echo "Error: at least one Yocto image flavor is required" >&2 + exit 1 +fi +for flavor in "${FLAVOR_LIST[@]}"; do + case "$flavor" in + prod|dev) + ;; + *) + echo "Error: unsupported Yocto image flavor: $flavor" >&2 exit 1 - fi - else - echo "Generating new WireGuard key" - GATEWAY_WG_KEY=$(wg genkey) - fi - - GATEWAY_WG_PUBKEY=$(echo $GATEWAY_WG_KEY | wg pubkey) - # kms - cat <kms.toml -log_level = "info" - -[rpc] -address = "127.0.0.1" -port = $KMS_RPC_LISTEN_PORT - -[rpc.tls] -key = "$CERTS_DIR/rpc.key" -certs = "$CERTS_DIR/rpc.crt" - -[rpc.tls.mutual] -ca_certs = "$CERTS_DIR/tmp-ca.crt" -mandatory = false - -[core] -cert_dir = "$CERTS_DIR" - -[core.gpu] -enabled = $VMM_ENABLE_GPU - -[core.auth_api] -type = "dev" - -[core.onboard] -quote_enabled = false -address = "127.0.0.1" -port = $KMS_RPC_LISTEN_PORT -auto_bootstrap_domain = "$KMS_DOMAIN" - -[core.image] -verify = false -EOF - - # dstack-gateway - cat <gateway.toml -log_level = "info" -address = "127.0.0.1" -port = $GATEWAY_RPC_LISTEN_PORT - -[tls] -key = "$CERTS_DIR/gateway-rpc.key" -certs = "$CERTS_DIR/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "$CERTS_DIR/gateway-ca.cert" -mandatory = false - -[core] -kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" -rpc_domain = "$GATEWAY_DOMAIN" -run_in_dstack = false - -[core.sync] -enabled = false - -[core.certbot] -enabled = $CERTBOT_ENABLED -# Path to the working directory -workdir = "$CERBOT_WORKDIR" -# ACME server URL -acme_url = "$ACME_URL" -# Cloudflare API token -cf_api_token = "$CF_API_TOKEN" -# Auto set CAA record -auto_set_caa = true -# Domain to issue certificates for -domain = "*.$GATEWAY_PUBLIC_DOMAIN" -# Check renewal interval -renew_interval = "30m" -# Number of days before expiration to trigger renewal -renew_days_before = "10d" -# Renew timeout -renew_timeout = "10m" - -[core.wg] -private_key = "$GATEWAY_WG_KEY" -public_key = "$GATEWAY_WG_PUBKEY" -listen_port = $GATEWAY_WG_LISTEN_PORT -ip = "$GATEWAY_WG_IP/24" -reserved_net = ["$GATEWAY_WG_IP/31"] -client_ip_range = "$GATEWAY_WG_IP/24" -config_path = "$RUN_DIR/wg.conf" -interface = "$GATEWAY_WG_INTERFACE" -endpoint = "10.0.2.2:$GATEWAY_WG_LISTEN_PORT" - -[core.proxy] -cert_chain = "$GATEWAY_CERT" -cert_key = "$GATEWAY_KEY" -base_domain = "$GATEWAY_PUBLIC_DOMAIN" -listen_addr = "$BIND_PUBLIC_IP" -listen_port = $GATEWAY_SERVE_PORT -agent_port = $AGENT_PORT -app_address_ns_prefix = "_tapp-address" -EOF - - # dstack-vmm config - cat <vmm.toml -log_level = "info" -address = "127.0.0.1" -port = $VMM_RPC_LISTEN_PORT -image_path = "$IMAGES_DIR" -run_path = "$RUN_DIR/vm" -kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" - -[cvm] -kms_urls = ["https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT"] -gateway_urls = ["https://$GATEWAY_DOMAIN:$GATEWAY_RPC_LISTEN_PORT"] -cid_start = $VMM_CID_POOL_START -cid_pool_size = $VMM_CID_POOL_SIZE -[cvm.port_mapping] -enabled = $VMM_PORT_MAPPING_ENABLED -address = "127.0.0.1" -range = [ - { protocol = "tcp", from = 1, to = 20000 }, - { protocol = "udp", from = 1, to = 20000 }, -] - -[gateway] -base_domain = "$GATEWAY_PUBLIC_DOMAIN" -port = $GATEWAY_SERVE_PORT -agent_port = $AGENT_PORT - -[host_api] -port = $VMM_VSOCK_LISTEN_PORT -EOF - - mkdir -p $RUN_DIR - mkdir -p $CERBOT_WORKDIR/backup/preinstalled -} - -download_image() { - local VERSION="" - local IS_DEV="" - - # Parse arguments to support both formats - if [[ "$1" == "-dev" ]]; then - IS_DEV=1 - VERSION=$2 - else - VERSION=$1 - fi - - echo "Downloading image $VERSION${IS_DEV:+ (dev)}" - - TAG=guest-os-v$VERSION - if [ x"$IS_DEV" = x"1" ]; then - BASENAME=dstack-cloud-dev-$VERSION - else - BASENAME=dstack-cloud-$VERSION - fi - URL=https://github.com/Dstack-TEE/dstack/releases/download/$TAG/$BASENAME.tar.gz - LEGACY_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v$VERSION/$BASENAME.tar.gz - if [ -d $IMAGES_DIR/$BASENAME ]; then - echo "Image already exists" - else - mkdir -p $IMAGES_DIR/$BASENAME.tmp - if ! curl -fL $URL -o $IMAGES_DIR/$BASENAME.tar.gz; then - echo "Falling back to the pre-monorepo release location" - curl -fL $LEGACY_URL -o $IMAGES_DIR/$BASENAME.tar.gz - fi - tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME.tmp - rm -f $IMAGES_DIR/$BASENAME.tar.gz - if [ -d $IMAGES_DIR/$BASENAME.tmp/$BASENAME ]; then - mv $IMAGES_DIR/$BASENAME.tmp/$BASENAME $IMAGES_DIR/$BASENAME - rm -rf $IMAGES_DIR/$BASENAME.tmp - else - mv $IMAGES_DIR/$BASENAME.tmp $IMAGES_DIR/$BASENAME - fi - fi -} - -usage() { - cat < [args] - -Actions: - host - Build host binaries only - image - Build guest images (backend interface) - guest - Alias for image - cfg - Generate configuration files only - dl - Download a specific image - hostcfg - Build host binaries and generate configuration files - all - Build everything (host, guest, and configuration) - help - Show this help - -Build a specific guest flavor (default builds production): set the FLAVORS env var. - flavor -> output image - prod -> dstack - dev -> dstack-dev - - Examples: - ./build.sh guest # build dstack (default) - FLAVORS=dev ./build.sh guest # build only dstack-dev - FLAVORS="prod dev" ./build.sh guest # build both flavors -EOF -} - -case $ACTION in -host) - build_host - ;; -image|guest) - build_guest $2 - ;; -cfg) - require_config - build_cfg - ;; -dl) - download_image $2 $3 - ;; -hostcfg) - require_config - build_host - build_cfg - ;; -all) - require_config - build_host - build_guest - build_cfg - ;; -help|-h|--help) - usage - ;; -*) - [ -n "$ACTION" ] && echo "Invalid action: $ACTION" >&2 - usage + ;; + esac +done + +if [ -z "${BBPATH:-}" ]; then + # Always pass a directory explicitly. A sourced script otherwise inherits + # this script's positional parameters and would mistake `image` for the + # build directory. + BUILD_DIR=${BUILD_DIR:-"$SCRIPT_DIR/bb-build"} + # shellcheck disable=SC1091 + source "$SCRIPT_DIR/dev-setup" "$BUILD_DIR" +elif [ -n "$BUILD_DIR" ] && [ "$(realpath "$BUILD_DIR")" != "$(realpath "$BBPATH")" ]; then + echo "Error: BBPATH is already set to $BBPATH, not requested build directory $BUILD_DIR" >&2 exit 1 - ;; -esac +fi + +cat <<'NOTE' +Note: the first docker-compose fetch contains hundreds of separately +checksummed Go modules. BitBake's 0-100% display resets for each module; this +is forward progress, not the same archive being downloaded repeatedly. +Interrupted downloads are retained and reused on the next run. +NOTE + +make -C "$SCRIPT_DIR" dist \ + DIST_DIR="$DIST_DIR" \ + BB_BUILD_DIR="$BBPATH" \ + FLAVORS="$FLAVORS" diff --git a/os/yocto/dev-setup b/os/yocto/dev-setup index f789c804e..31032a39e 100755 --- a/os/yocto/dev-setup +++ b/os/yocto/dev-setup @@ -1,9 +1,9 @@ #!/bin/bash shopt -s nullglob -if [ -n "$BASH_SOURCE" ]; then - THIS_SCRIPT=$BASH_SOURCE -elif [ -n "$ZSH_NAME" ]; then +if [ -n "${BASH_SOURCE[0]:-}" ]; then + THIS_SCRIPT=${BASH_SOURCE[0]} +elif [ -n "${ZSH_NAME:-}" ]; then THIS_SCRIPT=$0 else THIS_SCRIPT="$(pwd)/dev-setup" @@ -29,31 +29,33 @@ LAYERS="$THIS_DIR/deps/meta-yocto/meta-poky \ $THIS_DIR/deps/meta-security/meta-tpm \ $THIS_DIR/layers/meta-dstack" -if [ -z "$1" ]; then +if [ -z "${1:-}" ]; then BUILD_DIR=$THIS_DIR/bb-build else - BUILD_DIR=$(realpath "$1") + BUILD_DIR=$(realpath -m "$1") fi # Sync build config from the dstack layer (always overwrite) CONF_SRC=$THIS_DIR/layers/meta-dstack/conf mkdir -p "$BUILD_DIR/conf" cp -f "$CONF_SRC/local.conf" "$BUILD_DIR/conf/local.conf" -cp -rf "$CONF_SRC/multiconfig" "$BUILD_DIR/conf/" +rm -rf "$BUILD_DIR/conf/multiconfig" +cp -a "$CONF_SRC/multiconfig" "$BUILD_DIR/conf/" OE_INIT=$THIS_DIR/deps/openembedded-core/oe-init-build-env -for script in $THIS_DIR/setup.d/*.sh; do +for script in "$THIS_DIR"/setup.d/*.sh; do + # shellcheck disable=SC1090 source "$script" done -pushd "$BUILD_DIR" -BDIR="." TEMPLATECONF=$THIS_DIR/deps/openembedded-core/meta/conf/templates/default source $OE_INIT -popd +# shellcheck disable=SC2317 +pushd "$BUILD_DIR" >/dev/null || { return 1 2>/dev/null || exit 1; } +# shellcheck disable=SC1090 +BDIR="." TEMPLATECONF=$THIS_DIR/deps/openembedded-core/meta/conf/templates/default source "$OE_INIT" +# shellcheck disable=SC2317 +popd >/dev/null || { return 1 2>/dev/null || exit 1; } +# LAYERS is deliberately a space-separated list populated by setup.d hooks. +# shellcheck disable=SC2086 bitbake-layers add-layer $LAYERS - -# Add scripts/bin to PATH if not already present -if [[ ":$PATH:" != *":$THIS_DIR/scripts/bin:"* ]]; then - export PATH=$PATH:$THIS_DIR/scripts/bin -fi diff --git a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect index 1fdb6d2e6..7826547a1 100755 --- a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect +++ b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect @@ -8,18 +8,19 @@ # exit 0 -> hardware present (systemd runs the unit) # exit 1 -> hardware absent (systemd skips the unit, no failure) # -# Detection mirrors scripts/bin/enable_vfio_passthrough.sh, but reads PCI class -# codes from sysfs so it works before the nvidia driver is loaded and without -# relying on lspci / the pci.ids name database: +# Detection mirrors +# os/yocto/tools/vm-runner/enable-vfio-passthrough.sh, but reads PCI class codes +# from sysfs so it works before the nvidia driver is loaded and without relying +# on lspci / the pci.ids name database: # # GPU = NVIDIA (0x10de) VGA (0x0300xx) or 3D controller (0x0302xx) # NVSwitch = NVIDIA (0x10de) bridge (0x06xxxx, any bridge subclass) # # NVSwitch matches any NVIDIA bridge-class device (mirrors the `grep 'Bridge'` -# heuristic in scripts/bin/enable_vfio_passthrough.sh) rather than only the -# "Other bridge" subclass 0x0680, so it stays correct if a future NVSwitch -# enumerates under a different bridge subclass. This is safe here because these -# images run as TDX guests whose only NVIDIA-vendor devices are passed-through +# heuristic in os/yocto/tools/vm-runner/enable-vfio-passthrough.sh) rather than +# only the "Other bridge" subclass 0x0680, so it stays correct if a future +# NVSwitch enumerates under a different bridge subclass. This is safe here +# because these images run as TDX guests whose only NVIDIA-vendor devices are # GPUs (class 0x03xx) and NVSwitches (class 0x06xx). NVIDIA_VENDOR="0x10de" diff --git a/os/yocto/scripts/bin/dstack b/os/yocto/scripts/bin/dstack deleted file mode 120000 index 87fdc0b30..000000000 --- a/os/yocto/scripts/bin/dstack +++ /dev/null @@ -1 +0,0 @@ -dstack.py \ No newline at end of file diff --git a/os/yocto/tools/README.md b/os/yocto/tools/README.md new file mode 100644 index 000000000..82c787a73 --- /dev/null +++ b/os/yocto/tools/README.md @@ -0,0 +1,17 @@ +# Imported Yocto support tools + +These are auxiliary host-side tools retained from `meta-dstack`. They are not +part of the guest-OS backend interface; the supported backend entrypoint is +[`../build.sh`](../build.sh). + +- `dev-stack.sh` preserves the legacy all-in-one host/configuration helper. +- `fix-self-uid-map.sh` is an explicit host AppArmor workaround for affected + Ubuntu installations; it is not part of the image build. +- `vm-runner/` preserves the legacy direct-QEMU image runner and its GPU sample. + +Keeping these helpers under `tools/` prevents `dev-setup` from placing a second, +legacy command named `dstack` on `PATH`. Use the Rust `dstack` and `dstackup` +CLIs for supported deployment workflows. + +Invoke a retained helper by its explicit path, for example +`os/yocto/tools/dev-stack.sh --help`. diff --git a/os/yocto/tools/dev-stack.sh b/os/yocto/tools/dev-stack.sh new file mode 100755 index 000000000..46da266e9 --- /dev/null +++ b/os/yocto/tools/dev-stack.sh @@ -0,0 +1,398 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + +# Legacy all-in-one host development helper imported from meta-dstack. The +# actual Yocto backend entrypoint is ../build.sh; keep host/config/download +# actions here so the backend interface remains single-purpose. +TOOL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCRIPT_DIR=$(realpath "$TOOL_DIR/..") +ACTION=$1 + +META_DIR=$SCRIPT_DIR +DSTACK_DIR=$(realpath "$SCRIPT_DIR/../../dstack") +CERTS_DIR=$(pwd)/certs +IMAGES_DIR=$(pwd)/images +RUN_DIR=$(pwd)/run +RUST_BUILD_DIR=$(pwd)/rust-target +CERBOT_WORKDIR=$RUN_DIR/certbot +KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry +KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/ + +GATEWAY_CERT=${GATEWAY_CERT:-$CERTS_DIR/live/cert.pem} +GATEWAY_KEY=${GATEWAY_KEY:-$CERTS_DIR/live/key.pem} + +CONFIG_FILE=./build-config.sh + +check_config() { + local template_file=$1 + local config_file=$2 + + # extract all variables in template file + local variables=$(grep -oE '^\s*[A-Z_]+=' $template_file | sort) + + # check if each variable is set in config file + local var missing=0 + for var in $variables; do + if ! grep -qE "^\s*$var" $config_file; then + echo "Variable $var is not set in $config_file" + missing=1 + fi + done + if [ $missing -ne 0 ]; then + return 1 + fi + return 0 +} + +require_config() { + + # Base port for RPC services + BASE_PORT=$(($RANDOM % 1000 * 10 + 10000)) + CID_POOL_START=$(($RANDOM % 1000 * 1000 + 20000)) + SUBNET_INDEX=$(($RANDOM % 240 + 10)) + + cat <build-config.sh.tpl +# DNS domain of kms rpc and dstack-gateway rpc +# *.1022.dstack.org resolves to 10.0.2.2 which is the IP of the host system +# from CVMs point of view +KMS_DOMAIN=kms.1022.dstack.org +GATEWAY_DOMAIN=gateway.1022.dstack.org + +# CIDs allocated to VMs start from this number of type unsigned int32 +VMM_CID_POOL_START=$CID_POOL_START +# CID pool size +VMM_CID_POOL_SIZE=1000 + +VMM_RPC_LISTEN_PORT=$BASE_PORT +# Whether port mapping from host to CVM is allowed +VMM_PORT_MAPPING_ENABLED=true +# Host API configuration, type of uint32 +VMM_VSOCK_LISTEN_PORT=$BASE_PORT +# Whether to enable GPU support +VMM_ENABLE_GPU=false + +KMS_RPC_LISTEN_PORT=$(($BASE_PORT + 1)) +GATEWAY_RPC_LISTEN_PORT=$(($BASE_PORT + 2)) + +GATEWAY_WG_INTERFACE=dgw-$USER +GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3)) +GATEWAY_WG_IP=10.$SUBNET_INDEX.3.1 +GATEWAY_SERVE_PORT=$(($BASE_PORT + 4)) +GATEWAY_CERT=$CERBOT_WORKDIR/live/cert.pem +GATEWAY_KEY=$CERBOT_WORKDIR/live/key.pem + +BIND_PUBLIC_IP=0.0.0.0 + +GATEWAY_PUBLIC_DOMAIN= + +# for certbot +CERTBOT_ENABLED=false +CF_API_TOKEN= +ACME_URL=https://acme-staging-v02.api.letsencrypt.org/directory +EOF + if [ -f $CONFIG_FILE ]; then + source $CONFIG_FILE + # check if any variable in build-config.sh.tpl is not set in build-config.sh. + # This might occur if the build-config.sh is generated from and old repo. + check_config build-config.sh.tpl $CONFIG_FILE + if [ $? -ne 0 ]; then + exit 1 + fi + rm -f build-config.sh.tpl + + if [ -z "$GATEWAY_SERVE_PORT" ]; then + GATEWAY_SERVE_PORT=${GATEWAY_LISTEN_PORT1} + fi + AGENT_PORT=8090 + else + mv build-config.sh.tpl $CONFIG_FILE + echo "Config file $CONFIG_FILE created, please edit it to configure the build" + exit 1 + fi +} + +# Step 1: build binaries +build_host() { + echo "Building binaries" + (cd $DSTACK_DIR && cargo build --release --target-dir ${RUST_BUILD_DIR}) + for bin in dstack-gateway dstack-kms dstack-vmm supervisor; do + cp "${RUST_BUILD_DIR}/release/${bin}" ".${bin}.new" + mv -f ".${bin}.new" "./${bin}" + done +} + +# Step 2: build guest images +build_guest() { + echo "Building guest images" + if [ -z "$BBPATH" ]; then + source $SCRIPT_DIR/dev-setup $1 + fi + make -C $META_DIR dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} +} + +# Step 4: generate config files + +build_cfg() { + echo "Building config files" + if [ -f "gateway.toml" ]; then + echo "Reading existing WireGuard key from gateway.toml" + GATEWAY_WG_KEY=$(awk ' + /^\s*private_key\s*=/ { + # Remove leading whitespace and "private_key =" + gsub(/^\s*private_key\s*=\s*/, "") + # Remove quotes (both single and double) + gsub(/^["'"'"']|["'"'"']$/, "") + # Remove trailing whitespace and comments + gsub(/\s*(#.*)?$/, "") + if (length($0) > 0) { + print $0 + exit + } + } + ' gateway.toml) + + if [ -z "$GATEWAY_WG_KEY" ]; then + echo "Error: Could not read WireGuard key from existing gateway.toml" + exit 1 + fi + else + echo "Generating new WireGuard key" + GATEWAY_WG_KEY=$(wg genkey) + fi + + GATEWAY_WG_PUBKEY=$(echo $GATEWAY_WG_KEY | wg pubkey) + # kms + cat <kms.toml +log_level = "info" + +[rpc] +address = "127.0.0.1" +port = $KMS_RPC_LISTEN_PORT + +[rpc.tls] +key = "$CERTS_DIR/rpc.key" +certs = "$CERTS_DIR/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "$CERTS_DIR/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "$CERTS_DIR" + +[core.gpu] +enabled = $VMM_ENABLE_GPU + +[core.auth_api] +type = "dev" + +[core.onboard] +quote_enabled = false +address = "127.0.0.1" +port = $KMS_RPC_LISTEN_PORT +auto_bootstrap_domain = "$KMS_DOMAIN" + +[core.image] +verify = false +EOF + + # dstack-gateway + cat <gateway.toml +log_level = "info" +address = "127.0.0.1" +port = $GATEWAY_RPC_LISTEN_PORT + +[tls] +key = "$CERTS_DIR/gateway-rpc.key" +certs = "$CERTS_DIR/gateway-rpc.cert" + +[tls.mutual] +ca_certs = "$CERTS_DIR/gateway-ca.cert" +mandatory = false + +[core] +kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" +rpc_domain = "$GATEWAY_DOMAIN" +run_in_dstack = false + +[core.sync] +enabled = false + +[core.certbot] +enabled = $CERTBOT_ENABLED +# Path to the working directory +workdir = "$CERBOT_WORKDIR" +# ACME server URL +acme_url = "$ACME_URL" +# Cloudflare API token +cf_api_token = "$CF_API_TOKEN" +# Auto set CAA record +auto_set_caa = true +# Domain to issue certificates for +domain = "*.$GATEWAY_PUBLIC_DOMAIN" +# Check renewal interval +renew_interval = "30m" +# Number of days before expiration to trigger renewal +renew_days_before = "10d" +# Renew timeout +renew_timeout = "10m" + +[core.wg] +private_key = "$GATEWAY_WG_KEY" +public_key = "$GATEWAY_WG_PUBKEY" +listen_port = $GATEWAY_WG_LISTEN_PORT +ip = "$GATEWAY_WG_IP/24" +reserved_net = ["$GATEWAY_WG_IP/31"] +client_ip_range = "$GATEWAY_WG_IP/24" +config_path = "$RUN_DIR/wg.conf" +interface = "$GATEWAY_WG_INTERFACE" +endpoint = "10.0.2.2:$GATEWAY_WG_LISTEN_PORT" + +[core.proxy] +cert_chain = "$GATEWAY_CERT" +cert_key = "$GATEWAY_KEY" +base_domain = "$GATEWAY_PUBLIC_DOMAIN" +listen_addr = "$BIND_PUBLIC_IP" +listen_port = $GATEWAY_SERVE_PORT +agent_port = $AGENT_PORT +app_address_ns_prefix = "_tapp-address" +EOF + + # dstack-vmm config + cat <vmm.toml +log_level = "info" +address = "127.0.0.1" +port = $VMM_RPC_LISTEN_PORT +image_path = "$IMAGES_DIR" +run_path = "$RUN_DIR/vm" +kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" + +[cvm] +kms_urls = ["https://$KMS_DOMAIN:$KMS_RPC_LISTEN_PORT"] +gateway_urls = ["https://$GATEWAY_DOMAIN:$GATEWAY_RPC_LISTEN_PORT"] +cid_start = $VMM_CID_POOL_START +cid_pool_size = $VMM_CID_POOL_SIZE +[cvm.port_mapping] +enabled = $VMM_PORT_MAPPING_ENABLED +address = "127.0.0.1" +range = [ + { protocol = "tcp", from = 1, to = 20000 }, + { protocol = "udp", from = 1, to = 20000 }, +] + +[gateway] +base_domain = "$GATEWAY_PUBLIC_DOMAIN" +port = $GATEWAY_SERVE_PORT +agent_port = $AGENT_PORT + +[host_api] +port = $VMM_VSOCK_LISTEN_PORT +EOF + + mkdir -p $RUN_DIR + mkdir -p $CERBOT_WORKDIR/backup/preinstalled +} + +download_image() { + local VERSION="" + local IS_DEV="" + + # Parse arguments to support both formats + if [[ "$1" == "-dev" ]]; then + IS_DEV=1 + VERSION=$2 + else + VERSION=$1 + fi + + echo "Downloading image $VERSION${IS_DEV:+ (dev)}" + + TAG=guest-os-v$VERSION + if [ x"$IS_DEV" = x"1" ]; then + BASENAME=dstack-cloud-dev-$VERSION + else + BASENAME=dstack-cloud-$VERSION + fi + URL=https://github.com/Dstack-TEE/dstack/releases/download/$TAG/$BASENAME.tar.gz + LEGACY_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v$VERSION/$BASENAME.tar.gz + if [ -d $IMAGES_DIR/$BASENAME ]; then + echo "Image already exists" + else + mkdir -p $IMAGES_DIR/$BASENAME.tmp + if ! curl -fL $URL -o $IMAGES_DIR/$BASENAME.tar.gz; then + echo "Falling back to the pre-monorepo release location" + curl -fL $LEGACY_URL -o $IMAGES_DIR/$BASENAME.tar.gz + fi + tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME.tmp + rm -f $IMAGES_DIR/$BASENAME.tar.gz + if [ -d $IMAGES_DIR/$BASENAME.tmp/$BASENAME ]; then + mv $IMAGES_DIR/$BASENAME.tmp/$BASENAME $IMAGES_DIR/$BASENAME + rm -rf $IMAGES_DIR/$BASENAME.tmp + else + mv $IMAGES_DIR/$BASENAME.tmp $IMAGES_DIR/$BASENAME + fi + fi +} + +usage() { + cat < [args] + +Actions: + host - Build host binaries only + image - Build guest images (backend interface) + guest - Alias for image + cfg - Generate configuration files only + dl - Download a specific image + hostcfg - Build host binaries and generate configuration files + all - Build everything (host, guest, and configuration) + help - Show this help + +Build a specific guest flavor (default builds production): set the FLAVORS env var. + flavor -> output image + prod -> dstack + dev -> dstack-dev + + Examples: + $0 guest # build dstack (default) + FLAVORS=dev $0 guest # build only dstack-dev + FLAVORS="prod dev" $0 guest # build both flavors +EOF +} + +case $ACTION in +host) + build_host + ;; +image|guest) + build_guest $2 + ;; +cfg) + require_config + build_cfg + ;; +dl) + download_image $2 $3 + ;; +hostcfg) + require_config + build_host + build_cfg + ;; +all) + require_config + build_host + build_guest + build_cfg + ;; +help|-h|--help) + usage + ;; +*) + [ -n "$ACTION" ] && echo "Invalid action: $ACTION" >&2 + usage + exit 1 + ;; +esac diff --git a/os/yocto/scripts/fix_self_uid_map.sh b/os/yocto/tools/fix-self-uid-map.sh similarity index 57% rename from os/yocto/scripts/fix_self_uid_map.sh rename to os/yocto/tools/fix-self-uid-map.sh index be09997e5..226a599a3 100755 --- a/os/yocto/scripts/fix_self_uid_map.sh +++ b/os/yocto/tools/fix-self-uid-map.sh @@ -1,3 +1,7 @@ #!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + # BUG: https://bugs.launchpad.net/ubuntu/+source/apparmor/+bug/2056555 sudo apparmor_parser -R /etc/apparmor.d/unprivileged_userns diff --git a/os/yocto/tools/vm-runner/README.md b/os/yocto/tools/vm-runner/README.md new file mode 100644 index 000000000..ce742ba5d --- /dev/null +++ b/os/yocto/tools/vm-runner/README.md @@ -0,0 +1,15 @@ +# Legacy direct-QEMU runner + +This directory contains the imported direct-QEMU development runner. It is +retained for low-level image debugging; normal deployments should use +`dstack-vmm`, `dstack`, or `dstackup`. + +Run it explicitly instead of adding the directory to `PATH`: + +```bash +python3 os/yocto/tools/vm-runner/vm-runner.py --help +``` + +`enable-vfio-passthrough.sh` is the matching standalone host helper for NVIDIA +GPU/NVSwitch passthrough. The `samples/` directory contains the historical CUDA +notebook workload used with this runner. diff --git a/os/yocto/scripts/bin/enable_vfio_passthrough.sh b/os/yocto/tools/vm-runner/enable-vfio-passthrough.sh old mode 100644 new mode 100755 similarity index 94% rename from os/yocto/scripts/bin/enable_vfio_passthrough.sh rename to os/yocto/tools/vm-runner/enable-vfio-passthrough.sh index 941695ac8..f80b349aa --- a/os/yocto/scripts/bin/enable_vfio_passthrough.sh +++ b/os/yocto/tools/vm-runner/enable-vfio-passthrough.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - echo "==> Detecting NVIDIA GPUs and NVSwitches" mapfile -t GPU_BDFS < <( lspci -d 10de: -nn | grep '3D controller' | awk '{print $1}' diff --git a/os/yocto/scripts/bin/host_api.py b/os/yocto/tools/vm-runner/host_api.py similarity index 97% rename from os/yocto/scripts/bin/host_api.py rename to os/yocto/tools/vm-runner/host_api.py index 9c5df9a5c..e1e870df3 100644 --- a/os/yocto/scripts/bin/host_api.py +++ b/os/yocto/tools/vm-runner/host_api.py @@ -1,3 +1,5 @@ +"""Minimal host API used by the legacy direct-QEMU development runner.""" + import os import json import struct @@ -111,4 +113,3 @@ def handler(*args): server = HTTPServer(('localhost', 0), handler) chosen_port = server.server_port return server, chosen_port - diff --git a/os/yocto/scripts/samples/Dockerfile b/os/yocto/tools/vm-runner/samples/Dockerfile similarity index 100% rename from os/yocto/scripts/samples/Dockerfile rename to os/yocto/tools/vm-runner/samples/Dockerfile diff --git a/os/yocto/scripts/samples/docker-compose.yaml b/os/yocto/tools/vm-runner/samples/docker-compose.yaml similarity index 100% rename from os/yocto/scripts/samples/docker-compose.yaml rename to os/yocto/tools/vm-runner/samples/docker-compose.yaml diff --git a/os/yocto/scripts/bin/dstack.py b/os/yocto/tools/vm-runner/vm-runner.py similarity index 99% rename from os/yocto/scripts/bin/dstack.py rename to os/yocto/tools/vm-runner/vm-runner.py index 73c810d34..940a7b084 100755 --- a/os/yocto/scripts/bin/dstack.py +++ b/os/yocto/tools/vm-runner/vm-runner.py @@ -1,4 +1,9 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC +# +# SPDX-License-Identifier: BUSL-1.1 + +"""Legacy direct-QEMU development runner for images built by this backend.""" import argparse import json @@ -797,7 +802,7 @@ def write_to_sysfs(path, value): def main(): - parser = argparse.ArgumentParser(description='dstack VM Management Tool') + parser = argparse.ArgumentParser(description='Legacy direct-QEMU dstack VM runner') subparsers = parser.add_subparsers(dest='command', help='Commands') # Setup command diff --git a/sdk/run-tests.sh b/sdk/run-tests.sh index 8ab258f2a..a16e4a3c5 100755 --- a/sdk/run-tests.sh +++ b/sdk/run-tests.sh @@ -7,11 +7,13 @@ set -Eeuo pipefail -ROOT_DIR="$(pwd -P)" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" SIMULATOR_DIR="$ROOT_DIR/simulator" SIMULATOR_LOG="$SIMULATOR_DIR/dstack-simulator.log" DSTACK_SOCKET="$SIMULATOR_DIR/dstack.sock" TAPPD_SOCKET="$SIMULATOR_DIR/tappd.sock" +GUEST_SOCKET="$SIMULATOR_DIR/guest.sock" +EXTERNAL_SOCKET="$SIMULATOR_DIR/external.sock" SIMULATOR_PID="" cleanup() { @@ -19,6 +21,7 @@ cleanup() { kill "$SIMULATOR_PID" 2>/dev/null || true wait "$SIMULATOR_PID" 2>/dev/null || true fi + rm -f "$DSTACK_SOCKET" "$TAPPD_SOCKET" "$GUEST_SOCKET" "$EXTERNAL_SOCKET" } print_simulator_logs() { @@ -52,7 +55,12 @@ wait_for_socket() { trap 'print_simulator_logs' ERR trap cleanup EXIT INT TERM -rm -f "$DSTACK_SOCKET" "$TAPPD_SOCKET" "$SIMULATOR_LOG" +rm -f \ + "$DSTACK_SOCKET" \ + "$TAPPD_SOCKET" \ + "$GUEST_SOCKET" \ + "$EXTERNAL_SOCKET" \ + "$SIMULATOR_LOG" export DSTACK_SIMULATOR_ENDPOINT="$DSTACK_SOCKET" export TAPPD_SIMULATOR_ENDPOINT="$TAPPD_SOCKET" @@ -70,20 +78,20 @@ SIMULATOR_PID=$! wait_for_socket "$DSTACK_SOCKET" "dstack" wait_for_socket "$TAPPD_SOCKET" "tappd" -pushd rust/ +pushd "$ROOT_DIR/rust" cargo test -- --show-output cargo run --example tappd_client_usage cargo run --example dstack_client_usage cargo test -p dstack-sdk-types --test no_std_test --no-default-features popd -pushd go/ +pushd "$ROOT_DIR/go" go clean -testcache go test -v ./dstack DSTACK_SIMULATOR_ENDPOINT=$TAPPD_SIMULATOR_ENDPOINT go test -v ./tappd popd -pushd python/ +pushd "$ROOT_DIR/python" # Ensure PDM is installed if ! command -v pdm &> /dev/null; then echo "Installing PDM..." @@ -97,7 +105,7 @@ pdm run test pdm run check popd -pushd js/ +pushd "$ROOT_DIR/js" npm install npm run test -- --run popd diff --git a/sdk/simulator/.gitignore b/sdk/simulator/.gitignore index 1acd17bdb..fd1ce3bc3 100644 --- a/sdk/simulator/.gitignore +++ b/sdk/simulator/.gitignore @@ -1,3 +1,4 @@ dstack-simulator dstack-guest-agent *.lock +*.log diff --git a/sdk/simulator/build.sh b/sdk/simulator/build.sh index f3632fa98..8b81c0fd2 100755 --- a/sdk/simulator/build.sh +++ b/sdk/simulator/build.sh @@ -4,7 +4,13 @@ # # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" || exit -cargo build --release -p dstack-guest-agent-simulator -cp ../../dstack/target/release/dstack-simulator . +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cargo build \ + --manifest-path "$REPO_ROOT/dstack/Cargo.toml" \ + --release \ + -p dstack-guest-agent-simulator +cp "$REPO_ROOT/dstack/target/release/dstack-simulator" "$SCRIPT_DIR/" diff --git a/dstack/scripts/add-spdx-attribution.py b/tools/add-spdx-attribution.py similarity index 100% rename from dstack/scripts/add-spdx-attribution.py rename to tools/add-spdx-attribution.py From d7e0553c6925ad46bf9fa78b1a898582c0ac6020 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 18:19:36 -0700 Subject: [PATCH 429/435] fix: fetch nested Foundry dependencies --- .github/workflows/docker-build-check.yml | 2 +- .github/workflows/foundry-test.yml | 2 +- .github/workflows/kms-release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index 90cd645b5..25dcfc677 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -66,7 +66,7 @@ jobs: - name: Checkout KMS contract dependencies run: | - git submodule update --init --depth 1 -- \ + git submodule update --init --recursive --depth 1 -- \ dstack/kms/auth-eth/lib/forge-std \ dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades diff --git a/.github/workflows/foundry-test.yml b/.github/workflows/foundry-test.yml index c47ea0474..282d7056b 100644 --- a/.github/workflows/foundry-test.yml +++ b/.github/workflows/foundry-test.yml @@ -34,7 +34,7 @@ jobs: - name: Checkout contract dependencies working-directory: . run: | - git submodule update --init --depth 1 -- \ + git submodule update --init --recursive --depth 1 -- \ dstack/kms/auth-eth/lib/forge-std \ dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index e16df20e8..0b33c5f88 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -24,7 +24,7 @@ jobs: - name: Checkout contract dependencies run: | - git submodule update --init --depth 1 -- \ + git submodule update --init --recursive --depth 1 -- \ dstack/kms/auth-eth/lib/forge-std \ dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades From 1cf62f2de39815aea0981ed161d708b5fa8a8e40 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 18:34:20 -0700 Subject: [PATCH 430/435] fix: align CI Rust toolchain Keep the repository-wide Rust pin at the monorepo root so both the core workspace and SDK simulator inherit Rust 1.92. Update the CI action refs to the same toolchain and clean simulator sockets after core tests. --- .github/workflows/rust.yml | 3 +-- .github/workflows/sdk.yaml | 3 +-- dstack/run-tests.sh | 10 +++++++++- dstack/rust-toolchain.toml => rust-toolchain.toml | 0 4 files changed, 11 insertions(+), 5 deletions(-) rename dstack/rust-toolchain.toml => rust-toolchain.toml (100%) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 39231d377..a269d4c49 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,9 +23,8 @@ jobs: - uses: actions/checkout@v5 - name: Install Rust - uses: dtolnay/rust-toolchain@1.86 + uses: dtolnay/rust-toolchain@1.92.0 with: - toolchain: 1.92.0 components: clippy, rustfmt - name: Run Clippy diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 11222dbf0..b5f7ae11a 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -23,9 +23,8 @@ jobs: - uses: actions/checkout@v5 - name: Install Rust - uses: dtolnay/rust-toolchain@1.86 + uses: dtolnay/rust-toolchain@1.92.0 with: - toolchain: 1.92.0 components: clippy, rustfmt # This additional target is needed for wasm32 compatibility check. targets: wasm32-unknown-unknown, thumbv6m-none-eabi diff --git a/dstack/run-tests.sh b/dstack/run-tests.sh index 301abf56a..4fb8dbd94 100755 --- a/dstack/run-tests.sh +++ b/dstack/run-tests.sh @@ -12,6 +12,8 @@ SIMULATOR_DIR="$REPO_ROOT/sdk/simulator" SIMULATOR_LOG="$SIMULATOR_DIR/dstack-simulator.log" DSTACK_SOCKET="$SIMULATOR_DIR/dstack.sock" TAPPD_SOCKET="$SIMULATOR_DIR/tappd.sock" +GUEST_SOCKET="$SIMULATOR_DIR/guest.sock" +EXTERNAL_SOCKET="$SIMULATOR_DIR/external.sock" SIMULATOR_PID="" cleanup() { @@ -19,6 +21,7 @@ cleanup() { kill "$SIMULATOR_PID" 2>/dev/null || true wait "$SIMULATOR_PID" 2>/dev/null || true fi + rm -f "$DSTACK_SOCKET" "$TAPPD_SOCKET" "$GUEST_SOCKET" "$EXTERNAL_SOCKET" } print_simulator_logs() { @@ -52,7 +55,12 @@ wait_for_socket() { trap 'print_simulator_logs' ERR trap cleanup EXIT INT TERM -rm -f "$DSTACK_SOCKET" "$TAPPD_SOCKET" "$SIMULATOR_LOG" +rm -f \ + "$DSTACK_SOCKET" \ + "$TAPPD_SOCKET" \ + "$GUEST_SOCKET" \ + "$EXTERNAL_SOCKET" \ + "$SIMULATOR_LOG" ( cd "$SIMULATOR_DIR" ./build.sh diff --git a/dstack/rust-toolchain.toml b/rust-toolchain.toml similarity index 100% rename from dstack/rust-toolchain.toml rename to rust-toolchain.toml From cb51f440a10a5a56f44c90763746c34e2f9edfc2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 19:11:36 -0700 Subject: [PATCH 431/435] fix: route legacy guest OS releases --- .github/workflows/guest-os.yml | 10 +- README.md | 3 +- docs/amd-sev-snp-review-readiness.md | 207 ------------------ docs/amd-sev-snp.md | 151 +++++++++++++ docs/deployment.md | 2 +- docs/hardware-enablement.md | 3 + docs/onboarding.md | 7 +- docs/tutorials/guest-image-setup.md | 10 +- .../troubleshooting-dstack-installation.md | 8 +- dstack/crates/dstackup/src/image.rs | 100 +++++++-- dstack/kms/dstack-app/deploy-simple.sh | 2 +- dstack/scripts/bin/dstack-cloud | 33 ++- os/yocto/tools/dev-stack.sh | 33 ++- 13 files changed, 315 insertions(+), 254 deletions(-) delete mode 100644 docs/amd-sev-snp-review-readiness.md create mode 100644 docs/amd-sev-snp.md diff --git a/.github/workflows/guest-os.yml b/.github/workflows/guest-os.yml index 9f6c68525..dc53eb24d 100644 --- a/.github/workflows/guest-os.yml +++ b/.github/workflows/guest-os.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to create and release (e.g. guest-os-v0.5.9). Leave empty to build only.' + description: 'Tag to create and release (e.g. guest-os-v0.6.0). Leave empty to build only.' required: false default: '' type: string @@ -61,6 +61,14 @@ jobs: run: | echo "$TAG" | grep -Eq '^guest-os-v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' VERSION="${TAG#guest-os-v}" + python3 - "$VERSION" <<'PY' + import re + import sys + + match = re.match(r"^(\d+)\.(\d+)\.(\d+)", sys.argv[1]) + if not match or tuple(map(int, match.groups())) < (0, 6, 0): + raise SystemExit("guest OS versions below 0.6.0 belong in Dstack-TEE/meta-dstack") + PY BARE="os/yocto/repro-build/dist/dstack-${VERSION}.tar.gz" UKI="os/yocto/repro-build/dist/dstack-${VERSION}-uki.tar.gz" test -f "$BARE" diff --git a/README.md b/README.md index af93715eb..6cd58b4c2 100644 --- a/README.md +++ b/README.md @@ -144,9 +144,10 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use **For Operators** - [Hardware Enablement](./docs/hardware-enablement.md) - Prepare a TDX or AMD SEV-SNP host +- [AMD SEV-SNP](./docs/amd-sev-snp.md) - Image, attestation, and key-release requirements - [Self-hosted Quick Onboarding](./docs/onboarding.md) - First app on one host - [Build the Guest OS](./docs/building-guest-os.md) - Build and verify bootable images from source -- [Deployment](./docs/deployment.md) - Self-hosting on TDX hardware +- [Deployment](./docs/deployment.md) - Self-hosting on TDX or AMD SEV-SNP hardware - [On-Chain Governance](./docs/onchain-governance.md) - Smart contract authorization - [Gateway](./docs/dstack-gateway.md) - Gateway configuration diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md deleted file mode 100644 index 393c5b20e..000000000 --- a/docs/amd-sev-snp-review-readiness.md +++ /dev/null @@ -1,207 +0,0 @@ -# AMD SEV-SNP Review Readiness - -This branch adds AMD SEV-SNP support and now includes a controlled, explicitly opt-in KMS key/cert release gate for SNP. - -## Current review boundary - -Implemented and intended for review: - -- AMD SEV-SNP evidence plumbing in the v1 attestation format. -- SNP report verification with AMD Milan/Genoa/Turin ARK/ASK/VCEK chain verification (built-in ARK/ASK roots per product; Bergamo/Siena parts are canonicalized under the Genoa KDS endpoint). -- Report-data challenge binding and fail-closed report policy checks. -- SNP launch-measurement recomputation from OVMF/kernel/initrd/cmdline inputs. -- KMS SNP `BootInfo` construction from verified report measurement, chip id, launch inputs, TCB status, and advisory ids. -- Auth-policy evaluation through the existing KMS auth flow. -- Controlled SNP key/cert release guarded by both external auth policy and local KMS config. -- VMM-provided SNP launch inputs in `.sys-config.json` so KMS self/app auth can recompute the same launch measurement used by QEMU. -- Onboarding attestation-info reporting for SNP identity fields. -- VMM SNP launch path, selected either by host auto-detection (`/proc/cpuinfo` `sev_snp` CPU flag) or by an explicit `platform = "amd-sev-snp"` pin. - -Default posture: - -- SNP app key release, KMS/root/temp CA key release, and app certificate release are still disabled by default. -- Operators must explicitly set `core.sev_snp_key_release = true` before any SNP `BootInfo` can release sensitive material. -- The self-authorized `GetTempCaCert` path is gated per-RPC, not at startup: it runs `ensure_self_key_release_allowed` against the KMS's own self `BootInfo`. With the production default `enforce_self_authorization = true`, the KMS self-attests and any SNP self `BootInfo` must clear the same release gate as app requests. `enforce_self_authorization = false` is a dev/test-only escape hatch (it logs a startup warning, not a hard error); in that mode the self `BootInfo` is `None`, so the self-release gate is skipped — do not use it in production TEE deployments. -- Even with the local KMS gate enabled, the existing auth API must first allow the verified SNP `BootInfo` for the app/KMS identity. - -## Fail-closed policy summary - -- `platform` selects the guest TEE: omitted or the legacy `auto` value auto-detects the host TEE from `/proc/cpuinfo` (the `sev_snp` CPU flag selects AMD SEV-SNP; otherwise it falls back to TDX), and operators can pin `platform = "amd-sev-snp"` or `platform = "tdx"` to override detection. SNP key release stays fail-closed regardless of how the platform is selected: an auto-detected SNP launch still cannot release sensitive material until the SNP release gate below is explicitly enabled. -- SNP launch measurement is recomputed from the self-contained VMM launch inputs and compared to the hardware-verified report measurement. -- SNP `BootInfo.tcb_status` is verifier-derived from signed AMD SNP report TCB fields: - - `UpToDate` only when current/reported/committed/launch TCB versions all match. - - `OutOfDate` otherwise. -- SNP advisory ids are propagated from verifier output into `BootInfo`; currently this list is explicit and empty because the AMD report/VCEK evidence used here does not carry a direct advisory-list field. -- `auth-simple` defaults remain strict: only `UpToDate` is accepted and any advisory id is denied unless explicitly allowlisted. -- The local KMS release gate is intentionally only an operator opt-in switch: - - `core.sev_snp_key_release = false` by default. - - TCB/advisory policy is not duplicated in KMS config; it is decided by the auth API using the verified `BootInfo`. - -Example opt-in gate: - -```toml -[core] -sev_snp_key_release = true -``` - -Sensitive release surfaces using this gate: - -- `GetAppKey`: app disk/env/k256 key material. -- `GetKmsKey`: temp CA key plus root CA/k256 key material for authorized KMS transfer. -- `SignCert`: app certificate chain signing. -- `GetTempCaCert`: temp CA material for self-authorized KMS instances. - -## Live golden-vector proof - -The ignored live regression test cross-checks dstack's pure Rust SNP measurement recomputation against `sev-snp-measure` on the SNP-capable host. - -> Status: the captured vector below is **stale**. It predates the move of SNP app identity from the kernel cmdline into the MrConfigV3 `HOST_DATA` binding, so the recorded `sev_snp_measurement` no longer matches the current recomputation. It must be regenerated on an SNP host before relying on it as proof. The current end-to-end live evidence is the SNP E2E smoke section below, which exercises the updated HOST_DATA-bound path through real key release. - -Command: - -```bash -cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture -``` - -Last captured vector (STALE — regenerate before citing as proof): - -```text -DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_BEGIN -utc=2026-06-02T19:49:14Z -host=dedicated-m24-fork -uname=Linux dedicated-m24-fork 6.11.0-rc3-snp-host-85ef1ac03941 #2 SMP Sat May 3 11:42:34 EDT 2025 x86_64 GNU/Linux -sev_snp_measure=/usr/local/bin/sev-snp-measure -sev_snp_measure_version=sev-snp-measure 0.0.10 -ovmf_path=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd -ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a -kernel_fixture_sha256=3f73f96a321b35a4c5561b05cfa6e9b5c573159380d37abe76f9a8ebe113a72e -initrd_fixture_sha256=e8790816224329cd76675c2aba4e62e885b5a4e0ec056227da70e775191d6d56 -vcpus=2 -vcpu_type=EPYC-v4 -guest_features=0x1 -append=console=ttyS0 loglevel=7 -sev_snp_measurement=requires-refresh-after-mr-config-v3-host-data-binding -cargo_live_test=cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture -cargo_live_test_result=stale after SNP app identity moved from cmdline to HOST_DATA -DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_END -``` - -## Guest attestation proof - -A prior SNP guest smoke proof confirmed the guest kernel exposed SEV-SNP report support and could produce a report containing the expected challenge bytes. - -```text -Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP -SEV: SNP running at VMPL0. -sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0) -DSTACK_SEV_SNP_ATTESTATION_PROOF_BEGIN -source=configfs-tsm -report_size=1184 -report_data_offset=80 -report_contains_expected_report_data=true -DSTACK_SEV_SNP_ATTESTATION_PROOF_END -``` - -## Manual dstack E2E smoke status - -An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. The reusable version of that smoke is checked in at `dstack/test-scripts/snp-e2e-smoke.sh` for follow-up debugging on SNP hosts. - -That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: - -- `.sys-config.json` did not include the `sev_snp_measurement` launch input document needed by KMS SNP `BootInfo` recomputation. -- The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. -- The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. - -After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot and KMS self-bootstrap on the known-good remote host. Additional smoke/debug fixes made the host/KMS side reach the app-key boundary: - -- Minimal guest boot now keeps DNS usable when `systemd-resolved`/`chronyd` are unavailable early in smoke boots and detects `sev-guest` before trying the TDX guest module. -- SNP guests verify the SNP `HOST_DATA` value against the attached MrConfigV3 document instead of using TDX-only `mr_config_id`. -- Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. -- If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. -- KMS measurement recomputation now uses the image's original kernel cmdline for SNP launch measurement, while app identity is bound by MrConfigV3/HOST_DATA instead of appended cmdline fields. -- VMM now passes the image's split `measurement.snp.cbor` material plus per-launch SNP fields through the guest to KMS; KMS no longer needs a single locally configured `ovmf_path`, so different image/OVMF versions can be verified by their self-contained launch inputs. -- SNP `BootInfo.os_image_hash` is the unified image digest (`sha256(sha256sum.txt)`). The `measurement.snp.cbor` entry in `sha256sum.txt` commits to the cmdline, kernel/initrd hashes, and OVMF hash/sections while excluding per-deployment values like vCPU count/model and guest features. - -Latest sanitized remote smoke result with PR-built host binaries and a coherent `MACHINE = "sev-snp"` guest image: - -```text -remote_host=chris@173.234.27.162 -host_kernel=Linux 6.11.0-rc3-snp-host-85ef1ac03941 -qemu_version=10.0.2 -ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a -image=dstack-dev-0.6.0 -platform=amd-sev-snp -image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y -kms_guest=booted SNP Linux/userspace and started dstack-kms -kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready -kds_base_url=enabled for smoke via DSTACK_SNP_SMOKE_KDS_BASE_URL=https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 -strict_tcb_probe=denied_as_expected by auth API with tcb_status is not allowed -success_probe=GetTempCaCert HTTP 200; GetAppKey HTTP 200; SignCert HTTP 200; app container started -smoke_result=SNP E2E smoke success -no_secret_material_logged=true -``` - -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, auth-API strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_BASE_URL=https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`; the smoke writes this value to the top-level KMS `core.amd_kds_base_url` configuration. This is an AMD-KDS-compatible base URL; requests append relative KDS paths such as `/Milan/cert_chain` or `/Milan/?...`. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. - -### Fresh SNP host / image requirements - -The checked-in smoke is enough to reproduce the current boundary on a compatible SNP host, but reviewers should treat the guest image/kernel/userspace as part of the test matrix: - -- Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with QEMU 10.0.2, the SNP-capable OVMF above, and a coherent `dstack-dev-0.6.0` guest image built with `MACHINE = "sev-snp"`. -- Released images that do not carry PR #703 guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. -- A coherent PR #703 image must be built as an SNP image, not with `meta-dstack`'s default `tdx` machine. The default TDX build can emit a kernel without `CONFIG_AMD_MEM_ENCRYPT`, which fails before Linux serial output under SNP. -- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted SNP-capable kernels (`6.11.0-rc3-snp-host`, `6.9.0-rc7-snp-host`, and the `MACHINE = "sev-snp"` `6.18.24-dstack` kernel) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, command line, virtio wiring, or basic host SNP enablement. - -Practical implication for reviewers/testers on a fresh box: - -1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. -2. Build the PR binaries with `cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor -p dstack-kms`. -3. Run `dstack/test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_BASE_URL` to a trusted AMD-KDS-compatible mirror/cache base URL such as `https://mirror.example.com/vcek/v1` (or, for a path-prefix relay, `https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`) and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production auth policy should keep accepting only `UpToDate` and deny any advisory id unless explicitly allowlisted. -4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. Build the Yocto backend from the same monorepo revision, for example: - - ```bash - git clone https://github.com/Dstack-TEE/dstack.git - cd dstack - git fetch https://github.com/clawdbot-glitch003/dstack.git feat/amd-sev-snp-conversion - git checkout -B feat/amd-sev-snp-conversion FETCH_HEAD - git submodule update --init -- \ - os/yocto/deps/bitbake \ - os/yocto/deps/openembedded-core \ - os/yocto/deps/meta-yocto \ - os/yocto/deps/meta-confidential-compute \ - os/yocto/deps/meta-virtualization \ - os/yocto/deps/meta-openembedded \ - os/yocto/deps/meta-rust-bin \ - os/yocto/deps/meta-security - source os/yocto/dev-setup ./bb-build - sed -i 's/^MACHINE ??= .*/MACHINE = "sev-snp"/' ./bb-build/conf/local.conf - FLAVORS=dev make -C os/yocto dist DIST_DIR=$PWD/images BB_BUILD_DIR=$PWD/bb-build - # Use the resulting dstack-dev image directory with: - # DSTACK_SNP_SMOKE_IMAGE_NAME= - ``` - - Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. -5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. - -If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with AMD KDS `HTTP 429`, use the smoke KDS base URL hook above; if it fails with missing cert-chain/collateral without KDS base URL evidence, rebuild/use a coherent PR guest image rather than changing KMS release policy. - -## Validation commands - -Run locally for this review-ready staging branch: - -```bash -bash -n dstack/test-scripts/snp-e2e-smoke.sh -cargo fmt --manifest-path dstack/Cargo.toml --all -cargo test --manifest-path dstack/Cargo.toml -p dstack-kms --all-features -cargo test --manifest-path dstack/Cargo.toml -p dstack-attest --all-features -cargo test --manifest-path dstack/Cargo.toml -p dstack-vmm --all-features -cargo test --manifest-path dstack/Cargo.toml -p ra-rpc --all-features -cargo check --manifest-path dstack/Cargo.toml --workspace --all-features -cargo clippy --manifest-path dstack/Cargo.toml --workspace --all-features -- -D warnings --allow unused_variables -git diff --check -cd dstack/kms/auth-simple && bun install && bun run check -``` - -## Remaining production follow-up - -The release gate is controlled and production-oriented, but AMD advisory/revocation collateral is still limited by the evidence source available here: SNP reports/VCEKs do not directly carry an advisory list, so `advisory_ids` currently propagates as an explicit empty list. Future collateral fetchers can populate this field; auth policy should deny those advisories unless each one is explicitly allowlisted. diff --git a/docs/amd-sev-snp.md b/docs/amd-sev-snp.md new file mode 100644 index 000000000..a1320ce98 --- /dev/null +++ b/docs/amd-sev-snp.md @@ -0,0 +1,151 @@ +# AMD SEV-SNP Support + +This document describes how dstack uses AMD SEV-SNP on self-hosted bare-metal +systems. AMD SEV-SNP support is currently **experimental**; Intel TDX remains +the production bare-metal path. + +For platform firmware, kernel, QEMU, and OVMF preparation, start with +[Hardware Enablement](./hardware-enablement.md). This document covers the +dstack-specific image, installation, attestation, and key-release requirements. + +## Supported image line + +dstack OS 0.6.0 and later use one `dstack-` guest image for both Intel +TDX and AMD SEV-SNP. The unified Yocto machine includes both platform kernel +feature sets and detects the active TEE at runtime. Do not change the Yocto +`MACHINE` to a platform-specific value. + +Guest OS releases are split at the monorepo boundary: + +- versions below 0.6.0 are archived in + [`Dstack-TEE/meta-dstack`](https://github.com/Dstack-TEE/meta-dstack/releases) + under tags such as `v0.5.11`; +- versions 0.6.0 and later are in + [`Dstack-TEE/dstack`](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v) + under tags such as `guest-os-v0.6.0`. + +The 0.5.x images are legacy TDX images, not the current unified SEV-SNP image +line. Use a 0.6.0-or-later image for SEV-SNP. The image must include +`digest.txt`, `sha256sum.txt`, and the SNP measurement material. + +## Host requirements + +The host must provide: + +- an SEV-SNP-capable AMD processor and current platform firmware; +- SEV-SNP and the Reverse Map Table (RMP) enabled by the host kernel; +- `/dev/sev`; +- a QEMU and OVMF build with SEV-SNP support. + +After following the host platform's enablement procedure, check: + +```bash +test -e /dev/sev +sudo dmesg | grep -e SEV-SNP -e RMP +cat /sys/module/kvm_amd/parameters/sev_snp +``` + +The last command should print `Y`. `dstackup` checks `/dev/sev`, but that +preflight does not replace firmware, kernel, QEMU, or OVMF validation. + +## Install dstack on an SNP host + +Pull a current unified image and select the platform explicitly: + +```bash +VERSION=0.6.0 +sudo dstackup image pull --version "$VERSION" +sudo dstackup install --platform amd-sev-snp --image "dstack-$VERSION" +sudo dstackup status +``` + +The default `--platform auto` mode also selects SEV-SNP when the host CPU flags +advertise `sev_snp`. Explicit selection is preferable while commissioning a +host because it fails immediately when the required SNP device is absent. + +Unlike the TDX path, SEV-SNP does not use the local SGX key provider. + +## Attestation and image identity + +The guest collects an SNP attestation report through Linux configfs-tsm when +available, with `/dev/sev-guest` extended-report collection as the fallback. +See [Native TEE Interfaces](./native-tee-interfaces.md) before exposing either +kernel interface directly to an application container. + +Verification is fail-closed and includes: + +1. the AMD ARK/ASK/VCEK certificate chain and report signature; +2. the requested `REPORT_DATA` challenge binding and SNP policy fields; +3. the launch `MEASUREMENT` recomputed from the VMM's firmware, kernel, + initramfs, command line, and launch inputs; +4. the MrConfigV3 `HOST_DATA` application-identity binding; +5. the unified OS image identity, `sha256(sha256sum.txt)`, which must match + `digest.txt` and the SNP measurement document. + +The verifier supports the AMD Milan, Genoa, and Turin KDS product families. +Bergamo and Siena are handled through AMD's canonical Genoa KDS product path. + +`BootInfo.tcb_status` is `UpToDate` only when the current, reported, committed, +and launch TCB versions agree; otherwise it is `OutOfDate`. Authorization +policy should remain strict for non-up-to-date TCB values. The verifier +currently reports an explicit empty advisory-ID list because the SNP report +and VCEK evidence do not directly carry an advisory list. + +## KMS key-release policy + +SEV-SNP key and certificate release has two independent gates: + +1. the external KMS authorization policy must accept the verified `BootInfo`; +2. the local KMS operator must explicitly enable SNP release. + +The local gate is disabled by default: + +```toml +[core] +sev_snp_key_release = false +``` + +After the host, image, attestation, and external authorization policy have been +validated, enable it deliberately in the KMS configuration: + +```toml +[core] +sev_snp_key_release = true +``` + +This gate covers application keys, KMS key transfer, application certificate +signing, and self-authorized temporary CA material. Enabling it does not bypass +the external authorization decision. Keep `enforce_self_authorization = true` +for production TEE deployments. + +## AMD KDS collateral + +The verifier obtains AMD certificate collateral from the built-in AMD KDS URL +when the attestation evidence does not already contain the required chain. An +operator can set an AMD-KDS-compatible mirror or cache: + +```toml +[core] +amd_kds_base_url = "https://mirror.example.com/vcek/v1" +``` + +Leave the value empty to use the built-in default. A custom endpoint is part of +the verification trust and availability boundary: use a controlled mirror, +preserve TLS validation, and do not make verification succeed without valid +AMD signatures. + +## Troubleshooting + +- **`/dev/sev` is missing:** finish host firmware/kernel enablement before + running `dstackup install`. +- **The guest resets before Linux starts:** verify that the selected QEMU, + OVMF, and unified dstack OS image all support SNP. Do not debug KMS policy + until the guest boots reliably. +- **Image identity files are missing:** use a 0.6.0-or-later unified image. + `dstackup install` rejects an SNP image without `digest.txt`. +- **KDS requests fail:** check host time, DNS, outbound HTTPS, and KDS or mirror + availability. Do not disable certificate or signature verification. +- **Attestation succeeds but key release fails:** check both the external auth + response and `core.sev_snp_key_release`; either gate can deny the request. +- **TCB is `OutOfDate`:** update platform firmware and re-evaluate the reported, + committed, current, and launch TCB versions before changing auth policy. diff --git a/docs/deployment.md b/docs/deployment.md index e8cd9af9e..be439e30a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -190,7 +190,7 @@ AUTH_WEBHOOK_URL=http://your-auth-server:3001 KMS_RPC_ADDR=0.0.0.0:9201 GUEST_AGENT_ADDR=127.0.0.1:9205 OS_IMAGE=dstack-0.5.5 -IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v0.5.5/dstack-0.5.5.tar.gz +IMAGE_DOWNLOAD_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.5/dstack-0.5.5.tar.gz ``` Then run: diff --git a/docs/hardware-enablement.md b/docs/hardware-enablement.md index 46d6e7e76..fb5030d47 100644 --- a/docs/hardware-enablement.md +++ b/docs/hardware-enablement.md @@ -33,6 +33,9 @@ Do not install a generic QEMU package as a substitute for TDX host setup. Use th Use your vendor or distribution's SEV-SNP enablement path. The [AMDSEV project](https://github.com/AMDESE/AMDSEV) documents CPU, BIOS, firmware, kernel, QEMU, OVMF, and verification requirements for SEV-SNP hosts. Confidential Containers also keeps platform setup separate from its [quickstart](https://github.com/confidential-containers/documentation/blob/main/quickstart.md) and points SEV users to AMD host preparation from its [SEV guide](https://github.com/confidential-containers/documentation/blob/main/guides/sev.md). +After the host is enabled, follow [AMD SEV-SNP Support](./amd-sev-snp.md) for +the dstack guest image, installation, attestation, and KMS release policy. + For dstack, the host must have: - AMD SEV-SNP enabled in firmware and the host OS. diff --git a/docs/onboarding.md b/docs/onboarding.md index 209862051..48df63c11 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -12,7 +12,10 @@ sudo dstack deploy \ curl http://127.0.0.1:8080/ ``` -AMD SEV-SNP hosts use the same `dstackup` and `dstack` commands after you provide a guest image that contains the image digest (`digest.txt`). +AMD SEV-SNP hosts use the same `dstackup` and `dstack` commands after you +provide a guest image that contains the image digest (`digest.txt`); see +[AMD SEV-SNP Support](./amd-sev-snp.md) for the experimental platform's image, +attestation, and KMS requirements. The default onboarding flow uses a published image. To build or customize the guest OS first, follow [Build the dstack guest OS](./building-guest-os.md), then @@ -88,7 +91,7 @@ Run: sudo dstackup install ``` -`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest unified image from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. Current images include NVIDIA support conditionally and work on CPU-only hosts too. Pre-monorepo `meta-dstack` releases remain a fallback for pinned older versions. +`dstackup install` auto-detects TDX or AMD SEV-SNP. If no local guest image exists, it downloads the latest unified image from [dstack guest-OS releases](https://github.com/Dstack-TEE/dstack/releases?q=guest-os-v), requires the release SHA-256 digest by default, verifies the tarball, stages the unpack, and only then adopts the image. Current images include NVIDIA support conditionally and work on CPU-only hosts too. Pinned versions below 0.6.0 are read directly from the archived [`meta-dstack` releases](https://github.com/Dstack-TEE/meta-dstack/releases); versions 0.6.0 and later come from this repository. On TDX, `dstackup install` starts the SGX key provider automatically from `/usr/local/share/dstack/key-provider-build`. To use a different provider, pass one of: diff --git a/docs/tutorials/guest-image-setup.md b/docs/tutorials/guest-image-setup.md index 4f141b3a5..555b1ef1f 100644 --- a/docs/tutorials/guest-image-setup.md +++ b/docs/tutorials/guest-image-setup.md @@ -79,7 +79,13 @@ echo "Installing guest images for version: $DSTACK_VERSION" # Download the image archive cd /tmp -wget https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz +IFS=. read -r DSTACK_MAJOR DSTACK_MINOR _ <<< "$DSTACK_VERSION" +if (( DSTACK_MAJOR == 0 && DSTACK_MINOR < 6 )); then + IMAGE_URL="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/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz" +else + IMAGE_URL="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/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz" +fi +wget "$IMAGE_URL" ``` Verify the download: @@ -294,7 +300,7 @@ You can have multiple image versions installed simultaneously: ```bash # Download additional version DSTACK_VERSION="0.5.3" -wget https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz +wget https://github.com/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz # Extract to images directory (tarball already contains dstack-X.Y.Z/ folder) sudo tar -xvf dstack-${DSTACK_VERSION}.tar.gz -C /var/lib/dstack/images/ diff --git a/docs/tutorials/troubleshooting-dstack-installation.md b/docs/tutorials/troubleshooting-dstack-installation.md index c199d930d..c75198c1e 100644 --- a/docs/tutorials/troubleshooting-dstack-installation.md +++ b/docs/tutorials/troubleshooting-dstack-installation.md @@ -347,8 +347,14 @@ Try alternative download methods: ```bash # Using curl instead of wget +IFS=. read -r DSTACK_MAJOR DSTACK_MINOR _ <<< "$DSTACK_VERSION" +if (( DSTACK_MAJOR == 0 && DSTACK_MINOR < 6 )); then + IMAGE_URL="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/Dstack-TEE/meta-dstack/releases/download/v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz" +else + IMAGE_URL="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/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz" +fi curl -L -o dstack-${DSTACK_VERSION}.tar.gz \ - https://github.com/Dstack-TEE/dstack/releases/download/guest-os-v${DSTACK_VERSION}/dstack-${DSTACK_VERSION}.tar.gz + "$IMAGE_URL" ``` ### Image metadata missing diff --git a/dstack/crates/dstackup/src/image.rs b/dstack/crates/dstackup/src/image.rs index 4aa03c762..1874d31b9 100644 --- a/dstack/crates/dstackup/src/image.rs +++ b/dstack/crates/dstackup/src/image.rs @@ -27,6 +27,8 @@ const REPO: &str = "Dstack-TEE/dstack"; const LEGACY_REPO: &str = "Dstack-TEE/meta-dstack"; const RELEASE_TAG_PREFIX: &str = "guest-os-v"; pub(crate) const RELEASES_URL: &str = "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/Dstack-TEE/dstack/releases?q=guest-os-v"; +const LEGACY_RELEASES_URL: &str = "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/Dstack-TEE/meta-dstack/releases"; +const MONOREPO_GUEST_OS_MIN_VERSION: (u64, u64, u64) = (0, 6, 0); /// the single rule for where images live: `--image-path` if given, else the /// image directory from the install layout. `install` and every image subcommand resolve through @@ -593,27 +595,18 @@ fn missing_named_image_message(image_dir: &str, name: &str) -> String { /// Get the latest (or a tagged) guest-OS release from the GitHub API. /// -/// The legacy repository remains a read-only fallback so pinned deployments -/// keep working across the monorepo migration. +/// Versions before 0.6.0 were released from `meta-dstack`; 0.6.0 and later are +/// released from this monorepo. Do not probe the new repository first for old +/// versions: the version boundary is authoritative and avoids redundant or +/// misleading requests. async fn fetch_release(version: Option<&str>) -> Result { let client = reqwest::Client::new(); if let Some(version) = version { - let version = version - .trim_start_matches(RELEASE_TAG_PREFIX) - .trim_start_matches('v'); - let primary_url = format!( - "https://api.github.com/repos/{REPO}/releases/tags/{RELEASE_TAG_PREFIX}{version}" - ); - if let Some(release) = fetch_tagged_release(&client, &primary_url).await? { - return Ok(release); - } - - let legacy_url = - format!("https://api.github.com/repos/{LEGACY_REPO}/releases/tags/v{version}"); - return fetch_tagged_release(&client, &legacy_url) + let (version, url, releases_url) = tagged_release_location(version)?; + return fetch_tagged_release(&client, &url, releases_url) .await? .with_context(|| { - format!("guest-OS version {version} was not found; check {RELEASES_URL}") + format!("guest-OS version {version} was not found; check {releases_url}") }); } @@ -638,12 +631,51 @@ async fn fetch_release(version: Option<&str>) -> Result { } let legacy_url = format!("https://api.github.com/repos/{LEGACY_REPO}/releases/latest"); - fetch_tagged_release(&client, &legacy_url) + fetch_tagged_release(&client, &legacy_url, LEGACY_RELEASES_URL) .await? .with_context(|| format!("no guest-OS release found; check {RELEASES_URL}")) } -async fn fetch_tagged_release(client: &reqwest::Client, url: &str) -> Result> { +fn tagged_release_location(version: &str) -> Result<(String, String, &'static str)> { + let version = version + .trim_start_matches(RELEASE_TAG_PREFIX) + .trim_start_matches('v'); + let core = numeric_version_core(version)?; + let (repo, tag_prefix, releases_url) = if core < MONOREPO_GUEST_OS_MIN_VERSION { + (LEGACY_REPO, "v", LEGACY_RELEASES_URL) + } else { + (REPO, RELEASE_TAG_PREFIX, RELEASES_URL) + }; + Ok(( + version.to_string(), + format!("https://api.github.com/repos/{repo}/releases/tags/{tag_prefix}{version}"), + releases_url, + )) +} + +fn numeric_version_core(version: &str) -> Result<(u64, u64, u64)> { + let mut parts = version.split('.'); + let major = parts.next().unwrap_or_default(); + let minor = parts.next().unwrap_or_default(); + let patch = parts.next().unwrap_or_default(); + let patch = patch.split_once('-').map_or(patch, |(numeric, _)| numeric); + if major.is_empty() + || minor.is_empty() + || patch.is_empty() + || !major.chars().all(|c| c.is_ascii_digit()) + || !minor.chars().all(|c| c.is_ascii_digit()) + || !patch.chars().all(|c| c.is_ascii_digit()) + { + bail!("invalid guest-OS version {version:?}; expected MAJOR.MINOR.PATCH"); + } + Ok((major.parse()?, minor.parse()?, patch.parse()?)) +} + +async fn fetch_tagged_release( + client: &reqwest::Client, + url: &str, + releases_url: &str, +) -> Result> { let response = client .get(url) .header("user-agent", "dstackup") @@ -657,7 +689,7 @@ async fn fetch_tagged_release(client: &reqwest::Client, url: &str) -> Result str: + """Return the UKI release URL for a versioned guest image.""" + match = re.fullmatch( + r'(\d+)\.(\d+)\.(\d+)(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?', + version, + ) + if not match: + raise ValueError(f"Invalid guest OS version: {version}") + core = tuple(int(part) for part in match.groups()) + if core < MONOREPO_GUEST_OS_MIN_VERSION: + return ( + "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/Dstack-TEE/meta-dstack/releases/download/" + f"v{version}/{os_image}-uki.tar.gz" + ) + return ( + "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/Dstack-TEE/dstack/releases/download/" + f"guest-os-v{version}/{os_image}-uki.tar.gz" + ) @dataclass @@ -1008,7 +1030,6 @@ class CloudDeploymentManager: target_dir.mkdir(parents=True, exist_ok=True) # Check if os_image is an absolute URL - fallback_url = None if os_image.startswith("http://") or os_image.startswith("https://"): download_url = os_image # Derive image name from URL filename @@ -1023,7 +1044,6 @@ class CloudDeploymentManager: else: # Extract the release version from the unified image name (or a # legacy flavor name), including an optional prerelease suffix. - import re version_match = re.search( r'-(\d+\.\d+\.\d+(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?)$', os_image, @@ -1033,8 +1053,7 @@ class CloudDeploymentManager: logger.error("Expected format: dstack- (or a legacy dstack--)") return version = version_match.group(1) - download_url = f"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/Dstack-TEE/dstack/releases/download/guest-os-v{version}/{os_image}-uki.tar.gz" - fallback_url = f"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/Dstack-TEE/meta-dstack/releases/download/v{version}/{os_image}-uki.tar.gz" + download_url = guest_os_release_url(os_image, version) download_tar = target_dir / f"{os_image}-uki.tar.gz" if download_tar.exists(): @@ -1053,12 +1072,6 @@ class CloudDeploymentManager: ["curl", "-fL", "-o", str(download_tar), download_url], check=False ) - if result.returncode != 0 and fallback_url: - logger.warning("Image not found in the monorepo release; trying the legacy release") - result = subprocess.run( - ["curl", "-fL", "-o", str(download_tar), fallback_url], - check=False - ) result.check_returncode() logger.info(f"Successfully downloaded to {download_tar}") except subprocess.CalledProcessError as e: diff --git a/os/yocto/tools/dev-stack.sh b/os/yocto/tools/dev-stack.sh index 46da266e9..1f4b1f982 100755 --- a/os/yocto/tools/dev-stack.sh +++ b/os/yocto/tools/dev-stack.sh @@ -295,6 +295,24 @@ EOF mkdir -p $CERBOT_WORKDIR/backup/preinstalled } +guest_os_release_url() { + local VERSION=$1 + local BASENAME=$2 + + if [[ ! "$VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-.][0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid guest OS version: $VERSION" >&2 + return 1 + fi + + local MAJOR=$((10#${BASH_REMATCH[1]})) + local MINOR=$((10#${BASH_REMATCH[2]})) + if (( MAJOR == 0 && MINOR < 6 )); then + echo "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/Dstack-TEE/meta-dstack/releases/download/v$VERSION/$BASENAME.tar.gz" + else + echo "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/Dstack-TEE/dstack/releases/download/guest-os-v$VERSION/$BASENAME.tar.gz" + fi +} + download_image() { local VERSION="" local IS_DEV="" @@ -309,22 +327,19 @@ download_image() { echo "Downloading image $VERSION${IS_DEV:+ (dev)}" - TAG=guest-os-v$VERSION if [ x"$IS_DEV" = x"1" ]; then - BASENAME=dstack-cloud-dev-$VERSION + BASENAME=dstack-dev-$VERSION else - BASENAME=dstack-cloud-$VERSION + BASENAME=dstack-$VERSION + fi + if ! URL=$(guest_os_release_url "$VERSION" "$BASENAME"); then + return 1 fi - URL=https://github.com/Dstack-TEE/dstack/releases/download/$TAG/$BASENAME.tar.gz - LEGACY_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v$VERSION/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME ]; then echo "Image already exists" else mkdir -p $IMAGES_DIR/$BASENAME.tmp - if ! curl -fL $URL -o $IMAGES_DIR/$BASENAME.tar.gz; then - echo "Falling back to the pre-monorepo release location" - curl -fL $LEGACY_URL -o $IMAGES_DIR/$BASENAME.tar.gz - fi + curl -fL "$URL" -o $IMAGES_DIR/$BASENAME.tar.gz tar -xvf $IMAGES_DIR/$BASENAME.tar.gz -C $IMAGES_DIR/$BASENAME.tmp rm -f $IMAGES_DIR/$BASENAME.tar.gz if [ -d $IMAGES_DIR/$BASENAME.tmp/$BASENAME ]; then From 13e382afe5930b390379c1fa4e9d733f51932ec9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 19:26:40 -0700 Subject: [PATCH 432/435] fix: avoid hard-coded key false positive --- dstack/sodiumbox/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dstack/sodiumbox/src/lib.rs b/dstack/sodiumbox/src/lib.rs index 8375776c6..26f6ac827 100644 --- a/dstack/sodiumbox/src/lib.rs +++ b/dstack/sodiumbox/src/lib.rs @@ -59,9 +59,7 @@ fn derive_nonce(ephemeral_pk: &[u8], recipient_pk: &[u8]) -> Result [u8; 32] { let hsalsa_nonce = [0u8; 16]; let key = salsa20::hsalsa::(shared_secret.into(), &hsalsa_nonce.into()); - let mut result = [0u8; 32]; - result.copy_from_slice(key.as_slice()); - result + key.into() } /// Creates a sealed box for a message using a recipient's public key. From 1b47d904ed742a433957f6aef82490eddb9ea285 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 19:58:09 -0700 Subject: [PATCH 433/435] refactor: clarify docs and tool ownership --- CONTRIBUTING.md | 21 +++++++ README.md | 1 + REUSE.toml | 9 +++ attestation.md => docs/attestation-tdx.md | 26 ++++---- docs/verification.md | 4 ++ dstack/Makefile | 22 ------- dstack/kms/README.md | 2 +- dstack/run.sh | 61 ------------------- os/README.md | 2 +- os/yocto/README.md | 9 +-- .../nvidia/files/nvidia-gpu-detect | 4 +- os/yocto/tools/README.md | 19 +++--- prek.toml | 17 +++--- tools/README.md | 13 ++++ tools/meta-dstack/README.md | 14 +++++ .../tools => tools/meta-dstack}/dev-stack.sh | 16 ++--- .../meta-dstack}/vm-runner/README.md | 2 +- .../vm-runner/enable-vfio-passthrough.sh | 0 .../meta-dstack}/vm-runner/host_api.py | 0 .../meta-dstack}/vm-runner/samples/Dockerfile | 0 .../vm-runner/samples/docker-compose.yaml | 0 .../meta-dstack}/vm-runner/vm-runner.py | 2 +- 22 files changed, 112 insertions(+), 132 deletions(-) rename attestation.md => docs/attestation-tdx.md (82%) delete mode 100644 dstack/Makefile delete mode 100755 dstack/run.sh create mode 100644 tools/README.md create mode 100644 tools/meta-dstack/README.md rename {os/yocto/tools => tools/meta-dstack}/dev-stack.sh (96%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/README.md (89%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/enable-vfio-passthrough.sh (100%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/host_api.py (100%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/samples/Dockerfile (100%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/samples/docker-compose.yaml (100%) rename {os/yocto/tools => tools/meta-dstack}/vm-runner/vm-runner.py (99%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5cea75114..8c5df8bd5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,27 @@ Thank you for your interest in contributing to this project! 5. Push to the branch (`git push origin feature/amazing-feature`) 6. Open a Pull Request +## Repository layout + +- `dstack/` contains the core Rust workspace and component-owned assets. Keep a + component's build, deployment, test, and API documentation next to that + component when it is not useful outside the component. +- `sdk/` contains the public language SDKs and simulator. +- `os/` contains the backend-neutral guest-OS contract, image assembly, common + guest payload, and backend implementations. A file under `os//` + must be specific to that backend. +- `docs/` contains repository-wide developer, operator, architecture, and + security documentation. Do not put a product guide at the repository root. +- `examples/` contains supported end-user examples. +- `tools/` contains cross-component developer/operator utilities. Put a script + here instead of under an OS backend when it also builds or configures the + host, deploys services, or operates on multiple components. +- `.github/` contains GitHub Actions workflows and workflow-only helpers. + +Use the narrowest owning directory. Fixture explanations and component +READMEs should stay with their fixtures/components; general guides should be +linked from the root README and live under `docs/`. + ## Commit Convention This project uses [Conventional Commits](https://www.conventionalcommits.org/). Please format your commit messages as: diff --git a/README.md b/README.md index 6cd58b4c2..2021b3005 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ Apps communicate with the guest agent via HTTP over `/var/run/dstack.sock`. Use **Reference** - [App Compose Format](./docs/normalized-app-compose.md) - Compose file specification +- [Intel TDX Attestation](./docs/attestation-tdx.md) - Measurement and runtime-event verification - [Native TEE Interfaces](./docs/native-tee-interfaces.md) - Advanced compatibility with Linux TEE devices and configfs-tsm - [VMM CLI Guide](./docs/vmm-cli-user-guide.md) - Command-line reference - [Design Decisions](./docs/design-and-hardening-decisions.md) - Architecture rationale diff --git a/REUSE.toml b/REUSE.toml index 65914d1d8..5fcc0cc59 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -126,6 +126,15 @@ SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" SPDX-License-Identifier = "BUSL-1.1" precedence = "override" +# Legacy tools imported from meta-dstack but intentionally kept outside the +# Yocto backend because they also manage host and deployment concerns. + +[[annotations]] +path = "tools/meta-dstack/**" +SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" +SPDX-License-Identifier = "BUSL-1.1" +precedence = "override" + # Artworks diff --git a/attestation.md b/docs/attestation-tdx.md similarity index 82% rename from attestation.md rename to docs/attestation-tdx.md index 22c4b72f3..f0f433423 100644 --- a/attestation.md +++ b/docs/attestation-tdx.md @@ -1,4 +1,4 @@ -# TEE Attestation Guide for dstack Applications +# Intel TDX Attestation Guide for dstack Applications This document outlines the process of verifying the authenticity and integrity of data produced by dstack Applications running within Intel TDX environments. @@ -33,26 +33,30 @@ RTMR3 differs as it contains runtime information like compose hash and instance ### 2.2. Determining expected MRs MRTD, RTMR0, RTMR1, and RTMR2 correspond to the image. dstack OS builds all related software from source. -Build version v0.5.4 using these commands: +Build the exact image revision you intend to verify. See +[Build the dstack guest OS](./building-guest-os.md) for prerequisites and the +reproducible build workflow. At a high level: + ```bash -git clone https://github.com/Dstack-TEE/meta-dstack.git -cd meta-dstack/ -git checkout f7c795b76faa693f218e1c255007e3a68c541d79 -git submodule update --init --recursive -cd repro-build && ./repro-build.sh -n +git clone https://github.com/Dstack-TEE/dstack.git +cd dstack +git checkout +make os-image ``` -The resulting dstack-0.5.4.tar.gz contains: +The resulting `dstack-.tar.gz` contains: - ovmf.fd: virtual firmware - bzImage: kernel image - initramfs.cpio.gz: initrd -- rootfs.img.verity: root filesystem +- rootfs.img.parted.verity: partitioned dm-verity root filesystem - metadata.json: image metadata, including kernel boot cmdline -Calculate image MRs using [dstack-mr](dstack/dstack-mr/): +Calculate image MRs using [dstack-mr](../dstack/dstack-mr/): ```bash -cargo run --manifest-path ../dstack/Cargo.toml --bin dstack-mr measure -c 4 -m 4G dstack-0.5.4/metadata.json +VERSION=0.6.0 # replace with the image version being verified +cargo run --manifest-path dstack/Cargo.toml --bin dstack-mr measure \ + -c 4 -m 4G "dstack-$VERSION/metadata.json" ``` Once these verification steps are completed successfully, the report_data contained in the verified quote can be considered authentic and trustworthy. diff --git a/docs/verification.md b/docs/verification.md index 76376951e..e12af7a24 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -24,6 +24,10 @@ If any of these fail, the cryptographic proof won't verify. ## Learn More +- [Intel TDX attestation](./attestation-tdx.md) - Verify TDX measurements and runtime events +- [AMD SEV-SNP support](./amd-sev-snp.md) - SNP image, attestation, and key-release requirements +- [GCP attestation](./attestation-gcp.md) - Verify the GCP TDX and TPM evidence chain +- [AWS Nitro Enclaves attestation](./attestation-nitro-enclave.md) - Verify NSM evidence - [Attestation Documentation](https://docs.phala.com/phala-cloud/attestation/overview) - Generating quotes, programmatic verification, RTMR3 replay - [Confidential AI Verification](https://docs.phala.com/phala-cloud/confidential-ai/verify/overview) - GPU TEE attestation for AI workloads - [Domain Attestation](https://docs.phala.com/phala-cloud/networking/domain-attestation) - TLS certificates managed in TEE diff --git a/dstack/Makefile b/dstack/Makefile deleted file mode 100644 index 0d04c03a9..000000000 --- a/dstack/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -DOMAIN := local -TO := ./certs - -.PHONY: clean run all certs - -all: - -certs: ${TO} - -${TO}: - mkdir -p ${TO} - cargo run --bin certgen -- generate --domain ${DOMAIN} --output-dir ${TO} - -run: - $(MAKE) -C mkguest run - -clean: - rm -rf ${TO} diff --git a/dstack/kms/README.md b/dstack/kms/README.md index ba6aff882..c96033d69 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -124,7 +124,7 @@ When a KMS node receives a key provisioning request, it: #### Vanilla TDX Quote attestation -See [Attestation](../../attestation.md) for more details. +See [Intel TDX Attestation](../../docs/attestation-tdx.md) for more details. #### Validating Apps via the KMS Auth Chain diff --git a/dstack/run.sh b/dstack/run.sh deleted file mode 100755 index bf8a2af98..000000000 --- a/dstack/run.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2024 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -VMDIR=$1 -IMAGE_PATH=./images/$(jq -r '.image' ${VMDIR}/vm-manifest.json) -IMG_METADATA=${IMAGE_PATH}/metadata.json -MEM=$(jq -r '.memory' ${VMDIR}/vm-manifest.json) -VCPUS=$(jq -r '.vcpu' ${VMDIR}/vm-manifest.json) - -VDA=${VMDIR}/hda.img - -PROCESS_NAME=qemu - -INITRD=${IMAGE_PATH}/$(jq -r '.initrd' ${IMG_METADATA}) -KERNEL=${IMAGE_PATH}/$(jq -r '.kernel' ${IMG_METADATA}) -CDROM=${IMAGE_PATH}/$(jq -r '.rootfs' ${IMG_METADATA}) -TDVF_FIRMWARE=${IMAGE_PATH}/$(jq -r '.bios' ${IMG_METADATA}) -CMDLINE=$(jq -r '.cmdline' ${IMG_METADATA}) -CONFIG_DIR=${VMDIR}/shared -TD=${TD:-1} -RO=${RO:-on} -CID=$(( ( RANDOM % 10000 ) + 3 )) - -ARGS="${ARGS} -kernel ${KERNEL}" -ARGS="${ARGS} -initrd ${INITRD}" - -echo INITRD=${INITRD} -echo ARGS=${ARGS} -echo VDA=${VDA} -echo CMDLINE=${CMDLINE} -echo TD=${TD} - -if [ "${TD}" == "1" ]; then - MACHINE_ARGS=",confidential-guest-support=tdx,hpet=off" - PROCESS_NAME=td - TDX_ARGS="-device vhost-vsock-pci,guest-cid=${CID} -object tdx-guest,id=tdx" -fi -BIOS="-bios ${TDVF_FIRMWARE}" - -sleep 2 - -qemu-system-x86_64 \ - -accel kvm \ - -m ${MEM}M -smp ${VCPUS} \ - -name ${PROCESS_NAME},process=${PROCESS_NAME},debug-threads=on \ - -cpu host \ - -machine q35,kernel_irqchip=split${MACHINE_ARGS} \ - ${BIOS} \ - ${TDX_ARGS} \ - -nographic \ - -nodefaults \ - -chardev stdio,id=ser0,signal=on -serial chardev:ser0 \ - -device virtio-net-pci,netdev=nic0_td -netdev user,id=nic0_td \ - -drive file=${VDA},if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 \ - -cdrom ${CDROM} \ - -virtfs local,path=${CONFIG_DIR},mount_tag=host-shared,readonly=${RO},security_model=mapped,id=virtfs0 \ - ${ARGS} \ - -append "${CMDLINE}" diff --git a/os/README.md b/os/README.md index 335ab127f..73d5ed5cc 100644 --- a/os/README.md +++ b/os/README.md @@ -12,7 +12,7 @@ os/ └── yocto/ # the currently implemented backend ├── deps/ # external Yocto layers (git submodules) ├── layers/ # dstack-owned Yocto layers - └── tools/ # isolated legacy/imported host helpers + └── tools/ # Yocto-specific host workarounds ``` A future backend can be added at `os//` (for example `os/mkosi/`) diff --git a/os/yocto/README.md b/os/yocto/README.md index 9fadd694f..759f958c0 100644 --- a/os/yocto/README.md +++ b/os/yocto/README.md @@ -22,10 +22,11 @@ troubleshooting instructions are in the For an interactive native build, source `dev-setup` and use this directory's Makefile, or run the repository entrypoint `./os/build.sh --backend yocto`. -The supported backend script only builds guest images. Historical host-stack -and direct-QEMU helpers imported with the old repository are isolated under -`tools/`; they are not added to `PATH` and do not shadow the supported Rust -`dstack` CLI. +The supported backend script only builds guest images. The backend-specific +`tools/` directory contains only host workarounds needed while building Yocto. +Historical all-in-one host-stack and direct-QEMU helpers are isolated under +[`../../tools/meta-dstack/`](../../tools/meta-dstack/); they are not added to +`PATH` and do not shadow the supported Rust `dstack` CLI. ## Output boundary diff --git a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect index 7826547a1..925769ac6 100755 --- a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect +++ b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect @@ -9,7 +9,7 @@ # exit 1 -> hardware absent (systemd skips the unit, no failure) # # Detection mirrors -# os/yocto/tools/vm-runner/enable-vfio-passthrough.sh, but reads PCI class codes +# tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh, but reads PCI class codes # from sysfs so it works before the nvidia driver is loaded and without relying # on lspci / the pci.ids name database: # @@ -17,7 +17,7 @@ # NVSwitch = NVIDIA (0x10de) bridge (0x06xxxx, any bridge subclass) # # NVSwitch matches any NVIDIA bridge-class device (mirrors the `grep 'Bridge'` -# heuristic in os/yocto/tools/vm-runner/enable-vfio-passthrough.sh) rather than +# heuristic in tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh) rather than # only the "Other bridge" subclass 0x0680, so it stays correct if a future # NVSwitch enumerates under a different bridge subclass. This is safe here # because these images run as TDX guests whose only NVIDIA-vendor devices are diff --git a/os/yocto/tools/README.md b/os/yocto/tools/README.md index 82c787a73..430c040c3 100644 --- a/os/yocto/tools/README.md +++ b/os/yocto/tools/README.md @@ -1,17 +1,12 @@ -# Imported Yocto support tools +# Yocto host tools -These are auxiliary host-side tools retained from `meta-dstack`. They are not -part of the guest-OS backend interface; the supported backend entrypoint is -[`../build.sh`](../build.sh). +This directory is reserved for host-side workarounds that are specific to +building the Yocto backend. It is not part of the guest-OS backend interface; +the supported backend entrypoint is [`../build.sh`](../build.sh). -- `dev-stack.sh` preserves the legacy all-in-one host/configuration helper. - `fix-self-uid-map.sh` is an explicit host AppArmor workaround for affected Ubuntu installations; it is not part of the image build. -- `vm-runner/` preserves the legacy direct-QEMU image runner and its GPU sample. -Keeping these helpers under `tools/` prevents `dev-setup` from placing a second, -legacy command named `dstack` on `PATH`. Use the Rust `dstack` and `dstackup` -CLIs for supported deployment workflows. - -Invoke a retained helper by its explicit path, for example -`os/yocto/tools/dev-stack.sh --help`. +The legacy cross-cutting helpers imported from `meta-dstack` live under +[`../../../tools/meta-dstack/`](../../../tools/meta-dstack/) instead of inside +this backend. diff --git a/prek.toml b/prek.toml index e6e21be6b..590e13ca3 100644 --- a/prek.toml +++ b/prek.toml @@ -11,15 +11,15 @@ repo = "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/pre-commit/pre-commit-hooks" rev = "v5.0.0" hooks = [ - { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, - { id = "end-of-file-fixer", exclude = "^os/yocto/" }, + { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|tools/meta-dstack/|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, + { id = "end-of-file-fixer", exclude = "^(os/yocto/|tools/meta-dstack/)" }, { id = "check-yaml", args = ["--allow-multiple-documents"], exclude = "dstack/gateway/templates/" }, { id = "check-toml" }, { id = "check-json" }, { id = "check-merge-conflict" }, { id = "check-added-large-files", args = ["--maxkb=500"] }, { id = "check-symlinks" }, - { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^os/yocto/" }, + { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^(os/yocto/|tools/meta-dstack/)" }, ] # --- Rust: rustfmt --- @@ -35,14 +35,15 @@ types = ["rust"] pass_filenames = false # --- Python: ruff (lint + format) --- -# dstack-cloud and the imported Yocto backend follow their upstream styles; -# exclude them so this repository does not create unrelated formatting churn. +# dstack-cloud and the imported Yocto/legacy meta-dstack tools follow their +# upstream styles; exclude them so this repository does not create unrelated +# formatting churn. [[repos]] repo = "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/astral-sh/ruff-pre-commit" rev = "v0.11.4" hooks = [ - { id = "ruff", args = ["--fix", "--select", "E,F,I,D", "--ignore", "D203,D213,E501"], exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/)" }, - { id = "ruff-format", exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/)" }, + { id = "ruff", args = ["--fix", "--select", "E,F,I,D", "--ignore", "D203,D213,E501"], exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/meta-dstack/)" }, + { id = "ruff-format", exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/meta-dstack/)" }, ] # --- Go: go vet --- @@ -65,7 +66,7 @@ hooks = [ # The imported Yocto backend retains its upstream shell style. New common # OS scripts remain covered by this hook. The explicit legacy list records # pre-existing shellcheck debt exposed when the files moved into dstack/. - { id = "shellcheck", exclude = '^(os/yocto/.*|dstack/(build/shared/config-qemu\.sh|cargo-check-all\.sh|gateway/test-run/(cluster\.sh|e2e/run-e2e\.sh|test_certbot\.sh)|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|run\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, + { id = "shellcheck", exclude = '^(os/yocto/.*|tools/meta-dstack/.*|dstack/(build/shared/config-qemu\.sh|cargo-check-all\.sh|gateway/test-run/(cluster\.sh|e2e/run-e2e\.sh|test_certbot\.sh)|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, ] # --- Conventional commits (used by cliff.toml for changelog) --- diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 000000000..f01a65188 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,13 @@ +# Repository tools + +This directory contains developer and operator utilities that do not belong to +one runtime component or one guest-OS backend. + +- `add-spdx-attribution.py` updates SPDX attribution metadata. +- `mock-cf-dns-api/` provides a local Cloudflare DNS API test double. +- `sca/` builds self-contained application images. +- [`meta-dstack/`](meta-dstack/) preserves unsupported legacy host/deployment + and direct-QEMU helpers imported during the monorepo migration. These tools + are kept out of `os/yocto/` because they cross host, guest, and deployment + boundaries. Use the Rust `dstack` and `dstackup` CLIs for supported + workflows. diff --git a/tools/meta-dstack/README.md b/tools/meta-dstack/README.md new file mode 100644 index 000000000..437f37824 --- /dev/null +++ b/tools/meta-dstack/README.md @@ -0,0 +1,14 @@ +# Legacy meta-dstack tools + +This directory preserves host-side utilities imported from `meta-dstack` that +do not belong to a single monorepo component or guest-OS backend. They are +retained for compatibility and low-level debugging, not as supported deployment +entrypoints. + +- `dev-stack.sh` is the historical all-in-one host build, Yocto build, + configuration, and image-download helper. +- `vm-runner/` is the historical direct-QEMU runner and its GPU passthrough + helper and sample. + +Use `dstackup` for host installation, `dstack` for app deployment, and +`os/build.sh` or the repository Makefile for guest-OS builds. diff --git a/os/yocto/tools/dev-stack.sh b/tools/meta-dstack/dev-stack.sh similarity index 96% rename from os/yocto/tools/dev-stack.sh rename to tools/meta-dstack/dev-stack.sh index 1f4b1f982..c3e1e10b5 100755 --- a/os/yocto/tools/dev-stack.sh +++ b/tools/meta-dstack/dev-stack.sh @@ -3,15 +3,15 @@ # # SPDX-License-Identifier: BUSL-1.1 -# Legacy all-in-one host development helper imported from meta-dstack. The -# actual Yocto backend entrypoint is ../build.sh; keep host/config/download -# actions here so the backend interface remains single-purpose. +# Legacy all-in-one host development helper imported from meta-dstack. It is +# retained outside both the supported dstack CLI and the Yocto backend because +# it crosses host build, guest build, and deployment/configuration boundaries. TOOL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -SCRIPT_DIR=$(realpath "$TOOL_DIR/..") +REPO_ROOT=$(realpath "$TOOL_DIR/../..") ACTION=$1 -META_DIR=$SCRIPT_DIR -DSTACK_DIR=$(realpath "$SCRIPT_DIR/../../dstack") +YOCTO_DIR=$REPO_ROOT/os/yocto +DSTACK_DIR=$REPO_ROOT/dstack CERTS_DIR=$(pwd)/certs IMAGES_DIR=$(pwd)/images RUN_DIR=$(pwd)/run @@ -127,9 +127,9 @@ build_host() { build_guest() { echo "Building guest images" if [ -z "$BBPATH" ]; then - source $SCRIPT_DIR/dev-setup $1 + source "$YOCTO_DIR/dev-setup" "$1" fi - make -C $META_DIR dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} + make -C "$YOCTO_DIR" dist DIST_DIR=$IMAGES_DIR BB_BUILD_DIR=${BBPATH} } # Step 4: generate config files diff --git a/os/yocto/tools/vm-runner/README.md b/tools/meta-dstack/vm-runner/README.md similarity index 89% rename from os/yocto/tools/vm-runner/README.md rename to tools/meta-dstack/vm-runner/README.md index ce742ba5d..ee2895823 100644 --- a/os/yocto/tools/vm-runner/README.md +++ b/tools/meta-dstack/vm-runner/README.md @@ -7,7 +7,7 @@ retained for low-level image debugging; normal deployments should use Run it explicitly instead of adding the directory to `PATH`: ```bash -python3 os/yocto/tools/vm-runner/vm-runner.py --help +python3 tools/meta-dstack/vm-runner/vm-runner.py --help ``` `enable-vfio-passthrough.sh` is the matching standalone host helper for NVIDIA diff --git a/os/yocto/tools/vm-runner/enable-vfio-passthrough.sh b/tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh similarity index 100% rename from os/yocto/tools/vm-runner/enable-vfio-passthrough.sh rename to tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh diff --git a/os/yocto/tools/vm-runner/host_api.py b/tools/meta-dstack/vm-runner/host_api.py similarity index 100% rename from os/yocto/tools/vm-runner/host_api.py rename to tools/meta-dstack/vm-runner/host_api.py diff --git a/os/yocto/tools/vm-runner/samples/Dockerfile b/tools/meta-dstack/vm-runner/samples/Dockerfile similarity index 100% rename from os/yocto/tools/vm-runner/samples/Dockerfile rename to tools/meta-dstack/vm-runner/samples/Dockerfile diff --git a/os/yocto/tools/vm-runner/samples/docker-compose.yaml b/tools/meta-dstack/vm-runner/samples/docker-compose.yaml similarity index 100% rename from os/yocto/tools/vm-runner/samples/docker-compose.yaml rename to tools/meta-dstack/vm-runner/samples/docker-compose.yaml diff --git a/os/yocto/tools/vm-runner/vm-runner.py b/tools/meta-dstack/vm-runner/vm-runner.py similarity index 99% rename from os/yocto/tools/vm-runner/vm-runner.py rename to tools/meta-dstack/vm-runner/vm-runner.py index 940a7b084..689472fd0 100755 --- a/os/yocto/tools/vm-runner/vm-runner.py +++ b/tools/meta-dstack/vm-runner/vm-runner.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BUSL-1.1 -"""Legacy direct-QEMU development runner for images built by this backend.""" +"""Legacy direct-QEMU development runner for dstack guest OS images.""" import argparse import json From cac342b5c2f7ed9bdc02bc3ff4c8a942dd4875b7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 20:29:51 -0700 Subject: [PATCH 434/435] refactor: flatten tools and adopt Apache-2.0 --- LICENSES/BSD-2-Clause-Patent.txt | 19 +++ LICENSES/BSD-3-Clause.txt | 11 ++ LICENSES/BUSL-1.1.txt | 71 ----------- LICENSES/GPL-2.0.txt | 117 ++++++++++++++++++ README.md | 9 +- REUSE.toml | 13 +- os/README.md | 9 +- os/image/assemble.sh | 2 +- os/image/authenticode_hash.py | 2 +- os/image/mk-image-mr.sh | 2 +- os/yocto/LICENSE | 94 -------------- os/yocto/README.md | 8 +- os/yocto/build.sh | 2 +- .../nvidia/files/nvidia-gpu-detect | 4 +- os/yocto/mkimage.sh | 2 +- os/yocto/scripts/export-artifacts.sh | 2 +- os/yocto/tools/README.md | 5 +- os/yocto/tools/fix-self-uid-map.sh | 2 +- prek.toml | 17 ++- tools/README.md | 13 +- tools/{meta-dstack => }/dev-stack.sh | 10 +- tools/meta-dstack/README.md | 14 --- tools/{meta-dstack => }/vm-runner/README.md | 2 +- .../vm-runner/enable-vfio-passthrough.sh | 2 +- tools/{meta-dstack => }/vm-runner/host_api.py | 0 .../vm-runner/samples/Dockerfile | 0 .../vm-runner/samples/docker-compose.yaml | 0 .../{meta-dstack => }/vm-runner/vm-runner.py | 2 +- 28 files changed, 199 insertions(+), 235 deletions(-) create mode 100644 LICENSES/BSD-2-Clause-Patent.txt create mode 100644 LICENSES/BSD-3-Clause.txt delete mode 100644 LICENSES/BUSL-1.1.txt create mode 100644 LICENSES/GPL-2.0.txt delete mode 100644 os/yocto/LICENSE rename tools/{meta-dstack => }/dev-stack.sh (97%) delete mode 100644 tools/meta-dstack/README.md rename tools/{meta-dstack => }/vm-runner/README.md (89%) rename tools/{meta-dstack => }/vm-runner/enable-vfio-passthrough.sh (97%) rename tools/{meta-dstack => }/vm-runner/host_api.py (100%) rename tools/{meta-dstack => }/vm-runner/samples/Dockerfile (100%) rename tools/{meta-dstack => }/vm-runner/samples/docker-compose.yaml (100%) rename tools/{meta-dstack => }/vm-runner/vm-runner.py (99%) diff --git a/LICENSES/BSD-2-Clause-Patent.txt b/LICENSES/BSD-2-Clause-Patent.txt new file mode 100644 index 000000000..31de6e498 --- /dev/null +++ b/LICENSES/BSD-2-Clause-Patent.txt @@ -0,0 +1,19 @@ +Copyright (c) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder and contributor hereby grants to those receiving rights under this license a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except for failure to satisfy the conditions of this license) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer this software, where such license applies only to those patent claims, already acquired or hereafter acquired, licensable by such copyright holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and non-copyrightable additions of contributors, in source or binary form) alone; or + +(b) combination of their Contribution(s) with the work of authorship to which such Contribution(s) was added by such copyright holder or contributor, if, at the time the Contribution is added, such addition causes such combination to be necessarily infringed. The patent license shall not apply to any other combinations which include the Contribution. + +Except as expressly stated above, no rights or licenses from any copyright holder or contributor is granted under this license, whether expressly, by implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt new file mode 100644 index 000000000..086d3992c --- /dev/null +++ b/LICENSES/BSD-3-Clause.txt @@ -0,0 +1,11 @@ +Copyright (c) . + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSES/BUSL-1.1.txt b/LICENSES/BUSL-1.1.txt deleted file mode 100644 index 2ef98f1be..000000000 --- a/LICENSES/BUSL-1.1.txt +++ /dev/null @@ -1,71 +0,0 @@ -Business Source License 1.1 - -License text copyright © 2017 MariaDB Corporation Ab, All Rights Reserved. -"Business Source License" is a trademark of MariaDB Corporation Ab. - -Terms - -The Licensor hereby grants you the right to copy, modify, create derivative -works, redistribute, and make non-production use of the Licensed Work. The -Licensor may make an Additional Use Grant, above, permitting limited -production use. - -Effective on the Change Date, or the fourth anniversary of the first publicly -available distribution of a specific version of the Licensed Work under this -License, whichever comes first, the Licensor hereby grants you rights under -the terms of the Change License, and the rights granted in the paragraph -above terminate. - -If your use of the Licensed Work does not comply with the requirements -currently in effect as described in this License, you must purchase a -commercial license from the Licensor, its affiliated entities, or authorized -resellers, or you must refrain from using the Licensed Work. - -All copies of the original and modified Licensed Work, and derivative works -of the Licensed Work, are subject to this License. This License applies -separately for each version of the Licensed Work and the Change Date may vary -for each version of the Licensed Work released by Licensor. - -You must conspicuously display this License on each original or modified copy -of the Licensed Work. If you receive the Licensed Work in original or -modified form from a third party, the terms and conditions set forth in this -License apply to your use of that work. - -Any use of the Licensed Work in violation of this License will automatically -terminate your rights under this License for the current and all other -versions of the Licensed Work. - -This License does not grant you any right in any trademark or logo of -Licensor or its affiliates (provided that you may use a trademark or logo of -Licensor as expressly required by this License). - -TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON -AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, -EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND -TITLE. - -MariaDB hereby grants you permission to use this License’s text to license -your works, and to refer to it using the trademark “Business Source License”, -as long as you comply with the Covenants of Licensor below. - -Covenants of Licensor - -In consideration of the right to use this License’s text and the “Business -Source License” name and trademark, Licensor covenants to MariaDB, and to all -other recipients of the licensed work to be provided by Licensor: - -1. To specify as the Change License the GPL Version 2.0 or any later version, - or a license that is compatible with GPL Version 2.0 or a later version, - where “compatible” means that software provided under the Change License can - be included in a program with software provided under GPL Version 2.0 or a - later version. Licensor may specify additional Change Licenses without - limitation. - -2. To either: (a) specify an additional grant of rights to use that does not - impose any additional restriction on the right granted in this License, as - the Additional Use Grant; or (b) insert the text “None”. - -3. To specify a Change Date. - -4. Not to modify this License in any other way. diff --git a/LICENSES/GPL-2.0.txt b/LICENSES/GPL-2.0.txt new file mode 100644 index 000000000..17cb28643 --- /dev/null +++ b/LICENSES/GPL-2.0.txt @@ -0,0 +1,117 @@ +GNU GENERAL PUBLIC LICENSE +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. + + c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. + +3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + one line to give the program's name and an idea of what it does. Copyright (C) yyyy name of author + + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. + +signature of Ty Coon, 1 April 1989 Ty Coon, President of Vice diff --git a/README.md b/README.md index 2021b3005..cb3bdb6fb 100644 --- a/README.md +++ b/README.md @@ -256,8 +256,7 @@ Logo and branding assets: [dstack-logo-kit](./docs/assets/dstack-logo-kit/) ## License -This is a multi-license repository. Core services, SDKs, documentation, tools, -and the shared guest rootfs payload are Apache-2.0. The imported Yocto backend -and derived image-assembly code retain Business Source License 1.1 terms. See -[`os/README.md`](./os/README.md), file-level SPDX declarations, and -[`REUSE.toml`](./REUSE.toml) for the exact scope. +The dstack-owned source, SDKs, documentation, tools, guest OS backend, and +image-assembly code are Apache-2.0. Embedded and third-party components retain +their own license declarations and notices. See file-level SPDX declarations +and [`REUSE.toml`](./REUSE.toml) for the exact scope. diff --git a/REUSE.toml b/REUSE.toml index 5fcc0cc59..b442c4e96 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -118,22 +118,19 @@ path = "os/spec/*.json" SPDX-FileCopyrightText = "Copyright (c) 2026 The Project Contributors" SPDX-License-Identifier = "Apache-2.0" -# Imported OS backend +# dstack-owned OS backend [[annotations]] path = "os/yocto/**" SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" -SPDX-License-Identifier = "BUSL-1.1" -precedence = "override" +SPDX-License-Identifier = "Apache-2.0" -# Legacy tools imported from meta-dstack but intentionally kept outside the -# Yocto backend because they also manage host and deployment concerns. +# Legacy cross-component tools kept outside the Yocto backend. [[annotations]] -path = "tools/meta-dstack/**" +path = "tools/vm-runner/**" SPDX-FileCopyrightText = "Copyright (c) Hashforest Technology LLC" -SPDX-License-Identifier = "BUSL-1.1" -precedence = "override" +SPDX-License-Identifier = "Apache-2.0" # Artworks diff --git a/os/README.md b/os/README.md index 73d5ed5cc..a880d343f 100644 --- a/os/README.md +++ b/os/README.md @@ -72,8 +72,7 @@ It does not copy the entire repository or depend on a nested dstack submodule. ## Licensing -The monorepo contains multiple license scopes. Core, SDK, documentation, tools, -and `os/common/rootfs/` remain Apache-2.0. The imported Yocto implementation and -derived image-assembly code retain the Business Source License 1.1 terms in -`os/yocto/LICENSE`. Individual vendored recipes and patches may carry their own -notices; moving them into this repository does not relicense them. +The dstack-owned core, SDK, documentation, tools, guest payload, Yocto backend, +and image-assembly code are Apache-2.0. Individual vendored recipes, patches, +and embedded components may carry their own notices; moving them into this +repository does not relicense them. diff --git a/os/image/assemble.sh b/os/image/assemble.sh index 90780221e..fdf66a7f3 100755 --- a/os/image/assemble.sh +++ b/os/image/assemble.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/os/image/authenticode_hash.py b/os/image/authenticode_hash.py index c8ed5432e..c3fa3f484 100644 --- a/os/image/authenticode_hash.py +++ b/os/image/authenticode_hash.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 """Calculate the Authenticode SHA-256 hash of a PE/COFF image.""" diff --git a/os/image/mk-image-mr.sh b/os/image/mk-image-mr.sh index be5458207..3c69ab7fb 100755 --- a/os/image/mk-image-mr.sh +++ b/os/image/mk-image-mr.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/os/yocto/LICENSE b/os/yocto/LICENSE deleted file mode 100644 index 4a58670f1..000000000 --- a/os/yocto/LICENSE +++ /dev/null @@ -1,94 +0,0 @@ -Business Source License 1.1 - -Parameters - -Licensor: Hashforest Technology LLC - -Licensed Work: dstack-cloud - The Licensed Work is (c) Hashforest Technology LLC - -Additional Use Grant: Notwithstanding the foregoing, the Licensor grants - to certain commercial partners a license to use the - Licensed Work for production and commercial purposes - pursuant to separate agreements. - -Change Date: Two years from the date a MINOR version (SemVer) is - published. - -Change License: GNU Affero General Public License Version 3 (AGPL-3.0) - -Notice - -License text copyright (c) 2023 MariaDB plc, All Rights Reserved. -“Business Source License” is a trademark of MariaDB plc. - ------------------------------------------------------------------------------ - -Business Source License 1.1 - -Terms - -The Licensor hereby grants you the right to copy, modify, create derivative -works, redistribute, and make non-production use of the Licensed Work. The -Licensor may make an Additional Use Grant, above, permitting limited -production use. - -Effective on the Change Date, or the fourth anniversary of the first publicly -available distribution of a specific version of the Licensed Work under this -License, whichever comes first, the Licensor hereby grants you rights under -the terms of the Change License, and the rights granted in the paragraph -above terminate. - -If your use of the Licensed Work does not comply with the requirements -currently in effect as described in this License, you must purchase a -commercial license from the Licensor, its affiliated entities, or authorized -resellers, or you must refrain from using the Licensed Work. - -All copies of the original and modified Licensed Work, and derivative works -of the Licensed Work, are subject to this License. This License applies -separately for each version of the Licensed Work and the Change Date may vary -for each version of the Licensed Work released by Licensor. - -You must conspicuously display this License on each original or modified copy -of the Licensed Work. If you receive the Licensed Work in original or -modified form from a third party, the terms and conditions set forth in this -License apply to your use of that work. - -Any use of the Licensed Work in violation of this License will automatically -terminate your rights under this License for the current and all other -versions of the Licensed Work. - -This License does not grant you any right in any trademark or logo of -Licensor or its affiliates (provided that you may use a trademark or logo of -Licensor as expressly required by this License). - -TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON -AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, -EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND -TITLE. - -MariaDB hereby grants you permission to use this License’s text to license -your works, and to refer to it using the trademark “Business Source License”, -as long as you comply with the Covenants of Licensor below. - -Covenants of Licensor - -In consideration of the right to use this License’s text and the “Business -Source License” name and trademark, Licensor covenants to MariaDB, and to all -other recipients of the licensed work to be provided by Licensor: - -1. To specify as the Change License the GPL Version 2.0 or any later version, - or a license that is compatible with GPL Version 2.0 or a later version, - where “compatible” means that software provided under the Change License can - be included in a program with software provided under GPL Version 2.0 or a - later version. Licensor may specify additional Change Licenses without - limitation. - -2. To either: (a) specify an additional grant of rights to use that does not - impose any additional restriction on the right granted in this License, as - the Additional Use Grant; or (b) insert the text “None”. - -3. To specify a Change Date. - -4. Not to modify this License in any other way. \ No newline at end of file diff --git a/os/yocto/README.md b/os/yocto/README.md index 759f958c0..a17a622e4 100644 --- a/os/yocto/README.md +++ b/os/yocto/README.md @@ -25,8 +25,8 @@ Makefile, or run the repository entrypoint `./os/build.sh --backend yocto`. The supported backend script only builds guest images. The backend-specific `tools/` directory contains only host workarounds needed while building Yocto. Historical all-in-one host-stack and direct-QEMU helpers are isolated under -[`../../tools/meta-dstack/`](../../tools/meta-dstack/); they are not added to -`PATH` and do not shadow the supported Rust `dstack` CLI. +[`../../tools/`](../../tools/); they are not added to `PATH` and do not shadow +the supported Rust `dstack` CLI. ## Output boundary @@ -36,5 +36,5 @@ manifest and calls `../image/assemble.sh`. ## License -The imported backend retains the Business Source License 1.1 terms in -[`LICENSE`](LICENSE). Embedded components may have their own notices. +The dstack-owned backend code is Apache-2.0. Embedded and third-party +components retain their own license declarations and notices. diff --git a/os/yocto/build.sh b/os/yocto/build.sh index 593db2611..4f8d8baee 100755 --- a/os/yocto/build.sh +++ b/os/yocto/build.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 # Yocto implementation of the guest-OS backend contract in os/README.md. set -eo pipefail diff --git a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect index 925769ac6..8547c949a 100755 --- a/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect +++ b/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect @@ -9,7 +9,7 @@ # exit 1 -> hardware absent (systemd skips the unit, no failure) # # Detection mirrors -# tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh, but reads PCI class codes +# tools/vm-runner/enable-vfio-passthrough.sh, but reads PCI class codes # from sysfs so it works before the nvidia driver is loaded and without relying # on lspci / the pci.ids name database: # @@ -17,7 +17,7 @@ # NVSwitch = NVIDIA (0x10de) bridge (0x06xxxx, any bridge subclass) # # NVSwitch matches any NVIDIA bridge-class device (mirrors the `grep 'Bridge'` -# heuristic in tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh) rather than +# heuristic in tools/vm-runner/enable-vfio-passthrough.sh) rather than # only the "Other bridge" subclass 0x0680, so it stays correct if a future # NVSwitch enumerates under a different bridge subclass. This is safe here # because these images run as TDX guests whose only NVIDIA-vendor devices are diff --git a/os/yocto/mkimage.sh b/os/yocto/mkimage.sh index 1d3174214..5260b979d 100755 --- a/os/yocto/mkimage.sh +++ b/os/yocto/mkimage.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 # Compatibility entrypoint: export Yocto-specific outputs, then invoke the # backend-neutral image assembler. diff --git a/os/yocto/scripts/export-artifacts.sh b/os/yocto/scripts/export-artifacts.sh index 414c9b049..7fcbd283a 100755 --- a/os/yocto/scripts/export-artifacts.sh +++ b/os/yocto/scripts/export-artifacts.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/os/yocto/tools/README.md b/os/yocto/tools/README.md index 430c040c3..caac234a4 100644 --- a/os/yocto/tools/README.md +++ b/os/yocto/tools/README.md @@ -7,6 +7,5 @@ the supported backend entrypoint is [`../build.sh`](../build.sh). - `fix-self-uid-map.sh` is an explicit host AppArmor workaround for affected Ubuntu installations; it is not part of the image build. -The legacy cross-cutting helpers imported from `meta-dstack` live under -[`../../../tools/meta-dstack/`](../../../tools/meta-dstack/) instead of inside -this backend. +The legacy cross-cutting helpers live under [`../../../tools/`](../../../tools/) +instead of inside this backend. diff --git a/os/yocto/tools/fix-self-uid-map.sh b/os/yocto/tools/fix-self-uid-map.sh index 226a599a3..b262b9c47 100755 --- a/os/yocto/tools/fix-self-uid-map.sh +++ b/os/yocto/tools/fix-self-uid-map.sh @@ -1,7 +1,7 @@ #!/bin/sh # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 # BUG: https://bugs.launchpad.net/ubuntu/+source/apparmor/+bug/2056555 sudo apparmor_parser -R /etc/apparmor.d/unprivileged_userns diff --git a/prek.toml b/prek.toml index 590e13ca3..127d9c310 100644 --- a/prek.toml +++ b/prek.toml @@ -11,15 +11,15 @@ repo = "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/pre-commit/pre-commit-hooks" rev = "v5.0.0" hooks = [ - { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|tools/meta-dstack/|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, - { id = "end-of-file-fixer", exclude = "^(os/yocto/|tools/meta-dstack/)" }, + { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|tools/(dev-stack\.sh|vm-runner/)|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, + { id = "end-of-file-fixer", exclude = "^(os/yocto/|tools/(dev-stack\\.sh|vm-runner/))" }, { id = "check-yaml", args = ["--allow-multiple-documents"], exclude = "dstack/gateway/templates/" }, { id = "check-toml" }, { id = "check-json" }, { id = "check-merge-conflict" }, { id = "check-added-large-files", args = ["--maxkb=500"] }, { id = "check-symlinks" }, - { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^(os/yocto/|tools/meta-dstack/)" }, + { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^(os/yocto/|tools/(dev-stack\\.sh|vm-runner/))" }, ] # --- Rust: rustfmt --- @@ -35,15 +35,14 @@ types = ["rust"] pass_filenames = false # --- Python: ruff (lint + format) --- -# dstack-cloud and the imported Yocto/legacy meta-dstack tools follow their -# upstream styles; exclude them so this repository does not create unrelated -# formatting churn. +# dstack-cloud and the imported Yocto/legacy tools follow their upstream styles; +# exclude them so this repository does not create unrelated formatting churn. [[repos]] repo = "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/astral-sh/ruff-pre-commit" rev = "v0.11.4" hooks = [ - { id = "ruff", args = ["--fix", "--select", "E,F,I,D", "--ignore", "D203,D213,E501"], exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/meta-dstack/)" }, - { id = "ruff-format", exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/meta-dstack/)" }, + { id = "ruff", args = ["--fix", "--select", "E,F,I,D", "--ignore", "D203,D213,E501"], exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/vm-runner/)" }, + { id = "ruff-format", exclude = "^(dstack/scripts/bin/dstack-cloud|os/yocto/|tools/vm-runner/)" }, ] # --- Go: go vet --- @@ -66,7 +65,7 @@ hooks = [ # The imported Yocto backend retains its upstream shell style. New common # OS scripts remain covered by this hook. The explicit legacy list records # pre-existing shellcheck debt exposed when the files moved into dstack/. - { id = "shellcheck", exclude = '^(os/yocto/.*|tools/meta-dstack/.*|dstack/(build/shared/config-qemu\.sh|cargo-check-all\.sh|gateway/test-run/(cluster\.sh|e2e/run-e2e\.sh|test_certbot\.sh)|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, + { id = "shellcheck", exclude = '^(os/yocto/.*|tools/(dev-stack\.sh|vm-runner/.*)|dstack/(build/shared/config-qemu\.sh|cargo-check-all\.sh|gateway/test-run/(cluster\.sh|e2e/run-e2e\.sh|test_certbot\.sh)|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, ] # --- Conventional commits (used by cliff.toml for changelog) --- diff --git a/tools/README.md b/tools/README.md index f01a65188..87a2704cb 100644 --- a/tools/README.md +++ b/tools/README.md @@ -6,8 +6,11 @@ one runtime component or one guest-OS backend. - `add-spdx-attribution.py` updates SPDX attribution metadata. - `mock-cf-dns-api/` provides a local Cloudflare DNS API test double. - `sca/` builds self-contained application images. -- [`meta-dstack/`](meta-dstack/) preserves unsupported legacy host/deployment - and direct-QEMU helpers imported during the monorepo migration. These tools - are kept out of `os/yocto/` because they cross host, guest, and deployment - boundaries. Use the Rust `dstack` and `dstackup` CLIs for supported - workflows. +- `dev-stack.sh` preserves the unsupported legacy all-in-one host, guest, and + deployment helper. +- `vm-runner/` preserves the unsupported legacy direct-QEMU runner and GPU + passthrough helper. + +The legacy tools are kept out of `os/yocto/` because they cross host, guest, +and deployment boundaries. Use the Rust `dstack` and `dstackup` CLIs for +supported workflows. diff --git a/tools/meta-dstack/dev-stack.sh b/tools/dev-stack.sh similarity index 97% rename from tools/meta-dstack/dev-stack.sh rename to tools/dev-stack.sh index c3e1e10b5..af43e8351 100755 --- a/tools/meta-dstack/dev-stack.sh +++ b/tools/dev-stack.sh @@ -1,13 +1,13 @@ #!/bin/bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 -# Legacy all-in-one host development helper imported from meta-dstack. It is -# retained outside both the supported dstack CLI and the Yocto backend because -# it crosses host build, guest build, and deployment/configuration boundaries. +# Legacy all-in-one host development helper. It is retained outside both the +# supported dstack CLI and the Yocto backend because it crosses host build, +# guest build, and deployment/configuration boundaries. TOOL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -REPO_ROOT=$(realpath "$TOOL_DIR/../..") +REPO_ROOT=$(realpath "$TOOL_DIR/..") ACTION=$1 YOCTO_DIR=$REPO_ROOT/os/yocto diff --git a/tools/meta-dstack/README.md b/tools/meta-dstack/README.md deleted file mode 100644 index 437f37824..000000000 --- a/tools/meta-dstack/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Legacy meta-dstack tools - -This directory preserves host-side utilities imported from `meta-dstack` that -do not belong to a single monorepo component or guest-OS backend. They are -retained for compatibility and low-level debugging, not as supported deployment -entrypoints. - -- `dev-stack.sh` is the historical all-in-one host build, Yocto build, - configuration, and image-download helper. -- `vm-runner/` is the historical direct-QEMU runner and its GPU passthrough - helper and sample. - -Use `dstackup` for host installation, `dstack` for app deployment, and -`os/build.sh` or the repository Makefile for guest-OS builds. diff --git a/tools/meta-dstack/vm-runner/README.md b/tools/vm-runner/README.md similarity index 89% rename from tools/meta-dstack/vm-runner/README.md rename to tools/vm-runner/README.md index ee2895823..dc569fdbc 100644 --- a/tools/meta-dstack/vm-runner/README.md +++ b/tools/vm-runner/README.md @@ -7,7 +7,7 @@ retained for low-level image debugging; normal deployments should use Run it explicitly instead of adding the directory to `PATH`: ```bash -python3 tools/meta-dstack/vm-runner/vm-runner.py --help +python3 tools/vm-runner/vm-runner.py --help ``` `enable-vfio-passthrough.sh` is the matching standalone host helper for NVIDIA diff --git a/tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh b/tools/vm-runner/enable-vfio-passthrough.sh similarity index 97% rename from tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh rename to tools/vm-runner/enable-vfio-passthrough.sh index f80b349aa..b4259ad3b 100755 --- a/tools/meta-dstack/vm-runner/enable-vfio-passthrough.sh +++ b/tools/vm-runner/enable-vfio-passthrough.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/tools/meta-dstack/vm-runner/host_api.py b/tools/vm-runner/host_api.py similarity index 100% rename from tools/meta-dstack/vm-runner/host_api.py rename to tools/vm-runner/host_api.py diff --git a/tools/meta-dstack/vm-runner/samples/Dockerfile b/tools/vm-runner/samples/Dockerfile similarity index 100% rename from tools/meta-dstack/vm-runner/samples/Dockerfile rename to tools/vm-runner/samples/Dockerfile diff --git a/tools/meta-dstack/vm-runner/samples/docker-compose.yaml b/tools/vm-runner/samples/docker-compose.yaml similarity index 100% rename from tools/meta-dstack/vm-runner/samples/docker-compose.yaml rename to tools/vm-runner/samples/docker-compose.yaml diff --git a/tools/meta-dstack/vm-runner/vm-runner.py b/tools/vm-runner/vm-runner.py similarity index 99% rename from tools/meta-dstack/vm-runner/vm-runner.py rename to tools/vm-runner/vm-runner.py index 689472fd0..7bce7fe4f 100755 --- a/tools/meta-dstack/vm-runner/vm-runner.py +++ b/tools/vm-runner/vm-runner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) Hashforest Technology LLC # -# SPDX-License-Identifier: BUSL-1.1 +# SPDX-License-Identifier: Apache-2.0 """Legacy direct-QEMU development runner for dstack guest OS images.""" From 858419b2ae46f9420280443f7203f245d1935484 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 12 Jul 2026 20:37:04 -0700 Subject: [PATCH 435/435] fix: use current GPL SPDX identifier --- LICENSES/{GPL-2.0.txt => GPL-2.0-only.txt} | 0 .../0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename LICENSES/{GPL-2.0.txt => GPL-2.0-only.txt} (100%) diff --git a/LICENSES/GPL-2.0.txt b/LICENSES/GPL-2.0-only.txt similarity index 100% rename from LICENSES/GPL-2.0.txt rename to LICENSES/GPL-2.0-only.txt diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch index c868aa82f..98ce12641 100644 --- a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch @@ -70,7 +70,7 @@ index 0000000..34d09ca --- /dev/null +++ b/drivers/acpi/acpica/sandbox.h @@ -0,0 +1,154 @@ -+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ ++/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-only */ +/* SPDX-FileCopyrightText: Satoru Takekoshi, Manami Mori, Takaaki Fukai, + * Takahiro Shinagawa */ +/* SPDX-FileCopyrightText: Edgeless Systems GmbH */