From 762424d568683f44b423732fbb7ab0f6fde08148 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Fri, 7 Feb 2025 21:23:40 +0000 Subject: [PATCH 01/21] Add command for copying builds across buckets --- packages/orchestrator/cmd/copy-build/main.go | 110 +++++++++++++++++++ packages/shared/pkg/storage/gcs/bucket.go | 4 +- packages/shared/pkg/storage/gcs/object.go | 23 ++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 packages/orchestrator/cmd/copy-build/main.go diff --git a/packages/orchestrator/cmd/copy-build/main.go b/packages/orchestrator/cmd/copy-build/main.go new file mode 100644 index 0000000000..6fe0b77c4f --- /dev/null +++ b/packages/orchestrator/cmd/copy-build/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +func getReferencedData(ctx context.Context, bucket *gcs.BucketHandle, headerPath string, kind string) ([]string, error) { + obj := gcs.NewObject(ctx, bucket, headerPath) + + h, err := header.Deserialize(obj) + if err != nil { + return nil, fmt.Errorf("failed to deserialize header: %w", err) + } + + builds := make(map[string]struct{}) + + for _, mapping := range h.Mapping { + builds[mapping.BuildId.String()] = struct{}{} + } + + var dataReferences []string + + for build := range builds { + template := storage.NewTemplateFiles( + "", + build, + "", + "", + false, + ) + if kind == "memfile" { + dataReferences = append(dataReferences, template.StorageMemfilePath()) + } else if kind == "rootfs" { + dataReferences = append(dataReferences, template.StorageRootfsPath()) + } + } + + return dataReferences, nil +} + +func copyFromBucket(ctx context.Context, from *gcs.BucketHandle, to *gcs.BucketHandle, objectPath string) error { + fromObject := gcs.NewObject(ctx, from, objectPath) + toObject := gcs.NewObject(ctx, to, objectPath) + + return fromObject.Copy(ctx, toObject) +} + +func main() { + buildId := flag.String("build", "", "build id") + from := flag.String("from", "", "from bucket") + to := flag.String("to", "", "to bucket") + + flag.Parse() + + template := storage.NewTemplateFiles( + "", + *buildId, + "", + "", + false, + ) + + ctx := context.Background() + + fromBucket := gcs.NewBucket(*from) + toBucket := gcs.NewBucket(*to) + + var filesToCopy []string + + // Extract all files referenced by the build memfile header + buildHeaderPath := template.StorageMemfileHeaderPath() + dataReferences, err := getReferencedData(ctx, fromBucket, buildHeaderPath, "memfile") + if err != nil { + log.Fatalf("failed to get referenced data: %s", err) + } + + filesToCopy = append(filesToCopy, buildHeaderPath) + filesToCopy = append(filesToCopy, dataReferences...) + + // Extract all files referenced by the build rootfs header + buildHeaderPath = template.StorageRootfsHeaderPath() + dataReferences, err = getReferencedData(ctx, fromBucket, buildHeaderPath, "rootfs") + if err != nil { + log.Fatalf("failed to get referenced data: %s", err) + } + + filesToCopy = append(filesToCopy, buildHeaderPath) + filesToCopy = append(filesToCopy, dataReferences...) + + // Add the snapfile to the list of files to copy + snapfilePath := template.StorageSnapfilePath() + + filesToCopy = append(filesToCopy, snapfilePath) + + for _, file := range filesToCopy { + fmt.Printf("Copying %s", file) + err := copyFromBucket(ctx, fromBucket, toBucket, file) + if err != nil { + log.Fatalf("\nfailed to copy file: %s", err) + } + fmt.Printf(" done\n") + } +} diff --git a/packages/shared/pkg/storage/gcs/bucket.go b/packages/shared/pkg/storage/gcs/bucket.go index dd41206fbc..695ea04d35 100644 --- a/packages/shared/pkg/storage/gcs/bucket.go +++ b/packages/shared/pkg/storage/gcs/bucket.go @@ -8,12 +8,12 @@ import ( type BucketHandle = storage.BucketHandle -func newBucket(bucket string) *BucketHandle { +func NewBucket(bucket string) *BucketHandle { return client.Bucket(bucket) } var ( templateBucketName = utils.RequiredEnv("TEMPLATE_BUCKET_NAME", "bucket for storing template files") - TemplateBucket = newBucket(templateBucketName) + TemplateBucket = NewBucket(templateBucketName) ) diff --git a/packages/shared/pkg/storage/gcs/object.go b/packages/shared/pkg/storage/gcs/object.go index ad90629180..b6e160cb45 100644 --- a/packages/shared/pkg/storage/gcs/object.go +++ b/packages/shared/pkg/storage/gcs/object.go @@ -81,6 +81,29 @@ func (o *Object) ReadFrom(src io.Reader) (int64, error) { return n, nil } +func (o *Object) Copy(ctx context.Context, to *Object) error { + fromPath := fmt.Sprintf("gs://%s/%s", o.object.BucketName(), o.object.ObjectName()) + toPath := fmt.Sprintf("gs://%s/%s", to.object.BucketName(), to.object.ObjectName()) + + cmd := exec.CommandContext( + ctx, + "gcloud", + "storage", + "cp", + "--verbosity", + "error", + fromPath, + toPath, + ) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy GCS object: %w\n%s", err, string(output)) + } + + return nil +} + func (o *Object) UploadWithCli(ctx context.Context, path string) error { cmd := exec.CommandContext( ctx, From 9f6a22384cc7928dde2d846f1b33e93b03016c8c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Fri, 7 Feb 2025 21:40:09 +0000 Subject: [PATCH 02/21] Skip already copied build files during copy build command --- packages/orchestrator/cmd/copy-build/main.go | 37 +++++++++++++++++--- packages/shared/pkg/storage/gcs/object.go | 4 +++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/orchestrator/cmd/copy-build/main.go b/packages/orchestrator/cmd/copy-build/main.go index 6fe0b77c4f..8622d98496 100644 --- a/packages/orchestrator/cmd/copy-build/main.go +++ b/packages/orchestrator/cmd/copy-build/main.go @@ -2,10 +2,13 @@ package main import ( "context" + "errors" "flag" "fmt" "log" + googleStorage "cloud.google.com/go/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" @@ -45,11 +48,31 @@ func getReferencedData(ctx context.Context, bucket *gcs.BucketHandle, headerPath return dataReferences, nil } -func copyFromBucket(ctx context.Context, from *gcs.BucketHandle, to *gcs.BucketHandle, objectPath string) error { +func copyFromBucket(ctx context.Context, from *gcs.BucketHandle, to *gcs.BucketHandle, objectPath string) (bool, error) { fromObject := gcs.NewObject(ctx, from, objectPath) + + fromSize, err := fromObject.Size() + if err != nil { + return false, fmt.Errorf("failed to get size of object: %w", err) + } + toObject := gcs.NewObject(ctx, to, objectPath) - return fromObject.Copy(ctx, toObject) + toSize, err := toObject.Size() + if err != nil && !errors.Is(err, googleStorage.ErrObjectNotExist) { + return false, fmt.Errorf("failed to get size of object: %w", err) + } + + if fromSize == toSize { + return false, nil + } + + err = fromObject.Copy(ctx, toObject) + if err != nil { + return false, fmt.Errorf("failed to copy object: %w", err) + } + + return true, nil } func main() { @@ -101,10 +124,16 @@ func main() { for _, file := range filesToCopy { fmt.Printf("Copying %s", file) - err := copyFromBucket(ctx, fromBucket, toBucket, file) + + copied, err := copyFromBucket(ctx, fromBucket, toBucket, file) if err != nil { log.Fatalf("\nfailed to copy file: %s", err) } - fmt.Printf(" done\n") + + if copied { + fmt.Printf(" done\n") + } else { + fmt.Printf(" skipped\n") + } } } diff --git a/packages/shared/pkg/storage/gcs/object.go b/packages/shared/pkg/storage/gcs/object.go index b6e160cb45..0632c60bdb 100644 --- a/packages/shared/pkg/storage/gcs/object.go +++ b/packages/shared/pkg/storage/gcs/object.go @@ -159,6 +159,10 @@ func (o *Object) Size() (int64, error) { defer cancel() attrs, err := o.object.Attrs(ctx) + if errors.Is(err, storage.ErrObjectNotExist) { + return 0, storage.ErrObjectNotExist + } + if err != nil { return 0, fmt.Errorf("failed to get GCS object (%s) attributes: %w", o.object.ObjectName(), err) } From 3c209c8d99e9f6fa19f3d92bd89ed3f1fc59fb24 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Fri, 7 Feb 2025 21:43:01 +0000 Subject: [PATCH 03/21] Cleanup build copy command --- packages/orchestrator/cmd/copy-build/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/cmd/copy-build/main.go b/packages/orchestrator/cmd/copy-build/main.go index 8622d98496..80c9c940d7 100644 --- a/packages/orchestrator/cmd/copy-build/main.go +++ b/packages/orchestrator/cmd/copy-build/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "log" + "sort" googleStorage "cloud.google.com/go/storage" @@ -82,6 +83,8 @@ func main() { flag.Parse() + fmt.Printf("Copying build '%s' from bucket '%s' to bucket '%s'\n", *buildId, *from, *to) + template := storage.NewTemplateFiles( "", *buildId, @@ -122,8 +125,11 @@ func main() { filesToCopy = append(filesToCopy, snapfilePath) + // sort files to copy + sort.Strings(filesToCopy) + for _, file := range filesToCopy { - fmt.Printf("Copying %s", file) + fmt.Printf("- copying %s", file) copied, err := copyFromBucket(ctx, fromBucket, toBucket, file) if err != nil { @@ -136,4 +142,6 @@ func main() { fmt.Printf(" skipped\n") } } + + fmt.Printf("Build copied\n") } From 660040fe5fca1e0fe655af1527f59cdf5a29c7b6 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 13 Feb 2025 02:04:39 +0000 Subject: [PATCH 04/21] Add orchestrator rootfs mount command --- .../orchestrator/cmd/mount-rootfs/main.go | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 packages/orchestrator/cmd/mount-rootfs/main.go diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go new file mode 100644 index 0000000000..5e71c1cfde --- /dev/null +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" + "github.com/google/uuid" +) + +const blockSize = 4096 + +type DeviceWithClose struct { + *build.File + size int64 +} + +func (d *DeviceWithClose) Close() error { + return nil +} + +func (d *DeviceWithClose) Size() (int64, error) { + return d.size, nil +} + +func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { + fmt.Printf("ReadAt %d bytes at offset %d\n", len(p), off) + + return d.File.ReadAt(p, off) +} + +func main() { + buildId := flag.String("build", "", "build id") + + flag.Parse() + + template := storage.NewTemplateFiles( + "", + *buildId, + "", + "", + false, + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan os.Signal, 1) + signal.Notify(done, os.Interrupt) + + go func() { + <-done + + cancel() + }() + + storagePath := template.StorageRootfsHeaderPath() + obj := gcs.NewObject(ctx, gcs.TemplateBucket, storagePath) + + h, err := header.Deserialize(obj) + if err != nil { + id, err := uuid.Parse(*buildId) + if err != nil { + log.Fatalf("failed to parse build id: %s", err) + } + + object := gcs.NewObject(ctx, gcs.TemplateBucket, *buildId+"/"+string(build.Rootfs)) + + size, err := object.Size() + if err != nil { + log.Fatalf("failed to get object size: %s", err) + } + + h = header.NewHeader(&header.Metadata{ + BuildId: id, + BaseBuildId: id, + Size: uint64(size), + Version: 1, + BlockSize: uint64(blockSize), + Generation: 1, + }, nil) + } + + store, err := build.NewDiffStore(gcs.TemplateBucket, ctx) + if err != nil { + log.Fatalf("failed to create diff store: %s", err) + } + + rootfs := build.NewFile(h, store, build.Rootfs) + + random := uuid.New().String() + cachePath := filepath.Join(os.TempDir(), fmt.Sprintf("rootfs.cache-%s", random)) + + cache, err := block.NewCache(int64(h.Metadata.Size), blockSize, cachePath, false) + if err != nil { + log.Fatalf("failed to create cache: %s", err) + } + + fmt.Printf("cachePath: %+v\n", h.Metadata) + + overlay := block.NewOverlay(&DeviceWithClose{rootfs, int64(h.Metadata.Size)}, cache, blockSize) + defer overlay.Close() + + mnt := nbd.NewDirectPathMount(overlay) + + go func() { + <-ctx.Done() + + fmt.Println("Closing mnt") + + mnt.Close() + }() + + mntIndex, err := mnt.Open(ctx) + if err != nil { + log.Fatalf("failed to open: %s", err) + } + + devicePath := nbd.GetDevicePath(mntIndex) + + fmt.Printf("Mounted rootfs at %s\n", devicePath) + + <-ctx.Done() +} From 3521339d32647ef5b0df3d1403d7960649182d83 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Fri, 14 Feb 2025 22:35:40 +0000 Subject: [PATCH 05/21] [WIP] Add test for rootfs diff --- .../orchestrator/cmd/test-rootfs-diff/main.go | 383 ++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 packages/orchestrator/cmd/test-rootfs-diff/main.go diff --git a/packages/orchestrator/cmd/test-rootfs-diff/main.go b/packages/orchestrator/cmd/test-rootfs-diff/main.go new file mode 100644 index 0000000000..2bacf9212f --- /dev/null +++ b/packages/orchestrator/cmd/test-rootfs-diff/main.go @@ -0,0 +1,383 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "flag" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" + "github.com/google/uuid" +) + +type DeviceWithClose struct { + *build.File + id string + size int64 +} + +func (d *DeviceWithClose) Close() error { + return nil +} + +func (d *DeviceWithClose) Size() (int64, error) { + return d.size, nil +} + +func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { + return d.File.ReadAt(p, off) +} + +const ( + contentFileName = "test.txt" + content = `` + blockSize = 4096 +) + +// Execute the passed callback with the passed overlay mounted as a nbd device. +func executeForNbd( + ctx context.Context, + overlay *block.Overlay, + cb func(mountedPath string) error, +) error { + mnt := nbd.NewDirectPathMount(overlay) + + nbdCtx, nbdCancel := context.WithCancel(ctx) + defer nbdCancel() + + go func() { + <-nbdCtx.Done() + + mnt.Close() + }() + + mntIndex, err := mnt.Open(nbdCtx) + if err != nil { + return fmt.Errorf("failed to open: %w", err) + } + + devicePath := nbd.GetDevicePath(mntIndex) + + fmt.Printf("- created nbd device at %s\n", devicePath) + + // Check the block device health + out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to fsck: %w %s", err, out) + } + + fmt.Printf("- fscked base nbd device at %s\n", out) + + tmpDir, err := os.MkdirTemp("", "mount-*") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + + defer os.RemoveAll(tmpDir) + + out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to mount: %w - %s", err, out) + } + + defer func() { + out, err = exec.Command("umount", tmpDir).CombinedOutput() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to umount: %s - %s", err, out) + } + }() + + fmt.Printf("- mounted rootfs at %s\n", devicePath) + + err = cb(tmpDir) + if err != nil { + return fmt.Errorf("failed execute: %w", err) + } + + return nil +} + +func fileFromStorage( + ctx context.Context, + baseBuildId string, + store *build.DiffStore, +) (*build.File, *header.Header, error) { + buildId, err := uuid.Parse(baseBuildId) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse build id: %w", err) + } + + object := gcs.NewObject(ctx, gcs.TemplateBucket, buildId.String()+"/"+string(build.Rootfs)) + + size, err := object.Size() + if err != nil { + return nil, nil, fmt.Errorf("failed to get object size: %w", err) + } + + header := header.NewHeader(&header.Metadata{ + BuildId: buildId, + BaseBuildId: buildId, + Size: uint64(size), + Version: 1, + BlockSize: uint64(blockSize), + Generation: 1, + }, nil) + + rootfs := build.NewFile(header, store, build.Rootfs) + + return rootfs, header, nil +} + +// Create an overlay by extracting the diff from the passed overlay, +// putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. +func fileFromOverlay( + overlay *block.Overlay, + baseHeader *header.Header, + store *build.DiffStore, +) (*build.File, *header.Header, error) { + diffBuildId := uuid.New() + + // TODO: Diff file is not cleaned up after exit. The same goes for the base storage file in the cache. + // This should not affect the test. + diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) + if err != nil { + return nil, nil, fmt.Errorf("failed to create diff file: %w", err) + } + + cache, err := overlay.EjectCache() + if err != nil { + return nil, nil, fmt.Errorf("failed to eject cache: %w", err) + } + + dirtyBlocks, err := cache.Export(diffFile) + if err != nil { + return nil, nil, fmt.Errorf("failed to export cache: %w", err) + } + + diff, err := diffFile.ToDiff(blockSize) + if err != nil { + return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) + } + + newMappings := header.CreateMapping( + baseHeader.Metadata, + &diffBuildId, + dirtyBlocks, + ) + + diffMappings := header.MergeMappings( + baseHeader.Mapping, + newMappings, + ) + + diffHeader := header.NewHeader(&header.Metadata{ + BuildId: diffBuildId, + BaseBuildId: baseHeader.Metadata.BaseBuildId, + Size: baseHeader.Metadata.Size, + Version: baseHeader.Metadata.Version, + BlockSize: baseHeader.Metadata.BlockSize, + }, diffMappings) + + store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) + + // Create a build file that will use the diff and the base already in the store + diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) + + return diffRootfs, diffHeader, nil +} + +func compareSources( + s1, + s2 io.ReaderAt, + diffBuildId *uuid.UUID, + mappings []*header.BuildMap, +) error { + for _, mapping := range mappings { + if mapping.BuildId.String() != diffBuildId.String() { + continue + } + + for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { + c1 := make([]byte, blockSize) + n1, err := s1.ReadAt(c1, int64(off)) + if err != nil { + return fmt.Errorf("failed to read content1: %w", err) + } + + c2 := make([]byte, blockSize) + n2, err := s2.ReadAt(c2, int64(off)) + if err != nil { + return fmt.Errorf("failed to read content2: %w", err) + } + + if n1 != n2 { + return fmt.Errorf("content length mismatch: %d != %d", n1, n2) + } + + if !bytes.Equal(c1, c2) { + // Hash the content and show the mismatch + h1 := sha256.Sum256(c1) + h2 := sha256.Sum256(c2) + + return fmt.Errorf("content mismatch (showing hashes):\n%x != %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) + } + } + } + + return nil +} + +func checkNbd(ctx context.Context, buildId string) error { + store, err := build.NewDiffStore(gcs.TemplateBucket, ctx) + if err != nil { + return fmt.Errorf("failed to create diff store: %w", err) + } + + baseRootfs, baseHeader, err := fileFromStorage(ctx, buildId, store) + if err != nil { + return fmt.Errorf("failed to create base overlay: %w", err) + } + + baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) + baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) + if err != nil { + return fmt.Errorf("failed to create cache: %w", err) + } + defer baseCache.Close() + + baseOverlay := block.NewOverlay( + &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, + baseCache, + blockSize, + ) + + defer baseOverlay.Close() + + fmt.Printf("\n----- Base overlay mount -----\n\n") + + // Modify content in the overlay mounted from the base rootfs + err = executeForNbd(ctx, baseOverlay, func(mountedPath string) error { + contentPath := filepath.Join(mountedPath, contentFileName) + + err = os.WriteFile(contentPath, []byte(content), 0644) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + // fmt.Printf("- writing to content Path: %s\n", contentPath) + + // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() + // if err != nil { + // return fmt.Errorf("failed to sync: %w - %s", err, out) + // } + + // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() + // if err != nil { + // return fmt.Errorf("failed to sync: %w - %s", err, out) + // } + + return nil + }) + + diffRootfs, diffHeader, err := fileFromOverlay(baseOverlay, baseHeader, store) + if err != nil { + return fmt.Errorf("failed to create diff overlay: %w", err) + } + + fmt.Printf("\n----- Diff header -----\n") + + for _, mapping := range diffHeader.Mapping { + fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) + } + + // Compare the changed parts in the base overlay with the content in the diff overlay. + err = compareSources( + baseOverlay, + diffRootfs, + &diffHeader.Metadata.BuildId, + diffHeader.Mapping, + ) + if err != nil { + return fmt.Errorf("failed to compare overlays: %w", err) + } else { + fmt.Println("overlay comparison successful") + } + + diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) + + diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) + if err != nil { + return fmt.Errorf("failed to create cache: %w", err) + } + defer diffCache.Close() + + diffOverlay := block.NewOverlay( + &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, + diffCache, + blockSize, + ) + defer diffOverlay.Close() + + fmt.Printf("\n----- Diff overlay mount -----\n\n") + + // Check the modified content in the overlay created from the diff and the base + err = executeForNbd(ctx, diffOverlay, func(mountedPath string) error { + contentPath := filepath.Join(mountedPath, contentFileName) + + readContent, err := os.ReadFile(contentPath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + fmt.Printf("- reading from content Path: %s\n", contentPath) + fmt.Printf("- content: %s\n", readContent) + + if string(readContent) != content { + return fmt.Errorf("content mismatch: %s\n", readContent) + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to mount nbd: %w", err) + } + + return nil +} + +func main() { + buildId := flag.String("build", "", "template build id") + + flag.Parse() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan os.Signal, 1) + signal.Notify(done, os.Interrupt) + + go func() { + <-done + + cancel() + }() + + fmt.Printf("\n================== ROOTFS DIFF TEST ===================") + + err := checkNbd(ctx, *buildId) + if err != nil { + fmt.Fprintf(os.Stderr, "\n\n => failed:\n %s\n\n", err) + } else { + fmt.Println("\n\n => success") + } +} From 675858b2a6e2659bbf87a46a13eb89b29525740c Mon Sep 17 00:00:00 2001 From: Jakub Dobry Date: Sat, 15 Feb 2025 02:26:58 +0000 Subject: [PATCH 06/21] add rootfs_test with rootfs diff testing --- packages/orchestrator/Makefile | 4 + .../internal/sandbox/build/cache.go | 5 + .../internal/sandbox/rootfs_test.go | 445 ++++++++++++++++++ 3 files changed, 454 insertions(+) create mode 100644 packages/orchestrator/internal/sandbox/rootfs_test.go diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index f70f276217..94066cad29 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -68,3 +68,7 @@ kill-old: .PHONY: mock-snapshot mock-snapshot: sudo TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) CONSUL_TOKEN=$(CONSUL_TOKEN) NODE_ID="test-client" go run cmd/mock-snapshot/mock.go -template 5wzg6c91u51yaebviysf -build "f0370054-b669-4d7e-b33b-573d5287c6ef" -alive 1 -count 1 + +.PHONY: test +test: + sudo TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) CONSUL_TOKEN=$(CONSUL_TOKEN) NODE_ID="test-client" go test ./internal/sandbox/. diff --git a/packages/orchestrator/internal/sandbox/build/cache.go b/packages/orchestrator/internal/sandbox/build/cache.go index 999aa09690..2daa393dd0 100644 --- a/packages/orchestrator/internal/sandbox/build/cache.go +++ b/packages/orchestrator/internal/sandbox/build/cache.go @@ -78,3 +78,8 @@ func (s *DiffStore) Add(buildId string, t DiffType, d Diff) { s.cache.Set(storagePath, d, buildExpiration) } + +func (s *DiffStore) Close() { + s.cache.DeleteAll() + s.cache.Stop() +} diff --git a/packages/orchestrator/internal/sandbox/rootfs_test.go b/packages/orchestrator/internal/sandbox/rootfs_test.go new file mode 100644 index 0000000000..8ee2154f2a --- /dev/null +++ b/packages/orchestrator/internal/sandbox/rootfs_test.go @@ -0,0 +1,445 @@ +package sandbox + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +type DeviceWithClose struct { + *build.File + id string + size int64 +} + +func (d *DeviceWithClose) Close() error { + return nil +} + +func (d *DeviceWithClose) Size() (int64, error) { + return d.size, nil +} + +func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { + return d.File.ReadAt(p, off) +} + +const ( + contentFileName = "test.txt" + content = `test-file-content` + blockSize = 4096 +) + +// Execute the passed callback with the passed overlay mounted as a nbd device. +func executeForNbd( + t *testing.T, + ctx context.Context, + overlay *block.Overlay, + cb func(mountedPath string) error, +) error { + mnt := nbd.NewDirectPathMount(overlay) + + nbdCtx, nbdCancel := context.WithCancel(ctx) + defer nbdCancel() + + go func() { + <-nbdCtx.Done() + + mnt.Close() + }() + + mntIndex, err := mnt.Open(nbdCtx) + if err != nil { + return fmt.Errorf("failed to open: %w", err) + } + + devicePath := nbd.GetDevicePath(mntIndex) + + t.Logf("- created nbd device at %s\n", devicePath) + + // Check the block device health + out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to fsck: %w %s", err, out) + } + + t.Logf("- fscked base nbd device at %s\n", out) + + tmpDir, err := os.MkdirTemp("", "mount-*") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + + defer os.RemoveAll(tmpDir) + + out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to mount: %w - %s", err, out) + } + + defer func() { + out, err = exec.Command("umount", tmpDir).CombinedOutput() + if err != nil { + t.Errorf("failed to umount: %s - %s", err, out) + } + }() + + t.Logf("- mounted rootfs at %s\n", devicePath) + + err = cb(tmpDir) + if err != nil { + return fmt.Errorf("failed execute: %w", err) + } + + return nil +} + +// Create a test file with given size formatted as ext4 +func createRootfsFile(t *testing.T, size int64) (*os.File, error) { + f, err := os.CreateTemp("", "test-rootfs-*.ext4") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + t.Cleanup(func() { + f.Close() + os.Remove(f.Name()) + }) + + // Ensure the file is big enough + err = f.Truncate(size) + if err != nil { + return nil, fmt.Errorf("failed to truncate file: %w", err) + } + + // Close before formatting + err = f.Close() + if err != nil { + os.Remove(f.Name()) + return nil, fmt.Errorf("failed to close file: %w", err) + } + + // Format as ext4 + out, err := exec.Command("mkfs.ext4", "-F", "-b", fmt.Sprintf("%d", blockSize), f.Name()).CombinedOutput() + if err != nil { + os.Remove(f.Name()) + return nil, fmt.Errorf("failed to format ext4: %w - %s", err, out) + } + + // Reopen the file + f, err = os.OpenFile(f.Name(), os.O_RDWR, 0644) + if err != nil { + os.Remove(f.Name()) + return nil, fmt.Errorf("failed to reopen file: %w", err) + } + + return f, nil +} + +// TestDiff implements the build.Diff interface for testing +type TestDiff struct { + reader io.ReaderAt + size int64 +} + +func (d *TestDiff) Close() error { + return nil +} + +func (d *TestDiff) ReadAt(p []byte, off int64) (int, error) { + return d.reader.ReadAt(p, off) +} + +func (d *TestDiff) Slice(off, length int64) ([]byte, error) { + buf := make([]byte, length) + n, err := d.ReadAt(buf, off) + if err != nil { + return nil, err + } + return buf[:n], nil +} + +func (d *TestDiff) CachePath() (string, error) { + return "", nil +} + +func baseRootfs( + t *testing.T, + ctx context.Context, + store *build.DiffStore, +) (*build.File, *header.Header, error) { + buildId := uuid.New() + + // Create a test file of 10MB + rootfsFile, err := createRootfsFile(t, 10*1024*1024) + if err != nil { + return nil, nil, fmt.Errorf("failed to create test file: %w", err) + } + + fi, err := rootfsFile.Stat() + if err != nil { + return nil, nil, fmt.Errorf("failed to get file info: %w", err) + } + + header := header.NewHeader(&header.Metadata{ + BuildId: buildId, + BaseBuildId: buildId, + Size: uint64(fi.Size()), + Version: 1, + BlockSize: uint64(blockSize), + Generation: 1, + }, nil) + + // Create a test diff and add it to the store + diff := &TestDiff{ + reader: rootfsFile, + size: fi.Size(), + } + store.Add(buildId.String(), build.Rootfs, diff) + + rootfs := build.NewFile(header, store, build.Rootfs) + + return rootfs, header, nil +} + +// Create an overlay by extracting the diff from the passed overlay, +// putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. +func fileFromOverlay( + t *testing.T, + overlay *block.Overlay, + baseHeader *header.Header, + store *build.DiffStore, +) (*build.File, *header.Header, error) { + diffBuildId := uuid.New() + + diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) + if err != nil { + return nil, nil, fmt.Errorf("failed to create diff file: %w", err) + } + t.Cleanup(func() { + diffFile.Close() + os.Remove(diffFile.Name()) + }) + + cache, err := overlay.EjectCache() + if err != nil { + return nil, nil, fmt.Errorf("failed to eject cache: %w", err) + } + + dirtyBlocks, err := cache.Export(diffFile) + if err != nil { + return nil, nil, fmt.Errorf("failed to export cache: %w", err) + } + + diff, err := diffFile.ToDiff(blockSize) + if err != nil { + return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) + } + + newMappings := header.CreateMapping( + baseHeader.Metadata, + &diffBuildId, + dirtyBlocks, + ) + + diffMappings := header.MergeMappings( + baseHeader.Mapping, + newMappings, + ) + + diffHeader := header.NewHeader(&header.Metadata{ + BuildId: diffBuildId, + BaseBuildId: baseHeader.Metadata.BaseBuildId, + Size: baseHeader.Metadata.Size, + Version: baseHeader.Metadata.Version, + BlockSize: baseHeader.Metadata.BlockSize, + }, diffMappings) + + store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) + + // Create a build file that will use the diff and the base already in the store + diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) + + return diffRootfs, diffHeader, nil +} + +func compareSources( + s1, + s2 io.ReaderAt, + diffBuildId *uuid.UUID, + mappings []*header.BuildMap, +) error { + for _, mapping := range mappings { + if mapping.BuildId.String() != diffBuildId.String() { + continue + } + + for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { + c1 := make([]byte, blockSize) + n1, err := s1.ReadAt(c1, int64(off)) + if err != nil { + return fmt.Errorf("failed to read content1: %w", err) + } + + c2 := make([]byte, blockSize) + n2, err := s2.ReadAt(c2, int64(off)) + if err != nil { + return fmt.Errorf("failed to read content2: %w", err) + } + + if n1 != n2 { + return fmt.Errorf("content length mismatch: %d != %d", n1, n2) + } + + if !bytes.Equal(c1, c2) { + // Hash the content and show the mismatch + h1 := sha256.Sum256(c1) + h2 := sha256.Sum256(c2) + + return fmt.Errorf("content mismatch (showing hashes):\nBase: %x != Diff: %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) + } + } + } + + return nil +} + +func checkNbd(t *testing.T, ctx context.Context) error { + store, err := build.NewDiffStore(nil, ctx) + if err != nil { + return fmt.Errorf("failed to create diff store: %w", err) + } + t.Cleanup(func() { + store.Close() + }) + + baseRootfs, baseHeader, err := baseRootfs(t, ctx, store) + if err != nil { + return fmt.Errorf("failed to create base overlay: %w", err) + } + + baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) + baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) + if err != nil { + return fmt.Errorf("failed to create cache: %w", err) + } + defer baseCache.Close() + + baseOverlay := block.NewOverlay( + &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, + baseCache, + blockSize, + ) + + defer baseOverlay.Close() + + t.Logf("\n----- Base overlay mount -----\n\n") + + // Modify content in the overlay mounted from the base rootfs + err = executeForNbd(t, ctx, baseOverlay, func(mountedPath string) error { + contentPath := filepath.Join(mountedPath, contentFileName) + + err = os.WriteFile(contentPath, []byte(content), 0644) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + // fmt.Printf("- writing to content Path: %s\n", contentPath) + + // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() + // if err != nil { + // return fmt.Errorf("failed to sync: %w - %s", err, out) + // } + + // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() + // if err != nil { + // return fmt.Errorf("failed to sync: %w - %s", err, out) + // } + + return nil + }) + + diffRootfs, diffHeader, err := fileFromOverlay(t, baseOverlay, baseHeader, store) + if err != nil { + return fmt.Errorf("failed to create diff overlay: %w", err) + } + + t.Logf("\n----- Diff header -----\n") + + for _, mapping := range diffHeader.Mapping { + t.Log(mapping.Format(baseHeader.Metadata.BlockSize)) + } + + // Compare the changed parts in the base overlay with the content in the diff overlay. + err = compareSources( + baseOverlay, + diffRootfs, + &diffHeader.Metadata.BuildId, + diffHeader.Mapping, + ) + if err != nil { + return fmt.Errorf("failed to compare overlays: %w", err) + } else { + t.Logf("overlay comparison successful") + } + + diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) + + diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) + if err != nil { + return fmt.Errorf("failed to create cache: %w", err) + } + defer diffCache.Close() + + diffOverlay := block.NewOverlay( + &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, + diffCache, + blockSize, + ) + defer diffOverlay.Close() + + t.Logf("\n----- Diff overlay mount -----\n\n") + + // Check the modified content in the overlay created from the diff and the base + err = executeForNbd(t, ctx, diffOverlay, func(mountedPath string) error { + contentPath := filepath.Join(mountedPath, contentFileName) + + readContent, err := os.ReadFile(contentPath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + t.Logf("- reading from content Path: %s\n", contentPath) + t.Logf("- content: %s\n", readContent) + + assert.Equal(t, content, string(readContent)) + + return nil + }) + if err != nil { + return fmt.Errorf("failed to mount nbd: %w", err) + } + + return nil +} + +func TestNbd(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := checkNbd(t, ctx) + assert.NoError(t, err) +} From b94dbaf2d0b2b713daf26b8b152880c371f1064d Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 00:28:48 -0800 Subject: [PATCH 07/21] Cleanup --- .github/workflows/pr-tests.yml | 10 + .../orchestrator/cmd/mount-rootfs/main.go | 187 +++- .../orchestrator/cmd/mount-rootfs/start.sh | 7 +- .../orchestrator/cmd/test-rootfs-diff/main.go | 756 +++++++-------- .../internal/sandbox/build/cache.go | 2 +- .../internal/sandbox/rootfs_test.go | 888 +++++++++--------- 6 files changed, 993 insertions(+), 857 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index d3f3973974..760d9cfa2b 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -46,6 +46,16 @@ jobs: sudo mount -t hugetlbfs none /mnt/hugepages echo 128 | sudo tee /proc/sys/vm/nr_hugepages + - name: Enable NBD + run: | + sudo modprobe nbd nbds_max=256 + + - name: Disable inotify watching of change events for NBD devices + run: | + echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules + sudo udevadm control --reload-rules + sudo udevadm trigger + - name: Run tests working-directory: ${{ matrix.package }} run: go test -v ${{ matrix.test_path }} diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index e67d95f8a6..d042538747 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "os" + "os/exec" "os/signal" "path/filepath" "time" @@ -56,9 +57,70 @@ func (m *mountReadonlyDevice) Size() (int64, error) { return int64(m.header.Metadata.Size), nil } +type loggedOverlay struct { + overlay *block.Overlay +} + +func (l *loggedOverlay) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stdout, "[read panic recovered]: [%d, %d] -> %v\n", off, len(p), r) + } + }() + + fmt.Fprintf(os.Stdout, "[read started]: [%d, %d]\n", off, len(p)) + + n, err := l.overlay.ReadAt(ctx, p, off) + + fmt.Fprintf(os.Stdout, "[read completed]: [%d, %d] -> %d\n", off, len(p), n) + + return n, err +} + +func (l *loggedOverlay) WriteAt(p []byte, off int64) (int, error) { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stdout, "[write panic recovered]: [%d, %d] -> %v\n", off, len(p), r) + } + }() + + fmt.Fprintf(os.Stdout, "[write started]: [%d, %d]\n", off, len(p)) + + n, err := l.overlay.WriteAt(p, off) + + fmt.Fprintf(os.Stdout, "[write completed]: [%d, %d] -> %d\n", off, len(p), n) + + return n, err +} + +func (l *loggedOverlay) Size() (int64, error) { + return l.overlay.Size() +} + +func (l *loggedOverlay) BlockSize() int64 { + return l.overlay.BlockSize() +} + +func (l *loggedOverlay) Header() *header.Header { + return l.overlay.Header() +} + +func (l *loggedOverlay) Close() error { + return l.overlay.Close() +} + +func (l *loggedOverlay) EjectCache() (*block.Cache, error) { + return l.overlay.EjectCache() +} + +func (l *loggedOverlay) Slice(ctx context.Context, off, length int64) ([]byte, error) { + return l.overlay.Slice(ctx, off, length) +} + func main() { buildId := flag.String("build", "", "build id") mountPath := flag.String("mount", "", "mount path") + verify := flag.Bool("verify", false, "verify rootfs integrity") flag.Parse() @@ -84,13 +146,13 @@ func main() { // We use a separate ctx for majority of the operations as cancelling context for the NBD+storage and *then* doing cleanup for these often resulted in deadlocks. nbdContext := context.Background() - err := mountRootfs(ctx, nbdContext, *buildId, *mountPath) + err := mountRootfs(ctx, nbdContext, *buildId, *mountPath, *verify) if err != nil { panic(fmt.Errorf("failed to mount rootfs: %w", err)) } } -func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string) error { +func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string, verify bool) error { files := storage.TemplateFiles{ BuildID: buildID, } @@ -220,7 +282,7 @@ func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string) err close(poolClosed) }() - mnt := nbd.NewDirectPathMount(overlay, devicePool) + mnt := nbd.NewDirectPathMount(&loggedOverlay{overlay}, devicePool) mntIndex, err := mnt.Open(nbdContext) if err != nil { @@ -241,45 +303,112 @@ func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string) err fmt.Printf("rootfs exposed as device: %s\n", devicePath) - err = os.MkdirAll(mountPath, 0o755) - if err != nil { - return fmt.Errorf("failed to create mount path directory: %w", err) - } + if mountPath != "" { + err = os.MkdirAll(mountPath, 0o755) + if err != nil { + return fmt.Errorf("failed to create mount path directory: %w", err) + } - fmt.Fprintf(os.Stdout, "creating mount path directory: %s\n", mountPath) + fmt.Fprintf(os.Stdout, "creating mount path directory: %s\n", mountPath) - // We don't remote the dir as it might have been user created. + // We don't remote the dir as it might have been user created. - err = unix.Mount(devicePath, mountPath, "ext4", unix.MS_RDONLY, "") - if err != nil { - return fmt.Errorf("failed to mount device to mount path: %w", err) - } + err = unix.Mount(devicePath, mountPath, "ext4", 0, "") + if err != nil { + return fmt.Errorf("failed to mount device to mount path: %w", err) + } - defer func() { - ticker := time.NewTicker(600 * time.Millisecond) - defer ticker.Stop() + defer func() { + ticker := time.NewTicker(600 * time.Millisecond) + defer ticker.Stop() - cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cleanupCancel() + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() - for { - select { - case <-cleanupCtx.Done(): - fmt.Fprintf(os.Stderr, "failed to unmount device from mount path in time\n") + for { + select { + case <-cleanupCtx.Done(): + fmt.Fprintf(os.Stderr, "failed to unmount device from mount path in time\n") - return - case <-ticker.C: - err = unix.Unmount(mountPath, 0) - if err == nil { return + case <-ticker.C: + err = unix.Unmount(mountPath, 0) + if err == nil { + return + } + + fmt.Fprintf(os.Stderr, "failed to unmount device from mount path: %v\n", err) } + } + }() + + fmt.Printf("rootfs mounted at path: %s\n", mountPath) + } + + // cmd := exec.CommandContext(ctx, "dd", "if=/dev/zero", "of="+devicePath, "bs=4k", "count=1", "oflag=direct") + + // cmd.Stdout = os.Stdout + // cmd.Stderr = os.Stderr + + // err = cmd.Run() + // if err != nil { + // return fmt.Errorf("failed to write zero to device (with direct flag): %w", err) + // } + + // fmt.Println("> zero written to device (with direct flag)") + + // d, err := os.OpenFile(devicePath, unix.O_DIRECT|unix.O_RDWR, 0) + // if err != nil { + // return fmt.Errorf("failed to open device: %w", err) + // } + // defer d.Close() + + // buf := make([]byte, 4096) - fmt.Fprintf(os.Stderr, "failed to unmount device from mount path: %v\n", err) + // // fmt.Println("mmapped buffer start", unsafe.Pointer(&buf[0])) + // _, err = d.WriteAt(buf, 0) + // if err != nil { + // return fmt.Errorf("failed to write zero to device: %w", err) + // } + + // fmt.Println("zero written to device") + + if verify { + fmt.Println("\nverifying rootfs integrity...") + + cmd := exec.CommandContext(ctx, "e2fsck", "-nfv", devicePath) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to verify rootfs integrity: %w", err) + } + + fmt.Println("\nrootfs integrity verified") + + journalDir := filepath.Join(mountPath, "var", "log", "journal") + journalFiles, err := os.ReadDir(journalDir) + if err != nil { + return fmt.Errorf("failed to read journal directory: %w", err) + } + + for _, journalFile := range journalFiles { + cmd := exec.CommandContext(ctx, "journalctl", "--verify", "--directory", filepath.Join(journalDir, journalFile.Name())) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to verify journal file: %w", err) } } - }() - fmt.Printf("rootfs mounted at path: %s\n", mountPath) + fmt.Println("\njournal files verified") + + return nil + } <-ctx.Done() diff --git a/packages/orchestrator/cmd/mount-rootfs/start.sh b/packages/orchestrator/cmd/mount-rootfs/start.sh index 989f25f1bc..73da5567ab 100755 --- a/packages/orchestrator/cmd/mount-rootfs/start.sh +++ b/packages/orchestrator/cmd/mount-rootfs/start.sh @@ -2,13 +2,8 @@ set -euo pipefail -bucket=$1 -build=$2 -mount_path=$3 - go build -o ./bin/mount-rootfs ./cmd/mount-rootfs -TEMPLATE_BUCKET_NAME=$bucket \ GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/gcloud/application_default_credentials.json" \ sudo --preserve-env=GOOGLE_APPLICATION_CREDENTIALS,TEMPLATE_BUCKET_NAME \ -./bin/mount-rootfs -build $build -mount $mount_path +./bin/mount-rootfs "$@" \ No newline at end of file diff --git a/packages/orchestrator/cmd/test-rootfs-diff/main.go b/packages/orchestrator/cmd/test-rootfs-diff/main.go index 2bacf9212f..c68a4921fc 100644 --- a/packages/orchestrator/cmd/test-rootfs-diff/main.go +++ b/packages/orchestrator/cmd/test-rootfs-diff/main.go @@ -1,383 +1,383 @@ package main -import ( - "bytes" - "context" - "crypto/sha256" - "flag" - "fmt" - "io" - "os" - "os/exec" - "os/signal" - "path/filepath" - - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" - "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" - "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" - "github.com/google/uuid" -) - -type DeviceWithClose struct { - *build.File - id string - size int64 -} - -func (d *DeviceWithClose) Close() error { - return nil -} - -func (d *DeviceWithClose) Size() (int64, error) { - return d.size, nil -} - -func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { - return d.File.ReadAt(p, off) -} - -const ( - contentFileName = "test.txt" - content = `` - blockSize = 4096 -) - -// Execute the passed callback with the passed overlay mounted as a nbd device. -func executeForNbd( - ctx context.Context, - overlay *block.Overlay, - cb func(mountedPath string) error, -) error { - mnt := nbd.NewDirectPathMount(overlay) - - nbdCtx, nbdCancel := context.WithCancel(ctx) - defer nbdCancel() - - go func() { - <-nbdCtx.Done() - - mnt.Close() - }() - - mntIndex, err := mnt.Open(nbdCtx) - if err != nil { - return fmt.Errorf("failed to open: %w", err) - } - - devicePath := nbd.GetDevicePath(mntIndex) - - fmt.Printf("- created nbd device at %s\n", devicePath) - - // Check the block device health - out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to fsck: %w %s", err, out) - } - - fmt.Printf("- fscked base nbd device at %s\n", out) - - tmpDir, err := os.MkdirTemp("", "mount-*") - if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) - } - - defer os.RemoveAll(tmpDir) - - out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to mount: %w - %s", err, out) - } - - defer func() { - out, err = exec.Command("umount", tmpDir).CombinedOutput() - if err != nil { - fmt.Fprintf(os.Stderr, "failed to umount: %s - %s", err, out) - } - }() - - fmt.Printf("- mounted rootfs at %s\n", devicePath) - - err = cb(tmpDir) - if err != nil { - return fmt.Errorf("failed execute: %w", err) - } - - return nil -} - -func fileFromStorage( - ctx context.Context, - baseBuildId string, - store *build.DiffStore, -) (*build.File, *header.Header, error) { - buildId, err := uuid.Parse(baseBuildId) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse build id: %w", err) - } - - object := gcs.NewObject(ctx, gcs.TemplateBucket, buildId.String()+"/"+string(build.Rootfs)) - - size, err := object.Size() - if err != nil { - return nil, nil, fmt.Errorf("failed to get object size: %w", err) - } - - header := header.NewHeader(&header.Metadata{ - BuildId: buildId, - BaseBuildId: buildId, - Size: uint64(size), - Version: 1, - BlockSize: uint64(blockSize), - Generation: 1, - }, nil) - - rootfs := build.NewFile(header, store, build.Rootfs) - - return rootfs, header, nil -} - -// Create an overlay by extracting the diff from the passed overlay, -// putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. -func fileFromOverlay( - overlay *block.Overlay, - baseHeader *header.Header, - store *build.DiffStore, -) (*build.File, *header.Header, error) { - diffBuildId := uuid.New() - - // TODO: Diff file is not cleaned up after exit. The same goes for the base storage file in the cache. - // This should not affect the test. - diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) - if err != nil { - return nil, nil, fmt.Errorf("failed to create diff file: %w", err) - } - - cache, err := overlay.EjectCache() - if err != nil { - return nil, nil, fmt.Errorf("failed to eject cache: %w", err) - } - - dirtyBlocks, err := cache.Export(diffFile) - if err != nil { - return nil, nil, fmt.Errorf("failed to export cache: %w", err) - } - - diff, err := diffFile.ToDiff(blockSize) - if err != nil { - return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) - } - - newMappings := header.CreateMapping( - baseHeader.Metadata, - &diffBuildId, - dirtyBlocks, - ) - - diffMappings := header.MergeMappings( - baseHeader.Mapping, - newMappings, - ) - - diffHeader := header.NewHeader(&header.Metadata{ - BuildId: diffBuildId, - BaseBuildId: baseHeader.Metadata.BaseBuildId, - Size: baseHeader.Metadata.Size, - Version: baseHeader.Metadata.Version, - BlockSize: baseHeader.Metadata.BlockSize, - }, diffMappings) - - store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) - - // Create a build file that will use the diff and the base already in the store - diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) - - return diffRootfs, diffHeader, nil -} - -func compareSources( - s1, - s2 io.ReaderAt, - diffBuildId *uuid.UUID, - mappings []*header.BuildMap, -) error { - for _, mapping := range mappings { - if mapping.BuildId.String() != diffBuildId.String() { - continue - } - - for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { - c1 := make([]byte, blockSize) - n1, err := s1.ReadAt(c1, int64(off)) - if err != nil { - return fmt.Errorf("failed to read content1: %w", err) - } - - c2 := make([]byte, blockSize) - n2, err := s2.ReadAt(c2, int64(off)) - if err != nil { - return fmt.Errorf("failed to read content2: %w", err) - } - - if n1 != n2 { - return fmt.Errorf("content length mismatch: %d != %d", n1, n2) - } - - if !bytes.Equal(c1, c2) { - // Hash the content and show the mismatch - h1 := sha256.Sum256(c1) - h2 := sha256.Sum256(c2) - - return fmt.Errorf("content mismatch (showing hashes):\n%x != %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) - } - } - } - - return nil -} - -func checkNbd(ctx context.Context, buildId string) error { - store, err := build.NewDiffStore(gcs.TemplateBucket, ctx) - if err != nil { - return fmt.Errorf("failed to create diff store: %w", err) - } - - baseRootfs, baseHeader, err := fileFromStorage(ctx, buildId, store) - if err != nil { - return fmt.Errorf("failed to create base overlay: %w", err) - } - - baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) - baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) - if err != nil { - return fmt.Errorf("failed to create cache: %w", err) - } - defer baseCache.Close() - - baseOverlay := block.NewOverlay( - &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, - baseCache, - blockSize, - ) - - defer baseOverlay.Close() - - fmt.Printf("\n----- Base overlay mount -----\n\n") - - // Modify content in the overlay mounted from the base rootfs - err = executeForNbd(ctx, baseOverlay, func(mountedPath string) error { - contentPath := filepath.Join(mountedPath, contentFileName) - - err = os.WriteFile(contentPath, []byte(content), 0644) - if err != nil { - return fmt.Errorf("failed to write file: %w", err) - } - - // fmt.Printf("- writing to content Path: %s\n", contentPath) - - // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() - // if err != nil { - // return fmt.Errorf("failed to sync: %w - %s", err, out) - // } - - // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() - // if err != nil { - // return fmt.Errorf("failed to sync: %w - %s", err, out) - // } - - return nil - }) - - diffRootfs, diffHeader, err := fileFromOverlay(baseOverlay, baseHeader, store) - if err != nil { - return fmt.Errorf("failed to create diff overlay: %w", err) - } - - fmt.Printf("\n----- Diff header -----\n") - - for _, mapping := range diffHeader.Mapping { - fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) - } - - // Compare the changed parts in the base overlay with the content in the diff overlay. - err = compareSources( - baseOverlay, - diffRootfs, - &diffHeader.Metadata.BuildId, - diffHeader.Mapping, - ) - if err != nil { - return fmt.Errorf("failed to compare overlays: %w", err) - } else { - fmt.Println("overlay comparison successful") - } - - diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) - - diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) - if err != nil { - return fmt.Errorf("failed to create cache: %w", err) - } - defer diffCache.Close() - - diffOverlay := block.NewOverlay( - &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, - diffCache, - blockSize, - ) - defer diffOverlay.Close() - - fmt.Printf("\n----- Diff overlay mount -----\n\n") - - // Check the modified content in the overlay created from the diff and the base - err = executeForNbd(ctx, diffOverlay, func(mountedPath string) error { - contentPath := filepath.Join(mountedPath, contentFileName) - - readContent, err := os.ReadFile(contentPath) - if err != nil { - return fmt.Errorf("failed to read file: %w", err) - } - - fmt.Printf("- reading from content Path: %s\n", contentPath) - fmt.Printf("- content: %s\n", readContent) - - if string(readContent) != content { - return fmt.Errorf("content mismatch: %s\n", readContent) - } - - return nil - }) - if err != nil { - return fmt.Errorf("failed to mount nbd: %w", err) - } - - return nil -} +// import ( +// "bytes" +// "context" +// "crypto/sha256" +// "flag" +// "fmt" +// "io" +// "os" +// "os/exec" +// "os/signal" +// "path/filepath" + +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/disk/nbd" +// "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" +// "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +// "github.com/google/uuid" +// ) + +// type DeviceWithClose struct { +// *build.File +// id string +// size int64 +// } + +// func (d *DeviceWithClose) Close() error { +// return nil +// } + +// func (d *DeviceWithClose) Size() (int64, error) { +// return d.size, nil +// } + +// func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { +// return d.File.ReadAt(p, off) +// } + +// const ( +// contentFileName = "test.txt" +// content = `` +// blockSize = 4096 +// ) + +// // Execute the passed callback with the passed overlay mounted as a nbd device. +// func executeForNbd( +// ctx context.Context, +// overlay *block.Overlay, +// cb func(mountedPath string) error, +// ) error { +// mnt := nbd.NewDirectPathMount(overlay) + +// nbdCtx, nbdCancel := context.WithCancel(ctx) +// defer nbdCancel() + +// go func() { +// <-nbdCtx.Done() + +// mnt.Close() +// }() + +// mntIndex, err := mnt.Open(nbdCtx) +// if err != nil { +// return fmt.Errorf("failed to open: %w", err) +// } + +// devicePath := nbd.GetDevicePath(mntIndex) + +// fmt.Printf("- created nbd device at %s\n", devicePath) + +// // Check the block device health +// out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() +// if err != nil { +// return fmt.Errorf("failed to fsck: %w %s", err, out) +// } + +// fmt.Printf("- fscked base nbd device at %s\n", out) + +// tmpDir, err := os.MkdirTemp("", "mount-*") +// if err != nil { +// return fmt.Errorf("failed to create temp dir: %w", err) +// } + +// defer os.RemoveAll(tmpDir) + +// out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() +// if err != nil { +// return fmt.Errorf("failed to mount: %w - %s", err, out) +// } + +// defer func() { +// out, err = exec.Command("umount", tmpDir).CombinedOutput() +// if err != nil { +// fmt.Fprintf(os.Stderr, "failed to umount: %s - %s", err, out) +// } +// }() + +// fmt.Printf("- mounted rootfs at %s\n", devicePath) + +// err = cb(tmpDir) +// if err != nil { +// return fmt.Errorf("failed execute: %w", err) +// } + +// return nil +// } + +// func fileFromStorage( +// ctx context.Context, +// baseBuildId string, +// store *build.DiffStore, +// ) (*build.File, *header.Header, error) { +// buildId, err := uuid.Parse(baseBuildId) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to parse build id: %w", err) +// } + +// object := gcs.NewObject(ctx, gcs.TemplateBucket, buildId.String()+"/"+string(build.Rootfs)) + +// size, err := object.Size() +// if err != nil { +// return nil, nil, fmt.Errorf("failed to get object size: %w", err) +// } + +// header := header.NewHeader(&header.Metadata{ +// BuildId: buildId, +// BaseBuildId: buildId, +// Size: uint64(size), +// Version: 1, +// BlockSize: uint64(blockSize), +// Generation: 1, +// }, nil) + +// rootfs := build.NewFile(header, store, build.Rootfs) + +// return rootfs, header, nil +// } + +// // Create an overlay by extracting the diff from the passed overlay, +// // putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. +// func fileFromOverlay( +// overlay *block.Overlay, +// baseHeader *header.Header, +// store *build.DiffStore, +// ) (*build.File, *header.Header, error) { +// diffBuildId := uuid.New() + +// // TODO: Diff file is not cleaned up after exit. The same goes for the base storage file in the cache. +// // This should not affect the test. +// diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to create diff file: %w", err) +// } + +// cache, err := overlay.EjectCache() +// if err != nil { +// return nil, nil, fmt.Errorf("failed to eject cache: %w", err) +// } + +// dirtyBlocks, err := cache.Export(diffFile) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to export cache: %w", err) +// } + +// diff, err := diffFile.ToDiff(blockSize) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) +// } + +// newMappings := header.CreateMapping( +// baseHeader.Metadata, +// &diffBuildId, +// dirtyBlocks, +// ) + +// diffMappings := header.MergeMappings( +// baseHeader.Mapping, +// newMappings, +// ) + +// diffHeader := header.NewHeader(&header.Metadata{ +// BuildId: diffBuildId, +// BaseBuildId: baseHeader.Metadata.BaseBuildId, +// Size: baseHeader.Metadata.Size, +// Version: baseHeader.Metadata.Version, +// BlockSize: baseHeader.Metadata.BlockSize, +// }, diffMappings) + +// store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) + +// // Create a build file that will use the diff and the base already in the store +// diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) + +// return diffRootfs, diffHeader, nil +// } + +// func compareSources( +// s1, +// s2 io.ReaderAt, +// diffBuildId *uuid.UUID, +// mappings []*header.BuildMap, +// ) error { +// for _, mapping := range mappings { +// if mapping.BuildId.String() != diffBuildId.String() { +// continue +// } + +// for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { +// c1 := make([]byte, blockSize) +// n1, err := s1.ReadAt(c1, int64(off)) +// if err != nil { +// return fmt.Errorf("failed to read content1: %w", err) +// } + +// c2 := make([]byte, blockSize) +// n2, err := s2.ReadAt(c2, int64(off)) +// if err != nil { +// return fmt.Errorf("failed to read content2: %w", err) +// } + +// if n1 != n2 { +// return fmt.Errorf("content length mismatch: %d != %d", n1, n2) +// } + +// if !bytes.Equal(c1, c2) { +// // Hash the content and show the mismatch +// h1 := sha256.Sum256(c1) +// h2 := sha256.Sum256(c2) + +// return fmt.Errorf("content mismatch (showing hashes):\n%x != %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) +// } +// } +// } + +// return nil +// } + +// func checkNbd(ctx context.Context, buildId string) error { +// store, err := build.NewDiffStore(gcs.TemplateBucket, ctx) +// if err != nil { +// return fmt.Errorf("failed to create diff store: %w", err) +// } + +// baseRootfs, baseHeader, err := fileFromStorage(ctx, buildId, store) +// if err != nil { +// return fmt.Errorf("failed to create base overlay: %w", err) +// } + +// baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) +// baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) +// if err != nil { +// return fmt.Errorf("failed to create cache: %w", err) +// } +// defer baseCache.Close() + +// baseOverlay := block.NewOverlay( +// &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, +// baseCache, +// blockSize, +// ) + +// defer baseOverlay.Close() + +// fmt.Printf("\n----- Base overlay mount -----\n\n") + +// // Modify content in the overlay mounted from the base rootfs +// err = executeForNbd(ctx, baseOverlay, func(mountedPath string) error { +// contentPath := filepath.Join(mountedPath, contentFileName) + +// err = os.WriteFile(contentPath, []byte(content), 0644) +// if err != nil { +// return fmt.Errorf("failed to write file: %w", err) +// } + +// // fmt.Printf("- writing to content Path: %s\n", contentPath) + +// // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() +// // if err != nil { +// // return fmt.Errorf("failed to sync: %w - %s", err, out) +// // } + +// // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() +// // if err != nil { +// // return fmt.Errorf("failed to sync: %w - %s", err, out) +// // } + +// return nil +// }) + +// diffRootfs, diffHeader, err := fileFromOverlay(baseOverlay, baseHeader, store) +// if err != nil { +// return fmt.Errorf("failed to create diff overlay: %w", err) +// } + +// fmt.Printf("\n----- Diff header -----\n") + +// for _, mapping := range diffHeader.Mapping { +// fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) +// } + +// // Compare the changed parts in the base overlay with the content in the diff overlay. +// err = compareSources( +// baseOverlay, +// diffRootfs, +// &diffHeader.Metadata.BuildId, +// diffHeader.Mapping, +// ) +// if err != nil { +// return fmt.Errorf("failed to compare overlays: %w", err) +// } else { +// fmt.Println("overlay comparison successful") +// } + +// diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) + +// diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) +// if err != nil { +// return fmt.Errorf("failed to create cache: %w", err) +// } +// defer diffCache.Close() + +// diffOverlay := block.NewOverlay( +// &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, +// diffCache, +// blockSize, +// ) +// defer diffOverlay.Close() + +// fmt.Printf("\n----- Diff overlay mount -----\n\n") + +// // Check the modified content in the overlay created from the diff and the base +// err = executeForNbd(ctx, diffOverlay, func(mountedPath string) error { +// contentPath := filepath.Join(mountedPath, contentFileName) + +// readContent, err := os.ReadFile(contentPath) +// if err != nil { +// return fmt.Errorf("failed to read file: %w", err) +// } + +// fmt.Printf("- reading from content Path: %s\n", contentPath) +// fmt.Printf("- content: %s\n", readContent) + +// if string(readContent) != content { +// return fmt.Errorf("content mismatch: %s\n", readContent) +// } + +// return nil +// }) +// if err != nil { +// return fmt.Errorf("failed to mount nbd: %w", err) +// } + +// return nil +// } -func main() { - buildId := flag.String("build", "", "template build id") - - flag.Parse() +// func main() { +// buildId := flag.String("build", "", "template build id") + +// flag.Parse() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() - done := make(chan os.Signal, 1) - signal.Notify(done, os.Interrupt) - - go func() { - <-done - - cancel() - }() - - fmt.Printf("\n================== ROOTFS DIFF TEST ===================") - - err := checkNbd(ctx, *buildId) - if err != nil { - fmt.Fprintf(os.Stderr, "\n\n => failed:\n %s\n\n", err) - } else { - fmt.Println("\n\n => success") - } -} +// done := make(chan os.Signal, 1) +// signal.Notify(done, os.Interrupt) + +// go func() { +// <-done + +// cancel() +// }() + +// fmt.Printf("\n================== ROOTFS DIFF TEST ===================") + +// err := checkNbd(ctx, *buildId) +// if err != nil { +// fmt.Fprintf(os.Stderr, "\n\n => failed:\n %s\n\n", err) +// } else { +// fmt.Println("\n\n => success") +// } +// } diff --git a/packages/orchestrator/internal/sandbox/build/cache.go b/packages/orchestrator/internal/sandbox/build/cache.go index 9265c6457a..a40f05f1fe 100644 --- a/packages/orchestrator/internal/sandbox/build/cache.go +++ b/packages/orchestrator/internal/sandbox/build/cache.go @@ -304,7 +304,7 @@ func diskUsage(path string) (uint64, uint64, error) { return used, total, nil } -func (s *DiffStore) Close() { +func (s *DiffStore) Remove() { s.cache.DeleteAll() s.cache.Stop() } diff --git a/packages/orchestrator/internal/sandbox/rootfs_test.go b/packages/orchestrator/internal/sandbox/rootfs_test.go index 8ee2154f2a..3f32cff42b 100644 --- a/packages/orchestrator/internal/sandbox/rootfs_test.go +++ b/packages/orchestrator/internal/sandbox/rootfs_test.go @@ -1,445 +1,447 @@ package sandbox -import ( - "bytes" - "context" - "crypto/sha256" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" - "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" -) - -type DeviceWithClose struct { - *build.File - id string - size int64 -} - -func (d *DeviceWithClose) Close() error { - return nil -} - -func (d *DeviceWithClose) Size() (int64, error) { - return d.size, nil -} - -func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { - return d.File.ReadAt(p, off) -} - -const ( - contentFileName = "test.txt" - content = `test-file-content` - blockSize = 4096 -) - -// Execute the passed callback with the passed overlay mounted as a nbd device. -func executeForNbd( - t *testing.T, - ctx context.Context, - overlay *block.Overlay, - cb func(mountedPath string) error, -) error { - mnt := nbd.NewDirectPathMount(overlay) - - nbdCtx, nbdCancel := context.WithCancel(ctx) - defer nbdCancel() - - go func() { - <-nbdCtx.Done() - - mnt.Close() - }() - - mntIndex, err := mnt.Open(nbdCtx) - if err != nil { - return fmt.Errorf("failed to open: %w", err) - } - - devicePath := nbd.GetDevicePath(mntIndex) - - t.Logf("- created nbd device at %s\n", devicePath) - - // Check the block device health - out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to fsck: %w %s", err, out) - } - - t.Logf("- fscked base nbd device at %s\n", out) - - tmpDir, err := os.MkdirTemp("", "mount-*") - if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) - } - - defer os.RemoveAll(tmpDir) - - out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to mount: %w - %s", err, out) - } - - defer func() { - out, err = exec.Command("umount", tmpDir).CombinedOutput() - if err != nil { - t.Errorf("failed to umount: %s - %s", err, out) - } - }() - - t.Logf("- mounted rootfs at %s\n", devicePath) - - err = cb(tmpDir) - if err != nil { - return fmt.Errorf("failed execute: %w", err) - } - - return nil -} - -// Create a test file with given size formatted as ext4 -func createRootfsFile(t *testing.T, size int64) (*os.File, error) { - f, err := os.CreateTemp("", "test-rootfs-*.ext4") - if err != nil { - return nil, fmt.Errorf("failed to create temp file: %w", err) - } - t.Cleanup(func() { - f.Close() - os.Remove(f.Name()) - }) - - // Ensure the file is big enough - err = f.Truncate(size) - if err != nil { - return nil, fmt.Errorf("failed to truncate file: %w", err) - } - - // Close before formatting - err = f.Close() - if err != nil { - os.Remove(f.Name()) - return nil, fmt.Errorf("failed to close file: %w", err) - } - - // Format as ext4 - out, err := exec.Command("mkfs.ext4", "-F", "-b", fmt.Sprintf("%d", blockSize), f.Name()).CombinedOutput() - if err != nil { - os.Remove(f.Name()) - return nil, fmt.Errorf("failed to format ext4: %w - %s", err, out) - } - - // Reopen the file - f, err = os.OpenFile(f.Name(), os.O_RDWR, 0644) - if err != nil { - os.Remove(f.Name()) - return nil, fmt.Errorf("failed to reopen file: %w", err) - } - - return f, nil -} - -// TestDiff implements the build.Diff interface for testing -type TestDiff struct { - reader io.ReaderAt - size int64 -} - -func (d *TestDiff) Close() error { - return nil -} - -func (d *TestDiff) ReadAt(p []byte, off int64) (int, error) { - return d.reader.ReadAt(p, off) -} - -func (d *TestDiff) Slice(off, length int64) ([]byte, error) { - buf := make([]byte, length) - n, err := d.ReadAt(buf, off) - if err != nil { - return nil, err - } - return buf[:n], nil -} - -func (d *TestDiff) CachePath() (string, error) { - return "", nil -} - -func baseRootfs( - t *testing.T, - ctx context.Context, - store *build.DiffStore, -) (*build.File, *header.Header, error) { - buildId := uuid.New() - - // Create a test file of 10MB - rootfsFile, err := createRootfsFile(t, 10*1024*1024) - if err != nil { - return nil, nil, fmt.Errorf("failed to create test file: %w", err) - } - - fi, err := rootfsFile.Stat() - if err != nil { - return nil, nil, fmt.Errorf("failed to get file info: %w", err) - } - - header := header.NewHeader(&header.Metadata{ - BuildId: buildId, - BaseBuildId: buildId, - Size: uint64(fi.Size()), - Version: 1, - BlockSize: uint64(blockSize), - Generation: 1, - }, nil) - - // Create a test diff and add it to the store - diff := &TestDiff{ - reader: rootfsFile, - size: fi.Size(), - } - store.Add(buildId.String(), build.Rootfs, diff) - - rootfs := build.NewFile(header, store, build.Rootfs) - - return rootfs, header, nil -} - -// Create an overlay by extracting the diff from the passed overlay, -// putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. -func fileFromOverlay( - t *testing.T, - overlay *block.Overlay, - baseHeader *header.Header, - store *build.DiffStore, -) (*build.File, *header.Header, error) { - diffBuildId := uuid.New() - - diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) - if err != nil { - return nil, nil, fmt.Errorf("failed to create diff file: %w", err) - } - t.Cleanup(func() { - diffFile.Close() - os.Remove(diffFile.Name()) - }) - - cache, err := overlay.EjectCache() - if err != nil { - return nil, nil, fmt.Errorf("failed to eject cache: %w", err) - } - - dirtyBlocks, err := cache.Export(diffFile) - if err != nil { - return nil, nil, fmt.Errorf("failed to export cache: %w", err) - } - - diff, err := diffFile.ToDiff(blockSize) - if err != nil { - return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) - } - - newMappings := header.CreateMapping( - baseHeader.Metadata, - &diffBuildId, - dirtyBlocks, - ) - - diffMappings := header.MergeMappings( - baseHeader.Mapping, - newMappings, - ) - - diffHeader := header.NewHeader(&header.Metadata{ - BuildId: diffBuildId, - BaseBuildId: baseHeader.Metadata.BaseBuildId, - Size: baseHeader.Metadata.Size, - Version: baseHeader.Metadata.Version, - BlockSize: baseHeader.Metadata.BlockSize, - }, diffMappings) - - store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) - - // Create a build file that will use the diff and the base already in the store - diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) - - return diffRootfs, diffHeader, nil -} - -func compareSources( - s1, - s2 io.ReaderAt, - diffBuildId *uuid.UUID, - mappings []*header.BuildMap, -) error { - for _, mapping := range mappings { - if mapping.BuildId.String() != diffBuildId.String() { - continue - } - - for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { - c1 := make([]byte, blockSize) - n1, err := s1.ReadAt(c1, int64(off)) - if err != nil { - return fmt.Errorf("failed to read content1: %w", err) - } - - c2 := make([]byte, blockSize) - n2, err := s2.ReadAt(c2, int64(off)) - if err != nil { - return fmt.Errorf("failed to read content2: %w", err) - } - - if n1 != n2 { - return fmt.Errorf("content length mismatch: %d != %d", n1, n2) - } - - if !bytes.Equal(c1, c2) { - // Hash the content and show the mismatch - h1 := sha256.Sum256(c1) - h2 := sha256.Sum256(c2) - - return fmt.Errorf("content mismatch (showing hashes):\nBase: %x != Diff: %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) - } - } - } - - return nil -} - -func checkNbd(t *testing.T, ctx context.Context) error { - store, err := build.NewDiffStore(nil, ctx) - if err != nil { - return fmt.Errorf("failed to create diff store: %w", err) - } - t.Cleanup(func() { - store.Close() - }) - - baseRootfs, baseHeader, err := baseRootfs(t, ctx, store) - if err != nil { - return fmt.Errorf("failed to create base overlay: %w", err) - } - - baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) - baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) - if err != nil { - return fmt.Errorf("failed to create cache: %w", err) - } - defer baseCache.Close() - - baseOverlay := block.NewOverlay( - &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, - baseCache, - blockSize, - ) - - defer baseOverlay.Close() - - t.Logf("\n----- Base overlay mount -----\n\n") - - // Modify content in the overlay mounted from the base rootfs - err = executeForNbd(t, ctx, baseOverlay, func(mountedPath string) error { - contentPath := filepath.Join(mountedPath, contentFileName) - - err = os.WriteFile(contentPath, []byte(content), 0644) - if err != nil { - return fmt.Errorf("failed to write file: %w", err) - } - - // fmt.Printf("- writing to content Path: %s\n", contentPath) - - // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() - // if err != nil { - // return fmt.Errorf("failed to sync: %w - %s", err, out) - // } - - // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() - // if err != nil { - // return fmt.Errorf("failed to sync: %w - %s", err, out) - // } - - return nil - }) - - diffRootfs, diffHeader, err := fileFromOverlay(t, baseOverlay, baseHeader, store) - if err != nil { - return fmt.Errorf("failed to create diff overlay: %w", err) - } - - t.Logf("\n----- Diff header -----\n") - - for _, mapping := range diffHeader.Mapping { - t.Log(mapping.Format(baseHeader.Metadata.BlockSize)) - } - - // Compare the changed parts in the base overlay with the content in the diff overlay. - err = compareSources( - baseOverlay, - diffRootfs, - &diffHeader.Metadata.BuildId, - diffHeader.Mapping, - ) - if err != nil { - return fmt.Errorf("failed to compare overlays: %w", err) - } else { - t.Logf("overlay comparison successful") - } - - diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) - - diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) - if err != nil { - return fmt.Errorf("failed to create cache: %w", err) - } - defer diffCache.Close() - - diffOverlay := block.NewOverlay( - &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, - diffCache, - blockSize, - ) - defer diffOverlay.Close() - - t.Logf("\n----- Diff overlay mount -----\n\n") - - // Check the modified content in the overlay created from the diff and the base - err = executeForNbd(t, ctx, diffOverlay, func(mountedPath string) error { - contentPath := filepath.Join(mountedPath, contentFileName) - - readContent, err := os.ReadFile(contentPath) - if err != nil { - return fmt.Errorf("failed to read file: %w", err) - } - - t.Logf("- reading from content Path: %s\n", contentPath) - t.Logf("- content: %s\n", readContent) - - assert.Equal(t, content, string(readContent)) - - return nil - }) - if err != nil { - return fmt.Errorf("failed to mount nbd: %w", err) - } - - return nil -} - -func TestNbd(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err := checkNbd(t, ctx) - assert.NoError(t, err) -} +// import ( +// "bytes" +// "context" +// "crypto/sha256" +// "fmt" +// "io" +// "os" +// "os/exec" +// "path/filepath" +// "testing" + +// "github.com/google/uuid" +// "github.com/stretchr/testify/assert" +// "go.uber.org/zap" + +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" +// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/disk/nbd" +// "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +// ) + +// type DeviceWithClose struct { +// *build.File + +// id string +// size int64 +// } + +// func (d *DeviceWithClose) Close() error { +// return nil +// } + +// func (d *DeviceWithClose) Size() (int64, error) { +// return d.size, nil +// } + +// func (d *DeviceWithClose) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { +// return d.File.ReadAt(ctx, p, off) +// } + +// const ( +// contentFileName = "test.txt" +// content = `test-file-content` +// blockSize = 4096 +// ) + +// // Execute the passed callback with the passed overlay mounted as a nbd device. +// func executeForNbd( +// t *testing.T, +// ctx context.Context, +// overlay *block.Overlay, +// cb func(mountedPath string) error, +// ) error { +// mnt := nbd.NewDirectPathMount(overlay, nil, zap.NewNop()) + +// nbdCtx, nbdCancel := context.WithCancel(ctx) +// defer nbdCancel() + +// go func() { +// <-nbdCtx.Done() + +// mnt.Close() +// }() + +// mntIndex, err := mnt.Open(nbdCtx) +// if err != nil { +// return fmt.Errorf("failed to open: %w", err) +// } + +// devicePath := nbd.GetDevicePath(mntIndex) + +// t.Logf("- created nbd device at %s\n", devicePath) + +// // Check the block device health +// out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() +// if err != nil { +// return fmt.Errorf("failed to fsck: %w %s", err, out) +// } + +// t.Logf("- fscked base nbd device at %s\n", out) + +// tmpDir, err := os.MkdirTemp("", "mount-*") +// if err != nil { +// return fmt.Errorf("failed to create temp dir: %w", err) +// } + +// defer os.RemoveAll(tmpDir) + +// out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() +// if err != nil { +// return fmt.Errorf("failed to mount: %w - %s", err, out) +// } + +// defer func() { +// out, err = exec.Command("umount", tmpDir).CombinedOutput() +// if err != nil { +// t.Errorf("failed to umount: %s - %s", err, out) +// } +// }() + +// t.Logf("- mounted rootfs at %s\n", devicePath) + +// err = cb(tmpDir) +// if err != nil { +// return fmt.Errorf("failed execute: %w", err) +// } + +// return nil +// } + +// // Create a test file with given size formatted as ext4 +// func createRootfsFile(t *testing.T, size int64) (*os.File, error) { +// f, err := os.CreateTemp("", "test-rootfs-*.ext4") +// if err != nil { +// return nil, fmt.Errorf("failed to create temp file: %w", err) +// } +// t.Cleanup(func() { +// f.Close() +// os.Remove(f.Name()) +// }) + +// // Ensure the file is big enough +// err = f.Truncate(size) +// if err != nil { +// return nil, fmt.Errorf("failed to truncate file: %w", err) +// } + +// // Close before formatting +// err = f.Close() +// if err != nil { +// os.Remove(f.Name()) +// return nil, fmt.Errorf("failed to close file: %w", err) +// } + +// // Format as ext4 +// out, err := exec.Command("mkfs.ext4", "-F", "-b", fmt.Sprintf("%d", blockSize), f.Name()).CombinedOutput() +// if err != nil { +// os.Remove(f.Name()) +// return nil, fmt.Errorf("failed to format ext4: %w - %s", err, out) +// } + +// // Reopen the file +// f, err = os.OpenFile(f.Name(), os.O_RDWR, 0o644) +// if err != nil { +// os.Remove(f.Name()) +// return nil, fmt.Errorf("failed to reopen file: %w", err) +// } + +// return f, nil +// } + +// // TestDiff implements the build.Diff interface for testing +// type TestDiff struct { +// reader io.ReaderAt +// size int64 +// } + +// func (d *TestDiff) Close() error { +// return nil +// } + +// func (d *TestDiff) ReadAt(p []byte, off int64) (int, error) { +// return d.reader.ReadAt(p, off) +// } + +// func (d *TestDiff) Slice(off, length int64) ([]byte, error) { +// buf := make([]byte, length) +// n, err := d.ReadAt(buf, off) +// if err != nil { +// return nil, err +// } +// return buf[:n], nil +// } + +// func (d *TestDiff) CachePath() (string, error) { +// return "", nil +// } + +// func baseRootfs( +// t *testing.T, +// ctx context.Context, +// store *build.DiffStore, +// ) (*build.File, *header.Header, error) { +// buildId := uuid.New() + +// // Create a test file of 10MB +// rootfsFile, err := createRootfsFile(t, 10*1024*1024) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to create test file: %w", err) +// } + +// fi, err := rootfsFile.Stat() +// if err != nil { +// return nil, nil, fmt.Errorf("failed to get file info: %w", err) +// } + +// header := header.NewHeader(&header.Metadata{ +// BuildId: buildId, +// BaseBuildId: buildId, +// Size: uint64(fi.Size()), +// Version: 1, +// BlockSize: uint64(blockSize), +// Generation: 1, +// }, nil) + +// // Create a test diff and add it to the store +// diff := &TestDiff{ +// reader: rootfsFile, +// size: fi.Size(), +// } +// store.Add(buildId.String(), build.Rootfs, diff) + +// rootfs := build.NewFile(header, store, build.Rootfs) + +// return rootfs, header, nil +// } + +// // Create an overlay by extracting the diff from the passed overlay, +// // putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. +// func fileFromOverlay( +// t *testing.T, +// overlay *block.Overlay, +// baseHeader *header.Header, +// store *build.DiffStore, +// ) (*build.File, *header.Header, error) { +// diffBuildId := uuid.New() + +// diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to create diff file: %w", err) +// } +// t.Cleanup(func() { +// diffFile.Close() +// os.Remove(diffFile.Name()) +// }) + +// cache, err := overlay.EjectCache() +// if err != nil { +// return nil, nil, fmt.Errorf("failed to eject cache: %w", err) +// } + +// dirtyBlocks, err := cache.Export(diffFile) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to export cache: %w", err) +// } + +// diff, err := diffFile.ToDiff(blockSize) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) +// } + +// newMappings := header.CreateMapping( +// baseHeader.Metadata, +// &diffBuildId, +// dirtyBlocks, +// ) + +// diffMappings := header.MergeMappings( +// baseHeader.Mapping, +// newMappings, +// ) + +// diffHeader := header.NewHeader(&header.Metadata{ +// BuildId: diffBuildId, +// BaseBuildId: baseHeader.Metadata.BaseBuildId, +// Size: baseHeader.Metadata.Size, +// Version: baseHeader.Metadata.Version, +// BlockSize: baseHeader.Metadata.BlockSize, +// }, diffMappings) + +// store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) + +// // Create a build file that will use the diff and the base already in the store +// diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) + +// return diffRootfs, diffHeader, nil +// } + +// func compareSources( +// s1, +// s2 io.ReaderAt, +// diffBuildId *uuid.UUID, +// mappings []*header.BuildMap, +// ) error { +// for _, mapping := range mappings { +// if mapping.BuildId.String() != diffBuildId.String() { +// continue +// } + +// for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { +// c1 := make([]byte, blockSize) +// n1, err := s1.ReadAt(c1, int64(off)) +// if err != nil { +// return fmt.Errorf("failed to read content1: %w", err) +// } + +// c2 := make([]byte, blockSize) +// n2, err := s2.ReadAt(c2, int64(off)) +// if err != nil { +// return fmt.Errorf("failed to read content2: %w", err) +// } + +// if n1 != n2 { +// return fmt.Errorf("content length mismatch: %d != %d", n1, n2) +// } + +// if !bytes.Equal(c1, c2) { +// // Hash the content and show the mismatch +// h1 := sha256.Sum256(c1) +// h2 := sha256.Sum256(c2) + +// return fmt.Errorf("content mismatch (showing hashes):\nBase: %x != Diff: %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) +// } +// } +// } + +// return nil +// } + +// func checkNbd(t *testing.T, ctx context.Context) error { +// store, err := build.NewDiffStore(nil, ctx) +// if err != nil { +// return fmt.Errorf("failed to create diff store: %w", err) +// } +// t.Cleanup(func() { +// store.Close() +// }) + +// baseRootfs, baseHeader, err := baseRootfs(t, ctx, store) +// if err != nil { +// return fmt.Errorf("failed to create base overlay: %w", err) +// } + +// baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) +// baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) +// if err != nil { +// return fmt.Errorf("failed to create cache: %w", err) +// } +// defer baseCache.Close() + +// baseOverlay := block.NewOverlay( +// &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, +// baseCache, +// blockSize, +// ) + +// defer baseOverlay.Close() + +// t.Logf("\n----- Base overlay mount -----\n\n") + +// // Modify content in the overlay mounted from the base rootfs +// err = executeForNbd(t, ctx, baseOverlay, func(mountedPath string) error { +// contentPath := filepath.Join(mountedPath, contentFileName) + +// err = os.WriteFile(contentPath, []byte(content), 0o644) +// if err != nil { +// return fmt.Errorf("failed to write file: %w", err) +// } + +// // fmt.Printf("- writing to content Path: %s\n", contentPath) + +// // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() +// // if err != nil { +// // return fmt.Errorf("failed to sync: %w - %s", err, out) +// // } + +// // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() +// // if err != nil { +// // return fmt.Errorf("failed to sync: %w - %s", err, out) +// // } + +// return nil +// }) + +// diffRootfs, diffHeader, err := fileFromOverlay(t, baseOverlay, baseHeader, store) +// if err != nil { +// return fmt.Errorf("failed to create diff overlay: %w", err) +// } + +// t.Logf("\n----- Diff header -----\n") + +// for _, mapping := range diffHeader.Mapping { +// t.Log(mapping.Format(baseHeader.Metadata.BlockSize)) +// } + +// // Compare the changed parts in the base overlay with the content in the diff overlay. +// err = compareSources( +// baseOverlay, +// diffRootfs, +// &diffHeader.Metadata.BuildId, +// diffHeader.Mapping, +// ) +// if err != nil { +// return fmt.Errorf("failed to compare overlays: %w", err) +// } else { +// t.Logf("overlay comparison successful") +// } + +// diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) + +// diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) +// if err != nil { +// return fmt.Errorf("failed to create cache: %w", err) +// } +// defer diffCache.Close() + +// diffOverlay := block.NewOverlay( +// &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, +// diffCache, +// blockSize, +// ) +// defer diffOverlay.Close() + +// t.Logf("\n----- Diff overlay mount -----\n\n") + +// // Check the modified content in the overlay created from the diff and the base +// err = executeForNbd(t, ctx, diffOverlay, func(mountedPath string) error { +// contentPath := filepath.Join(mountedPath, contentFileName) + +// readContent, err := os.ReadFile(contentPath) +// if err != nil { +// return fmt.Errorf("failed to read file: %w", err) +// } + +// t.Logf("- reading from content Path: %s\n", contentPath) +// t.Logf("- content: %s\n", readContent) + +// assert.Equal(t, content, string(readContent)) + +// return nil +// }) +// if err != nil { +// return fmt.Errorf("failed to mount nbd: %w", err) +// } + +// return nil +// } + +// func TestNbd(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() + +// err := checkNbd(t, ctx) +// assert.NoError(t, err) +// } From c71cd11ed87ac3f907ccd9c777bc7dfe5764628a Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 00:55:34 -0800 Subject: [PATCH 08/21] Add nbd fix --- .../internal/sandbox/nbd/dispatch.go | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/nbd/dispatch.go b/packages/orchestrator/internal/sandbox/nbd/dispatch.go index 43dd0f9a4a..4985e57f32 100644 --- a/packages/orchestrator/internal/sandbox/nbd/dispatch.go +++ b/packages/orchestrator/internal/sandbox/nbd/dispatch.go @@ -21,7 +21,13 @@ type Provider interface { Size() (int64, error) } -const dispatchBufferSize = 4 * 1024 * 1024 +const ( + // We increase the buffer size by 28 bytes to account for the usual max requests. + dispatchBufferSize = 4 * 1024 * 1024 + // https://sourceforge.net/p/nbd/mailman/message/35081223/ + // 32MB is the maximum buffer size for a single request that should be universally supported. + dispatchMaxWriteBufferSize = 32 * 1024 * 1024 +) // NBD Commands const ( @@ -130,7 +136,6 @@ func (d *Dispatch) Handle(ctx context.Context) error { // Now go through processing complete packets rp := 0 - process: for { // Check if there is a fatal error from an async read/write to return select { @@ -169,14 +174,37 @@ func (d *Dispatch) Handle(ctx context.Context) error { } case NBDCmdWrite: rp += 28 - if wp-rp < int(request.Length) { - rp -= 28 - break process // We don't have enough data yet... Wait for next read + if request.Length > dispatchMaxWriteBufferSize { + return fmt.Errorf("nbd write request length is too large: %d", request.Length) } + data := make([]byte, request.Length) - copy(data, buffer[rp:rp+int(request.Length)]) - rp += int(request.Length) + + dataCopied := copy(data, buffer[rp:wp]) + + rp += dataCopied + + // We need to wait for more data here, otherwise we will deadlock if the buffer is Xmb and the length is Xmb because of the headers's extra 28 bytes needed. + // At the same time we don't want to increase the buffer size as the max would be 32mb which is too large for hundreds sandbox connections. + + for dataCopied < int(request.Length) { + n, err := d.fp.Read(data[dataCopied:]) + if err != nil { + return fmt.Errorf("nbd write read error: %w", err) + } + + dataCopied += n + + select { + case err := <-d.fatal: + return err + case <-ctx.Done(): + return ctx.Err() + default: + } + } + err := d.cmdWrite(ctx, request.Handle, request.From, data) if err != nil { return err From 1f2920d4d8c3e72edcfb2b834bb11fc4ff275c22 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 18:14:23 -0800 Subject: [PATCH 09/21] Refactor nbd mount for testing --- packages/orchestrator/Makefile | 4 - packages/orchestrator/cmd/mock-nbd/mock.go | 202 ------------ .../orchestrator/cmd/mount-rootfs/main.go | 293 ++---------------- .../internal/sandbox/build/cache.go | 3 +- .../sandbox/nbd/testutils/build_device.go | 40 +++ .../internal/sandbox/nbd/testutils/cleaner.go | 35 +++ .../sandbox/nbd/testutils/logger_overlay.go | 74 +++++ .../internal/sandbox/nbd/testutils/mount.go | 46 +++ .../sandbox/nbd/testutils/nbd_device.go | 63 ++++ .../sandbox/nbd/testutils/template_rootfs.go | 120 +++++++ .../sandbox/nbd/testutils/zero_device.go | 40 +++ 11 files changed, 450 insertions(+), 470 deletions(-) delete mode 100644 packages/orchestrator/cmd/mock-nbd/mock.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/build_device.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/logger_overlay.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/mount.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/nbd_device.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/template_rootfs.go create mode 100644 packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index 8d1e8d0bfb..b620e92026 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -85,10 +85,6 @@ build-and-upload/template-manager: build upload/template-manager mock: sudo TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) CONSUL_TOKEN=$(CONSUL_TOKEN) NODE_ID="testclient" go run cmd/mock-sandbox/mock.go -template 5wzg6c91u51yaebviysf -build "f0370054-b669-eeee-b33b-573d5287c6ef" -alive 1 -count 2 -.PHONY: mock-nbd -mock-nbd: - sudo go run -gcflags=all="-N -l" cmd/mock-nbd/mock.go - .PHONY: mock-snapshot mock-snapshot: sudo TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) CONSUL_TOKEN=$(CONSUL_TOKEN) NODE_ID="testclient" go run cmd/mock-snapshot/mock.go -template 5wzg6c91u51yaebviysf -build "f0370054-b669-4d7e-b33b-573d5287c6ef" -alive 1 -count 1 diff --git a/packages/orchestrator/cmd/mock-nbd/mock.go b/packages/orchestrator/cmd/mock-nbd/mock.go deleted file mode 100644 index e1f300d4a9..0000000000 --- a/packages/orchestrator/cmd/mock-nbd/mock.go +++ /dev/null @@ -1,202 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/rand" - "fmt" - "os" - "os/signal" - - "github.com/google/uuid" - "github.com/pojntfx/go-nbd/pkg/backend" - - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" - "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" -) - -const blockSize = 4096 - -type DeviceWithClose struct { - b backend.Backend -} - -var _ block.Device = (*DeviceWithClose)(nil) - -func (d *DeviceWithClose) ReadAt(_ context.Context, p []byte, off int64) (n int, err error) { - return d.b.ReadAt(p, off) -} - -func (d *DeviceWithClose) Size() (int64, error) { - return d.b.Size() -} - -func (d *DeviceWithClose) WriteAt(p []byte, off int64) (n int, err error) { - return d.b.WriteAt(p, off) -} - -func (d *DeviceWithClose) Close() error { - return nil -} - -func (d *DeviceWithClose) Slice(_ context.Context, offset, length int64) ([]byte, error) { - b := make([]byte, length) - - _, err := d.b.ReadAt(b, offset) - if err != nil { - return nil, err - } - - return b, nil -} - -func (d *DeviceWithClose) BlockSize() int64 { - return blockSize -} - -func (d *DeviceWithClose) Header() *header.Header { - size, err := d.b.Size() - if err != nil { - panic(err) - } - - h, err := header.NewHeader(header.NewTemplateMetadata( - uuid.New(), - uint64(blockSize), - uint64(size), - ), nil) - if err != nil { - panic(err) - } - - return h -} - -func main() { - data := make([]byte, blockSize*8) - rand.Read(data) - - device := &DeviceWithClose{ - b: backend.NewMemoryBackend(data), - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - done := make(chan os.Signal, 1) - signal.Notify(done, os.Interrupt) - devicePool, err := nbd.NewDevicePool() - if err != nil { - fmt.Fprintf(os.Stderr, "failed to create device pool: %v\n", err) - - return - } - go func() { - devicePool.Populate(ctx) - fmt.Fprintf(os.Stderr, "device pool done populating\n") - }() - defer func() { - err = devicePool.Close(ctx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to close device pool: %v\n", err) - - return - } - }() - - go func() { - <-done - - cancel() - }() - - for i := 0; ; i++ { - select { - case <-ctx.Done(): - return - default: - } - fmt.Printf("----------------------------------------\n") - fmt.Printf("[%d] starting mock nbd server\n", i) - - readData, err := MockNbd(ctx, device, i, devicePool) - if err != nil { - fmt.Fprintf(os.Stderr, "[%d] failed to mock nbd: %v\n", i, err) - - return - } - - if !bytes.Equal(data, readData) { - fmt.Fprintf(os.Stderr, "[%d] data mismatch\n", i) - - return - } - } -} - -func MockNbd(ctx context.Context, device *DeviceWithClose, index int, devicePool *nbd.DevicePool) ([]byte, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - size, err := device.Size() - if err != nil { - return nil, fmt.Errorf("failed to get size: %w", err) - } - - deviceIndex, err := devicePool.GetDevice(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get device: %w", err) - } - - var mnt *nbd.DirectPathMount - - defer func() { - counter := 0 - - for { - counter++ - err = devicePool.ReleaseDevice(ctx, deviceIndex) - if err != nil { - if counter%10 == 0 { - fmt.Printf("[%d] failed to release device: %v\n", index, err) - } - - if mnt != nil { - mnt.Close(ctx) - } - - continue - } - - fmt.Printf("[%d] released device: %d\n", index, deviceIndex) - - return - } - }() - - mnt = nbd.NewDirectPathMount(device, devicePool) - - go func() { - <-ctx.Done() - - mnt.Close(context.Background()) //nolint:contextcheck // TODO: fix this later - }() - - _, err = mnt.Open(ctx) - if err != nil { - return nil, fmt.Errorf("failed to open: %w", err) - } - - data := make([]byte, size) - _, err = mnt.Backend.ReadAt(ctx, data, 0) - if err != nil { - return nil, fmt.Errorf("failed to read: %w", err) - } - - fmt.Printf("[%d] Read %d bytes from nbd\n", index, len(data)) - - cancel() - - return data, nil -} diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index d042538747..04ba7e652e 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -4,138 +4,45 @@ import ( "context" "flag" "fmt" + "log" "os" "os/exec" "os/signal" "path/filepath" - "time" "github.com/google/uuid" - "go.opentelemetry.io/otel/metric/noop" - "golang.org/x/sys/unix" + "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/orchestrator/internal/cfg" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block/metrics" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" - "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" - featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" - "github.com/e2b-dev/infra/packages/shared/pkg/storage" - "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd/testutils" ) -var _ block.ReadonlyDevice = (*mountReadonlyDevice)(nil) - -type mountReadonlyDevice struct { - *build.File - - header *header.Header - blockSize int64 -} - -func newReadonlyDevice(file *build.File, header *header.Header, blockSize int64) *mountReadonlyDevice { - return &mountReadonlyDevice{ - File: file, - header: header, - blockSize: blockSize, - } -} - -func (m *mountReadonlyDevice) Close() error { - return nil -} - -func (m *mountReadonlyDevice) BlockSize() int64 { - return m.blockSize -} - -func (m *mountReadonlyDevice) Header() *header.Header { - return m.header -} - -func (m *mountReadonlyDevice) Size() (int64, error) { - return int64(m.header.Metadata.Size), nil -} - -type loggedOverlay struct { - overlay *block.Overlay -} - -func (l *loggedOverlay) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { - defer func() { - if r := recover(); r != nil { - fmt.Fprintf(os.Stdout, "[read panic recovered]: [%d, %d] -> %v\n", off, len(p), r) - } - }() - - fmt.Fprintf(os.Stdout, "[read started]: [%d, %d]\n", off, len(p)) - - n, err := l.overlay.ReadAt(ctx, p, off) - - fmt.Fprintf(os.Stdout, "[read completed]: [%d, %d] -> %d\n", off, len(p), n) - - return n, err -} - -func (l *loggedOverlay) WriteAt(p []byte, off int64) (int, error) { - defer func() { - if r := recover(); r != nil { - fmt.Fprintf(os.Stdout, "[write panic recovered]: [%d, %d] -> %v\n", off, len(p), r) - } - }() - - fmt.Fprintf(os.Stdout, "[write started]: [%d, %d]\n", off, len(p)) - - n, err := l.overlay.WriteAt(p, off) - - fmt.Fprintf(os.Stdout, "[write completed]: [%d, %d] -> %d\n", off, len(p), n) - - return n, err -} - -func (l *loggedOverlay) Size() (int64, error) { - return l.overlay.Size() -} - -func (l *loggedOverlay) BlockSize() int64 { - return l.overlay.BlockSize() -} - -func (l *loggedOverlay) Header() *header.Header { - return l.overlay.Header() -} - -func (l *loggedOverlay) Close() error { - return l.overlay.Close() -} - -func (l *loggedOverlay) EjectCache() (*block.Cache, error) { - return l.overlay.EjectCache() -} - -func (l *loggedOverlay) Slice(ctx context.Context, off, length int64) ([]byte, error) { - return l.overlay.Slice(ctx, off, length) -} - func main() { buildId := flag.String("build", "", "build id") mountPath := flag.String("mount", "", "mount path") verify := flag.Bool("verify", false, "verify rootfs integrity") + logging := flag.Bool("log", false, "enable logging (it is pretty spammy)") flag.Parse() + if *verify && *mountPath == "" { + log.Fatalf("verify flag is only supported when mount path is provided") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan os.Signal, 1) signal.Notify(done, os.Interrupt) - // Disabling the logger for normal use—is very spammy, because Populate on device pool periodically logs errors if the number of acquirable devices is less than the number of requested devices. - // logger, err := zap.NewDevelopment() - // if err != nil { - // log.Fatalf("failed to create logger: %s", err) - // } - // zap.ReplaceGlobals(logger) + // Logger is very spammy, because Populate on device pool periodically logs errors if the number of acquirable devices is less than the number of requested devices. + if *logging { + logger, err := zap.NewDevelopment() + if err != nil { + log.Fatalf("failed to create logger: %s", err) + } + zap.ReplaceGlobals(logger) + } go func() { <-done @@ -146,103 +53,28 @@ func main() { // We use a separate ctx for majority of the operations as cancelling context for the NBD+storage and *then* doing cleanup for these often resulted in deadlocks. nbdContext := context.Background() - err := mountRootfs(ctx, nbdContext, *buildId, *mountPath, *verify) + err := run(ctx, nbdContext, *buildId, *mountPath, *verify) if err != nil { panic(fmt.Errorf("failed to mount rootfs: %w", err)) } } -func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string, verify bool) error { - files := storage.TemplateFiles{ - BuildID: buildID, - } +func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool) error { + cleanupCtx := context.Background() //nolint:contextcheck // we need to use separate context otherwise the cleanup can be problematic - s, err := storage.GetTemplateStorageProvider(nbdContext, nil) + rootfs, rootfsCleanup, err := testutils.TemplateRootfs(ctx, buildID) + defer rootfsCleanup.Run(cleanupCtx) if err != nil { - return fmt.Errorf("failed to get storage provider: %w", err) + return fmt.Errorf("failed to get template rootfs: %w", err) } - obj, err := s.OpenObject(nbdContext, files.StorageRootfsHeaderPath(), storage.RootFSHeaderObjectType) - if err != nil { - return fmt.Errorf("failed to open object: %w", err) - } - - h, err := header.Deserialize(nbdContext, obj) - if err != nil { - id, err := uuid.Parse(buildID) - if err != nil { - return fmt.Errorf("failed to parse build id: %w", err) - } - - r, err := s.OpenSeekableObject(nbdContext, files.StorageRootfsPath(), storage.RootFSObjectType) - if err != nil { - return fmt.Errorf("failed to open object: %w", err) - } - - size, err := r.Size(nbdContext) - if err != nil { - return fmt.Errorf("failed to get object size: %w", err) - } - - h, err = header.NewHeader(&header.Metadata{ - BuildId: id, - BaseBuildId: id, - Size: uint64(size), - Version: 1, - BlockSize: header.RootfsBlockSize, - Generation: 1, - }, nil) - if err != nil { - return fmt.Errorf("failed to create header for rootfs without header: %w", err) - } - } - - diffCacheDir := filepath.Join(os.TempDir(), fmt.Sprintf("%s-rootfs.diff.cache-%s", buildID, uuid.New().String())) - - err = os.MkdirAll(diffCacheDir, 0o755) - if err != nil { - return fmt.Errorf("failed to create diff cache directory: %w", err) - } - - defer os.RemoveAll(diffCacheDir) - - flags, err := featureflags.NewClient() - if err != nil { - return fmt.Errorf("failed to create feature flags client: %w", err) - } - - store, err := build.NewDiffStore( - nbdContext, - cfg.Config{}, - flags, - diffCacheDir, - 24*time.Hour, - 24*time.Hour, - ) - if err != nil { - return fmt.Errorf("failed to create diff store: %w", err) - } - - defer store.Close() - - fmt.Printf("caching diffs to: %+v\n", diffCacheDir) - - m, err := metrics.NewMetrics(noop.NewMeterProvider()) - if err != nil { - return fmt.Errorf("failed to create metrics: %w", err) - } - - rootfs := build.NewFile(h, store, build.Rootfs, s, m) - - readonlyDevice := newReadonlyDevice(rootfs, h, int64(h.Metadata.BlockSize)) - cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-rootfs.ext4.cow.cache-%s", buildID, uuid.New().String())) defer os.RemoveAll(cowCachePath) cache, err := block.NewCache( - int64(h.Metadata.Size), - int64(h.Metadata.BlockSize), + int64(rootfs.Header().Metadata.Size), + int64(rootfs.Header().Metadata.BlockSize), cowCachePath, false, ) @@ -252,55 +84,15 @@ func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string, ver fmt.Printf("caching writes to: %+v\n", cowCachePath) - overlay := block.NewOverlay(readonlyDevice, cache) + overlay := block.NewOverlay(rootfs, cache) defer overlay.Close() - devicePool, err := nbd.NewDevicePool() + devicePath, deviceCleanup, err := testutils.GetNBDDevice(nbdContext, overlay) + defer deviceCleanup.Run(cleanupCtx) if err != nil { - return fmt.Errorf("failed to create device pool: %w", err) + return fmt.Errorf("failed to get nbd device: %w", err) } - poolClosed := make(chan struct{}) - - defer func() { //nolint:contextcheck // we need to use separate context otherwise the cleanup can be problematic - <-poolClosed - - cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) - defer cancelCleanup() - - err = devicePool.Close(cleanupCtx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to close device pool: %v\n", err) - } - }() - - poolCtx, poolCancel := context.WithCancel(nbdContext) - defer poolCancel() - - go func() { - devicePool.Populate(poolCtx) - close(poolClosed) - }() - - mnt := nbd.NewDirectPathMount(&loggedOverlay{overlay}, devicePool) - - mntIndex, err := mnt.Open(nbdContext) - if err != nil { - return fmt.Errorf("failed to open nbd mount: %w", err) - } - - defer func() { //nolint:contextcheck // we need to use separate context otherwise the cleanup can be problematic - cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 30*time.Second) - defer cancelCleanup() - - err = mnt.Close(cleanupCtx) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to close nbd mount: %v\n", err) - } - }() - - devicePath := nbd.GetDevicePath(mntIndex) - fmt.Printf("rootfs exposed as device: %s\n", devicePath) if mountPath != "" { @@ -311,36 +103,13 @@ func mountRootfs(ctx, nbdContext context.Context, buildID, mountPath string, ver fmt.Fprintf(os.Stdout, "creating mount path directory: %s\n", mountPath) - // We don't remote the dir as it might have been user created. - - err = unix.Mount(devicePath, mountPath, "ext4", 0, "") + mountCleanup, err := testutils.MountNBDDevice(devicePath, mountPath) + defer mountCleanup.Run(cleanupCtx) if err != nil { return fmt.Errorf("failed to mount device to mount path: %w", err) } - defer func() { - ticker := time.NewTicker(600 * time.Millisecond) - defer ticker.Stop() - - cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cleanupCancel() - - for { - select { - case <-cleanupCtx.Done(): - fmt.Fprintf(os.Stderr, "failed to unmount device from mount path in time\n") - - return - case <-ticker.C: - err = unix.Unmount(mountPath, 0) - if err == nil { - return - } - - fmt.Fprintf(os.Stderr, "failed to unmount device from mount path: %v\n", err) - } - } - }() + // We don't remove the dir as it might have been user created. fmt.Printf("rootfs mounted at path: %s\n", mountPath) } diff --git a/packages/orchestrator/internal/sandbox/build/cache.go b/packages/orchestrator/internal/sandbox/build/cache.go index a40f05f1fe..8d1016ccf9 100644 --- a/packages/orchestrator/internal/sandbox/build/cache.go +++ b/packages/orchestrator/internal/sandbox/build/cache.go @@ -304,7 +304,6 @@ func diskUsage(path string) (uint64, uint64, error) { return used, total, nil } -func (s *DiffStore) Remove() { +func (s *DiffStore) RemoveCache() { s.cache.DeleteAll() - s.cache.Stop() } diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/build_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/build_device.go new file mode 100644 index 0000000000..4f167834e1 --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/build_device.go @@ -0,0 +1,40 @@ +package testutils + +import ( + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +var _ block.ReadonlyDevice = (*BuildDevice)(nil) + +type BuildDevice struct { + *build.File + + header *header.Header + blockSize int64 +} + +func NewBuildDevice(file *build.File, header *header.Header, blockSize int64) *BuildDevice { + return &BuildDevice{ + File: file, + header: header, + blockSize: blockSize, + } +} + +func (m *BuildDevice) Close() error { + return nil +} + +func (m *BuildDevice) BlockSize() int64 { + return m.blockSize +} + +func (m *BuildDevice) Header() *header.Header { + return m.header +} + +func (m *BuildDevice) Size() (int64, error) { + return int64(m.header.Metadata.Size), nil +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go b/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go new file mode 100644 index 0000000000..cfdabec96a --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go @@ -0,0 +1,35 @@ +package testutils + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" +) + +type Cleaner struct { + steps []func(ctx context.Context) error + once sync.Once +} + +func (c *Cleaner) Add(f func(ctx context.Context) error) { + c.steps = append(c.steps, f) +} + +func (c *Cleaner) Run(ctx context.Context) (err error) { + c.once.Do(func() { + var errs []error + + for _, step := range slices.Backward(c.steps) { + err := step(ctx) + if err != nil { + errs = append(errs, fmt.Errorf("failed to run step: %w", err)) + } + } + + err = errors.Join(errs...) + }) + + return err +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/logger_overlay.go b/packages/orchestrator/internal/sandbox/nbd/testutils/logger_overlay.go new file mode 100644 index 0000000000..8eb5a0f65a --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/logger_overlay.go @@ -0,0 +1,74 @@ +package testutils + +import ( + "context" + "fmt" + "os" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +type LoggerOverlay struct { + overlay *block.Overlay +} + +func NewLoggerOverlay(overlay *block.Overlay) *LoggerOverlay { + return &LoggerOverlay{overlay: overlay} +} + +func (l *LoggerOverlay) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stdout, "[read panic recovered]: [%d, %d] -> %v\n", off, len(p), r) + } + }() + + fmt.Fprintf(os.Stdout, "[read started]: [%d, %d]\n", off, len(p)) + + n, err := l.overlay.ReadAt(ctx, p, off) + + fmt.Fprintf(os.Stdout, "[read completed]: [%d, %d] -> %d\n", off, len(p), n) + + return n, err +} + +func (l *LoggerOverlay) WriteAt(p []byte, off int64) (int, error) { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stdout, "[write panic recovered]: [%d, %d] -> %v\n", off, len(p), r) + } + }() + + fmt.Fprintf(os.Stdout, "[write started]: [%d, %d]\n", off, len(p)) + + n, err := l.overlay.WriteAt(p, off) + + fmt.Fprintf(os.Stdout, "[write completed]: [%d, %d] -> %d\n", off, len(p), n) + + return n, err +} + +func (l *LoggerOverlay) Size() (int64, error) { + return l.overlay.Size() +} + +func (l *LoggerOverlay) BlockSize() int64 { + return l.overlay.BlockSize() +} + +func (l *LoggerOverlay) Header() *header.Header { + return l.overlay.Header() +} + +func (l *LoggerOverlay) Close() error { + return l.overlay.Close() +} + +func (l *LoggerOverlay) EjectCache() (*block.Cache, error) { + return l.overlay.EjectCache() +} + +func (l *LoggerOverlay) Slice(ctx context.Context, off, length int64) ([]byte, error) { + return l.overlay.Slice(ctx, off, length) +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go b/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go new file mode 100644 index 0000000000..a55908ab6d --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go @@ -0,0 +1,46 @@ +package testutils + +import ( + "context" + "fmt" + "os" + "time" + + "golang.org/x/sys/unix" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" +) + +func MountNBDDevice(device nbd.DevicePath, mountPath string) (*Cleaner, error) { + var cleaner Cleaner + + err := unix.Mount(device, mountPath, "ext4", 0, "") + if err != nil { + return &cleaner, fmt.Errorf("failed to mount device to mount path: %w", err) + } + + cleaner.Add(func(cleanupCtx context.Context) error { + ticker := time.NewTicker(600 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-cleanupCtx.Done(): + fmt.Fprintf(os.Stderr, "failed to unmount device from mount path in time\n") + + return nil + case <-ticker.C: + err = unix.Unmount(mountPath, 0) + if err == nil { + return nil + } + + fmt.Fprintf(os.Stderr, "failed to unmount device from mount path: %v\n", err) + } + } + + return nil + }) + + return &cleaner, nil +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/nbd_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/nbd_device.go new file mode 100644 index 0000000000..1b53d25aa4 --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/nbd_device.go @@ -0,0 +1,63 @@ +package testutils + +import ( + "context" + "fmt" + "os" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd" +) + +func GetNBDDevice(ctx context.Context, backend block.Device) (nbd.DevicePath, *Cleaner, error) { + var cleaner Cleaner + + devicePool, err := nbd.NewDevicePool() + if err != nil { + return "", &cleaner, fmt.Errorf("failed to create device pool: %w", err) + } + + poolClosed := make(chan struct{}) + + cleaner.Add(func(cleanupCtx context.Context) error { + <-poolClosed + + err = devicePool.Close(cleanupCtx) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to close device pool: %v\n", err) + } + + return nil + }) + + poolCtx, poolCancel := context.WithCancel(ctx) + + cleaner.Add(func(context.Context) error { + poolCancel() + + return nil + }) + + go func() { + devicePool.Populate(poolCtx) + close(poolClosed) + }() + + mnt := nbd.NewDirectPathMount(backend, devicePool) + + mntIndex, err := mnt.Open(ctx) + if err != nil { + return "", &cleaner, fmt.Errorf("failed to open nbd mount: %w", err) + } + + cleaner.Add(func(cleanupCtx context.Context) error { + err = mnt.Close(cleanupCtx) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to close nbd mount: %v\n", err) + } + + return nil + }) + + return nbd.GetDevicePath(mntIndex), &cleaner, nil +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/template_rootfs.go b/packages/orchestrator/internal/sandbox/nbd/testutils/template_rootfs.go new file mode 100644 index 0000000000..b039c7679c --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/template_rootfs.go @@ -0,0 +1,120 @@ +package testutils + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" + "go.opentelemetry.io/otel/metric/noop" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/cfg" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block/metrics" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" + featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +func TemplateRootfs(ctx context.Context, buildID string) (*BuildDevice, *Cleaner, error) { + var cleaner Cleaner + + files := storage.TemplateFiles{ + BuildID: buildID, + } + + s, err := storage.GetTemplateStorageProvider(ctx, nil) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to get storage provider: %w", err) + } + + obj, err := s.OpenObject(ctx, files.StorageRootfsHeaderPath(), storage.RootFSHeaderObjectType) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to open object: %w", err) + } + + h, err := header.Deserialize(ctx, obj) + if err != nil { + id, err := uuid.Parse(buildID) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to parse build id: %w", err) + } + + r, err := s.OpenSeekableObject(ctx, files.StorageRootfsPath(), storage.RootFSObjectType) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to open object: %w", err) + } + + size, err := r.Size(ctx) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to get object size: %w", err) + } + + h, err = header.NewHeader(&header.Metadata{ + BuildId: id, + BaseBuildId: id, + Size: uint64(size), + Version: 1, + BlockSize: header.RootfsBlockSize, + Generation: 1, + }, nil) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to create header for rootfs without header: %w", err) + } + } + + diffCacheDir := filepath.Join(os.TempDir(), fmt.Sprintf("%s-rootfs.diff.cache-%s", buildID, uuid.New().String())) + + err = os.MkdirAll(diffCacheDir, 0o755) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to create diff cache directory: %w", err) + } + + cleaner.Add(func(context.Context) error { + return os.RemoveAll(diffCacheDir) + }) + + flags, err := featureflags.NewClient() + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to create feature flags client: %w", err) + } + + store, err := build.NewDiffStore( + ctx, + cfg.Config{}, + flags, + diffCacheDir, + 24*time.Hour, + 24*time.Hour, + ) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to create diff store: %w", err) + } + + cleaner.Add(func(context.Context) error { + store.RemoveCache() + + return nil + }) + + cleaner.Add(func(context.Context) error { + store.Close() + + return nil + }) + + m, err := metrics.NewMetrics(noop.NewMeterProvider()) + if err != nil { + return nil, &cleaner, fmt.Errorf("failed to create metrics: %w", err) + } + + buildDevice := NewBuildDevice( + build.NewFile(h, store, build.Rootfs, s, m), + h, + int64(h.Metadata.BlockSize), + ) + + return buildDevice, &cleaner, nil +} diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go new file mode 100644 index 0000000000..088dcb1ab0 --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go @@ -0,0 +1,40 @@ +package testutils + +import ( + "context" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +var _ block.ReadonlyDevice = (*ZeroDevice)(nil) + +type ZeroDevice struct { + blockSize int64 +} + +func (z *ZeroDevice) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { + clear(p) + + return len(p), nil +} + +func (z *ZeroDevice) BlockSize() int64 { + return z.blockSize +} + +func (z *ZeroDevice) Slice(ctx context.Context, off, length int64) ([]byte, error) { + return make([]byte, length), nil +} + +func (z *ZeroDevice) Header() *header.Header { + return nil +} + +func (z *ZeroDevice) Close() error { + return nil +} + +func (z *ZeroDevice) Size() (int64, error) { + return 0, nil +} From a9748a0b52b10db99928d5a635f6d959e582c530 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 18:22:42 -0800 Subject: [PATCH 10/21] Cleanup --- packages/orchestrator/cmd/mount-rootfs/main.go | 11 +++++------ .../internal/sandbox/nbd/testutils/cleaner.go | 6 +++++- .../internal/sandbox/nbd/testutils/mount.go | 2 -- .../internal/sandbox/nbd/testutils/zero_device.go | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index 04ba7e652e..def723b348 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -9,6 +9,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "time" "github.com/google/uuid" "go.uber.org/zap" @@ -39,7 +40,7 @@ func main() { if *logging { logger, err := zap.NewDevelopment() if err != nil { - log.Fatalf("failed to create logger: %s", err) + panic(fmt.Errorf("failed to create logger: %w", err)) } zap.ReplaceGlobals(logger) } @@ -60,10 +61,8 @@ func main() { } func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool) error { - cleanupCtx := context.Background() //nolint:contextcheck // we need to use separate context otherwise the cleanup can be problematic - rootfs, rootfsCleanup, err := testutils.TemplateRootfs(ctx, buildID) - defer rootfsCleanup.Run(cleanupCtx) + defer rootfsCleanup.Run(ctx, 30*time.Second) if err != nil { return fmt.Errorf("failed to get template rootfs: %w", err) } @@ -88,7 +87,7 @@ func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool defer overlay.Close() devicePath, deviceCleanup, err := testutils.GetNBDDevice(nbdContext, overlay) - defer deviceCleanup.Run(cleanupCtx) + defer deviceCleanup.Run(ctx, 30*time.Second) if err != nil { return fmt.Errorf("failed to get nbd device: %w", err) } @@ -104,7 +103,7 @@ func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool fmt.Fprintf(os.Stdout, "creating mount path directory: %s\n", mountPath) mountCleanup, err := testutils.MountNBDDevice(devicePath, mountPath) - defer mountCleanup.Run(cleanupCtx) + defer mountCleanup.Run(ctx, 30*time.Second) if err != nil { return fmt.Errorf("failed to mount device to mount path: %w", err) } diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go b/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go index cfdabec96a..59cf3ee7a6 100644 --- a/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/cleaner.go @@ -6,6 +6,7 @@ import ( "fmt" "slices" "sync" + "time" ) type Cleaner struct { @@ -17,7 +18,10 @@ func (c *Cleaner) Add(f func(ctx context.Context) error) { c.steps = append(c.steps, f) } -func (c *Cleaner) Run(ctx context.Context) (err error) { +func (c *Cleaner) Run(ctx context.Context, timeout time.Duration) (err error) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + defer cancel() + c.once.Do(func() { var errs []error diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go b/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go index a55908ab6d..cab0033e8c 100644 --- a/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/mount.go @@ -38,8 +38,6 @@ func MountNBDDevice(device nbd.DevicePath, mountPath string) (*Cleaner, error) { fmt.Fprintf(os.Stderr, "failed to unmount device from mount path: %v\n", err) } } - - return nil }) return &cleaner, nil diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go index 088dcb1ab0..e9c49c4b22 100644 --- a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go @@ -13,7 +13,7 @@ type ZeroDevice struct { blockSize int64 } -func (z *ZeroDevice) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { +func (z *ZeroDevice) ReadAt(_ context.Context, p []byte, _ int64) (n int, err error) { clear(p) return len(p), nil @@ -23,7 +23,7 @@ func (z *ZeroDevice) BlockSize() int64 { return z.blockSize } -func (z *ZeroDevice) Slice(ctx context.Context, off, length int64) ([]byte, error) { +func (z *ZeroDevice) Slice(_ context.Context, _, length int64) ([]byte, error) { return make([]byte, length), nil } From 05215cb1822dda0ad7a4846a9a9dd52b49c260e9 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 18:56:42 -0800 Subject: [PATCH 11/21] Cleanup --- .../orchestrator/cmd/mount-rootfs/main.go | 60 +++++++++++++++++-- .../sandbox/nbd/testutils/zero_device.go | 33 +++++++++- packages/shared/pkg/storage/header/header.go | 1 + 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index def723b348..4708368fb8 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -19,10 +19,13 @@ import ( ) func main() { - buildId := flag.String("build", "", "build id") - mountPath := flag.String("mount", "", "mount path") - verify := flag.Bool("verify", false, "verify rootfs integrity") + buildId := flag.String("build", "", "build id (only used when empty flag is false)") + mountPath := flag.String("mount", "", "mount path (only used when empty flag is false)") + verify := flag.Bool("verify", false, "verify rootfs integrity (only used when empty flag is false)") logging := flag.Bool("log", false, "enable logging (it is pretty spammy)") + empty := flag.Bool("empty", false, "create an empty rootfs") + size := flag.Int64("size", 1024*1024*1024, "size of the rootfs (only used when empty flag is true)") + blockSize := flag.Int64("block-size", 4096, "block size of the rootfs (only used when empty flag is true)") flag.Parse() @@ -54,10 +57,57 @@ func main() { // We use a separate ctx for majority of the operations as cancelling context for the NBD+storage and *then* doing cleanup for these often resulted in deadlocks. nbdContext := context.Background() - err := run(ctx, nbdContext, *buildId, *mountPath, *verify) + if *empty { + err := runEmpty(ctx, nbdContext, *size, *blockSize) + if err != nil { + panic(fmt.Errorf("failed to create empty rootfs: %w", err)) + } + } else { + err := run(ctx, nbdContext, *buildId, *mountPath, *verify) + if err != nil { + panic(fmt.Errorf("failed to mount rootfs: %w", err)) + } + } +} + +func runEmpty(ctx, nbdContext context.Context, size int64, blockSize int64) error { + cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-rootfs.ext4.cow.cache-%s", uuid.New().String())) + + emptyDevice, err := testutils.NewZeroDevice(size, blockSize) + if err != nil { + return fmt.Errorf("failed to create zero device: %w", err) + } + + defer os.RemoveAll(cowCachePath) + + cache, err := block.NewCache( + size, + blockSize, + cowCachePath, + false, + ) if err != nil { - panic(fmt.Errorf("failed to mount rootfs: %w", err)) + return fmt.Errorf("failed to create cache: %w", err) } + + fmt.Printf("caching writes to: %+v\n", cowCachePath) + + overlay := block.NewOverlay(emptyDevice, cache) + defer overlay.Close() + + devicePath, deviceCleanup, err := testutils.GetNBDDevice(nbdContext, testutils.NewLoggerOverlay(overlay)) + defer deviceCleanup.Run(ctx, 30*time.Second) + if err != nil { + return fmt.Errorf("failed to get nbd device: %w", err) + } + + fmt.Printf("rootfs exposed as device: %s\n", devicePath) + + <-ctx.Done() + + fmt.Println("closing rootfs mount") + + return nil } func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool) error { diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go index e9c49c4b22..c7f167f24b 100644 --- a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go @@ -2,6 +2,9 @@ package testutils import ( "context" + "fmt" + + "github.com/google/uuid" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" @@ -11,6 +14,34 @@ var _ block.ReadonlyDevice = (*ZeroDevice)(nil) type ZeroDevice struct { blockSize int64 + size int64 + header *header.Header +} + +func NewZeroDevice(size int64, blockSize int64) (*ZeroDevice, error) { + h, err := header.NewHeader(header.NewTemplateMetadata( + uuid.Nil, + uint64(blockSize), + uint64(size), + ), + []*header.BuildMap{ + { + Offset: 0, + Length: uint64(size), + BuildId: uuid.Nil, + BuildStorageOffset: 0, + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create header: %w", err) + } + + return &ZeroDevice{ + size: size, + blockSize: blockSize, + header: h, + }, nil } func (z *ZeroDevice) ReadAt(_ context.Context, p []byte, _ int64) (n int, err error) { @@ -28,7 +59,7 @@ func (z *ZeroDevice) Slice(_ context.Context, _, length int64) ([]byte, error) { } func (z *ZeroDevice) Header() *header.Header { - return nil + return z.header } func (z *ZeroDevice) Close() error { diff --git a/packages/shared/pkg/storage/header/header.go b/packages/shared/pkg/storage/header/header.go index 74b51be985..fd9e4dfc34 100644 --- a/packages/shared/pkg/storage/header/header.go +++ b/packages/shared/pkg/storage/header/header.go @@ -85,6 +85,7 @@ func (t *Header) GetShiftedMapping(offset int64) (mappedOffset int64, mappedLeng return mappedOffset, mappedLength, buildID, nil } +// TODO: Maybe we can optimize mapping by automatically assuming the mapping is uuid.Nil if we don't find it + stopping storing the nil mapping. func (t *Header) getMapping(offset int64) (*BuildMap, int64, error) { if offset < 0 || offset >= int64(t.Metadata.Size) { if t.IsNormalizeFixApplied() { From 0a62de9a59793db7e8ea22d01fb4796bb4df8f26 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 19:04:23 -0800 Subject: [PATCH 12/21] Fix format --- packages/orchestrator/cmd/mount-rootfs/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index 4708368fb8..74762fa182 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -71,7 +71,7 @@ func main() { } func runEmpty(ctx, nbdContext context.Context, size int64, blockSize int64) error { - cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-rootfs.ext4.cow.cache-%s", uuid.New().String())) + cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("rootfs.ext4.cow.cache-%s", uuid.New().String())) emptyDevice, err := testutils.NewZeroDevice(size, blockSize) if err != nil { From d32c6835121d3793987a4b8627c8cfe835e1c56a Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 19:13:46 -0800 Subject: [PATCH 13/21] Temporarily remove incomplete tests --- .../orchestrator/cmd/test-rootfs-diff/main.go | 383 --------------- .../internal/sandbox/rootfs_test.go | 447 ------------------ 2 files changed, 830 deletions(-) delete mode 100644 packages/orchestrator/cmd/test-rootfs-diff/main.go delete mode 100644 packages/orchestrator/internal/sandbox/rootfs_test.go diff --git a/packages/orchestrator/cmd/test-rootfs-diff/main.go b/packages/orchestrator/cmd/test-rootfs-diff/main.go deleted file mode 100644 index c68a4921fc..0000000000 --- a/packages/orchestrator/cmd/test-rootfs-diff/main.go +++ /dev/null @@ -1,383 +0,0 @@ -package main - -// import ( -// "bytes" -// "context" -// "crypto/sha256" -// "flag" -// "fmt" -// "io" -// "os" -// "os/exec" -// "os/signal" -// "path/filepath" - -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/disk/nbd" -// "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" -// "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" -// "github.com/google/uuid" -// ) - -// type DeviceWithClose struct { -// *build.File -// id string -// size int64 -// } - -// func (d *DeviceWithClose) Close() error { -// return nil -// } - -// func (d *DeviceWithClose) Size() (int64, error) { -// return d.size, nil -// } - -// func (d *DeviceWithClose) ReadAt(p []byte, off int64) (int, error) { -// return d.File.ReadAt(p, off) -// } - -// const ( -// contentFileName = "test.txt" -// content = `` -// blockSize = 4096 -// ) - -// // Execute the passed callback with the passed overlay mounted as a nbd device. -// func executeForNbd( -// ctx context.Context, -// overlay *block.Overlay, -// cb func(mountedPath string) error, -// ) error { -// mnt := nbd.NewDirectPathMount(overlay) - -// nbdCtx, nbdCancel := context.WithCancel(ctx) -// defer nbdCancel() - -// go func() { -// <-nbdCtx.Done() - -// mnt.Close() -// }() - -// mntIndex, err := mnt.Open(nbdCtx) -// if err != nil { -// return fmt.Errorf("failed to open: %w", err) -// } - -// devicePath := nbd.GetDevicePath(mntIndex) - -// fmt.Printf("- created nbd device at %s\n", devicePath) - -// // Check the block device health -// out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() -// if err != nil { -// return fmt.Errorf("failed to fsck: %w %s", err, out) -// } - -// fmt.Printf("- fscked base nbd device at %s\n", out) - -// tmpDir, err := os.MkdirTemp("", "mount-*") -// if err != nil { -// return fmt.Errorf("failed to create temp dir: %w", err) -// } - -// defer os.RemoveAll(tmpDir) - -// out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() -// if err != nil { -// return fmt.Errorf("failed to mount: %w - %s", err, out) -// } - -// defer func() { -// out, err = exec.Command("umount", tmpDir).CombinedOutput() -// if err != nil { -// fmt.Fprintf(os.Stderr, "failed to umount: %s - %s", err, out) -// } -// }() - -// fmt.Printf("- mounted rootfs at %s\n", devicePath) - -// err = cb(tmpDir) -// if err != nil { -// return fmt.Errorf("failed execute: %w", err) -// } - -// return nil -// } - -// func fileFromStorage( -// ctx context.Context, -// baseBuildId string, -// store *build.DiffStore, -// ) (*build.File, *header.Header, error) { -// buildId, err := uuid.Parse(baseBuildId) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to parse build id: %w", err) -// } - -// object := gcs.NewObject(ctx, gcs.TemplateBucket, buildId.String()+"/"+string(build.Rootfs)) - -// size, err := object.Size() -// if err != nil { -// return nil, nil, fmt.Errorf("failed to get object size: %w", err) -// } - -// header := header.NewHeader(&header.Metadata{ -// BuildId: buildId, -// BaseBuildId: buildId, -// Size: uint64(size), -// Version: 1, -// BlockSize: uint64(blockSize), -// Generation: 1, -// }, nil) - -// rootfs := build.NewFile(header, store, build.Rootfs) - -// return rootfs, header, nil -// } - -// // Create an overlay by extracting the diff from the passed overlay, -// // putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. -// func fileFromOverlay( -// overlay *block.Overlay, -// baseHeader *header.Header, -// store *build.DiffStore, -// ) (*build.File, *header.Header, error) { -// diffBuildId := uuid.New() - -// // TODO: Diff file is not cleaned up after exit. The same goes for the base storage file in the cache. -// // This should not affect the test. -// diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to create diff file: %w", err) -// } - -// cache, err := overlay.EjectCache() -// if err != nil { -// return nil, nil, fmt.Errorf("failed to eject cache: %w", err) -// } - -// dirtyBlocks, err := cache.Export(diffFile) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to export cache: %w", err) -// } - -// diff, err := diffFile.ToDiff(blockSize) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) -// } - -// newMappings := header.CreateMapping( -// baseHeader.Metadata, -// &diffBuildId, -// dirtyBlocks, -// ) - -// diffMappings := header.MergeMappings( -// baseHeader.Mapping, -// newMappings, -// ) - -// diffHeader := header.NewHeader(&header.Metadata{ -// BuildId: diffBuildId, -// BaseBuildId: baseHeader.Metadata.BaseBuildId, -// Size: baseHeader.Metadata.Size, -// Version: baseHeader.Metadata.Version, -// BlockSize: baseHeader.Metadata.BlockSize, -// }, diffMappings) - -// store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) - -// // Create a build file that will use the diff and the base already in the store -// diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) - -// return diffRootfs, diffHeader, nil -// } - -// func compareSources( -// s1, -// s2 io.ReaderAt, -// diffBuildId *uuid.UUID, -// mappings []*header.BuildMap, -// ) error { -// for _, mapping := range mappings { -// if mapping.BuildId.String() != diffBuildId.String() { -// continue -// } - -// for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { -// c1 := make([]byte, blockSize) -// n1, err := s1.ReadAt(c1, int64(off)) -// if err != nil { -// return fmt.Errorf("failed to read content1: %w", err) -// } - -// c2 := make([]byte, blockSize) -// n2, err := s2.ReadAt(c2, int64(off)) -// if err != nil { -// return fmt.Errorf("failed to read content2: %w", err) -// } - -// if n1 != n2 { -// return fmt.Errorf("content length mismatch: %d != %d", n1, n2) -// } - -// if !bytes.Equal(c1, c2) { -// // Hash the content and show the mismatch -// h1 := sha256.Sum256(c1) -// h2 := sha256.Sum256(c2) - -// return fmt.Errorf("content mismatch (showing hashes):\n%x != %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) -// } -// } -// } - -// return nil -// } - -// func checkNbd(ctx context.Context, buildId string) error { -// store, err := build.NewDiffStore(gcs.TemplateBucket, ctx) -// if err != nil { -// return fmt.Errorf("failed to create diff store: %w", err) -// } - -// baseRootfs, baseHeader, err := fileFromStorage(ctx, buildId, store) -// if err != nil { -// return fmt.Errorf("failed to create base overlay: %w", err) -// } - -// baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) -// baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) -// if err != nil { -// return fmt.Errorf("failed to create cache: %w", err) -// } -// defer baseCache.Close() - -// baseOverlay := block.NewOverlay( -// &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, -// baseCache, -// blockSize, -// ) - -// defer baseOverlay.Close() - -// fmt.Printf("\n----- Base overlay mount -----\n\n") - -// // Modify content in the overlay mounted from the base rootfs -// err = executeForNbd(ctx, baseOverlay, func(mountedPath string) error { -// contentPath := filepath.Join(mountedPath, contentFileName) - -// err = os.WriteFile(contentPath, []byte(content), 0644) -// if err != nil { -// return fmt.Errorf("failed to write file: %w", err) -// } - -// // fmt.Printf("- writing to content Path: %s\n", contentPath) - -// // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() -// // if err != nil { -// // return fmt.Errorf("failed to sync: %w - %s", err, out) -// // } - -// // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() -// // if err != nil { -// // return fmt.Errorf("failed to sync: %w - %s", err, out) -// // } - -// return nil -// }) - -// diffRootfs, diffHeader, err := fileFromOverlay(baseOverlay, baseHeader, store) -// if err != nil { -// return fmt.Errorf("failed to create diff overlay: %w", err) -// } - -// fmt.Printf("\n----- Diff header -----\n") - -// for _, mapping := range diffHeader.Mapping { -// fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) -// } - -// // Compare the changed parts in the base overlay with the content in the diff overlay. -// err = compareSources( -// baseOverlay, -// diffRootfs, -// &diffHeader.Metadata.BuildId, -// diffHeader.Mapping, -// ) -// if err != nil { -// return fmt.Errorf("failed to compare overlays: %w", err) -// } else { -// fmt.Println("overlay comparison successful") -// } - -// diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) - -// diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) -// if err != nil { -// return fmt.Errorf("failed to create cache: %w", err) -// } -// defer diffCache.Close() - -// diffOverlay := block.NewOverlay( -// &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, -// diffCache, -// blockSize, -// ) -// defer diffOverlay.Close() - -// fmt.Printf("\n----- Diff overlay mount -----\n\n") - -// // Check the modified content in the overlay created from the diff and the base -// err = executeForNbd(ctx, diffOverlay, func(mountedPath string) error { -// contentPath := filepath.Join(mountedPath, contentFileName) - -// readContent, err := os.ReadFile(contentPath) -// if err != nil { -// return fmt.Errorf("failed to read file: %w", err) -// } - -// fmt.Printf("- reading from content Path: %s\n", contentPath) -// fmt.Printf("- content: %s\n", readContent) - -// if string(readContent) != content { -// return fmt.Errorf("content mismatch: %s\n", readContent) -// } - -// return nil -// }) -// if err != nil { -// return fmt.Errorf("failed to mount nbd: %w", err) -// } - -// return nil -// } - -// func main() { -// buildId := flag.String("build", "", "template build id") - -// flag.Parse() - -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// done := make(chan os.Signal, 1) -// signal.Notify(done, os.Interrupt) - -// go func() { -// <-done - -// cancel() -// }() - -// fmt.Printf("\n================== ROOTFS DIFF TEST ===================") - -// err := checkNbd(ctx, *buildId) -// if err != nil { -// fmt.Fprintf(os.Stderr, "\n\n => failed:\n %s\n\n", err) -// } else { -// fmt.Println("\n\n => success") -// } -// } diff --git a/packages/orchestrator/internal/sandbox/rootfs_test.go b/packages/orchestrator/internal/sandbox/rootfs_test.go deleted file mode 100644 index 3f32cff42b..0000000000 --- a/packages/orchestrator/internal/sandbox/rootfs_test.go +++ /dev/null @@ -1,447 +0,0 @@ -package sandbox - -// import ( -// "bytes" -// "context" -// "crypto/sha256" -// "fmt" -// "io" -// "os" -// "os/exec" -// "path/filepath" -// "testing" - -// "github.com/google/uuid" -// "github.com/stretchr/testify/assert" -// "go.uber.org/zap" - -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/build" -// "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/disk/nbd" -// "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" -// ) - -// type DeviceWithClose struct { -// *build.File - -// id string -// size int64 -// } - -// func (d *DeviceWithClose) Close() error { -// return nil -// } - -// func (d *DeviceWithClose) Size() (int64, error) { -// return d.size, nil -// } - -// func (d *DeviceWithClose) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { -// return d.File.ReadAt(ctx, p, off) -// } - -// const ( -// contentFileName = "test.txt" -// content = `test-file-content` -// blockSize = 4096 -// ) - -// // Execute the passed callback with the passed overlay mounted as a nbd device. -// func executeForNbd( -// t *testing.T, -// ctx context.Context, -// overlay *block.Overlay, -// cb func(mountedPath string) error, -// ) error { -// mnt := nbd.NewDirectPathMount(overlay, nil, zap.NewNop()) - -// nbdCtx, nbdCancel := context.WithCancel(ctx) -// defer nbdCancel() - -// go func() { -// <-nbdCtx.Done() - -// mnt.Close() -// }() - -// mntIndex, err := mnt.Open(nbdCtx) -// if err != nil { -// return fmt.Errorf("failed to open: %w", err) -// } - -// devicePath := nbd.GetDevicePath(mntIndex) - -// t.Logf("- created nbd device at %s\n", devicePath) - -// // Check the block device health -// out, err := exec.CommandContext(nbdCtx, "fsck.ext4", "-n", devicePath).CombinedOutput() -// if err != nil { -// return fmt.Errorf("failed to fsck: %w %s", err, out) -// } - -// t.Logf("- fscked base nbd device at %s\n", out) - -// tmpDir, err := os.MkdirTemp("", "mount-*") -// if err != nil { -// return fmt.Errorf("failed to create temp dir: %w", err) -// } - -// defer os.RemoveAll(tmpDir) - -// out, err = exec.CommandContext(nbdCtx, "mount", devicePath, tmpDir).CombinedOutput() -// if err != nil { -// return fmt.Errorf("failed to mount: %w - %s", err, out) -// } - -// defer func() { -// out, err = exec.Command("umount", tmpDir).CombinedOutput() -// if err != nil { -// t.Errorf("failed to umount: %s - %s", err, out) -// } -// }() - -// t.Logf("- mounted rootfs at %s\n", devicePath) - -// err = cb(tmpDir) -// if err != nil { -// return fmt.Errorf("failed execute: %w", err) -// } - -// return nil -// } - -// // Create a test file with given size formatted as ext4 -// func createRootfsFile(t *testing.T, size int64) (*os.File, error) { -// f, err := os.CreateTemp("", "test-rootfs-*.ext4") -// if err != nil { -// return nil, fmt.Errorf("failed to create temp file: %w", err) -// } -// t.Cleanup(func() { -// f.Close() -// os.Remove(f.Name()) -// }) - -// // Ensure the file is big enough -// err = f.Truncate(size) -// if err != nil { -// return nil, fmt.Errorf("failed to truncate file: %w", err) -// } - -// // Close before formatting -// err = f.Close() -// if err != nil { -// os.Remove(f.Name()) -// return nil, fmt.Errorf("failed to close file: %w", err) -// } - -// // Format as ext4 -// out, err := exec.Command("mkfs.ext4", "-F", "-b", fmt.Sprintf("%d", blockSize), f.Name()).CombinedOutput() -// if err != nil { -// os.Remove(f.Name()) -// return nil, fmt.Errorf("failed to format ext4: %w - %s", err, out) -// } - -// // Reopen the file -// f, err = os.OpenFile(f.Name(), os.O_RDWR, 0o644) -// if err != nil { -// os.Remove(f.Name()) -// return nil, fmt.Errorf("failed to reopen file: %w", err) -// } - -// return f, nil -// } - -// // TestDiff implements the build.Diff interface for testing -// type TestDiff struct { -// reader io.ReaderAt -// size int64 -// } - -// func (d *TestDiff) Close() error { -// return nil -// } - -// func (d *TestDiff) ReadAt(p []byte, off int64) (int, error) { -// return d.reader.ReadAt(p, off) -// } - -// func (d *TestDiff) Slice(off, length int64) ([]byte, error) { -// buf := make([]byte, length) -// n, err := d.ReadAt(buf, off) -// if err != nil { -// return nil, err -// } -// return buf[:n], nil -// } - -// func (d *TestDiff) CachePath() (string, error) { -// return "", nil -// } - -// func baseRootfs( -// t *testing.T, -// ctx context.Context, -// store *build.DiffStore, -// ) (*build.File, *header.Header, error) { -// buildId := uuid.New() - -// // Create a test file of 10MB -// rootfsFile, err := createRootfsFile(t, 10*1024*1024) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to create test file: %w", err) -// } - -// fi, err := rootfsFile.Stat() -// if err != nil { -// return nil, nil, fmt.Errorf("failed to get file info: %w", err) -// } - -// header := header.NewHeader(&header.Metadata{ -// BuildId: buildId, -// BaseBuildId: buildId, -// Size: uint64(fi.Size()), -// Version: 1, -// BlockSize: uint64(blockSize), -// Generation: 1, -// }, nil) - -// // Create a test diff and add it to the store -// diff := &TestDiff{ -// reader: rootfsFile, -// size: fi.Size(), -// } -// store.Add(buildId.String(), build.Rootfs, diff) - -// rootfs := build.NewFile(header, store, build.Rootfs) - -// return rootfs, header, nil -// } - -// // Create an overlay by extracting the diff from the passed overlay, -// // putting it into the store and then creating a mapping that combines the diff and the base to a new overlay. -// func fileFromOverlay( -// t *testing.T, -// overlay *block.Overlay, -// baseHeader *header.Header, -// store *build.DiffStore, -// ) (*build.File, *header.Header, error) { -// diffBuildId := uuid.New() - -// diffFile, err := build.NewLocalDiffFile(diffBuildId.String(), build.Rootfs) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to create diff file: %w", err) -// } -// t.Cleanup(func() { -// diffFile.Close() -// os.Remove(diffFile.Name()) -// }) - -// cache, err := overlay.EjectCache() -// if err != nil { -// return nil, nil, fmt.Errorf("failed to eject cache: %w", err) -// } - -// dirtyBlocks, err := cache.Export(diffFile) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to export cache: %w", err) -// } - -// diff, err := diffFile.ToDiff(blockSize) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to convert diff file to diff: %w", err) -// } - -// newMappings := header.CreateMapping( -// baseHeader.Metadata, -// &diffBuildId, -// dirtyBlocks, -// ) - -// diffMappings := header.MergeMappings( -// baseHeader.Mapping, -// newMappings, -// ) - -// diffHeader := header.NewHeader(&header.Metadata{ -// BuildId: diffBuildId, -// BaseBuildId: baseHeader.Metadata.BaseBuildId, -// Size: baseHeader.Metadata.Size, -// Version: baseHeader.Metadata.Version, -// BlockSize: baseHeader.Metadata.BlockSize, -// }, diffMappings) - -// store.Add(diffHeader.Metadata.BuildId.String(), build.Rootfs, diff) - -// // Create a build file that will use the diff and the base already in the store -// diffRootfs := build.NewFile(diffHeader, store, build.Rootfs) - -// return diffRootfs, diffHeader, nil -// } - -// func compareSources( -// s1, -// s2 io.ReaderAt, -// diffBuildId *uuid.UUID, -// mappings []*header.BuildMap, -// ) error { -// for _, mapping := range mappings { -// if mapping.BuildId.String() != diffBuildId.String() { -// continue -// } - -// for off := mapping.Offset; off < mapping.Offset+mapping.Length; off += blockSize { -// c1 := make([]byte, blockSize) -// n1, err := s1.ReadAt(c1, int64(off)) -// if err != nil { -// return fmt.Errorf("failed to read content1: %w", err) -// } - -// c2 := make([]byte, blockSize) -// n2, err := s2.ReadAt(c2, int64(off)) -// if err != nil { -// return fmt.Errorf("failed to read content2: %w", err) -// } - -// if n1 != n2 { -// return fmt.Errorf("content length mismatch: %d != %d", n1, n2) -// } - -// if !bytes.Equal(c1, c2) { -// // Hash the content and show the mismatch -// h1 := sha256.Sum256(c1) -// h2 := sha256.Sum256(c2) - -// return fmt.Errorf("content mismatch (showing hashes):\nBase: %x != Diff: %x\n, offset: %d, length: %d", h1, h2, mapping.Offset, mapping.Length) -// } -// } -// } - -// return nil -// } - -// func checkNbd(t *testing.T, ctx context.Context) error { -// store, err := build.NewDiffStore(nil, ctx) -// if err != nil { -// return fmt.Errorf("failed to create diff store: %w", err) -// } -// t.Cleanup(func() { -// store.Close() -// }) - -// baseRootfs, baseHeader, err := baseRootfs(t, ctx, store) -// if err != nil { -// return fmt.Errorf("failed to create base overlay: %w", err) -// } - -// baseCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("cache-base-rootfs.ext4-%s", baseHeader.Metadata.BuildId)) -// baseCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, baseCachePath, false) -// if err != nil { -// return fmt.Errorf("failed to create cache: %w", err) -// } -// defer baseCache.Close() - -// baseOverlay := block.NewOverlay( -// &DeviceWithClose{baseRootfs, "base", int64(baseHeader.Metadata.Size)}, -// baseCache, -// blockSize, -// ) - -// defer baseOverlay.Close() - -// t.Logf("\n----- Base overlay mount -----\n\n") - -// // Modify content in the overlay mounted from the base rootfs -// err = executeForNbd(t, ctx, baseOverlay, func(mountedPath string) error { -// contentPath := filepath.Join(mountedPath, contentFileName) - -// err = os.WriteFile(contentPath, []byte(content), 0o644) -// if err != nil { -// return fmt.Errorf("failed to write file: %w", err) -// } - -// // fmt.Printf("- writing to content Path: %s\n", contentPath) - -// // out, err := exec.CommandContext(ctx, "sync").CombinedOutput() -// // if err != nil { -// // return fmt.Errorf("failed to sync: %w - %s", err, out) -// // } - -// // out, err = exec.CommandContext(ctx, "/bin/bash", "-c", "echo 3 | sudo tee /proc/sys/vm/drop_caches").CombinedOutput() -// // if err != nil { -// // return fmt.Errorf("failed to sync: %w - %s", err, out) -// // } - -// return nil -// }) - -// diffRootfs, diffHeader, err := fileFromOverlay(t, baseOverlay, baseHeader, store) -// if err != nil { -// return fmt.Errorf("failed to create diff overlay: %w", err) -// } - -// t.Logf("\n----- Diff header -----\n") - -// for _, mapping := range diffHeader.Mapping { -// t.Log(mapping.Format(baseHeader.Metadata.BlockSize)) -// } - -// // Compare the changed parts in the base overlay with the content in the diff overlay. -// err = compareSources( -// baseOverlay, -// diffRootfs, -// &diffHeader.Metadata.BuildId, -// diffHeader.Mapping, -// ) -// if err != nil { -// return fmt.Errorf("failed to compare overlays: %w", err) -// } else { -// t.Logf("overlay comparison successful") -// } - -// diffCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("diff-cache-rootfs.ext4-%s", diffHeader.Metadata.BuildId)) - -// diffCache, err := block.NewCache(int64(baseHeader.Metadata.Size), blockSize, diffCachePath, false) -// if err != nil { -// return fmt.Errorf("failed to create cache: %w", err) -// } -// defer diffCache.Close() - -// diffOverlay := block.NewOverlay( -// &DeviceWithClose{diffRootfs, "diff", int64(baseHeader.Metadata.Size)}, -// diffCache, -// blockSize, -// ) -// defer diffOverlay.Close() - -// t.Logf("\n----- Diff overlay mount -----\n\n") - -// // Check the modified content in the overlay created from the diff and the base -// err = executeForNbd(t, ctx, diffOverlay, func(mountedPath string) error { -// contentPath := filepath.Join(mountedPath, contentFileName) - -// readContent, err := os.ReadFile(contentPath) -// if err != nil { -// return fmt.Errorf("failed to read file: %w", err) -// } - -// t.Logf("- reading from content Path: %s\n", contentPath) -// t.Logf("- content: %s\n", readContent) - -// assert.Equal(t, content, string(readContent)) - -// return nil -// }) -// if err != nil { -// return fmt.Errorf("failed to mount nbd: %w", err) -// } - -// return nil -// } - -// func TestNbd(t *testing.T) { -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() - -// err := checkNbd(t, ctx) -// assert.NoError(t, err) -// } From 469e8877caeaabc12f5eb0277d9bc0dd92bb4234 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 19:14:59 -0800 Subject: [PATCH 14/21] Update deps --- packages/orchestrator/go.mod | 1 - packages/orchestrator/go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index a7dbb4a090..f7a6210bb0 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -41,7 +41,6 @@ require ( github.com/ngrok/firewall_toolkit v0.0.18 github.com/oapi-codegen/gin-middleware v1.0.2 github.com/pkg/errors v0.9.1 - github.com/pojntfx/go-nbd v0.3.2 github.com/shirou/gopsutil/v4 v4.25.6 github.com/soheilhy/cmux v0.1.5 github.com/stretchr/testify v1.11.1 diff --git a/packages/orchestrator/go.sum b/packages/orchestrator/go.sum index bf295884da..f7663be826 100644 --- a/packages/orchestrator/go.sum +++ b/packages/orchestrator/go.sum @@ -989,8 +989,6 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pojntfx/go-nbd v0.3.2 h1:qI6S4qsHD87V9fTH6jiS4DIqq/rWmI0El0xSToMUDeg= -github.com/pojntfx/go-nbd v0.3.2/go.mod h1:SehHnbi2e8NiSAKby42Itm8SIoS7b+wAprsfPH3qgYk= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= From 44480a4bfedf27ad62f51ac0f510d06ef0e155a0 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 19:56:39 -0800 Subject: [PATCH 15/21] Add basic test for the current dispatch bug --- .github/workflows/pr-tests.yml | 2 +- .../orchestrator/cmd/mount-rootfs/main.go | 28 ----- .../internal/sandbox/nbd/path_direct_test.go | 113 ++++++++++++++++++ 3 files changed, 114 insertions(+), 29 deletions(-) create mode 100644 packages/orchestrator/internal/sandbox/nbd/path_direct_test.go diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 760d9cfa2b..a4268fb7d3 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -58,7 +58,7 @@ jobs: - name: Run tests working-directory: ${{ matrix.package }} - run: go test -v ${{ matrix.test_path }} + run: sudo go test -race -v ${{ matrix.test_path }} validate-iac: name: Validate terraform diff --git a/packages/orchestrator/cmd/mount-rootfs/main.go b/packages/orchestrator/cmd/mount-rootfs/main.go index 74762fa182..b66b90191c 100644 --- a/packages/orchestrator/cmd/mount-rootfs/main.go +++ b/packages/orchestrator/cmd/mount-rootfs/main.go @@ -163,34 +163,6 @@ func run(ctx, nbdContext context.Context, buildID, mountPath string, verify bool fmt.Printf("rootfs mounted at path: %s\n", mountPath) } - // cmd := exec.CommandContext(ctx, "dd", "if=/dev/zero", "of="+devicePath, "bs=4k", "count=1", "oflag=direct") - - // cmd.Stdout = os.Stdout - // cmd.Stderr = os.Stderr - - // err = cmd.Run() - // if err != nil { - // return fmt.Errorf("failed to write zero to device (with direct flag): %w", err) - // } - - // fmt.Println("> zero written to device (with direct flag)") - - // d, err := os.OpenFile(devicePath, unix.O_DIRECT|unix.O_RDWR, 0) - // if err != nil { - // return fmt.Errorf("failed to open device: %w", err) - // } - // defer d.Close() - - // buf := make([]byte, 4096) - - // // fmt.Println("mmapped buffer start", unsafe.Pointer(&buf[0])) - // _, err = d.WriteAt(buf, 0) - // if err != nil { - // return fmt.Errorf("failed to write zero to device: %w", err) - // } - - // fmt.Println("zero written to device") - if verify { fmt.Println("\nverifying rootfs integrity...") diff --git a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go new file mode 100644 index 0000000000..b87f99e8cc --- /dev/null +++ b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go @@ -0,0 +1,113 @@ +package nbd_test + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + "golang.org/x/sys/unix" + + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" + "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd/testutils" +) + +func TestPathDirect4MBWrite(t *testing.T) { + if os.Geteuid() != 0 { + t.Fatalf("the nbd requires root privileges to run") + } + + // Create a device that's at least 4MB (use 10MB to be safe) + size := int64(10 * 1024 * 1024) + blockSize := int64(4096) + + // Create zero device + emptyDevice, err := testutils.NewZeroDevice(size, blockSize) + if err != nil { + t.Fatalf("failed to create zero device: %v", err) + } + + // Create cache path + cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("test-rootfs.ext4.cow.cache-%s", uuid.New().String())) + t.Cleanup(func() { + os.RemoveAll(cowCachePath) + }) + + // Create cache + cache, err := block.NewCache( + size, + blockSize, + cowCachePath, + false, + ) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + // Create overlay + overlay := block.NewOverlay(emptyDevice, cache) + t.Cleanup(func() { + overlay.Close() + }) + + // Get NBD device + nbdContext := context.Background() + devicePath, deviceCleanup, err := testutils.GetNBDDevice(nbdContext, overlay) + t.Cleanup(func() { + deviceCleanup.Run(t.Context(), 30*time.Second) + }) + if err != nil { + t.Fatalf("failed to get nbd device: %v", err) + } + + t.Logf("NBD device path: %s", devicePath) + + // We need to ensure buffer is page aligned to be able to use O_DIRECT. + const bs = 4 * 1024 * 1024 + buf, err := unix.Mmap(-1, 0, bs, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_ANON) + if err != nil { + panic(err) + } + + t.Cleanup(func() { + unix.Munmap(buf) + }) + + // Open device with direct I/O to trigger unbuffered write of 4MB. + deviceFile, err := os.OpenFile(string(devicePath), unix.O_DIRECT|unix.O_RDWR, 0) + if err != nil { + t.Fatalf("failed to open device: %v", err) + } + t.Cleanup(func() { + deviceFile.Close() + }) + + // Write 4MB at offset 0 + n, err := deviceFile.WriteAt(buf, 0) + if err != nil { + t.Fatalf("failed to write to device: %v", err) + } + if n != len(buf) { + t.Fatalf("partial write: expected %d bytes, wrote %d bytes", len(buf), n) + } + + // Verify the write by reading it back + readData := make([]byte, bs) + n, err = deviceFile.ReadAt(readData, 0) + if err != nil { + t.Fatalf("failed to read from device: %v", err) + } + if n != len(readData) { + t.Fatalf("partial read: expected %d bytes, read %d bytes", len(readData), n) + } + + if !bytes.Equal(buf, readData) { + t.Fatalf("data mismatch: expected %v, got %v", buf, readData) + } + + t.Logf("Successfully wrote and verified 4MB to NBD device") +} From 1036a97748004160a773530e3c533196d9fcba6a Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 19:57:19 -0800 Subject: [PATCH 16/21] Remove conversion --- packages/orchestrator/internal/sandbox/nbd/path_direct_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go index b87f99e8cc..ee672ae712 100644 --- a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go +++ b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go @@ -78,7 +78,7 @@ func TestPathDirect4MBWrite(t *testing.T) { }) // Open device with direct I/O to trigger unbuffered write of 4MB. - deviceFile, err := os.OpenFile(string(devicePath), unix.O_DIRECT|unix.O_RDWR, 0) + deviceFile, err := os.OpenFile(devicePath, unix.O_DIRECT|unix.O_RDWR, 0) if err != nil { t.Fatalf("failed to open device: %v", err) } From a675be464cbd6ed7775c45c29d621e35c38d7b48 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 20:00:40 -0800 Subject: [PATCH 17/21] Fix size --- .../orchestrator/internal/sandbox/nbd/testutils/zero_device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go index c7f167f24b..f266a1a2fb 100644 --- a/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go +++ b/packages/orchestrator/internal/sandbox/nbd/testutils/zero_device.go @@ -67,5 +67,5 @@ func (z *ZeroDevice) Close() error { } func (z *ZeroDevice) Size() (int64, error) { - return 0, nil + return z.size, nil } From 71a1e683df5258fd22593389624067fa3060a299 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 20:16:14 -0800 Subject: [PATCH 18/21] Disable race for now --- .github/workflows/pr-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index a4268fb7d3..330b963bf8 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -58,7 +58,7 @@ jobs: - name: Run tests working-directory: ${{ matrix.package }} - run: sudo go test -race -v ${{ matrix.test_path }} + run: sudo go test -v ${{ matrix.test_path }} validate-iac: name: Validate terraform From 176979402aaefab827e410dc0f42db5e52c7f3ed Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 21:32:14 -0800 Subject: [PATCH 19/21] Add more nbd tests --- .../internal/sandbox/nbd/path_direct_test.go | 193 ++++++++++++------ 1 file changed, 132 insertions(+), 61 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go index ee672ae712..75621cde15 100644 --- a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go +++ b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go @@ -1,113 +1,184 @@ package nbd_test import ( - "bytes" "context" + "crypto/rand" "fmt" "os" + "os/exec" "path/filepath" "testing" "time" "github.com/google/uuid" + "github.com/stretchr/testify/require" "golang.org/x/sys/unix" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/nbd/testutils" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" ) -func TestPathDirect4MBWrite(t *testing.T) { - if os.Geteuid() != 0 { - t.Fatalf("the nbd requires root privileges to run") - } - - // Create a device that's at least 4MB (use 10MB to be safe) +func TestPathDirect_Direct4MBWrite(t *testing.T) { size := int64(10 * 1024 * 1024) - blockSize := int64(4096) - // Create zero device - emptyDevice, err := testutils.NewZeroDevice(size, blockSize) + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, unix.O_DIRECT|unix.O_RDWR) + + const bs = 4 * 1024 * 1024 + buf, err := unix.Mmap(-1, 0, bs, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_ANON) if err != nil { - t.Fatalf("failed to create zero device: %v", err) + panic(err) } - // Create cache path + t.Cleanup(func() { + unix.Munmap(buf) + }) + + n, err := deviceFile.WriteAt(buf, 0) + require.NoError(t, err, "failed to write to device") + require.Equal(t, len(buf), n, "partial write") + + readData := make([]byte, bs) + n, err = deviceFile.ReadAt(readData, 0) + require.NoError(t, err, "failed to read from device") + require.Equal(t, len(readData), n, "partial read") + require.Equal(t, buf, readData, "data mismatch") +} + +// We usually see the 32MB write be split into smaller writes, even on O_DIRECT. +func TestPathDirect_Direct32MBWrite(t *testing.T) { + size := int64(256 * 1024 * 1024) + + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, unix.O_DIRECT|unix.O_RDWR) + + const bs = 32 * 1024 * 1024 + buf, err := unix.Mmap(-1, 0, bs, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_ANON) + require.NoError(t, err, "failed to mmap") + + t.Cleanup(func() { + unix.Munmap(buf) + }) + + n, err := deviceFile.WriteAt(buf, 0) + require.NoError(t, err, "failed to write to device") + require.Equal(t, len(buf), n, "partial write") + + readData := make([]byte, bs) + n, err = deviceFile.ReadAt(readData, 0) + require.NoError(t, err, "failed to read from device") + require.Equal(t, len(readData), n, "partial read") + require.Equal(t, buf, readData, "data mismatch") +} + +func TestPathDirect_Write(t *testing.T) { + size := int64(5 * 1024 * 1024) + + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDWR) + + const writeSize = 1024 * 1024 + testData := make([]byte, writeSize) + _, err := rand.Read(testData) + require.NoError(t, err, "failed to generate random data") + + n, err := deviceFile.WriteAt(testData, 0) + require.NoError(t, err, "failed to write data to device") + require.Equal(t, len(testData), n, "partial write") + + readData := make([]byte, writeSize) + n, err = deviceFile.ReadAt(readData, 0) + require.NoError(t, err, "failed to read data from device") + require.Equal(t, len(readData), n, "partial read") + require.Equal(t, testData, readData, "data mismatch") +} + +func TestPathDirect_WriteAtOffset(t *testing.T) { + size := int64(5 * 1024 * 1024) + + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDWR) + + const writeSize = 512 * 1024 + const writeOffset = 512 * 1024 + testData := make([]byte, writeSize) + _, err := rand.Read(testData) + require.NoError(t, err, "failed to generate random data") + + n, err := deviceFile.WriteAt(testData, writeOffset) + require.NoError(t, err, "failed to write data to device") + require.Equal(t, len(testData), n, "partial write") + + readData := make([]byte, writeSize) + n, err = deviceFile.ReadAt(readData, writeOffset) + require.NoError(t, err, "failed to read data from device") + require.Equal(t, len(readData), n, "partial read") + require.Equal(t, testData, readData, "data mismatch") +} + +func TestPathDirect_LargeWrite(t *testing.T) { + size := int64(1200 * 1024 * 1024) + + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDWR) + + time.Sleep(1 * time.Second) + cmd := exec.Command("dd", "if=/dev/zero", "of="+deviceFile.Name(), "bs=1G", "count=1") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + require.NoError(t, err, "failed to execute dd command") +} + +func TestPathLargeRead(t *testing.T) { + size := int64(1200 * 1024 * 1024) + + deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDONLY) + time.Sleep(1 * time.Second) + + cmd := exec.Command("dd", "if="+deviceFile.Name(), "of=/dev/null", "bs=1G", "count=1") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + require.NoError(t, err, "failed to execute dd command") +} + +func setupNBDDevice(t *testing.T, size, blockSize int64, flags int) *os.File { + require.Equal(t, 0, os.Geteuid(), "the nbd requires root privileges to run") + + emptyDevice, err := testutils.NewZeroDevice(size, blockSize) + require.NoError(t, err, "failed to create zero device") + cowCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("test-rootfs.ext4.cow.cache-%s", uuid.New().String())) t.Cleanup(func() { os.RemoveAll(cowCachePath) }) - // Create cache cache, err := block.NewCache( size, blockSize, cowCachePath, false, ) - if err != nil { - t.Fatalf("failed to create cache: %v", err) - } + require.NoError(t, err, "failed to create cache") - // Create overlay overlay := block.NewOverlay(emptyDevice, cache) t.Cleanup(func() { overlay.Close() }) - // Get NBD device nbdContext := context.Background() devicePath, deviceCleanup, err := testutils.GetNBDDevice(nbdContext, overlay) t.Cleanup(func() { deviceCleanup.Run(t.Context(), 30*time.Second) }) - if err != nil { - t.Fatalf("failed to get nbd device: %v", err) - } + require.NoError(t, err, "failed to get nbd device") t.Logf("NBD device path: %s", devicePath) - // We need to ensure buffer is page aligned to be able to use O_DIRECT. - const bs = 4 * 1024 * 1024 - buf, err := unix.Mmap(-1, 0, bs, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_ANON) - if err != nil { - panic(err) - } - - t.Cleanup(func() { - unix.Munmap(buf) - }) - - // Open device with direct I/O to trigger unbuffered write of 4MB. - deviceFile, err := os.OpenFile(devicePath, unix.O_DIRECT|unix.O_RDWR, 0) - if err != nil { - t.Fatalf("failed to open device: %v", err) - } + deviceFile, err := os.OpenFile(devicePath, flags, 0) + require.NoError(t, err, "failed to open device") t.Cleanup(func() { deviceFile.Close() }) - // Write 4MB at offset 0 - n, err := deviceFile.WriteAt(buf, 0) - if err != nil { - t.Fatalf("failed to write to device: %v", err) - } - if n != len(buf) { - t.Fatalf("partial write: expected %d bytes, wrote %d bytes", len(buf), n) - } - - // Verify the write by reading it back - readData := make([]byte, bs) - n, err = deviceFile.ReadAt(readData, 0) - if err != nil { - t.Fatalf("failed to read from device: %v", err) - } - if n != len(readData) { - t.Fatalf("partial read: expected %d bytes, read %d bytes", len(readData), n) - } - - if !bytes.Equal(buf, readData) { - t.Fatalf("data mismatch: expected %v, got %v", buf, readData) - } - - t.Logf("Successfully wrote and verified 4MB to NBD device") + return deviceFile } From 516898550e1dab83a6a37733d12b982221803c95 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 21:32:36 -0800 Subject: [PATCH 20/21] Remove invalid test --- .../shared/pkg/storage/storage_cache_test.go | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/packages/shared/pkg/storage/storage_cache_test.go b/packages/shared/pkg/storage/storage_cache_test.go index fa5950c1c2..86f89243b3 100644 --- a/packages/shared/pkg/storage/storage_cache_test.go +++ b/packages/shared/pkg/storage/storage_cache_test.go @@ -236,25 +236,3 @@ func TestMoveWithoutReplace_FailWhenExists(t *testing.T) { _, err = os.Stat(src) assert.ErrorIs(t, err, os.ErrNotExist) } - -func TestMoveWithoutReplace_Fail(t *testing.T) { - td := t.TempDir() - content := []byte("alpha") - src := filepath.Join(td, "src") - require.NoError(t, os.WriteFile(src, content, 0o644)) - - roDir := filepath.Join(td, "ro") - require.NoError(t, os.Mkdir(roDir, 0o555)) // r-x only, no write - t.Cleanup(func() { - // ensure cleanup possible - err := os.Chmod(roDir, 0o755) - assert.NoError(t, err) - }) - - dst := filepath.Join(roDir, "dst") - err := moveWithoutReplace(src, dst) - require.Error(t, err) - - _, err = os.Stat(src) - assert.ErrorIs(t, err, os.ErrNotExist) -} From da8c36dd007e5ff51b7802892b23c7b1e2af3374 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 18 Nov 2025 21:39:42 -0800 Subject: [PATCH 21/21] Fix lint --- .../orchestrator/internal/sandbox/nbd/path_direct_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go index 75621cde15..8094985077 100644 --- a/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go +++ b/packages/orchestrator/internal/sandbox/nbd/path_direct_test.go @@ -119,7 +119,7 @@ func TestPathDirect_LargeWrite(t *testing.T) { deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDWR) time.Sleep(1 * time.Second) - cmd := exec.Command("dd", "if=/dev/zero", "of="+deviceFile.Name(), "bs=1G", "count=1") + cmd := exec.CommandContext(t.Context(), "dd", "if=/dev/zero", "of="+deviceFile.Name(), "bs=1G", "count=1") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -133,7 +133,7 @@ func TestPathLargeRead(t *testing.T) { deviceFile := setupNBDDevice(t, size, header.RootfsBlockSize, os.O_RDONLY) time.Sleep(1 * time.Second) - cmd := exec.Command("dd", "if="+deviceFile.Name(), "of=/dev/null", "bs=1G", "count=1") + cmd := exec.CommandContext(t.Context(), "dd", "if="+deviceFile.Name(), "of=/dev/null", "bs=1G", "count=1") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -142,6 +142,8 @@ func TestPathLargeRead(t *testing.T) { } func setupNBDDevice(t *testing.T, size, blockSize int64, flags int) *os.File { + t.Helper() + require.Equal(t, 0, os.Geteuid(), "the nbd requires root privileges to run") emptyDevice, err := testutils.NewZeroDevice(size, blockSize)