Dirty pages deduplication between diff and base layer for memory

.github/actions/start-services/action.yml

@@ -18,6 +18,10 @@ inputs:
     description: "Number of frame encode workers"
     required: false
     default: ""
+  dedup_mode:
+    description: "Memfile dedup mode"
+    required: false
+    default: "default"
 
 runs:
   using: "composite"
@@ -129,6 +133,7 @@ runs:
         COMPRESS_TYPE: ${{ inputs.compress_type }}
         COMPRESS_LEVEL: ${{ inputs.compress_level }}
         COMPRESS_FRAME_ENCODE_WORKERS: ${{ inputs.compress_workers }}
+        TESTS_MEMFILE_DIFF_DEDUP_MODE: ${{ inputs.dedup_mode }}
         E2B_DEBUG: "true"
       run: |
         mkdir -p $SHARED_CHUNK_CACHE_PATH

.github/workflows/integration_tests.yml

@@ -25,16 +25,19 @@ jobs:
             compress_type: ""
             compress_level: ""
             compress_workers: ""
+            dedup_mode: "default"
           - name: zstd1
             compress_enabled: "true"
             compress_type: "zstd"
             compress_level: "1"
             compress_workers: "8"
+            dedup_mode: "best_effort"
           - name: lz4
             compress_enabled: "true"
             compress_type: "lz4"
             compress_level: "0"
             compress_workers: "8"
+            dedup_mode: "direct_io"
     env:
       # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -64,6 +67,7 @@ jobs:
           compress_type: ${{ matrix.compress_type }}
           compress_level: ${{ matrix.compress_level }}
           compress_workers: ${{ matrix.compress_workers }}
+          dedup_mode: ${{ matrix.dedup_mode }}
 
       - name: Run Integration Tests
         env:

packages/orchestrator/main.go

@@ -4,22 +4,41 @@ package main
 
 import (
 	"context"
+	"os"
+
+	"github.com/launchdarkly/go-sdk-common/v3/ldvalue"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/factories"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/tcpfirewall"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/version"
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
 )
 
 var commitSHA string
 
 func main() {
+	applyTestFlagOverrides()
+
 	factories.Run(factories.Options{
 		Version:       version.Version,
 		CommitSHA:     commitSHA,
 		EgressFactory: defaultEgressFactory,
 	})
 }
 
+func applyTestFlagOverrides() {
+	mode := os.Getenv("TESTS_MEMFILE_DIFF_DEDUP_MODE")
+	if mode == "" {
+		return
+	}
+
+	featureflags.OverrideJSONFlag(featureflags.MemfileDiffDedupFlag, ldvalue.FromJSONMarshal(map[string]any{
+		"enabled":    true,
+		"bestEffort": mode == "best_effort",
+		"directIO":   mode == "direct_io",
+	}))
+}
+
 func defaultEgressFactory(_ context.Context, deps *factories.Deps) (*factories.EgressSetup, error) {
 	fw := tcpfirewall.New(
 		deps.Logger,