From 1172331f43e7bb09076f6ab71d97fb59da5636fd Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 26 May 2026 01:30:28 -0700 Subject: [PATCH 01/11] feat(storage): write-through compressed templates to NFS on upload cachedSeekable.StoreFile previously skipped NFS write-through whenever compression was enabled, so even the orchestrator that just built a template would miss NFS on the first read and hit GCS. Add a per-frame FrameSink PutOption and hook it from the compressed StoreFile path so each compressed frame is teed into a .frm file at its C-space offset as it is produced, matching the layout used by the read-miss writeback. Gated by the existing write-to-cache-on-writes feature flag. --- .../shared/pkg/storage/compress_encode.go | 2 +- .../shared/pkg/storage/compress_upload.go | 7 ++++- .../pkg/storage/compress_upload_test.go | 8 +++--- packages/shared/pkg/storage/storage.go | 3 +++ .../pkg/storage/storage_cache_seekable.go | 21 ++++++++++++--- .../storage/storage_cache_seekable_test.go | 26 +++++++++++++++++++ packages/shared/pkg/storage/storage_fs.go | 9 ++++--- packages/shared/pkg/storage/storage_google.go | 2 +- .../pkg/storage/storageopts/storageopts.go | 8 ++++++ 9 files changed, 73 insertions(+), 13 deletions(-) diff --git a/packages/shared/pkg/storage/compress_encode.go b/packages/shared/pkg/storage/compress_encode.go index 3515a9d765..98364b46c5 100644 --- a/packages/shared/pkg/storage/compress_encode.go +++ b/packages/shared/pkg/storage/compress_encode.go @@ -113,7 +113,7 @@ func CompressBytes(ctx context.Context, data []byte, cfg CompressConfig) (*Frame up := &memPartUploader{} const compressBytesConcurrency = 1 - ft, checksum, err := compressStream(ctx, bytes.NewReader(data), cfg, up, compressBytesConcurrency) + ft, checksum, err := compressStream(ctx, bytes.NewReader(data), cfg, up, compressBytesConcurrency, nil) if err != nil { return nil, nil, [32]byte{}, err } diff --git a/packages/shared/pkg/storage/compress_upload.go b/packages/shared/pkg/storage/compress_upload.go index 861befbbb8..243f76f2b9 100644 --- a/packages/shared/pkg/storage/compress_upload.go +++ b/packages/shared/pkg/storage/compress_upload.go @@ -104,7 +104,7 @@ func (p *part) addFrame(ctx context.Context, uncompressedData []byte, pool *sync }) } -func compressStream(ctx context.Context, in io.Reader, cfg CompressConfig, uploader partUploader, maxUploadConcurrency int) (*FrameTable, [32]byte, error) { +func compressStream(ctx context.Context, in io.Reader, cfg CompressConfig, uploader partUploader, maxUploadConcurrency int, sink FrameSink) (*FrameTable, [32]byte, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -132,6 +132,7 @@ func compressStream(ctx context.Context, in io.Reader, cfg CompressConfig, uploa // Upload loop. var frameSizes []FrameSize + var cOffset int64 var loopErr error for p := range q { if err := p.compress.Wait(); err != nil { @@ -145,6 +146,10 @@ func compressStream(ctx context.Context, in io.Reader, cfg CompressConfig, uploa for _, f := range p.frames { frameSizes = append(frameSizes, FrameSize{U: int32(f.uncompressedSize), C: int32(len(f.compressed))}) compressed = append(compressed, f.compressed) + if sink != nil { + sink(cOffset, f.compressed) + } + cOffset += int64(len(f.compressed)) } pi := p.index diff --git a/packages/shared/pkg/storage/compress_upload_test.go b/packages/shared/pkg/storage/compress_upload_test.go index 6ac670e2f4..0721cddc92 100644 --- a/packages/shared/pkg/storage/compress_upload_test.go +++ b/packages/shared/pkg/storage/compress_upload_test.go @@ -166,6 +166,7 @@ func TestCompressStreamRoundTrip(t *testing.T) { cfg, up, 4, + nil, ) require.NoError(t, err) @@ -203,7 +204,7 @@ func TestCompressStreamContextCancel(t *testing.T) { up := &memPartUploader{} cfg := defaultCfg(CompressionZstd, 4, 2*megabyte) - _, _, err := compressStream(ctx, bytes.NewReader(data), cfg, up, 4) + _, _, err := compressStream(ctx, bytes.NewReader(data), cfg, up, 4, nil) require.Error(t, err) require.ErrorIs(t, err, context.Canceled) } @@ -234,7 +235,7 @@ func TestCompressStreamPartSizeMinimum(t *testing.T) { cfg := defaultCfg(CompressionZstd, 4, tc.frameSize) cfg.MinPartSizeMB = tc.minPartSizeMB - _, _, err := compressStream(t.Context(), bytes.NewReader(data), cfg, up, 4) + _, _, err := compressStream(t.Context(), bytes.NewReader(data), cfg, up, 4, nil) require.NoError(t, err) // Verify: no non-final part is under 5 MiB. @@ -290,7 +291,7 @@ func TestCompressStreamRace(t *testing.T) { cfg.EncoderConcurrency = 4 // multi-threaded zstd encoders for more contention } - ft, checksum, err := compressStream(ctx, bytes.NewReader(data), cfg, up, 4) + ft, checksum, err := compressStream(ctx, bytes.NewReader(data), cfg, up, 4, nil) if err != nil { return fmt.Errorf("stream %d: compress: %w", i, err) } @@ -357,6 +358,7 @@ func BenchmarkCompress(b *testing.B) { bytes.NewReader(data), compCfg, up, 4, + nil, ) if err != nil { b.Fatal(err) diff --git a/packages/shared/pkg/storage/storage.go b/packages/shared/pkg/storage/storage.go index 03d31ffab0..668ca49fb4 100644 --- a/packages/shared/pkg/storage/storage.go +++ b/packages/shared/pkg/storage/storage.go @@ -94,6 +94,7 @@ type ( ObjectMetadata = storageopts.ObjectMetadata PutOptions = storageopts.PutOptions PutOption = storageopts.PutOption + FrameSink = storageopts.FrameSink ) const ObjectMetadataTeamID = storageopts.ObjectMetadataTeamID @@ -105,6 +106,8 @@ func WithMetadata(metadata ObjectMetadata) PutOption { return storageopts.WithMe // backends use CompressConfigFromOpts to pull it back out. func WithCompressConfig(cfg CompressConfig) PutOption { return storageopts.WithCompression(cfg) } +func WithFrameSink(s FrameSink) PutOption { return storageopts.WithFrameSink(s) } + func WithChecksumSHA256() PutOption { return func(o *PutOptions) { o.Checksum = true } } diff --git a/packages/shared/pkg/storage/storage_cache_seekable.go b/packages/shared/pkg/storage/storage_cache_seekable.go index 91e25be9f6..9884b9fea4 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable.go +++ b/packages/shared/pkg/storage/storage_cache_seekable.go @@ -306,11 +306,13 @@ func (c *cachedSeekable) StoreFile(ctx context.Context, path string, opts ...Put }() cfg := CompressConfigFromOpts(ApplyPutOptions(opts)) + writeThrough := c.flags.BoolFlag(ctx, featureflags.EnableWriteThroughCacheFlag) - // write the file to the disk and the remote system at the same time. - // this opens the file twice, but the API makes it difficult to use a MultiWriter + if cfg.IsCompressionEnabled() && writeThrough { + opts = append(opts, WithFrameSink(c.frameSink(ctx))) + } - if !cfg.IsCompressionEnabled() && c.flags.BoolFlag(ctx, featureflags.EnableWriteThroughCacheFlag) { + if !cfg.IsCompressionEnabled() && writeThrough { c.goCtx(ctx, func(ctx context.Context) { ctx, span := c.tracer.Start(ctx, "write cache object from file system", trace.WithAttributes(attribute.String("path", path))) @@ -336,6 +338,19 @@ func (c *cachedSeekable) StoreFile(ctx context.Context, path string, opts ...Put return c.inner.StoreFile(ctx, path, opts...) } +// frameSink tees each compressed frame to a .frm file at its C-space offset, +// matching the layout openReaderCompressed expects. Async; best-effort. +func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { + return func(cOffset int64, data []byte) { + framePath := makeFrameFilename(c.path, Range{Offset: cOffset, Length: len(data)}) + c.goCtx(ctx, func(ctx context.Context) { + if err := c.writeToCache(ctx, cOffset, framePath, data); err != nil { + recordCacheWriteError(ctx, cacheTypeSeekable, cacheOpWriteFromFileSystem, err) + } + }) + } +} + func (c *cachedSeekable) goCtx(ctx context.Context, fn func(context.Context)) { c.wg.Go(func() { fn(context.WithoutCancel(ctx)) diff --git a/packages/shared/pkg/storage/storage_cache_seekable_test.go b/packages/shared/pkg/storage/storage_cache_seekable_test.go index e0ea301f70..4cb812afb2 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable_test.go +++ b/packages/shared/pkg/storage/storage_cache_seekable_test.go @@ -605,6 +605,32 @@ func TestCachedSeekable_OpenRangeReader(t *testing.T) { }) } +func TestCachedSeekable_FrameSinkPopulatesNFS(t *testing.T) { + t.Parallel() + + const frameSize = 64 * 1024 + data := generateSemiRandomData(3 * frameSize) + + c := &cachedSeekable{path: t.TempDir(), tracer: noopTracer} + up := &memPartUploader{} + cfg := defaultCfg(CompressionZstd, 2, frameSize) + + ft, _, err := compressStream(t.Context(), bytes.NewReader(data), cfg, up, 4, c.frameSink(t.Context())) + require.NoError(t, err) + c.wg.Wait() + + require.Equal(t, 3, ft.NumFrames()) + assembled := up.Assemble() + + for i := range ft.NumFrames() { + _, _, startC, endC := ft.FrameAt(i) + framePath := makeFrameFilename(c.path, Range{Offset: startC, Length: int(endC - startC)}) + onDisk, err := os.ReadFile(framePath) + require.NoError(t, err) + assert.Equal(t, assembled[startC:endC], onDisk) + } +} + func TestCacheWriteThroughReader(t *testing.T) { t.Parallel() diff --git a/packages/shared/pkg/storage/storage_fs.go b/packages/shared/pkg/storage/storage_fs.go index 045f61e921..25b08f7b10 100644 --- a/packages/shared/pkg/storage/storage_fs.go +++ b/packages/shared/pkg/storage/storage_fs.go @@ -131,9 +131,10 @@ func (o *fsObject) Put(_ context.Context, data []byte, _ ...PutOption) error { } func (o *fsObject) StoreFile(ctx context.Context, path string, opts ...PutOption) (*FrameTable, [32]byte, error) { - cfg := CompressConfigFromOpts(ApplyPutOptions(opts)) + putOpts := ApplyPutOptions(opts) + cfg := CompressConfigFromOpts(putOpts) if cfg.IsCompressionEnabled() { - ft, checksum, err := o.storeFileCompressed(ctx, path, cfg) + ft, checksum, err := o.storeFileCompressed(ctx, path, cfg, putOpts.FrameSink) if err == nil { logger.L().Debug(ctx, "Stored file to filesystem", zap.String("object", o.path), @@ -173,7 +174,7 @@ func (o *fsObject) StoreFile(ctx context.Context, path string, opts ...PutOption return nil, [32]byte{}, err } -func (o *fsObject) storeFileCompressed(ctx context.Context, localPath string, cfg CompressConfig) (*FrameTable, [32]byte, error) { +func (o *fsObject) storeFileCompressed(ctx context.Context, localPath string, cfg CompressConfig, sink FrameSink) (*FrameTable, [32]byte, error) { file, err := os.Open(localPath) if err != nil { return nil, [32]byte{}, fmt.Errorf("failed to open local file %s: %w", localPath, err) @@ -188,7 +189,7 @@ func (o *fsObject) storeFileCompressed(ctx context.Context, localPath string, cf uploader := &fsPartUploader{fullPath: o.path} const noConcurrencyForMemUploader = 1 - ft, checksum, err := compressStream(ctx, file, cfg, uploader, noConcurrencyForMemUploader) + ft, checksum, err := compressStream(ctx, file, cfg, uploader, noConcurrencyForMemUploader, sink) if err != nil { return nil, [32]byte{}, err } diff --git a/packages/shared/pkg/storage/storage_google.go b/packages/shared/pkg/storage/storage_google.go index 8288b671e3..0556cecb73 100644 --- a/packages/shared/pkg/storage/storage_google.go +++ b/packages/shared/pkg/storage/storage_google.go @@ -576,7 +576,7 @@ func (o *gcpObject) storeFileCompressed(ctx context.Context, localPath string, c return nil, [32]byte{}, fmt.Errorf("failed to create multipart uploader: %w", err) } - return compressStream(ctx, file, cfg, uploader, maxConcurrency) + return compressStream(ctx, file, cfg, uploader, maxConcurrency, putOpts.FrameSink) } type gcpServiceToken struct { diff --git a/packages/shared/pkg/storage/storageopts/storageopts.go b/packages/shared/pkg/storage/storageopts/storageopts.go index 526d776324..059918d654 100644 --- a/packages/shared/pkg/storage/storageopts/storageopts.go +++ b/packages/shared/pkg/storage/storageopts/storageopts.go @@ -8,15 +8,23 @@ type ObjectMetadata map[string]string const ObjectMetadataTeamID = "team_id" +// FrameSink fires once per compressed frame produced by a compressed +// StoreFile, with the frame's absolute C-space offset and bytes. +// Best-effort, non-blocking. +type FrameSink func(cOffset int64, compressed []byte) + // PutOptions holds parameters for blob/seekable writes. Compression is held // as `any` so that storage.CompressConfig (which has heavy storage-internal // dependencies) doesn't have to be moved here. Backends type-assert it back. type PutOptions struct { Metadata ObjectMetadata Compression any + FrameSink FrameSink Checksum bool } +func WithFrameSink(s FrameSink) PutOption { return func(o *PutOptions) { o.FrameSink = s } } + type PutOption func(*PutOptions) func WithMetadata(metadata ObjectMetadata) PutOption { From dbc94d9dce1f4778de7472f6c00bcc999d9312ed Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Tue, 26 May 2026 14:07:43 -0700 Subject: [PATCH 02/11] perf(storage): cap NFS write concurrency in compressed frame sink Match the uncompressed write-through path: read MaxCacheWriterConcurrencyFlag and gate concurrent writeToCache calls through a per-upload semaphore. Same operator knob, same fallback-to-1 warning, no timeout, still fire-and-forget via goCtx so the upload doesn't wait on NFS. Also widen FrameSink to take context.Context so the callback is a plain method signature rather than something that has to be wrapped in a closure just to carry ctx. --- .../shared/pkg/storage/compress_upload.go | 2 +- .../pkg/storage/storage_cache_seekable.go | 24 +++++++++++++++---- .../storage/storage_cache_seekable_test.go | 4 +++- .../pkg/storage/storageopts/storageopts.go | 12 ++++++---- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/shared/pkg/storage/compress_upload.go b/packages/shared/pkg/storage/compress_upload.go index 243f76f2b9..c4022c8fcb 100644 --- a/packages/shared/pkg/storage/compress_upload.go +++ b/packages/shared/pkg/storage/compress_upload.go @@ -147,7 +147,7 @@ func compressStream(ctx context.Context, in io.Reader, cfg CompressConfig, uploa frameSizes = append(frameSizes, FrameSize{U: int32(f.uncompressedSize), C: int32(len(f.compressed))}) compressed = append(compressed, f.compressed) if sink != nil { - sink(cOffset, f.compressed) + sink(ctx, cOffset, f.compressed) } cOffset += int64(len(f.compressed)) } diff --git a/packages/shared/pkg/storage/storage_cache_seekable.go b/packages/shared/pkg/storage/storage_cache_seekable.go index 9884b9fea4..e0850f5454 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable.go +++ b/packages/shared/pkg/storage/storage_cache_seekable.go @@ -338,12 +338,28 @@ func (c *cachedSeekable) StoreFile(ctx context.Context, path string, opts ...Put return c.inner.StoreFile(ctx, path, opts...) } -// frameSink tees each compressed frame to a .frm file at its C-space offset, -// matching the layout openReaderCompressed expects. Async; best-effort. +// frameSink builds a FrameSink that writes each compressed frame to a .frm +// file at its C-space offset, matching the layout openReaderCompressed expects. +// +// Fire-and-forget: writes are spawned via goCtx so they survive caller +// cancellation, the upload doesn't wait. Concurrent NFS writes for this +// upload are capped by MaxCacheWriterConcurrencyFlag, the same knob +// createCacheBlocksFromFile reads for the uncompressed write-through path. func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { - return func(cOffset int64, data []byte) { - framePath := makeFrameFilename(c.path, Range{Offset: cOffset, Length: len(data)}) + maxConcurrency := c.flags.IntFlag(ctx, featureflags.MaxCacheWriterConcurrencyFlag) + if maxConcurrency <= 0 { + logger.L().Warn(ctx, "max cache writer concurrency is too low, falling back to 1", + zap.Int("max_concurrency", maxConcurrency)) + maxConcurrency = 1 + } + sem := make(chan struct{}, maxConcurrency) + + return func(ctx context.Context, cOffset int64, data []byte) { c.goCtx(ctx, func(ctx context.Context) { + sem <- struct{}{} + defer func() { <-sem }() + + framePath := makeFrameFilename(c.path, Range{Offset: cOffset, Length: len(data)}) if err := c.writeToCache(ctx, cOffset, framePath, data); err != nil { recordCacheWriteError(ctx, cacheTypeSeekable, cacheOpWriteFromFileSystem, err) } diff --git a/packages/shared/pkg/storage/storage_cache_seekable_test.go b/packages/shared/pkg/storage/storage_cache_seekable_test.go index 4cb812afb2..7d51b3ac96 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable_test.go +++ b/packages/shared/pkg/storage/storage_cache_seekable_test.go @@ -611,7 +611,9 @@ func TestCachedSeekable_FrameSinkPopulatesNFS(t *testing.T) { const frameSize = 64 * 1024 data := generateSemiRandomData(3 * frameSize) - c := &cachedSeekable{path: t.TempDir(), tracer: noopTracer} + flags := NewMockFeatureFlagsClient(t) + flags.EXPECT().IntFlag(mock.Anything, mock.Anything).Return(4) + c := &cachedSeekable{path: t.TempDir(), flags: flags, tracer: noopTracer} up := &memPartUploader{} cfg := defaultCfg(CompressionZstd, 2, frameSize) diff --git a/packages/shared/pkg/storage/storageopts/storageopts.go b/packages/shared/pkg/storage/storageopts/storageopts.go index 059918d654..9432dc90f1 100644 --- a/packages/shared/pkg/storage/storageopts/storageopts.go +++ b/packages/shared/pkg/storage/storageopts/storageopts.go @@ -2,16 +2,20 @@ // separate so generated mocks can reference them without an import cycle. package storageopts -import "maps" +import ( + "context" + "maps" +) type ObjectMetadata map[string]string const ObjectMetadataTeamID = "team_id" // FrameSink fires once per compressed frame produced by a compressed -// StoreFile, with the frame's absolute C-space offset and bytes. -// Best-effort, non-blocking. -type FrameSink func(cOffset int64, compressed []byte) +// StoreFile, with the frame's absolute C-space offset and bytes. Best-effort +// and expected to return quickly — implementations should schedule any I/O +// asynchronously and bound their own concurrency. +type FrameSink func(ctx context.Context, cOffset int64, compressed []byte) // PutOptions holds parameters for blob/seekable writes. Compression is held // as `any` so that storage.CompressConfig (which has heavy storage-internal From ca9e8c55723a83cd3c3c07240ecc44f7c0fa32d1 Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Tue, 26 May 2026 14:27:39 -0700 Subject: [PATCH 03/11] feat(orch): attach upload use-case to ctx for flag targeting Upload.Run now adds CompressUseCaseContext(u.useCase) to the ctx that flows through the whole upload, so downstream flag reads can target template builds ("build") and snapshot pauses ("pause") independently. Enables turning on EnableWriteThroughCacheFlag for template builds only. Uses the same LD kind already wired through resolveCompressConfig. --- packages/orchestrator/pkg/sandbox/build_upload.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/orchestrator/pkg/sandbox/build_upload.go b/packages/orchestrator/pkg/sandbox/build_upload.go index b7a39cb2b1..942714a9ae 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload.go +++ b/packages/orchestrator/pkg/sandbox/build_upload.go @@ -75,6 +75,12 @@ func NewUpload( } func (u *Upload) Run(ctx context.Context) error { + // Attach the upload's use case ("build" for template builds, "pause" for + // snapshots) so downstream flag reads can target only one — e.g. the NFS + // write-through-on-write FF firing only for template builds. Matches the + // kind already wired into resolveCompressConfig. + ctx = featureflags.AddToContext(ctx, featureflags.CompressUseCaseContext(u.useCase)) + if !u.mem.IsCompressionEnabled() && !u.root.IsCompressionEnabled() && !u.useV4 { return u.runV3(ctx) } From 2356bae02d4b837b2305e9efacefaba4b9f535a9 Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Tue, 26 May 2026 14:42:00 -0700 Subject: [PATCH 04/11] test(storage): cover compressed StoreFile write-through to NFS TestCachedSeekable_FrameSinkPopulatesNFS exercises the sink directly but skips the StoreFile gating branch. Add two tests modeled on the existing uncompressed TestCachedFileObjectProvider_WriteFromFileSystem: - _WriteThrough: routes through StoreFile with compressed cfg + FF on, verifies every frame in the returned FrameTable lands at its expected .frm path on the temp NFS dir. Mock inner runs compressStream with the sink pulled from opts, mirroring fs/GCS backends. - _FlagOff_NoSink: asserts no FrameSink is attached when the EnableWriteThroughCacheFlag is false. --- .../storage/storage_cache_seekable_test.go | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/packages/shared/pkg/storage/storage_cache_seekable_test.go b/packages/shared/pkg/storage/storage_cache_seekable_test.go index 7d51b3ac96..bc5a8a7393 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable_test.go +++ b/packages/shared/pkg/storage/storage_cache_seekable_test.go @@ -605,6 +605,89 @@ func TestCachedSeekable_OpenRangeReader(t *testing.T) { }) } +func TestCachedSeekable_StoreFile_Compressed_WriteThrough(t *testing.T) { + t.Parallel() + + const frameSize = 64 * 1024 + cfg := defaultCfg(CompressionZstd, 2, frameSize) + data := generateSemiRandomData(3 * frameSize) + + tempDir := t.TempDir() + cacheDir := filepath.Join(tempDir, "cache") + require.NoError(t, os.MkdirAll(cacheDir, os.ModePerm)) + + srcPath := filepath.Join(tempDir, "src.bin") + require.NoError(t, os.WriteFile(srcPath, data, 0o644)) + + // Stub inner.StoreFile: open the file and run compressStream with the sink + // pulled from opts — mirrors what fs/GCS backends do for compressed puts. + up := &memPartUploader{} + var capturedFT *FrameTable + inner := NewMockSeekable(t) + inner.EXPECT(). + StoreFile(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(ctx context.Context, path string, opts ...PutOption) (*FrameTable, [32]byte, error) { + po := ApplyPutOptions(opts) + require.NotNil(t, po.FrameSink, "cachedSeekable must attach a FrameSink for compressed+writeThrough") + + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + ft, sum, err := compressStream(ctx, f, cfg, up, 4, po.FrameSink) + capturedFT = ft + return ft, sum, err + }) + + flags := NewMockFeatureFlagsClient(t) + flags.EXPECT().BoolFlag(mock.Anything, mock.Anything).Return(true) + flags.EXPECT().IntFlag(mock.Anything, mock.Anything).Return(4) + + c := cachedSeekable{path: cacheDir, inner: inner, chunkSize: frameSize, flags: flags, tracer: noopTracer} + + _, _, err := c.StoreFile(t.Context(), srcPath, WithCompressConfig(cfg)) + require.NoError(t, err) + + c.wg.Wait() + + require.Equal(t, 3, capturedFT.NumFrames()) + assembled := up.Assemble() + for i := range capturedFT.NumFrames() { + _, _, startC, endC := capturedFT.FrameAt(i) + framePath := makeFrameFilename(c.path, Range{Offset: startC, Length: int(endC - startC)}) + onDisk, err := os.ReadFile(framePath) + require.NoError(t, err) + assert.Equal(t, assembled[startC:endC], onDisk) + } +} + +func TestCachedSeekable_StoreFile_Compressed_FlagOff_NoSink(t *testing.T) { + t.Parallel() + + cfg := defaultCfg(CompressionZstd, 1, 64*1024) + tempDir := t.TempDir() + srcPath := filepath.Join(tempDir, "src.bin") + require.NoError(t, os.WriteFile(srcPath, []byte("x"), 0o644)) + + inner := NewMockSeekable(t) + inner.EXPECT(). + StoreFile(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(_ context.Context, _ string, opts ...PutOption) (*FrameTable, [32]byte, error) { + po := ApplyPutOptions(opts) + assert.Nil(t, po.FrameSink, "FrameSink must NOT be attached when write-through flag is off") + return nil, [32]byte{}, nil + }) + + flags := NewMockFeatureFlagsClient(t) + flags.EXPECT().BoolFlag(mock.Anything, mock.Anything).Return(false) + + c := cachedSeekable{path: t.TempDir(), inner: inner, chunkSize: 64 * 1024, flags: flags, tracer: noopTracer} + + _, _, err := c.StoreFile(t.Context(), srcPath, WithCompressConfig(cfg)) + require.NoError(t, err) + c.wg.Wait() +} + func TestCachedSeekable_FrameSinkPopulatesNFS(t *testing.T) { t.Parallel() From 5b3246d4fea2bdee13bc11894a1446ede2a0311f Mon Sep 17 00:00:00 2001 From: Lev Brouk Date: Tue, 26 May 2026 14:50:39 -0700 Subject: [PATCH 05/11] PR feedback --- .../shared/pkg/storage/storage_cache_seekable.go | 12 +++++++++++- .../pkg/storage/storage_cache_seekable_test.go | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/shared/pkg/storage/storage_cache_seekable.go b/packages/shared/pkg/storage/storage_cache_seekable.go index e0850f5454..e8297ef7eb 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable.go +++ b/packages/shared/pkg/storage/storage_cache_seekable.go @@ -355,8 +355,18 @@ func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { sem := make(chan struct{}, maxConcurrency) return func(ctx context.Context, cOffset int64, data []byte) { + // Acquire BEFORE spawning so the goroutine count is bounded — otherwise + // every frame still spawns a goroutine that just blocks on sem, and + // large uploads with a low concurrency cap pile up thousands of live + // goroutines in wg. Pushes backpressure into the sink caller (the + // upload loop in compressStream), which is the right place to slow. + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } + c.goCtx(ctx, func(ctx context.Context) { - sem <- struct{}{} defer func() { <-sem }() framePath := makeFrameFilename(c.path, Range{Offset: cOffset, Length: len(data)}) diff --git a/packages/shared/pkg/storage/storage_cache_seekable_test.go b/packages/shared/pkg/storage/storage_cache_seekable_test.go index bc5a8a7393..03709943da 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable_test.go +++ b/packages/shared/pkg/storage/storage_cache_seekable_test.go @@ -636,6 +636,7 @@ func TestCachedSeekable_StoreFile_Compressed_WriteThrough(t *testing.T) { ft, sum, err := compressStream(ctx, f, cfg, up, 4, po.FrameSink) capturedFT = ft + return ft, sum, err }) @@ -675,6 +676,7 @@ func TestCachedSeekable_StoreFile_Compressed_FlagOff_NoSink(t *testing.T) { RunAndReturn(func(_ context.Context, _ string, opts ...PutOption) (*FrameTable, [32]byte, error) { po := ApplyPutOptions(opts) assert.Nil(t, po.FrameSink, "FrameSink must NOT be attached when write-through flag is off") + return nil, [32]byte{}, nil }) From 2f05fd051cf384c49971b1433f992b1cc0371ff6 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Thu, 28 May 2026 18:46:20 -0700 Subject: [PATCH 06/11] docs(storage): document goCtx WithoutCancel rationale --- packages/shared/pkg/storage/storage_cache_seekable.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/shared/pkg/storage/storage_cache_seekable.go b/packages/shared/pkg/storage/storage_cache_seekable.go index e8297ef7eb..3aaec3391e 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable.go +++ b/packages/shared/pkg/storage/storage_cache_seekable.go @@ -377,6 +377,11 @@ func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { } } +// goCtx runs fn on c.wg, detached from the caller's cancellation via +// WithoutCancel so an in-flight cache write isn't aborted when the upload's +// context is cancelled (it still inherits values for tracing/flags). Tracked +// by c.wg so it can be awaited; pre-existing pattern from the uncompressed +// write-through path. func (c *cachedSeekable) goCtx(ctx context.Context, fn func(context.Context)) { c.wg.Go(func() { fn(context.WithoutCancel(ctx)) From d87dd0d69a61cfe83b8b7003736d11f80f2e5de9 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Thu, 28 May 2026 18:48:14 -0700 Subject: [PATCH 07/11] refactor(storage): trim comments and redundant write-through tests --- .../orchestrator/pkg/sandbox/build_upload.go | 5 +- .../pkg/storage/storage_cache_seekable.go | 24 +++----- .../storage/storage_cache_seekable_test.go | 56 ------------------- .../pkg/storage/storageopts/storageopts.go | 6 +- 4 files changed, 10 insertions(+), 81 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/build_upload.go b/packages/orchestrator/pkg/sandbox/build_upload.go index 942714a9ae..5ad332dc57 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload.go +++ b/packages/orchestrator/pkg/sandbox/build_upload.go @@ -75,10 +75,7 @@ func NewUpload( } func (u *Upload) Run(ctx context.Context) error { - // Attach the upload's use case ("build" for template builds, "pause" for - // snapshots) so downstream flag reads can target only one — e.g. the NFS - // write-through-on-write FF firing only for template builds. Matches the - // kind already wired into resolveCompressConfig. + // Attach the upload use case so flag reads can target it (e.g. write-through only for builds). ctx = featureflags.AddToContext(ctx, featureflags.CompressUseCaseContext(u.useCase)) if !u.mem.IsCompressionEnabled() && !u.root.IsCompressionEnabled() && !u.useV4 { diff --git a/packages/shared/pkg/storage/storage_cache_seekable.go b/packages/shared/pkg/storage/storage_cache_seekable.go index 3aaec3391e..a3b20c1118 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable.go +++ b/packages/shared/pkg/storage/storage_cache_seekable.go @@ -338,13 +338,9 @@ func (c *cachedSeekable) StoreFile(ctx context.Context, path string, opts ...Put return c.inner.StoreFile(ctx, path, opts...) } -// frameSink builds a FrameSink that writes each compressed frame to a .frm -// file at its C-space offset, matching the layout openReaderCompressed expects. -// -// Fire-and-forget: writes are spawned via goCtx so they survive caller -// cancellation, the upload doesn't wait. Concurrent NFS writes for this -// upload are capped by MaxCacheWriterConcurrencyFlag, the same knob -// createCacheBlocksFromFile reads for the uncompressed write-through path. +// frameSink writes each compressed frame to a .frm file at its C-space offset, +// the layout openReaderCompressed expects. Writes are async (goCtx) and capped +// by MaxCacheWriterConcurrencyFlag. func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { maxConcurrency := c.flags.IntFlag(ctx, featureflags.MaxCacheWriterConcurrencyFlag) if maxConcurrency <= 0 { @@ -355,11 +351,8 @@ func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { sem := make(chan struct{}, maxConcurrency) return func(ctx context.Context, cOffset int64, data []byte) { - // Acquire BEFORE spawning so the goroutine count is bounded — otherwise - // every frame still spawns a goroutine that just blocks on sem, and - // large uploads with a low concurrency cap pile up thousands of live - // goroutines in wg. Pushes backpressure into the sink caller (the - // upload loop in compressStream), which is the right place to slow. + // Acquire before spawning so goroutine count stays bounded; this also + // backpressures the upload loop in compressStream. select { case sem <- struct{}{}: case <-ctx.Done(): @@ -377,11 +370,8 @@ func (c *cachedSeekable) frameSink(ctx context.Context) FrameSink { } } -// goCtx runs fn on c.wg, detached from the caller's cancellation via -// WithoutCancel so an in-flight cache write isn't aborted when the upload's -// context is cancelled (it still inherits values for tracing/flags). Tracked -// by c.wg so it can be awaited; pre-existing pattern from the uncompressed -// write-through path. +// goCtx runs fn on c.wg with WithoutCancel so an in-flight cache write isn't +// aborted when the upload's context is cancelled. func (c *cachedSeekable) goCtx(ctx context.Context, fn func(context.Context)) { c.wg.Go(func() { fn(context.WithoutCancel(ctx)) diff --git a/packages/shared/pkg/storage/storage_cache_seekable_test.go b/packages/shared/pkg/storage/storage_cache_seekable_test.go index 03709943da..d989eb084f 100644 --- a/packages/shared/pkg/storage/storage_cache_seekable_test.go +++ b/packages/shared/pkg/storage/storage_cache_seekable_test.go @@ -662,62 +662,6 @@ func TestCachedSeekable_StoreFile_Compressed_WriteThrough(t *testing.T) { } } -func TestCachedSeekable_StoreFile_Compressed_FlagOff_NoSink(t *testing.T) { - t.Parallel() - - cfg := defaultCfg(CompressionZstd, 1, 64*1024) - tempDir := t.TempDir() - srcPath := filepath.Join(tempDir, "src.bin") - require.NoError(t, os.WriteFile(srcPath, []byte("x"), 0o644)) - - inner := NewMockSeekable(t) - inner.EXPECT(). - StoreFile(mock.Anything, mock.Anything, mock.Anything). - RunAndReturn(func(_ context.Context, _ string, opts ...PutOption) (*FrameTable, [32]byte, error) { - po := ApplyPutOptions(opts) - assert.Nil(t, po.FrameSink, "FrameSink must NOT be attached when write-through flag is off") - - return nil, [32]byte{}, nil - }) - - flags := NewMockFeatureFlagsClient(t) - flags.EXPECT().BoolFlag(mock.Anything, mock.Anything).Return(false) - - c := cachedSeekable{path: t.TempDir(), inner: inner, chunkSize: 64 * 1024, flags: flags, tracer: noopTracer} - - _, _, err := c.StoreFile(t.Context(), srcPath, WithCompressConfig(cfg)) - require.NoError(t, err) - c.wg.Wait() -} - -func TestCachedSeekable_FrameSinkPopulatesNFS(t *testing.T) { - t.Parallel() - - const frameSize = 64 * 1024 - data := generateSemiRandomData(3 * frameSize) - - flags := NewMockFeatureFlagsClient(t) - flags.EXPECT().IntFlag(mock.Anything, mock.Anything).Return(4) - c := &cachedSeekable{path: t.TempDir(), flags: flags, tracer: noopTracer} - up := &memPartUploader{} - cfg := defaultCfg(CompressionZstd, 2, frameSize) - - ft, _, err := compressStream(t.Context(), bytes.NewReader(data), cfg, up, 4, c.frameSink(t.Context())) - require.NoError(t, err) - c.wg.Wait() - - require.Equal(t, 3, ft.NumFrames()) - assembled := up.Assemble() - - for i := range ft.NumFrames() { - _, _, startC, endC := ft.FrameAt(i) - framePath := makeFrameFilename(c.path, Range{Offset: startC, Length: int(endC - startC)}) - onDisk, err := os.ReadFile(framePath) - require.NoError(t, err) - assert.Equal(t, assembled[startC:endC], onDisk) - } -} - func TestCacheWriteThroughReader(t *testing.T) { t.Parallel() diff --git a/packages/shared/pkg/storage/storageopts/storageopts.go b/packages/shared/pkg/storage/storageopts/storageopts.go index 9432dc90f1..99b02e8cc2 100644 --- a/packages/shared/pkg/storage/storageopts/storageopts.go +++ b/packages/shared/pkg/storage/storageopts/storageopts.go @@ -11,10 +11,8 @@ type ObjectMetadata map[string]string const ObjectMetadataTeamID = "team_id" -// FrameSink fires once per compressed frame produced by a compressed -// StoreFile, with the frame's absolute C-space offset and bytes. Best-effort -// and expected to return quickly — implementations should schedule any I/O -// asynchronously and bound their own concurrency. +// FrameSink fires once per compressed frame with its absolute C-space offset. +// Best-effort; implementations should return quickly and bound their own I/O. type FrameSink func(ctx context.Context, cOffset int64, compressed []byte) // PutOptions holds parameters for blob/seekable writes. Compression is held From 0f73db67701234aaea479ec60f462741e8cc89be Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Fri, 29 May 2026 01:26:50 -0700 Subject: [PATCH 08/11] chore(orchestrator): add cross-artifact dedup benchmarks --- .../cmd/sample-dedup-gains/main.go | 824 ++++++++++++++++++ .../cmd/synthetic-dedup-corpus/main.go | 278 ++++++ .../orchestrator/docs/cross-artifact-dedup.md | 162 ++++ 3 files changed, 1264 insertions(+) create mode 100644 packages/orchestrator/cmd/sample-dedup-gains/main.go create mode 100644 packages/orchestrator/cmd/synthetic-dedup-corpus/main.go create mode 100644 packages/orchestrator/docs/cross-artifact-dedup.md diff --git a/packages/orchestrator/cmd/sample-dedup-gains/main.go b/packages/orchestrator/cmd/sample-dedup-gains/main.go new file mode 100644 index 0000000000..154a46a62e --- /dev/null +++ b/packages/orchestrator/cmd/sample-dedup-gains/main.go @@ -0,0 +1,824 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "errors" + "flag" + "fmt" + "io" + "log" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "go.opentelemetry.io/otel/metric/noop" + + "github.com/e2b-dev/infra/packages/orchestrator/cmd/internal/cmdutil" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/cfg" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block" + blockmetrics "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block/metrics" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build" + sboxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +const ( + artifactMemfile = "memfile" + artifactRootfs = "rootfs" +) + +type buildPair struct { + BuildID string + ParentBuildID string + SiblingBuildID string + SiblingMemfileBuildID string + SiblingRootfsBuildID string + Family string +} + +type pool struct { + Name string + TargetArtifact string + CandidateArtifact string + CandidateBuild func(buildPair) string + Positional bool + ValidationOnly bool +} + +type rowResult struct { + BuildID string + ParentBuildID string + Family string + TargetArtifact string + Pool string + CandidateBuildID string + CandidateArtifact string + ValidationOnly bool + Positional bool + TargetPages int64 + SampledTargetPages int64 + CandidatePages int64 + IndexedCandidatePages int64 + Hits int64 + ZeroPages int64 + EligibleBytes int64 + SavingsBytes int64 + SavingsRatio float64 + FrameSizeBytes int64 + FrameTargetFrames int64 + FrameHits int64 + FrameSavingsBytes int64 + FrameSavingsRatio float64 + IndexMS int64 + CompareMS int64 + Error string +} + +type summary struct { + Rows int64 + EligibleBytes int64 + SavingsBytes int64 + Hits int64 + TargetPages int64 + FrameTargetFrames int64 + FrameHits int64 + Errors int64 + Ratios []float64 +} + +type frameRef struct { + off int64 + length int64 +} + +type analyzer struct { + ctx context.Context + store storage.StorageProvider + diffStore *build.DiffStore + cacheDir string + metrics blockmetrics.Metrics + devices map[string]*sboxtemplate.Storage +} + +func main() { + storagePath := flag.String("storage", ".local-build", "storage: local path or gs://bucket") + buildsFile := flag.String("builds-file", "", "CSV with build_id,parent_build_id and optional sibling columns") + buildID := flag.String("build", "", "single current build ID") + parentBuildID := flag.String("parent-build", "", "single parent build ID") + artifacts := flag.String("artifacts", "both", "memfile, rootfs, or both") + maxTargetPages := flag.Int("max-target-pages", 50000, "target pages to sample per artifact; 0 scans all") + maxCandidatePages := flag.Int("max-candidate-pages", 100000, "candidate pages to index per pool; 0 scans all") + frameSize := flag.Int("frame-size", 2<<20, "compression-frame size for whole-frame dedup estimate") + seed := flag.Int64("seed", 1, "sampling seed") + outputPath := flag.String("csv-path", "", "write detailed CSV here; default stdout") + includeValidation := flag.Bool("include-validation", true, "include validation-only pools") + + flag.Parse() + cmdutil.SuppressNoisyLogs() + + if *buildsFile == "" && (*buildID == "" || *parentBuildID == "") { + log.Fatal("provide -builds-file or both -build and -parent-build") + } + + pairs, err := loadPairs(*buildsFile, *buildID, *parentBuildID) + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + if err := cmdutil.SetupStorage(*storagePath); err != nil { + log.Fatal(err) + } + + ff, err := featureflags.NewClientWithLogLevel(ldlog.Error) + if err != nil { + log.Fatalf("feature flags: %s", err) + } + metrics, err := blockmetrics.NewMetrics(noop.NewMeterProvider()) + if err != nil { + log.Fatalf("metrics: %s", err) + } + persistence, err := storage.GetStorageProvider(ctx, storage.TemplateStorageConfig) + if err != nil { + log.Fatalf("storage: %s", err) + } + cacheDir, err := os.MkdirTemp("", "sample-dedup-gains-*") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(cacheDir) + + diffStore, err := build.NewDiffStore(cfg.Config{}, ff, cacheDir, time.Hour, time.Second) + if err != nil { + log.Fatal(err) + } + defer diffStore.Close() + + out := io.Writer(os.Stdout) + var outFile *os.File + if *outputPath != "" { + if err := os.MkdirAll(filepath.Dir(*outputPath), 0o755); err != nil { + log.Fatal(err) + } + outFile, err = os.Create(*outputPath) + if err != nil { + log.Fatal(err) + } + defer outFile.Close() + out = outFile + } + + a := &analyzer{ + ctx: ctx, + store: persistence, + diffStore: diffStore, + cacheDir: cacheDir, + metrics: metrics, + devices: make(map[string]*sboxtemplate.Storage), + } + defer a.close() + + writer := csv.NewWriter(out) + if err := writer.Write(resultHeader()); err != nil { + log.Fatal(err) + } + + selectedArtifacts, err := parseArtifacts(*artifacts) + if err != nil { + log.Fatal(err) + } + + summaries := make(map[string]*summary) + for i, pair := range pairs { + for _, artifact := range selectedArtifacts { + results := a.analyzeArtifact(pair, artifact, *maxTargetPages, *maxCandidatePages, int64(*frameSize), *seed+int64(i), *includeValidation) + for _, result := range results { + if err := writer.Write(result.Record()); err != nil { + log.Fatal(err) + } + addSummary(summaries, result) + } + } + } + writer.Flush() + if err := writer.Error(); err != nil { + log.Fatal(err) + } + + printSummary(os.Stderr, summaries) +} + +func (a *analyzer) analyzeArtifact(pair buildPair, artifact string, maxTargetPages, maxCandidatePages int, frameSize int64, seed int64, includeValidation bool) []rowResult { + target, err := a.device(pair.BuildID, artifact) + if err != nil { + return []rowResult{errorResult(pair, artifact, "open_target", err)} + } + + targetOffsets, targetPages := sampledSelfPages(target.Header(), pair.BuildID, maxTargetPages, seed) + results := make([]rowResult, 0) + for _, p := range candidatePools(pair, artifact, includeValidation) { + candidateBuildID := p.CandidateBuild(pair) + base := rowResult{ + BuildID: pair.BuildID, + ParentBuildID: pair.ParentBuildID, + Family: pair.Family, + TargetArtifact: artifact, + Pool: p.Name, + CandidateBuildID: candidateBuildID, + CandidateArtifact: p.CandidateArtifact, + ValidationOnly: p.ValidationOnly, + Positional: p.Positional, + FrameSizeBytes: frameSize, + TargetPages: targetPages, + SampledTargetPages: int64(len(targetOffsets)), + } + if candidateBuildID == "" { + base.Error = "candidate build missing" + results = append(results, base) + continue + } + + candidate, err := a.device(candidateBuildID, p.CandidateArtifact) + if err != nil { + base.Error = err.Error() + results = append(results, base) + continue + } + if p.Positional { + results = append(results, a.comparePositional(base, target, candidate, targetOffsets)) + continue + } + results = append(results, a.compareIndexed(base, target, candidate, targetOffsets, maxCandidatePages, seed)) + } + + return results +} + +func (a *analyzer) comparePositional(base rowResult, target, candidate block.ReadonlyDevice, targetOffsets []int64) rowResult { + start := time.Now() + var hits, zeroes int64 + targetPage := make([]byte, header.PageSize) + candidatePage := make([]byte, header.PageSize) + for _, off := range targetOffsets { + if _, err := target.ReadAt(a.ctx, targetPage, off); err != nil { + base.Error = fmt.Sprintf("read target at %d: %s", off, err) + break + } + if header.IsZero(targetPage) { + zeroes++ + continue + } + if _, err := candidate.ReadAt(a.ctx, candidatePage, off); err != nil { + continue + } + if bytes.Equal(targetPage, candidatePage) { + hits++ + } + } + base.Hits = hits + base.ZeroPages = zeroes + base.CompareMS = time.Since(start).Milliseconds() + base.CandidatePages = base.TargetPages + base.IndexedCandidatePages = base.SampledTargetPages + base.measureFramePositional(a.ctx, target, candidate, targetOffsets) + base.finish() + + return base +} + +func (a *analyzer) compareIndexed(base rowResult, target, candidate block.ReadonlyDevice, targetOffsets []int64, maxCandidatePages int, seed int64) rowResult { + indexStart := time.Now() + candidateOffsets, candidatePages := sampledAllPages(candidate.Header(), maxCandidatePages, seed) + base.CandidatePages = candidatePages + + index := make(map[[32]byte][]int64, len(candidateOffsets)) + page := make([]byte, header.PageSize) + for _, off := range candidateOffsets { + if _, err := candidate.ReadAt(a.ctx, page, off); err != nil { + continue + } + if header.IsZero(page) { + continue + } + sum := sha256.Sum256(page) + index[sum] = append(index[sum], off) + base.IndexedCandidatePages++ + } + base.IndexMS = time.Since(indexStart).Milliseconds() + + compareStart := time.Now() + targetPage := make([]byte, header.PageSize) + candidatePage := make([]byte, header.PageSize) + var hits, zeroes int64 + for _, off := range targetOffsets { + if _, err := target.ReadAt(a.ctx, targetPage, off); err != nil { + base.Error = fmt.Sprintf("read target at %d: %s", off, err) + break + } + if header.IsZero(targetPage) { + zeroes++ + continue + } + for _, candidateOff := range index[sha256.Sum256(targetPage)] { + if _, err := candidate.ReadAt(a.ctx, candidatePage, candidateOff); err != nil { + continue + } + if bytes.Equal(targetPage, candidatePage) { + hits++ + break + } + } + } + base.Hits = hits + base.ZeroPages = zeroes + base.CompareMS = time.Since(compareStart).Milliseconds() + base.measureFrameIndexed(a.ctx, target, candidate, targetOffsets) + base.finish() + + return base +} + +func (r *rowResult) finish() { + if r.SampledTargetPages == 0 { + return + } + r.EligibleBytes = r.TargetPages * header.PageSize + r.SavingsBytes = int64(float64(r.Hits) / float64(r.SampledTargetPages) * float64(r.EligibleBytes)) + if r.EligibleBytes > 0 { + r.SavingsRatio = float64(r.SavingsBytes) / float64(r.EligibleBytes) + } + if r.FrameTargetFrames > 0 { + r.FrameSavingsBytes = r.FrameHits * r.FrameSizeBytes + r.FrameSavingsRatio = float64(r.FrameHits) / float64(r.FrameTargetFrames) + } +} + +func (r *rowResult) measureFramePositional(ctx context.Context, target, candidate block.ReadonlyDevice, targetOffsets []int64) { + frames := targetFrames(targetOffsets, r.FrameSizeBytes) + r.FrameTargetFrames = int64(len(frames)) + if r.FrameSizeBytes <= 0 { + return + } + + targetSize, err := target.Size(ctx) + if err != nil { + return + } + candidateSize, err := candidate.Size(ctx) + if err != nil { + return + } + for _, off := range frames { + length := min(r.FrameSizeBytes, targetSize-off) + if length <= 0 || off+length > candidateSize { + continue + } + targetFrame, err := readRange(ctx, target, off, length) + if err != nil { + continue + } + candidateFrame, err := readRange(ctx, candidate, off, length) + if err != nil { + continue + } + if bytes.Equal(targetFrame, candidateFrame) { + r.FrameHits++ + } + } +} + +func (r *rowResult) measureFrameIndexed(ctx context.Context, target, candidate block.ReadonlyDevice, targetOffsets []int64) { + frames := targetFrames(targetOffsets, r.FrameSizeBytes) + r.FrameTargetFrames = int64(len(frames)) + if r.FrameSizeBytes <= 0 { + return + } + + candidateSize, err := candidate.Size(ctx) + if err != nil { + return + } + index := make(map[[32]byte][]frameRef) + for off := int64(0); off < candidateSize; off += r.FrameSizeBytes { + length := min(r.FrameSizeBytes, candidateSize-off) + frame, err := readRange(ctx, candidate, off, length) + if err != nil || header.IsZero(frame) { + continue + } + index[sha256.Sum256(frame)] = append(index[sha256.Sum256(frame)], frameRef{off: off, length: length}) + } + + targetSize, err := target.Size(ctx) + if err != nil { + return + } + for _, off := range frames { + length := min(r.FrameSizeBytes, targetSize-off) + if length <= 0 { + continue + } + targetFrame, err := readRange(ctx, target, off, length) + if err != nil || header.IsZero(targetFrame) { + continue + } + sum := sha256.Sum256(targetFrame) + for _, ref := range index[sum] { + if ref.length != length { + continue + } + candidateFrame, err := readRange(ctx, candidate, ref.off, ref.length) + if err != nil { + continue + } + if bytes.Equal(targetFrame, candidateFrame) { + r.FrameHits++ + break + } + } + } +} + +func targetFrames(targetOffsets []int64, frameSize int64) []int64 { + if frameSize <= 0 { + return nil + } + seen := make(map[int64]struct{}) + frames := make([]int64, 0) + for _, off := range targetOffsets { + frameOff := (off / frameSize) * frameSize + if _, ok := seen[frameOff]; ok { + continue + } + seen[frameOff] = struct{}{} + frames = append(frames, frameOff) + } + + return frames +} + +func readRange(ctx context.Context, d block.ReadonlyDevice, off, length int64) ([]byte, error) { + buf := make([]byte, length) + _, err := d.ReadAt(ctx, buf, off) + + return buf, err +} + +func (a *analyzer) device(buildID, artifact string) (*sboxtemplate.Storage, error) { + key := buildID + "/" + artifact + if d, ok := a.devices[key]; ok { + return d, nil + } + fileType, err := diffType(artifact) + if err != nil { + return nil, err + } + d, err := sboxtemplate.NewStorage(a.ctx, a.diffStore, buildID, fileType, nil, a.store, a.metrics) + if err != nil { + return nil, err + } + a.devices[key] = d + + return d, nil +} + +func (a *analyzer) close() { + for _, d := range a.devices { + _ = d.Close() + } +} + +func sampledSelfPages(h *header.Header, buildID string, maxPages int, seed int64) ([]int64, int64) { + id, err := uuid.Parse(buildID) + if err != nil || h == nil { + return nil, 0 + } + var total int64 + return sampleOffsets(h.Mapping, func(m header.BuildMap) bool { + return m.BuildId == id + }, maxPages, seed, &total), total +} + +func sampledAllPages(h *header.Header, maxPages int, seed int64) ([]int64, int64) { + if h == nil || h.Metadata == nil { + return nil, 0 + } + total := int64(h.Metadata.Size / header.PageSize) + return sampleOffsets([]header.BuildMap{{ + Offset: 0, + Length: uint64(total * header.PageSize), + }}, func(header.BuildMap) bool { return true }, maxPages, seed, nil), total +} + +func sampleOffsets(mappings []header.BuildMap, include func(header.BuildMap) bool, maxPages int, seed int64, totalOut *int64) []int64 { + rng := rand.New(rand.NewSource(seed)) + var sampled []int64 + var seen int64 + for _, m := range mappings { + if !include(m) { + continue + } + start := alignUp(int64(m.Offset), header.PageSize) + end := alignDown(int64(m.Offset+m.Length), header.PageSize) + for off := start; off < end; off += header.PageSize { + seen++ + if maxPages <= 0 { + sampled = append(sampled, off) + continue + } + if int64(len(sampled)) < int64(maxPages) { + sampled = append(sampled, off) + continue + } + j := rng.Int63n(seen) + if j < int64(maxPages) { + sampled[j] = off + } + } + } + if totalOut != nil { + *totalOut = seen + } + + return sampled +} + +func alignUp(v, by int64) int64 { + if v%by == 0 { + return v + } + return v + by - v%by +} + +func alignDown(v, by int64) int64 { + return v - v%by +} + +func candidatePools(pair buildPair, targetArtifact string, includeValidation bool) []pool { + siblingFor := func(artifact string) func(buildPair) string { + return func(p buildPair) string { + if artifact == artifactMemfile && p.SiblingMemfileBuildID != "" { + return p.SiblingMemfileBuildID + } + if artifact == artifactRootfs && p.SiblingRootfsBuildID != "" { + return p.SiblingRootfsBuildID + } + return p.SiblingBuildID + } + } + parent := func(p buildPair) string { return p.ParentBuildID } + current := func(p buildPair) string { return p.BuildID } + + var pools []pool + switch targetArtifact { + case artifactMemfile: + pools = append(pools, + pool{Name: "memfile_parent_memfile_positional", TargetArtifact: artifactMemfile, CandidateArtifact: artifactMemfile, CandidateBuild: parent, Positional: true}, + pool{Name: "memfile_current_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: current}, + pool{Name: "memfile_sibling_memfile", TargetArtifact: artifactMemfile, CandidateArtifact: artifactMemfile, CandidateBuild: siblingFor(artifactMemfile)}, + ) + if includeValidation { + pools = append(pools, + pool{Name: "memfile_parent_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: parent, ValidationOnly: true}, + pool{Name: "memfile_sibling_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: siblingFor(artifactRootfs), ValidationOnly: true}, + ) + } + case artifactRootfs: + pools = append(pools, + pool{Name: "rootfs_parent_rootfs_positional", TargetArtifact: artifactRootfs, CandidateArtifact: artifactRootfs, CandidateBuild: parent, Positional: true}, + pool{Name: "rootfs_parent_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: parent}, + pool{Name: "rootfs_sibling_rootfs", TargetArtifact: artifactRootfs, CandidateArtifact: artifactRootfs, CandidateBuild: siblingFor(artifactRootfs)}, + ) + if includeValidation { + pools = append(pools, + pool{Name: "rootfs_current_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: current, ValidationOnly: true}, + pool{Name: "rootfs_sibling_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: siblingFor(artifactMemfile), ValidationOnly: true}, + ) + } + } + + return pools +} + +func parseArtifacts(value string) ([]string, error) { + switch strings.ToLower(value) { + case "both": + return []string{artifactMemfile, artifactRootfs}, nil + case artifactMemfile: + return []string{artifactMemfile}, nil + case artifactRootfs: + return []string{artifactRootfs}, nil + default: + return nil, fmt.Errorf("unknown -artifacts value %q", value) + } +} + +func diffType(artifact string) (build.DiffType, error) { + switch artifact { + case artifactMemfile: + return build.Memfile, nil + case artifactRootfs: + return build.Rootfs, nil + default: + return "", fmt.Errorf("unknown artifact %q", artifact) + } +} + +func loadPairs(path, buildID, parentBuildID string) ([]buildPair, error) { + if path == "" { + return []buildPair{{BuildID: buildID, ParentBuildID: parentBuildID}}, nil + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + r.TrimLeadingSpace = true + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return nil, errors.New("builds file is empty") + } + + start := 0 + index := map[string]int{} + if looksLikeHeader(records[0]) { + start = 1 + for i, name := range records[0] { + index[strings.TrimSpace(name)] = i + } + } + + pairs := make([]buildPair, 0, len(records)-start) + for _, record := range records[start:] { + pair := buildPair{ + BuildID: getCSV(record, index, "build_id", 0), + ParentBuildID: getCSV(record, index, "parent_build_id", 1), + SiblingBuildID: getCSV(record, index, "sibling_build_id", 2), + SiblingMemfileBuildID: getCSV(record, index, "sibling_memfile_build_id", -1), + SiblingRootfsBuildID: getCSV(record, index, "sibling_rootfs_build_id", -1), + Family: getCSV(record, index, "family", -1), + } + if pair.BuildID == "" || pair.ParentBuildID == "" { + return nil, fmt.Errorf("build_id and parent_build_id are required in record %q", strings.Join(record, ",")) + } + pairs = append(pairs, pair) + } + + return pairs, nil +} + +func looksLikeHeader(record []string) bool { + for _, field := range record { + if strings.Contains(strings.ToLower(field), "build") { + return true + } + } + return false +} + +func getCSV(record []string, index map[string]int, name string, fallback int) string { + if i, ok := index[name]; ok && i >= 0 && i < len(record) { + return strings.TrimSpace(record[i]) + } + if fallback >= 0 && fallback < len(record) { + return strings.TrimSpace(record[fallback]) + } + return "" +} + +func resultHeader() []string { + return []string{ + "build_id", "parent_build_id", "family", "target_artifact", "pool", + "candidate_build_id", "candidate_artifact", "validation_only", "positional", + "target_pages", "sampled_target_pages", "candidate_pages", "indexed_candidate_pages", + "hits", "zero_pages", "eligible_bytes", "savings_bytes", "savings_ratio", + "frame_size_bytes", "frame_target_frames", "frame_hits", "frame_savings_bytes", "frame_savings_ratio", + "index_ms", "compare_ms", "error", + } +} + +func (r rowResult) Record() []string { + return []string{ + r.BuildID, r.ParentBuildID, r.Family, r.TargetArtifact, r.Pool, + r.CandidateBuildID, r.CandidateArtifact, + strconv.FormatBool(r.ValidationOnly), strconv.FormatBool(r.Positional), + strconv.FormatInt(r.TargetPages, 10), + strconv.FormatInt(r.SampledTargetPages, 10), + strconv.FormatInt(r.CandidatePages, 10), + strconv.FormatInt(r.IndexedCandidatePages, 10), + strconv.FormatInt(r.Hits, 10), + strconv.FormatInt(r.ZeroPages, 10), + strconv.FormatInt(r.EligibleBytes, 10), + strconv.FormatInt(r.SavingsBytes, 10), + strconv.FormatFloat(r.SavingsRatio, 'f', 6, 64), + strconv.FormatInt(r.FrameSizeBytes, 10), + strconv.FormatInt(r.FrameTargetFrames, 10), + strconv.FormatInt(r.FrameHits, 10), + strconv.FormatInt(r.FrameSavingsBytes, 10), + strconv.FormatFloat(r.FrameSavingsRatio, 'f', 6, 64), + strconv.FormatInt(r.IndexMS, 10), + strconv.FormatInt(r.CompareMS, 10), + r.Error, + } +} + +func errorResult(pair buildPair, artifact, pool string, err error) rowResult { + return rowResult{ + BuildID: pair.BuildID, + ParentBuildID: pair.ParentBuildID, + Family: pair.Family, + TargetArtifact: artifact, + Pool: pool, + Error: err.Error(), + } +} + +func addSummary(summaries map[string]*summary, r rowResult) { + s := summaries[r.Pool] + if s == nil { + s = &summary{} + summaries[r.Pool] = s + } + s.Rows++ + s.EligibleBytes += r.EligibleBytes + s.SavingsBytes += r.SavingsBytes + s.Hits += r.Hits + s.TargetPages += r.SampledTargetPages + s.FrameTargetFrames += r.FrameTargetFrames + s.FrameHits += r.FrameHits + if r.Error != "" { + s.Errors++ + } + if r.Error == "" && r.EligibleBytes > 0 { + s.Ratios = append(s.Ratios, r.SavingsRatio) + } +} + +func printSummary(w io.Writer, summaries map[string]*summary) { + fmt.Fprintln(w, "\nSUMMARY") + fmt.Fprintln(w, "pool,rows,errors,sampled_pages,hits,eligible_bytes,savings_bytes,weighted_savings_ratio,frame_target_frames,frame_hits,frame_savings_ratio,mean_row_ratio,ci95_low,ci95_high") + for pool, s := range summaries { + ratio := 0.0 + if s.EligibleBytes > 0 { + ratio = float64(s.SavingsBytes) / float64(s.EligibleBytes) + } + frameRatio := 0.0 + if s.FrameTargetFrames > 0 { + frameRatio = float64(s.FrameHits) / float64(s.FrameTargetFrames) + } + mean, low, high := bootstrapMeanCI(s.Ratios, 1000, 1) + fmt.Fprintf(w, "%s,%d,%d,%d,%d,%d,%d,%.6f,%d,%d,%.6f,%.6f,%.6f,%.6f\n", + pool, s.Rows, s.Errors, s.TargetPages, s.Hits, s.EligibleBytes, s.SavingsBytes, ratio, s.FrameTargetFrames, s.FrameHits, frameRatio, mean, low, high) + } +} + +func bootstrapMeanCI(values []float64, iterations int, seed int64) (mean, low, high float64) { + if len(values) == 0 { + return 0, 0, 0 + } + for _, v := range values { + mean += v + } + mean /= float64(len(values)) + if len(values) == 1 { + return mean, mean, mean + } + + rng := rand.New(rand.NewSource(seed)) + samples := make([]float64, iterations) + for i := range samples { + var total float64 + for range values { + total += values[rng.Intn(len(values))] + } + samples[i] = total / float64(len(values)) + } + sortFloat64s(samples) + + return mean, samples[iterations*25/1000], samples[iterations*975/1000] +} + +func sortFloat64s(values []float64) { + for i := 1; i < len(values); i++ { + v := values[i] + j := i - 1 + for ; j >= 0 && values[j] > v; j-- { + values[j+1] = values[j] + } + values[j+1] = v + } +} diff --git a/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go b/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go new file mode 100644 index 0000000000..738d6b7f7e --- /dev/null +++ b/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go @@ -0,0 +1,278 @@ +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sort" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/google/uuid" + + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +const pageSize = int(header.PageSize) + +type scenario struct { + name string + buildID uuid.UUID + parentID uuid.UUID + siblingID uuid.UUID + mutate func(*corpus, []int) +} + +type corpus struct { + pages int + + parentMem []byte + parentRoot []byte + childMem []byte + childRoot []byte + siblingMem []byte + siblingRoot []byte +} + +func main() { + out := flag.String("out", ".synthetic-dedup", "output storage root") + pages := flag.Int("pages", 2048, "pages per artifact") + dirtyPages := flag.Int("dirty-pages", 512, "dirty pages per child artifact") + seed := flag.Int64("seed", 1, "random seed") + flag.Parse() + + if *dirtyPages > *pages { + log.Fatal("-dirty-pages cannot exceed -pages") + } + if err := os.RemoveAll(*out); err != nil { + log.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(*out, "templates"), 0o755); err != nil { + log.Fatal(err) + } + + rng := rand.New(rand.NewSource(*seed)) + pairsPath := filepath.Join(*out, "pairs.csv") + pairs, err := os.Create(pairsPath) + if err != nil { + log.Fatal(err) + } + defer pairs.Close() + pairsCSV := csv.NewWriter(pairs) + if err := pairsCSV.Write([]string{"build_id", "parent_build_id", "sibling_build_id", "family"}); err != nil { + log.Fatal(err) + } + + for _, s := range scenarios() { + dirty := pickPages(*pages, *dirtyPages, rng) + c := newCorpus(*pages, rng) + s.mutate(c, dirty) + if err := writeScenario(*out, s, c, dirty); err != nil { + log.Fatal(err) + } + if err := pairsCSV.Write([]string{s.buildID.String(), s.parentID.String(), s.siblingID.String(), s.name}); err != nil { + log.Fatal(err) + } + } + pairsCSV.Flush() + if err := pairsCSV.Error(); err != nil { + log.Fatal(err) + } + + fmt.Printf("storage=%s\npairs=%s\n", *out, pairsPath) +} + +func scenarios() []scenario { + return []scenario{ + { + name: "writeback_current_rootfs", + buildID: uuid.MustParse("10000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("10000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("10000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for i, p := range dirty { + fillPage(c.childRoot, p, byte(40+i%100)) + copyPage(c.childMem, p, c.childRoot, p) + } + }, + }, + { + name: "rootfs_from_parent_memfile", + buildID: uuid.MustParse("20000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("20000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("20000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childRoot, p, c.parentMem, p) + fillPage(c.childMem, p, 0x91) + } + }, + }, + { + name: "sibling_memfile", + buildID: uuid.MustParse("30000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("30000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("30000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childMem, p, c.siblingMem, p) + fillPage(c.childRoot, p, 0xa1) + } + }, + }, + { + name: "parent_rootfs_only", + buildID: uuid.MustParse("40000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("40000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("40000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childMem, p, c.parentRoot, p) + fillPage(c.childRoot, p, 0xb1) + } + }, + }, + { + name: "random", + buildID: uuid.MustParse("50000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("50000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("50000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for i, p := range dirty { + fillPage(c.childMem, p, byte(0xc0+i%31)) + fillPage(c.childRoot, p, byte(0xe0+i%31)) + } + }, + }, + } +} + +func newCorpus(pages int, rng *rand.Rand) *corpus { + size := pages * pageSize + c := &corpus{ + pages: pages, + parentMem: make([]byte, size), + parentRoot: make([]byte, size), + childMem: make([]byte, size), + childRoot: make([]byte, size), + siblingMem: make([]byte, size), + siblingRoot: make([]byte, size), + } + fillRandom(c.parentMem, rng) + fillRandom(c.parentRoot, rng) + fillRandom(c.siblingMem, rng) + fillRandom(c.siblingRoot, rng) + copy(c.childMem, c.parentMem) + copy(c.childRoot, c.parentRoot) + + return c +} + +func writeScenario(root string, s scenario, c *corpus, dirty []int) error { + if err := writeFullBuild(root, s.parentID, c.parentMem, c.parentRoot); err != nil { + return err + } + if err := writeFullBuild(root, s.siblingID, c.siblingMem, c.siblingRoot); err != nil { + return err + } + if err := writeChildBuild(root, s.buildID, s.parentID, c, dirty); err != nil { + return err + } + + return nil +} + +func writeFullBuild(root string, id uuid.UUID, mem, rootfs []byte) error { + if err := writeArtifact(root, id, id, storage.MemfileName, mem, nil); err != nil { + return err + } + return writeArtifact(root, id, id, storage.RootfsName, rootfs, nil) +} + +func writeChildBuild(root string, id, parent uuid.UUID, c *corpus, dirty []int) error { + if err := writeArtifact(root, id, parent, storage.MemfileName, c.childMem, dirty); err != nil { + return err + } + return writeArtifact(root, id, parent, storage.RootfsName, c.childRoot, dirty) +} + +func writeArtifact(root string, id, parent uuid.UUID, name string, data []byte, dirty []int) error { + dir := filepath.Join(root, "templates", id.String()) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + + var mappings []header.BuildMap + var object []byte + if dirty == nil { + object = data + mappings = []header.BuildMap{{ + Offset: 0, + Length: uint64(len(data)), + BuildId: id, + BuildStorageOffset: 0, + }} + } else { + sort.Ints(dirty) + dirtyMap := roaring.New() + for _, p := range dirty { + dirtyMap.Add(uint32(p)) + start := p * pageSize + object = append(object, data[start:start+pageSize]...) + } + parentMapping := []header.BuildMap{{ + Offset: 0, + Length: uint64(len(data)), + BuildId: parent, + BuildStorageOffset: 0, + }} + selfMapping := header.CreateMapping(&id, dirtyMap, header.PageSize) + mappings = header.NormalizeMappings(header.MergeMappings(parentMapping, selfMapping)) + } + + if err := os.WriteFile(filepath.Join(dir, name), object, 0o644); err != nil { + return err + } + h, err := header.NewHeader(&header.Metadata{ + Version: 3, + BlockSize: uint64(header.PageSize), + Size: uint64(len(data)), + Generation: 1, + BuildId: id, + BaseBuildId: parent, + }, mappings) + if err != nil { + return err + } + raw, err := header.SerializeHeader(h) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, name+storage.HeaderSuffix), raw, 0o644) +} + +func pickPages(pages, n int, rng *rand.Rand) []int { + perm := rng.Perm(pages) + return perm[:n] +} + +func fillRandom(b []byte, rng *rand.Rand) { + for i := range b { + b[i] = byte(rng.Intn(255) + 1) + } +} + +func fillPage(b []byte, page int, value byte) { + start := page * pageSize + for i := start; i < start+pageSize; i++ { + b[i] = value + } +} + +func copyPage(dst []byte, dstPage int, src []byte, srcPage int) { + copy(dst[dstPage*pageSize:(dstPage+1)*pageSize], src[srcPage*pageSize:(srcPage+1)*pageSize]) +} diff --git a/packages/orchestrator/docs/cross-artifact-dedup.md b/packages/orchestrator/docs/cross-artifact-dedup.md new file mode 100644 index 0000000000..731accc748 --- /dev/null +++ b/packages/orchestrator/docs/cross-artifact-dedup.md @@ -0,0 +1,162 @@ +# Cross-Artifact Dedup + +## Measurement + +Use `sample-dedup-gains` before changing the snapshot format. It compares sampled 4 KiB target pages against assembled candidate artifacts and emits detailed CSV plus a pool summary. + +Input is either `-build` plus `-parent-build`, or a CSV: + +```csv +build_id,parent_build_id,sibling_build_id,sibling_memfile_build_id,sibling_rootfs_build_id,family +``` + +Only `build_id` and `parent_build_id` are required. `sibling_build_id` is used for both artifacts unless the artifact-specific sibling columns are set. DB-backed sampling should export this same CSV shape from snapshot/template-build queries, keeping team and customer fields out of the file. + +Example: + +```bash +go run ./cmd/sample-dedup-gains \ + -storage gs://$TEMPLATE_BUCKET_NAME \ + -builds-file pairs.csv \ + -csv-path dedup-gains.csv \ + -max-target-pages 50000 \ + -max-candidate-pages 100000 +``` + +Set either max page flag to `0` for an exact scan. Keep the default sampled mode for broad GCS runs. + +For controlled checks, generate a local corpus first: + +```bash +go run ./cmd/synthetic-dedup-corpus \ + -out /tmp/synthetic-dedup-corpus \ + -pages 1024 \ + -dirty-pages 256 + +go run ./cmd/sample-dedup-gains \ + -storage /tmp/synthetic-dedup-corpus \ + -builds-file /tmp/synthetic-dedup-corpus/pairs.csv \ + -max-target-pages 0 \ + -max-candidate-pages 0 \ + -csv-path /tmp/synthetic-dedup-corpus/results.csv +``` + +## Pools + +Primary pools: + +- `memfile_current_rootfs` +- `rootfs_parent_memfile` +- `memfile_sibling_memfile` +- `rootfs_sibling_rootfs` + +Baseline pools: + +- `memfile_parent_memfile_positional` +- `rootfs_parent_rootfs_positional` + +Validation pools: + +- `memfile_parent_rootfs` +- `memfile_sibling_rootfs` +- `rootfs_current_memfile` +- `rootfs_sibling_memfile` + +The validation pools are for deciding what to skip. They should not ship without a separate dependency-cycle review. + +## Pool Ordering + +Treat dedup as canonicalization, not independent pool checks. Prefer sources in this order: + +1. Zero pages. +2. Parent or ancestor mappings. +3. Already-canonical sibling mappings. +4. Current-build rootfs mappings. +5. New self bytes. + +Rootfs should be canonicalized before memfile. First dedup current rootfs against parent rootfs, parent memfile, and one recent sibling rootfs. Resolve every hit through the candidate header before storing the mapping, so a match in a sibling that already points to a parent becomes a parent reference, not a sibling reference. Then build the rootfs page index from this resolved header. + +Memfile should run after rootfs. Its `current_rootfs` pool should use the resolved rootfs index, so writeback/page-cache duplicates point at the same canonical source chosen by rootfs dedup. This largely subsumes direct `memfile_parent_rootfs`: if the matching byte exists in the assembled current rootfs, the resolved rootfs mapping will already point back to the parent when appropriate. + +Sibling pools should be fallback pools, not first-choice pools. Use them only after parent/current-rootfs canonical pools miss, and resolve through sibling headers before emitting mappings. This reduces read fragmentation and keeps cache locality centered on parent artifacts instead of scattering references across sibling builds. + +Do not allow cycles. In the first shippable version, permit `memfile -> current rootfs` after rootfs finalization, but keep `rootfs -> current memfile` measurement-only. If both directions are ever needed, enforce a single artifact order for each build and reject mappings that point backward in that order. + +## Synthetic Results + +Ran an exact synthetic benchmark with 5 scenario families, 4096 pages per artifact, and 1024 dirty pages per child: + +```bash +go run ./cmd/synthetic-dedup-corpus \ + -out /tmp/synthetic-dedup-corpus \ + -pages 4096 \ + -dirty-pages 1024 \ + -seed 11 + +go run ./cmd/sample-dedup-gains \ + -storage /tmp/synthetic-dedup-corpus \ + -builds-file /tmp/synthetic-dedup-corpus/pairs.csv \ + -max-target-pages 0 \ + -max-candidate-pages 0 \ + -csv-path /tmp/synthetic-dedup-corpus/results.csv +``` + +Per-scenario hits: + +- `writeback_current_rootfs`: `memfile_current_rootfs` recovered 1024/1024 dirty memfile pages, saving 4 MiB. `rootfs_current_memfile` also recovered 1024/1024 pages, but stays validation-only because it can create same-build cycles. +- `rootfs_from_parent_memfile`: `rootfs_parent_memfile` recovered 1024/1024 dirty rootfs pages, saving 4 MiB. +- `sibling_memfile`: `memfile_sibling_memfile` recovered 1024/1024 dirty memfile pages, saving 4 MiB. +- `parent_rootfs_only`: `memfile_parent_rootfs` recovered 1024/1024 dirty memfile pages, saving 4 MiB. This confirms the theoretical edge case but should remain validation-only until real storage data shows meaningful frequency. +- `random`: no pools hit. + +Across all 5 families, each planted pool shows 20% weighted savings because it applies to exactly one family. The random family produced no false positives. This validates that the sampler separates pool-specific signal correctly; it does not predict production frequency. + +With `-frame-size 2097152`, the same run found 0/8 whole-frame hits for every planted page-level scenario. The page-level pools recovered 1024/1024 dirty pages, but whole-frame dedup recovered 0% because matching pages were sparse inside 2 MiB frames. That argues against frame-only dedup as the primary mechanism; we still need 4 KiB mappings that reference source uncompressed offsets and use the source frame table for reads. + +Ordering takeaways from the synthetic cases: + +- Canonicalizing rootfs first would make the `writeback_current_rootfs` memfile hits point at the already-resolved rootfs source instead of making rootfs an isolated current-build source. +- `rootfs_parent_memfile` should run before rootfs sibling fallback, because it turns RAM-persisted-to-disk pages into parent-backed references. +- `memfile_sibling_memfile` is real when siblings share runtime state, but should run after parent/current-rootfs pools so siblings do not become unnecessary hubs. +- `memfile_parent_rootfs` only appears in its constructed edge case; keep measuring it, but do not prioritize it over current-rootfs canonicalization. + +## Statistics + +The detailed CSV has one row per build, artifact, and pool. The command also prints a summary with weighted savings and a bootstrap 95% CI over per-row `savings_ratio`. For broad estimates, prefer bootstrapping by build or family so a large artifact does not dominate the confidence interval. + +Recommended first pass: + +- Stratify pairs by family, generation depth, artifact size, and target dirty ratio. +- Run sampled mode on at least 30 builds per stratum. +- Re-run exact mode on a smaller calibration set. +- Treat sub-1% pools as noise unless exact scans reproduce them. + +## Implementation If Worth It + +Cross-artifact references need a new header format. Today `BuildMap` only identifies `BuildId` and `BuildStorageOffset`, and the read path opens data using the artifact type of the current header. A memfile header cannot point at rootfs bytes safely. + +Preferred V5 shape: + +- Add a source artifact to each mapping, defaulting to the owner artifact when absent. +- Key build data by `(source_artifact, build_id)`, not just `build_id`. +- Make read path open the mapped artifact type. +- Make upload finalization wait for every referenced `(source_artifact, build_id)` and copy the matching frame table into the referencing header. + +For production, store per-artifact page hash index sidecars. An index entry should resolve to `{source_artifact, build_id, storage_offset}` after header resolution. The snapshot path can then hash dirty target pages, verify candidate bytes, and map hits without rescanning full artifacts. + +Start with acyclic pools only. `memfile -> current rootfs` requires rootfs data/header to be available before the memfile header publishes. `rootfs -> current memfile` should stay measurement-only at first because it can create same-build cycles. + +Compression-time dedup is the better integration point. Keep pause/export producing normal local diffs, then run canonicalization while compressing/uploading rootfs first and memfile second. Redis can advertise in-flight candidates across orchestrators as `{artifact, build_id, generation, frame_table, page_index, owner_orchestrator, ttl}`. That is only discovery state; the final header must still include enough `(source_artifact, build_id)` build data and frame metadata to read after Redis expires. + +Sibling dedup should use finalized storage first. In-flight sibling candidates are an optimization: discover through Redis, verify bytes from the peer or storage, and fall back cleanly if the peer disappears. Do not make durable headers depend on Redis-only metadata. + +Whole-frame sharing is cheaper but likely misses most page-cache/writeback duplicates unless pages align with compression frames. Prefer 4 KiB page matches that reference source uncompressed offsets; the source frame table then tells the reader which compressed frame to fetch. The sampler reports both page-level `savings_ratio` and whole-frame `frame_savings_ratio` to quantify this gap. + +Minimal implementation order: + +1. Add V5 mapping source support and read-path artifact selection. +2. Compress/upload rootfs first, dedup it, and publish its resolved page index/frame table. +3. Compress/upload memfile second, dedup against parent memfile, resolved current rootfs, then one sibling memfile. +4. Gate memfile header upload on referenced rootfs header/data availability. +5. Add Redis in-flight sibling discovery after finalized-storage sibling dedup proves useful. +6. Keep validation pools metric-only until real storage sampling justifies them. From bc2087825530571657601e79676bc5f1be03fde2 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 30 May 2026 15:22:04 -0700 Subject: [PATCH 09/11] feat(orchestrator): add rootfs reboot fallback for snapshots --- packages/api/internal/api/api.gen.go | 314 +++++++++--------- packages/api/internal/handlers/proxy_grpc.go | 1 + packages/api/internal/handlers/sandbox.go | 4 + .../api/internal/handlers/sandbox_connect.go | 2 + .../api/internal/handlers/sandbox_create.go | 3 + .../api/internal/handlers/sandbox_pause.go | 20 +- .../api/internal/handlers/sandbox_resume.go | 5 + .../handlers/snapshot_template_create.go | 3 + .../internal/orchestrator/create_instance.go | 10 +- .../internal/orchestrator/delete_instance.go | 11 +- .../orchestrator/nodemanager/sandboxes.go | 1 + .../internal/orchestrator/pause_instance.go | 24 +- .../orchestrator/snapshot_template.go | 8 +- .../internal/sandbox/sandboxtypes/sandbox.go | 3 + .../internal/sandbox/sandboxtypes/states.go | 5 +- packages/api/internal/sandbox/store_test.go | 1 + packages/db/pkg/types/types.go | 9 +- .../orchestrator/pkg/sandbox/build_upload.go | 11 +- .../pkg/sandbox/build_upload_test.go | 39 +++ .../pkg/sandbox/build_upload_v3.go | 29 +- .../pkg/sandbox/build_upload_v4.go | 16 +- .../orchestrator/pkg/sandbox/fc/process.go | 7 +- packages/orchestrator/pkg/sandbox/reclaim.go | 21 ++ packages/orchestrator/pkg/sandbox/sandbox.go | 101 +++--- packages/orchestrator/pkg/sandbox/snapshot.go | 1 + packages/orchestrator/pkg/server/sandboxes.go | 180 ++++++++-- .../pkg/grpc/orchestrator/internal_flags.go | 7 + spec/openapi.yml | 14 + tests/integration/internal/api/generated.go | 12 + 29 files changed, 606 insertions(+), 256 deletions(-) create mode 100644 packages/shared/pkg/grpc/orchestrator/internal_flags.go diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go index b1ccf2943f..608e497941 100644 --- a/packages/api/internal/api/api.gen.go +++ b/packages/api/internal/api/api.gen.go @@ -330,6 +330,9 @@ type CPUCount = int32 // ConnectSandbox defines model for ConnectSandbox. type ConnectSandbox struct { + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Timeout in seconds from the current time after which the sandbox should expire Timeout int32 `json:"timeout"` } @@ -558,6 +561,9 @@ type NewSandbox struct { // AutoPause Automatically pauses the sandbox after the timeout AutoPause *bool `json:"autoPause,omitempty"` + // AutoPauseMemory Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. + AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` + // AutoResume Auto-resume configuration for paused sandboxes. AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` EnvVars *EnvVars `json:"envVars,omitempty"` @@ -714,6 +720,9 @@ type ResumedSandbox struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set AutoPause *bool `json:"autoPause,omitempty"` + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Time to live for the sandbox in seconds. Timeout *int32 `json:"timeout,omitempty"` } @@ -1510,6 +1519,9 @@ type PostSandboxesSandboxIDRefreshesJSONBody struct { // PostSandboxesSandboxIDSnapshotsJSONBody defines parameters for PostSandboxesSandboxIDSnapshots. type PostSandboxesSandboxIDSnapshotsJSONBody struct { + // Memory Whether to persist memory state. Set false to snapshot disk only and reboot on next start. + Memory *bool `json:"memory,omitempty"` + // Name Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. Name *string `json:"name,omitempty"` } @@ -13003,156 +13015,158 @@ var swaggerSpec = []string{ "DZwAUHaJCinAG07TxQfmFQUxuSFxnwQ6TRen0O5+EiRECKWEN5Z0mi6Q+Yis3PPgQ0iSNTtfSJIpQiix", "nvEU2DcnMaDeUGKcLhCBpfhwTRMiJE48E1zaTxbZ7kDFJkZYkh01Sj/1FVOVKJkYbBZov5BY5uKcYCPv", "a6jXm2L+Ki4rv19NPJglumUdHQJmQFxP4dBN13ZWScJzclv3+KPZX3sOqvNPUJhzTpiMV4iTLOWSsgVK", - "WawFMOgppsdIynBYcO/OWODVLhydfW7hx0dnn1GYciIANFiK5suB76bYcTecKL2PkVAa0eNhtDQhaS79", - "NJnmUtG9IGHKIgEXRYDGYBKpzgjPJeHodknDpQsqEss0jyNE7jLKSSfge71yxULpUzKOOFFEd1jaPjwK", - "hmkje86eNqAgqUZB0EkrUEPO4CSg0RC+7c4xhEcnWFz3HZpylo9YXFO2OCYS01io/vr+2RD5OCEtEDU5", - "l9+gcLkkyGhgGr09A9X2FFYLwNkZzFonznZdlRt8SXByeHZiFOv19vfw7ARdk9X4rTUTvIe5cRz/Og8O", - "fu/eEwXvZ6GI+WoSsDyO8Swm+so/mFYMvEPI5Np34TjHt+gGxzlpDtgYIMZCfhbEA9cpFuasyyUVBRJv", - "sUC5AKbnRWJ1zc9C2a3L9dGibmhI0BBmlRKPSUwkGaTA9sPmKFQD9TKr/kYAxvqKmD10VjU9puL6I5Gc", - "hh6NNCI3NPQs5Rh+R3asOgBzGhOxEpIkl95L60/Fd6T6oh/IdDGdIHIn303Q3Vz86GWFSlyepdQnMz+q", - "byhTHy2GIwpb6eFnEsfvV5L4cKy+IZHhEHT/GbRyjx9l8q/vvFcsdRZaRlXnap1B69pDuf6J3ZgGql1A", - "Kmu1W31B/00+vvfsKBXXSNB/k7rWoWD+SN+PleGT4AO7+YKN+yKKqJoHx2c18nJB+MBuKE9ZopSLG8yp", - "Yh8+Jah5mj+wm+gL4cJr2zEfLF0QdhMhnjOmNECj17eOPQm0iaspc9LIQ9fQGME3D7qaKGrVZvWsfYzL", - "TOSqlT/xNDlJ8IK4JraIqrETyrDUa0lwlqkBtcGtjfu6hrpJsAiztoZ/PzpzGvJi5pbWhBGO46LH/cTi", - "dvXJWOHVqu8nQcrIAFHrgnk/6W7rQtrbtg6nwq87QIMoBOHqVB6GoTqq/xA+arzQbZBphP5x8esnoPG/", - "H51twAiodnGoEdCzHJ8KXsdTAy0ZFuI25R7d4sx8UXItFyXr4SU1PToGirGvPIPngnC/8P5svgwH1Y/U", - "YoZJiRcfVltVnwZ6lc5Coi9K0TvjZE7vPHiG30FfUyxP90A3Vcao7z0pb1MRnXku8rl3Hv37A+fJuhcB", - "F25qsSMaQyKD6Ma4oAqfEraQS4+WC793g9gmmA3A1Rkmnn3x4VAxlVMqJIlab+k4pthnqFM/D9Enw5gS", - "Jq1dMeNEuzGMYt53C9G9veNmeWHC6GKkhanjfqJEkaOCdPVylJV7dXpb73fodkkqYhzd0jj2mB4673ik", - "qkJ0er2cpiDEk5Sv+hf00baDPhJHWPY62AxNfLTN6972vs3rUGwgDoCMwSoWyHQajFUhFU0OW+QFtG14", - "6fuWWBjrwUClLVFUVCA39zgvUwDHPFwf4IgNslIagL+UffvN325ggRsQURxOd0ecs+XQV+X02CNhcVyl", - "YOAq1jTuMVwqfDVIxErIiMzyBcSEzNNgEtxiDvITVFKf0DxNF+KYchJKr/5dfHLs28Z1ZayEM2ICaWCP", - "LBjzlN9irn6Z4fAa/tmYfRLc7aj2OzcYpKpQHSvw/FSMUvn5fTGkWcBFmnPfTVf/PhJ0tdspx6AVZGpL", - "BPgchoOvZ710hil/PXMGvJ8EH3G4pIycqM1qXlOy/JCHSypJKHNO/MZm7LSwC2X6auHj+T/hhMYr/1Bz", - "+DZgkI9p5KNMNUaiPg0d4pNXWSuHYY7NxT9W/U5VLNCBszbfpIFXvRF3lwQn2pbiYaoEJyiBj8ZJ4fhp", - "mmZ5x1nULbEb7iMzxxgPkuOf+sx8ulfnJErVU920lfAH6zAQlIUEkSwNlz/WrsMtNhTQn/ymZhMRV7Vn", - "mjglEllwzHV+QW8IQ2pgfoMdj7gO4Ot0mFXxYEGC7Q2zDlNGIwLm49EZClM2p4uc67CmpiGjxUZaXgI+", - "OqpF3d2lvqxjq3mz/58+3H8it51OlIc6EnxWyCs9b4fiG6e3f8A+MiL/0BP4FOE4vS1QINMCkiVBtvMU", - "/ab0GUGkajDHsSATRCWakSW+IVZdSAhSSk5GQjpfUbZAEWGrX3PoszeF/+3uWSpjRN6m/Nrs8rRc8ixN", - "Y4JBN8S5TM9wLkjFj6qnbwbBpQlWF9Y4XqFMdapqMdrVBiqPcYi1zXhORJ4MVbsOiw5HsBCjDFvTXY8i", - "DM2UQqtPR6f+G2YPVH0Nxgf2/KRbl6sSJPTKwAv4HeE4RsYoHaZJkjMbjwjcuqFJOzgfp7DaY9DtA3A9", - "szZW+C8+3q9oM6Y3XrutYcXT8cbbZ9CLDTfo8vQ9ns/H5T8a3nVm01gCU45UfCY4CP7/73jn34c7/29v", - "529/7Fz97/8YCImH+X8yJuaaRhfnQhI+jNRMY68ClSbeYPcj+N0OkPJwSYTkYDhu9Yz+ZA1TPYFl5iIG", - "4RJD/Sq6y4WORyNjZhFFn2EzDXPKtimkSVUN72SETlPNEK3zrauXIgfrpyvNACNi+qzPI2XuQiqYaXFT", - "iOKCDnEz/XOahujCTl5jQP5ZtLn5hAmJWehlptZ4Tk2b0g7Yuz8muGcAknVoFDDBgS6l7lPi8zY31zpx", - "TnYBbW2bS1ppnovqWWzZs3JJBQOoUu6V4Tva2OyL1Q2XJIIoLc9RPKUCOIduZaNqaVQjueFxmltmt2V2", - "G2d2WzbUy4YqbKCfF/mYTsHIfOzHiUepP52JrO1BNMwm6roIhpKjs89dVFK0Q0Wo5UDaKHrq63dLvMch", - "RGpUZ9JG3LFBJa6HxRepUr5ILINGx1N8mOVnhIfEe7YUwtXgOUTXZrqdDikeMnZExbXwxQ9J/WrB7KWO", - "wsXhEsJ2dpMynGdo5LAbxuSNG1b4v+yN/WGawNbZLN3rc3sc0CdnbOsiXTsaqELsLZRZ2domgB4vg4Mg", - "u3f2TF4UnKvpTMiFy/emX9kOijimigMfFD8jKtAszRl4+2cEiWUuUZTesik6kdpnx1IJxptMIkZuHXaO", - "WaRbCJlmKFU8F4OPjwpQNJ2WnKAoZRoIxdai2aoKg55E0hsS632YoFkuEZUoxMy+6IW3vThawcxhyiRl", - "OUHAL9kCSY7ncxpOv1YDC3CkLp525cDuIP5a/5GzJcGxXK40Y1WADfQIlOg/N3OUvxyXs5U/Hrnzlj9/", - "diAof72wsFQ2+miJ2eLx7p+9EazjBWPtQJgB1Cq0OavDoV61ynXb1x/JLve8Bh2FrFcXXxClCaYetec9", - "FuqMq4/Oa83C/qvPpjrp2g5MZ/GggGTCburvCGoIcd8HAAMHqcVuoqrB8HHDCx7L379Jr7rZg05sws+l", - "MVSh0uxXyc/RDcUo4+ndatq/g2t43Osu8zaTeJMUcpnucGji8fIAk4hKoTRtqKqEqYVEo030H0y/+mLt", - "eD47Yusgg1wSdpVmBjSP8cK/SHSsB9PeFb8/xMDSZl54KCcCh9GJcfkctriLflsSuSS8cA1Zd9EtFojc", - "ZTENqYxXxYJTrnRSs/gqR56iT3kco4RgJpT+oEZQ2oUziiCyg3QdzHwPQVobZ9gbiAl7gRIhpnMSrsJ4", - "qIfvtGi/+Wi1h7rstsFu22C3IcFuDVJv3lMNforTg7I0puGqiN9Cs5Wjas/TptSu+tb9UqWyFZghXMpQ", - "v0hM2WV5NRiwEb8W7RsmgRI8d9gOneA0Xfhfy+u4nmqYElyFY8pIAy/wo3cc9aXryf0zPYsHgK8qeGhJ", - "QjCnxLg62t4wtTkxSmRvPJHBc2EV4HeTDhjsVTEt+vMNVO1LPIcgu0hHXzaY/Rgu1pVaIE59zy5PH2PO", - "Xo4Jc09cPNRw9mX/3KQC82KvL09DwfzUakSB0UfDnm85zgo+OgrEsLeAtkevbK9M4g28/OiGKg5lae2W", - "909Nm/uwx35hln8WJDoLW3I+dFnY53Hq5p2xgYxaSILRts2gHcG7ztbHp+3mbNXR/yIcnoq2GrA7DeRH", - "OFz64nW1w9jYxn/IgL+p334cP0UnNjos+52D+hHxsceW3z7knzPCd0TcraMSOuem3Atnqx3CcqjWPRoO", - "J6reMPwRp7/6MqLY2AZoQSIUESFN2k3jvVpwuAsa1wD6gMOlwZ7SA2cEYXR0cnyOZnEaXusn7+hr8J9T", - "+N/u2/2vwY8ThNEMc4JOzhCOIhiw1hBapRxhe6GGCHfbiNzhJIvJNEyTr8EEfQ3+17Ty049TdGgWYNP2", - "4PgWrwSS+JogRYckImpX0xvCUUQYLZtOR8VuAKLO8llMw0uNk4qM8hH6hQ68RbTC89Hn81PhvLcojQQ6", - "gQ+w9OpzT7+mbYJ52/fWLLfcJaEwXe4F8e/0cbkR2v/EUolEnmWpuuFAFzU14nk8FokJFtcmy8TPqfCA", - "blG2TIWE95bmUgjmjhkpjRIQ3WoQauLmvamOAMguOT1GYTCn7TzXtof66mrvOwnfMTQtOWZCsRONMwQZ", - "KnVqqQTLcEnZwu7Cz5eXZ7vq/y6KZU3RL2RlPYFqvPIQ4YxOG2ekcULs0YrhXSYCi5V1HFa8QZYZ7MCT", - "YMs8FJgZ4QnVaXYrzsLa5eC+/W7m4q7JrSsIcvFj0GJxUeKrYBgmVrV5vS2wPm53C1iGLOfSnaNlTYap", - "dmx6sboZmaccPGW3mEeULZqrWhIcET7uClcFTFEXMsMoaChTa1OsQTFJTiOiHzsbGEsyPGSlh1r3d9KT", - "KbYNvJuq9WQxDkk0RfA0WNNuFqvd0kCJ/4OETh3KiUjjHFwNS5xlhAlj/N0RChCDEEFYBI7q1EZFr0l+", - "nzOlVLS5QT5VnkBYF0gOfTTXKQKxCpP5uV6rqOQSs/sKp73AUcbTGxqRqDr+FP2aUCk1TcMNE4UxwVwg", - "KqfeoKCtRH88if6qn+O8Qg3gJcpj5xBXonDM8VU7a4/udBTT+dW1hdZdLcBbFFGzprkerg8ozYEB2IAc", - "kw4XHJXeh8sVW/zYV9KGsdkJoq4ZXOP5Q56ZNHNlYF8yizNcprJo6+vPCgbjdRiMifiNymVrurDCGdxF", - "sMM8SJyGNgW+E+NTjA9XOoYzsUyl/+mziZ5qpB7L49gcVLu1Zhg37W8Y51p+Epzo1mBBwQzS4hrmrD7u", - "iDhf7CarHTvKwc3+j6MOuO040MfVBewScv9O0WfFpwqod8Grrs8M1sLlFotSuHYtxlzXlICSS8JvqSCK", - "W8cCzXB4bdULjm9LeE6OzYh4Fr7Zf1sMMe2lQQcTE7N9PlK8JDjx3NyhRImHb5h8hdZ1r9bpTd8pju3V", - "tMu1AwRhPG5mZbUhHXEzJA2gH5qy9EW/R9A3QsMhZ4plmGNukOWu+spgdptkszVE8U+fI9NQjzdP6yOl", - "PghTZu4EF64saSYEKEPfyy5OGHDtuA8wz7sPps69CoE3Ub52Y0MFIG0WHWS239p/++y/Hjrw7JGlPOAC", - "DZ5FEhNl1ofhD6qhXXguIGK593AO4y9mtB7m4jttGnq9QhPy5g+Ys0ekJ45ZN31ARn+Tvb/XL6aV8Uq4", - "iuKEqrMcdhZHFEJQTNkpV2FS9Kvjr7OAdIULzspM+H1c1m6Bkzx/3ei6HklaXm4q2CvDep5JnK6fuW3t", - "ODcs5EWGb9loZAFRPEzyrhEm13Lj+OReNgowf6jr5xpOvaDiWzTuLpGBu6VPg7UXB4F0e7BksnjlelZm", - "K4926ai2Qu3LupygvjMdftK1ouN850EbJdcjJN11zQAZNxauLHk4IOTNbKbLMNxluEe8flYq+1Nh29Xz", - "OCkEiKXeKlN0hQ/Ij/YglM2R3mPRRNdGmdW46wfO//CaPUMuGk8qVbSAXEekbF4CzCmjYjluVbbP4GWt", - "w+rFQ5SGwayoXNTD+VDJeopH3a18xcObGifhJxqTz1mcYs+ZyDgR3tfELjOY0xgYAY71Q0vTyZrzQxPQ", - "0Tz/OfeE93zmsfO0AMYuIwZygBNcib14srA3Fuy3Mq5x/Jt2haHVlgCOdWP4eksrDQgjLAEY58Moykz1", - "AlipS/XQg7YJSeE5V/5gzgqMp+lCPCig8ylJoS2Ys7KC1tonD35Nus6DpTS8Jlydek+0YvHNMQq1T7+O", - "NAAGdpR47AHwXhuFSxJew4sgrF+/kzsS5rq+YEUvKp8StzILMDh55wKryCPN8sj2Z2d/2gjpy/7LIKV1", - "9t/F1tiXe4PwpxHRirq3nagbYBaqI3OKjotuE4ht0pEqTEiCo+lz4np4aaUpOsLMuMcIwuC4A+tzmMYp", - "Q4JkGFLlFKEWyWrH9v0aqJtJ5aeDmzcQbXEyh5GosENHELlgA+6krYYmbBgjzOs63Ox5xAuBgOVOx1eB", - "8hdBLQ70iFKoj0+7dTIFxuQ73/O0SLnd9QzZ1Rpvl2lsFeNSwYOBgOfxnCFOFphHMREFXbcrk3NbL8fD", - "69TPttwHFhDII5pCpJ2Jzn21eLrovFm8x4ziGoDrrhUDxQPg/P7El5Ak6y1dakJ+oG3XfI0zNUQTvZAk", - "82pWHnd2U3ftSbLTAM3GpMDfOijlFlOT/8XmpWlP4G9BOCULHK56vAxbn8Kj6xxbj8B36hHY2uO39vj1", - "7PGurm/UfGsvaFX3N+wHfnpeOsah9kL9ZB1aPC4K7T+CEr9JQ1dxEJrOo6KqbEUP6qypX122La7fjOPn", - "vTaxQ77IE8WLyzw9avYxiIRqqj9j4QmqVb9aDEKz4t2nM1PzDjD+iqOGepS7TXelxHaofYUL3T29xIuH", - "G8IV+achhbuyefBBhboTD3KQDVaYzN3bnrXh0VJ44Y87UyP6yx7X0QZLcSRMLVxO41K/r2k1pm6KUd17", - "QGqzTj93mIMncr3Kc36jclmmKH9+QdmRKd2kSPcYpkfdNrVv2pdAfSM3i+dUy7dBN1slf1Ash09dadPk", - "+7V3zXE0q1yjYg251RZ1e/BHl615hJo1LenBvLt+PPI5UTHUpL2+jV7CIYvWrjjWvhTZkq8vV4oGZOtz", - "yjjDo0QM6ZK1KqLH7cqw9ISossBf+R7OQQkrKlcXSgBoNDkJCtXyQBcjmBP+kz03msP8YQuzgfAAzgLN", - "SgCXUoLN8jBKKKsMSNXK9AtmC+ZB8H93oOHOZbXgm3lbosaBf/WNcXay84tL+U7/XC7PtNOEvwdYH2uB", - "zsBweLtBVE12YIMaIF7kGZ5hQd4MQZdt3I4x22J/AFTlaBVxYQdT1EJNyIqkUknQ4MP+e8VpnIoMB8He", - "9M10D/LQZYThjAYHwdvp3nTPPAYEEtvVCN4BBGulypuM4kjXIsGQvL1WDlCdbHgQdBIFB8FZKqRDuCLQ", - "Z4II+T6NVuYhiDQxRJAIQCf22P2nCSTRGlBvuvBqUcPae0JjxuRG04WF7e+9ebTZj4x0qUPQkUPUCCTH", - "YBIDYbzTYPlmK8DfVY3uJ8Ff9vb626pGLkMBU7CPmn+/up98az2Kv1/dX1kjx+9BlUyu1PhV0tn9hktk", - "nBzfaxKKic+zdQy/I8y6KUk3c2np0J0CyJjjhEjIAdFi7y6b7FYABLt3jT7e9aSB1et52Ba+07P0tX33", - "ArdbiYRdpcGK3W/at3y/qzWm3RCzUCdCbGEf8F2/KadsJ+OpfmaPWYQyk9CidmHSuQEgS4jmgh42A9JM", - "AXQJ4OiboZ6rSR2eh2tARMCE4XV0wYKLR5VVjjJxuEPfq58mde09GveBdcNi9VrPichj6eNAFw6lIr1J", - "cVG+62WSbl1F0YQp8iTBfFWQElCSQzG4uFpZClbDdFFu8dBvF5IJtJLuLzQ2hNt8grgGjRZv3n6xKQy+", - "ZyI1q1VrHUikajPc3POvk0jVgj00002lGd25JivYiAVpy2SiBoWX6uZeKRpU93citXquVa8HbO9AQ1Vx", - "RW7a87r3ukjP3FzUMytGXn29T4C23T5qotVus7rrD1C2Xbz4OYyz2U+iZ7s7/Cxqdh0AD5Os5CZ4YVr2", - "ZojJZSG73/RdcaAe3k1jRg3XVHZoxh2vfNuOw/Tuyqa+dr17Y9wEy9Dj3tSmzL5tPlOdH3mXH58dNcyy", - "gzjSXg+BGQPxlsB6OYyuMteqovwMn3VYqU8x0d+DQRxgaYryheBaseXtRu0KEMcuSyMyQKvSzTxAfzIf", - "OlX1rgp5oLz/KyeQ0sNo75VarBtT2AuNbliQINSpv796kF6n0boxEezXzH36NgC2+039x8hJL338nUhT", - "X5LN01by+ASjjOaXevJAHdTXSE59tGOKgQ2ml6K67Su69NVJq1Wph6qbSBTh3tgW8m2q9I9BUk8kgBtl", - "RO+NBB7Cz+EcGQxANAIM8Rrk7nC2UkmU2C1xagl0/dLHTZLUKYGKuhrAGnQIm0zRnMayWryIlHlqc0H4", - "f+FZ+DXf29v/K86y/8p4GsFjKEiJq5QjzCJ0o3MXJ7mQaEbQ5/NTRFiYmiygPoZUFBZz+dEzi7NTSPpt", - "i6U+UK41Nw+IcW8IMe5tUB46nlqtEW5chaym9uwxedi0r1AFrRaE0mSU7uF4IutHQS6bNX1UpvVoxk4x", - "u3abx5YYO4mxwq53kzL1bTvbdova6yj3Yczb5tXt4eFHaZLgHfN+lURQpsDJSohOjuE924JUIAkmAbnL", - "YqVN2BhkH0s2g/xBI9HpNWiPj0vw3Yn++GZvr8Y8J0HO6L9yYhrA+XhSBdObt/hhLFzHCFlC2B6hsUfo", - "W1ERstPuqL0jTuJtn8Gx2N4Lp8rkOFW4rE850OhYY6zWF/XytdOXJuRbb9KlgJ+tENxJ23nmE238o3Og", - "dW65lva35PQAHrMbpowRXXTPr1aeA85FQXSRTqCsUz04p50KU/S8kvBB13aNpujy8lQ1gaBucicJMxef", - "DoW0IN4jA+NDafjxlVsD2SgFd+85FFybR8zWZbifPJeqbShiY6r29rxXzrvNnlWIl+5nPkrgCKc8KpRP", - "QLs3+x3Du5lwBoimU50QbO2jPfHmyoCSD55yvgLJJZbOC8JCplCGEhrH1OTvbjHKQIoOv4XYvhlJKKNJ", - "ngQHe74k3w2bE75TrZ2U7V1QtkAV04RWoSpeVL7ZU/eMRgqITiA3IPFh19eR9zrB3pYJPIQJ9N3T3VOf", - "FNfuAWe59Y7+gONcpNbXR7l8vYy5tAcbHqnc4HiiTrE5wBNoqgs+lin7n/Bc+4YlUPLAPZYDlkZYtN7C", - "xoF8tYnYtlrNonXNxS4D2IBxYcsvKvzCVI/Tb65la3hMs9CcLXq4gWqHnlqHtbtF7mFYpubbC7xadFSa", - "HHTN8BhnKkfLBuy4J8u7eU95dt7t/W1I27/9Sc6ZLsbXehU/U59rBfeG3J+h38ZNf9oaULnngSPUVAc1", - "98AtdW2MujiZcyKWRHQZe6BJhSloa41isVQKU8crRTG9IQPJ77yY93m4bPXhdGTZWjOW15VWpXpl8VBe", - "xq5JJhFWGHC0Mij9dae1rbd/VXeubn2x8X75fjwPtzu6IYPoK6Z8YfMBFGTfbfc4hx5r8Frd8QXqExqw", - "6OX74tsNhFspsYGzYuuMDnnJnhEuqJBQXNCWXi2iTsyY/1MUmr2QkHjaVqYVVhWwcVY6CgOe7JT+ZP2+", - "FeZBM7JKmWZ2KacLynDsTBPTOVHiaahboYDjRcglf0KPXzNdJLiaEKVR5xa8MdhT/7ZMHgYDWDcAFKUW", - "E/M+ylRSMS6beua9ooC1U1QX7LtQXhIKmUKxeTVSyoi/km1D2G2U+bilkH0mB4u2TYcBbc0MFdYjy/Lm", - "fsZzQaQOaNMNy/NgDQoVWUKh+npGOUF3VktzguJomSrHnOApOsJxrItFU4ESIpdphJI8ljSLiS2jfkP4", - "LafSWC0uL08niOBQl21FubC1pi3TK817WJSGS9UqSymDav0JwSI3xXns0qyaOpSZ2dLwL4GVybYy9QZI", - "R2su98PFl8li3qqD610Nxro1mgVWFZRXj6KKC0Oabs19NfqWIwzkCK7a0R3zXTSt1zFuOijaJXxHdF/t", - "BUitQr6OBS9hmK2QSHMeEifC0CcCe09ihpVCo6Y5BW/eqC6fyJ00SXQ2Y9aviNR1rfrlpv+pgv+KVWvS", - "h6waw1IWeN1wl+bDJt8BQFqtB4b/6wVtbufrqeC6tn9Enh/1m7ORZXqUIV5WNxi6i5M56U/W9bGaZCdb", - "B+v35WBVRPEY3lV42b4R1+rbIW3fvno238sYdhN818kcgPZMeJCPUdj06voVhaXkYezjI77bcpAXz0Em", - "nheKnIaQ7179i9yQCpXAI0PznqXlSSGHbL/tT1dstaQwZeZK9of7Pse+gIHN+INjSTxFk540gOwjvnN5", - "3pbHvRQep62DgzRZ29TLqsqPA+5sRQq3tgM8uDDf1aY1aPMo88FatMXXM96hRunWT0iPJTKq72S7nX21", - "ZGEdj2Vd4nwKJ523iPAga/n+o8Ng6t21eOzK6us4DEkmbSTHi3vs9wIps8Iud03i/d1v8I/2LCpHUGiS", - "zmvOHa386YIN2q3TyVVN3Q74TwuHrebIxKZlu8bQUnPTdtykbmAnh+UVRVr6+KkSKToDqS22tDLQjybp", - "4XL/T2Ppbaf8svxXZ3K90gGDF9Zh0KpB6D4FuV/ixVPx6upMaqJRDPtdS+Gz9kx9Wx/kw+jQVl/z+xYP", - "Tc1ovPhB/AhFpJs15zoUgickNA3Z2oT25pEBIZELilc/wIsyimBLx09Dx1VW+q0s6jM0X2mLmltnoJVi", - "QSPNRUXX4RHElVpHj5G19Pu/5nRftmsZ3lt23VURH2nL13RfruEo3YRG6ZTNG2lycpV1KsWalQO+GxWw", - "kmO3+15unxW13sjVQE/CqZ7uZl8tabl26t1G/brW9LsvP/vO6zYenRNTsZUNNB29Djp9vRaorVWprhDa", - "EkPfTL3d+zEvISCbQlmZeChta1n5viyL/YR6hC0j7FEE9v28U9POEgvto9ySzmDSaSYQaXitqyXLY53e", - "Y4i6WaGZtdKCrEk3m00hEuZcgIf2NeUQ8cXa2udub/pfu/mddxHlJIQ1TAaKBkUVx0Wv1oFjcgP1ywYP", - "egodPKi90LGdQ3Z/ztOkLXQBRhm1Sj3xhuzlcObUrINt5v5LjnPkX6b5xs98X7SVvJsd61TgYxhyWxL+", - "PoasU5Y/G0s+YRG5s+eweENREFzrqSwySLjl430sI12IX+dzQVp44OgkSt8Nl16bmW6Mc7U+7OrlWFs2", - "9SRsak5j9dMSi2V3WRDMUJ7FKY5QTNm1tVJijtQIUGMcU+YcdLwi+ttQnfIn1fZnLJYPZVwe1/hSDzvU", - "M66gsAzMLqHfOf7maY6MwstnwHzb/drdl9sl4ZD2wfwIR8js0nfg1Hntx8060nvC7sB9vo4fwDg3H9P9", - "8ySR8IVb8qGh8EZhArw+Yeznn9Fn6aYOHfDWr6s0wJf977myy6Tt6WEB6GyFUkZQylGScl0VCDAxqJKB", - "1Md/vXR+F9LoTtVDNgmEXMXqB6WGviYX4LYMzmt8NNyZhnhQdtM266TDWl5pouJXaWXsu8DujYW5sBUO", - "wWwLyI9ipayhUi8CDBdwvTYvenLOpkj1RjMSp7c6D4VugDlB5C6M86gdt49m9TzCguwIwgSV9IYgkc+0", - "WEIJluESpQwgT4gQeKGvaYrLtkgagnm4rICV4LtTwhbqgO//5a+bDRF28k9/2V/P3LnNRL0er648EXr8", - "xxlf9p/jecaX/ZfuHjeY2JY3W+/O7RJuI86yu154d+SSQ6/fd+zSkwDRzri3wVHPeSp6gk3GhpZ4D8nz", - "BZc8sUwBjIySKC8rtuUFcu+3bWrHmkrG22dRMt4+l5JhALD81gKy1TeejmLTOE/IwIRQyLb22TCKT09v", - "e9dzjTa7x2BHa67mz7T/ds0DKjtr7lTgyc+gnF1/kmrOdqs3+/ZMz3rIImMj7iEsm8m1ibMtG+ojQ4cJ", - "7X7T/xj+qqydOHUjQ55fzLCjlTcLz8AnZRWisM/JcJMgtgacTr7UEVtWILI1sOwpt3zvuRiMTb+0pabR", - "7AWg4zd293MeBwfBUspMHOzu4oxOyf5sirMscPp/K/P9lOluvtUSt1Z/hNxE7t+weztSLbjaMKM712RV", - "+c14/ou/C8Xk6v6/AwAA//8=", + "WawFMOgppsdIynBYcO/OWODVLhydfW7hx0dnn1GYciIANFiK5suB76bYcTecKL2PkVAa0dPcZ05maeoB", + "4VyJesUAndnhpqh/YDgTS6XE0piIlZAkQZhFKKIixDxCCUlSvgJ0kmmJlFmaxgQzexrSXPrPQppLdd4E", + "CVMWiXJas4NIdUZ4LglHt0saLitAimWaxxEidxnlpBNhe73yzELpU26OAD/RYWlz8Sg2po3sOfPacIOk", + "GgVBJ624DTn7k4BGQ+SFO8cQ2ZBgcd13WMtZPmJxTdnimEhMY6H663tvQ9XACWmBqMkx/YaMyyVBRvPT", + "6O0ZqLansFoAzs5g1jpxtuuq3OBLgpPDsxOj0K+3v4dnJ+iarMZvrZngPcyN4/jXeXDwe/eeKHg/C0XM", + "V5OA5XGMZzHRpobBtGLgHUIm176Lzjm+RTc4zklzwMYAMRbysyAeuE6xMGddLqkokHiLBcoFMFsvEqtr", + "fhbKbl2ujxZ1Q0OChjCrlHhMYiLJIMW5HzZHkRuoD1q1OwIw1lcA7aGzKvExFdcfieQ09GjCEbmhoWcp", + "x/A7smPVASgF0qX3svxTKbBUX/QDmS6mE0Tu5LsJupuLH72sUInps5T6ZPVH9Q1l6qPFcERhKz38TOL4", + "/UoSH47VNyQyHMKdYwat3ONHmfzrO+/VTp2FllHVuVpn0LrWUq5/YjemgWoXkMpa7VZf0H+Tj+89O0rF", + "NRL036Su7SiYP9L3Y2X4JPjAbr5g4zaJIqrmwfFZjbxcED6wG8pTlijl4gZzqtiHT/lqnuYP7Cb6Qrjw", + "2pTMB0sXhN1EiOeMKc3T3Cdax54E2rTWlDlp5KFraIzgmwddTRS1atF61j7GZSZy1dmfeJqcJHhBXNOe", + "Ugg5TSjDUq8lwVmmBtSGvjbu6xoIJ8EizNoa/v3ozGnIi5lbWhNGOI6LHvcTi9vVJ2P9V6u+nwQpIwNE", + "rQvm/aS7rQtpb9s6nAq/7gANohCEq1N5GIbqqP5D+KjxQrdBphH6x8Wvn4DG/350tgHjo9rFocZHz3J8", + "KngdTw20ZFiI25R7dIsz80XJtVyUrIeX1PToGCjGvvIMngvC/cL7s/kyHFQ/UosZJiVefFhtVX0a6FU6", + "C4m+KEXvjJM5vfPgGX4HfU2xPN0D3VQZo773pLxNRXTmucjn3nn07w+cJ+teBFz0qcWOaAyJDKIb44Iq", + "fErYQi49Wi783g1im2A2AFdnmHj2xYdDxVROqZAkarUO4Jhin4FQ/TxEnwxjSpi09syME+0+MYp53y1E", + "9/aOm+WF6aSLkRYmlvuJEkWOCtLVy1FW7tXpbb3fodslqYhxdEvj2GN66LzjkaoK0eltc5qCEE9Svupf", + "0EfbDvpIHGHZ69gzNPHRNq97+fs2r0OxgfgDMgarWCDTaTBWwfY0cJEX0LYRHdC3xMJJAAYqbYmiogK5", + "ucd5mQIEBMD1AY7YIOuoAfhL2bff7O4GNLiBGMXhdHfEOVsOfVVOjz0SFsdVCgauYk3yHoOpwleDRKyE", + "jMgsX0AsyjwNJsEt5iA/QSX1Cc3TdCGOKSeh9OrfxSfHrm5cZsZKOCMmgAf2yIIxT/kt5uqXGQ6v4Z+N", + "2SfB3Y5qv3ODQaoK1bECz0/FKJWf3xdDmgVcpDn33XT17yNBV7udcgxaQaa2RICvYzj4etZLZ5jy1zNn", + "wPtJ8BGHS8rIidqs5jUlyw95uKSShDLnxG/kxk4Lu1CmrxY+nv8TTmi88g81h28DBvmYRj7KVGMk6tPQ", + "IT55lbVyGObYXPxj1e9UxQIdOGvzTRp41Rtxd0lwom0pHqZKcIIS+GicI45/qOZ3rTqpuiV2w21l5hjj", + "uXL8Yp+ZT/fqnESpeqqbthL+YB0GgrKQIJKl4fLH2nW4xYYC+pPf1Gwi8ar2TBMfRSILjrnOL+gNYUgN", + "zG+w44nXgYOdjroqHixIsL1h1mHKaETefDw6Q2HK5nSRcx1O1TRktNhIy0vAR0e1qLvZwLezhq3mzf5/", + "+nD/idx2OlEe6kjwWSGv9Lwdim+c3v4B+8iI/ENP4FOE4/S2QIFMC0iWBNnOU/Sb0mcEkarBHMeCTBCV", + "aEaW+IZYdSEhSCk5GQnpfEXZAkWErX7Noc/eFP63u2epjBF5m/Jrs8t+BxvOZXqGc0Eq/ls9fTP4Lk2w", + "urDG8QplqlNVi9GuNlB5jEOsc0ZNIZV5fezjtyWRS8LVfTndgVmtaDOCq+pGRBdEavRBDKv1QCrFRLtx", + "MYuQdmeilCFG7qRWGtvxc05EngxVEg+LDkeAdqO6W0Njj9oOzZT6rc9yp7YeZg9U1A19DOz5SbcuVyVI", + "6JXYF/A7wnGMjAk9TJMkZzZqE2RLQ+93nb6j1Gt7aLs9Fq4f2UZU/8UnqRTVxPTGa2U2gmM63tT8DFq8", + "4V1dfsnH81C53FLDu85sGktgeJKKKwYHwf//He/8+3Dn/+3t/O2Pnav//R8DIfGIqk/GIF7TP+NcSMKH", + "kZpp7FX30sT7JOAIfrcDpDxcEiE5mLlb/bg/WTNaT/iduTZCUMlQL5DucqGj9siYWUTRZ9hMw1zIbepz", + "Ur00dDJCp6lmiNZV2NVLkYP1KpZGixGRj9ZDkzJ3IRXMtDhVRGFOgOii/jlNQ3RhJ68xIP8s2jh+woTE", + "LPQyU2vqp6ZNabXs3R8TAjUAyTqADJjgQAdY9ynx+caba504J7uAtrbNJa00z0X1LLbsWbmkggFUKffK", + "8B1tGvdFNIdLEkEsm+conirlJp0j3crGHtOoRnLDo1m3zG7L7DbO7LZsqJcNVdhAPy/yMZ2CkfnYjxM9", + "U39gFFlLiWgYedTlFsw6R2efu6ikaIeKgNSBtFH01FfBluiUQ4grqc5krnwjQ2Bcf5AvrqZ8t1mG1o6n", + "+DDLzwgPifdsKYSrwXOIQc50Ox14PWRsdY0Vvmgnqd92mL3Usco4XMK9dzcpg4+Gxle7QVfe6GqF/8ve", + "SCWmCWydzdK9PrdHLX1yxrYO3bVjlyrE3kKZla1tAujxiTgIsntnz+RFwbmaro9cuHxv+pXtoIhjqjjw", + "QfEzogLN0pxBbMKMILHMJYrSWzZFJ1J7GFkqwdSUScTIrcPOMYt0CyHTDKWK52LwSFIBiqbTkhMUpUwD", + "odhaNFtVYdCTSHpDYr0PEzTLJaIShZjZd8/wAhpH2voSpkxSlhME/JItkOR4Pqfh9Gs1DAJH6uJpVw7s", + "DqLU9R85WxIcy+VKM1YF2ED/RYn+czNH+ctxOVv545E7b/nzZweC8tcLC0tlo4+WmC0e7/7ZG287XjDW", + "DoQZQK1Cm7M63P9VG2K3N+CRrIgv4kHCxg1JapNeXRRGlCaYetSt91go3qI+Om9pCyu55gmKw2hrOZ3F", + "g8K2Cbupv7aoIcR9RQGCA6Qlu4mqhsrHDcJ4rKiITcYemD3oxCb8XBphFSrNfpVyBN1QjDKe3q2m/Tu4", + "RlxCPbCgzRTfJIVcpjscmnh8YcCcolIYThsqMmFqIdFo18AH06++WDuez37ZOsggx41dpZkBzWO88C8S", + "HevBtBPFzwUNLG1mjYdyInCrnRjH2GGLU836hawDzTrVbrFA5C6LaUhlvCoWnHLF5c3iqxx5ij7lcYwS", + "gplQeosaQWk1ziiCyA7SdTDzPYSybZxhbyBy7gVKhJjOSbgK46GexdOi/eZj+h7qKtyGBG5DAoeEBDZI", + "vXk/NvgpTg/K0piGqyLKDc1Wjqo9T5tSu+rT90uVylZgpqMPtAz1i8SUXZZXgwEb8WvRvmGKKMFzh+3Q", + "CU7ThT+XgY5+qgZzwYUnpow08AI/esdRX7oSIjxT0gIA+KqCh5YUEXNKjIul7aVXm/OkRPbG00w8F1YB", + "fjclhMFeFdOiPxtE1a7FcwhFjHSMaoPZj+FiXYkf4tT3OPX0Mebs5Zgw98TFQw1nX/bPTaI2L/b6smgU", + "zE+tRhQYfTTs+ZbjrOCjo0AMezFpe/TK9sok3vDUj25A51CW1m7x/9S09Q97Ehlm+WdBorOwJSNHl2V/", + "HqduViAb7qmFJBiL2wzpEL/W/kS33YyuOvrfzcOD2lbDeadh/giHS19Us3ZUGyvaDxnwN/Xbj+On6MRG", + "h0ehc1A/Ij72+BDah/xzxkGPiE52VELn3JR74Wy1Q1gO1bpHw+FE1RuGPy73V1/eGBtTAS1IhCIipEmK", + "arxmCw53QeOSQB9wuDTYU3rgjCCMjk6Oz9EsTsNrnRgAfQ3+cwr/2327/zX4cYIwmmFO0MkZwlEEA9Ya", + "QquUI2wv1PAOwDYidzjJYjIN0+RrMEFfg/81rfz04xQdmgXYpEo4vsUrgSS+JkjRIYmI2tX0hnAUEUbL", + "ptNRMSOAqLN8FtPwUuOkN2L3QocnI1rh+ejz+alwXqWURgIdlwssvfoo1q9pm5Dn9r01yy13SShMl3tB", + "/Dt9XG6E9nuxVCKRZ1mqbjjQRU2NeB6PRWKCxbXJxfFzKjygW5QtUyHhVaq5FIK5Y0ZKowRE1RqEmtcF", + "3kRUAGSXnB6jMJjTdp5r20N9dbVXsITvGJqWHDOh2InGGYL8oTrxV4JluKRsYXfh58vLs131fxfFsqbo", + "F7KyHkg1XnmIcEanjTPSOCH2aMXwehWBxco6LCteKMsMduDhtGUeCsyM8ITqJMgVJ2XtcnDffjdzcdfk", + "1hUEufgxaLG4KPFVMAwTI9u83hZYH7e7BSxDlnPpztGyJsNUOza9WN2MzFMOnrJbzCPKFs1VLQmOCB93", + "hasCpqgLmWEUNJSptSnWoJgkpxHRT8INjCUZHrLSM677O8njFNsG3k3VerIYhySaInhArWk3i9VuaaDE", + "/0FCJ3blRKRxDq6GJc4ywoQx/u4IBYhBiCAsAgd5aqOx1yS/z5lSKtrcIJ8qD0WsCySHPprrFAFghcn8", + "XK9VVDKu2X2F017gKOPpDY1IVB1/in5NqJSapuGGicKYYC4QlVNvMNJWoj+eRH/Vj5ZeoQbwEuWxc4gr", + "0T/m+KqdtUd3Oorp/OraQuuuFuAtiqhZ01wP1weU5sAAbCCQSVYMjkrv8+6KLX7sW3LD2OwEUdcMrvH8", + "Ic9bmhlFsC/lxxkuE3609fXnToPxOgzGRPxG5bI1qVrhDO4i2GEeJE5DW6DAiS0qxocrnQnK8T8QN1Fb", + "jQRteRybg2q31sb2OEmZwzjX8pPgRLcGCwpmkLTYMGf1cUfE+WI3We3YUQ5u9n8cdcBtx4E+ri5gl5CZ", + "eYo+Kz5VQL0LXnV9ZrAWLrdYlMK1azHmuqYElFwSfksFUdw6FmiGw2urXnB8W8JzcmxGxLPwzf7bYohp", + "Lw06mJiY7fOR4iXBiefmDgVkPHzDZHW0rnu1Tm+SU3Fsr6Zdrh0gCONxMyurDemImyHJEv3QlIVJ+j2C", + "vhEaDjlTysQcc4Msd9VXBrPbVKStoZF/+kyihnq82WwfKUFEmDJzJ7hwZUkzbUIZcl92ccKPa8d9gHne", + "fah17lUIvGUMtBsb6jNps+ggs/3W/ttn//XQgWePLOUBF2jwLJKYKLM+DH9QDe3CcwGR0r2Hcxh/MaP1", + "MBffadPQ6xWakDd/wJw9Ij3x07rpA+otmNoKvX4xrYxXwlUUJ1Sd5bCzOKJMhWLKTjERU0BBHX+dK6Ur", + "XHBW1ino47J2C5zSButG1/VI0vJyU8FeGdbzTOJ0/fx2a8e5YSEvMnzLRiMLiOJhkneNMLmWG8cn97JR", + "gPlDXT/XcOoFFd+icXeJDNwtfRqsvTgIpNuDJRMynJSeldnKo106qq1Q+7IuJ6jvTIefdK3oON950EbJ", + "9QhJd10zQMaNhSsLUg4IeTOb6TIMdxnuEa+flcr+VNh29TxOCgFiqbfKFF3hA/KjPQhlc6T3WDTRtVFm", + "Ne76gfM/vKLSkIvGk0oVLSDXESmblwBzyqhYjluV7TN4WeuwevEQpWEwKyoX9XA+VLKe4jF5K1/x8KbG", + "SfiJxuRzFqfYcyYyToT3FbPLDOY0BkaAY/3A03Sy5vzQBHQ0z3/OPeE9n3nsPC2AscuIgRzgBFdiL54s", + "7I0F+62Maxz/pl1haC0sgGPdGL7ewlcDwghLAMb5MIoiYL0AVqqGPfSgbUJSeM6VP5izAuNpuhAPCuh8", + "SlJoC+asrKC1QsyDX5Ou82ApDa8JV6feE61YfHOMQu3TryMNgIEdJZHvVbFibeGShNfwIgjrV/fkjoS5", + "rv5Y0YvKJ8ytzAIMTt65wCrySLM8sv3Z2Z82Qvqy/zJIaZ39d7E19uXeIPxpRLSi7m0n6gaYherInKLj", + "otsEYpt0pAoTkuBo+py4Hl6AaoqOMDPuMYIwOO7A+hymccqQIBmGFD1FqEWy2rF9vwbqZlL56eDmDURb", + "nMxhJCrs0BFELtiAO2lrxgkbxgjzug43ex7xQiBgudPxtbL8JWqLAz2iUO3j026dTIEx+c73PC0Sk3c9", + "Q3a1xttlGlvFuFTwYCDgeTxniJMF5lFMREHX7crk3FYV8vA69bMtioIFBPKIphBpZ6JzX8WiLjpvljgy", + "o7gG4LprxUDxADi/P/ElJMl6C8uakB9o2zVf40wN0UQvJMm8mpXHnd3UXXuS+zRAszEp8LcOSrnF1OSd", + "sflw2sscWBBOyQKHqx4vw9an8Og6x9Yj8J16BLb2+K09fj17vKvrGzXf2gta1f0N+4GfnpeOcai9UD9Z", + "hxavVfiynP2DlPhNGrqKg9B0HhW1dyt6kL+uvVfPUrckCCtvxPHzXpvYIV/kieLFZZ4eNfsYREJ6t5+x", + "8ATVql8tBqFZ8e7Tmal5Bxh/xVFDPcrdprueZDvUvvKO7p5e4sXDDeGK/NOQwl3ZPPigQt2JBznIBitM", + "5u5tz9rwaCm88MedqRH9xaHraIOlOBKmFi6ncanf17QaUzfFqO49ILVZp587zMETuV7lOb9RuSxToz+/", + "oOzI0G5Ss3sM06Num9o37UvcvpGbxXOq5dugm62SPyiWw6eutGny/dq75jiaVa5RKYfcaou6Pfijy+U8", + "Qq2clvRg3l0/HvmcqBhq0l5XRy/hkEVr12VrX4psydeXK0UDsvU5xa7hUSKGNM1aFdHjdmVYekJUWeCv", + "fA/noHQWlasLJQA0mpwEhWp5oIsRzAn/yZ4bzWH+sOXrQHgAZ4FmJYBLKcFmeRgllFUGpGpl+gWzBfMg", + "+L870HDnsloWz7wtUePAv/rGODvZ+cWlfKd/Lpdn2mnC3wOsj7VAZ2A4vN0gqiY7sEENEC/yDM+wIG+G", + "oMs2bseYbbE/AKpytIq4sIMpaqEmZEVSqSRo8GH/veI0TiWIg2Bv+ma6B3noMsJwRoOD4O10b7pnHgMC", + "ie1qBO8AgrVS5U1GcaQzaGNIGl8rmqhONjwIOomCg+AsFdIhXBHoM0GEfJ9GK/MQRJoYIkgEoBN77P7T", + "BJJoDag3TXm19GPtPaExY3Kj6cLC9vfePNrsR0a61CHoyCFqBJJjMImBMN5psHyzFeDvqkb3k+Ave3v9", + "bVUjl6GAKdhHzb9f3U++tR7F36/ur6yR4/egSiZXavwq6ex+wyUyTo7vNQnFxOfZOobfEWbdlKSbubR0", + "6E4BZMxxQiTkgGixd5dNdisAgt27Rh/vetLA6vU8bAvf6Vn62r57gdutRMKu0mDF7jftW77f1RrTbohZ", + "qBMhtrAP+K7flFO2k/FUP7PHLEKZSWhRuzDp3ACQJURzQQ+bAWmmALoEcPTNUM/VpA7PwzUgImDC8Dq6", + "YMHFo8oqR5k43KHv1U+TuvYejfvAumGxeq3nROSx9HGgC4dSkd6kuCgb9jJJt66iaMIUeZJgvipICSjJ", + "oRhcXK0sBathuii3eOi3C8kEWkn3Fxobwm0+QVyDRos3b7/YFAbfM5Ga1aq1DiRStRlu7vnXSaRqwR6a", + "6abSjO5ckxVsxIK0ZTJRg8JLdXOvFA2q+zuRWj3XqtcDtnegoaq4Ijfted17XaRnbi7qmRUjr77eJ0Db", + "bh810Wq3Wd31ByjbLl78HMbZ7CfRs90dfhY1uw6Ah0lWchO8MC17M8TkspDdb/quOFAP76Yxo4ZrKjs0", + "445Xvm3HYXp3ZVNfu969MW6CZehxb2pTZt82n6nOj7zLj8+OGmbZQRxpr4fAjIF4S2C9HEZXt2tVUX6G", + "zzqs1KeY6O/BIA6wNMUAQ3Ct2LJ6o3YFiGOXpREZoFXpZh6gP5kPnap6V2U+UN7/lRNI6WG090oN2I0p", + "7IVGNyxIEOrj3189SK/TaN2YCPZr5j59GwDb/ab+Y+Sklz7+TqSpa8nmaSt5fIJRRvNLPXmgDuprJKc+", + "2jHFwAbTS1FV9xVd+uqk1arUQ7VPJIpwb2wLCDdV+scgqScSwI3ypfdGAg/h53CODAYgGgGGeA1ydzhb", + "qSRK7JY4tQS6funjJknqlEBFXQ1gDTqETaZoTmNZLV5Eyjy1uSD8v/As/Jrv7e3/FWfZf2U8jeAxFKTE", + "VcoRZhG60bmLk1xINCPo8/kpIixMTRZQH0MqCou5/OiZxdkpJP22xVIfKNeamwfEuDeEGPc2KA8dT63W", + "CDeuQlZTe/aYPBr1eZ0glCajdA/HE1k/CnLZrOmjMq1HM3aK2bXbPLbE2EmMFXa9m5Spb9vZtltMX0e5", + "D2PeNq9uDw8/SpME75j3qySCMgVOVkJ0cgzv2RakAkkwCchdFittwsYg+1iyGeQPGolOr0F7fFyC7070", + "xzd7ezXmOQlyRv+VE9MAzseTKpjevMUPY+E6RsgSwvYIjT1C34qKkJ12R+0dcRJv+wyOxfZeOFUmx6nC", + "ZX3KgUbHGmO1vqiXr52+NCHfepMuBfxsheBO2s4zn2jjH50DrXPLtbS/JacH8JjdMGWM6KJ7frXyHHAu", + "CqKLdAJlnerBOe1UmKLnlYQPurZrNEWXl6eqCQR1kztJmLn4dCikBfEeGRgfSsOPr9wayEYpuHvPoeDa", + "PGK2LsP95LlUbUMRG1O1t+e9ct5t9qxCvHQ/81ECRzjlUaF8Atq92e8Y3s2EM0A0neqEYGsf7Yk3VwaU", + "fPCU8xVILrF0XhAWMoUylNA4piZ/d4tRBlJ0+C3E9s1IQhlN8iQ42PMl+W7YnPCdau2kbO+CsgWqmCa0", + "ClXxovLNnrpnNFJAdAK5AYkPu76OvNcJ9rZM4CFMoO+e7p76pLh2DzjLrXf0BxznIrW+Psrl62XMpT3Y", + "8EjlBscTdYrNAZ5AU13wsUzZ/4Tn2jcsgZIH7rEcsDTCovUWNg7kq03EttVqFq1rLnYZwAaMC1t+UeEX", + "pnqcfnMtW8NjmoXmbNHDDVQ79NQ6rN0tcg/DMjXfXuDVoqPS5KBrhsc4UzlaNmDHPVnezXvKs/Nu729D", + "2v7tT3LOdDG+1qv4mfpcK7g35P4M/TZu+tPWgMo9DxyhpjqouQduqWtj1MXJnBOxJKLL2ANNKkxBW2sU", + "i6VSmDpeKYrpDRlIfufFvM/DZasPpyPL1pqxvK60KtUri4fyMnZNMomwwoCjlUHprzutbb39q7pzdeuL", + "jffL9+N5uN3RDRlEXzHlC5sPoCD7brvHOfRYg9fqji9Qn9CARS/fF99uINxKiQ2cFVtndMhL9oxwQYWE", + "4oK29GoRdWLG/J+i0OyFhMTTtjKtsKqAjbPSURjwZKf0J+v3rTAPmpFVyjSzSzldUIZjZ5qYzokST0Pd", + "CgUcL0Iu6cSKFdOdjzEVeXPSAiu6Z4FdInXJcNWk2JWIimuTUYdFiJNZmkqUMsTIndSmlOmIIq2/Zrp0", + "cTVNS6P6LviIsKcqb5nSDAawzgkolS0m5tWWqe9iHEn1fIBFWW2n1C9YnaHoJZRXhRL4aqSUEX993YYI", + "3ihLdAs0+wwhFm2bDk7aGj8qDFGWRdf97FCdOAiz0w3L82DNHBUJR6EmfEY5QXdWd3RC9WiZwMfwlSk6", + "wnGsS1hTgRIil2mEkjyWNIuJLe5+Q/gtp9LYUi4vTyeI4FAXk0W5sBWwLSsujY5YlOZU1SpLqfqeooRg", + "kZuSQXZpVnkeymJtwfqXwGBlW/F8A6Sjy5f74eLL5FZvvRnoXQ3GOluaZV8VlFePckEQhjQtpHb0LUcY", + "yBFcZag7Er1oWq+u3HSbtOsdHTGHtXcptbr9OkK9hGG2QiLNeUicuEefCOw9iRlWapaa5hR8jKO6fCJ3", + "0qT22YyzoSJS1/U1lJv+pwpJLFatSR9yfQxLpOB1Dl6aD5t8nQDJvh74KEEvaHM7X09Q17X9I7IPqd+c", + "jSyTtgzx/boh2l2czEnKsq7n16Rg2bp9vy+3ryKKx/D5wnv7jTh83w5p+/bVs/lexrCb4LtO5gC0Z4KW", + "fIzCJn3XbzssJQ9jHx/x3ZaDvHgOMvG8m+Q0hCz86l/khlSoBJ4+mlc2LQ8dOeQgbn9QY2s4hSkzV7I/", + "3FdD9l0ObMYfHEviKeX0pGFtH/Gdy/O2PO6l8DhtHRykydqmXlZVfhxwZysSy7Ud4MHlAq82rUGbp6IP", + "1qItvp7xDjVKt35CeiyRUX292+2CrKUw63jC6xLnU7gOvaWNB1nL9x8dBlOFr8WPWNaEx2FIMmnjS17c", + "E8QXSJkVdrlrygHsfoN/tOd2OYLyl3Rec+5o5U+XkdBunU6uaqqJwH9aOGw1cyc2Lds1hpZKoLbjJnUD", + "Ozksrygd08dPlUjReVFtCaiVgX40SQ+X+38aS2875ZdFyTpT/pUOGLywDoNWDUL3Kcj9Ei+eildXZ1IT", + "jWLY71rKsbXnD9z6IB9Gh7YmnN+3eGgqWePFD+JHKG3drITXoRA8IaFpyNYmtDePDAiJXFC8+gFelFEE", + "Wzp+GjqustJvZamhoVlUW9TcOgOtlDAaaS4qug6Pa65UYHqMXKrf/zWn+7JdyzvfsuuuivhIW76m+3IN", + "R+kmNEqnmN9Ik5OrrFMp1qxn8N2ogJXMv933cvvYqfVGrgZ6Ek71dDf7aqHNtRMCN6rqtSYFfvk5gV63", + "8eicmDqybKDp6HXQ6eu1QG2tSnWF0BY++maqAN+PeZ8BOR7KeslDaVvLyvdlse4n1CNscWOPIrDv552a", + "dpZYaB/llnQGk04zrUnDa10tpB7rpCND1M0KzayVrGRNutlsYpMw5wI8tK8ps4kv1tY+wnvT/wbP77yL", + "KCchrGEyUDQoqjguerUOHJMbqKo2eNBT6OBB7YWO7Ryy+3OeJm2hCzDKqFXqiTdkL4czp2YdbDP3X3Kc", + "I/8yzTd+5vuireTd7FgnKB/DkNtKA/QxZJ1I/dlY8gmLyJ09h8UbioLgWk9lkdfCLWrvYxnpQvw6nwvS", + "wgNHp3b6brj02sx0Y5yr9WFXL8fasqknYVNzGqufllgsu4uVYIbyLE5xhGLKrq2VEnOkRoDK55gy56Dj", + "FdHfhuqUP6m2P2OxfCjj8rjGl3rYoZ5xBYVlYHYJ/c7xN09zZBRePgPm2+7X7r7cLgmHZ67mRzhCZpe+", + "A6fOaz9u1pHeE3YH7vN1/ADGufmY7p8niYQv3JIPDYU3ChPg9QljP/+MPks3oemAt35dBQu+7H/P9WYm", + "bU8PC0BnK5QyglKOkpTrWkWAiUH1FaQ+/uslGbyQRneqHrJJIOQqVj8oNfQ1uQC3xXle46PhzuTIg3Ku", + "tlknHdbyStMnv0orY98Fdm8szIWtcAhmW0B+FCtlDZV6EWC4gOu1edGTczZFqjeakTi91XkodAPMCSJ3", + "YZxH7bh9NKvnERZkRxAmqKQ3BIl8psUSSrAMlyhlAHlChMALfU1TXLZF0hDMw2UFrATfnRK2UAd8/y9/", + "3WyIsJMV+8v+eubObX7s9Xh15YnQ4z/O+LL/HM8zvuy/dPe4wcS26Np6d26XcBtxlt1VzLsjlxx6/b5j", + "l54EiHbGvQ2Oes5T0RNsMja0xHtIni+45IllCmBklER5WbEtL5B7v21TO9ZUMt4+i5Lx9rmUDAOA5bcW", + "kK2+8XQUm8Z5QgYmhEK2tc+GUXx6etu7nmu02T0GO1pzNX+m/bdrHlBvWnOnAk9+BuXs+pPUmLZbvdm3", + "Z3rWQxYZG3EPYdlMrk2cbdlQHxk6TGj3m/7H8Fdl7cSpGxny/GKGHa28WXgGPimrEIV9ToabBLE14HTy", + "pY7YsgKRrYFlT7nle8/FYGz6pS01jWYvAB2/sbuf8zg4CJZSZuJgdxdndEr2Z1OcZYHT/1uZ76dMd/Ot", + "lri1+iPkJnL/ht3bkWrB1YYZ3bkmq8pvxvNf/F0oJlf3/x0AAP//", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/packages/api/internal/handlers/proxy_grpc.go b/packages/api/internal/handlers/proxy_grpc.go index 605f547b39..d653f41a3b 100644 --- a/packages/api/internal/handlers/proxy_grpc.go +++ b/packages/api/internal/handlers/proxy_grpc.go @@ -257,6 +257,7 @@ func (s *SandboxService) ResumeSandbox(ctx context.Context, req *proxygrpc.Sandb s.api.buildResumeSandboxData(sandboxID, nil), &headers, true, + false, nil, // mcp ) if apiErr != nil { diff --git a/packages/api/internal/handlers/sandbox.go b/packages/api/internal/handlers/sandbox.go index 4790692a49..b7f47fced0 100644 --- a/packages/api/internal/handlers/sandbox.go +++ b/packages/api/internal/handlers/sandbox.go @@ -28,6 +28,7 @@ func (a *APIStore) startSandbox( getSandboxData orchestrator.SandboxDataFetcher, requestHeader *http.Header, isResume bool, + rebootFromRootfs bool, mcp api.Mcp, ) (*api.Sandbox, *api.APIError) { sbx, apiErr := a.startSandboxInternal( @@ -38,6 +39,7 @@ func (a *APIStore) startSandbox( getSandboxData, requestHeader, isResume, + rebootFromRootfs, mcp, ) if apiErr != nil { @@ -56,6 +58,7 @@ func (a *APIStore) startSandboxInternal( getSandboxData orchestrator.SandboxDataFetcher, requestHeader *http.Header, isResume bool, + rebootFromRootfs bool, mcp api.Mcp, ) (sandbox.Sandbox, *api.APIError) { startTime := time.Now() @@ -77,6 +80,7 @@ func (a *APIStore) startSandboxInternal( timeout, isResume, creationMeta, + rebootFromRootfs, ) if instanceErr != nil { telemetry.ReportError(ctx, "error when creating instance", instanceErr.Err) diff --git a/packages/api/internal/handlers/sandbox_connect.go b/packages/api/internal/handlers/sandbox_connect.go index 8128da2547..813f9cba63 100644 --- a/packages/api/internal/handlers/sandbox_connect.go +++ b/packages/api/internal/handlers/sandbox_connect.go @@ -54,6 +54,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S return } + rebootFromRootfs := body.Reboot != nil && *body.Reboot teamID := teamInfo.Team.ID @@ -158,6 +159,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S a.buildResumeSandboxData(sandboxID, nil), &c.Request.Header, true, + rebootFromRootfs, nil, // mcp ) if createErr != nil { diff --git a/packages/api/internal/handlers/sandbox_create.go b/packages/api/internal/handlers/sandbox_create.go index 66434a3af1..897df8573e 100644 --- a/packages/api/internal/handlers/sandbox_create.go +++ b/packages/api/internal/handlers/sandbox_create.go @@ -148,6 +148,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { ) autoPause := sharedUtils.DerefOrDefault(body.AutoPause, sandbox.AutoPauseDefault) + autoPauseMemory := body.AutoPauseMemory envVars := sharedUtils.DerefOrDefault(body.EnvVars, nil) mcp := sharedUtils.DerefOrDefault(body.Mcp, nil) metadata := sharedUtils.DerefOrDefault(body.Metadata, nil) @@ -257,6 +258,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { TemplateID: env.TemplateID, BaseTemplateID: env.TemplateID, AutoPause: autoPause, + AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, VolumeMounts: sbxVolumeMounts, EnvdAccessToken: envdAccessToken, @@ -271,6 +273,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { getSandboxData, &c.Request.Header, false, + false, mcp, ) if createErr != nil { diff --git a/packages/api/internal/handlers/sandbox_pause.go b/packages/api/internal/handlers/sandbox_pause.go index 5040682997..be7227f24c 100644 --- a/packages/api/internal/handlers/sandbox_pause.go +++ b/packages/api/internal/handlers/sandbox_pause.go @@ -18,10 +18,15 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/auth/pkg/auth" + "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) +type pauseSandboxBody struct { + Memory *bool `json:"memory,omitempty"` +} + func (a *APIStore) PostSandboxesSandboxIDPause(c *gin.Context, sandboxID api.SandboxID) { ctx := c.Request.Context() // Get team from context, use TeamContextKey @@ -49,9 +54,22 @@ func (a *APIStore) PostSandboxesSandboxIDPause(c *gin.Context, sandboxID api.San traceID := span.SpanContext().TraceID().String() c.Set("traceID", traceID) + memory := true + if c.Request.ContentLength != 0 { + body, err := ginutils.ParseBody[pauseSandboxBody](ctx, c) + if err != nil { + a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Error when parsing request: %s", err)) + + return + } + if body.Memory != nil { + memory = *body.Memory + } + } + pause.LogInitiated(ctx, sandboxID, teamID.String(), pause.ReasonRequest) - err = a.orchestrator.RemoveSandbox(ctx, teamID, sandboxID, sandbox.RemoveOpts{Action: sandbox.StateActionPause}) + err = a.orchestrator.RemoveSandbox(ctx, teamID, sandboxID, sandbox.RemoveOpts{Action: sandbox.StateActionPause, SkipMemory: !memory}) var transErr *sandbox.InvalidStateTransitionError switch { diff --git a/packages/api/internal/handlers/sandbox_resume.go b/packages/api/internal/handlers/sandbox_resume.go index 6ee26259e2..d2d78228fc 100644 --- a/packages/api/internal/handlers/sandbox_resume.go +++ b/packages/api/internal/handlers/sandbox_resume.go @@ -60,6 +60,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa } telemetry.ReportEvent(ctx, "Parsed body") + rebootFromRootfs := body.Reboot != nil && *body.Reboot timeout := sandbox.SandboxTimeoutDefault if body.Timeout != nil { @@ -166,6 +167,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa a.buildResumeSandboxData(sandboxID, body.AutoPause), &c.Request.Header, true, + rebootFromRootfs, nil, // mcp ) if createErr != nil { @@ -233,8 +235,10 @@ func (a *APIStore) buildResumeSandboxData(sandboxID string, autoPauseOverride *b var network *types.SandboxNetworkConfig var autoResume *types.SandboxAutoResumeConfig var volumes []*types.SandboxVolumeMountConfig + var autoPauseMemory *bool if snap.Config != nil { network = snap.Config.Network + autoPauseMemory = snap.Config.AutoPauseMemory autoResume = snap.Config.AutoResume volumes = snap.Config.VolumeMounts } @@ -248,6 +252,7 @@ func (a *APIStore) buildResumeSandboxData(sandboxID string, autoPauseOverride *b TemplateID: snap.EnvID, BaseTemplateID: snap.BaseEnvID, AutoPause: autoPause, + AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, VolumeMounts: convertDatabaseMountsToOrchestratorMounts(volumes), EnvdAccessToken: envdAccessToken, diff --git a/packages/api/internal/handlers/snapshot_template_create.go b/packages/api/internal/handlers/snapshot_template_create.go index ab0a19f2dd..501bc45ab3 100644 --- a/packages/api/internal/handlers/snapshot_template_create.go +++ b/packages/api/internal/handlers/snapshot_template_create.go @@ -65,6 +65,9 @@ func (a *APIStore) PostSandboxesSandboxIDSnapshots(c *gin.Context, sandboxID api opts := orchestrator.SnapshotTemplateOpts{ Tag: id.DefaultTag, } + if body.Memory != nil { + opts.SkipMemory = !*body.Memory + } if body.Name != nil { identifier, tag, err := id.ParseName(*body.Name) diff --git a/packages/api/internal/orchestrator/create_instance.go b/packages/api/internal/orchestrator/create_instance.go index b1b9fb0445..0dde32f8f4 100644 --- a/packages/api/internal/orchestrator/create_instance.go +++ b/packages/api/internal/orchestrator/create_instance.go @@ -10,6 +10,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/api/internal/api" @@ -46,6 +47,7 @@ type SandboxMetadata struct { TemplateID string BaseTemplateID string AutoPause bool + AutoPauseMemory *bool AutoResume *types.SandboxAutoResumeConfig VolumeMounts []*orchestrator.SandboxVolumeMount EnvdAccessToken *string @@ -127,6 +129,7 @@ func (o *Orchestrator) CreateSandbox( timeout time.Duration, isResume bool, creationMeta sandbox.CreationMetadata, + rebootFromRootfsOpt ...bool, ) (sbx sandbox.Sandbox, apiErr *api.APIError) { ctx, childSpan := tracer.Start(ctx, "create-sandbox") defer childSpan.End() @@ -252,6 +255,10 @@ func (o *Orchestrator) CreateSandbox( TimeoutSeconds: sbxData.AutoResume.Timeout, } } + rebootFromRootfs := len(rebootFromRootfsOpt) > 0 && rebootFromRootfsOpt[0] + if rebootFromRootfs { + ctx = metadata.AppendToOutgoingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey, "true") + } sbxRequest := &orchestrator.SandboxCreateRequest{ Sandbox: &orchestrator.SandboxConfig{ @@ -347,6 +354,7 @@ func (o *Orchestrator) CreateSandbox( node.ID, node.ClusterID, sbxData.AutoPause, + sbxData.AutoPauseMemory, sbxData.AutoResume, sbxData.EnvdAccessToken, sbxData.AllowInternetAccess, @@ -365,7 +373,7 @@ func (o *Orchestrator) CreateSandbox( // Copy to a new variable to avoid race conditions sbxToRemove := sbx go func() { - killErr := o.removeSandboxFromNode(context.WithoutCancel(ctx), sbxToRemove, sandbox.StateActionKill) + killErr := o.removeSandboxFromNode(context.WithoutCancel(ctx), sbxToRemove, sandbox.RemoveOpts{Action: sandbox.StateActionKill}) if killErr != nil { logger.L().Error(ctx, "Error removing sandbox", zap.Error(killErr), logger.WithSandboxID(sbxToRemove.SandboxID)) } diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index 1768106189..67340768fd 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -94,7 +94,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand defer func() { go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) }() // Once we start the removal process, we want to make sure it gets removed from the store defer o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID) - err = o.removeSandboxFromNode(ctx, sbx, opts.Action) + err = o.removeSandboxFromNode(ctx, sbx, opts) if err != nil { logger.L().Error(ctx, "Error removing sandbox", zap.String("state_action", opts.Action.Name), @@ -108,7 +108,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand return nil } -func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sandbox, stateAction sandbox.StateAction) error { +func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sandbox, opts sandbox.RemoveOpts) error { ctx, span := tracer.Start(ctx, "remove-sandbox-from-node") defer span.End() @@ -131,12 +131,13 @@ func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sa sbxlogger.I(sbx).Debug(ctx, "Removing sandbox", zap.Bool("auto_pause", sbx.AutoPause), - zap.String("state_action", stateAction.Name), + zap.String("state_action", opts.Action.Name), ) - switch stateAction { + switch opts.Action { case sandbox.StateActionPause: - err := o.pauseSandbox(ctx, node, sbx) + skipMemory := opts.SkipMemory || (opts.Eviction && sbx.AutoPauseMemory != nil && !*sbx.AutoPauseMemory) + err := o.pauseSandbox(ctx, node, sbx, skipMemory) if err != nil { if dberrors.IsForeignKeyViolation(err) { killErr := o.killSandboxOnNode(ctx, node, sbx) diff --git a/packages/api/internal/orchestrator/nodemanager/sandboxes.go b/packages/api/internal/orchestrator/nodemanager/sandboxes.go index 658f27eaed..074fd216d2 100644 --- a/packages/api/internal/orchestrator/nodemanager/sandboxes.go +++ b/packages/api/internal/orchestrator/nodemanager/sandboxes.go @@ -131,6 +131,7 @@ func (n *Node) GetSandboxes(ctx context.Context) ([]sandbox.Sandbox, error) { n.ID, n.ClusterID, config.GetAutoPause(), + nil, autoResume, config.EnvdAccessToken, //nolint:protogetter // we need the nil check too config.AllowInternetAccess, //nolint:protogetter // we need the nil check too diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index c1863cebd8..f3111d5708 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" "github.com/e2b-dev/infra/packages/api/internal/sandbox" @@ -26,18 +27,18 @@ func (PauseQueueExhaustedError) Error() string { return "The pause queue is exhausted" } -func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox) error { +func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, skipMemory bool) error { ctx, span := tracer.Start(ctx, "pause-sandbox") defer span.End() - result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node)) + result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, skipMemory)) if err != nil { telemetry.ReportCriticalError(ctx, "error inserting snapshot for env", err) return err } - err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String()) + err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String(), skipMemory) if errors.Is(err, PauseQueueExhaustedError{}) { telemetry.ReportCriticalError(ctx, "pause queue exhausted", err) @@ -68,11 +69,14 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, return nil } -func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string) error { +func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string, skipMemory bool) error { childCtx, childSpan := tracer.Start(ctx, "snapshot-instance") defer childSpan.End() client, childCtx := node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID) + if skipMemory { + childCtx = metadata.AppendToOutgoingContext(childCtx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey, "false") + } _, err := client.Sandbox.Pause( childCtx, &orchestrator.SandboxPauseRequest{ SandboxId: sbx.SandboxID, @@ -103,14 +107,13 @@ func (o *Orchestrator) WaitForStateChange(ctx context.Context, teamID uuid.UUID, return o.sandboxStore.WaitForStateChange(ctx, teamID, sandboxID) } -func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node) queries.UpsertSnapshotParams { +func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, skipMemory bool) queries.UpsertSnapshotParams { machineInfo := node.MachineInfo() metadata := types.JSONBStringMap(sbx.Metadata) if metadata == nil { metadata = types.JSONBStringMap{} } - return queries.UpsertSnapshotParams{ // Used if there's no snapshot for this sandbox yet TemplateID: id.Generate(), @@ -131,10 +134,11 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node) quer AllowInternetAccess: sbx.AllowInternetAccess, AutoPause: sbx.AutoPause, Config: &types.PausedSandboxConfig{ - Version: types.PausedSandboxConfigVersion, - Network: sbx.Network, - AutoResume: sbx.AutoResume, - VolumeMounts: sbx.VolumeMounts, + Version: types.PausedSandboxConfigVersion, + Network: sbx.Network, + AutoPauseMemory: sbx.AutoPauseMemory, + AutoResume: sbx.AutoResume, + VolumeMounts: sbx.VolumeMounts, }, OriginNodeID: node.ID, Status: types.BuildStatusSnapshotting, diff --git a/packages/api/internal/orchestrator/snapshot_template.go b/packages/api/internal/orchestrator/snapshot_template.go index 4d08816d1e..e1aa972ba4 100644 --- a/packages/api/internal/orchestrator/snapshot_template.go +++ b/packages/api/internal/orchestrator/snapshot_template.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/db/pkg/types" @@ -30,6 +31,8 @@ type SnapshotTemplateOpts struct { Namespace *string // Tag is the build tag parsed from the name, defaults to "default". Tag string + // SkipMemory controls whether the snapshot template should omit VM memory. + SkipMemory bool } // CreateSnapshotTemplate creates a persistent snapshot template from a running sandbox and immediately resumes it. @@ -67,7 +70,7 @@ func (o *Orchestrator) CreateSnapshotTemplate(ctx context.Context, teamID uuid.U return SnapshotTemplateResult{}, fmt.Errorf("node '%s' not found", sbx.NodeID) } - upsertResult, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node)) + upsertResult, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, opts.SkipMemory)) if err != nil { return SnapshotTemplateResult{}, fmt.Errorf("error upserting snapshot: %w", err) } @@ -84,6 +87,9 @@ func (o *Orchestrator) CreateSnapshotTemplate(ctx context.Context, teamID uuid.U // kills the sandbox itself; RemoveSandbox is still needed to clean up // API-side state (store, routing, analytics). client, childCtx := node.GetClient(ctx) + if opts.SkipMemory { + childCtx = metadata.AppendToOutgoingContext(childCtx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey, "false") + } _, err = client.Sandbox.Checkpoint(childCtx, &orchestrator.SandboxCheckpointRequest{ SandboxId: sbx.SandboxID, BuildId: upsertResult.BuildID.String(), diff --git a/packages/api/internal/sandbox/sandboxtypes/sandbox.go b/packages/api/internal/sandbox/sandboxtypes/sandbox.go index ee0c9256c4..9f89d9fd6b 100644 --- a/packages/api/internal/sandbox/sandboxtypes/sandbox.go +++ b/packages/api/internal/sandbox/sandboxtypes/sandbox.go @@ -31,6 +31,7 @@ func NewSandbox( nodeID string, clusterID uuid.UUID, autoPause bool, + autoPauseMemory *bool, autoResume *types.SandboxAutoResumeConfig, envdAccessToken *string, allowInternetAccess *bool, @@ -66,6 +67,7 @@ func NewSandbox( NodeID: nodeID, ClusterID: clusterID, AutoPause: autoPause, + AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, State: StateRunning, BaseTemplateID: baseTemplateID, @@ -101,6 +103,7 @@ type Sandbox struct { NodeID string `json:"nodeID"` ClusterID uuid.UUID `json:"clusterID"` AutoPause bool `json:"autoPause"` + AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` AutoResume *types.SandboxAutoResumeConfig `json:"autoResume,omitempty"` Network *types.SandboxNetworkConfig `json:"network"` VolumeMounts []*types.SandboxVolumeMountConfig `json:"volumeMounts"` diff --git a/packages/api/internal/sandbox/sandboxtypes/states.go b/packages/api/internal/sandbox/sandboxtypes/states.go index 8dee7265c5..76d52883fa 100644 --- a/packages/api/internal/sandbox/sandboxtypes/states.go +++ b/packages/api/internal/sandbox/sandboxtypes/states.go @@ -48,8 +48,9 @@ var ( // RemoveOpts bundles the parameters that control sandbox removal. type RemoveOpts struct { - Action StateAction - Eviction bool + Action StateAction + Eviction bool + SkipMemory bool } var AllowedTransitions = map[State]map[State]bool{ diff --git a/packages/api/internal/sandbox/store_test.go b/packages/api/internal/sandbox/store_test.go index a40b323b46..97793e97e0 100644 --- a/packages/api/internal/sandbox/store_test.go +++ b/packages/api/internal/sandbox/store_test.go @@ -206,6 +206,7 @@ func createTestSandbox() Sandbox { "node-1", uuid.New(), false, // autoPause + nil, // autoPauseMemory nil, // autoResume nil, // envdAccessToken nil, // allowInternetAccess diff --git a/packages/db/pkg/types/types.go b/packages/db/pkg/types/types.go index 996dd1523f..d551ea2c03 100644 --- a/packages/db/pkg/types/types.go +++ b/packages/db/pkg/types/types.go @@ -109,10 +109,11 @@ type SandboxAutoResumeConfig struct { } type PausedSandboxConfig struct { - Version string `json:"version"` - Network *SandboxNetworkConfig `json:"network,omitempty"` - AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` - VolumeMounts []*SandboxVolumeMountConfig `json:"volumeMounts,omitempty"` + Version string `json:"version"` + Network *SandboxNetworkConfig `json:"network,omitempty"` + AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` + AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` + VolumeMounts []*SandboxVolumeMountConfig `json:"volumeMounts,omitempty"` } func (c PausedSandboxConfig) Value() (driver.Value, error) { diff --git a/packages/orchestrator/pkg/sandbox/build_upload.go b/packages/orchestrator/pkg/sandbox/build_upload.go index 5ad332dc57..f3ab462c9e 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload.go +++ b/packages/orchestrator/pkg/sandbox/build_upload.go @@ -41,9 +41,14 @@ func NewUpload( useCase string, objectMetadata storage.ObjectMetadata, ) (*Upload, error) { - mem, memV4, err := resolveCompressConfig(ctx, cfg, ff, storage.MemfileName, snap.MemfileBlockSize, useCase) - if err != nil { - return nil, fmt.Errorf("resolve memfile compress config: %w", err) + var mem storage.CompressConfig + var memV4 bool + if snap.MemorySnapshot { + var err error + mem, memV4, err = resolveCompressConfig(ctx, cfg, ff, storage.MemfileName, snap.MemfileBlockSize, useCase) + if err != nil { + return nil, fmt.Errorf("resolve memfile compress config: %w", err) + } } root, rootV4, err := resolveCompressConfig(ctx, cfg, ff, storage.RootfsName, snap.RootfsBlockSize, useCase) if err != nil { diff --git a/packages/orchestrator/pkg/sandbox/build_upload_test.go b/packages/orchestrator/pkg/sandbox/build_upload_test.go index bd783e4845..41df6dc532 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_test.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_test.go @@ -3,11 +3,16 @@ package sandbox import ( + "os" "testing" + "github.com/google/uuid" "github.com/launchdarkly/go-server-sdk/v7/testhelpers/ldtestdata" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build" + sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/storage" ) @@ -55,3 +60,37 @@ func TestResolveCompressConfig_V4_FlagOn(t *testing.T) { ff := newV4HeaderFF(t, true) require.True(t, resolveV4(t, ff)) } + +func TestUploadRunV3MemorylessSkipsMemoryArtifacts(t *testing.T) { + t.Parallel() + + buildID := uuid.New() + metaPath := t.TempDir() + "/metadata.json" + require.NoError(t, os.WriteFile(metaPath, []byte("{}"), 0o644)) + + store := storage.NewMockStorageProvider(t) + metadataBlob := storage.NewMockBlob(t) + store.EXPECT(). + OpenBlob(mock.Anything, mock.MatchedBy(func(path string) bool { + return path == (storage.Paths{BuildID: buildID.String()}).Metadata() + }), storage.MetadataObjectType). + Return(metadataBlob, nil) + metadataBlob.EXPECT().Put(mock.Anything, []byte("{}"), mock.Anything).Return(nil) + + upload := &Upload{ + buildID: buildID, + snap: &Snapshot{ + BuildID: buildID, + MemorySnapshot: false, + MemfileDiff: &build.NoDiff{}, + MemfileDiffHeader: NewResolvedDiffHeader(nil), + RootfsDiff: &build.NoDiff{}, + RootfsDiffHeader: NewResolvedDiffHeader(nil), + Metafile: sbxtemplate.NewLocalFileLink(metaPath), + }, + paths: storage.Paths{BuildID: buildID.String()}, + store: store, + } + + require.NoError(t, upload.runV3(t.Context())) +} diff --git a/packages/orchestrator/pkg/sandbox/build_upload_v3.go b/packages/orchestrator/pkg/sandbox/build_upload_v3.go index a5b2b25329..b095a60bcd 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_v3.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_v3.go @@ -15,9 +15,13 @@ import ( ) func (u *Upload) runV3(ctx context.Context) error { - memfilePath, err := u.snap.MemfileDiff.CachePath(ctx) - if err != nil { - return fmt.Errorf("error getting memfile diff path: %w", err) + memfilePath := "" + if u.snap.MemorySnapshot { + var err error + memfilePath, err = u.snap.MemfileDiff.CachePath(ctx) + if err != nil { + return fmt.Errorf("error getting memfile diff path: %w", err) + } } rootfsPath, err := u.snap.RootfsDiff.CachePath(ctx) @@ -28,6 +32,9 @@ func (u *Upload) runV3(ctx context.Context) error { eg, egCtx := errgroup.WithContext(ctx) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } h, err := u.snap.MemfileDiffHeader.WaitWithContext(egCtx) if err != nil { return fmt.Errorf("wait memfile diff header: %w", err) @@ -54,6 +61,9 @@ func (u *Upload) runV3(ctx context.Context) error { meta := storage.WithMetadata(u.objectMetadata) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } if memfilePath == "" { return nil } @@ -90,6 +100,9 @@ func (u *Upload) runV3(ctx context.Context) error { }) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } return storage.UploadBlob(egCtx, u.store, u.paths.Snapfile(), storage.SnapfileObjectType, u.snap.Snapfile.Path(), meta) }) @@ -103,9 +116,13 @@ func (u *Upload) runV3(ctx context.Context) error { // Body uploads done; headers must be ready by now (the per-file Goroutines // above already Wait-ed). Wait() is a fast lookup here. - memfileDiffHeader, err := u.snap.MemfileDiffHeader.WaitWithContext(ctx) - if err != nil { - return fmt.Errorf("wait memfile diff header: %w", err) + var memfileDiffHeader *headers.Header + if u.snap.MemorySnapshot { + var err error + memfileDiffHeader, err = u.snap.MemfileDiffHeader.WaitWithContext(ctx) + if err != nil { + return fmt.Errorf("wait memfile diff header: %w", err) + } } rootfsDiffHeader, err := u.snap.RootfsDiffHeader.WaitWithContext(ctx) if err != nil { diff --git a/packages/orchestrator/pkg/sandbox/build_upload_v4.go b/packages/orchestrator/pkg/sandbox/build_upload_v4.go index a243728c67..41969cb9e7 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_v4.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_v4.go @@ -16,9 +16,13 @@ import ( ) func (u *Upload) runV4(ctx context.Context) error { - memSrc, err := u.snap.MemfileDiff.CachePath(ctx) - if err != nil { - return fmt.Errorf("memfile diff path: %w", err) + memSrc := "" + if u.snap.MemorySnapshot { + var err error + memSrc, err = u.snap.MemfileDiff.CachePath(ctx) + if err != nil { + return fmt.Errorf("memfile diff path: %w", err) + } } rootfsSrc, err := u.snap.RootfsDiff.CachePath(ctx) @@ -29,6 +33,9 @@ func (u *Upload) runV4(ctx context.Context) error { eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } h, err := u.snap.MemfileDiffHeader.WaitWithContext(ctx) if err != nil { return fmt.Errorf("wait memfile diff header: %w", err) @@ -55,6 +62,9 @@ func (u *Upload) runV4(ctx context.Context) error { meta := storage.WithMetadata(u.objectMetadata) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } return storage.UploadBlob(ctx, u.store, u.paths.Snapfile(), storage.SnapfileObjectType, u.snap.Snapfile.Path(), meta) }) diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index 0074176a48..5806397603 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -103,6 +103,11 @@ type ProcessOptions struct { Stderr io.Writer } +// ext4RootFlags must not include "noload": filesystem-only reboot fallback +// relies on ext4 replaying the journal after a snapshot was taken from a +// previously running guest. +const ext4RootFlags = "discard" + // TokenBucketConfig holds parameters for a single Firecracker token bucket. // BucketSize < 0 disables the bucket. type TokenBucketConfig struct { @@ -374,7 +379,7 @@ func (p *Process) Create( "random.trust_cpu": "on", // discard: ext4 issues TRIM on freed blocks so they are elided from the snapshot diff. - "rootflags": "discard", + "rootflags": ext4RootFlags, } if options.KvmClock { diff --git a/packages/orchestrator/pkg/sandbox/reclaim.go b/packages/orchestrator/pkg/sandbox/reclaim.go index 270be40e8d..f941de0973 100644 --- a/packages/orchestrator/pkg/sandbox/reclaim.go +++ b/packages/orchestrator/pkg/sandbox/reclaim.go @@ -108,6 +108,27 @@ func (s *Sandbox) bestEffortReclaim(ctx context.Context) { } } +func (s *Sandbox) bestEffortGuestSync(ctx context.Context) { + const syncTimeout = 2 * time.Second + + rcCtx, cancel := context.WithTimeout(ctx, syncTimeout) + defer cancel() + + stream, err := s.StartEnvdSystemShell(rcCtx, "/bin/sh", []string{"-c", "sync"}, "root", syncTimeout) + if err != nil { + logger.L().Warn(ctx, "envd sync failed", logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err)) + + return + } + defer stream.Close() + + for stream.Receive() { + } + if err := stream.Err(); err != nil { + logger.L().Warn(ctx, "envd sync stream error", logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err)) + } +} + // envdSupportsCgroupFreeze reports whether the sandbox's envd exposes the // native /freeze and /unfreeze endpoints. Bad version strings log and return // false so we never accidentally call an unsupported endpoint. diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index f57fa26bd4..c16770106c 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -1043,6 +1043,18 @@ func (s *Sandbox) Shutdown(ctx context.Context) error { return nil } +type pauseOptions struct { + memorySnapshot bool +} + +type PauseOption func(*pauseOptions) + +func WithMemorySnapshot(enabled bool) PauseOption { + return func(opts *pauseOptions) { + opts.memorySnapshot = enabled + } +} + // Pause creates a snapshot of the sandbox. // // Currently the memory snapshotting works like this: @@ -1058,9 +1070,14 @@ func (s *Sandbox) Pause( ctx context.Context, m metadata.Template, useCase SnapshotUseCase, + opts ...PauseOption, ) (st *Snapshot, e error) { ctx, span := tracer.Start(ctx, "sandbox-snapshot") defer span.End() + pauseOpts := pauseOptions{memorySnapshot: true} + for _, opt := range opts { + opt(&pauseOpts) + } cleanup := NewCleanup() defer func() { @@ -1089,6 +1106,10 @@ func (s *Sandbox) Pause( // compact_memory) on the live VM via envd. Per-step caps are LD-flag-driven; // all default to 0 which disables the chain entirely. Non-fatal. s.bestEffortReclaim(ctx) + if !pauseOpts.memorySnapshot { + s.bestEffortGuestSync(ctx) + m.Prefetch = nil + } // reclaim freezes user cgroups; if pause/snapshot fails the sandbox stays // live, so unfreeze on error to avoid a permanently frozen live VM. // Only runs via cleanup.Run on the error path; success leaves the frozen @@ -1126,49 +1147,54 @@ func (s *Sandbox) Pause( return nil, fmt.Errorf("error creating snapshot: %w", err) } - // Gather data for postprocessing - originalMemfile, err := s.Template.Memfile(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get original memfile: %w", err) - } - originalRootfs, err := s.Template.Rootfs() if err != nil { return nil, fmt.Errorf("failed to get original rootfs: %w", err) } - memfileDiffMetadata, err := s.Resources.memory.DiffMetadata(ctx, s.process) - if err != nil { - return nil, fmt.Errorf("failed to get memfile metadata: %w", err) - } - recordSnapshotDiff(ctx, "memfile", memfileDiffMetadata, originalMemfile.Header()) - // Start POSTPROCESSING - var dedupBase block.ReadonlyDevice - var dedupBestEffort, dedupDirectIO bool - dedupCfg := s.featureFlags.JSONFlag(ctx, featureflags.MemfileDiffDedupFlag, sandboxLDContext(s.Runtime, s.Config)).AsValueMap() - if dedupCfg.Get("enabled").BoolValue() { - dedupBase = originalMemfile - dedupBestEffort = dedupCfg.Get("bestEffort").BoolValue() - dedupDirectIO = dedupCfg.Get("directIO").BoolValue() - } - memfileDiff, memfileDiffHeader, err := pauseProcessMemory( - ctx, - buildID, - originalMemfile.Header(), - memfileDiffMetadata, - s.config.DefaultCacheDir, - s.process, - s.memory.Memfd(ctx), - s.featureFlags.BoolFlag(ctx, featureflags.MemfdBackgroundCopyFlag, sandboxLDContext(s.Runtime, s.Config)), - dedupBase, - dedupBestEffort, - dedupDirectIO, - ) - if err != nil { - return nil, fmt.Errorf("error while post processing: %w", err) + memfileDiff := build.Diff(&build.NoDiff{}) + memfileDiffHeader := NewResolvedDiffHeader(nil) + memfileBlockSize := uint64(0) + if pauseOpts.memorySnapshot { + originalMemfile, err := s.Template.Memfile(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get original memfile: %w", err) + } + memfileBlockSize = originalMemfile.Header().Metadata.BlockSize + + memfileDiffMetadata, err := s.Resources.memory.DiffMetadata(ctx, s.process) + if err != nil { + return nil, fmt.Errorf("failed to get memfile metadata: %w", err) + } + recordSnapshotDiff(ctx, "memfile", memfileDiffMetadata, originalMemfile.Header()) + + var dedupBase block.ReadonlyDevice + var dedupBestEffort, dedupDirectIO bool + dedupCfg := s.featureFlags.JSONFlag(ctx, featureflags.MemfileDiffDedupFlag, sandboxLDContext(s.Runtime, s.Config)).AsValueMap() + if dedupCfg.Get("enabled").BoolValue() { + dedupBase = originalMemfile + dedupBestEffort = dedupCfg.Get("bestEffort").BoolValue() + dedupDirectIO = dedupCfg.Get("directIO").BoolValue() + } + memfileDiff, memfileDiffHeader, err = pauseProcessMemory( + ctx, + buildID, + originalMemfile.Header(), + memfileDiffMetadata, + s.config.DefaultCacheDir, + s.process, + s.memory.Memfd(ctx), + s.featureFlags.BoolFlag(ctx, featureflags.MemfdBackgroundCopyFlag, sandboxLDContext(s.Runtime, s.Config)), + dedupBase, + dedupBestEffort, + dedupDirectIO, + ) + if err != nil { + return nil, fmt.Errorf("error while post processing: %w", err) + } + cleanup.AddNoContext(ctx, memfileDiff.Close) } - cleanup.AddNoContext(ctx, memfileDiff.Close) rootfsDiff, rootfsHeader, err := pauseProcessRootfs( ctx, @@ -1200,7 +1226,8 @@ func (s *Sandbox) Pause( MemfileDiffHeader: memfileDiffHeader, RootfsDiff: rootfsDiff, RootfsDiffHeader: NewResolvedDiffHeader(rootfsHeader), - MemfileBlockSize: originalMemfile.Header().Metadata.BlockSize, + MemorySnapshot: pauseOpts.memorySnapshot, + MemfileBlockSize: memfileBlockSize, RootfsBlockSize: originalRootfs.Header().Metadata.BlockSize, BuildID: buildID, diff --git a/packages/orchestrator/pkg/sandbox/snapshot.go b/packages/orchestrator/pkg/sandbox/snapshot.go index c597dc629c..05f9ff61df 100644 --- a/packages/orchestrator/pkg/sandbox/snapshot.go +++ b/packages/orchestrator/pkg/sandbox/snapshot.go @@ -33,6 +33,7 @@ type Snapshot struct { Snapfile template.File Metafile template.File BuildID uuid.UUID + MemorySnapshot bool // Template block sizes captured sync at Pause time. They equal // MemfileDiffHeader.Metadata.BlockSize once that header resolves, but diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index c29eda76c1..2e489dc487 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -19,21 +19,27 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "google.golang.org/grpc/codes" + grpcmetadata "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/fc" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/constants" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/units" "github.com/e2b-dev/infra/packages/shared/pkg/events" + fcmodels "github.com/e2b-dev/infra/packages/shared/pkg/fc/models" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" "github.com/e2b-dev/infra/packages/shared/pkg/logger" sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) @@ -66,6 +72,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ defer childSpan.End() isResume := req.GetSandbox().GetSnapshot() + rebootFromRootfs := rebootFromRootfsEnabled(ctx) createStart := time.Now() defer func() { if createErr != nil { @@ -75,6 +82,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ s.sandboxCreateDuration.Record(ctx, time.Since(createStart).Milliseconds(), metric.WithAttributes( attribute.Bool("sandbox.resume", isResume), + attribute.Bool("sandbox.reboot_from_rootfs", rebootFromRootfs), ), ) }() @@ -86,6 +94,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ telemetry.WithKernelVersion(req.GetSandbox().GetKernelVersion()), telemetry.WithSandboxID(req.GetSandbox().GetSandboxId()), telemetry.WithEnvdVersion(req.GetSandbox().GetEnvdVersion()), + attribute.Bool("sandbox.reboot_from_rootfs", rebootFromRootfs), ) // setup launch darkly @@ -185,15 +194,26 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ SandboxType: sandbox.SandboxTypeSandbox, } - sbx, err := s.sandboxFactory.ResumeSandbox( - ctx, - template, - config, - runtime, - req.GetStartTime().AsTime(), - req.GetEndTime().AsTime(), - req.GetSandbox(), - ) + var sbx *sandbox.Sandbox + if rebootFromRootfs { + sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req) + } else { + sbx, err = s.sandboxFactory.ResumeSandbox( + ctx, + template, + config, + runtime, + req.GetStartTime().AsTime(), + req.GetEndTime().AsTime(), + req.GetSandbox(), + ) + if errors.Is(err, storage.ErrObjectNotExist) { + telemetry.ReportEvent(ctx, "memory snapshot files missing, rebooting from rootfs") + rebootFromRootfs = true + childSpan.SetAttributes(attribute.Bool("sandbox.reboot_from_rootfs", true)) + sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req) + } + } if err != nil { if errors.Is(err, storage.ErrObjectNotExist) { // Snapshot data not found, let the API know the data aren't probably upload yet @@ -271,6 +291,85 @@ func createVolumeMountModelsFromAPI(volumeMounts []*orchestrator.SandboxVolumeMo return results, errors.Join(errs...) } +func memorySnapshotEnabled(ctx context.Context) bool { + values := grpcmetadata.ValueFromIncomingContext(ctx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey) + if len(values) == 0 { + return true + } + + return values[0] != orchestrator.SandboxMemorySnapshotGRPCValueFalse +} + +func rebootFromRootfsEnabled(ctx context.Context) bool { + values := grpcmetadata.ValueFromIncomingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey) + return len(values) > 0 && values[0] == "true" +} + +func (s *Server) createSandboxFromRootfs( + ctx context.Context, + template sbxtemplate.Template, + config *sandbox.Config, + runtime sandbox.RuntimeMetadata, + req *orchestrator.SandboxCreateRequest, +) (*sandbox.Sandbox, error) { + pageSize := int64(header.PageSize) + if config.HugePages { + pageSize = int64(header.HugepageSize) + } + + buildID, err := uuid.Parse(template.Files().BuildID) + if err != nil { + return nil, fmt.Errorf("parse build id: %w", err) + } + + memfile, err := block.NewEmpty( + units.MBToBytes(config.RamMB), + pageSize, + buildID, + ) + if err != nil { + return nil, fmt.Errorf("create empty memfile: %w", err) + } + + maskedTemplate := sbxtemplate.NewMaskTemplate(template, sbxtemplate.WithMemfile(memfile)) + ioEngine := fcmodels.DriveIoEngineSync + kvmClock, err := utils.IsGTEVersion(config.Envd.Version, "0.2.11") + if err != nil { + return nil, fmt.Errorf("compare envd version: %w", err) + } + + timeout := req.GetEndTime().AsTime().Sub(req.GetStartTime().AsTime()) + if timeout <= 0 { + timeout = s.config.EnvdTimeout + } + sbx, err := s.sandboxFactory.CreateSandbox( + ctx, + config, + runtime, + maskedTemplate, + timeout, + "", + fc.ProcessOptions{ + InitScriptPath: constants.SystemdInitPath, + KvmClock: kvmClock, + IoEngine: &ioEngine, + }, + req.GetSandbox(), + nil, + ) + if err != nil { + return nil, err + } + + if err := sbx.WaitForEnvd(ctx, timeout); err != nil { + closeErr := sbx.Close(context.WithoutCancel(ctx)) + + return nil, errors.Join(fmt.Errorf("wait for envd after rootfs reboot: %w", err), closeErr) + } + + return sbx, nil +} + func (s *Server) Update(ctx context.Context, req *orchestrator.SandboxUpdateRequest) (*emptypb.Empty, error) { ctx, childSpan := tracer.Start(ctx, "sandbox-update") defer childSpan.End() @@ -495,8 +594,9 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest // Stop the old sandbox in background after we're done defer s.stopSandboxAsync(context.WithoutCancel(ctx), sbx) + memorySnapshot := memorySnapshotEnabled(ctx) // Fire and forget - upload completes in the background - res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId()) + res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId(), memorySnapshot) if err != nil { telemetry.ReportCriticalError(ctx, "error snapshotting sandbox", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -575,7 +675,8 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo sbxlogger.E(sbx).Info(ctx, "Checkpointing sandbox") - res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId()) + memorySnapshot := memorySnapshotEnabled(ctx) + res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId(), memorySnapshot) if err != nil { telemetry.ReportCriticalError(ctx, "error snapshotting sandbox for checkpoint", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -591,26 +692,36 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo return nil, status.Errorf(codes.Internal, "error getting template for resume: %s", err) } + runtime := sandbox.RuntimeMetadata{ + TemplateID: sbx.Runtime.TemplateID, + SandboxID: sbx.Runtime.SandboxID, + ExecutionID: sbx.Runtime.ExecutionID, + TeamID: sbx.Runtime.TeamID, + BuildID: sbx.Runtime.BuildID, + SandboxType: sbx.Runtime.SandboxType, + } + // Resume the sandbox keeping the same ExecutionID (stable identity for // the API, routing catalog, and analytics) but with a fresh LifecycleID // so the old sandbox's cleanup goroutine won't // accidentally evict the resumed sandbox from the map. - resumedSbx, err := s.sandboxFactory.ResumeSandbox( - ctx, - template, - sbx.Config, - sandbox.RuntimeMetadata{ - TemplateID: sbx.Runtime.TemplateID, - SandboxID: sbx.Runtime.SandboxID, - ExecutionID: sbx.Runtime.ExecutionID, - TeamID: sbx.Runtime.TeamID, - BuildID: sbx.Runtime.BuildID, - SandboxType: sbx.Runtime.SandboxType, - }, - sbx.GetStartedAt(), - sbx.GetEndAt(), - sbx.APIStoredConfig, - ) + var resumedSbx *sandbox.Sandbox + if memorySnapshot { + resumedSbx, err = s.sandboxFactory.ResumeSandbox( + ctx, + template, + sbx.Config, + runtime, + sbx.GetStartedAt(), + sbx.GetEndAt(), + sbx.APIStoredConfig, + ) + } else { + resumedSbx, err = s.createSandboxFromRootfs(ctx, template, sbx.Config, runtime, &orchestrator.SandboxCreateRequest{ + StartTime: timestamppb.New(sbx.GetStartedAt()), + EndTime: timestamppb.New(sbx.GetEndAt()), + }) + } if err != nil { telemetry.ReportCriticalError(ctx, "error resuming sandbox after checkpoint", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -618,16 +729,20 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo } // Collect prefetch data immediately after resume while it's most accurate - prefetchData, prefetchErr := resumedSbx.MemoryPrefetchData(ctx) - if prefetchErr != nil { - sbxlogger.I(resumedSbx).Warn(ctx, "failed to get prefetch data for checkpoint", zap.Error(prefetchErr)) + var prefetchData block.PrefetchData + var prefetchErr error + if memorySnapshot { + prefetchData, prefetchErr = resumedSbx.MemoryPrefetchData(ctx) + if prefetchErr != nil { + sbxlogger.I(resumedSbx).Warn(ctx, "failed to get prefetch data for checkpoint", zap.Error(prefetchErr)) + } } // Setup lifecycle for the resumed sandbox s.setupSandboxLifecycle(ctx, resumedSbx) // Embed prefetch data into the metadata so it's uploaded with the snapshot files in a single pass. - if prefetchErr == nil { + if memorySnapshot && prefetchErr == nil { prefetchMapping := metadata.PrefetchEntriesToMapping(slices.Collect(maps.Values(prefetchData.BlockEntries)), prefetchData.BlockSize) if prefetchMapping != nil { res.meta = res.meta.WithPrefetch(&metadata.Prefetch{ @@ -716,6 +831,7 @@ func (s *Server) snapshotAndCacheSandbox( ctx context.Context, sbx *sandbox.Sandbox, buildID string, + memorySnapshot bool, ) (*snapshotResult, error) { meta, err := sbx.Template.Metadata() if err != nil { @@ -728,7 +844,7 @@ func (s *Server) snapshotAndCacheSandbox( FirecrackerVersion: sbx.Config.FirecrackerConfig.FirecrackerVersion, }) - snapshot, err := sbx.Pause(ctx, meta, sandbox.SnapshotUseCasePause) + snapshot, err := sbx.Pause(ctx, meta, sandbox.SnapshotUseCasePause, sandbox.WithMemorySnapshot(memorySnapshot)) if err != nil { return nil, fmt.Errorf("error snapshotting sandbox: %w", err) } diff --git a/packages/shared/pkg/grpc/orchestrator/internal_flags.go b/packages/shared/pkg/grpc/orchestrator/internal_flags.go new file mode 100644 index 0000000000..b2c0ddf40d --- /dev/null +++ b/packages/shared/pkg/grpc/orchestrator/internal_flags.go @@ -0,0 +1,7 @@ +package orchestrator + +const SandboxRebootFromRootfsGRPCMetadataKey = "x-e2b-reboot-from-rootfs" + +const SandboxMemorySnapshotGRPCMetadataKey = "x-e2b-memory-snapshot" + +const SandboxMemorySnapshotGRPCValueFalse = "false" diff --git a/spec/openapi.yml b/spec/openapi.yml index 6e39265fd0..bfddf2c3b9 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -689,6 +689,10 @@ components: type: boolean default: false description: Automatically pauses the sandbox after the timeout + autoPauseMemory: + type: boolean + default: true + description: Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. autoResume: $ref: "#/components/schemas/SandboxAutoResumeConfig" secure: @@ -724,6 +728,9 @@ components: type: boolean deprecated: true description: Automatically pauses the sandbox after the timeout + reboot: + type: boolean + description: Recreate the sandbox from the snapshot filesystem and discard memory state. ConnectSandbox: type: object @@ -735,6 +742,9 @@ components: type: integer format: int32 minimum: 0 + reboot: + type: boolean + description: Recreate the sandbox from the snapshot filesystem and discard memory state. TeamMetric: description: Team metric with timestamp @@ -2554,6 +2564,10 @@ paths: name: type: string description: Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. + memory: + type: boolean + default: true + description: Whether to persist memory state. Set false to snapshot disk only and reboot on next start. responses: "201": description: Snapshot created successfully diff --git a/tests/integration/internal/api/generated.go b/tests/integration/internal/api/generated.go index d245e339c6..eda599fd79 100644 --- a/tests/integration/internal/api/generated.go +++ b/tests/integration/internal/api/generated.go @@ -325,6 +325,9 @@ type CPUCount = int32 // ConnectSandbox defines model for ConnectSandbox. type ConnectSandbox struct { + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Timeout in seconds from the current time after which the sandbox should expire Timeout int32 `json:"timeout"` } @@ -553,6 +556,9 @@ type NewSandbox struct { // AutoPause Automatically pauses the sandbox after the timeout AutoPause *bool `json:"autoPause,omitempty"` + // AutoPauseMemory Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. + AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` + // AutoResume Auto-resume configuration for paused sandboxes. AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` EnvVars *EnvVars `json:"envVars,omitempty"` @@ -709,6 +715,9 @@ type ResumedSandbox struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set AutoPause *bool `json:"autoPause,omitempty"` + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Time to live for the sandbox in seconds. Timeout *int32 `json:"timeout,omitempty"` } @@ -1505,6 +1514,9 @@ type PostSandboxesSandboxIDRefreshesJSONBody struct { // PostSandboxesSandboxIDSnapshotsJSONBody defines parameters for PostSandboxesSandboxIDSnapshots. type PostSandboxesSandboxIDSnapshotsJSONBody struct { + // Memory Whether to persist memory state. Set false to snapshot disk only and reboot on next start. + Memory *bool `json:"memory,omitempty"` + // Name Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. Name *string `json:"name,omitempty"` } From faf648b16a3711697436ff53f876deb3743343c6 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Sat, 30 May 2026 15:28:09 -0700 Subject: [PATCH 10/11] fix(orchestrator): pass config to rootfs reboot checkpoint Reuse the stored sandbox config when checkpoint resumes through the rootfs reboot path. --- packages/orchestrator/pkg/server/sandboxes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 2e489dc487..bf1d9df71e 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -718,6 +718,7 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo ) } else { resumedSbx, err = s.createSandboxFromRootfs(ctx, template, sbx.Config, runtime, &orchestrator.SandboxCreateRequest{ + Sandbox: sbx.APIStoredConfig, StartTime: timestamppb.New(sbx.GetStartedAt()), EndTime: timestamppb.New(sbx.GetEndAt()), }) From 31db71f551fc7bf6863b9a9d26668761b4eac9a4 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 30 May 2026 15:28:56 -0700 Subject: [PATCH 11/11] refactor(snapshot): use artifacts for memoryless resume fallback --- packages/api/internal/api/api.gen.go | 252 +++++++++--------- .../api/internal/handlers/sandbox_create.go | 2 - .../api/internal/handlers/sandbox_resume.go | 3 - .../internal/orchestrator/create_instance.go | 2 - .../internal/orchestrator/delete_instance.go | 3 +- .../orchestrator/nodemanager/sandboxes.go | 1 - .../internal/orchestrator/pause_instance.go | 9 +- .../internal/sandbox/sandboxtypes/sandbox.go | 3 - packages/api/internal/sandbox/store_test.go | 1 - packages/db/pkg/types/types.go | 9 +- spec/openapi.yml | 4 - tests/integration/internal/api/generated.go | 3 - 12 files changed, 133 insertions(+), 159 deletions(-) diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go index 608e497941..b72cd424d4 100644 --- a/packages/api/internal/api/api.gen.go +++ b/packages/api/internal/api/api.gen.go @@ -561,9 +561,6 @@ type NewSandbox struct { // AutoPause Automatically pauses the sandbox after the timeout AutoPause *bool `json:"autoPause,omitempty"` - // AutoPauseMemory Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. - AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` - // AutoResume Auto-resume configuration for paused sandboxes. AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` EnvVars *EnvVars `json:"envVars,omitempty"` @@ -13042,131 +13039,130 @@ var swaggerSpec = []string{ "zG+w44nXgYOdjroqHixIsL1h1mHKaETefDw6Q2HK5nSRcx1O1TRktNhIy0vAR0e1qLvZwLezhq3mzf5/", "+nD/idx2OlEe6kjwWSGv9Lwdim+c3v4B+8iI/ENP4FOE4/S2QIFMC0iWBNnOU/Sb0mcEkarBHMeCTBCV", "aEaW+IZYdSEhSCk5GQnpfEXZAkWErX7Noc/eFP63u2epjBF5m/Jrs8t+BxvOZXqGc0Eq/ls9fTP4Lk2w", - "urDG8QplqlNVi9GuNlB5jEOsc0ZNIZV5fezjtyWRS8LVfTndgVmtaDOCq+pGRBdEavRBDKv1QCrFRLtx", - "MYuQdmeilCFG7qRWGtvxc05EngxVEg+LDkeAdqO6W0Njj9oOzZT6rc9yp7YeZg9U1A19DOz5SbcuVyVI", - "6JXYF/A7wnGMjAk9TJMkZzZqE2RLQ+93nb6j1Gt7aLs9Fq4f2UZU/8UnqRTVxPTGa2U2gmM63tT8DFq8", - "4V1dfsnH81C53FLDu85sGktgeJKKKwYHwf//He/8+3Dn/+3t/O2Pnav//R8DIfGIqk/GIF7TP+NcSMKH", - "kZpp7FX30sT7JOAIfrcDpDxcEiE5mLlb/bg/WTNaT/iduTZCUMlQL5DucqGj9siYWUTRZ9hMw1zIbepz", - "Ur00dDJCp6lmiNZV2NVLkYP1KpZGixGRj9ZDkzJ3IRXMtDhVRGFOgOii/jlNQ3RhJ68xIP8s2jh+woTE", - "LPQyU2vqp6ZNabXs3R8TAjUAyTqADJjgQAdY9ynx+caba504J7uAtrbNJa00z0X1LLbsWbmkggFUKffK", - "8B1tGvdFNIdLEkEsm+conirlJp0j3crGHtOoRnLDo1m3zG7L7DbO7LZsqJcNVdhAPy/yMZ2CkfnYjxM9", - "U39gFFlLiWgYedTlFsw6R2efu6ikaIeKgNSBtFH01FfBluiUQ4grqc5krnwjQ2Bcf5AvrqZ8t1mG1o6n", - "+DDLzwgPifdsKYSrwXOIQc50Ox14PWRsdY0Vvmgnqd92mL3Usco4XMK9dzcpg4+Gxle7QVfe6GqF/8ve", - "SCWmCWydzdK9PrdHLX1yxrYO3bVjlyrE3kKZla1tAujxiTgIsntnz+RFwbmaro9cuHxv+pXtoIhjqjjw", - "QfEzogLN0pxBbMKMILHMJYrSWzZFJ1J7GFkqwdSUScTIrcPOMYt0CyHTDKWK52LwSFIBiqbTkhMUpUwD", - "odhaNFtVYdCTSHpDYr0PEzTLJaIShZjZd8/wAhpH2voSpkxSlhME/JItkOR4Pqfh9Gs1DAJH6uJpVw7s", - "DqLU9R85WxIcy+VKM1YF2ED/RYn+czNH+ctxOVv545E7b/nzZweC8tcLC0tlo4+WmC0e7/7ZG287XjDW", - "DoQZQK1Cm7M63P9VG2K3N+CRrIgv4kHCxg1JapNeXRRGlCaYetSt91go3qI+Om9pCyu55gmKw2hrOZ3F", - "g8K2Cbupv7aoIcR9RQGCA6Qlu4mqhsrHDcJ4rKiITcYemD3oxCb8XBphFSrNfpVyBN1QjDKe3q2m/Tu4", - "RlxCPbCgzRTfJIVcpjscmnh8YcCcolIYThsqMmFqIdFo18AH06++WDuez37ZOsggx41dpZkBzWO88C8S", - "HevBtBPFzwUNLG1mjYdyInCrnRjH2GGLU836hawDzTrVbrFA5C6LaUhlvCoWnHLF5c3iqxx5ij7lcYwS", - "gplQeosaQWk1ziiCyA7SdTDzPYSybZxhbyBy7gVKhJjOSbgK46GexdOi/eZj+h7qKtyGBG5DAoeEBDZI", - "vXk/NvgpTg/K0piGqyLKDc1Wjqo9T5tSu+rT90uVylZgpqMPtAz1i8SUXZZXgwEb8WvRvmGKKMFzh+3Q", - "CU7ThT+XgY5+qgZzwYUnpow08AI/esdRX7oSIjxT0gIA+KqCh5YUEXNKjIul7aVXm/OkRPbG00w8F1YB", - "fjclhMFeFdOiPxtE1a7FcwhFjHSMaoPZj+FiXYkf4tT3OPX0Mebs5Zgw98TFQw1nX/bPTaI2L/b6smgU", - "zE+tRhQYfTTs+ZbjrOCjo0AMezFpe/TK9sok3vDUj25A51CW1m7x/9S09Q97Ehlm+WdBorOwJSNHl2V/", - "HqduViAb7qmFJBiL2wzpEL/W/kS33YyuOvrfzcOD2lbDeadh/giHS19Us3ZUGyvaDxnwN/Xbj+On6MRG", - "h0ehc1A/Ij72+BDah/xzxkGPiE52VELn3JR74Wy1Q1gO1bpHw+FE1RuGPy73V1/eGBtTAS1IhCIipEmK", - "arxmCw53QeOSQB9wuDTYU3rgjCCMjk6Oz9EsTsNrnRgAfQ3+cwr/2327/zX4cYIwmmFO0MkZwlEEA9Ya", - "QquUI2wv1PAOwDYidzjJYjIN0+RrMEFfg/81rfz04xQdmgXYpEo4vsUrgSS+JkjRIYmI2tX0hnAUEUbL", - "ptNRMSOAqLN8FtPwUuOkN2L3QocnI1rh+ejz+alwXqWURgIdlwssvfoo1q9pm5Dn9r01yy13SShMl3tB", - "/Dt9XG6E9nuxVCKRZ1mqbjjQRU2NeB6PRWKCxbXJxfFzKjygW5QtUyHhVaq5FIK5Y0ZKowRE1RqEmtcF", - "3kRUAGSXnB6jMJjTdp5r20N9dbVXsITvGJqWHDOh2InGGYL8oTrxV4JluKRsYXfh58vLs131fxfFsqbo", - "F7KyHkg1XnmIcEanjTPSOCH2aMXwehWBxco6LCteKMsMduDhtGUeCsyM8ITqJMgVJ2XtcnDffjdzcdfk", - "1hUEufgxaLG4KPFVMAwTI9u83hZYH7e7BSxDlnPpztGyJsNUOza9WN2MzFMOnrJbzCPKFs1VLQmOCB93", - "hasCpqgLmWEUNJSptSnWoJgkpxHRT8INjCUZHrLSM677O8njFNsG3k3VerIYhySaInhArWk3i9VuaaDE", - "/0FCJ3blRKRxDq6GJc4ywoQx/u4IBYhBiCAsAgd5aqOx1yS/z5lSKtrcIJ8qD0WsCySHPprrFAFghcn8", - "XK9VVDKu2X2F017gKOPpDY1IVB1/in5NqJSapuGGicKYYC4QlVNvMNJWoj+eRH/Vj5ZeoQbwEuWxc4gr", - "0T/m+KqdtUd3Oorp/OraQuuuFuAtiqhZ01wP1weU5sAAbCCQSVYMjkrv8+6KLX7sW3LD2OwEUdcMrvH8", - "Ic9bmhlFsC/lxxkuE3609fXnToPxOgzGRPxG5bI1qVrhDO4i2GEeJE5DW6DAiS0qxocrnQnK8T8QN1Fb", - "jQRteRybg2q31sb2OEmZwzjX8pPgRLcGCwpmkLTYMGf1cUfE+WI3We3YUQ5u9n8cdcBtx4E+ri5gl5CZ", - "eYo+Kz5VQL0LXnV9ZrAWLrdYlMK1azHmuqYElFwSfksFUdw6FmiGw2urXnB8W8JzcmxGxLPwzf7bYohp", - "Lw06mJiY7fOR4iXBiefmDgVkPHzDZHW0rnu1Tm+SU3Fsr6Zdrh0gCONxMyurDemImyHJEv3QlIVJ+j2C", - "vhEaDjlTysQcc4Msd9VXBrPbVKStoZF/+kyihnq82WwfKUFEmDJzJ7hwZUkzbUIZcl92ccKPa8d9gHne", - "fah17lUIvGUMtBsb6jNps+ggs/3W/ttn//XQgWePLOUBF2jwLJKYKLM+DH9QDe3CcwGR0r2Hcxh/MaP1", - "MBffadPQ6xWakDd/wJw9Ij3x07rpA+otmNoKvX4xrYxXwlUUJ1Sd5bCzOKJMhWLKTjERU0BBHX+dK6Ur", - "XHBW1ino47J2C5zSButG1/VI0vJyU8FeGdbzTOJ0/fx2a8e5YSEvMnzLRiMLiOJhkneNMLmWG8cn97JR", - "gPlDXT/XcOoFFd+icXeJDNwtfRqsvTgIpNuDJRMynJSeldnKo106qq1Q+7IuJ6jvTIefdK3oON950EbJ", - "9QhJd10zQMaNhSsLUg4IeTOb6TIMdxnuEa+flcr+VNh29TxOCgFiqbfKFF3hA/KjPQhlc6T3WDTRtVFm", - "Ne76gfM/vKLSkIvGk0oVLSDXESmblwBzyqhYjluV7TN4WeuwevEQpWEwKyoX9XA+VLKe4jF5K1/x8KbG", - "SfiJxuRzFqfYcyYyToT3FbPLDOY0BkaAY/3A03Sy5vzQBHQ0z3/OPeE9n3nsPC2AscuIgRzgBFdiL54s", - "7I0F+62Maxz/pl1haC0sgGPdGL7ewlcDwghLAMb5MIoiYL0AVqqGPfSgbUJSeM6VP5izAuNpuhAPCuh8", - "SlJoC+asrKC1QsyDX5Ou82ApDa8JV6feE61YfHOMQu3TryMNgIEdJZHvVbFibeGShNfwIgjrV/fkjoS5", - "rv5Y0YvKJ8ytzAIMTt65wCrySLM8sv3Z2Z82Qvqy/zJIaZ39d7E19uXeIPxpRLSi7m0n6gaYherInKLj", - "otsEYpt0pAoTkuBo+py4Hl6AaoqOMDPuMYIwOO7A+hymccqQIBmGFD1FqEWy2rF9vwbqZlL56eDmDURb", - "nMxhJCrs0BFELtiAO2lrxgkbxgjzug43ex7xQiBgudPxtbL8JWqLAz2iUO3j026dTIEx+c73PC0Sk3c9", - "Q3a1xttlGlvFuFTwYCDgeTxniJMF5lFMREHX7crk3FYV8vA69bMtioIFBPKIphBpZ6JzX8WiLjpvljgy", - "o7gG4LprxUDxADi/P/ElJMl6C8uakB9o2zVf40wN0UQvJMm8mpXHnd3UXXuS+zRAszEp8LcOSrnF1OSd", - "sflw2sscWBBOyQKHqx4vw9an8Og6x9Yj8J16BLb2+K09fj17vKvrGzXf2gta1f0N+4GfnpeOcai9UD9Z", - "hxavVfiynP2DlPhNGrqKg9B0HhW1dyt6kL+uvVfPUrckCCtvxPHzXpvYIV/kieLFZZ4eNfsYREJ6t5+x", - "8ATVql8tBqFZ8e7Tmal5Bxh/xVFDPcrdprueZDvUvvKO7p5e4sXDDeGK/NOQwl3ZPPigQt2JBznIBitM", - "5u5tz9rwaCm88MedqRH9xaHraIOlOBKmFi6ncanf17QaUzfFqO49ILVZp587zMETuV7lOb9RuSxToz+/", - "oOzI0G5Ss3sM06Num9o37UvcvpGbxXOq5dugm62SPyiWw6eutGny/dq75jiaVa5RKYfcaou6Pfijy+U8", - "Qq2clvRg3l0/HvmcqBhq0l5XRy/hkEVr12VrX4psydeXK0UDsvU5xa7hUSKGNM1aFdHjdmVYekJUWeCv", - "fA/noHQWlasLJQA0mpwEhWp5oIsRzAn/yZ4bzWH+sOXrQHgAZ4FmJYBLKcFmeRgllFUGpGpl+gWzBfMg", - "+L870HDnsloWz7wtUePAv/rGODvZ+cWlfKd/Lpdn2mnC3wOsj7VAZ2A4vN0gqiY7sEENEC/yDM+wIG+G", - "oMs2bseYbbE/AKpytIq4sIMpaqEmZEVSqSRo8GH/veI0TiWIg2Bv+ma6B3noMsJwRoOD4O10b7pnHgMC", - "ie1qBO8AgrVS5U1GcaQzaGNIGl8rmqhONjwIOomCg+AsFdIhXBHoM0GEfJ9GK/MQRJoYIkgEoBN77P7T", - "BJJoDag3TXm19GPtPaExY3Kj6cLC9vfePNrsR0a61CHoyCFqBJJjMImBMN5psHyzFeDvqkb3k+Ave3v9", - "bVUjl6GAKdhHzb9f3U++tR7F36/ur6yR4/egSiZXavwq6ex+wyUyTo7vNQnFxOfZOobfEWbdlKSbubR0", - "6E4BZMxxQiTkgGixd5dNdisAgt27Rh/vetLA6vU8bAvf6Vn62r57gdutRMKu0mDF7jftW77f1RrTbohZ", - "qBMhtrAP+K7flFO2k/FUP7PHLEKZSWhRuzDp3ACQJURzQQ+bAWmmALoEcPTNUM/VpA7PwzUgImDC8Dq6", - "YMHFo8oqR5k43KHv1U+TuvYejfvAumGxeq3nROSx9HGgC4dSkd6kuCgb9jJJt66iaMIUeZJgvipICSjJ", - "oRhcXK0sBathuii3eOi3C8kEWkn3Fxobwm0+QVyDRos3b7/YFAbfM5Ga1aq1DiRStRlu7vnXSaRqwR6a", - "6abSjO5ckxVsxIK0ZTJRg8JLdXOvFA2q+zuRWj3XqtcDtnegoaq4Ijfted17XaRnbi7qmRUjr77eJ0Db", - "bh810Wq3Wd31ByjbLl78HMbZ7CfRs90dfhY1uw6Ah0lWchO8MC17M8TkspDdb/quOFAP76Yxo4ZrKjs0", - "445Xvm3HYXp3ZVNfu969MW6CZehxb2pTZt82n6nOj7zLj8+OGmbZQRxpr4fAjIF4S2C9HEZXt2tVUX6G", - "zzqs1KeY6O/BIA6wNMUAQ3Ct2LJ6o3YFiGOXpREZoFXpZh6gP5kPnap6V2U+UN7/lRNI6WG090oN2I0p", - "7IVGNyxIEOrj3189SK/TaN2YCPZr5j59GwDb/ab+Y+Sklz7+TqSpa8nmaSt5fIJRRvNLPXmgDuprJKc+", - "2jHFwAbTS1FV9xVd+uqk1arUQ7VPJIpwb2wLCDdV+scgqScSwI3ypfdGAg/h53CODAYgGgGGeA1ydzhb", - "qSRK7JY4tQS6funjJknqlEBFXQ1gDTqETaZoTmNZLV5Eyjy1uSD8v/As/Jrv7e3/FWfZf2U8jeAxFKTE", - "VcoRZhG60bmLk1xINCPo8/kpIixMTRZQH0MqCou5/OiZxdkpJP22xVIfKNeamwfEuDeEGPc2KA8dT63W", - "CDeuQlZTe/aYPBr1eZ0glCajdA/HE1k/CnLZrOmjMq1HM3aK2bXbPLbE2EmMFXa9m5Spb9vZtltMX0e5", - "D2PeNq9uDw8/SpME75j3qySCMgVOVkJ0cgzv2RakAkkwCchdFittwsYg+1iyGeQPGolOr0F7fFyC7070", - "xzd7ezXmOQlyRv+VE9MAzseTKpjevMUPY+E6RsgSwvYIjT1C34qKkJ12R+0dcRJv+wyOxfZeOFUmx6nC", - "ZX3KgUbHGmO1vqiXr52+NCHfepMuBfxsheBO2s4zn2jjH50DrXPLtbS/JacH8JjdMGWM6KJ7frXyHHAu", - "CqKLdAJlnerBOe1UmKLnlYQPurZrNEWXl6eqCQR1kztJmLn4dCikBfEeGRgfSsOPr9wayEYpuHvPoeDa", - "PGK2LsP95LlUbUMRG1O1t+e9ct5t9qxCvHQ/81ECRzjlUaF8Atq92e8Y3s2EM0A0neqEYGsf7Yk3VwaU", - "fPCU8xVILrF0XhAWMoUylNA4piZ/d4tRBlJ0+C3E9s1IQhlN8iQ42PMl+W7YnPCdau2kbO+CsgWqmCa0", - "ClXxovLNnrpnNFJAdAK5AYkPu76OvNcJ9rZM4CFMoO+e7p76pLh2DzjLrXf0BxznIrW+Psrl62XMpT3Y", - "8EjlBscTdYrNAZ5AU13wsUzZ/4Tn2jcsgZIH7rEcsDTCovUWNg7kq03EttVqFq1rLnYZwAaMC1t+UeEX", - "pnqcfnMtW8NjmoXmbNHDDVQ79NQ6rN0tcg/DMjXfXuDVoqPS5KBrhsc4UzlaNmDHPVnezXvKs/Nu729D", - "2v7tT3LOdDG+1qv4mfpcK7g35P4M/TZu+tPWgMo9DxyhpjqouQduqWtj1MXJnBOxJKLL2ANNKkxBW2sU", - "i6VSmDpeKYrpDRlIfufFvM/DZasPpyPL1pqxvK60KtUri4fyMnZNMomwwoCjlUHprzutbb39q7pzdeuL", - "jffL9+N5uN3RDRlEXzHlC5sPoCD7brvHOfRYg9fqji9Qn9CARS/fF99uINxKiQ2cFVtndMhL9oxwQYWE", - "4oK29GoRdWLG/J+i0OyFhMTTtjKtsKqAjbPSURjwZKf0J+v3rTAPmpFVyjSzSzldUIZjZ5qYzokST0Pd", - "CgUcL0Iu6cSKFdOdjzEVeXPSAiu6Z4FdInXJcNWk2JWIimuTUYdFiJNZmkqUMsTIndSmlOmIIq2/Zrp0", - "cTVNS6P6LviIsKcqb5nSDAawzgkolS0m5tWWqe9iHEn1fIBFWW2n1C9YnaHoJZRXhRL4aqSUEX993YYI", - "3ihLdAs0+wwhFm2bDk7aGj8qDFGWRdf97FCdOAiz0w3L82DNHBUJR6EmfEY5QXdWd3RC9WiZwMfwlSk6", - "wnGsS1hTgRIil2mEkjyWNIuJLe5+Q/gtp9LYUi4vTyeI4FAXk0W5sBWwLSsujY5YlOZU1SpLqfqeooRg", - "kZuSQXZpVnkeymJtwfqXwGBlW/F8A6Sjy5f74eLL5FZvvRnoXQ3GOluaZV8VlFePckEQhjQtpHb0LUcY", - "yBFcZag7Er1oWq+u3HSbtOsdHTGHtXcptbr9OkK9hGG2QiLNeUicuEefCOw9iRlWapaa5hR8jKO6fCJ3", - "0qT22YyzoSJS1/U1lJv+pwpJLFatSR9yfQxLpOB1Dl6aD5t8nQDJvh74KEEvaHM7X09Q17X9I7IPqd+c", - "jSyTtgzx/boh2l2czEnKsq7n16Rg2bp9vy+3ryKKx/D5wnv7jTh83w5p+/bVs/lexrCb4LtO5gC0Z4KW", - "fIzCJn3XbzssJQ9jHx/x3ZaDvHgOMvG8m+Q0hCz86l/khlSoBJ4+mlc2LQ8dOeQgbn9QY2s4hSkzV7I/", - "3FdD9l0ObMYfHEviKeX0pGFtH/Gdy/O2PO6l8DhtHRykydqmXlZVfhxwZysSy7Ud4MHlAq82rUGbp6IP", - "1qItvp7xDjVKt35CeiyRUX292+2CrKUw63jC6xLnU7gOvaWNB1nL9x8dBlOFr8WPWNaEx2FIMmnjS17c", - "E8QXSJkVdrlrygHsfoN/tOd2OYLyl3Rec+5o5U+XkdBunU6uaqqJwH9aOGw1cyc2Lds1hpZKoLbjJnUD", - "Ozksrygd08dPlUjReVFtCaiVgX40SQ+X+38aS2875ZdFyTpT/pUOGLywDoNWDUL3Kcj9Ei+eildXZ1IT", - "jWLY71rKsbXnD9z6IB9Gh7YmnN+3eGgqWePFD+JHKG3drITXoRA8IaFpyNYmtDePDAiJXFC8+gFelFEE", - "Wzp+GjqustJvZamhoVlUW9TcOgOtlDAaaS4qug6Pa65UYHqMXKrf/zWn+7JdyzvfsuuuivhIW76m+3IN", - "R+kmNEqnmN9Ik5OrrFMp1qxn8N2ogJXMv933cvvYqfVGrgZ6Ek71dDf7aqHNtRMCN6rqtSYFfvk5gV63", - "8eicmDqybKDp6HXQ6eu1QG2tSnWF0BY++maqAN+PeZ8BOR7KeslDaVvLyvdlse4n1CNscWOPIrDv552a", - "dpZYaB/llnQGk04zrUnDa10tpB7rpCND1M0KzayVrGRNutlsYpMw5wI8tK8ps4kv1tY+wnvT/wbP77yL", - "KCchrGEyUDQoqjguerUOHJMbqKo2eNBT6OBB7YWO7Ryy+3OeJm2hCzDKqFXqiTdkL4czp2YdbDP3X3Kc", - "I/8yzTd+5vuireTd7FgnKB/DkNtKA/QxZJ1I/dlY8gmLyJ09h8UbioLgWk9lkdfCLWrvYxnpQvw6nwvS", - "wgNHp3b6brj02sx0Y5yr9WFXL8fasqknYVNzGqufllgsu4uVYIbyLE5xhGLKrq2VEnOkRoDK55gy56Dj", - "FdHfhuqUP6m2P2OxfCjj8rjGl3rYoZ5xBYVlYHYJ/c7xN09zZBRePgPm2+7X7r7cLgmHZ67mRzhCZpe+", - "A6fOaz9u1pHeE3YH7vN1/ADGufmY7p8niYQv3JIPDYU3ChPg9QljP/+MPks3oemAt35dBQu+7H/P9WYm", - "bU8PC0BnK5QyglKOkpTrWkWAiUH1FaQ+/uslGbyQRneqHrJJIOQqVj8oNfQ1uQC3xXle46PhzuTIg3Ku", - "tlknHdbyStMnv0orY98Fdm8szIWtcAhmW0B+FCtlDZV6EWC4gOu1edGTczZFqjeakTi91XkodAPMCSJ3", - "YZxH7bh9NKvnERZkRxAmqKQ3BIl8psUSSrAMlyhlAHlChMALfU1TXLZF0hDMw2UFrATfnRK2UAd8/y9/", - "3WyIsJMV+8v+eubObX7s9Xh15YnQ4z/O+LL/HM8zvuy/dPe4wcS26Np6d26XcBtxlt1VzLsjlxx6/b5j", - "l54EiHbGvQ2Oes5T0RNsMja0xHtIni+45IllCmBklER5WbEtL5B7v21TO9ZUMt4+i5Lx9rmUDAOA5bcW", - "kK2+8XQUm8Z5QgYmhEK2tc+GUXx6etu7nmu02T0GO1pzNX+m/bdrHlBvWnOnAk9+BuXs+pPUmLZbvdm3", - "Z3rWQxYZG3EPYdlMrk2cbdlQHxk6TGj3m/7H8Fdl7cSpGxny/GKGHa28WXgGPimrEIV9ToabBLE14HTy", - "pY7YsgKRrYFlT7nle8/FYGz6pS01jWYvAB2/sbuf8zg4CJZSZuJgdxdndEr2Z1OcZYHT/1uZ76dMd/Ot", - "lri1+iPkJnL/ht3bkWrB1YYZ3bkmq8pvxvNf/F0oJlf3/x0AAP//", + "urDG8QplqlNVi9GuNlB5jEOsbcZzIvJkqNp1WHQ4goUYZdia7noUYWimFFp9Ojr13zB7oOprMD6w5yfd", + "ulyVIKFXBl7A7wjHMTJG6TBNkpzZOEjg1g1N2nWjjlJY7THo9gG4nlkbo/wXH+9XtBnTG6/d1rDi6Xjj", + "7TPoxYYbdHn6Hs/n4/IfDe86s2ksgSlHKj4THAT//3e88+/Dnf+3t/O3P3au/vd/DITEw/w/GRNzTaOL", + "cyEJH0ZqprFXgUoTb5D9EfxuB0h5uCRCcjAct3pGf7KGqZ6ANnMRgzCNoX4V3eVCx8GRMbOIos+wmYY5", + "ZdsU0qSqhncyQqepZojW+dbVS5GD9dOVZoARsYTW55EydyEVzLS4KURxQYd4nf45TUN0YSevMSD/LNrc", + "fMKExCz0MlNrPKemTWkH7N0fE1Q0AMk6JAuY4ECXUvcp8Xmbm2udOCe7gLa2zSWtNM9F9Sy27Fm5pIIB", + "VCn3yvAdbWz2xQiHSxJBdJjnKJ5SAZxDt7LRvDSqkdzw+NAts9syu40zuy0b6mVDFTbQz4t8TKdgZD72", + "48Sj1J/sRNb2IBpmE3VdBEPJ0dnnLiop2qEixHMgbRQ99fW7Jd7jECI1qjOZWMyRQSWuh8UXqVK+hCyD", + "VcdTfJjlZ4SHxHu2FMLV4DlE9Wa6nQ5lHjJ2RMW18MUPSf1awuyljv7F4RLCdnaTMpxnaMSyG8bkjVdW", + "+L/sjf1hmsDW2Szd63N7HNAnZ2zrIl07GqhC7C2UWdnaJoAeL4ODILt39kxeFJyr6UzIhcv3pl/ZDoo4", + "pooDHxQ/IyrQLM0ZePtnBIllLlGU3rIpOpHaZ8dSCcabTCJGbh12jlmkWwiZZihVPBeDj48KUDSdlpyg", + "KGUaCMXWotmqCoOeRNIbEut9mKBZLhGVKMTMviSGN8U4WsHMYcokZTlBwC/ZAkmO53MaTr9WAwtwpC6e", + "duXA7iDuW/+RsyXBsVyuNGNVgA30CJToPzdzlL8cl7OVPx6585Y/f3YgKH+9sLBUNvpoidni8e6fvRGs", + "4wVj7UCYAdQqtDmrw6Fetcp129cfyS73IkL8N25IUpv06uIaojTB1KNuvcdC8Rb10XmdWtidNU9QHEbb", + "n+ksHhQITdhN/f1CDSHuuwQQHCAt2U1UNVQ+bljDY8UZbNKbb/agE5vwc2mEVag0+1XKEXRDMcp4erea", + "9u/gGp7+uqu+zRTfJIVcpjscmni8S8CcolIYThsqMmFqIdFo18AH06++WDuez37ZOsggV4hdpZkBzWO8", + "8C8SHevBtFfHzwUNLG1mjYdyInBUnRhX02GLm+q3JZFLwguXlHVT3WKByF0W05DKeFUsOOWKy5vFVzny", + "FH3K4xglBDOh9BY1gtJqnFEEkR2k62DmewgO2zjD3kAs2guUCDGdk3AVxkM9i6dF+81HyT3UVbgNstsG", + "2Q0JsmuQevN+bPBTnB6UpTENV0XcGJqtHFV7njaldtWn75cqla3ADOFShvpFYsouy6vBgI34tWjfMEWU", + "4LnDdugEp+nCnx1AxxNVw6PgwhNTRhp4gR+946gvXSkGnikNAAB8VcFDS9KFOSXGxdL2dqrNeVIie+OJ", + "G54LqwC/m2TBYK+KadGfX6Fq1+I5BPdFOuqzwezHcLGuVApx6nvuefoYc/ZyTJh74uKhhrMv++cm9ZkX", + "e315KQrmp1YjCow+GvZ8y3FW8NFRIIa9QbQ9emV7ZRJvwOdHN0RyKEtrt/h/atr6hz0yDLP8syDRWdiS", + "46LLsj+PUzfPjg2g1EISjMVthvQI3pO2PnptN6Orjv6X6PBEtdVw3mmYP8Lh0hcnrB3Vxor2Qwb8Tf32", + "4/gpOrHR4VHoHNSPiI89PoT2If+ckcUj4n0dldA5N+VeOFvtEJZDte7RcDhR9Ybhj3T91ZeJxcZUQAsS", + "oYgIadKMGq/ZgsNd0Lgk0AccLg32lB44Iwijo5PjczSL0/BaP7VHX4P/nML/dt/ufw1+nCCMZpgTdHKG", + "cBTBgLWG0CrlCNsLNUTW20bkDidZTKZhmnwNJuhr8L+mlZ9+nKJDswCbpgjHt3glkMTXBCk6JBFRu5re", + "EI4iwmjZdDoqZgQQdZbPYhpeapxUZJSP0C90wC+iFZ6PPp+fCuedR2kk0AmLgKVXn5n6NW0TRNy+t2a5", + "5S4JhelyL4h/p4/LjdB+L5ZKJPIsS9UNB7qoqRHP47FITLC4Ntktfk6FB3SLsmUqJLzzNJdCMHfMSGmU", + "gKhag1ATr+9N7QRAdsnpMQqDOW3nubY91FdXe1dK+I6hackxE4qdaJwhyMipU2klWIZLyhZ2F36+vDzb", + "Vf93USxrin4hK+uBVOOVhwhndNo4I40TYo9WDO9BEVisrMOy4oWyzGAHniJb5qHAzAhPqE4rXHFS1i4H", + "9+13Mxd3TW5dQZCLH4MWi4sSXwXDMDGyzettgfVxu1vAMmQ5l+4cLWsyTLVj04vVzcg85eApu8U8omzR", + "XNWS4IjwcVe4KmCKupAZRkFDmVqbYg2KSXIaEf3I2sBYkuEhKz3jur+Tjk2xbeDdVK0ni3FIoimCJ8ma", + "drNY7ZYGSvwfJHSqVE5EGufgaljiLCNMGOPvjlCAGIQIwiJwkKc2GntN8vucKaWizQ3yqfL0wrpAcuij", + "uU4RAFaYzM/1WkUlh5ndVzjtBY4ynt7QiETV8afo14RKqWkabpgojAnmAlE59QYjbSX640n0V/0M6BVq", + "AC9RHjuHuBL9Y46v2ll7dKejmM6vri207moB3qKImjXN9XB9QGkODMAGApn0v+Co9D6Yrtjix77ONozN", + "ThB1zeAazx/yvKWZowP7kmic4TKFRltffzYyGK/DYEzEb1QuW9OUFc7gLoId5kHiNLQp/53YomJ8uNKZ", + "oBz/k2sTtdVIeZbHsTmodmttbI+T5jiMcy0/CU50a7CgYAZpgA1zVh93RJwvdpPVjh3l4Gb/x1EH3HYc", + "6OPqAnYJuY6n6LPiUwXUu+BV12cGa+Fyi0UpXLsWY65rSkDJJeG3VBDFrWOBZji8tuoFx7clPCfHZkQ8", + "C9/svy2GmPbSoIOJidk+HyleEpx4bu5QksXDN0yeROu6V+v0pg0Vx/Zq2uXaAYIwHjezstqQjrgZkn7Q", + "D01Z6qPfI+gboeGQM8VBzDE3yHJXfWUwu03u2Roa+afPzWmox5sf9pFSLoQpM3eCC1eWNBMRlCH3ZRcn", + "/Lh23AeY592HWudehcBbGEC7saHikTaLDjLbb+2/ffZfDx149shSHnCBBs8iiYky68PwB9XQLjwXECnd", + "eziH8RczWg9z8Z02Db1eoQl58wfM2SPSEz+tmz6ggoGpVtDrF9PKeCVcRXFC1VkOO4sjCj8opuyU5zAl", + "CdTx19lHusIFZ2Xm/z4ua7fAKRawbnRdjyQtLzcV7JVhPc8kTtfPGLd2nBsW8iLDt2w0soAoHiZ51wiT", + "a7lxfHIvGwWYP9T1cw2nXlDxLRp3l8jA3dKnwdqLg0C6PVgyWbxyPSuzlUe7dFRbofZlXU5Q35kOP+la", + "0XG+86CNkusRku66ZoCMGwtXlngcEPJmNtNlGO4y3CNePyuV/amw7ep5nBQCxFJvlSm6wgfkR3sQyuZI", + "77FoomujzGrc9QPnf3iNoiEXjSeVKlpAriNSNi8B5pRRsRy3Kttn8LLWYfXiIUrDYFZULurhfKhkPcVj", + "8la+4uFNjZPwE43J5yxOsedMZJwI7ytmlxnMaQyMAMf6gafpZM35oQnoaJ7/nHvCez7z2HlaAGOXEQM5", + "wAmuxF48WdgbC/ZbGdc4/k27wtDqUgDHujF8vaWkBoQRlgCM82EUZbV6AazU4XroQduEpPCcK38wZwXG", + "03QhHhTQ+ZSk0BbMWVlBa82VB78mXefBUhpeE65OvSdasfjmGIXap19HGgADO0oi36tixdrCJQmv4UUQ", + "1q/uyR0Jc11PsaIXlU+YW5kFGJy8c4FV5JFmeWT7s7M/bYT0Zf9lkNI6++9ia+zLvUH404hoRd3bTtQN", + "MAvVkTlFx0W3CcQ26UgVJiTB0fQ5cT28pNMUHWFm3GMEYXDcgfU5TOOUIUEyDCl6ilCLZLVj+34N1M2k", + "8tPBzRuItjiZw0hU2KEjiFywAXfSVmETNowR5nUdbvY84oVAwHKn46tP+Yu+Fgd6ROnXx6fdOpkCY/Kd", + "73lapPrueobsao23yzS2inGp4MFAwPN4zhAnC8yjmIiCrtuVybmt0+PhdepnW2YECwjkEU0h0s5E574a", + "QF103iwaZEZxDcB114qB4gFwfn/iS0iS9ZZqNSE/0LZrvsaZGqKJXkiSeTUrjzu7qbv2JPdpgGZjUuBv", + "HZRyi6nJO2Pz4bQXDrAgnJIFDlc9XoatT+HRdY6tR+A79Qhs7fFbe/x69nhX1zdqvrUXtKr7G/YDPz0v", + "HeNQe6F+sg4tXqvwZYH4BynxmzR0FQeh6TwqqtlW9CB/pXivnqVuSRBW3ojj5702sUO+yBPFi8s8PWr2", + "MYiE9G4/Y+EJqlW/WgxCs+LdpzNT8w4w/oqjhnqUu013hcZ2qH0FE909vcSLhxvCFfmnIYW7snnwQYW6", + "Ew9ykA1WmMzd25614dFSeOGPO1Mj+sst19EGS3EkTC1cTuNSv69pNaZuilHde0Bqs04/d5iDJ3K9ynN+", + "o3JZpkZ/fkHZkaHdpGb3GKZH3Ta1b9qXuH0jN4vnVMu3QTdbJX9QLIdPXWnT5Pu1d81xNKtco1IOudUW", + "dXvwR5fLeYRaOS3pwby7fjzyOVEx1KS9ro5ewiGL1q501r4U2ZKvL1eKBmTrc8pHw6NEDGmatSqix+3K", + "sPSEqLLAX/kezkHpLCpXF0oAaDQ5CQrV8kAXI5gT/pM9N5rD/GELwoHwAM4CzUoAl1KCzfIwSiirDEjV", + "yvQLZgvmQfB/d6DhzmW10Jx5W6LGgX/1jXF2svOLS/lO/1wuz7TThL8HWB9rgc7AcHi7QVRNdmCDGiBe", + "5BmeYUHeDEGXbdyOMdtifwBU5WgVcWEHU9RCTciKpFJJ0ODD/nvFaZxKEAfB3vTNdA/y0GWE4YwGB8Hb", + "6d50zzwGBBLb1QjeAQRrpcqbjOJIZ9DGkDS+VoZQnWx4EHQSBQfBWSqkQ7gi0GeCCPk+jVbmIYg0MUSQ", + "CEAn9tj9pwkk0RpQb5ryajHF2ntCY8bkRtOFhe3vvXm02Y+MdKlD0JFD1Agkx2ASA2G802D5ZivA31WN", + "7ifBX/b2+tuqRi5DAVOwj5p/v7qffGs9ir9f3V9ZI8fvQZVMrtT4VdLZ/YZLZJwc32sSionPs3UMvyPM", + "uilJN3Np6dCdAsiY44RIyAHRYu8um+xWAAS7d40+3vWkgdXredgWvtOz9LV99wK3W4mEXaXBit1v2rd8", + "v6s1pt0Qs1AnQmxhH/BdvymnbCfjqX5mj1mEMpPQonZh0rkBIEuI5oIeNgPSTAF0CeDom6Geq0kdnodr", + "QETAhOF1dMGCi0eVVY4ycbhD36ufJnXtPRr3gXXDYvVaz4nIY+njQBcOpSK9SXFRNuxlkm5dRdGEKfIk", + "wXxVkBJQkkMxuLhaWQpWw3RRbvHQbxeSCbSS7i80NoTbfIK4Bo0Wb95+sSkMvmciNatVax1IpGoz3Nzz", + "r5NI1YI9NNNNpRnduSYr2IgFactkogaFl+rmXikaVPd3IrV6rlWvB2zvQENVcUVu2vO697pIz9xc1DMr", + "Rl59vU+Att0+aqLVbrO66w9Qtl28+DmMs9lPome7O/wsanYdAA+TrOQmeGFa9maIyWUhu9/0XXGgHt5N", + "Y0YN11R2aMYdr3zbjsP07sqmvna9e2PcBMvQ497Upsy+bT5TnR95lx+fHTXMsoM40l4PgRkD8ZbAejmM", + "rm7XqqL8DJ91WKlPMdHfg0EcYGmKAYbgWrFl9UbtChDHLksjMkCr0s08QH8yHzpV9a7KfKC8/ysnkNLD", + "aO+VGrAbU9gLjW5YkCDUx7+/epBep9G6MRHs18x9+jYAtvtN/cfISS99/J1IU9eSzdNW8vgEo4zml3ry", + "QB3U10hOfbRjioENppeiqu4ruvTVSatVqYdqn0gU4d7YFhBuqvSPQVJPJIAb5UvvjQQews/hHBkMQDQC", + "DPEa5O5wtlJJlNgtcWoJdP3Sx02S1CmBiroawBp0CJtM0ZzGslq8iJR5anNB+H/hWfg139vb/yvOsv/K", + "eBrBYyhIiauUI8widKNzFye5kGhG0OfzU0RYmJosoD6GVBQWc/nRM4uzU0j6bYulPlCuNTcPiHFvCDHu", + "bVAeOp5arRFuXIWspvbsMXk06vM6QShNRukejieyfhTkslnTR2Vaj2bsFLNrt3lsibGTGCvsejcpU9+2", + "s223mL6Och/GvG1e3R4efpQmCd4x71dJBGUKnKyE6OQY3rMtSAWSYBKQuyxW2oSNQfaxZDPIHzQSnV6D", + "9vi4BN+d6I9v9vZqzHMS5Iz+KyemAZyPJ1UwvXmLH8bCdYyQJYTtERp7hL4VFSE77Y7aO+Ik3vYZHIvt", + "vXCqTI5Thcv6lAONjjXGan1RL187fWlCvvUmXQr42QrBnbSdZz7Rxj86B1rnlmtpf0tOD+Axu2HKGNFF", + "9/xq5TngXBREF+kEyjrVg3PaqTBFzysJH3Rt12iKLi9PVRMI6iZ3kjBz8elQSAviPTIwPpSGH1+5NZCN", + "UnD3nkPBtXnEbF2G+8lzqdqGIjamam/Pe+W82+xZhXjpfuajBI5wyqNC+QS0e7PfMbybCWeAaDrVCcHW", + "PtoTb64MKPngKecrkFxi6bwgLGQKZSihcUxN/u4Wowyk6PBbiO2bkYQymuRJcLDnS/LdsDnhO9XaSdne", + "BWULVDFNaBWq4kXlmz11z2ikgOgEcgMSH3Z9HXmvE+xtmcBDmEDfPd099Ulx7R5wllvv6A84zkVqfX2U", + "y9fLmEt7sOGRyg2OJ+oUmwM8gaa64GOZsv8Jz7VvWAIlD9xjOWBphEXrLWwcyFebiG2r1Sxa11zsMoAN", + "GBe2/KLCL0z1OP3mWraGxzQLzdmihxuoduipdVi7W+QehmVqvr3Aq0VHpclB1wyPcaZytGzAjnuyvJv3", + "lGfn3d7fhrT925/knOlifK1X8TP1uVZwb8j9Gfpt3PSnrQGVex44Qk11UHMP3FLXxqiLkzknYklEl7EH", + "mlSYgrbWKBZLpTB1vFIU0xsykPzOi3mfh8tWH05Hlq01Y3ldaVWqVxYP5WXsmmQSYYUBRyuD0l93Wtt6", + "+1d15+rWFxvvl+/H83C7oxsyiL5iyhc2H0BB9t12j3PosQav1R1foD6hAYtevi++3UC4lRIbOCu2zuiQ", + "l+wZ4YIKCcUFbenVIurEjPk/RaHZCwmJp21lWmFVARtnpaMw4MlO6U/W71thHjQjq5RpZpdyuqAMx840", + "MZ0TJZ6GuhUKOF6EXNKJFSumOx9jKvLmpAVWdM8Cu0TqkuGqSbErERXXJqMOixAnszSVKGWIkTupTSnT", + "EUVaf8106eJqmpZG9V3wEWFPVd4ypRkMYJ0TUCpbTMyrLVPfxTiS6vkAi7LaTqlfsDpD0Usorwol8NVI", + "KSP++roNEbxRlugWaPYZQizaNh2ctDV+VBiiLIuu+9mhOnEQZqcblufBmjkqEo5CTfiMcoLurO7ohOrR", + "MoGP4StTdITjWJewpgIlRC7TCCV5LGkWE1vc/YbwW06lsaVcXp5OEMGhLiaLcmErYFtWXBodsSjNqapV", + "llL1PUUJwSI3JYPs0qzyPJTF2oL1L4HByrbi+QZIR5cv98PFl8mt3noz0LsajHW2NMu+KiivHuWCIAxp", + "Wkjt6FuOMJAjuMpQdyR60bReXbnpNmnXOzpiDmvvUmp1+3WEegnDbIVEmvOQOHGPPhHYexIzrNQsNc0p", + "+BhHdflE7qRJ7bMZZ0NFpK7rayg3/U8VklisWpM+5PoYlkjB6xy8NB82+ToBkn098FGCXtDmdr6eoK5r", + "+0dkH1K/ORtZJm0Z4vt1Q7S7OJmTlGVdz69JwbJ1+35fbl9FFI/h84X39htx+L4d0vbtq2fzvYxhN8F3", + "ncwBaM8ELfkYhU36rt92WEoexj4+4rstB3nxHGTieTfJaQhZ+NW/yA2pUAk8fTSvbFoeOnLIQdz+oMbW", + "cApTZq5kf7ivhuy7HNiMPziWxFPK6UnD2j7iO5fnbXncS+Fx2jo4SJO1Tb2sqvw44M5WJJZrO8CDywVe", + "bVqDNk9FH6xFW3w94x1qlG79hPRYIqP6erfbBVlLYdbxhNclzqdwHXpLGw+ylu8/OgymCl+LH7GsCY/D", + "kGTSxpe8uCeIL5AyK+xy15QD2P0G/2jP7XIE5S/pvObc0cqfLiOh3TqdXNVUE4H/tHDYauZObFq2awwt", + "lUBtx03qBnZyWF5ROqaPnyqRovOi2hJQKwP9aJIeLvf/NJbedsovi5J1pvwrHTB4YR0GrRqE7lOQ+yVe", + "PBWvrs6kJhrFsN+1lGNrzx+49UE+jA5tTTi/b/HQVLLGix/Ej1DaulkJr0MheEJC05CtTWhvHhkQErmg", + "ePUDvCijCLZ0/DR0XGWl38pSQ0OzqLaouXUGWilhNNJcVHQdHtdcqcD0GLlUv/9rTvdlu5Z3vmXXXRXx", + "kbZ8TfflGo7STWiUTjG/kSYnV1mnUqxZz+C7UQErmX+77+X2sVPrjVwN9CSc6ulu9tVCm2snBG5U1WtN", + "CvzycwK9buPROTF1ZNlA09HroNPXa4HaWpXqCqEtfPTNVAG+H/M+A3I8lPWSh9K2lpXvy2LdT6hH2OLG", + "HkVg3887Ne0ssdA+yi3pDCadZlqThte6Wkg91klHhqibFZpZK1nJmnSz2cQmYc4FeGhfU2YTX6ytfYT3", + "pv8Nnt95F1FOQljDZKBoUFRxXPRqHTgmN1BVbfCgp9DBg9oLHds5ZPfnPE3aQhdglFGr1BNvyF4OZ07N", + "Othm7r/kOEf+ZZpv/Mz3RVvJu9mxTlA+hiG3lQboY8g6kfqzseQTFpE7ew6LNxQFwbWeyiKvhVvU3scy", + "0oX4dT4XpIUHjk7t9N1w6bWZ6cY4V+vDrl6OtWVTT8Km5jRWPy2xWHYXK8EM5Vmc4gjFlF1bKyXmSI0A", + "lc8xZc5Bxyuivw3VKX9SbX/GYvlQxuVxjS/1sEM94woKy8DsEvqd42+e5sgovHwGzLfdr919uV0SDs9c", + "zY9whMwufQdOndd+3KwjvSfsDtzn6/gBjHPzMd0/TxIJX7glHxoKbxQmwOsTxn7+GX2WbkLTAW/9ugoW", + "fNn/nuvNTNqeHhaAzlYoZQSlHCUp17WKABOD6itIffzXSzJ4IY3uVD1kk0DIVax+UGroa3IBbovzvMZH", + "w53JkQflXG2zTjqs5ZWmT36VVsa+C+zeWJgLW+EQzLaA/ChWyhoq9SLAcAHXa/OiJ+dsilRvNCNxeqvz", + "UOgGmBNE7sI4j9px+2hWzyMsyI4gTFBJbwgS+UyLJZRgGS5RygDyhAiBF/qaprhsi6QhmIfLClgJvjsl", + "bKEO+P5f/rrZEGEnK/aX/fXMndv82Ovx6soTocd/nPFl/zmeZ3zZf+nucYOJbdG19e7cLuE24iy7q5h3", + "Ry459Pp9xy49CRDtjHsbHPWcp6In2GRsaIn3kDxfcMkTyxTAyCiJ8rJiW14g937bpnasqWS8fRYl4+1z", + "KRkGAMtvLSBbfePpKDaN84QMTAiFbGufDaP49PS2dz3XaLN7DHa05mr+TPtv1zyg3rTmTgWe/AzK2fUn", + "qTFtt3qzb8/0rIcsMjbiHsKymVybONuyoT4ydJjQ7jf9j+GvytqJUzcy5PnFDDtaebPwDHxSViEK+5wM", + "Nwlia8Dp5EsdsWUFIlsDy55yy/eei8HY9EtbahrNXgA6fmN3P+dxcBAspczEwe4uzuiU7M+mOMsCp/+3", + "Mt9Pme7mWy1xa/VHyE3k/g27tyPVgqsNM7pzTVaV34znv/i7UEyu7v87AAD//w==", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/packages/api/internal/handlers/sandbox_create.go b/packages/api/internal/handlers/sandbox_create.go index 897df8573e..cc1760ad0c 100644 --- a/packages/api/internal/handlers/sandbox_create.go +++ b/packages/api/internal/handlers/sandbox_create.go @@ -148,7 +148,6 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { ) autoPause := sharedUtils.DerefOrDefault(body.AutoPause, sandbox.AutoPauseDefault) - autoPauseMemory := body.AutoPauseMemory envVars := sharedUtils.DerefOrDefault(body.EnvVars, nil) mcp := sharedUtils.DerefOrDefault(body.Mcp, nil) metadata := sharedUtils.DerefOrDefault(body.Metadata, nil) @@ -258,7 +257,6 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { TemplateID: env.TemplateID, BaseTemplateID: env.TemplateID, AutoPause: autoPause, - AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, VolumeMounts: sbxVolumeMounts, EnvdAccessToken: envdAccessToken, diff --git a/packages/api/internal/handlers/sandbox_resume.go b/packages/api/internal/handlers/sandbox_resume.go index d2d78228fc..9b0f64ab6b 100644 --- a/packages/api/internal/handlers/sandbox_resume.go +++ b/packages/api/internal/handlers/sandbox_resume.go @@ -235,10 +235,8 @@ func (a *APIStore) buildResumeSandboxData(sandboxID string, autoPauseOverride *b var network *types.SandboxNetworkConfig var autoResume *types.SandboxAutoResumeConfig var volumes []*types.SandboxVolumeMountConfig - var autoPauseMemory *bool if snap.Config != nil { network = snap.Config.Network - autoPauseMemory = snap.Config.AutoPauseMemory autoResume = snap.Config.AutoResume volumes = snap.Config.VolumeMounts } @@ -252,7 +250,6 @@ func (a *APIStore) buildResumeSandboxData(sandboxID string, autoPauseOverride *b TemplateID: snap.EnvID, BaseTemplateID: snap.BaseEnvID, AutoPause: autoPause, - AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, VolumeMounts: convertDatabaseMountsToOrchestratorMounts(volumes), EnvdAccessToken: envdAccessToken, diff --git a/packages/api/internal/orchestrator/create_instance.go b/packages/api/internal/orchestrator/create_instance.go index 0dde32f8f4..f1389cfc0f 100644 --- a/packages/api/internal/orchestrator/create_instance.go +++ b/packages/api/internal/orchestrator/create_instance.go @@ -47,7 +47,6 @@ type SandboxMetadata struct { TemplateID string BaseTemplateID string AutoPause bool - AutoPauseMemory *bool AutoResume *types.SandboxAutoResumeConfig VolumeMounts []*orchestrator.SandboxVolumeMount EnvdAccessToken *string @@ -354,7 +353,6 @@ func (o *Orchestrator) CreateSandbox( node.ID, node.ClusterID, sbxData.AutoPause, - sbxData.AutoPauseMemory, sbxData.AutoResume, sbxData.EnvdAccessToken, sbxData.AllowInternetAccess, diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index 67340768fd..ca47c489e3 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -136,8 +136,7 @@ func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sa switch opts.Action { case sandbox.StateActionPause: - skipMemory := opts.SkipMemory || (opts.Eviction && sbx.AutoPauseMemory != nil && !*sbx.AutoPauseMemory) - err := o.pauseSandbox(ctx, node, sbx, skipMemory) + err := o.pauseSandbox(ctx, node, sbx, opts.SkipMemory) if err != nil { if dberrors.IsForeignKeyViolation(err) { killErr := o.killSandboxOnNode(ctx, node, sbx) diff --git a/packages/api/internal/orchestrator/nodemanager/sandboxes.go b/packages/api/internal/orchestrator/nodemanager/sandboxes.go index 074fd216d2..658f27eaed 100644 --- a/packages/api/internal/orchestrator/nodemanager/sandboxes.go +++ b/packages/api/internal/orchestrator/nodemanager/sandboxes.go @@ -131,7 +131,6 @@ func (n *Node) GetSandboxes(ctx context.Context) ([]sandbox.Sandbox, error) { n.ID, n.ClusterID, config.GetAutoPause(), - nil, autoResume, config.EnvdAccessToken, //nolint:protogetter // we need the nil check too config.AllowInternetAccess, //nolint:protogetter // we need the nil check too diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index f3111d5708..a17aa6390b 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -134,11 +134,10 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, skip AllowInternetAccess: sbx.AllowInternetAccess, AutoPause: sbx.AutoPause, Config: &types.PausedSandboxConfig{ - Version: types.PausedSandboxConfigVersion, - Network: sbx.Network, - AutoPauseMemory: sbx.AutoPauseMemory, - AutoResume: sbx.AutoResume, - VolumeMounts: sbx.VolumeMounts, + Version: types.PausedSandboxConfigVersion, + Network: sbx.Network, + AutoResume: sbx.AutoResume, + VolumeMounts: sbx.VolumeMounts, }, OriginNodeID: node.ID, Status: types.BuildStatusSnapshotting, diff --git a/packages/api/internal/sandbox/sandboxtypes/sandbox.go b/packages/api/internal/sandbox/sandboxtypes/sandbox.go index 9f89d9fd6b..ee0c9256c4 100644 --- a/packages/api/internal/sandbox/sandboxtypes/sandbox.go +++ b/packages/api/internal/sandbox/sandboxtypes/sandbox.go @@ -31,7 +31,6 @@ func NewSandbox( nodeID string, clusterID uuid.UUID, autoPause bool, - autoPauseMemory *bool, autoResume *types.SandboxAutoResumeConfig, envdAccessToken *string, allowInternetAccess *bool, @@ -67,7 +66,6 @@ func NewSandbox( NodeID: nodeID, ClusterID: clusterID, AutoPause: autoPause, - AutoPauseMemory: autoPauseMemory, AutoResume: autoResume, State: StateRunning, BaseTemplateID: baseTemplateID, @@ -103,7 +101,6 @@ type Sandbox struct { NodeID string `json:"nodeID"` ClusterID uuid.UUID `json:"clusterID"` AutoPause bool `json:"autoPause"` - AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` AutoResume *types.SandboxAutoResumeConfig `json:"autoResume,omitempty"` Network *types.SandboxNetworkConfig `json:"network"` VolumeMounts []*types.SandboxVolumeMountConfig `json:"volumeMounts"` diff --git a/packages/api/internal/sandbox/store_test.go b/packages/api/internal/sandbox/store_test.go index 97793e97e0..a40b323b46 100644 --- a/packages/api/internal/sandbox/store_test.go +++ b/packages/api/internal/sandbox/store_test.go @@ -206,7 +206,6 @@ func createTestSandbox() Sandbox { "node-1", uuid.New(), false, // autoPause - nil, // autoPauseMemory nil, // autoResume nil, // envdAccessToken nil, // allowInternetAccess diff --git a/packages/db/pkg/types/types.go b/packages/db/pkg/types/types.go index d551ea2c03..996dd1523f 100644 --- a/packages/db/pkg/types/types.go +++ b/packages/db/pkg/types/types.go @@ -109,11 +109,10 @@ type SandboxAutoResumeConfig struct { } type PausedSandboxConfig struct { - Version string `json:"version"` - Network *SandboxNetworkConfig `json:"network,omitempty"` - AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` - AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` - VolumeMounts []*SandboxVolumeMountConfig `json:"volumeMounts,omitempty"` + Version string `json:"version"` + Network *SandboxNetworkConfig `json:"network,omitempty"` + AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` + VolumeMounts []*SandboxVolumeMountConfig `json:"volumeMounts,omitempty"` } func (c PausedSandboxConfig) Value() (driver.Value, error) { diff --git a/spec/openapi.yml b/spec/openapi.yml index bfddf2c3b9..babbf23767 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -689,10 +689,6 @@ components: type: boolean default: false description: Automatically pauses the sandbox after the timeout - autoPauseMemory: - type: boolean - default: true - description: Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. autoResume: $ref: "#/components/schemas/SandboxAutoResumeConfig" secure: diff --git a/tests/integration/internal/api/generated.go b/tests/integration/internal/api/generated.go index eda599fd79..153b5a2a33 100644 --- a/tests/integration/internal/api/generated.go +++ b/tests/integration/internal/api/generated.go @@ -556,9 +556,6 @@ type NewSandbox struct { // AutoPause Automatically pauses the sandbox after the timeout AutoPause *bool `json:"autoPause,omitempty"` - // AutoPauseMemory Whether auto-pause should persist memory state. Set false to snapshot disk only and reboot on next start. - AutoPauseMemory *bool `json:"autoPauseMemory,omitempty"` - // AutoResume Auto-resume configuration for paused sandboxes. AutoResume *SandboxAutoResumeConfig `json:"autoResume,omitempty"` EnvVars *EnvVars `json:"envVars,omitempty"`