diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go index 6cea337545..6cb5aeb971 100644 --- a/packages/api/internal/api/api.gen.go +++ b/packages/api/internal/api/api.gen.go @@ -330,6 +330,9 @@ type CPUCount = int32 // ConnectSandbox defines model for ConnectSandbox. type ConnectSandbox struct { + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Timeout in seconds from the current time after which the sandbox should expire Timeout int32 `json:"timeout"` } @@ -714,6 +717,9 @@ type ResumedSandbox struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set AutoPause *bool `json:"autoPause,omitempty"` + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Time to live for the sandbox in seconds. Timeout *int32 `json:"timeout,omitempty"` } @@ -1510,6 +1516,9 @@ type PostSandboxesSandboxIDRefreshesJSONBody struct { // PostSandboxesSandboxIDSnapshotsJSONBody defines parameters for PostSandboxesSandboxIDSnapshots. type PostSandboxesSandboxIDSnapshotsJSONBody struct { + // Memory Whether to persist memory state. Set false to snapshot disk only and reboot on next start. + Memory *bool `json:"memory,omitempty"` + // Name Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. Name *string `json:"name,omitempty"` } @@ -13409,175 +13418,177 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7H37UyM50uC/oqj7Im7mzhia7t34lovvBxq6d9gBmuDRc3HT3IRcJdta6rWSCvB28L9/oZRUpapSvYxt", - "oMexETuNS49UKjOVykxlfvf8JEqTmMSCewffvRQzHBFBGPyFfZ9wfp3ckfjkWP5AY+/AS7GYeyMvxhHx", - "DiptRh4j/8ooI4F3IFhGRh735yTCsrNYpLIDF4zGM+/paeThlP5KFs1Dm8/DRp1kNAwaBzVfh40ZJwFp", - "HFJ/HDZiimc0xoIm8SmNqJCNAsJ9RlP5m3fgneFHGmURirNoQhhKpogKEnEkEsSIyFiMUsJQimfEGymo", - "/pURtijACmFcG4qATHEWCu/g3d7eyJsmLMLCO/BoLN7veyMvUjPqzxGN9V8jAz6NBZkRVoH/nDwK2P/6", - "Go4yxhMmQeYCM4HEnKCQcoGmLIkawI7z4doRyHEcTJLHxl0pvg/bGEFw1Dio/jh0xCgNsSAto+YNho18", - "n4RZ1Dxu/nnIqE+yMU+TmBMQAh/29uR//CQWJAY6xWkaUh/2fvefPIF9L8b7D0am3oH3P3YLybKrvvLd", - "T4wlTM1RJpSPOEASRMKF9zTyPuy9W/+ch5mYk1joURFR7eTk79c/+eeETWgQkFjN+GH9M54nAk2TLA7U", - "jH9b/4xHSTwNqQ87+pdNUNEVYfeEmZ18MlQOZHz429UlmVEu2AIOOpakhAmqaBw/8EM4x+R5E9Tl2OFv", - "V0g1QL+SBTo5RtOEoU9HlwiXiMgbVdlpJMeWEyexe1j1DT3MCSMgH+WoTEOKKEdh4mNBgoahr4jPiMiB", - "d8+hGtkr6A+++qE66vUiJfJIygGtDURieXb8LmH0bkcO2VVIpN/V11F1G5wLtBFajJtM/kkUoR0GEY0/", - "ykP+CMc+CS8JhyOvuuU+fA1JcJRkseP4Pc+PXdAYOOIZwDDNwnCB8t5e/XAceVNMBwws5lgg1UWelGpo", - "z3no2jirLKA8663BxJU6BX+lYSMmekKrz1NSA/iOhqETDfLDoIFLKFa9u/Fgz+JAAud0Fl/rA/Yaz/il", - "PmZqeBB4xh2Ujmegc2EYSP5LMqk5saUOI7Uyx0GaA44Zwwv4G7MZEa4p5O/5mIjG6Bsc4QcCz755SCtq", - "nUykhh+phRSLJ4G9/Pq6LX25DNdJIDl6StU2yWVDU4mKxKdSKKEHKubyCycIZrW0yiyjTqHlRrMBFYYx", - "0y2B5RpOACizRIkUkA2nyexT7DwKQnJPwq4T6DSZnUK7p5EXEc6lEl5b0mkyQ/ojMueeAx9ckLTe+UqQ", - "VBJCgfWUJSC+GQkB9ZoSw2SGCCzFhWsaES5w5Jjg2nwyyLYHyjcxwILsyFG6qS+fqkDJSGMzR/uVwCLj", - "lwTr876CerUp+q/8svL77ciBWaJaVtHBYQbE1BQW3bRtZ5kkHJzbuMdnen8NH5TnHyE/Y4zEIlwgRtKE", - "CRrPUBKH6gAGPUX3GEgZlgju3BkDvNyFo4ubBnl8dHGD/IQRDqDBUpRc9lw3xZa74UjqfTHxhT56HIKW", - "RiTJhJsmk0xIuufET+KAw0URoNGYRLIzwlNBGHqYU39ug4r4PMnCAJHHlDLSCvhe57lioHQpGUeMSKI7", - "LGwfDgVDtxEdvKcMKEjIURB0UgpUHx4ceTToI7ftOfrI6Ajzuy6mKWY5w/yOxrNjIjANueyv7p+1Ix9H", - "pAGiuuRyGxSu5wRpDUyht2Ogyp7CagE4M4Ne68jarttig68Jjg4vTrRivdz+Hl6coDuyGL61eoKPMDcO", - "wy9T7+D39j2R8N5wScy3Iy/OwhBPQqKu/L1pRcPbh0zuXBeOS/yA7nGYkfqAtQFCzMUNJw64TjHXvC7m", - "lOdIfMAcZRyEnhOJ5TW/CGU3LtdFi6qhJkFNmGVKPCYhEaSXAtsNm6VQ9dTLjPobABjLK2KG6Yxqekz5", - "3RkRjPoOjTQg99R3LOUYfkdmrCoAUxoSvuCCRNfOS+vn/DuSfdFPZDwbjxB5FB9G6HHKf3aKQnlcXiTU", - "dWaeyW8olR8NhgMKW+mQZwKHHxeCuHAsvyGeYh90/wm0stmPxuKvH5xXLMkLDaNKvlpm0Kr2UKx/ZDam", - "hmobkNJazVZf0X+Ts4+OHaX8DnH6b1LVOiTMZ/Tj0DN85H2K779i7b4IAirnweFFhbxsED7F95QlcSSV", - "i3vMqBQfLiWozs2f4vvgK2HcadvRHwxdkPg+QCyLY6kBar2+ceyRp0xc9TMnCRx0DY0RfHOgq46iRm1W", - "zdoluPREtlr5mSXRSYRnxDaxBVSOHdEYC7WWCKepHFAZ3Jqkr22oG3kzP21q+PejC6shy2duaE1iwnCY", - "93gaGdwuzrUVXq76aeQlMelx1NpgPo3a29qQdratwinxaw9QIwpOmOTKQ9+XrPoP7qLGK9UG6UboH1df", - "zoHG/350sQEjoNzFvkZAx3JcKngVTzW0pJjzh4Q5dIsL/UWeaxkvRA8rqGnlGMjHvnUMnnHC3If3jf7S", - "H1Q3UvMZRgVeXFhtVH1q6JU6Cwm+SkXvgpEpfXTgGX4HfU2KPNUD3ZcFo7r3JKxJRbTmucqmznnU78+c", - "J21fBFy4qcEOrw2JNKJr44IqfErimZg7tFz4vR3EpoNZA1yeYeTYFxcOpVA5pVyQoPGWjkOKXYY6+XMf", - "fdIPKYmFsSumjCg3hlbMu24hqrdz3DTLTRhtgjQ3dTyN5FFkqSBtvSxl5Ulyb+P9Dj3MSekYRw80DB2m", - "h9Y7HimrEK1eL6spHOJRwhbdCzoz7aCPwAEWnQ42TRNnpnnV2961eS2KDcQBkCFYxRzpTr2xyoWkyX6L", - "vIK2NS991xJzYz0YqJQlivIS5Poe5xQK4JiH6wOwWC8rpQb4a9G32/xtBxbYARE5c9o7YvGWRV8l7jEs", - "YXBcpmCQKsY07jBcSnzVSMSckAGZZDOICZkm3sh7wAzOT1BJXYfmaTLjx5QRXzj17/yTZd/WrittJZwQ", - "HUgDe2TAmCbsATP5ywT7d/DP2uwj73FHtt+5x3CqctmxBM/nfJTSzx/zIfUCrpKMuW666veBoMvdThgG", - "rSCVW8LB59AffDXrtTVM8euFNeDTyDvD/pzG5ERuVv2akmaHzJ9TQXyRMeI2NmOrhVlorK4WLpn/GUc0", - "XLiHmsK3HoOcJYGLMuUYkfzUd4hzp7JWDBNbNhf3WNU7Vb5AC87KfKMaXtVGPF4THClbikOoEhyhCD5q", - "J4Xlp6mb5S1nUfuJXXMf6TmGeJAs/9RN7NK9WieRqp7spqyEPxmHAaexTxBJE3/+c+U63GBDAf3JbWrW", - "EXFle6aOUyKBAUdf52f0nsRIDszuseURVwF8rQ6zMh4MSLC9ftpiyqhFwJwdXSA/iad0ljEV1lQ3ZDTY", - "SItLwJmlWlTdXfLLMraad/v/6cL9OXlodaI815HgskLeqnlbFN8wefgD9jEm4g81gUsRDpOHHAUiySGZ", - "E2Q6j9FvUp/hRMgGUxxyMkJUoAmZ43ti1IWIIKnkpMSn0wWNZygg8eJLBn32xvC/3T1DZTERDwm707s8", - "LpY8SZKQYNANcSaSC5xxUvKjqunrQXBJhOWFNQwXKJWdylqMcrWByqMdYk0zXhKeRX3VrsO8wxEsRCvD", - "xnTXoQhDM6nQKu5o1X/99Jmqr8Z4z57nqnWxKk585xl4Bb8jHIZIG6X9JIqy2MQjgrSuadIWzocprIYN", - "2n0AtmfWxAr/xSX7JW2G9N5pt9WieDzcePsCerGWBm2evtX5fGz5o+BdZjaFJTDlCClnvAPv//+Od/59", - "uPP/9nb+9sfO7f/+j56QOIT/uTYxVzS6MOOCsH6kphs7Fagkcga7H8HvZoCE+XPCBQPDcaNn9LMxTHUE", - "lumLGIRL9PWrqC5XKh6NDJmF5336zdTPKdukkEZlNbxVEFpNlUA0zre2XpIcjJ+uMAMMiOkzPo8kthdS", - "wkyDm4LnF3SIm+meUzdEV2byigByz6LMzScxFzj2ncLUGM+pblPYATv3Rwf39ECyCo0CIdjTpdTOJS5v", - "c32tI4uzc2gr21zQSp0vyrzYsGfFknIBUKbcWy13lLHZFavrz0kAUVoOVjylHCSHamWiamlQIbn+cZpb", - "YbcVdhsXdlsx1CmGSmKgWxa5hE4uyFzix4pHqT6dCYztgdfMJvK6CIaSo4ubNirJ26E81LInbeQ91fW7", - "Id7jECI1yjMpI+7QoBLbw+KKVCleJBZBo8Mp3k+zC8J84uQtiXA5eAbRtalqp0KK+4wdUH7HXfFDQr1a", - "0HuponCxP4ewnd2oCOfpGzlshzE544Yl/q87Y39iRWDLbJbqddMcB3RujW1cpEtHA5WIvYEyS1tbB9Dh", - "ZbAQZPbO8ORVLrnqzoSM23Jv/C3eQQHDVErgg/xnRDmaJFkM3v4JQXyeCRQkD/EYnQjls4sTAcabVKCY", - "PFjiHMeBasFFkqJEylwMPj7KQdG0WjKCgiRWQEixFkwWZRjUJILek1DtwwhNMoGoQD6OzYteeNuLgwXM", - "7CexoHFGEMjLeIYEw9Mp9cffyoEFOJAXT7NyEHcQf63+yOI5waGYL5RglYD19AgU6L/UcxS/HBezFT8e", - "2fMWP99YEBS/XhlYSht9NMfxbHX3z84I1uEHY4Uh9AByFcqc1eJQL1vl2u3rK7LLvaxBRyLrzcUXBEmE", - "qUPt+Yi55HH50Xqtmdt/FW9KTld2YDoJewUkk/i++o6gghD7fQAIcDi14vugbDBcbXjBqvz9m/Sq6z1o", - "xSb8XBhDJSr1fhXyHN1TjFKWPC7G3Tu4hMe96jJvMonXSSETyQ6DJg4vDwiJoDiUxjVVlcRyIcFgE/0n", - "3a+6WDOey47YOEgvl4RZpZ4BTUM8cy8SHavBlHfF7Q/RsDSZF54ricBhdKJdPocN7qLf5kTMCctdQ8Zd", - "9IA5Io9pSH0qwkW+4IRJnVQvviyRx+g8C0MUERxzqT/IEaR2YY3CiWghXQszP0KQ1sYF9gZiwl7hiRDS", - "KfEXftjXw3eat998tNpzXXbbYLdtsFufYLcaqdfvqRo/OfegNAmpv8jjt9BkYana06R+apd96+5TpbQV", - "OEa4OEPdR2ISXxdXgx4b8SVvXzMJFODZw7boBKfJzP1aXsX1lMOU4Coc0pjU8AI/OseRX9qe3L/Qs3gA", - "+LaEh4YkBFNKtKuj6Q1TkxOjQPbGExm8FFYBfjvpgMZeGdO8O99A2b7EMgiyC1T0ZU3YD5FibakFwsT1", - "7PJ0FXN2SkyYe2TjoYKzr/uXOhWYE3tdeRpy4SdXw3OMrgx7ruVYKzizFIh+bwFNj86zvTSJM/DyzA5V", - "7CvSmi3v53Wbe7/Hfn6a3XASXPgNOR/aLOzTMLHzzphARnVIgtG2yaAdwLvOxsenzeZs2dH9IhyeijYa", - "sFsN5EfYn7vidZXDWNvGf0pBvsnffh4+RSs2Wiz7rYO6EXHWYctvHvLPGeE7IO7WUgktvin2wtpqi7As", - "qrVZw5JE5RuGO+L0iysjioltgBYkQAHhQqfd1N6rGYO7oHYNoE/Yn2vsST1wQhBGRyfHl2gSJv6devKO", - "vnn/OYb/7b7f/+b9PEIYTTAj6OQC4SCAASsNoVXCEDYXaohwN43II47SkIz9JPrmjdA373+NSz/9PEaH", - "egEmbQ8OH/CCI4HvCJJ0SAIidzW5JwwFJKZF0/Gg2A1A1EU2Cal/rXBSOqNchH6lAm8RLcl8dHN5yq33", - "FoWRQCXwAZFefu7p1rR1MG/z3urlFrvEJaaLvSDunT4uNkL5n+JEIJ6laSJvONBFTo1YFg5FYoT5nc4y", - "8UvCHaAblM0TLuC9pb4UgrljQgqjBES3aoTquHlnqiMAsu2cHqIwaG67zJTtobq6yvtOwnY0TQuGYy7F", - "icIZggyVKrVUhIU/p/HM7MIv19cXu/L/rvJljdGvZGE8gXK8golwSsc1HqlxiGGtEN5lIrBYGcdhyRtk", - "hMEOPAk2wkOCmRIWUZVmt+QsrFwOnprvZjbu6tK6hCAbPxotBhcFvnKBoWNV69fbHOvDdjeHpc9yru05", - "GtakhWrLpuerm5BpwsBT9oBZQONZfVVzggPChl3hyoBJ6kJ6GAkNjeXapGiQQpLRgKjHzhrGggwP48JD", - "rfpb6cmk2AbZTeV60hD7JBgjeBqsaDcN5W4poPj/QVylDmWEJ2EGroY5TlMSc2383eESEI0QTuIAHNWJ", - "iYpekvxuUqlUNLlBzktPIIwLJIM+SurkgVi5yfxSrZWXcomZfQVuz3GUsuSeBiQojz9GXyIqhKJpuGEi", - "PySYcUTF2BkUtD3RV3eiv+nnOG9QA3iN57HFxKUoHM2+cmcN644HCZ0vti206moB2SKJOq6b6+H6gJIM", - "BIAJyNHpcMFR6Xy4XLLFD30lrQWbmSBom8E2nj/nmUk9VwZ2JbO4wEUqi6a+7qxgMF6LwZjw36iYN6YL", - "y53BbQTbz4PEqG9S4FsxPvn4cKWLccrniXA/fdbRU7XUY1kYakY1W6uHsdP++mGmzk+CI9UaLCg4hrS4", - "WjjLjzs8zGa70WLHjHJwv//zIAY3HXv6uNqAnUPu3zG6kXIqh3oXvOqKZ7A6XB4wLw7XtsXo65o8oMSc", - "sAfKiZTWIUcT7N8Z9YLhhwKek2M9Ip747/bf50OMO2nQwsRIb5+LFK8Jjhw3dyhR4pAbOl+hcd3LdTrT", - "d/JjczVtc+0AQWiPm15ZZUjruOmTBtANTVH6otsj6Bqh5pDTxTI0m2tk2au+1ZjdJtlsDFH80+fI1NTj", - "zNO6otQHfhLrO8GVfZbUEwIUoe9FFysMuMLuPczz9oOpS6dC4EyUr9zYUAFImUV7me239t8u+6+DDhx7", - "ZCgPpEBNZpFIR5l1YfiTbGgWnnGIWO5kzn7yRY/WIVxc3KagVyvUIW/ugDnDIh1xzKrpMzL66+z9nX4x", - "pYyXwlWkJJSdRT9eHFAIQQplq1yFTtEv2V9lAWkLF5wUmfC7pKzZAit5/rLRdR0naXG5KWGvCOt5oeN0", - "+cxtS8e5YS6uUvwQD0YWEMXzTt4lwuQabhzn9mUjB/Onqn6u4FQLyr8Fw+4SKbhbujRYc3HgSLUHS2Yc", - "LmzPymTh0C4t1ZbLfVlWElR3psVPulR0nIsflFFyOUJSXZcMkLFj4YqShz1C3vRm2gLDXobN4lVeKe1P", - "SWyX+XGUHyCGestC0T584PxoDkLZHOmtiibaNkqvxl4/SP7n1+zpc9FY66miDshljpTNnwBTGlM+H7Yq", - "06f3spYR9fw5SkNvUVQs6vlyqBA9+aPuRrnikE01TvhMQ3KThgl28ETKCHe+JraFwZSGIAhwqB5a6k7G", - "nO/rgI46/2fMEd5zw0LraQGMXUQMZAAnuBI78WRgry3YbWVcgv3rdoW+1ZYAjmVj+DpLK/UIIywAGObD", - "yMtMdQJYqkv1XEbbxEnh4Ct3MGcJxtNkxp8V0LlOUmgK5iytoLH2ybNfky7zYCnx7wiTXO+IVsy/WUah", - "5umXOQ1AgB1FDnsAvNdG/pz4d/AiCKvX7+SR+JmqL1jSi4qnxI3CAgxOzrnAKrKiWVZsf7b2p4mQvu6/", - "DlJaZv9tbA19udcLfwoRjah734q6HmahKjLH6DjvNoLYJhWpEnNBcDB+SVz3L600Rkc41u4xgjA47sD6", - "7CdhEiNOUgypcvJQi2ixY/p+8+TNpPTTwf07iLY4mcJIlJuhA4hcMAF3wlRD4yaMEea1HW6GH/GMIxC5", - "4+FVoNxFUHOGHlAKdfW0WyVTEEwu/p4mecrttmfIttb4ME9CoxgXCh4MBDKPZTFiZIZZEBKe03WzMjk1", - "9XIcsk7+bMp9YA6BPLx+iDQL0amrFk8bndeL9+hRbANw1bWioXgGnD/e8cUFSTtLl+qQH2jbNl+Np/po", - "oleCpE7NyuHOruuuHUl2aqCZmBT4WwWlPGCq87+YvDTNCfwNCKdkhv1Fh5dh61NYuc6x9Qj8oB6BrT1+", - "a49fzh5v6/pazTf2gkZ1f8N+4PXL0iEOtVfqJ2vR4nFeaH8FSvwmDV05I9SdR3lV2ZIe1FpTv7xsU1y/", - "HsfPOm1ih2yWRVIWF3l65OxDEAnVVH/B3BFUK381GIRm+btPa6b6HWD4FUcOtZK7TXulxGaoXYUL7T29", - "xrPnG8Il+Sc+hbuyfvBBubwT93KQ9VaY9N3b8Fr/aCk8c8edyRHdZY+raIOlWCdMJVxO4VK9r2k0pm5K", - "UD05QGqyTr90mIMjcr0sc36jYl6kKH/5g7IlU7pOke4wTA+6bSrftCuB+kZuFi+plm+DbrZKfq9YDpe6", - "0qTJd2vvSuIoUblExRryoCzqhvEHl61ZQc2ahvRgzl0/HvicKB9q1FzfRi3hMA6WrjjWvBTRkK8vk4oG", - "ZOuzyjjDo0QM6ZKVKqLGbcuwtEZUGeBvXQ/noIQVFYsreQAoNFkJCuXyQBcjmBH22fCNkjB/mMJscHiA", - "ZIFmBYBzIcBmeRhENC4NSOXK1AtmA+aB9393oOHOdbngm35bIseBf3WNcXGy86tN+Vb/TMwvlNOEfQRY", - "V7VAa2Bg3nYQZZMd2KAaiFdZiieYk3d90GUaN2PMtNjvAVUxWum4MINJaqE6ZEVQIU9Q79P+RylprIoM", - "B97e+N14D/LQpSTGKfUOvPfjvfGefgwIJLarELwDCFZKlTMZxZGqRYIheXulHKDkbHgQdBJ4B95FwoVF", - "uNxTPEG4+JgEC/0QROgYIkgEoBJ77P5TB5IoDagzXXi5qGHlPaE2YzKt6cLC9vferWz2I326VCFoySGq", - "DyTLYBICYXxQYLlmy8HflY2eRt5f9va628pGtkABU7CLmn+/fRp9b2TF32+fbo2R43evTCa3cvwy6ex+", - "xwUyTo6fFAmFxOXZOobfEY7bKUk1s2np0J4CyJjhiAjIAdFg7y6a7JYABLt3hT4+dKSBVet53hZ+ULN0", - "tf3wCrdbHgm7UoPlu9+Vb/lpF6d0544sOiUHR1g9sLQfbaqUAThEulgNekjY3TRMHuAhvUOswOklAbiG", - "6dUpxOuE4HijBvQC8hYeQufSNn8/WRYeI0sQdD3wuV2bhLPUz5cRcFUAHIgtvQp1MsdeH4LfG85I7/u0", - "fb85pqsqV4qleBZFmC2KM7TEBpgj4CvP4j34uwfP7X5XGkEvaftsBtTS2M2ChxqQl2bFUfchYCDtJ/9L", - "JN4s/9dF4q+BbPVp/WyyVZfrXR/HvsqZ23BewHeVfoTGOylLVEYWHAco1bmPKrY1lUYGEkophbn76FBG", - "RDXXqzg/SoS4tzI5DuuGxaq1XhKehcIlzK8sokZqk8K80uPbpFy1ZKAki2JwboXrS7n5m/BdyDvTSLq/", - "0lATbv21+hI0mj+P/tVku/mRiVSvVq61J5HKzbDLlLxNIpULdtBMO5VaWveMNCW9koPaUpvXqO7vRBQ6", - "9LO2t6dPo9Ama66f9r3OM/nXF/XCd2inaafrrtVkqKrcwsw23z6N+thlbLy4JYy12dsLyys0yGyGmGwR", - "MugSUblDNN0Ruu4EL6Kivw0TzcakCRa+IxJGeb26tvlCdl7xLq9eHNU8eL0k0l4HgWlf4pbAOiWMKkja", - "qKL8Ap/VCwSXYqK+e70kwJzkFgzKkamEOmhXgDh24yQgPbQq1cwB9Ln+0KqqtxVTBeX9XxmB7E9aey+V", - "7d6Ywp5rdP3iyeXKvafbZ+l1Cq0bO4LdmrlL3wbAdr/L/+hz0kkffydClyKOp0kjeZzDKIPlpZrck4z6", - "Fsmpi3Z03cje9JIXQn9Dl74qaTUq9VCgGfH8ZRA2Nd/rKv0qSGpdDoxqxeknfQL3kefARxoDELgGQ7yF", - "c7e/WCnl1G0/cSq51t2nj51Pr/UEykswgWhQ0c4iQVMainKdO1KkNM84Yf+FJ/63bG9v/684Tf8rZUkA", - "72YhezqYheMA3as091HGBZoQdHN5ikjsJzphtEsg5TUobXn0wsfZKdSHMHW1n3mu1TdvjU6DZxCuFdSj", - "NMKNq5DlLNAdJg+TIRwKZlbiFeuC0maONVk/cnLZrOmjNK1DM7bqnm7aSfvDEGNJXO9GRZb0ZrGtG1kP", - "ovoJb5OCvUOGHyVRhHd0qgMSQEUbK4EtOjmGp88zUoLEG3nkMQ2lNmGeq7hEsh7kDxrwVq9Bcyh1hB9P", - "1Md3e3sV4Tnyspj+KyO6AfDHWhVMZ4r754lwFU5qCGHLQkNZ6HtePLjV7qi8I1aNBpfBMd/eK6sg8TBV", - "uChl3NPoWBGsxhf1+rXT13bIN96kiwN+skBwJ22WmWva+JVLoGVuuYb2t+T0DBmz6ydxTFR9VrdaeQk4", - "5znRBSrXvsoKZHE75UhVhSnlBlJlwIMxur4+lU3g/Q95FCTWF58WhTQn3iMN43NpePXKrYZskIK79xIK", - "rkk5aUr4PI1eStXWFPGjBYu9FX43iRbz46X9Rag8cLhVSRsq7aDd+/2W4e2kaT2OplOVO3Jp1h450ypB", - "dSBH5XeOxBwL67F5fqbQGEU0DKku9dBglIFsTm4LsXleGNGYRlnkHey56kHUbE74Uba2qnu0QdkAVUgj", - "WoYqf3z/bk/eM2rZglqB3MCJD7u+zHmvcrFuhcBzhEDXPd3m+ii/dvfg5cY7+jPYOa/Coli5SHSBmTCM", - "DQHb9zgcSS7WDDyCpqo2cFHdZY187RqWQHUcmy17LI3EwXILGwby7SZi2yrl7ZY1F9sCYAPGha28KMkL", - "XWhUpecQjeEx9Zqkpj7uBgrjOsriVu4WmUNg6fKgr/Bq0VKUuNc1w2GcKbGWCdixOcu5eevknQ97f+vT", - "9m9/Ej5TdVsbr+IX8nOlNmuf+zP027jpT1kDSvc8cITqQtL6Hrilro1RFyNTRvic8DZjDzQpCQVlrZEi", - "lgquSz4mKKT3pCf5XebzvoyULefYCIxYq8fy2qdVoV4ZPBSXsTuSCoQlBiytDKpEPipt6/1f5Z2rXV+s", - "pbp4Gi7DzY5uyCD6himfm9QxOdm32z0uoccSslZ1fIX6hAIseP2++GYD4faU2ACvmJLUfZKepIRxygXU", - "oTVVuvOoEz3m/+S5Zs8F1CgwRcy5UQVMnJWKwoAnO4U/Wb1vhXnQhCySWAm7hNEZhSfa+TQhnRJ5PPV1", - "K+RwvIpzyZ376Uuq6smXc2fVSqKDNwY7SqUXeSZhAOMGII+UCz7S76N00S3tsqkmaYW2qjJ8Xn8d7LtQ", - "iRhqXsczPVISE3fR89pht1HhY1fNd5kcDNo2n6tha2awRI9k3iRr8UVeEaEC2lTDgh+MQaF0lkh6Jo8p", - "ZQQ9Gi3NCoqjRVY1zcFjdITDEEhdckxExDwJUJSFgqYh0VkGk3vCHhgV2mpxfX06QgT7qsI3yrjqXpgz", - "CvMe5oXhUrZKEyq/JygimGe6jptZmlFT+wqza4271yDKrH2sZ0CUiyu05mI/bHzpgheNOrjaVW+oW6Ne", - "i1tCebsSVZxr0jSQmtG3EqGnRLDVjvaY77xpteR93UHRfMK3RPdVXoCUofisYsELGCYLxJOM+cSKMFwq", - "A0uKpUIjpzkFb96gLufkUeh8a5sx65eO1GWt+sWm/6mC//JVK9KHrBr9UhY43XDX+sMm3wFABsZnhv+r", - "BW1u56tZQ9u2f0BKOPmbtZFFepQ+XlY7GLpNklnpT5b1sepkJ1sH64/lYJVEsQrvKrxs34hrtX9+ujct", - "5jsFw26EH1uFA9CeDg9yCQpTiUO9ojCU3E98nOHHrQR59RJk5HihyKgPpVHkv8g9KVEJPDLU71kanhQy", - "SAzf/HTFFNbzk1hfyf6w3+eYFzCwGX8wLIijvt5aA8jO8KMt87Yy7rXIOGUd7KXJmqZOUVV87HFny1O4", - "NTFw7xqut5vWoPWjzGdr0QZfL3iHGqRbr5EeC2SU38m2O/sqycJaHsvaxLkOJ52z3nwva/n+ymHQpVEb", - "PHbaZ4A55CpPhYnkeHWP/V4hZZbE5a6u0bL7Hf7RnEXlCGoS02nFuaOUP1XbR7l1WqWqLvEE/2mQsOUc", - "mVi3bNYYGsozm46b1A3M5LC8vJ5XlzyVR4rKQGrq8i009INJ+tXl3n55HaGZ8otKka3J9QoHDJ4Zh0Gj", - "BqH65OR+jWfrktXlmeREgwT2h4YamT9oMu1XQIemUKfbt3gIbm9JYz/xn+UFC9fLk7YoBGskNAXZ0oT2", - "bsWAkMAGxakf4FkRRbCl4/XQcVmUfi/qv/XNV9qg5lYFaKmu3EBzUd61fwRxqSzeKrKW/vjXnPbLdiXD", - "e8Ou2yriirZ8SfflEo7STWiUVoXVgSYnW1mngi9ZOeCHUQFLOXbb7+XmWVHjjVwOtBZJtb6bfbn68dKp", - "d2ulThvT777+7Dtv23h0SXRx77in6eht0OnbtUBtrUpVhdCUGPquS7M/DXkJAdkUiiL2fWlbnZUf81rw", - "69QjTMV5hyKw75adinbmmCsf5ZZ0epNOPYFIzWtdujWrdA391M0SzSyVFmRJutlsChE/Yxw8tG8ph4gr", - "1tY8d3vX/drN7bwLKCM+rGHU82iQVHGc92ocOCT3UL+s96Cn0MGB2isV29ln96csiZpCF2CUQatUE2/I", - "Xg48J2ftbTN3X3Isln+d5hu38H3VVvJ2caxSgQ8RyE1J+LsEskpZ/mIi+SQOyKPhw/wNRU5wjVyZZ5Cw", - "tHOnyEhm/Mt0ykmDDBycROmHkdJLC9ONSa7Gh12dEmsrptYipqY0lD/NMZ+3lwXBMcrSMMEBCml8Z6yU", - "mCE5ApKUgmlsMTpeEPWtr075Wbb9BfP5cwWXwzU+V8P29YxLKIwAM0vodo6/Ww/LSLzcAOab7tf2vjzM", - "CYO0D/pHYCG9Sz+AU+ets5txpHeE3YH7fBk/gHZurtL9s5ZI+Nwt+dxQeK0wAV7ffv31V+WztFOH9njr", - "11Ya4Ov+j1zZZdT09DAHdLJASUxQwlCUMFUVCDDRq5KBUOy/XDq/K6F1pzKTjTwuFqH8Qaqhb8kFuC2D", - "8xYfDbemIe6V3bTJOmmJljeaqPhNWhm7LrB7Q2HObYV9MNsA8kqslBVUqkWA4QKu1/pFT8biMZK90YSE", - "yYPKQ6EaYEYQefTDLGjG7cqsnkeYkx1OYk4FvSeIZxN1LKEIC3+OkhggjwjneKauaVLKNpw0BDN/XgIr", - "wo+nJJ5JBt//y183GyJs5Z/+ur+cuXObiXo5WV16IrT6xxlf91/iecbX/dfuHteY2JY3W+7ObRNuLc6y", - "vV54e+SSRa8/duzSWoBoFtzb4KiX5IqOYJOhoSVOJnm54JI1nymAkUEnyuuKbXmF0vt9k9qxpJLx/kWU", - "jPcvpWRoAIy8NYC8Ln3jT/PevkbdSZhFpGfyKGRau+wd+af12+nVXINN9CHY3Oqr+TNJN7PmHlWglSTL", - "8eQWZtaur6Xys9nqzb5TU7MexoG2J3cQlsn6WsfZ9orURYaWENr9rv7R/wVaM3GqRpo8v+phByt6Bp6e", - "z89KRGGenuE6QWyNPa1yqSUOLUdkYxDaOrd876UEjEnVtKWmweIFoGP3ZvczFnoH3lyIlB/s7uKUjsn+", - "ZIzT1LP6fy9yAxWpcb5XkryWf4Q8RvbfsHs7Qi643DClO3dkUfpNRwnkf+eKye3TfwcAAP//", + "7L3/U+Q4siD+ryj8eRGfnbuioOnejbdcvB9o6N7hDd1DAN1zcdPchMpWVWmxLa8kA7Ud/O8XSkm2bMvf", + "Ciigp2IjdpqyvqRSqcxUZirzexCyJGMpSaUIDr4HGeY4IZJw+AuHIRHikl2T9ORY/UDT4CDIsFwGkyDF", + "CQkOam0mASf/yiknUXAgeU4mgQiXJMGqs1xlqoOQnKaL4P5+EuCM/kJW7UPbz+NGneU0jloHtV/HjZmy", + "iLQOaT6OGzHDC5piSVl6ShMqVaOIiJDTTP0WHASf8B1N8gSleTIjHLE5opIkAkmGOJE5T1FGOMrwggQT", + "DdW/csJXJVgxjOtCEZE5zmMZHLzZ25sEc8YTLIODgKby7X4wCRI9o/mc0NT8NbHg01SSBeE1+D+TOwn7", + "31zDUc4F4wpkITGXSC4JiqmQaM5Z0gJ2WgzXjUCB02jG7lp3pfw+bmMkwUnroObj2BGTLMaSdIxaNBg3", + "8g2L86R93OLzmFHvVWORsVQQYALv9vbUf0KWSpICneIsi2kIe7/7T8Fg38vx/oOTeXAQ/H+7JWfZ1V/F", + "7gfOGddzVAnlPY6QApEIGdxPgnd7b55+zsNcLkkqzaiI6HZq8rdPP/lHxmc0ikiqZ3z39DN+ZhLNWZ5G", + "esa/P/2MRyydxzSEHf3rJqjogvAbwu1O3lsqBzI+/O3inCyokHwFgo6zjHBJNY3jW3EIckzJm6jJxw5/", + "u0C6AfqFrNDJMZozjj4cnSNcIaJgUj9OEzW2mpil/mH1N3S7JJwAf1SjcgMpogLFLMSSRC1DX5CQE1kA", + "759DN3JXMBx8/UN91MtVRpRIKgBtDERSJTt+VzAGVxMP7yo50u/666S+Dd4Fuggtx2WzfxJNaIdRQtP3", + "Ssgf4TQk8TkRIPLqWx7C15hERyxPPeL3cyF2QWMQSOQAwzyP4xUqegdN4TgJ5piOGFgusUS6i5KUeujA", + "K3RdnNUWUJ31ymLiQkvBX2jciomB0Bp5ShoAX9M49qJBfRg1cAXFunc/HtxZPEgQgi7SSyNgL/FCnBsx", + "08CDxAvhoXS8AJ0Lw0DqX+qQWomtdBillXkEaQE45hyv4G/MF0T6plC/F2MimqJvIMIPJF58C5BR1HoP", + "kR5+ohdSLp5E7vKb63b05SpcJ5E60XOqt0ktG5oqVLCQKqaEbqlcqi+CIJjV0SrznHqZlh/NFlQYxk63", + "BpYbOAGg7BIVUoA3nLLFh9QrCmJyQ+I+CXTKFqfQ7n4SJEQIpYQ3lnTKFsh8RFbuefAhJMmanS8kyRQh", + "lFjPOAP2zUkMqDeUGLMFIrAUH65pQoTEiWeCS/vJItsdqNjECEuyo0bpp75iqhIlE4PNAu0XEstcnBNs", + "5H0N9XpTzF/FZeX3q4kHs0S3rKNDwAyI6ykcuunazipJeE5u6x5/Mvtrz0F1/gkKc85JKuMV4iRjXNJ0", + "gVgaawEMeorpMZIyHBbcuzMWeLULR2dfWvjx0dkXFDJOBIAGS9F8OfDdFDvuhhOl96UklEb0NPeZkxlj", + "HhDOlahXDNCZHW6K+ocUZ2KplFgaE7ESkiQIpxGKqAgxj1BCEsZXgE4yLZEyYywmOLWngeXSfxZYLtV5", + "EyRkaSTKac0OItUZ4bkkHN0uabisACmWLI8jRO4yykknwvZ65ZmF0qfcHAF+osPS5uJRbEwb2XPmteEG", + "STUKgk5acRty9icBjYbIC3eOIbIhweK677CWs3zC4pqmi2MiMY2F6q/vvQ1VAyekBaImx/QbMi6XBBnN", + "T6O3Z6DansJqATg7g1nrxNmuq3KDLwlODs9OjEK/3v4enp2ga7Iav7VmgvcwN47jX+fBwe/de6Lg/SIU", + "MV9NgjSPYzyLiTY1DKYVA+8QMrn2XXTO8S26wXFOmgM2BoixkF8E8cB1ioU563JJRYHEWyxQLoDZepFY", + "XfOzUHbrcn20qBsaEjSEWaXEYxITSQYpzv2wOYrcQH3Qqt0RgLG+AmgPnVWJj6m4/kQkp6FHE47IDQ09", + "SzmG35Edqw5AKZAuvZflj6XAUn3RX8h0MZ0gciffTdDdXPzkZYVKTJ8x6pPVn9Q3lKmPFsMRha308DOJ", + "4/crSXw4Vt+QyHAId44ZtHKPH03l3955r3bqLLSMqs7VOoPWtZZy/RO7MQ1Uu4BU1mq3+oL+m3x679lR", + "Kq6RoP8mdW1HwfyJvh8rwyfBh/TmKzZukyiiah4cn9XIywXhQ3pDOUsTpVzcYE4V+/ApX83T/CG9ib4S", + "Lrw2JfPB0gVJbyLE8zRVmqe5T7SOPQm0aa0pc1jkoWtojOCbB11NFLVq0XrWPsZlJnLV2Y+cJScJXhDX", + "tKcUQk4TmmKp15LgLFMDakNfG/d1DYSTYBFmbQ3/cXTmNOTFzC2tSUo4jose9xOL29VnY/1Xq76fBCwl", + "A0StC+b9pLutC2lv2zqcCr/uAA2iEISrU3kYhuqo/rfwUeOFboNMI/TfF79+Bhr/x9HZBoyPaheHGh89", + "y/Gp4HU8NdCSYSFuGffoFmfmi5JruShZDy+p6dExUIx95Rk8F4T7hfcX82U4qH6kFjNMSrz4sNqq+jTQ", + "q3QWEn1Vit4ZJ3N658Ez/A76mmJ5uge6qTJGfe9hvE1FdOa5yOfeefTvD5wn614EXPSpxY5oDIkMohvj", + "gip8StKFXHq0XPi9G8Q2wWwArs4w8eyLD4eKqZxSIUnUah3AMcU+A6H6eYg+GcaUpNLaMzNOtPvEKOZ9", + "txDd2ztulhemky5GWphY7idKFDkqSFcvR1m5V6e39X6HbpekIsbRLY1jj+mh845HqipEp7fNaQpCPGF8", + "1b+gT7Yd9JE4wrLXsWdo4pNtXvfy921eh2ID8QdkDFaxQKbTYKyC7WngIi+gbSM6oG+JhZMADFTaEkVF", + "BXJzj/MyBQgIgOsDHLFB1lED8Neyb7/Z3Q1ocAMxisPp7ohzthz6qpweeyQsjqsUDFzFmuQ9BlOFrwaJ", + "WAkZkVm+gFiUOQsmwS3mID9BJfUJzVO2EMeUk1B69e/ik2NXNy4zYyWcERPAA3tkwZgzfou5+mWGw2v4", + "Z2P2SXC3o9rv3GCQqkJ1rMDzsRil8vP7YkizgAuWc99NV/8+EnS124xj0AoytSUCfB3DwdezXjrDlL+e", + "OQPeT4JPOFzSlJyozWpeU7L8kIdLKkkoc078Rm7stLALTfXVwsfzP+KExiv/UHP4NmCQTyzyUaYaI1Gf", + "hg7x2auslcOkjs3FP1b9TlUs0IGzNt+kgVe9EXeXBCfaluJhqgQnKIGPxjni+Idqfteqk6pbYjfcVmaO", + "MZ4rxy/2JfXpXp2TKFVPddNWwr9Yh4GgaUgQyVi4/Kl2HW6xoYD+5Dc1m0i8qj3TxEeRyIJjrvMLekNS", + "pAbmN9jxxOvAwU5HXRUPFiTY3jDrMGU0Im8+HZ2hkKVzusi5DqdqGjJabKTlJeCTo1rU3Wzg21nDVvNm", + "/z99uP9MbjudKA91JPiskFd63g7FN2a3f8A+pkT+oSfwKcIxuy1QIFkByZIg23mKflP6jCBSNZjjWJAJ", + "ohLNyBLfEKsuJAQpJScjIZ2vaLpAEUlXv+bQZ28K/9vds1SWEnnL+LXZZb+DDeeSneFckIr/Vk/fDL5j", + "CVYX1jheoUx1qmox2tUGKo9xiLXNeE5EngxVuw6LDkewEKMMW9NdjyIMzZRCq09Hp/4bZg9UfQ3GB/b8", + "rFuXqxIk9MrAC/gd4ThGxigdsiTJUxsHCdy6oUm7btRRCqs9Bt0+ANcza2OU/+rj/Yo2Y3rjtdsaVjwd", + "b7x9Br3YcIMuT9/j+Xxc/qPhXWc2jSUw5UjFZ4KD4P/+jnf+fbjzf/Z2/v7HztX//I+BkHiY/2djYq5p", + "dHEuJOHDSM009ipQLPEG2R/B73YAxsMlEZKD4bjVM/rRGqZ6AtrMRQzCNIb6VXSXCx0HR8bMIoo+w2Ya", + "5pRtU0iTqhreyQidppohWudbVy9FDtZPV5oBRsQSWp8HS92FVDDT4qYQxQUd4nX65zQN0YWdvMaA/LNo", + "c/NJKiROQy8ztcZzatqUdsDe/TFBRQOQrEOygAkOdCl1nxKft7m51olzsgtoa9tc0krzXFTPYsuelUsq", + "GECVcq8M39HGZl+McLgkEUSHeY7iKRXAOXQrG81LoxrJDY8P3TK7LbPbOLPbsqFeNlRhA/28yMd0Ckbm", + "Yz9OPEr9yU5kbQ+iYTZR10UwlBydfemikqIdKkI8B9JG0VNfv1viPQ4hUqM6k4nFHBlU4npYfJEq5UvI", + "Mlh1PMWHWX5GeEi8Z0shXA2eQ1RvptvpUOYhY0dUXAtf/JDUryXMXuroXxwuIWxnNynDeYZGLLthTN54", + "ZYX/y97Yn1QT2DqbpXt9aY8D+uyMbV2ka0cDVYi9hTIrW9sE0ONlcBBk986eyYuCczWdCblw+d70W7qD", + "Io6p4sAHxc+ICjRjeQre/hlBYplLFLHbdIpOpPbZpUyC8SaTKCW3DjvHaaRbCMkyxBTPxeDjowIUTacl", + "JyhiqQZCsbVotqrCoCeR9IbEeh8maJZLRCUKcWpfEsObYhytYOaQpZKmOUHAL9MFkhzP5zScfqsGFuBI", + "XTztyoHdQdy3/iNPlwTHcrnSjFUBNtAjUKL/3MxR/nJczlb+eOTOW/78xYGg/PXCwlLZ6KMlThePd//s", + "jWAdLxhrB8IMoFahzVkdDvWqVa7bvv5IdrkXEeK/cUOS2qRXF9cQsQRTj7r1HgvFW9RH53VqYXfWPEFx", + "GG1/prN4UCA0SW/q7xdqCHHfJYDgAGmZ3kRVQ+XjhjU8VpzBJr35Zg86sQk/l0ZYhUqzX6UcQTcUo4yz", + "u9W0fwfX8PTXXfVtpvgmKeSS7XBo4vEuAXOKSmE4bajIJFULiUa7Bj6YfvXF2vF89svWQQa5QuwqzQxo", + "HuOFf5HoWA+mvTp+LmhgaTNrPJQTgaPqxLiaDlvcVL8tiVwSXrikrJvqFgtE7rKYhlTGq2LBjCsubxZf", + "5chT9DmPY5QQnAqlt6gRlFbjjCKI7CBdBzM/QnDYxhn2BmLRXqBEiOmchKswHupZPC3abz5K7qGuwm2Q", + "3TbIbkiQXYPUm/djg5/i9KCMxTRcFXFjaLZyVO05a0rtqk/fL1UqW4FThEsZ6heJLL0srwYDNuLXon3D", + "FFGC5w7boROcsoU/O4COJ6qGR8GFJ6YpaeAFfvSOo750pRh4pjQAAPBVBQ8tSRfmlBgXS9vbqTbnSYns", + "jSdueC6sAvxukgWDvSqmRX9+hapdi+cQ3BfpqM8Gsx/DxbpSKcTM99zz9DHm7OWYMPfExUMNZ1/3z03q", + "My/2+vJSFMxPrUYUGH007PmW46zgk6NADHuDaHv0yvbKJN6Az09uiORQltZu8f/ctPUPe2QYZvkXQaKz", + "sCXHRZdlfx4zN8+ODaDUQhKMxW2G9Ajek7Y+em03o6uO/pfo8ES11XDeaZg/wuHSFyesHdXGivaXDPib", + "+u2n8VN0YqPDo9A5qB8Rn3p8CO1D/jkji0fE+zoqoXNuyr1wttohLIdq3aPhcKLqDcMf6fqrLxOLjamA", + "FiRCERHSpBk1XrMFh7ugcUmgDzhcGuwpPXBGEEZHJ8fnaBaz8Fo/tUffgv+cwv923+5/C36aIIxmmBN0", + "coZwFMGAtYbQinGE7YUaIuttI3KHkywm05Al34IJ+hb8j2nlp5+m6NAswKYpwvEtXgkk8TVBig5JRNSu", + "shvCUURSWjadjooZAUSd5bOYhpcaJxUZ5SP0Cx3wi2iF56Mv56fCeedRGgl0wiJg6dVnpn5N2wQRt++t", + "WW65S0JhutwL4t/p43IjtN8rZRKJPMuYuuFAFzU14nk8FokJFtcmu8XPTHhAtyhbMiHhnae5FIK5Y0ZK", + "owRE1RqEmnh9b2onALJLTo9RGMxpO8+17aG+utq7UsJ3DE1LjlOh2InGGYKMnDqVVoJluKTpwu7Cz5eX", + "Z7vq/y6KZU3RL2RlPZBqvPIQ4YxOG2ekcULs0YrhPSgCi5V1WFa8UJYZ7MBTZMs8FJgZ4QnVaYUrTsra", + "5eC+/W7m4q7JrSsIcvFj0GJxUeKrYBgmRrZ5vS2wPm53C1iGLOfSnaNlTYapdmx6sboZmTMOnrJbzCOa", + "LpqrWhIcET7uClcFTFEXMsMoaGiq1qZYg2KSnEZEP7I2MJZkeJiWnnHd30nHptg28G6q1pPFOCTRFMGT", + "ZE27Wax2SwMl/hcSOlUqJ4LFObgaljjLSCqM8XdHKEAMQgRJI3CQMxuNvSb5fcmUUtHmBvlceXphXSA5", + "9NFcpwgAK0zm53qtopLDzO4rnPYCRxlnNzQiUXX8Kfo1oVJqmoYbJgpjgrlAVE69wUhbif54Ev1VPwN6", + "hRrAS5THziGuRP+Y46t21h7d6Sim86trC627WoC3KKJOm+Z6uD4glgMDsIFAJv0vOCq9D6Yrtvixr7MN", + "Y7MTRF0zuMbzhzxvaebowL4kGme4TKHR1tefjQzG6zAYE/EblcvWNGWFM7iLYId5kDgNbcp/J7aoGB+u", + "dCYox//k2kRtNVKe5XFsDqrdWhvb46Q5DuNcy0+CE90aLCg4hTTAhjmrjzsizhe7yWrHjnJws//TqANu", + "Ow70cXUBu4Rcx1P0RfGpAupd8KrrM4O1cLnFohSuXYsx1zUloOSS8FsqiOLWsUAzHF5b9YLj2xKek2Mz", + "Ip6Fb/bfFkNMe2nQwcTEbJ+PFC8JTjw3dyjJ4uEbJk+idd2rdXrThopjezXtcu0AQRiPm1lZbUhH3AxJ", + "P+iHpiz10e8R9I3QcMiZ4iDmmBtkuau+MpjdJvdsDY380+fmNNTjzQ/7SCkXQpaaO8GFK0uaiQjKkPuy", + "ixN+XDvuA8zz7kOtc69C4C0MoN3YUPFIm0UHme239t8++6+HDjx7ZCkPuECDZ5HERJn1YfiDamgXnguI", + "lO49nMP4ixmth7n4TpuGXq/QhLz5A+bsEemJn9ZNH1DBwFQr6PWLaWW8Eq6iOKHqLIedxRGFHxRTdspz", + "mJIE6vjr7CNd4YKzMvN/H5e1W+AUC1g3uq5HkpaXmwr2yrCeZxKn62eMWzvODQt5keHbdDSygCgeJnnX", + "CJNruXF8di8bBZh/qevnGk69oOJbNO4ukYG7pU+DtRcHgXR7sGSm8cr1rMxWHu3SUW2F2pd1OUF9Zzr8", + "pGtFx/nOgzZKrkdIuuuaATJuLFxZ4nFAyJvZTJdhuMtwj3j9rFT2p8K2q+dxUggQS71VpugKH5Af7UEo", + "myO9x6KJro0yq3HXD5z/4TWKhlw0nlSqaAG5jkjZvASY05SK5bhV2T6Dl7UOqxcPURoGs6JyUQ/nQyXr", + "KR6Tt/IVD29qnISPNCZfsphhz5nIOBHeV8wuM5jTGBgBjvUDT9PJmvNDE9DRPP8594T3fOGx87QAxi4j", + "BnKAE1yJvXiysDcW7LcyrnH8m3aFodWlAI51Y/h6S0kNCCMsARjnwyjKavUCWKnD9dCDtglJ4TlX/mDO", + "CoynbCEeFND5lKTQFsxZWUFrzZUHvyZd58ESC68JV6feE61YfHOMQu3TryMNgIEdJZHvVbFibeGShNfw", + "IgjrV/fkjoS5rqdY0YvKJ8ytzAIMTt65wCrySLM8sv3Z2Z82Qvq6/zJIaZ39d7E19uXeIPxpRLSi7m0n", + "6gaYherInKLjotsEYpt0pEoqJMHR9DlxPbyk0xQd4dS4xwjC4LgD63PIYpYiQTIMKXqKUItktWP7fgvU", + "zaTy08HNG4i2OJnDSFTYoSOIXLABd9JWYRM2jBHmdR1u9jzihUDAcqfjq0/5i74WB3pE6dfHp906mQJj", + "8p3vOStSfXc9Q3a1xtsli61iXCp4MBDwPJ6niJMF5lFMREHX7crk3Nbp8fA69bMtM4IFBPKIphBpZ6Jz", + "Xw2gLjpvFg0yo7gG4LprxUDxADh/PPElJMl6S7WakB9o2zVf40wN0UQvJMm8mpXHnd3UXXuS+zRAszEp", + "8LcOSrnF1OSdsflw2gsHWBBOyQKHqx4vw9an8Og6x9Yj8IN6BLb2+K09fj17vKvrGzXf2gta1f0N+4Gf", + "npeOcai9UD9ZhxavVfiyQPyDlPhNGrqKg9B0HhXVbCt6kL9SvFfPUrckCCtvxPHzXpvYIV/kieLFZZ4e", + "NfsYREJ6t5+x8ATVql8tBqFZ8e7Tmal5Bxh/xVFDPcrdprtCYzvUvoKJ7p5e4sXDDeGK/FlI4a5sHnxQ", + "oe7EgxxkgxUmc/e2Z214tBRe+OPO1Ij+cst1tMFSHAlTC5fTuNTva1qNqZtiVPcekNqs088d5uCJXK/y", + "nN+oXJap0Z9fUHZkaDep2T2G6VG3Te2b9iVu38jN4jnV8m3QzVbJHxTL4VNX2jT5fu1dcxzNKteolENu", + "tUXdHvzR5XIeoVZOS3ow764fj3xOVAw1aa+ro5dwmEZrVzprX4psydeXK0UDsvU55aPhUSKGNM1aFdHj", + "dmVYekJUWeCvfA/noHQWlasLJQA0mpwEhWp5oIsRzAn/aM+N5jB/2IJwIDyAs0CzEsCllGCzPIwSmlYG", + "pGpl+gWzBfMg+N870HDnslpozrwtUePAv/rGODvZ+cWlfKd/Lpdn2mnC3wOsj7VAZ2A4vN0gqiY7sEEN", + "EC/yDM+wIG+GoMs2bseYbbE/AKpytIq4sIMpaqEmZEVSqSRo8GH/veI0TiWIg2Bv+ma6B3noMpLijAYH", + "wdvp3nTPPAYEEtvVCN4BBGulypuM4khn0MaQNL5WhlCdbHgQdBIFB8EZE9IhXBHoM0GEfM+ilXkIIk0M", + "ESQC0Ik9dv9pAkm0BtSbprxaTLH2ntCYMbnRdGFh+3tvHm32IyNd6hB05BA1AskxmMRAGO80WL7ZCvB3", + "VaP7SfDXvb3+tqqRy1DAFOyj5t+v7iffW4/i71f3V9bI8XtQJZMrNX6VdHa/4xIZJ8f3moRi4vNsHcPv", + "CKfdlKSbubR06E4BZMxxQiTkgGixd5dNdisAgt27Rh/vetLA6vU8bAvf6Vn62r57gdutRMKu0mDF7nft", + "W77fxRnduSarXs4hENYPLN1HmzplAI6RKZKDbhm/nsfsFh7Se9gKSC8FwCVMr6WQaBKC540a0AvwW3gI", + "XXDb4v1klXlMHEbQ98Dn6sk4nKN+Pg+DqwPgQWzlVaj3cOwNIfi98Qfp7ZC2bzd36OrKlT5SIk8SzFel", + "DK0cAywQnKvAOXvw94Azt/tdawSDuO2DD6Dhxv4jeGgAee6jOOkXAhbSYfy/QuLt/P+pSPwlkK2R1g8m", + "W3253g1xGuqcuS3yAr7r9CM03ck40xlZcBqhzOQ+qtnWdBoZSCilFeZ+0aGNiHquFyE/KoS492h8HNYN", + "i9VrPScij6WPmV84RI30JsVFhcnXSbl6yUBJDsXgwgo3lHKLN+G7kHemlXR/obEh3OZr9TVotHge/YvN", + "dvMjE6lZrVrrQCJVm+GWKXmdRKoW7KGZbip1tO4FaUt6pQZ1ubZoUN0/iCx16Adt70CfRqlNNlw/3Xtd", + "ZPJvLuqZ79Be007fXavNUFW7hdltvrqfDLHLuHjxcxhns7cXlhdokNkMMbksZNQlonaHaLsj9N0JnkVF", + "fx0mmo1xEyxDTySM9nr1bfOZ6vzIu/z47KjhwRvEkfZ6CMz4ErcE1sthdCHUVhXlZ/isXyD4FBP9PRjE", + "AZaksGBQgWwF1lG7AsSxm7KIDNCqdDMP0J/Nh05VvauIKyjv/8oJZH8y2nulXPjGFPZCoxsWT65WHtxf", + "PUiv02jdmAj2a+Y+fRsA2/2u/mPkpJc+/kGkKYGczlkreXyGUUbzSz15oA7qaySnPtoxdSMH00tRgP0V", + "XfrqpNWq1ENhaCSKl0HY1ppvqvSPQVJP5cCoV7q+NxJ4CD+Hc2QwAIFrMMRrkLvD2Uolp263xKnlWvdL", + "HzefXqcEKkowAWvQ0c6SoTmNZbXOHSlTmueC8P/Cs/Bbvre3/zecZf+VcRbBu1nIng5m4TRCNzrNfZIL", + "iWYEfTk/RSQNmUkY7WNIRQ1Klx89szg7hfoQtq72A+Vac/Oe0GnwAMJ1gnq0RrhxFbKaBbrH5NEo5e7E", + "KzYZpXs4nsj6UZDLZk0flWk9mrFT93TTTtofhhgr7Ho3KbOkt7Nt08h5EDWMedsU7D08/IglCd4xqQ5I", + "BBVtnAS26OQYnj4vSAWSYBKQuyxW2oR9ruJjyWaQP2gkOr0G7aHUCb470R/f7O3VmOckyFP6r5yYBnA+", + "nlTB9Ka4fxgL1+GklhC2R2jsEfpeFA/utDtq74hTo8FncCy298IpSDxOFS5LGQ80OtYYq/VFvXzt9KUJ", + "+dabdCngZysEd9J2nvlEG//oHGidW66l/S05PYDH7IYsTYmuz+pXK88B56Igukjn2tdZgZzTTgXSVWEq", + "uYF0GfBoii4vT1UTeP9D7iRJzcWnQyEtiPfIwPhQGn585dZANkrB3XsOBdemnLQlfO4nz6VqG4r40YLF", + "Xst5t4kWC/HS/SJUCRzhVNKGSjto92a/Y3g3adoA0XSqc0eufbQn3rRKUB3IU/ldILnE0nlsXsgUmqKE", + "xjE1pR5ajDKQzclvIbbPCxOa0iRPgoM9Xz2Ihs0J36nWTnWPLihboIppQqtQFY/v3+ype0YjW1AnkBuQ", + "+LDr68h7nYt1ywQewgT67unuqU+Ka/eAs9x6R3/AcS6qsOijXCa6wFzagw0B2zc4nqhTbA7wBJrq2sBl", + "dZcnPNe+YQlUx3GP5YClkTRab2HjQL7aRGxbrbzduuZilwFswLiw5RcVfmEKjer0HLI1PKZZk9TWx91A", + "YVxPWdza3SL3MCxTHvQFXi06ihIPumZ4jDOVo2UDdtyT5d28pzw77/b+PqTt3/8k50zXbW29ip+pz7Xa", + "rEPuz9Bv46Y/bQ2o3PPAEWoKSZt74Ja6NkZdnMw5EUsiuow90KTCFLS1RrFYKoUp+chQTG/IQPI7L+Z9", + "Hi5bzbERWbbWjOV1pVWpXlk8lJexa5JJhBUGHK0MqkTeaW3r7d/UnatbX2ykurgfz8Ptjm7IIPqKKV/Y", + "1DEF2XfbPc6hxxq8Vnd8gfqEBix6+b74dgPhVkps4KzYktRDkp5khAsqJNShtVW6i6gTM+b/LwrNXkio", + "UWCLmAurCtg4Kx2FAU92Sn+yft8K86AZWbFUMzvG6YLCE+1impjOiRJPQ90KBRwvQi7pHLwV052PMRUp", + "1liBFd2zwC6RCCIYVJNiVyIqrk3ytTRCnMwYk4ilKCV3UptSpiPqef+a6Sr31YxejULt4CPCngLuZfZL", + "GMA6J8gdFVJMzKstUwrMOJLqqWOhra5XX1SFB6sz1EeGStzpwozEUuIvxd4QwRtliW4tf58hxKJt8xkk", + "tsYPhyEqlsLyDg+pOnEQZqcblufBmjkqEk7RM7nLKCfozuqOTqgeLXO9Gb4yRUc4joHU1YlJiFyyCCV5", + "LGkWE5P7kN0QfsupNLaUy8vTCSI41HXHUS5099LIUhodsSjNqapVxqj6zlBCsMhNdTm7NKs8D2WxlwZ3", + "L4HBOvvYzMuoFlfq8uV+uPgyZThabwZ6V4OxzpZmhXAF5dWjXBCEIU0LqR19yxEGcgRXGeqORC+a1gvx", + "N90m7XpHR8xh7V1KFYqPOkK9hGG2QoLlPCRO3ONaeWEyrNQsNc0p+BhHdflM7qTJArcZZ0NFpK7rayg3", + "/U8VklisWpM+5PoYlkjB6xy8NB82+ToB8kI+8FGCXtDmdr6ey7Rr+0ckqlO/ORtZJm0Z4vt1Q7S7OJmT", + "lGVdz69JwbJ1+/5Ybl9FFI/h84X39htx+A7Pmveq2XwvY9hN8F0ncwDaM0FLPkZh64Potx2Wkoexj0/4", + "bstBXjwHmXjeTXIaQsEW9S9yQypUAk8fzSubloeOHNLVtz+oseX+QpaaK9kf7qsh+y4HNuMPjiXxVP17", + "0rC2T/jO5XlbHvdSeJy2Dg7SZG1TL6sqPw64sxWJ5doO8ODKsleb1qDNU9EHa9EWX894hxqlWz8hPZbI", + "qL7e7XZB1lKYdTzhdYnzKVyH3ir4g6zl+48OgynY2uJHND4DLCCDeiZtfMmLe4L4Aimzwi53TeWY3e/w", + "j/bcLkdQKZnOa84drfzpikPardPJVU3hKfhPC4etZu7EpmW7xtBSNNp23KRuYCeH5RVVxvr4qRIpOi+q", + "rRa4MtCPJukXlxH8+XWEdsov61d2pvwrHTB4YR0GrRqE7lOQ+yVePBWvrs6kJhrFsN+1VO78QVN8vwA6", + "tOVD/b7FQ3B7Kxr7i/hJXbBws2hqh0LwhISmIVub0N48MiAkckHx6gd4UUYRbOn4aei4ykq/l1XphmZR", + "bVFz6wy0Uu1upLmo6Do8rrlSrO8xcqn++Nec7st2Le98y667KuIjbfma7ss1HKWb0Ciduq8jTU6usk6l", + "WLOewQ+jAlYy/3bfy+1jp9YbuRroSTjV093sqzWZ104I3CjA2poU+OXnBHrdxqNzYkqOpwNNR6+DTl+v", + "BWprVaorhLbw0XdTMP5+zPsMyPFQltYfSttaVr4vKtQ/pR5h6+B7FIF9P+/UtLPEQvsot6QzmHSaaU0a", + "XuvKrVknkRimblZoZq1kJWvSzWYTm4Q5F+ChfU2ZTXyxtvYR3pv+N3h+511EOQlhDZOBokFRxXHRq3Xg", + "mNxAVbXBg55CBw9qL3Rs55Ddn3OWtIUuwCijVqkn3pC9HM6cmnWwzdx/yXGO/Ms03/iZ74u2knezY52g", + "fAxDbisN0MeQdSL1Z2PJJ2lE7uw5LN5QFATXeiqLvBaOdu5lGWwhfp3PBWnhgaNTO/0wXHptZroxztX6", + "sKuXY23Z1JOwqTmN1U9LLJbdxUpwivIsZjhCMU2vrZUSc6RGQIpSME2dg45XRH8bqlN+VG1/xmL5UMbl", + "cY0v9bBDPeMKCsvA7BL6neNvnubIKLx8Acy33a/dfbldEg7PXM2PcITMLv0ATp3XftysI70n7A7c5+v4", + "AYxz8zHdP08SCV+4JR8aCm8UJsDr668K/6J8lm5C0wFv/boKFnzd/5HrzUzanh4WgM5WiKUEMY4SxnWt", + "IsDEoPoKUh//9ZIMXkijO1UP2SQQchWrH5Qa+ppcgNviPK/x0XBncuRBOVfbrJMOa3ml6ZNfpZWx7wK7", + "NxbmwlY4BLMtID+KlbKGSr0IMFzA9dq86Ml5OkWqN5qRmN3qPBS6AeYEkbswzqN23D6a1fMIC7IjSCqo", + "pDcEiXymxRJKsAyXiKUAeUKEwAt9TVNctkXSEMzDZQWsBN+dknShDvj+X/+22RBhJyv21/31zJ3b/Njr", + "8erKE6HHf5zxdf85nmd83X/p7nGDiW3RtfXu3C7hNuIsu6uYd0cuOfT6Y8cuPQkQ7Yx7Gxz1nKeiJ9hk", + "bGiJ95A8X3DJE8sUwMgoifKyYlteIPd+26Z2rKlkvH0WJePtcykZBgDLby0gL0vf+NO8t29QN4vzhAxM", + "HoVsa5+9o/j09HZ6PddoE30MNrfmav5M3M2ueUBtas3JCjz5mZmz609Sj9pu9WbfqelZD9PI2JN7CMtm", + "fW3ibHtF6iNDhwntftf/GP4CrZ04dSNDnl/NsKMVPQvPwOdnFaKwT89wkyC2xp5OvtQRh1YgsjUI7Sm3", + "fO+5GIxN1bSlptHsBaDjN3b3cx4HB8FSykwc7O7ijE7J/myKsyxw+n8vcwOVqXG+15K8Vn+EPEbu37B7", + "O1ItuNowozvXZFX5zUQJFH8XisnV/f8LAAD//w==", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/packages/api/internal/handlers/proxy_grpc.go b/packages/api/internal/handlers/proxy_grpc.go index 605f547b39..d653f41a3b 100644 --- a/packages/api/internal/handlers/proxy_grpc.go +++ b/packages/api/internal/handlers/proxy_grpc.go @@ -257,6 +257,7 @@ func (s *SandboxService) ResumeSandbox(ctx context.Context, req *proxygrpc.Sandb s.api.buildResumeSandboxData(sandboxID, nil), &headers, true, + false, nil, // mcp ) if apiErr != nil { diff --git a/packages/api/internal/handlers/sandbox.go b/packages/api/internal/handlers/sandbox.go index 4790692a49..b7f47fced0 100644 --- a/packages/api/internal/handlers/sandbox.go +++ b/packages/api/internal/handlers/sandbox.go @@ -28,6 +28,7 @@ func (a *APIStore) startSandbox( getSandboxData orchestrator.SandboxDataFetcher, requestHeader *http.Header, isResume bool, + rebootFromRootfs bool, mcp api.Mcp, ) (*api.Sandbox, *api.APIError) { sbx, apiErr := a.startSandboxInternal( @@ -38,6 +39,7 @@ func (a *APIStore) startSandbox( getSandboxData, requestHeader, isResume, + rebootFromRootfs, mcp, ) if apiErr != nil { @@ -56,6 +58,7 @@ func (a *APIStore) startSandboxInternal( getSandboxData orchestrator.SandboxDataFetcher, requestHeader *http.Header, isResume bool, + rebootFromRootfs bool, mcp api.Mcp, ) (sandbox.Sandbox, *api.APIError) { startTime := time.Now() @@ -77,6 +80,7 @@ func (a *APIStore) startSandboxInternal( timeout, isResume, creationMeta, + rebootFromRootfs, ) if instanceErr != nil { telemetry.ReportError(ctx, "error when creating instance", instanceErr.Err) diff --git a/packages/api/internal/handlers/sandbox_connect.go b/packages/api/internal/handlers/sandbox_connect.go index 746165556c..9dcc2bef53 100644 --- a/packages/api/internal/handlers/sandbox_connect.go +++ b/packages/api/internal/handlers/sandbox_connect.go @@ -48,6 +48,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S return } + rebootFromRootfs := body.Reboot != nil && *body.Reboot teamID := teamInfo.Team.ID @@ -152,6 +153,7 @@ func (a *APIStore) PostSandboxesSandboxIDConnect(c *gin.Context, sandboxID api.S a.buildResumeSandboxData(sandboxID, nil), &c.Request.Header, true, + rebootFromRootfs, nil, // mcp ) if createErr != nil { diff --git a/packages/api/internal/handlers/sandbox_create.go b/packages/api/internal/handlers/sandbox_create.go index 95f2c0ba5e..efa6b8c09d 100644 --- a/packages/api/internal/handlers/sandbox_create.go +++ b/packages/api/internal/handlers/sandbox_create.go @@ -265,6 +265,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) { getSandboxData, &c.Request.Header, false, + false, mcp, ) if createErr != nil { diff --git a/packages/api/internal/handlers/sandbox_pause.go b/packages/api/internal/handlers/sandbox_pause.go index 3ef7eeb6d3..3abc832f6f 100644 --- a/packages/api/internal/handlers/sandbox_pause.go +++ b/packages/api/internal/handlers/sandbox_pause.go @@ -18,10 +18,15 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/auth/pkg/auth" + "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) +type pauseSandboxBody struct { + Memory *bool `json:"memory,omitempty"` +} + func (a *APIStore) PostSandboxesSandboxIDPause(c *gin.Context, sandboxID api.SandboxID) { ctx := c.Request.Context() // Get team from context, use TeamContextKey @@ -42,9 +47,22 @@ func (a *APIStore) PostSandboxesSandboxIDPause(c *gin.Context, sandboxID api.San traceID := span.SpanContext().TraceID().String() c.Set("traceID", traceID) + memory := true + if c.Request.ContentLength != 0 { + body, err := ginutils.ParseBody[pauseSandboxBody](ctx, c) + if err != nil { + a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Error when parsing request: %s", err)) + + return + } + if body.Memory != nil { + memory = *body.Memory + } + } + pause.LogInitiated(ctx, sandboxID, teamID.String(), pause.ReasonRequest) - err = a.orchestrator.RemoveSandbox(ctx, teamID, sandboxID, sandbox.RemoveOpts{Action: sandbox.StateActionPause}) + err = a.orchestrator.RemoveSandbox(ctx, teamID, sandboxID, sandbox.RemoveOpts{Action: sandbox.StateActionPause, SkipMemory: !memory}) var transErr *sandbox.InvalidStateTransitionError switch { diff --git a/packages/api/internal/handlers/sandbox_resume.go b/packages/api/internal/handlers/sandbox_resume.go index 1535e35d6f..482255a7ab 100644 --- a/packages/api/internal/handlers/sandbox_resume.go +++ b/packages/api/internal/handlers/sandbox_resume.go @@ -54,6 +54,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa } telemetry.ReportEvent(ctx, "Parsed body") + rebootFromRootfs := body.Reboot != nil && *body.Reboot timeout := sandbox.SandboxTimeoutDefault if body.Timeout != nil { @@ -160,6 +161,7 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa a.buildResumeSandboxData(sandboxID, body.AutoPause), &c.Request.Header, true, + rebootFromRootfs, nil, // mcp ) if createErr != nil { diff --git a/packages/api/internal/handlers/snapshot_template_create.go b/packages/api/internal/handlers/snapshot_template_create.go index 7908b5c9c0..7dae81f6b7 100644 --- a/packages/api/internal/handlers/snapshot_template_create.go +++ b/packages/api/internal/handlers/snapshot_template_create.go @@ -59,6 +59,9 @@ func (a *APIStore) PostSandboxesSandboxIDSnapshots(c *gin.Context, sandboxID api opts := orchestrator.SnapshotTemplateOpts{ Tag: id.DefaultTag, } + if body.Memory != nil { + opts.SkipMemory = !*body.Memory + } if body.Name != nil { identifier, tag, err := id.ParseName(*body.Name) diff --git a/packages/api/internal/orchestrator/create_instance.go b/packages/api/internal/orchestrator/create_instance.go index c4f2239ee7..fc051722c9 100644 --- a/packages/api/internal/orchestrator/create_instance.go +++ b/packages/api/internal/orchestrator/create_instance.go @@ -10,6 +10,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/api/internal/api" @@ -127,6 +128,7 @@ func (o *Orchestrator) CreateSandbox( timeout time.Duration, isResume bool, creationMeta sandbox.CreationMetadata, + rebootFromRootfsOpt ...bool, ) (sbx sandbox.Sandbox, apiErr *api.APIError) { ctx, childSpan := tracer.Start(ctx, "create-sandbox") defer childSpan.End() @@ -252,6 +254,10 @@ func (o *Orchestrator) CreateSandbox( TimeoutSeconds: sbxData.AutoResume.Timeout, } } + rebootFromRootfs := len(rebootFromRootfsOpt) > 0 && rebootFromRootfsOpt[0] + if rebootFromRootfs { + ctx = metadata.AppendToOutgoingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey, "true") + } sbxRequest := &orchestrator.SandboxCreateRequest{ Sandbox: &orchestrator.SandboxConfig{ @@ -365,12 +371,10 @@ func (o *Orchestrator) CreateSandbox( // Copy to a new variable to avoid race conditions sbxToRemove := sbx go func() { - killErr := o.removeSandboxFromNode( - context.WithoutCancel(ctx), - sbxToRemove, - sandbox.StateActionKill, - sandbox.KillReasonUnknown, - ) + killErr := o.removeSandboxFromNode(context.WithoutCancel(ctx), sbxToRemove, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, + Reason: sandbox.KillReasonUnknown, + }) if killErr != nil { logger.L().Error(ctx, "Error removing sandbox", zap.Error(killErr), diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index 6258961ca1..f4c1db19cb 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -104,7 +104,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand defer func() { go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) }() // Once we start the removal process, we want to make sure it gets removed from the store defer o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID) - err = o.removeSandboxFromNode(ctx, sbx, opts.Action, opts.Reason) + err = o.removeSandboxFromNode(ctx, sbx, opts) if err != nil { fields := []zap.Field{ zap.String("state_action", opts.Action.Name), @@ -123,12 +123,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand return nil } -func (o *Orchestrator) removeSandboxFromNode( - ctx context.Context, - sbx sandbox.Sandbox, - stateAction sandbox.StateAction, - reason sandbox.KillReason, -) error { +func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sandbox, opts sandbox.RemoveOpts) error { ctx, span := tracer.Start(ctx, "remove-sandbox-from-node") defer span.End() @@ -137,8 +132,8 @@ func (o *Orchestrator) removeSandboxFromNode( fields := []zap.Field{ logger.WithNodeID(sbx.NodeID), } - if stateAction == sandbox.StateActionKill { - fields = append(fields, zap.String("kill_reason", reason.String())) + if opts.Action == sandbox.StateActionKill { + fields = append(fields, zap.String("kill_reason", opts.Reason.String())) } logger.L().Error(ctx, "failed to get node", fields...) @@ -156,8 +151,8 @@ func (o *Orchestrator) removeSandboxFromNode( zap.Error(err), logger.WithSandboxID(sbx.SandboxID), } - if stateAction == sandbox.StateActionKill { - fields = append(fields, zap.String("kill_reason", reason.String())) + if opts.Action == sandbox.StateActionKill { + fields = append(fields, zap.String("kill_reason", opts.Reason.String())) } logger.L().Error(ctx, "error removing routing record from catalog", fields...) @@ -166,12 +161,12 @@ func (o *Orchestrator) removeSandboxFromNode( sbxlogger.I(sbx).Debug(ctx, "Removing sandbox", zap.Bool("auto_pause", sbx.AutoPause), - zap.String("state_action", stateAction.Name), + zap.String("state_action", opts.Action.Name), ) - switch stateAction { + switch opts.Action { case sandbox.StateActionPause: - err := o.pauseSandbox(ctx, node, sbx) + err := o.pauseSandbox(ctx, node, sbx, opts.SkipMemory) if err != nil { if dberrors.IsForeignKeyViolation(err) { killErr := o.killSandboxOnNode(ctx, node, sbx, sandbox.KillReasonBaseTemplateMissing) @@ -191,7 +186,7 @@ func (o *Orchestrator) removeSandboxFromNode( return nil case sandbox.StateActionKill: - return o.killSandboxOnNode(ctx, node, sbx, reason) + return o.killSandboxOnNode(ctx, node, sbx, opts.Reason) } return nil diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index c1863cebd8..a17aa6390b 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" "github.com/e2b-dev/infra/packages/api/internal/sandbox" @@ -26,18 +27,18 @@ func (PauseQueueExhaustedError) Error() string { return "The pause queue is exhausted" } -func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox) error { +func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, skipMemory bool) error { ctx, span := tracer.Start(ctx, "pause-sandbox") defer span.End() - result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node)) + result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, skipMemory)) if err != nil { telemetry.ReportCriticalError(ctx, "error inserting snapshot for env", err) return err } - err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String()) + err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String(), skipMemory) if errors.Is(err, PauseQueueExhaustedError{}) { telemetry.ReportCriticalError(ctx, "pause queue exhausted", err) @@ -68,11 +69,14 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, return nil } -func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string) error { +func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string, skipMemory bool) error { childCtx, childSpan := tracer.Start(ctx, "snapshot-instance") defer childSpan.End() client, childCtx := node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID) + if skipMemory { + childCtx = metadata.AppendToOutgoingContext(childCtx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey, "false") + } _, err := client.Sandbox.Pause( childCtx, &orchestrator.SandboxPauseRequest{ SandboxId: sbx.SandboxID, @@ -103,14 +107,13 @@ func (o *Orchestrator) WaitForStateChange(ctx context.Context, teamID uuid.UUID, return o.sandboxStore.WaitForStateChange(ctx, teamID, sandboxID) } -func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node) queries.UpsertSnapshotParams { +func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, skipMemory bool) queries.UpsertSnapshotParams { machineInfo := node.MachineInfo() metadata := types.JSONBStringMap(sbx.Metadata) if metadata == nil { metadata = types.JSONBStringMap{} } - return queries.UpsertSnapshotParams{ // Used if there's no snapshot for this sandbox yet TemplateID: id.Generate(), diff --git a/packages/api/internal/orchestrator/snapshot_template.go b/packages/api/internal/orchestrator/snapshot_template.go index 4d08816d1e..e1aa972ba4 100644 --- a/packages/api/internal/orchestrator/snapshot_template.go +++ b/packages/api/internal/orchestrator/snapshot_template.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "google.golang.org/grpc/metadata" "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/db/pkg/types" @@ -30,6 +31,8 @@ type SnapshotTemplateOpts struct { Namespace *string // Tag is the build tag parsed from the name, defaults to "default". Tag string + // SkipMemory controls whether the snapshot template should omit VM memory. + SkipMemory bool } // CreateSnapshotTemplate creates a persistent snapshot template from a running sandbox and immediately resumes it. @@ -67,7 +70,7 @@ func (o *Orchestrator) CreateSnapshotTemplate(ctx context.Context, teamID uuid.U return SnapshotTemplateResult{}, fmt.Errorf("node '%s' not found", sbx.NodeID) } - upsertResult, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node)) + upsertResult, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, opts.SkipMemory)) if err != nil { return SnapshotTemplateResult{}, fmt.Errorf("error upserting snapshot: %w", err) } @@ -84,6 +87,9 @@ func (o *Orchestrator) CreateSnapshotTemplate(ctx context.Context, teamID uuid.U // kills the sandbox itself; RemoveSandbox is still needed to clean up // API-side state (store, routing, analytics). client, childCtx := node.GetClient(ctx) + if opts.SkipMemory { + childCtx = metadata.AppendToOutgoingContext(childCtx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey, "false") + } _, err = client.Sandbox.Checkpoint(childCtx, &orchestrator.SandboxCheckpointRequest{ SandboxId: sbx.SandboxID, BuildId: upsertResult.BuildID.String(), diff --git a/packages/api/internal/sandbox/sandboxtypes/states.go b/packages/api/internal/sandbox/sandboxtypes/states.go index 409b30253d..cad8c5ca67 100644 --- a/packages/api/internal/sandbox/sandboxtypes/states.go +++ b/packages/api/internal/sandbox/sandboxtypes/states.go @@ -70,9 +70,10 @@ func (r KillReason) String() string { // RemoveOpts bundles the parameters that control sandbox removal. type RemoveOpts struct { - Action StateAction - Eviction bool - Reason KillReason + Action StateAction + Eviction bool + SkipMemory bool + Reason KillReason } var AllowedTransitions = map[State]map[State]bool{ diff --git a/packages/orchestrator/cmd/sample-dedup-gains/main.go b/packages/orchestrator/cmd/sample-dedup-gains/main.go new file mode 100644 index 0000000000..154a46a62e --- /dev/null +++ b/packages/orchestrator/cmd/sample-dedup-gains/main.go @@ -0,0 +1,824 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "errors" + "flag" + "fmt" + "io" + "log" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" + "go.opentelemetry.io/otel/metric/noop" + + "github.com/e2b-dev/infra/packages/orchestrator/cmd/internal/cmdutil" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/cfg" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block" + blockmetrics "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block/metrics" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build" + sboxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +const ( + artifactMemfile = "memfile" + artifactRootfs = "rootfs" +) + +type buildPair struct { + BuildID string + ParentBuildID string + SiblingBuildID string + SiblingMemfileBuildID string + SiblingRootfsBuildID string + Family string +} + +type pool struct { + Name string + TargetArtifact string + CandidateArtifact string + CandidateBuild func(buildPair) string + Positional bool + ValidationOnly bool +} + +type rowResult struct { + BuildID string + ParentBuildID string + Family string + TargetArtifact string + Pool string + CandidateBuildID string + CandidateArtifact string + ValidationOnly bool + Positional bool + TargetPages int64 + SampledTargetPages int64 + CandidatePages int64 + IndexedCandidatePages int64 + Hits int64 + ZeroPages int64 + EligibleBytes int64 + SavingsBytes int64 + SavingsRatio float64 + FrameSizeBytes int64 + FrameTargetFrames int64 + FrameHits int64 + FrameSavingsBytes int64 + FrameSavingsRatio float64 + IndexMS int64 + CompareMS int64 + Error string +} + +type summary struct { + Rows int64 + EligibleBytes int64 + SavingsBytes int64 + Hits int64 + TargetPages int64 + FrameTargetFrames int64 + FrameHits int64 + Errors int64 + Ratios []float64 +} + +type frameRef struct { + off int64 + length int64 +} + +type analyzer struct { + ctx context.Context + store storage.StorageProvider + diffStore *build.DiffStore + cacheDir string + metrics blockmetrics.Metrics + devices map[string]*sboxtemplate.Storage +} + +func main() { + storagePath := flag.String("storage", ".local-build", "storage: local path or gs://bucket") + buildsFile := flag.String("builds-file", "", "CSV with build_id,parent_build_id and optional sibling columns") + buildID := flag.String("build", "", "single current build ID") + parentBuildID := flag.String("parent-build", "", "single parent build ID") + artifacts := flag.String("artifacts", "both", "memfile, rootfs, or both") + maxTargetPages := flag.Int("max-target-pages", 50000, "target pages to sample per artifact; 0 scans all") + maxCandidatePages := flag.Int("max-candidate-pages", 100000, "candidate pages to index per pool; 0 scans all") + frameSize := flag.Int("frame-size", 2<<20, "compression-frame size for whole-frame dedup estimate") + seed := flag.Int64("seed", 1, "sampling seed") + outputPath := flag.String("csv-path", "", "write detailed CSV here; default stdout") + includeValidation := flag.Bool("include-validation", true, "include validation-only pools") + + flag.Parse() + cmdutil.SuppressNoisyLogs() + + if *buildsFile == "" && (*buildID == "" || *parentBuildID == "") { + log.Fatal("provide -builds-file or both -build and -parent-build") + } + + pairs, err := loadPairs(*buildsFile, *buildID, *parentBuildID) + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + if err := cmdutil.SetupStorage(*storagePath); err != nil { + log.Fatal(err) + } + + ff, err := featureflags.NewClientWithLogLevel(ldlog.Error) + if err != nil { + log.Fatalf("feature flags: %s", err) + } + metrics, err := blockmetrics.NewMetrics(noop.NewMeterProvider()) + if err != nil { + log.Fatalf("metrics: %s", err) + } + persistence, err := storage.GetStorageProvider(ctx, storage.TemplateStorageConfig) + if err != nil { + log.Fatalf("storage: %s", err) + } + cacheDir, err := os.MkdirTemp("", "sample-dedup-gains-*") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(cacheDir) + + diffStore, err := build.NewDiffStore(cfg.Config{}, ff, cacheDir, time.Hour, time.Second) + if err != nil { + log.Fatal(err) + } + defer diffStore.Close() + + out := io.Writer(os.Stdout) + var outFile *os.File + if *outputPath != "" { + if err := os.MkdirAll(filepath.Dir(*outputPath), 0o755); err != nil { + log.Fatal(err) + } + outFile, err = os.Create(*outputPath) + if err != nil { + log.Fatal(err) + } + defer outFile.Close() + out = outFile + } + + a := &analyzer{ + ctx: ctx, + store: persistence, + diffStore: diffStore, + cacheDir: cacheDir, + metrics: metrics, + devices: make(map[string]*sboxtemplate.Storage), + } + defer a.close() + + writer := csv.NewWriter(out) + if err := writer.Write(resultHeader()); err != nil { + log.Fatal(err) + } + + selectedArtifacts, err := parseArtifacts(*artifacts) + if err != nil { + log.Fatal(err) + } + + summaries := make(map[string]*summary) + for i, pair := range pairs { + for _, artifact := range selectedArtifacts { + results := a.analyzeArtifact(pair, artifact, *maxTargetPages, *maxCandidatePages, int64(*frameSize), *seed+int64(i), *includeValidation) + for _, result := range results { + if err := writer.Write(result.Record()); err != nil { + log.Fatal(err) + } + addSummary(summaries, result) + } + } + } + writer.Flush() + if err := writer.Error(); err != nil { + log.Fatal(err) + } + + printSummary(os.Stderr, summaries) +} + +func (a *analyzer) analyzeArtifact(pair buildPair, artifact string, maxTargetPages, maxCandidatePages int, frameSize int64, seed int64, includeValidation bool) []rowResult { + target, err := a.device(pair.BuildID, artifact) + if err != nil { + return []rowResult{errorResult(pair, artifact, "open_target", err)} + } + + targetOffsets, targetPages := sampledSelfPages(target.Header(), pair.BuildID, maxTargetPages, seed) + results := make([]rowResult, 0) + for _, p := range candidatePools(pair, artifact, includeValidation) { + candidateBuildID := p.CandidateBuild(pair) + base := rowResult{ + BuildID: pair.BuildID, + ParentBuildID: pair.ParentBuildID, + Family: pair.Family, + TargetArtifact: artifact, + Pool: p.Name, + CandidateBuildID: candidateBuildID, + CandidateArtifact: p.CandidateArtifact, + ValidationOnly: p.ValidationOnly, + Positional: p.Positional, + FrameSizeBytes: frameSize, + TargetPages: targetPages, + SampledTargetPages: int64(len(targetOffsets)), + } + if candidateBuildID == "" { + base.Error = "candidate build missing" + results = append(results, base) + continue + } + + candidate, err := a.device(candidateBuildID, p.CandidateArtifact) + if err != nil { + base.Error = err.Error() + results = append(results, base) + continue + } + if p.Positional { + results = append(results, a.comparePositional(base, target, candidate, targetOffsets)) + continue + } + results = append(results, a.compareIndexed(base, target, candidate, targetOffsets, maxCandidatePages, seed)) + } + + return results +} + +func (a *analyzer) comparePositional(base rowResult, target, candidate block.ReadonlyDevice, targetOffsets []int64) rowResult { + start := time.Now() + var hits, zeroes int64 + targetPage := make([]byte, header.PageSize) + candidatePage := make([]byte, header.PageSize) + for _, off := range targetOffsets { + if _, err := target.ReadAt(a.ctx, targetPage, off); err != nil { + base.Error = fmt.Sprintf("read target at %d: %s", off, err) + break + } + if header.IsZero(targetPage) { + zeroes++ + continue + } + if _, err := candidate.ReadAt(a.ctx, candidatePage, off); err != nil { + continue + } + if bytes.Equal(targetPage, candidatePage) { + hits++ + } + } + base.Hits = hits + base.ZeroPages = zeroes + base.CompareMS = time.Since(start).Milliseconds() + base.CandidatePages = base.TargetPages + base.IndexedCandidatePages = base.SampledTargetPages + base.measureFramePositional(a.ctx, target, candidate, targetOffsets) + base.finish() + + return base +} + +func (a *analyzer) compareIndexed(base rowResult, target, candidate block.ReadonlyDevice, targetOffsets []int64, maxCandidatePages int, seed int64) rowResult { + indexStart := time.Now() + candidateOffsets, candidatePages := sampledAllPages(candidate.Header(), maxCandidatePages, seed) + base.CandidatePages = candidatePages + + index := make(map[[32]byte][]int64, len(candidateOffsets)) + page := make([]byte, header.PageSize) + for _, off := range candidateOffsets { + if _, err := candidate.ReadAt(a.ctx, page, off); err != nil { + continue + } + if header.IsZero(page) { + continue + } + sum := sha256.Sum256(page) + index[sum] = append(index[sum], off) + base.IndexedCandidatePages++ + } + base.IndexMS = time.Since(indexStart).Milliseconds() + + compareStart := time.Now() + targetPage := make([]byte, header.PageSize) + candidatePage := make([]byte, header.PageSize) + var hits, zeroes int64 + for _, off := range targetOffsets { + if _, err := target.ReadAt(a.ctx, targetPage, off); err != nil { + base.Error = fmt.Sprintf("read target at %d: %s", off, err) + break + } + if header.IsZero(targetPage) { + zeroes++ + continue + } + for _, candidateOff := range index[sha256.Sum256(targetPage)] { + if _, err := candidate.ReadAt(a.ctx, candidatePage, candidateOff); err != nil { + continue + } + if bytes.Equal(targetPage, candidatePage) { + hits++ + break + } + } + } + base.Hits = hits + base.ZeroPages = zeroes + base.CompareMS = time.Since(compareStart).Milliseconds() + base.measureFrameIndexed(a.ctx, target, candidate, targetOffsets) + base.finish() + + return base +} + +func (r *rowResult) finish() { + if r.SampledTargetPages == 0 { + return + } + r.EligibleBytes = r.TargetPages * header.PageSize + r.SavingsBytes = int64(float64(r.Hits) / float64(r.SampledTargetPages) * float64(r.EligibleBytes)) + if r.EligibleBytes > 0 { + r.SavingsRatio = float64(r.SavingsBytes) / float64(r.EligibleBytes) + } + if r.FrameTargetFrames > 0 { + r.FrameSavingsBytes = r.FrameHits * r.FrameSizeBytes + r.FrameSavingsRatio = float64(r.FrameHits) / float64(r.FrameTargetFrames) + } +} + +func (r *rowResult) measureFramePositional(ctx context.Context, target, candidate block.ReadonlyDevice, targetOffsets []int64) { + frames := targetFrames(targetOffsets, r.FrameSizeBytes) + r.FrameTargetFrames = int64(len(frames)) + if r.FrameSizeBytes <= 0 { + return + } + + targetSize, err := target.Size(ctx) + if err != nil { + return + } + candidateSize, err := candidate.Size(ctx) + if err != nil { + return + } + for _, off := range frames { + length := min(r.FrameSizeBytes, targetSize-off) + if length <= 0 || off+length > candidateSize { + continue + } + targetFrame, err := readRange(ctx, target, off, length) + if err != nil { + continue + } + candidateFrame, err := readRange(ctx, candidate, off, length) + if err != nil { + continue + } + if bytes.Equal(targetFrame, candidateFrame) { + r.FrameHits++ + } + } +} + +func (r *rowResult) measureFrameIndexed(ctx context.Context, target, candidate block.ReadonlyDevice, targetOffsets []int64) { + frames := targetFrames(targetOffsets, r.FrameSizeBytes) + r.FrameTargetFrames = int64(len(frames)) + if r.FrameSizeBytes <= 0 { + return + } + + candidateSize, err := candidate.Size(ctx) + if err != nil { + return + } + index := make(map[[32]byte][]frameRef) + for off := int64(0); off < candidateSize; off += r.FrameSizeBytes { + length := min(r.FrameSizeBytes, candidateSize-off) + frame, err := readRange(ctx, candidate, off, length) + if err != nil || header.IsZero(frame) { + continue + } + index[sha256.Sum256(frame)] = append(index[sha256.Sum256(frame)], frameRef{off: off, length: length}) + } + + targetSize, err := target.Size(ctx) + if err != nil { + return + } + for _, off := range frames { + length := min(r.FrameSizeBytes, targetSize-off) + if length <= 0 { + continue + } + targetFrame, err := readRange(ctx, target, off, length) + if err != nil || header.IsZero(targetFrame) { + continue + } + sum := sha256.Sum256(targetFrame) + for _, ref := range index[sum] { + if ref.length != length { + continue + } + candidateFrame, err := readRange(ctx, candidate, ref.off, ref.length) + if err != nil { + continue + } + if bytes.Equal(targetFrame, candidateFrame) { + r.FrameHits++ + break + } + } + } +} + +func targetFrames(targetOffsets []int64, frameSize int64) []int64 { + if frameSize <= 0 { + return nil + } + seen := make(map[int64]struct{}) + frames := make([]int64, 0) + for _, off := range targetOffsets { + frameOff := (off / frameSize) * frameSize + if _, ok := seen[frameOff]; ok { + continue + } + seen[frameOff] = struct{}{} + frames = append(frames, frameOff) + } + + return frames +} + +func readRange(ctx context.Context, d block.ReadonlyDevice, off, length int64) ([]byte, error) { + buf := make([]byte, length) + _, err := d.ReadAt(ctx, buf, off) + + return buf, err +} + +func (a *analyzer) device(buildID, artifact string) (*sboxtemplate.Storage, error) { + key := buildID + "/" + artifact + if d, ok := a.devices[key]; ok { + return d, nil + } + fileType, err := diffType(artifact) + if err != nil { + return nil, err + } + d, err := sboxtemplate.NewStorage(a.ctx, a.diffStore, buildID, fileType, nil, a.store, a.metrics) + if err != nil { + return nil, err + } + a.devices[key] = d + + return d, nil +} + +func (a *analyzer) close() { + for _, d := range a.devices { + _ = d.Close() + } +} + +func sampledSelfPages(h *header.Header, buildID string, maxPages int, seed int64) ([]int64, int64) { + id, err := uuid.Parse(buildID) + if err != nil || h == nil { + return nil, 0 + } + var total int64 + return sampleOffsets(h.Mapping, func(m header.BuildMap) bool { + return m.BuildId == id + }, maxPages, seed, &total), total +} + +func sampledAllPages(h *header.Header, maxPages int, seed int64) ([]int64, int64) { + if h == nil || h.Metadata == nil { + return nil, 0 + } + total := int64(h.Metadata.Size / header.PageSize) + return sampleOffsets([]header.BuildMap{{ + Offset: 0, + Length: uint64(total * header.PageSize), + }}, func(header.BuildMap) bool { return true }, maxPages, seed, nil), total +} + +func sampleOffsets(mappings []header.BuildMap, include func(header.BuildMap) bool, maxPages int, seed int64, totalOut *int64) []int64 { + rng := rand.New(rand.NewSource(seed)) + var sampled []int64 + var seen int64 + for _, m := range mappings { + if !include(m) { + continue + } + start := alignUp(int64(m.Offset), header.PageSize) + end := alignDown(int64(m.Offset+m.Length), header.PageSize) + for off := start; off < end; off += header.PageSize { + seen++ + if maxPages <= 0 { + sampled = append(sampled, off) + continue + } + if int64(len(sampled)) < int64(maxPages) { + sampled = append(sampled, off) + continue + } + j := rng.Int63n(seen) + if j < int64(maxPages) { + sampled[j] = off + } + } + } + if totalOut != nil { + *totalOut = seen + } + + return sampled +} + +func alignUp(v, by int64) int64 { + if v%by == 0 { + return v + } + return v + by - v%by +} + +func alignDown(v, by int64) int64 { + return v - v%by +} + +func candidatePools(pair buildPair, targetArtifact string, includeValidation bool) []pool { + siblingFor := func(artifact string) func(buildPair) string { + return func(p buildPair) string { + if artifact == artifactMemfile && p.SiblingMemfileBuildID != "" { + return p.SiblingMemfileBuildID + } + if artifact == artifactRootfs && p.SiblingRootfsBuildID != "" { + return p.SiblingRootfsBuildID + } + return p.SiblingBuildID + } + } + parent := func(p buildPair) string { return p.ParentBuildID } + current := func(p buildPair) string { return p.BuildID } + + var pools []pool + switch targetArtifact { + case artifactMemfile: + pools = append(pools, + pool{Name: "memfile_parent_memfile_positional", TargetArtifact: artifactMemfile, CandidateArtifact: artifactMemfile, CandidateBuild: parent, Positional: true}, + pool{Name: "memfile_current_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: current}, + pool{Name: "memfile_sibling_memfile", TargetArtifact: artifactMemfile, CandidateArtifact: artifactMemfile, CandidateBuild: siblingFor(artifactMemfile)}, + ) + if includeValidation { + pools = append(pools, + pool{Name: "memfile_parent_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: parent, ValidationOnly: true}, + pool{Name: "memfile_sibling_rootfs", TargetArtifact: artifactMemfile, CandidateArtifact: artifactRootfs, CandidateBuild: siblingFor(artifactRootfs), ValidationOnly: true}, + ) + } + case artifactRootfs: + pools = append(pools, + pool{Name: "rootfs_parent_rootfs_positional", TargetArtifact: artifactRootfs, CandidateArtifact: artifactRootfs, CandidateBuild: parent, Positional: true}, + pool{Name: "rootfs_parent_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: parent}, + pool{Name: "rootfs_sibling_rootfs", TargetArtifact: artifactRootfs, CandidateArtifact: artifactRootfs, CandidateBuild: siblingFor(artifactRootfs)}, + ) + if includeValidation { + pools = append(pools, + pool{Name: "rootfs_current_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: current, ValidationOnly: true}, + pool{Name: "rootfs_sibling_memfile", TargetArtifact: artifactRootfs, CandidateArtifact: artifactMemfile, CandidateBuild: siblingFor(artifactMemfile), ValidationOnly: true}, + ) + } + } + + return pools +} + +func parseArtifacts(value string) ([]string, error) { + switch strings.ToLower(value) { + case "both": + return []string{artifactMemfile, artifactRootfs}, nil + case artifactMemfile: + return []string{artifactMemfile}, nil + case artifactRootfs: + return []string{artifactRootfs}, nil + default: + return nil, fmt.Errorf("unknown -artifacts value %q", value) + } +} + +func diffType(artifact string) (build.DiffType, error) { + switch artifact { + case artifactMemfile: + return build.Memfile, nil + case artifactRootfs: + return build.Rootfs, nil + default: + return "", fmt.Errorf("unknown artifact %q", artifact) + } +} + +func loadPairs(path, buildID, parentBuildID string) ([]buildPair, error) { + if path == "" { + return []buildPair{{BuildID: buildID, ParentBuildID: parentBuildID}}, nil + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + r.TrimLeadingSpace = true + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return nil, errors.New("builds file is empty") + } + + start := 0 + index := map[string]int{} + if looksLikeHeader(records[0]) { + start = 1 + for i, name := range records[0] { + index[strings.TrimSpace(name)] = i + } + } + + pairs := make([]buildPair, 0, len(records)-start) + for _, record := range records[start:] { + pair := buildPair{ + BuildID: getCSV(record, index, "build_id", 0), + ParentBuildID: getCSV(record, index, "parent_build_id", 1), + SiblingBuildID: getCSV(record, index, "sibling_build_id", 2), + SiblingMemfileBuildID: getCSV(record, index, "sibling_memfile_build_id", -1), + SiblingRootfsBuildID: getCSV(record, index, "sibling_rootfs_build_id", -1), + Family: getCSV(record, index, "family", -1), + } + if pair.BuildID == "" || pair.ParentBuildID == "" { + return nil, fmt.Errorf("build_id and parent_build_id are required in record %q", strings.Join(record, ",")) + } + pairs = append(pairs, pair) + } + + return pairs, nil +} + +func looksLikeHeader(record []string) bool { + for _, field := range record { + if strings.Contains(strings.ToLower(field), "build") { + return true + } + } + return false +} + +func getCSV(record []string, index map[string]int, name string, fallback int) string { + if i, ok := index[name]; ok && i >= 0 && i < len(record) { + return strings.TrimSpace(record[i]) + } + if fallback >= 0 && fallback < len(record) { + return strings.TrimSpace(record[fallback]) + } + return "" +} + +func resultHeader() []string { + return []string{ + "build_id", "parent_build_id", "family", "target_artifact", "pool", + "candidate_build_id", "candidate_artifact", "validation_only", "positional", + "target_pages", "sampled_target_pages", "candidate_pages", "indexed_candidate_pages", + "hits", "zero_pages", "eligible_bytes", "savings_bytes", "savings_ratio", + "frame_size_bytes", "frame_target_frames", "frame_hits", "frame_savings_bytes", "frame_savings_ratio", + "index_ms", "compare_ms", "error", + } +} + +func (r rowResult) Record() []string { + return []string{ + r.BuildID, r.ParentBuildID, r.Family, r.TargetArtifact, r.Pool, + r.CandidateBuildID, r.CandidateArtifact, + strconv.FormatBool(r.ValidationOnly), strconv.FormatBool(r.Positional), + strconv.FormatInt(r.TargetPages, 10), + strconv.FormatInt(r.SampledTargetPages, 10), + strconv.FormatInt(r.CandidatePages, 10), + strconv.FormatInt(r.IndexedCandidatePages, 10), + strconv.FormatInt(r.Hits, 10), + strconv.FormatInt(r.ZeroPages, 10), + strconv.FormatInt(r.EligibleBytes, 10), + strconv.FormatInt(r.SavingsBytes, 10), + strconv.FormatFloat(r.SavingsRatio, 'f', 6, 64), + strconv.FormatInt(r.FrameSizeBytes, 10), + strconv.FormatInt(r.FrameTargetFrames, 10), + strconv.FormatInt(r.FrameHits, 10), + strconv.FormatInt(r.FrameSavingsBytes, 10), + strconv.FormatFloat(r.FrameSavingsRatio, 'f', 6, 64), + strconv.FormatInt(r.IndexMS, 10), + strconv.FormatInt(r.CompareMS, 10), + r.Error, + } +} + +func errorResult(pair buildPair, artifact, pool string, err error) rowResult { + return rowResult{ + BuildID: pair.BuildID, + ParentBuildID: pair.ParentBuildID, + Family: pair.Family, + TargetArtifact: artifact, + Pool: pool, + Error: err.Error(), + } +} + +func addSummary(summaries map[string]*summary, r rowResult) { + s := summaries[r.Pool] + if s == nil { + s = &summary{} + summaries[r.Pool] = s + } + s.Rows++ + s.EligibleBytes += r.EligibleBytes + s.SavingsBytes += r.SavingsBytes + s.Hits += r.Hits + s.TargetPages += r.SampledTargetPages + s.FrameTargetFrames += r.FrameTargetFrames + s.FrameHits += r.FrameHits + if r.Error != "" { + s.Errors++ + } + if r.Error == "" && r.EligibleBytes > 0 { + s.Ratios = append(s.Ratios, r.SavingsRatio) + } +} + +func printSummary(w io.Writer, summaries map[string]*summary) { + fmt.Fprintln(w, "\nSUMMARY") + fmt.Fprintln(w, "pool,rows,errors,sampled_pages,hits,eligible_bytes,savings_bytes,weighted_savings_ratio,frame_target_frames,frame_hits,frame_savings_ratio,mean_row_ratio,ci95_low,ci95_high") + for pool, s := range summaries { + ratio := 0.0 + if s.EligibleBytes > 0 { + ratio = float64(s.SavingsBytes) / float64(s.EligibleBytes) + } + frameRatio := 0.0 + if s.FrameTargetFrames > 0 { + frameRatio = float64(s.FrameHits) / float64(s.FrameTargetFrames) + } + mean, low, high := bootstrapMeanCI(s.Ratios, 1000, 1) + fmt.Fprintf(w, "%s,%d,%d,%d,%d,%d,%d,%.6f,%d,%d,%.6f,%.6f,%.6f,%.6f\n", + pool, s.Rows, s.Errors, s.TargetPages, s.Hits, s.EligibleBytes, s.SavingsBytes, ratio, s.FrameTargetFrames, s.FrameHits, frameRatio, mean, low, high) + } +} + +func bootstrapMeanCI(values []float64, iterations int, seed int64) (mean, low, high float64) { + if len(values) == 0 { + return 0, 0, 0 + } + for _, v := range values { + mean += v + } + mean /= float64(len(values)) + if len(values) == 1 { + return mean, mean, mean + } + + rng := rand.New(rand.NewSource(seed)) + samples := make([]float64, iterations) + for i := range samples { + var total float64 + for range values { + total += values[rng.Intn(len(values))] + } + samples[i] = total / float64(len(values)) + } + sortFloat64s(samples) + + return mean, samples[iterations*25/1000], samples[iterations*975/1000] +} + +func sortFloat64s(values []float64) { + for i := 1; i < len(values); i++ { + v := values[i] + j := i - 1 + for ; j >= 0 && values[j] > v; j-- { + values[j+1] = values[j] + } + values[j+1] = v + } +} diff --git a/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go b/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go new file mode 100644 index 0000000000..738d6b7f7e --- /dev/null +++ b/packages/orchestrator/cmd/synthetic-dedup-corpus/main.go @@ -0,0 +1,278 @@ +package main + +import ( + "encoding/csv" + "flag" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sort" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/google/uuid" + + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" +) + +const pageSize = int(header.PageSize) + +type scenario struct { + name string + buildID uuid.UUID + parentID uuid.UUID + siblingID uuid.UUID + mutate func(*corpus, []int) +} + +type corpus struct { + pages int + + parentMem []byte + parentRoot []byte + childMem []byte + childRoot []byte + siblingMem []byte + siblingRoot []byte +} + +func main() { + out := flag.String("out", ".synthetic-dedup", "output storage root") + pages := flag.Int("pages", 2048, "pages per artifact") + dirtyPages := flag.Int("dirty-pages", 512, "dirty pages per child artifact") + seed := flag.Int64("seed", 1, "random seed") + flag.Parse() + + if *dirtyPages > *pages { + log.Fatal("-dirty-pages cannot exceed -pages") + } + if err := os.RemoveAll(*out); err != nil { + log.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(*out, "templates"), 0o755); err != nil { + log.Fatal(err) + } + + rng := rand.New(rand.NewSource(*seed)) + pairsPath := filepath.Join(*out, "pairs.csv") + pairs, err := os.Create(pairsPath) + if err != nil { + log.Fatal(err) + } + defer pairs.Close() + pairsCSV := csv.NewWriter(pairs) + if err := pairsCSV.Write([]string{"build_id", "parent_build_id", "sibling_build_id", "family"}); err != nil { + log.Fatal(err) + } + + for _, s := range scenarios() { + dirty := pickPages(*pages, *dirtyPages, rng) + c := newCorpus(*pages, rng) + s.mutate(c, dirty) + if err := writeScenario(*out, s, c, dirty); err != nil { + log.Fatal(err) + } + if err := pairsCSV.Write([]string{s.buildID.String(), s.parentID.String(), s.siblingID.String(), s.name}); err != nil { + log.Fatal(err) + } + } + pairsCSV.Flush() + if err := pairsCSV.Error(); err != nil { + log.Fatal(err) + } + + fmt.Printf("storage=%s\npairs=%s\n", *out, pairsPath) +} + +func scenarios() []scenario { + return []scenario{ + { + name: "writeback_current_rootfs", + buildID: uuid.MustParse("10000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("10000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("10000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for i, p := range dirty { + fillPage(c.childRoot, p, byte(40+i%100)) + copyPage(c.childMem, p, c.childRoot, p) + } + }, + }, + { + name: "rootfs_from_parent_memfile", + buildID: uuid.MustParse("20000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("20000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("20000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childRoot, p, c.parentMem, p) + fillPage(c.childMem, p, 0x91) + } + }, + }, + { + name: "sibling_memfile", + buildID: uuid.MustParse("30000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("30000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("30000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childMem, p, c.siblingMem, p) + fillPage(c.childRoot, p, 0xa1) + } + }, + }, + { + name: "parent_rootfs_only", + buildID: uuid.MustParse("40000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("40000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("40000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for _, p := range dirty { + copyPage(c.childMem, p, c.parentRoot, p) + fillPage(c.childRoot, p, 0xb1) + } + }, + }, + { + name: "random", + buildID: uuid.MustParse("50000000-0000-0000-0000-000000000001"), + parentID: uuid.MustParse("50000000-0000-0000-0000-000000000002"), + siblingID: uuid.MustParse("50000000-0000-0000-0000-000000000003"), + mutate: func(c *corpus, dirty []int) { + for i, p := range dirty { + fillPage(c.childMem, p, byte(0xc0+i%31)) + fillPage(c.childRoot, p, byte(0xe0+i%31)) + } + }, + }, + } +} + +func newCorpus(pages int, rng *rand.Rand) *corpus { + size := pages * pageSize + c := &corpus{ + pages: pages, + parentMem: make([]byte, size), + parentRoot: make([]byte, size), + childMem: make([]byte, size), + childRoot: make([]byte, size), + siblingMem: make([]byte, size), + siblingRoot: make([]byte, size), + } + fillRandom(c.parentMem, rng) + fillRandom(c.parentRoot, rng) + fillRandom(c.siblingMem, rng) + fillRandom(c.siblingRoot, rng) + copy(c.childMem, c.parentMem) + copy(c.childRoot, c.parentRoot) + + return c +} + +func writeScenario(root string, s scenario, c *corpus, dirty []int) error { + if err := writeFullBuild(root, s.parentID, c.parentMem, c.parentRoot); err != nil { + return err + } + if err := writeFullBuild(root, s.siblingID, c.siblingMem, c.siblingRoot); err != nil { + return err + } + if err := writeChildBuild(root, s.buildID, s.parentID, c, dirty); err != nil { + return err + } + + return nil +} + +func writeFullBuild(root string, id uuid.UUID, mem, rootfs []byte) error { + if err := writeArtifact(root, id, id, storage.MemfileName, mem, nil); err != nil { + return err + } + return writeArtifact(root, id, id, storage.RootfsName, rootfs, nil) +} + +func writeChildBuild(root string, id, parent uuid.UUID, c *corpus, dirty []int) error { + if err := writeArtifact(root, id, parent, storage.MemfileName, c.childMem, dirty); err != nil { + return err + } + return writeArtifact(root, id, parent, storage.RootfsName, c.childRoot, dirty) +} + +func writeArtifact(root string, id, parent uuid.UUID, name string, data []byte, dirty []int) error { + dir := filepath.Join(root, "templates", id.String()) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + + var mappings []header.BuildMap + var object []byte + if dirty == nil { + object = data + mappings = []header.BuildMap{{ + Offset: 0, + Length: uint64(len(data)), + BuildId: id, + BuildStorageOffset: 0, + }} + } else { + sort.Ints(dirty) + dirtyMap := roaring.New() + for _, p := range dirty { + dirtyMap.Add(uint32(p)) + start := p * pageSize + object = append(object, data[start:start+pageSize]...) + } + parentMapping := []header.BuildMap{{ + Offset: 0, + Length: uint64(len(data)), + BuildId: parent, + BuildStorageOffset: 0, + }} + selfMapping := header.CreateMapping(&id, dirtyMap, header.PageSize) + mappings = header.NormalizeMappings(header.MergeMappings(parentMapping, selfMapping)) + } + + if err := os.WriteFile(filepath.Join(dir, name), object, 0o644); err != nil { + return err + } + h, err := header.NewHeader(&header.Metadata{ + Version: 3, + BlockSize: uint64(header.PageSize), + Size: uint64(len(data)), + Generation: 1, + BuildId: id, + BaseBuildId: parent, + }, mappings) + if err != nil { + return err + } + raw, err := header.SerializeHeader(h) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, name+storage.HeaderSuffix), raw, 0o644) +} + +func pickPages(pages, n int, rng *rand.Rand) []int { + perm := rng.Perm(pages) + return perm[:n] +} + +func fillRandom(b []byte, rng *rand.Rand) { + for i := range b { + b[i] = byte(rng.Intn(255) + 1) + } +} + +func fillPage(b []byte, page int, value byte) { + start := page * pageSize + for i := start; i < start+pageSize; i++ { + b[i] = value + } +} + +func copyPage(dst []byte, dstPage int, src []byte, srcPage int) { + copy(dst[dstPage*pageSize:(dstPage+1)*pageSize], src[srcPage*pageSize:(srcPage+1)*pageSize]) +} diff --git a/packages/orchestrator/docs/cross-artifact-dedup.md b/packages/orchestrator/docs/cross-artifact-dedup.md new file mode 100644 index 0000000000..731accc748 --- /dev/null +++ b/packages/orchestrator/docs/cross-artifact-dedup.md @@ -0,0 +1,162 @@ +# Cross-Artifact Dedup + +## Measurement + +Use `sample-dedup-gains` before changing the snapshot format. It compares sampled 4 KiB target pages against assembled candidate artifacts and emits detailed CSV plus a pool summary. + +Input is either `-build` plus `-parent-build`, or a CSV: + +```csv +build_id,parent_build_id,sibling_build_id,sibling_memfile_build_id,sibling_rootfs_build_id,family +``` + +Only `build_id` and `parent_build_id` are required. `sibling_build_id` is used for both artifacts unless the artifact-specific sibling columns are set. DB-backed sampling should export this same CSV shape from snapshot/template-build queries, keeping team and customer fields out of the file. + +Example: + +```bash +go run ./cmd/sample-dedup-gains \ + -storage gs://$TEMPLATE_BUCKET_NAME \ + -builds-file pairs.csv \ + -csv-path dedup-gains.csv \ + -max-target-pages 50000 \ + -max-candidate-pages 100000 +``` + +Set either max page flag to `0` for an exact scan. Keep the default sampled mode for broad GCS runs. + +For controlled checks, generate a local corpus first: + +```bash +go run ./cmd/synthetic-dedup-corpus \ + -out /tmp/synthetic-dedup-corpus \ + -pages 1024 \ + -dirty-pages 256 + +go run ./cmd/sample-dedup-gains \ + -storage /tmp/synthetic-dedup-corpus \ + -builds-file /tmp/synthetic-dedup-corpus/pairs.csv \ + -max-target-pages 0 \ + -max-candidate-pages 0 \ + -csv-path /tmp/synthetic-dedup-corpus/results.csv +``` + +## Pools + +Primary pools: + +- `memfile_current_rootfs` +- `rootfs_parent_memfile` +- `memfile_sibling_memfile` +- `rootfs_sibling_rootfs` + +Baseline pools: + +- `memfile_parent_memfile_positional` +- `rootfs_parent_rootfs_positional` + +Validation pools: + +- `memfile_parent_rootfs` +- `memfile_sibling_rootfs` +- `rootfs_current_memfile` +- `rootfs_sibling_memfile` + +The validation pools are for deciding what to skip. They should not ship without a separate dependency-cycle review. + +## Pool Ordering + +Treat dedup as canonicalization, not independent pool checks. Prefer sources in this order: + +1. Zero pages. +2. Parent or ancestor mappings. +3. Already-canonical sibling mappings. +4. Current-build rootfs mappings. +5. New self bytes. + +Rootfs should be canonicalized before memfile. First dedup current rootfs against parent rootfs, parent memfile, and one recent sibling rootfs. Resolve every hit through the candidate header before storing the mapping, so a match in a sibling that already points to a parent becomes a parent reference, not a sibling reference. Then build the rootfs page index from this resolved header. + +Memfile should run after rootfs. Its `current_rootfs` pool should use the resolved rootfs index, so writeback/page-cache duplicates point at the same canonical source chosen by rootfs dedup. This largely subsumes direct `memfile_parent_rootfs`: if the matching byte exists in the assembled current rootfs, the resolved rootfs mapping will already point back to the parent when appropriate. + +Sibling pools should be fallback pools, not first-choice pools. Use them only after parent/current-rootfs canonical pools miss, and resolve through sibling headers before emitting mappings. This reduces read fragmentation and keeps cache locality centered on parent artifacts instead of scattering references across sibling builds. + +Do not allow cycles. In the first shippable version, permit `memfile -> current rootfs` after rootfs finalization, but keep `rootfs -> current memfile` measurement-only. If both directions are ever needed, enforce a single artifact order for each build and reject mappings that point backward in that order. + +## Synthetic Results + +Ran an exact synthetic benchmark with 5 scenario families, 4096 pages per artifact, and 1024 dirty pages per child: + +```bash +go run ./cmd/synthetic-dedup-corpus \ + -out /tmp/synthetic-dedup-corpus \ + -pages 4096 \ + -dirty-pages 1024 \ + -seed 11 + +go run ./cmd/sample-dedup-gains \ + -storage /tmp/synthetic-dedup-corpus \ + -builds-file /tmp/synthetic-dedup-corpus/pairs.csv \ + -max-target-pages 0 \ + -max-candidate-pages 0 \ + -csv-path /tmp/synthetic-dedup-corpus/results.csv +``` + +Per-scenario hits: + +- `writeback_current_rootfs`: `memfile_current_rootfs` recovered 1024/1024 dirty memfile pages, saving 4 MiB. `rootfs_current_memfile` also recovered 1024/1024 pages, but stays validation-only because it can create same-build cycles. +- `rootfs_from_parent_memfile`: `rootfs_parent_memfile` recovered 1024/1024 dirty rootfs pages, saving 4 MiB. +- `sibling_memfile`: `memfile_sibling_memfile` recovered 1024/1024 dirty memfile pages, saving 4 MiB. +- `parent_rootfs_only`: `memfile_parent_rootfs` recovered 1024/1024 dirty memfile pages, saving 4 MiB. This confirms the theoretical edge case but should remain validation-only until real storage data shows meaningful frequency. +- `random`: no pools hit. + +Across all 5 families, each planted pool shows 20% weighted savings because it applies to exactly one family. The random family produced no false positives. This validates that the sampler separates pool-specific signal correctly; it does not predict production frequency. + +With `-frame-size 2097152`, the same run found 0/8 whole-frame hits for every planted page-level scenario. The page-level pools recovered 1024/1024 dirty pages, but whole-frame dedup recovered 0% because matching pages were sparse inside 2 MiB frames. That argues against frame-only dedup as the primary mechanism; we still need 4 KiB mappings that reference source uncompressed offsets and use the source frame table for reads. + +Ordering takeaways from the synthetic cases: + +- Canonicalizing rootfs first would make the `writeback_current_rootfs` memfile hits point at the already-resolved rootfs source instead of making rootfs an isolated current-build source. +- `rootfs_parent_memfile` should run before rootfs sibling fallback, because it turns RAM-persisted-to-disk pages into parent-backed references. +- `memfile_sibling_memfile` is real when siblings share runtime state, but should run after parent/current-rootfs pools so siblings do not become unnecessary hubs. +- `memfile_parent_rootfs` only appears in its constructed edge case; keep measuring it, but do not prioritize it over current-rootfs canonicalization. + +## Statistics + +The detailed CSV has one row per build, artifact, and pool. The command also prints a summary with weighted savings and a bootstrap 95% CI over per-row `savings_ratio`. For broad estimates, prefer bootstrapping by build or family so a large artifact does not dominate the confidence interval. + +Recommended first pass: + +- Stratify pairs by family, generation depth, artifact size, and target dirty ratio. +- Run sampled mode on at least 30 builds per stratum. +- Re-run exact mode on a smaller calibration set. +- Treat sub-1% pools as noise unless exact scans reproduce them. + +## Implementation If Worth It + +Cross-artifact references need a new header format. Today `BuildMap` only identifies `BuildId` and `BuildStorageOffset`, and the read path opens data using the artifact type of the current header. A memfile header cannot point at rootfs bytes safely. + +Preferred V5 shape: + +- Add a source artifact to each mapping, defaulting to the owner artifact when absent. +- Key build data by `(source_artifact, build_id)`, not just `build_id`. +- Make read path open the mapped artifact type. +- Make upload finalization wait for every referenced `(source_artifact, build_id)` and copy the matching frame table into the referencing header. + +For production, store per-artifact page hash index sidecars. An index entry should resolve to `{source_artifact, build_id, storage_offset}` after header resolution. The snapshot path can then hash dirty target pages, verify candidate bytes, and map hits without rescanning full artifacts. + +Start with acyclic pools only. `memfile -> current rootfs` requires rootfs data/header to be available before the memfile header publishes. `rootfs -> current memfile` should stay measurement-only at first because it can create same-build cycles. + +Compression-time dedup is the better integration point. Keep pause/export producing normal local diffs, then run canonicalization while compressing/uploading rootfs first and memfile second. Redis can advertise in-flight candidates across orchestrators as `{artifact, build_id, generation, frame_table, page_index, owner_orchestrator, ttl}`. That is only discovery state; the final header must still include enough `(source_artifact, build_id)` build data and frame metadata to read after Redis expires. + +Sibling dedup should use finalized storage first. In-flight sibling candidates are an optimization: discover through Redis, verify bytes from the peer or storage, and fall back cleanly if the peer disappears. Do not make durable headers depend on Redis-only metadata. + +Whole-frame sharing is cheaper but likely misses most page-cache/writeback duplicates unless pages align with compression frames. Prefer 4 KiB page matches that reference source uncompressed offsets; the source frame table then tells the reader which compressed frame to fetch. The sampler reports both page-level `savings_ratio` and whole-frame `frame_savings_ratio` to quantify this gap. + +Minimal implementation order: + +1. Add V5 mapping source support and read-path artifact selection. +2. Compress/upload rootfs first, dedup it, and publish its resolved page index/frame table. +3. Compress/upload memfile second, dedup against parent memfile, resolved current rootfs, then one sibling memfile. +4. Gate memfile header upload on referenced rootfs header/data availability. +5. Add Redis in-flight sibling discovery after finalized-storage sibling dedup proves useful. +6. Keep validation pools metric-only until real storage sampling justifies them. diff --git a/packages/orchestrator/pkg/sandbox/build_upload.go b/packages/orchestrator/pkg/sandbox/build_upload.go index bb6a73b4b9..b122b30c9f 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload.go +++ b/packages/orchestrator/pkg/sandbox/build_upload.go @@ -42,9 +42,14 @@ func NewUpload( useCase string, objectMetadata storage.ObjectMetadata, ) (*Upload, error) { - mem, memV4, err := resolveCompressConfig(ctx, cfg, ff, storage.MemfileName, snap.MemfileBlockSize, useCase) - if err != nil { - return nil, fmt.Errorf("resolve memfile compress config: %w", err) + var mem storage.CompressConfig + var memV4 bool + if snap.MemorySnapshot { + var err error + mem, memV4, err = resolveCompressConfig(ctx, cfg, ff, storage.MemfileName, snap.MemfileBlockSize, useCase) + if err != nil { + return nil, fmt.Errorf("resolve memfile compress config: %w", err) + } } root, rootV4, err := resolveCompressConfig(ctx, cfg, ff, storage.RootfsName, snap.RootfsBlockSize, useCase) if err != nil { diff --git a/packages/orchestrator/pkg/sandbox/build_upload_test.go b/packages/orchestrator/pkg/sandbox/build_upload_test.go index bd783e4845..41df6dc532 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_test.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_test.go @@ -3,11 +3,16 @@ package sandbox import ( + "os" "testing" + "github.com/google/uuid" "github.com/launchdarkly/go-server-sdk/v7/testhelpers/ldtestdata" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build" + sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/storage" ) @@ -55,3 +60,37 @@ func TestResolveCompressConfig_V4_FlagOn(t *testing.T) { ff := newV4HeaderFF(t, true) require.True(t, resolveV4(t, ff)) } + +func TestUploadRunV3MemorylessSkipsMemoryArtifacts(t *testing.T) { + t.Parallel() + + buildID := uuid.New() + metaPath := t.TempDir() + "/metadata.json" + require.NoError(t, os.WriteFile(metaPath, []byte("{}"), 0o644)) + + store := storage.NewMockStorageProvider(t) + metadataBlob := storage.NewMockBlob(t) + store.EXPECT(). + OpenBlob(mock.Anything, mock.MatchedBy(func(path string) bool { + return path == (storage.Paths{BuildID: buildID.String()}).Metadata() + }), storage.MetadataObjectType). + Return(metadataBlob, nil) + metadataBlob.EXPECT().Put(mock.Anything, []byte("{}"), mock.Anything).Return(nil) + + upload := &Upload{ + buildID: buildID, + snap: &Snapshot{ + BuildID: buildID, + MemorySnapshot: false, + MemfileDiff: &build.NoDiff{}, + MemfileDiffHeader: NewResolvedDiffHeader(nil), + RootfsDiff: &build.NoDiff{}, + RootfsDiffHeader: NewResolvedDiffHeader(nil), + Metafile: sbxtemplate.NewLocalFileLink(metaPath), + }, + paths: storage.Paths{BuildID: buildID.String()}, + store: store, + } + + require.NoError(t, upload.runV3(t.Context())) +} diff --git a/packages/orchestrator/pkg/sandbox/build_upload_v3.go b/packages/orchestrator/pkg/sandbox/build_upload_v3.go index a5b2b25329..b095a60bcd 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_v3.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_v3.go @@ -15,9 +15,13 @@ import ( ) func (u *Upload) runV3(ctx context.Context) error { - memfilePath, err := u.snap.MemfileDiff.CachePath(ctx) - if err != nil { - return fmt.Errorf("error getting memfile diff path: %w", err) + memfilePath := "" + if u.snap.MemorySnapshot { + var err error + memfilePath, err = u.snap.MemfileDiff.CachePath(ctx) + if err != nil { + return fmt.Errorf("error getting memfile diff path: %w", err) + } } rootfsPath, err := u.snap.RootfsDiff.CachePath(ctx) @@ -28,6 +32,9 @@ func (u *Upload) runV3(ctx context.Context) error { eg, egCtx := errgroup.WithContext(ctx) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } h, err := u.snap.MemfileDiffHeader.WaitWithContext(egCtx) if err != nil { return fmt.Errorf("wait memfile diff header: %w", err) @@ -54,6 +61,9 @@ func (u *Upload) runV3(ctx context.Context) error { meta := storage.WithMetadata(u.objectMetadata) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } if memfilePath == "" { return nil } @@ -90,6 +100,9 @@ func (u *Upload) runV3(ctx context.Context) error { }) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } return storage.UploadBlob(egCtx, u.store, u.paths.Snapfile(), storage.SnapfileObjectType, u.snap.Snapfile.Path(), meta) }) @@ -103,9 +116,13 @@ func (u *Upload) runV3(ctx context.Context) error { // Body uploads done; headers must be ready by now (the per-file Goroutines // above already Wait-ed). Wait() is a fast lookup here. - memfileDiffHeader, err := u.snap.MemfileDiffHeader.WaitWithContext(ctx) - if err != nil { - return fmt.Errorf("wait memfile diff header: %w", err) + var memfileDiffHeader *headers.Header + if u.snap.MemorySnapshot { + var err error + memfileDiffHeader, err = u.snap.MemfileDiffHeader.WaitWithContext(ctx) + if err != nil { + return fmt.Errorf("wait memfile diff header: %w", err) + } } rootfsDiffHeader, err := u.snap.RootfsDiffHeader.WaitWithContext(ctx) if err != nil { diff --git a/packages/orchestrator/pkg/sandbox/build_upload_v4.go b/packages/orchestrator/pkg/sandbox/build_upload_v4.go index baea16b40f..ef5b3c5d7d 100644 --- a/packages/orchestrator/pkg/sandbox/build_upload_v4.go +++ b/packages/orchestrator/pkg/sandbox/build_upload_v4.go @@ -16,9 +16,13 @@ import ( ) func (u *Upload) runV4(ctx context.Context) error { - memSrc, err := u.snap.MemfileDiff.CachePath(ctx) - if err != nil { - return fmt.Errorf("memfile diff path: %w", err) + memSrc := "" + if u.snap.MemorySnapshot { + var err error + memSrc, err = u.snap.MemfileDiff.CachePath(ctx) + if err != nil { + return fmt.Errorf("memfile diff path: %w", err) + } } rootfsSrc, err := u.snap.RootfsDiff.CachePath(ctx) @@ -29,6 +33,9 @@ func (u *Upload) runV4(ctx context.Context) error { eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } h, err := u.snap.MemfileDiffHeader.WaitWithContext(ctx) if err != nil { return fmt.Errorf("wait memfile diff header: %w", err) @@ -55,6 +62,9 @@ func (u *Upload) runV4(ctx context.Context) error { meta := storage.WithMetadata(u.objectMetadata) eg.Go(func() error { + if !u.snap.MemorySnapshot { + return nil + } return storage.UploadBlob(ctx, u.store, u.paths.Snapfile(), storage.SnapfileObjectType, u.snap.Snapfile.Path(), meta) }) diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index 0074176a48..5806397603 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -103,6 +103,11 @@ type ProcessOptions struct { Stderr io.Writer } +// ext4RootFlags must not include "noload": filesystem-only reboot fallback +// relies on ext4 replaying the journal after a snapshot was taken from a +// previously running guest. +const ext4RootFlags = "discard" + // TokenBucketConfig holds parameters for a single Firecracker token bucket. // BucketSize < 0 disables the bucket. type TokenBucketConfig struct { @@ -374,7 +379,7 @@ func (p *Process) Create( "random.trust_cpu": "on", // discard: ext4 issues TRIM on freed blocks so they are elided from the snapshot diff. - "rootflags": "discard", + "rootflags": ext4RootFlags, } if options.KvmClock { diff --git a/packages/orchestrator/pkg/sandbox/reclaim.go b/packages/orchestrator/pkg/sandbox/reclaim.go index 270be40e8d..f941de0973 100644 --- a/packages/orchestrator/pkg/sandbox/reclaim.go +++ b/packages/orchestrator/pkg/sandbox/reclaim.go @@ -108,6 +108,27 @@ func (s *Sandbox) bestEffortReclaim(ctx context.Context) { } } +func (s *Sandbox) bestEffortGuestSync(ctx context.Context) { + const syncTimeout = 2 * time.Second + + rcCtx, cancel := context.WithTimeout(ctx, syncTimeout) + defer cancel() + + stream, err := s.StartEnvdSystemShell(rcCtx, "/bin/sh", []string{"-c", "sync"}, "root", syncTimeout) + if err != nil { + logger.L().Warn(ctx, "envd sync failed", logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err)) + + return + } + defer stream.Close() + + for stream.Receive() { + } + if err := stream.Err(); err != nil { + logger.L().Warn(ctx, "envd sync stream error", logger.WithSandboxID(s.Runtime.SandboxID), zap.Error(err)) + } +} + // envdSupportsCgroupFreeze reports whether the sandbox's envd exposes the // native /freeze and /unfreeze endpoints. Bad version strings log and return // false so we never accidentally call an unsupported endpoint. diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index f57fa26bd4..c16770106c 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -1043,6 +1043,18 @@ func (s *Sandbox) Shutdown(ctx context.Context) error { return nil } +type pauseOptions struct { + memorySnapshot bool +} + +type PauseOption func(*pauseOptions) + +func WithMemorySnapshot(enabled bool) PauseOption { + return func(opts *pauseOptions) { + opts.memorySnapshot = enabled + } +} + // Pause creates a snapshot of the sandbox. // // Currently the memory snapshotting works like this: @@ -1058,9 +1070,14 @@ func (s *Sandbox) Pause( ctx context.Context, m metadata.Template, useCase SnapshotUseCase, + opts ...PauseOption, ) (st *Snapshot, e error) { ctx, span := tracer.Start(ctx, "sandbox-snapshot") defer span.End() + pauseOpts := pauseOptions{memorySnapshot: true} + for _, opt := range opts { + opt(&pauseOpts) + } cleanup := NewCleanup() defer func() { @@ -1089,6 +1106,10 @@ func (s *Sandbox) Pause( // compact_memory) on the live VM via envd. Per-step caps are LD-flag-driven; // all default to 0 which disables the chain entirely. Non-fatal. s.bestEffortReclaim(ctx) + if !pauseOpts.memorySnapshot { + s.bestEffortGuestSync(ctx) + m.Prefetch = nil + } // reclaim freezes user cgroups; if pause/snapshot fails the sandbox stays // live, so unfreeze on error to avoid a permanently frozen live VM. // Only runs via cleanup.Run on the error path; success leaves the frozen @@ -1126,49 +1147,54 @@ func (s *Sandbox) Pause( return nil, fmt.Errorf("error creating snapshot: %w", err) } - // Gather data for postprocessing - originalMemfile, err := s.Template.Memfile(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get original memfile: %w", err) - } - originalRootfs, err := s.Template.Rootfs() if err != nil { return nil, fmt.Errorf("failed to get original rootfs: %w", err) } - memfileDiffMetadata, err := s.Resources.memory.DiffMetadata(ctx, s.process) - if err != nil { - return nil, fmt.Errorf("failed to get memfile metadata: %w", err) - } - recordSnapshotDiff(ctx, "memfile", memfileDiffMetadata, originalMemfile.Header()) - // Start POSTPROCESSING - var dedupBase block.ReadonlyDevice - var dedupBestEffort, dedupDirectIO bool - dedupCfg := s.featureFlags.JSONFlag(ctx, featureflags.MemfileDiffDedupFlag, sandboxLDContext(s.Runtime, s.Config)).AsValueMap() - if dedupCfg.Get("enabled").BoolValue() { - dedupBase = originalMemfile - dedupBestEffort = dedupCfg.Get("bestEffort").BoolValue() - dedupDirectIO = dedupCfg.Get("directIO").BoolValue() - } - memfileDiff, memfileDiffHeader, err := pauseProcessMemory( - ctx, - buildID, - originalMemfile.Header(), - memfileDiffMetadata, - s.config.DefaultCacheDir, - s.process, - s.memory.Memfd(ctx), - s.featureFlags.BoolFlag(ctx, featureflags.MemfdBackgroundCopyFlag, sandboxLDContext(s.Runtime, s.Config)), - dedupBase, - dedupBestEffort, - dedupDirectIO, - ) - if err != nil { - return nil, fmt.Errorf("error while post processing: %w", err) + memfileDiff := build.Diff(&build.NoDiff{}) + memfileDiffHeader := NewResolvedDiffHeader(nil) + memfileBlockSize := uint64(0) + if pauseOpts.memorySnapshot { + originalMemfile, err := s.Template.Memfile(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get original memfile: %w", err) + } + memfileBlockSize = originalMemfile.Header().Metadata.BlockSize + + memfileDiffMetadata, err := s.Resources.memory.DiffMetadata(ctx, s.process) + if err != nil { + return nil, fmt.Errorf("failed to get memfile metadata: %w", err) + } + recordSnapshotDiff(ctx, "memfile", memfileDiffMetadata, originalMemfile.Header()) + + var dedupBase block.ReadonlyDevice + var dedupBestEffort, dedupDirectIO bool + dedupCfg := s.featureFlags.JSONFlag(ctx, featureflags.MemfileDiffDedupFlag, sandboxLDContext(s.Runtime, s.Config)).AsValueMap() + if dedupCfg.Get("enabled").BoolValue() { + dedupBase = originalMemfile + dedupBestEffort = dedupCfg.Get("bestEffort").BoolValue() + dedupDirectIO = dedupCfg.Get("directIO").BoolValue() + } + memfileDiff, memfileDiffHeader, err = pauseProcessMemory( + ctx, + buildID, + originalMemfile.Header(), + memfileDiffMetadata, + s.config.DefaultCacheDir, + s.process, + s.memory.Memfd(ctx), + s.featureFlags.BoolFlag(ctx, featureflags.MemfdBackgroundCopyFlag, sandboxLDContext(s.Runtime, s.Config)), + dedupBase, + dedupBestEffort, + dedupDirectIO, + ) + if err != nil { + return nil, fmt.Errorf("error while post processing: %w", err) + } + cleanup.AddNoContext(ctx, memfileDiff.Close) } - cleanup.AddNoContext(ctx, memfileDiff.Close) rootfsDiff, rootfsHeader, err := pauseProcessRootfs( ctx, @@ -1200,7 +1226,8 @@ func (s *Sandbox) Pause( MemfileDiffHeader: memfileDiffHeader, RootfsDiff: rootfsDiff, RootfsDiffHeader: NewResolvedDiffHeader(rootfsHeader), - MemfileBlockSize: originalMemfile.Header().Metadata.BlockSize, + MemorySnapshot: pauseOpts.memorySnapshot, + MemfileBlockSize: memfileBlockSize, RootfsBlockSize: originalRootfs.Header().Metadata.BlockSize, BuildID: buildID, diff --git a/packages/orchestrator/pkg/sandbox/snapshot.go b/packages/orchestrator/pkg/sandbox/snapshot.go index c597dc629c..05f9ff61df 100644 --- a/packages/orchestrator/pkg/sandbox/snapshot.go +++ b/packages/orchestrator/pkg/sandbox/snapshot.go @@ -33,6 +33,7 @@ type Snapshot struct { Snapfile template.File Metafile template.File BuildID uuid.UUID + MemorySnapshot bool // Template block sizes captured sync at Pause time. They equal // MemfileDiffHeader.Metadata.BlockSize once that header resolves, but diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index cfd4a507f8..a9fbae25b3 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -19,21 +19,27 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "google.golang.org/grpc/codes" + grpcmetadata "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/block" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/fc" sbxtemplate "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/template" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/constants" "github.com/e2b-dev/infra/packages/orchestrator/pkg/template/metadata" + "github.com/e2b-dev/infra/packages/orchestrator/pkg/units" "github.com/e2b-dev/infra/packages/shared/pkg/events" + fcmodels "github.com/e2b-dev/infra/packages/shared/pkg/fc/models" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" "github.com/e2b-dev/infra/packages/shared/pkg/logger" sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) @@ -68,6 +74,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ defer childSpan.End() isResume := req.GetSandbox().GetSnapshot() + rebootFromRootfs := rebootFromRootfsEnabled(ctx) createStart := time.Now() defer func() { if createErr != nil { @@ -77,6 +84,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ s.sandboxCreateDuration.Record(ctx, time.Since(createStart).Milliseconds(), metric.WithAttributes( attribute.Bool("sandbox.resume", isResume), + attribute.Bool("sandbox.reboot_from_rootfs", rebootFromRootfs), ), ) }() @@ -88,6 +96,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ telemetry.WithKernelVersion(req.GetSandbox().GetKernelVersion()), telemetry.WithSandboxID(req.GetSandbox().GetSandboxId()), telemetry.WithEnvdVersion(req.GetSandbox().GetEnvdVersion()), + attribute.Bool("sandbox.reboot_from_rootfs", rebootFromRootfs), ) // setup launch darkly @@ -187,15 +196,26 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ SandboxType: sandbox.SandboxTypeSandbox, } - sbx, err := s.sandboxFactory.ResumeSandbox( - ctx, - template, - config, - runtime, - req.GetStartTime().AsTime(), - req.GetEndTime().AsTime(), - req.GetSandbox(), - ) + var sbx *sandbox.Sandbox + if rebootFromRootfs { + sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req) + } else { + sbx, err = s.sandboxFactory.ResumeSandbox( + ctx, + template, + config, + runtime, + req.GetStartTime().AsTime(), + req.GetEndTime().AsTime(), + req.GetSandbox(), + ) + if errors.Is(err, storage.ErrObjectNotExist) { + telemetry.ReportEvent(ctx, "memory snapshot files missing, rebooting from rootfs") + rebootFromRootfs = true + childSpan.SetAttributes(attribute.Bool("sandbox.reboot_from_rootfs", true)) + sbx, err = s.createSandboxFromRootfs(ctx, template, config, runtime, req) + } + } if err != nil { if errors.Is(err, storage.ErrObjectNotExist) { // Snapshot data not found, let the API know the data aren't probably upload yet @@ -273,6 +293,85 @@ func createVolumeMountModelsFromAPI(volumeMounts []*orchestrator.SandboxVolumeMo return results, errors.Join(errs...) } +func memorySnapshotEnabled(ctx context.Context) bool { + values := grpcmetadata.ValueFromIncomingContext(ctx, orchestrator.SandboxMemorySnapshotGRPCMetadataKey) + if len(values) == 0 { + return true + } + + return values[0] != orchestrator.SandboxMemorySnapshotGRPCValueFalse +} + +func rebootFromRootfsEnabled(ctx context.Context) bool { + values := grpcmetadata.ValueFromIncomingContext(ctx, orchestrator.SandboxRebootFromRootfsGRPCMetadataKey) + return len(values) > 0 && values[0] == "true" +} + +func (s *Server) createSandboxFromRootfs( + ctx context.Context, + template sbxtemplate.Template, + config *sandbox.Config, + runtime sandbox.RuntimeMetadata, + req *orchestrator.SandboxCreateRequest, +) (*sandbox.Sandbox, error) { + pageSize := int64(header.PageSize) + if config.HugePages { + pageSize = int64(header.HugepageSize) + } + + buildID, err := uuid.Parse(template.Files().BuildID) + if err != nil { + return nil, fmt.Errorf("parse build id: %w", err) + } + + memfile, err := block.NewEmpty( + units.MBToBytes(config.RamMB), + pageSize, + buildID, + ) + if err != nil { + return nil, fmt.Errorf("create empty memfile: %w", err) + } + + maskedTemplate := sbxtemplate.NewMaskTemplate(template, sbxtemplate.WithMemfile(memfile)) + ioEngine := fcmodels.DriveIoEngineSync + kvmClock, err := utils.IsGTEVersion(config.Envd.Version, "0.2.11") + if err != nil { + return nil, fmt.Errorf("compare envd version: %w", err) + } + + timeout := req.GetEndTime().AsTime().Sub(req.GetStartTime().AsTime()) + if timeout <= 0 { + timeout = s.config.EnvdTimeout + } + sbx, err := s.sandboxFactory.CreateSandbox( + ctx, + config, + runtime, + maskedTemplate, + timeout, + "", + fc.ProcessOptions{ + InitScriptPath: constants.SystemdInitPath, + KvmClock: kvmClock, + IoEngine: &ioEngine, + }, + req.GetSandbox(), + nil, + ) + if err != nil { + return nil, err + } + + if err := sbx.WaitForEnvd(ctx, timeout); err != nil { + closeErr := sbx.Close(context.WithoutCancel(ctx)) + + return nil, errors.Join(fmt.Errorf("wait for envd after rootfs reboot: %w", err), closeErr) + } + + return sbx, nil +} + func (s *Server) Update(ctx context.Context, req *orchestrator.SandboxUpdateRequest) (*emptypb.Empty, error) { ctx, childSpan := tracer.Start(ctx, "sandbox-update") defer childSpan.End() @@ -556,8 +655,9 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest // Stop the old sandbox in background after we're done defer s.stopSandboxAsync(context.WithoutCancel(ctx), sbx) + memorySnapshot := memorySnapshotEnabled(ctx) // Fire and forget - upload completes in the background - res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId()) + res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId(), memorySnapshot) if err != nil { telemetry.ReportCriticalError(ctx, "error snapshotting sandbox", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -649,7 +749,8 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo sbxlogger.E(sbx).Info(ctx, "Checkpointing sandbox") - res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId()) + memorySnapshot := memorySnapshotEnabled(ctx) + res, err := s.snapshotAndCacheSandbox(ctx, sbx, in.GetBuildId(), memorySnapshot) if err != nil { telemetry.ReportCriticalError(ctx, "error snapshotting sandbox for checkpoint", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -665,26 +766,37 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo return nil, status.Errorf(codes.Internal, "error getting template for resume: %s", err) } + runtime := sandbox.RuntimeMetadata{ + TemplateID: sbx.Runtime.TemplateID, + SandboxID: sbx.Runtime.SandboxID, + ExecutionID: sbx.Runtime.ExecutionID, + TeamID: sbx.Runtime.TeamID, + BuildID: sbx.Runtime.BuildID, + SandboxType: sbx.Runtime.SandboxType, + } + // Resume the sandbox keeping the same ExecutionID (stable identity for // the API, routing catalog, and analytics) but with a fresh LifecycleID // so the old sandbox's cleanup goroutine won't // accidentally evict the resumed sandbox from the map. - resumedSbx, err := s.sandboxFactory.ResumeSandbox( - ctx, - template, - sbx.Config, - sandbox.RuntimeMetadata{ - TemplateID: sbx.Runtime.TemplateID, - SandboxID: sbx.Runtime.SandboxID, - ExecutionID: sbx.Runtime.ExecutionID, - TeamID: sbx.Runtime.TeamID, - BuildID: sbx.Runtime.BuildID, - SandboxType: sbx.Runtime.SandboxType, - }, - sbx.GetStartedAt(), - sbx.GetEndAt(), - sbx.APIStoredConfig, - ) + var resumedSbx *sandbox.Sandbox + if memorySnapshot { + resumedSbx, err = s.sandboxFactory.ResumeSandbox( + ctx, + template, + sbx.Config, + runtime, + sbx.GetStartedAt(), + sbx.GetEndAt(), + sbx.APIStoredConfig, + ) + } else { + resumedSbx, err = s.createSandboxFromRootfs(ctx, template, sbx.Config, runtime, &orchestrator.SandboxCreateRequest{ + Sandbox: sbx.APIStoredConfig, + StartTime: timestamppb.New(sbx.GetStartedAt()), + EndTime: timestamppb.New(sbx.GetEndAt()), + }) + } if err != nil { telemetry.ReportCriticalError(ctx, "error resuming sandbox after checkpoint", err, telemetry.WithSandboxID(in.GetSandboxId())) @@ -692,16 +804,20 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo } // Collect prefetch data immediately after resume while it's most accurate - prefetchData, prefetchErr := resumedSbx.MemoryPrefetchData(ctx) - if prefetchErr != nil { - sbxlogger.I(resumedSbx).Warn(ctx, "failed to get prefetch data for checkpoint", zap.Error(prefetchErr)) + var prefetchData block.PrefetchData + var prefetchErr error + if memorySnapshot { + prefetchData, prefetchErr = resumedSbx.MemoryPrefetchData(ctx) + if prefetchErr != nil { + sbxlogger.I(resumedSbx).Warn(ctx, "failed to get prefetch data for checkpoint", zap.Error(prefetchErr)) + } } // Setup lifecycle for the resumed sandbox s.setupSandboxLifecycle(ctx, resumedSbx) // Embed prefetch data into the metadata so it's uploaded with the snapshot files in a single pass. - if prefetchErr == nil { + if memorySnapshot && prefetchErr == nil { prefetchMapping := metadata.PrefetchEntriesToMapping(slices.Collect(maps.Values(prefetchData.BlockEntries)), prefetchData.BlockSize) if prefetchMapping != nil { res.meta = res.meta.WithPrefetch(&metadata.Prefetch{ @@ -790,6 +906,7 @@ func (s *Server) snapshotAndCacheSandbox( ctx context.Context, sbx *sandbox.Sandbox, buildID string, + memorySnapshot bool, ) (*snapshotResult, error) { meta, err := sbx.Template.Metadata() if err != nil { @@ -802,7 +919,7 @@ func (s *Server) snapshotAndCacheSandbox( FirecrackerVersion: sbx.Config.FirecrackerConfig.FirecrackerVersion, }) - snapshot, err := sbx.Pause(ctx, meta, sandbox.SnapshotUseCasePause) + snapshot, err := sbx.Pause(ctx, meta, sandbox.SnapshotUseCasePause, sandbox.WithMemorySnapshot(memorySnapshot)) if err != nil { return nil, fmt.Errorf("error snapshotting sandbox: %w", err) } diff --git a/packages/shared/pkg/grpc/orchestrator/internal_flags.go b/packages/shared/pkg/grpc/orchestrator/internal_flags.go new file mode 100644 index 0000000000..b2c0ddf40d --- /dev/null +++ b/packages/shared/pkg/grpc/orchestrator/internal_flags.go @@ -0,0 +1,7 @@ +package orchestrator + +const SandboxRebootFromRootfsGRPCMetadataKey = "x-e2b-reboot-from-rootfs" + +const SandboxMemorySnapshotGRPCMetadataKey = "x-e2b-memory-snapshot" + +const SandboxMemorySnapshotGRPCValueFalse = "false" diff --git a/spec/openapi.yml b/spec/openapi.yml index 3dd67764f5..7bbdebd1a1 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -724,6 +724,9 @@ components: type: boolean deprecated: true description: Automatically pauses the sandbox after the timeout + reboot: + type: boolean + description: Recreate the sandbox from the snapshot filesystem and discard memory state. ConnectSandbox: type: object @@ -735,6 +738,9 @@ components: type: integer format: int32 minimum: 0 + reboot: + type: boolean + description: Recreate the sandbox from the snapshot filesystem and discard memory state. TeamMetric: description: Team metric with timestamp @@ -2554,6 +2560,10 @@ paths: name: type: string description: Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. + memory: + type: boolean + default: true + description: Whether to persist memory state. Set false to snapshot disk only and reboot on next start. responses: "201": description: Snapshot created successfully diff --git a/tests/integration/internal/api/generated.go b/tests/integration/internal/api/generated.go index a9b495038b..7cede3a147 100644 --- a/tests/integration/internal/api/generated.go +++ b/tests/integration/internal/api/generated.go @@ -325,6 +325,9 @@ type CPUCount = int32 // ConnectSandbox defines model for ConnectSandbox. type ConnectSandbox struct { + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Timeout in seconds from the current time after which the sandbox should expire Timeout int32 `json:"timeout"` } @@ -709,6 +712,9 @@ type ResumedSandbox struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set AutoPause *bool `json:"autoPause,omitempty"` + // Reboot Recreate the sandbox from the snapshot filesystem and discard memory state. + Reboot *bool `json:"reboot,omitempty"` + // Timeout Time to live for the sandbox in seconds. Timeout *int32 `json:"timeout,omitempty"` } @@ -1505,6 +1511,9 @@ type PostSandboxesSandboxIDRefreshesJSONBody struct { // PostSandboxesSandboxIDSnapshotsJSONBody defines parameters for PostSandboxesSandboxIDSnapshots. type PostSandboxesSandboxIDSnapshotsJSONBody struct { + // Memory Whether to persist memory state. Set false to snapshot disk only and reboot on next start. + Memory *bool `json:"memory,omitempty"` + // Name Optional name for the snapshot template. If a snapshot template with this name already exists, a new build will be assigned to the existing template instead of creating a new one. Name *string `json:"name,omitempty"` }