Optimize sparse delta apply path

crates/surge-core/src/releases/delta/mod.rs

@@ -21,6 +21,7 @@ pub use self::format::{
     has_archive_bsdiff_magic_prefix, has_archive_chunked_magic_prefix, has_sparse_file_ops_magic_prefix,
     is_supported_delta, patch_format_from_magic_prefix,
 };
+pub(crate) use self::sparse_ops::apply_sparse_file_patch_to_directory;
 pub use self::sparse_ops::build_sparse_file_patch;
 
 pub fn decode_delta_patch(data: &[u8], delta: &DeltaArtifact) -> Result<Vec<u8>> {
@@ -75,3 +76,14 @@ pub fn apply_delta_patch(older: &[u8], patch: &[u8], delta: &DeltaArtifact) -> R
         delta.algorithm, delta.patch_format
     )))
 }
+
+#[must_use]
+pub fn is_sparse_file_ops_delta(delta: &DeltaArtifact) -> bool {
+    let patch_format = normalized_or_default(&delta.patch_format, PATCH_FORMAT_BSDIFF4);
+    if !patch_format.eq_ignore_ascii_case(PATCH_FORMAT_SPARSE_FILE_OPS_V1) {
+        return false;
+    }
+
+    let algorithm = delta.algorithm.trim();
+    algorithm.is_empty() || algorithm.eq_ignore_ascii_case(DIFF_ALGORITHM_FILE_OPS)
+}

crates/surge-core/src/releases/delta/sparse_ops.rs

@@ -1,4 +1,5 @@
 use std::fs;
+use std::path::Path;
 
 use serde::{Deserialize, Serialize};
 
@@ -191,6 +192,11 @@ pub(super) fn apply_sparse_file_patch(older: &[u8], patch: &[u8]) -> Result<Vec<
     packer.finalize()
 }
 
+pub(crate) fn apply_sparse_file_patch_to_directory(root: &Path, patch: &[u8]) -> Result<(i32, u32)> {
+    let (manifest, payloads) = decode_sparse_file_ops_payload(patch)?;
+    apply_sparse_file_ops(root, &manifest.ops, payloads).map(|()| (manifest.compression_level, manifest.zstd_workers))
+}
+
 fn encode_sparse_file_ops_payload(manifest: &SparseFileDeltaManifest, payloads: &[u8]) -> Result<Vec<u8>> {
     let manifest_bytes = serde_json::to_vec(manifest)?;
     let manifest_len = u64::try_from(manifest_bytes.len())

crates/surge-core/src/releases/delta/tests.rs

@@ -198,3 +198,87 @@ fn test_sparse_file_patch_roundtrip_rebuilds_full_archive_bytes() {
     let rebuilt = apply_delta_patch(&full_v1, &decoded, &delta).unwrap();
     assert_eq!(rebuilt, full_v2);
 }
+
+#[test]
+fn test_sparse_file_patch_can_apply_directly_to_directory() {
+    let dir = tempfile::tempdir().unwrap();
+    let old_dir = dir.path().join("old");
+    let new_dir = dir.path().join("new");
+    std::fs::create_dir_all(old_dir.join("bin")).unwrap();
+    std::fs::create_dir_all(new_dir.join("bin")).unwrap();
+    std::fs::create_dir_all(new_dir.join("models")).unwrap();
+    std::fs::write(old_dir.join("bin").join("runtime.bin"), vec![b'A'; 256 * 1024]).unwrap();
+    std::fs::write(old_dir.join("config.json"), br#"{"version":1}"#).unwrap();
+    std::fs::write(new_dir.join("bin").join("runtime.bin"), {
+        let mut bytes = vec![b'A'; 256 * 1024];
+        bytes[2048] = b'B';
+        bytes
+    })
+    .unwrap();
+    std::fs::write(new_dir.join("config.json"), br#"{"version":2}"#).unwrap();
+    std::fs::write(new_dir.join("models").join("model-v2.bin"), vec![b'Z'; 128 * 1024]).unwrap();
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+
+        std::fs::set_permissions(old_dir.join("bin"), std::fs::Permissions::from_mode(0o700)).unwrap();
+        std::fs::set_permissions(new_dir.join("bin"), std::fs::Permissions::from_mode(0o700)).unwrap();
+    }
+
+    let mut old_packer = ArchivePacker::new(7).unwrap();
+    old_packer.add_directory(&old_dir, "").unwrap();
+    let full_v1 = old_packer.finalize().unwrap();
+
+    let mut new_packer = ArchivePacker::new(7).unwrap();
+    new_packer.add_directory(&new_dir, "").unwrap();
+    let full_v2 = new_packer.finalize().unwrap();
+
+    let patch = build_sparse_file_patch(
+        &full_v1,
+        &full_v2,
+        7,
+        0,
+        &ChunkedDiffOptions {
+            chunk_size: 64 * 1024,
+            max_threads: 1,
+        },
+    )
+    .unwrap();
+    let delta_bytes = zstd::encode_all(patch.as_slice(), 3).unwrap();
+    let delta = DeltaArtifact::sparse_file_ops_zstd(
+        "primary",
+        "1.0.0",
+        "demo-1.1.0-delta.tar.zst",
+        i64::try_from(delta_bytes.len()).unwrap(),
+        &sha256_hex(&delta_bytes),
+    );
+    assert!(is_sparse_file_ops_delta(&delta));
+
+    let working_dir = tempfile::tempdir().unwrap();
+    crate::archive::extractor::extract_to(&full_v1, working_dir.path(), None).unwrap();
+
+    let decoded = decode_delta_patch(&delta_bytes, &delta).unwrap();
+    let archive_settings = apply_sparse_file_patch_to_directory(working_dir.path(), &decoded).unwrap();
+    assert_eq!(archive_settings, (7, 0));
+
+    let mut rebuilt_packer = ArchivePacker::new(7).unwrap();
+    rebuilt_packer.add_directory(working_dir.path(), "").unwrap();
+    let rebuilt = rebuilt_packer.finalize().unwrap();
+
+    assert_eq!(rebuilt, full_v2);
+    assert_eq!(
+        std::fs::read_to_string(working_dir.path().join("config.json")).unwrap(),
+        r#"{"version":2}"#
+    );
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+
+        let mode = std::fs::metadata(working_dir.path().join("bin"))
+            .unwrap()
+            .permissions()
+            .mode()
+            & 0o777;
+        assert_eq!(mode, 0o700);
+    }
+}

crates/surge-core/src/update/manager.rs

@@ -1710,7 +1710,7 @@ mod tests {
                     os,
                     rid: rid.clone(),
                     is_genesis: false,
-                    full_filename: full_v3_name,
+                    full_filename: full_v3_name.clone(),
                     full_size: full_v3.len() as i64,
                     full_sha256: sha256_hex(&full_v3),
                     deltas: vec![DeltaArtifact::bsdiff_zstd(
@@ -1830,7 +1830,7 @@ mod tests {
                     os,
                     rid: rid.clone(),
                     is_genesis: false,
-                    full_filename: full_v3_name,
+                    full_filename: full_v3_name.clone(),
                     full_size: full_v3.len() as i64,
                     full_sha256: sha256_hex(&full_v3),
                     deltas: vec![DeltaArtifact::sparse_file_ops_zstd(
@@ -1927,6 +1927,161 @@ mod tests {
 
         let cached_current_full = install_root.join(".surge-cache").join("artifacts").join(&full_v2_name);
         assert!(!cached_current_full.exists());
+        let cached_latest_full = install_root.join(".surge-cache").join("artifacts").join(&full_v3_name);
+        assert!(cached_latest_full.exists());
+    }
+
+    #[tokio::test]
+    async fn test_download_and_apply_sparse_delta_uses_pristine_base_when_persistent_assets_diverge() {
+        let tmp = tempfile::tempdir().unwrap();
+        let store_root = tmp.path().join("store");
+        let install_root = tmp.path().join("install");
+        let app_id = "test-app";
+        std::fs::create_dir_all(&store_root).unwrap();
+        std::fs::create_dir_all(&install_root).unwrap();
+        let app_store = app_scoped_store_root(&store_root, app_id);
+
+        let rid = current_rid();
+        let os = current_os_label_for_tests();
+
+        let source_v2 = tmp.path().join("source-v2");
+        let source_v3 = tmp.path().join("source-v3");
+        std::fs::create_dir_all(&source_v2).unwrap();
+        std::fs::create_dir_all(&source_v3).unwrap();
+        std::fs::write(source_v2.join("payload.txt"), "v2 payload").unwrap();
+        std::fs::write(source_v2.join("settings.json"), r#"{"theme":"light"}"#).unwrap();
+        std::fs::write(source_v3.join("payload.txt"), "v3 payload").unwrap();
+        std::fs::write(source_v3.join("settings.json"), r#"{"theme":"light"}"#).unwrap();
+
+        let mut packer_v2 = ArchivePacker::new(3).unwrap();
+        packer_v2.add_directory(&source_v2, "").unwrap();
+        let full_v2 = packer_v2.finalize().unwrap();
+
+        let mut packer_v3 = ArchivePacker::new(3).unwrap();
+        packer_v3.add_directory(&source_v3, "").unwrap();
+        let full_v3 = packer_v3.finalize().unwrap();
+
+        let patch_v3 = build_sparse_file_patch(&full_v2, &full_v3, 3, 0, &ChunkedDiffOptions::default()).unwrap();
+        let delta_v3 = zstd::encode_all(patch_v3.as_slice(), 3).unwrap();
+
+        let full_v2_name = format!("{app_id}-1.1.0-{rid}-full.tar.zst");
+        let full_v3_name = format!("{app_id}-1.2.0-{rid}-full.tar.zst");
+        let delta_v3_name = format!("{app_id}-1.2.0-{rid}-delta.tar.zst");
+
+        std::fs::write(app_store.join(&full_v2_name), &full_v2).unwrap();
+        std::fs::write(app_store.join(&delta_v3_name), &delta_v3).unwrap();
+
+        let persistent_assets = vec!["settings.json".to_string()];
+        let index = ReleaseIndex {
+            app_id: app_id.to_string(),
+            releases: vec![
+                ReleaseEntry {
+                    version: "1.1.0".to_string(),
+                    channels: vec!["stable".to_string()],
+                    os: os.clone(),
+                    rid: rid.clone(),
+                    is_genesis: true,
+                    full_filename: full_v2_name.clone(),
+                    full_size: full_v2.len() as i64,
+                    full_sha256: sha256_hex(&full_v2),
+                    deltas: Vec::new(),
+                    preferred_delta_id: String::new(),
+                    created_utc: chrono::Utc::now().to_rfc3339(),
+                    release_notes: String::new(),
+                    name: String::new(),
+                    main_exe: app_id.to_string(),
+                    install_directory: app_id.to_string(),
+                    supervisor_id: String::new(),
+                    icon: String::new(),
+                    shortcuts: Vec::new(),
+                    persistent_assets: persistent_assets.clone(),
+                    installers: Vec::new(),
+                    environment: std::collections::BTreeMap::new(),
+                },
+                ReleaseEntry {
+                    version: "1.2.0".to_string(),
+                    channels: vec!["stable".to_string()],
+                    os,
+                    rid: rid.clone(),
+                    is_genesis: false,
+                    full_filename: full_v3_name.clone(),
+                    full_size: full_v3.len() as i64,
+                    full_sha256: sha256_hex(&full_v3),
+                    deltas: vec![DeltaArtifact::sparse_file_ops_zstd(
+                        "primary",
+                        "1.1.0",
+                        &delta_v3_name,
+                        delta_v3.len() as i64,
+                        &sha256_hex(&delta_v3),
+                    )],
+                    preferred_delta_id: "primary".to_string(),
+                    created_utc: chrono::Utc::now().to_rfc3339(),
+                    release_notes: String::new(),
+                    name: String::new(),
+                    main_exe: app_id.to_string(),
+                    install_directory: app_id.to_string(),
+                    supervisor_id: String::new(),
+                    icon: String::new(),
+                    shortcuts: Vec::new(),
+                    persistent_assets,
+                    installers: Vec::new(),
+                    environment: std::collections::BTreeMap::new(),
+                },
+            ],
+            ..ReleaseIndex::default()
+        };
+
+        write_app_scoped_release_index(&store_root, app_id, &index);
+
+        let active_app_dir = install_root.join("app");
+        std::fs::create_dir_all(active_app_dir.join(".surge")).unwrap();
+        std::fs::write(active_app_dir.join("payload.txt"), "v2 payload").unwrap();
+        std::fs::write(active_app_dir.join("settings.json"), r#"{"theme":"dark"}"#).unwrap();
+        std::fs::write(
+            active_app_dir.join(crate::install::RUNTIME_MANIFEST_RELATIVE_PATH),
+            format!("id: {app_id}\nversion: 1.1.0\n"),
+        )
+        .unwrap();
+        std::fs::write(
+            active_app_dir.join(crate::install::LEGACY_RUNTIME_MANIFEST_RELATIVE_PATH),
+            format!("id: {app_id}\nversion: 1.1.0\n"),
+        )
+        .unwrap();
+
+        let ctx = Arc::new(Context::new());
+        ctx.set_storage(
+            StorageProvider::Filesystem,
+            store_root.to_str().unwrap(),
+            "",
+            "",
+            "",
+            "",
+        );
+
+        let mut manager = UpdateManager::new(ctx, app_id, "1.1.0", "stable", install_root.to_str().unwrap()).unwrap();
+        let info = manager.check_for_updates().await.unwrap().unwrap();
+        assert_eq!(info.apply_strategy, ApplyStrategy::Delta);
+        manager
+            .download_and_apply(&info, None::<fn(ProgressInfo)>)
+            .await
+            .unwrap();
+
+        let installed_payload = std::fs::read_to_string(install_root.join("app").join("payload.txt")).unwrap();
+        assert_eq!(installed_payload, "v3 payload");
+        let installed_settings = std::fs::read_to_string(install_root.join("app").join("settings.json")).unwrap();
+        assert_eq!(installed_settings, r#"{"theme":"dark"}"#);
+
+        let cached_latest_full = install_root.join(".surge-cache").join("artifacts").join(&full_v3_name);
+        assert!(cached_latest_full.exists());
+        let extracted_cached = tempfile::tempdir().unwrap();
+        crate::archive::extractor::extract_to(
+            &std::fs::read(&cached_latest_full).unwrap(),
+            extracted_cached.path(),
+            None,
+        )
+        .unwrap();
+        let cached_settings = std::fs::read_to_string(extracted_cached.path().join("settings.json")).unwrap();
+        assert_eq!(cached_settings, r#"{"theme":"light"}"#);
     }
 
     #[tokio::test]