diff --git a/.claude/skills/uvm-info/SKILL.md b/.claude/skills/uvm-info/SKILL.md new file mode 100644 index 00000000..3b8c8f0f --- /dev/null +++ b/.claude/skills/uvm-info/SKILL.md @@ -0,0 +1,57 @@ +--- +name: uvm-info +description: Query Unity version information using uvm - list installed versions, available modules, detect project versions. +license: MIT +metadata: + author: larusso + version: "1.0" +--- + +Use this skill to query Unity version information without making changes. + +## Available Commands + +Run these via the development binary or installed `uvm`: + +```bash +# List installed Unity versions +cargo run --bin uvm -- list + +# Show available modules for a specific version +cargo run --bin uvm -- modules + +# Detect Unity version from current project +cargo run --bin uvm -- detect + +# Show all available Unity versions from Unity's API +cargo run --bin uvm -- versions +``` + +## When to Use + +- **Exploring what's installed**: Use `list` to see current Unity installations +- **Planning an installation**: Use `modules ` to see what modules are available +- **Understanding a project**: Use `detect` to find which Unity version a project needs +- **Finding versions**: Use `versions` to see what's available to install + +## Examples + +```bash +# What Unity versions are installed? +cargo run --bin uvm -- list + +# What modules can I install for Unity 2022.3.0f1? +cargo run --bin uvm -- modules 2022.3.0f1 + +# What version does this project need? +cargo run --bin uvm -- detect + +# What LTS versions are available? +cargo run --bin uvm -- versions --lts +``` + +## Notes + +- These commands are read-only and safe to run +- Use the development binary (`cargo run --bin uvm --`) when working on the codebase +- Output helps understand the current state before making changes diff --git a/.claude/skills/uvm-test-install/SKILL.md b/.claude/skills/uvm-test-install/SKILL.md new file mode 100644 index 00000000..b4bc9d31 --- /dev/null +++ b/.claude/skills/uvm-test-install/SKILL.md @@ -0,0 +1,89 @@ +--- +name: uvm-test-install +description: Install Unity to a temporary directory for testing. Use this to see real installation output and verify installer behavior. +license: MIT +metadata: + author: larusso + version: "1.0" +--- + +Use this skill to perform a real Unity installation to a temporary directory. This lets you observe actual installer output, verify fixes, and debug installation issues. + +## Basic Usage + +```bash +# Create a temp directory and install +TEMP_UNITY=$(mktemp -d) +cargo run --bin uvm -- install "$TEMP_UNITY" + +# Example: Install Unity 2022.3.0f1 to temp +TEMP_UNITY=$(mktemp -d) +cargo run --bin uvm -- install 2022.3.0f1 "$TEMP_UNITY" +``` + +## With Modules + +```bash +# Install with specific modules +TEMP_UNITY=$(mktemp -d) +cargo run --bin uvm -- install -m "$TEMP_UNITY" + +# Example: Install with Android support +TEMP_UNITY=$(mktemp -d) +cargo run --bin uvm -- install 2022.3.0f1 -m android "$TEMP_UNITY" +``` + +## With Verbose Logging + +```bash +# Enable debug logging to see detailed installation steps +RUST_LOG=debug cargo run --bin uvm -- install "$TEMP_UNITY" + +# Or trace level for maximum detail +RUST_LOG=trace cargo run --bin uvm -- install "$TEMP_UNITY" +``` + +## Cleanup + +After testing, clean up the temporary installation: + +```bash +rm -rf "$TEMP_UNITY" +``` + +## When to Use + +- **Verifying a bug fix**: See if changes to installer code work correctly +- **Understanding error messages**: Observe real failure modes +- **Testing module installation**: Verify module dependencies resolve correctly +- **Debugging platform-specific issues**: See actual extraction/installation output + +## Cautions + +- **Downloads are large**: Unity editor is several GB; modules add more +- **Takes time**: Full installation can take 10+ minutes depending on network +- **Disk space**: Ensure temp directory has enough space (5-10 GB minimum) +- **Consider using small modules**: Language packs (.po files) are tiny and fast for quick tests + +## Quick Test with Minimal Download + +For a fast test of the installation flow, install just a language pack: + +```bash +# First install editor (required base) +TEMP_UNITY=$(mktemp -d) +cargo run --bin uvm -- install 2022.3.0f1 "$TEMP_UNITY" + +# Then add a small language module +cargo run --bin uvm -- install 2022.3.0f1 -m language-ja "$TEMP_UNITY" +``` + +## Architecture Selection (macOS) + +```bash +# Force x86_64 architecture +cargo run --bin uvm -- install --architecture x86-64 "$TEMP_UNITY" + +# Force ARM64 architecture +cargo run --bin uvm -- install --architecture arm64 "$TEMP_UNITY" +``` diff --git a/CLAUDE.md b/CLAUDE.md index f662ed6c..450a5956 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,6 +52,10 @@ Lower-level crates don't depend on higher-level ones. - **Installation**: `uvm_install/src/` - core install/uninstall logic - **Hub integration**: `unity-hub/src/` - Unity Hub paths and installations +### Deep Dives + +- [Installer Architecture](docs/installer-architecture.md) - execution flow, platform-specific logic, external dependencies + ## Code Conventions - **Rust Editions**: mixed (2018, 2021, 2024); check each crate's `Cargo.toml` for its edition (e.g., `unity-version` is 2021; `uvm_detect`/`uvm_gc` are 2024) @@ -64,11 +68,18 @@ Lower-level crates don't depend on higher-level ones. Use imperative mood: "Fix bug" not "Fixed bug" or "Fixes bug" +Do NOT add Co-Authored-By lines. + ``` Short summary (50 chars or less) -Detailed explanation if needed, wrapped at 72 chars. -Use markdown formatting. Bullet points are okay. +## Description + +Detailed explanation of the change. + +## Changes + +- Bullet list of specific changes made ``` ## Pull Requests diff --git a/Cargo.lock b/Cargo.lock index 64d07d85..c175edc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2875,6 +2875,7 @@ dependencies = [ "console", "dirs-2", "dmg", + "flate2", "lazy_static", "log", "mach_object", diff --git a/docs/installer-architecture.md b/docs/installer-architecture.md new file mode 100644 index 00000000..2499fa02 --- /dev/null +++ b/docs/installer-architecture.md @@ -0,0 +1,648 @@ +# Unity Version Manager - Installer Logic Research + +## Executive Summary + +The UVM installer is a sophisticated cross-platform system for installing Unity Editor versions and their modules. It fetches release metadata from Unity's GraphQL API, constructs a dependency graph, downloads installer packages, and extracts them using platform-specific strategies. + +--- + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ CLI Layer (uvm) │ +│ uvm/src/commands/install.rs │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ Installation Orchestration │ +│ uvm_install/src/lib.rs │ +│ ┌──────────────┐ ┌────────────────┐ ┌──────────────────────┐ │ +│ │InstallOptions│ │InstallGraph │ │install_module_and_ │ │ +│ │ (Builder) │──│ (DAG Walker) │──│ dependencies() │ │ +│ └──────────────┘ └────────────────┘ └──────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌──────────────────────────────┐ +│ Live Platform │ │ Install Graph │ │ Platform Installers │ +│ (API Client) │ │ (Dependency Res)│ │ uvm_install/src/sys/ │ +│ │ │ │ │ ┌───────┬────────┬───────┐ │ +│ ┌─────────────┐ │ │ ┌─────────────┐ │ │ │macOS │ Linux │Windows│ │ +│ │FetchRelease │ │ │ │ DAG with │ │ │ ├───────┼────────┼───────┤ │ +│ │(GraphQL API)│ │ │ │ Install │ │ │ │.pkg │.xz │.exe │ │ +│ └─────────────┘ │ │ │ Status │ │ │ │.dmg │.zip │.msi │ │ +│ │ │ └─────────────┘ │ │ │.zip │.pkg │.zip │ │ +└─────────────────┘ └─────────────────┘ │ │.po │.po │.po │ │ + │ └───────┴────────┴───────┘ │ + └──────────────────────────────┘ +``` + +--- + +## Core Components + +### 1. InstallOptions (Entry Point) + +**Location**: `uvm_install/src/lib.rs:127-376` + +The `InstallOptions` struct serves as a builder for installation configuration: + +```rust +pub struct InstallOptions { + version: Version, + requested_modules: HashSet, + install_sync: bool, // Include synced/optional dependencies + destination: Option, // Custom install location + architecture: Option, // x86_64 or arm64 +} +``` + +**Execution Flow** (`install()` method): + +1. **Acquire Lock**: Uses file-based locking (`locks_dir().join(".lock")`) to prevent concurrent installations of the same version +2. **Fetch Release**: Calls Unity's GraphQL API to get release metadata +3. **Build Dependency Graph**: Creates `InstallGraph` from the release +4. **Detect Existing Installation**: Checks if Unity is already installed at destination +5. **Architecture Check** (macOS ARM64 only): Verifies binary architecture matches system +6. **Resolve Dependencies**: Collects all required modules including transitive dependencies +7. **Filter Graph**: Removes unrequested modules from the graph +8. **Install Components**: Walks the graph in topological order, installing each component +9. **Update Module Manifest**: Writes installed modules to Unity's module list + +### 2. Release Fetching (Live Platform API) + +**Location**: `uvm_live_platform/src/api/fetch_release.rs` + +Uses Unity's GraphQL API at `https://live-platform-api.prd.ld.unity3d.com/graphql`: + +``` +┌────────────────────────────────────────────────────────────────┐ +│ FetchReleaseBuilder │ +├────────────────────────────────────────────────────────────────┤ +│ .with_current_platform() → MAC_OS / LINUX / WINDOWS │ +│ .with_architecture() → X86_64 / ARM64 │ +│ .with_extended_lts() → Include XLTS versions │ +│ .with_u7_alpha() → Include Unity 7 alpha │ +│ .fetch() → Execute GraphQL query │ +└────────────────────────────────────────────────────────────────┘ +``` + +**Response Structure** (`Release`): +- `version`: Version string (e.g., "2022.3.0f1") +- `short_revision`: Git revision hash +- `downloads[]`: Platform-specific download info + - `release_file`: URL and integrity hash + - `modules[]`: Available add-on modules (recursive structure) + +**Caching**: 7-day cache by default, controlled via `UVM_LIVE_PLATFORM_FETCH_RELEASE_CACHE_*` env vars. + +### 3. Dependency Resolution (Install Graph) + +**Location**: `uvm_install_graph/src/lib.rs` + +Uses `daggy` (DAG library built on `petgraph`) to model the module dependency tree: + +``` + ┌──────────┐ + │ Editor │ + └────┬─────┘ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + ┌────────┐ ┌──────────┐ ┌────────┐ + │Android │ │ iOS │ │ WebGL │ + └────┬───┘ └──────────┘ └────────┘ + │ + ┌──────┴──────┐ + ▼ ▼ +┌───────┐ ┌──────────┐ +│SDK/NDK│ │OpenJDK │ +└───────┘ └──────────┘ +``` + +**Key Operations**: +- `from(release)`: Builds DAG from release metadata +- `mark_installed(components)`: Marks already-installed components +- `keep(requested_modules)`: Filters graph to only requested components +- `topo()`: Returns topological traversal for ordered installation +- `get_dependend_modules()`: Walks parent chain (dependencies) +- `get_sub_modules()`: Walks child chain (optional sub-dependencies) + +**Install Status**: +- `Unknown`: Initial state +- `Missing`: Needs to be installed +- `Installed`: Already present + +### 4. Download/Load System + +**Location**: `uvm_install/src/install/loader.rs` + +The `Loader` handles downloading installer packages: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Download Flow │ +├─────────────────────────────────────────────────────────────┤ +│ 1. Calculate cache paths: │ +│ - installer_dir: ~/.cache/uvm/installer/-/ │ +│ - temp_dir: ~/.cache/uvm/tmp/-/ │ +│ │ +│ 2. Check for cached installer: │ +│ - If exists & checksum matches → return cached path │ +│ - If checksum mismatch → delete and re-download │ +│ │ +│ 3. Download with resume support: │ +│ - Uses HTTP Range headers for partial downloads │ +│ - Saves as .part file, renames on completion │ +│ │ +│ 4. Verify integrity (ssri/SRI hash) │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Features**: +- **Resume Support**: Checks existing `.part` file size, requests with `Range: bytes=N-` +- **Integrity Verification**: Uses SRI (Subresource Integrity) hashes +- **Process Locking**: Per-file locks prevent concurrent downloads of same package + +--- + +## Platform-Specific Installers + +### InstallHandler Trait + +**Location**: `uvm_install/src/install/mod.rs:15-45` + +```rust +pub trait InstallHandler { + fn install_handler(&self) -> Result<(), InstallerError>; // Core logic + fn installer(&self) -> &Path; // Source file + fn error_handler(&self) {} // Cleanup on error + fn before_install(&self) -> Result<()> { Ok(()) } // Pre-install hook + fn after_install(&self) -> Result<()> { Ok(()) } // Post-install hook + + fn install(&self) -> Result<()> { + self.before_install()?; + self.install_handler().map_err(|e| { self.error_handler(); e })?; + self.after_install()?; + Ok(()) + } +} +``` + +--- + +### macOS Installers + +**Location**: `uvm_install/src/sys/mac/` + +#### Supported Formats + +| Format | Editor | Module | Handler | +|--------|--------|--------|---------| +| `.pkg` | ✓ | ✓ | `EditorPkgInstaller`, `ModulePkgInstaller`, `ModulePkgNativeInstaller` | +| `.dmg` | ✗ | ✓ | `ModuleDmgInstaller`, `ModuleDmgWithDestinationInstaller` | +| `.zip` | ✗ | ✓ | `ModuleZipInstaller` | +| `.po` | ✗ | ✓ | `ModulePoInstaller` (language files) | + +#### PKG Installation Flow (mac/pkg.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ PKG Installation (Editor) │ +├────────────────────────────────────────────────────────────────────┤ +│ 1. before_install(): │ +│ - Remove existing destination directory │ +│ │ +│ 2. install_handler(): │ +│ ┌────────────────┐ │ +│ │ xar -x -f pkg │ Extract PKG using macOS xar │ +│ │ -C tmp_dest │ │ +│ └───────┬────────┘ │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ find_payload() │ Locate *.pkg.tmp/Payload or Payload~ │ +│ └───────┬────────┘ │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ tar -zmxf │ Extract gzipped tar payload │ +│ └───────┬────────┘ │ +│ ▼ │ +│ ┌────────────────┐ │ +│ │ cleanup_editor │ Move Unity/ contents up, remove temp │ +│ └────────────────┘ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +**Module PKG Variants**: +- `ModulePkgInstaller`: Extracts to specific destination using xar/tar +- `ModulePkgNativeInstaller`: Uses `sudo installer -package -target /` for system-level packages + +#### DMG Installation Flow (mac/dmg.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ DMG Installation │ +├────────────────────────────────────────────────────────────────────┤ +│ 1. Mount DMG using `dmg` crate (Attach::new().with()) │ +│ │ +│ 2. Find .app bundle in mounted volume │ +│ │ +│ 3. Copy .app to destination using `cp -a` │ +│ │ +│ 4. DMG auto-unmounts when Attach drops │ +└────────────────────────────────────────────────────────────────────┘ +``` + +#### Architecture Check (mac/arch.rs) + +**Only on macOS ARM64 systems**, when `UVM_ARCHITECTURE_CHECK_ENABLED=true`: + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Architecture Verification (ARM64 Mac) │ +├────────────────────────────────────────────────────────────────────┤ +│ For Unity >= 2021.2.0f1: │ +│ │ +│ 1. Read Mach-O binary from Unity.app/Contents/MacOS/Unity │ +│ │ +│ 2. Parse architectures (handles Fat binaries with multiple archs) │ +│ │ +│ 3. Compare against system arch (sysctl hw.machine) │ +│ │ +│ 4. If mismatch: reinstall with correct architecture │ +│ - Preserves installed modules list │ +│ - Deletes existing installation & installer cache │ +│ - Downloads ARM64 version │ +└────────────────────────────────────────────────────────────────────┘ +``` + +--- + +### Linux Installers + +**Location**: `uvm_install/src/sys/linux/` + +#### Supported Formats + +| Format | Editor | Module | Handler | +|--------|--------|--------|---------| +| `.xz` | ✓ | ✓ | `EditorXzInstaller`, `ModuleXzInstaller` | +| `.zip` | ✓ | ✓ | `EditorZipInstaller`, `ModuleZipInstaller` | +| `.pkg` | ✗ | ✓ | `ModulePkgInstaller` | +| `.po` | ✗ | ✓ | `ModulePoInstaller` | + +#### XZ Installation Flow (linux/xz.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ XZ/Tar Installation │ +├────────────────────────────────────────────────────────────────────┤ +│ tar -C -amxf │ +│ │ +│ Flags: │ +│ -a : Auto-detect compression (xz in this case) │ +│ -m : Don't extract modification time │ +│ -x : Extract │ +│ -f : File to extract │ +├────────────────────────────────────────────────────────────────────┤ +│ QUIRK: PlaybackEngines destination adjustment │ +│ │ +│ If destination ends with "Editor/Data/PlaybackEngines": │ +│ → Navigate 3 levels up to find correct root │ +│ → Tar extracts with full internal path structure │ +│ │ +│ If destination ends with "Editor/Data/PlaybackEngines/iOSSupport":│ +│ → Strip iOSSupport suffix first, then apply above logic │ +└────────────────────────────────────────────────────────────────────┘ +``` + +#### PKG on Linux (linux/pkg.rs) + +Linux can install macOS-style `.pkg` files using 7z + gzip + cpio: + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ PKG on Linux (via 7z) │ +├────────────────────────────────────────────────────────────────────┤ +│ 1. Extract PKG: 7z x -y -o │ +│ │ +│ 2. Find payload: Payload or Payload~ file │ +│ │ +│ 3. Extract payload: │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ If Payload~: │ │ +│ │ cat Payload~ | cpio -iu │ │ +│ │ │ │ +│ │ If Payload (gzipped): │ │ +│ │ gzip -dc Payload | cpio -iu │ │ +│ └─────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +**External Dependency**: Requires `7z` command (p7zip package). + +--- + +### Windows Installers + +**Location**: `uvm_install/src/sys/win/` + +#### Supported Formats + +| Format | Editor | Module | Handler | +|--------|--------|--------|---------| +| `.exe` | ✓ | ✓ | `EditorExeInstaller`, `ModuleExeInstaller`, `ModuleExeTargetInstaller` | +| `.msi` | ✗ | ✓ | `ModuleMsiInstaller` | +| `.zip` | ✗ | ✓ | `ModuleZipInstaller` | +| `.po` | ✗ | ✓ | `ModulePoInstaller` | + +#### EXE Installation Flow (win/exe.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ EXE Installation (Windows) │ +├────────────────────────────────────────────────────────────────────┤ +│ Creates temporary .cmd script: │ +│ │ +│ ┌─────────────────────────────────────────────────────────────┐ │ +│ │ ECHO OFF │ │ +│ │ CALL "" /S /D= │ │ +│ │ │ │ +│ │ Or with custom UI flag for editor: │ │ +│ │ CALL "" -UI=reduced /D= │ │ +│ └─────────────────────────────────────────────────────────────┘ │ +│ │ +│ Executes script and waits for completion │ +├────────────────────────────────────────────────────────────────────┤ +│ QUIRK: Editor gets "-UI=reduced" for minimal UI during install │ +│ QUIRK: Modules without destination use just /S (silent) │ +└────────────────────────────────────────────────────────────────────┘ +``` + +#### MSI Installation Flow (win/msi.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ MSI Installation │ +├────────────────────────────────────────────────────────────────────┤ +│ Uses command string from module metadata: │ +│ │ +│ Original: msiexec /i │ +│ Modified: msiexec /i "" │ +│ │ +│ Wrapped in .cmd script and executed │ +└────────────────────────────────────────────────────────────────────┘ +``` + +#### Windows Path Handling (utils.rs) + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Long Path Support (Windows-specific) │ +├────────────────────────────────────────────────────────────────────┤ +│ prepend_long_path_support(): │ +│ │ +│ Input: c:/path/to/file.txt │ +│ Output: \\?\c:\path\to\file.txt │ +│ │ +│ - Handles paths > 260 characters │ +│ - Converts forward slashes to backslashes │ +│ - Skips already-prefixed verbatim paths │ +└────────────────────────────────────────────────────────────────────┘ +``` + +#### Windows Move Directory (uvm_move_dir) + +Uses Win32 `MoveFileW` API instead of Rust's `fs::rename` for cross-volume moves: + +```rust +// win_move_file.rs +winbase::MoveFileW(from.as_ptr(), to.as_ptr()) +``` + +--- + +## Shared Installers (Cross-Platform) + +### ZIP Installer (installer/zip.rs) + +Uses the `zip` crate for pure-Rust extraction: + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ ZIP Extraction │ +├────────────────────────────────────────────────────────────────────┤ +│ Features: │ +│ - In-memory extraction using zip crate │ +│ - Preserves Unix permissions (chmod) on Unix systems │ +│ - Supports rename mapping (from → to path prefix replacement) │ +│ - Creates directories as needed │ +└────────────────────────────────────────────────────────────────────┘ +``` + +### PO Installer (installer/po.rs) + +Simple file copy for language pack `.po` files: + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ PO File Installation │ +├────────────────────────────────────────────────────────────────────┤ +│ 1. Extract filename from source path │ +│ 2. Create destination directory if needed │ +│ 3. Copy file to destination/ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Module Rename/Destination Logic + +**Location**: `uvm_install/src/lib.rs:437-472` + +Unity modules can specify: +- **destination**: Where to install (relative to Unity base, uses `{UNITY_PATH}` placeholder) +- **extracted_path_rename**: Rename files after extraction + +```rust +fn strip_unity_base_url, Q: AsRef>(path: P, base_dir: Q) -> PathBuf { + // Replaces "{UNITY_PATH}" with actual base directory + base_dir.join(path.strip_prefix("{UNITY_PATH}").unwrap_or(path)) +} +``` + +**Special Case - iOS Module**: +```rust +// In Module::destination() +if &self.id == "ios" { + self.destination.map(|d| format!("{}/iOSSupport", d)) +} +``` + +--- + +## Process Locking + +**Location**: `uvm_install/src/install/utils.rs:12-40` + +Uses `cluFlock` for file-based process locking: + +```rust +macro_rules! lock_process { + ($lock_path:expr) => { + let lock_file = fs::File::create($lock_path)?; + let _lock = utils::lock_process_or_wait(&lock_file)?; + }; +} +``` + +**Lock Types**: +1. **Version Lock**: `~/.unity/locks/.lock` - Prevents concurrent installs of same version +2. **Download Lock**: `~/.cache/uvm/tmp//.lock` - Prevents concurrent downloads of same file + +**Behavior**: If lock is held, waits for it to be released rather than failing. + +--- + +## Error Handling + +**Error Types** (`uvm_install/src/error.rs`): + +| Error | Description | +|-------|-------------| +| `ReleaseLoadFailure` | Failed to fetch from Unity API | +| `LockProcessFailure` | Can't acquire installation lock | +| `UnsupportedModule` | Requested module not available for version | +| `LoadingInstallerFailed` | Download or checksum failure | +| `InstallerCreatedFailed` | Can't create appropriate installer for format | +| `InstallFailed` | Installation execution failed | + +**Cleanup Strategy**: +- `error_handler()` is called on install failure +- Most installers delete the destination directory on failure +- Temp directories are always cleaned up + +--- + +## Platform Quirks Summary + +### macOS +- Uses `xar` for PKG extraction (system command) +- Uses `tar -zmxf` for payload extraction +- Uses `dmg` crate for DMG mounting +- Some modules require `sudo installer` for system-level packages +- ARM64 architecture verification for Unity >= 2021.2.0f1 + +### Linux +- Uses `7z` for PKG extraction (external dependency) +- Uses `gzip` + `cpio` pipeline for payload extraction +- PlaybackEngines path adjustment for correct extraction location +- Always defaults to x86_64 architecture regardless of system + +### Windows +- Uses temp `.cmd` scripts to invoke installers +- Editor installs use `-UI=reduced` flag for minimal UI +- Uses Win32 `MoveFileW` API for reliable directory moves +- Supports `\\?\` long path prefix for > 260 char paths +- MSI modules require command string from module metadata + +### All Platforms +- HTTP Range headers for download resumption +- SRI hash verification of downloaded packages +- File-based process locking for concurrent operation safety +- Modules may have `extracted_path_rename` for post-install reorganization + +--- + +## Data Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ User Request │ +│ uvm install 2022.3.0f1 -m android │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ InstallOptions │ +│ version: 2022.3.0f1 │ +│ modules: ["android"] │ +│ architecture: x86_64 │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ FetchRelease (GraphQL API) │ +│ POST https://live-platform-api.prd.ld.unity3d.com/graphql │ +│ Returns: Release with downloads[] and modules[] │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ InstallGraph │ +│ ┌─────────┐ │ +│ │ Editor │ ◄── Already installed? mark_installed() │ +│ └────┬────┘ │ +│ │ │ +│ ┌────▼────┐ │ +│ │ Android │ ◄── Requested, mark as needed │ +│ └────┬────┘ │ +│ │ │ +│ ┌────▼─────┐ │ +│ │ SDK/NDK │ ◄── Dependency (if --with-sync) │ +│ └──────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ For each MISSING component (topo order): │ +│ │ +│ ┌─────────────┐ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ Loader │ ──► │ Platform-specific│ ──► │ InstallHandler │ │ +│ │ (download) │ │ create_installer │ │ .install() │ │ +│ └─────────────┘ └──────────────────┘ └──────────────────┘ │ +│ │ +│ Cache dir: ~/.cache/uvm/installer/-/ │ +│ Install dir: /Applications/Unity// (or custom) │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Write Module Manifest │ +│ Updates modules.json in Unity installation │ +│ Registers with Unity Hub if custom destination │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Configuration & Environment Variables + +| Variable | Purpose | +|----------|---------| +| `UVM_ARCHITECTURE_CHECK_ENABLED` | Enable ARM64 binary verification (macOS) | +| `UVM_LIVE_PLATFORM_CACHE_*` | Control API response caching | +| `UVM_LIVE_PLATFORM_FETCH_RELEASE_CACHE_*` | Release-specific cache settings | + +--- + +## External Dependencies + +| Platform | Dependency | Used For | +|----------|------------|----------| +| macOS | `xar` | PKG extraction | +| macOS | `tar` | Payload extraction | +| Linux | `7z` (p7zip) | PKG extraction | +| Linux | `gzip` | Payload decompression | +| Linux | `cpio` | Payload extraction | +| Linux | `tar` | XZ extraction | + +--- + +## Future Considerations + +1. **Parallel Downloads**: Currently downloads are sequential; could parallelize +2. **Better Progress Reporting**: `ProgressHandler` trait exists but implementation is partial +3. **Checksum Caching**: Currently reads entire file for verification; could use incremental hash +4. **Linux ARM64**: Currently forces x86_64; could support native ARM64 when Unity provides it diff --git a/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/.openspec.yaml b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/.openspec.yaml new file mode 100644 index 00000000..85cf50d8 --- /dev/null +++ b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-03-03 diff --git a/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/design.md b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/design.md new file mode 100644 index 00000000..2c8b6575 --- /dev/null +++ b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/design.md @@ -0,0 +1,93 @@ +## Context + +The macOS PKG installer extracts Unity packages using a two-step process: +1. Extract the PKG container using `xar` +2. Extract the payload using `tar -zmxf` (assumes gzipped tar) + +Unity has started shipping PKG files with cpio payloads instead of tar. The Linux implementation already handles both formats by using a `gzip | cpio` pipeline. The macOS implementation needs similar logic. + +**Current code** (`uvm_install/src/sys/mac/pkg.rs`): +```rust +fn tar(&self, source: P, destination: D) -> InstallerResult<()> { + Command::new("tar").arg("-zmxf").arg(source)... +} +``` + +**Linux code** (`uvm_install/src/sys/linux/pkg.rs`): +```rust +// Handles both Payload~ (raw cpio) and Payload (gzipped cpio) +gzip -dc Payload | cpio -iu +``` + +## Goals / Non-Goals + +**Goals:** +- Support cpio-format payloads on macOS +- Maintain backwards compatibility with tar-format payloads +- Detect payload format automatically (no user configuration required) + +**Non-Goals:** +- Changing the PKG extraction (xar) step - it works correctly +- Supporting other archive formats beyond tar and cpio +- Modifying the Linux implementation (already works) + +## Decisions + +### 1. Format Detection Strategy + +**Decision**: Detect format by examining magic bytes after gzip decompression + +**Rationale**: +- cpio archives start with `070707` (ODC) or `070701`/`070702` (newc) +- tar archives can be detected by `ustar` at offset 257, or by absence of cpio magic +- Checking first 6 bytes of decompressed stream is sufficient + +**Alternatives considered**: +- Try tar first, fall back to cpio on failure: Wasteful for large files (5+ GB), poor UX +- Check file extension: Not reliable, both are named "Payload" +- Assume cpio always: Breaks backwards compatibility + +### 2. Implementation Approach + +**Decision**: Add a `detect_payload_format()` function, then dispatch to `extract_tar()` or `extract_cpio()` + +**Rationale**: +- Clean separation of concerns +- Easy to test format detection independently +- Mirrors the structure in the Linux implementation + +**Implementation outline**: +```rust +enum PayloadFormat { Tar, Cpio } + +fn detect_payload_format(path: &Path) -> Result { + // Read first ~512 bytes, decompress, check magic +} + +fn extract_payload(&self, payload: &Path, dest: &Path) -> Result<()> { + match detect_payload_format(payload)? { + PayloadFormat::Tar => self.extract_tar(payload, dest), + PayloadFormat::Cpio => self.extract_cpio(payload, dest), + } +} +``` + +### 3. cpio Extraction Method + +**Decision**: Use `gzip -dc | cpio -iu` via shell pipeline + +**Rationale**: +- Consistent with Linux implementation +- Uses standard macOS system tools (cpio is available by default) +- Handles large files efficiently via streaming + +## Risks / Trade-offs + +**[Risk]** cpio command not available on some macOS systems +→ **Mitigation**: cpio is part of macOS base system since at least 10.4. Not a realistic concern. + +**[Risk]** Format detection reads beginning of large file +→ **Mitigation**: Only decompress first ~512 bytes for detection, not the entire file. Use `flate2` crate for in-process decompression to avoid spawning extra processes. + +**[Trade-off]** Adding format detection adds complexity +→ **Accepted**: Necessary for backwards compatibility. The alternative (breaking old packages) is worse. diff --git a/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/proposal.md b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/proposal.md new file mode 100644 index 00000000..013fcc48 --- /dev/null +++ b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/proposal.md @@ -0,0 +1,26 @@ +## Why + +Unity PKG installer payloads can be either gzipped tar or gzipped cpio archives. The macOS installer currently only supports tar format, causing installation failures for newer Unity versions (e.g., Unity 6000.3.10f1) that use cpio payloads. + +## What Changes + +- Add payload format detection to the macOS PKG installer +- Implement cpio extraction support for macOS (gzip -dc | cpio -iu pipeline) +- Maintain backwards compatibility with existing tar-based payloads +- Align macOS behavior with the Linux implementation which already handles both formats + +## Capabilities + +### New Capabilities + +- `payload-format-detection`: Detect whether a PKG payload is tar or cpio format by examining magic bytes after decompression + +### Modified Capabilities + +- None (no existing spec-level requirements are changing, this is a bug fix to support an additional format) + +## Impact + +- **Code**: `uvm_install/src/sys/mac/pkg.rs` - the `tar()` and `untar()` methods need to detect format and dispatch to appropriate extractor +- **Dependencies**: Uses existing system commands (`gzip`, `cpio`, `tar`) - no new dependencies +- **Compatibility**: Fully backwards compatible - existing tar payloads continue to work, cpio payloads now also work diff --git a/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/specs/payload-format-detection/spec.md b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/specs/payload-format-detection/spec.md new file mode 100644 index 00000000..4a474eda --- /dev/null +++ b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/specs/payload-format-detection/spec.md @@ -0,0 +1,47 @@ +## ADDED Requirements + +### Requirement: Detect gzipped tar payload format + +The macOS PKG installer SHALL detect when a payload is a gzipped tar archive and extract it using tar. + +#### Scenario: Payload is gzipped tar + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content does NOT start with cpio magic (070707, 070701, 070702) +- **THEN** the installer SHALL extract the payload using `tar -zmxf` + +### Requirement: Detect gzipped cpio payload format + +The macOS PKG installer SHALL detect when a payload is a gzipped cpio archive and extract it using cpio. + +#### Scenario: Payload is gzipped cpio ODC format + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content starts with `070707` (cpio ODC magic) +- **THEN** the installer SHALL extract the payload using `gzip -dc | cpio -iu` + +#### Scenario: Payload is gzipped cpio newc format + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content starts with `070701` or `070702` (cpio newc magic) +- **THEN** the installer SHALL extract the payload using `gzip -dc | cpio -iu` + +### Requirement: Extract to correct destination directory + +The macOS PKG installer SHALL extract payload contents to the specified destination directory regardless of payload format. + +#### Scenario: cpio extraction respects destination + +- **WHEN** extracting a cpio payload to destination directory D +- **THEN** the cpio command SHALL run with `-D` or `--directory` set to D, OR the process SHALL change to directory D before extraction + +#### Scenario: tar extraction respects destination + +- **WHEN** extracting a tar payload to destination directory D +- **THEN** the tar command SHALL use `-C D` to extract to the destination + +### Requirement: Backwards compatibility with tar payloads + +The installer SHALL continue to successfully install Unity versions that use tar-format payloads. + +#### Scenario: Install older Unity version with tar payload + +- **WHEN** installing a Unity version that ships with a tar-format PKG payload +- **THEN** the installation SHALL complete successfully using tar extraction diff --git a/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/tasks.md b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/tasks.md new file mode 100644 index 00000000..31e543cc --- /dev/null +++ b/openspec/changes/archive/2026-03-03-fix-macos-cpio-installation/tasks.md @@ -0,0 +1,22 @@ +## 1. Payload Format Detection + +- [x] 1.1 Add `PayloadFormat` enum with `Tar` and `Cpio` variants to `uvm_install/src/sys/mac/pkg.rs` +- [x] 1.2 Implement `detect_payload_format()` function that reads first bytes of gzipped payload and checks for cpio magic (070707, 070701, 070702) +- [x] 1.3 Add `flate2` crate dependency to `uvm_install/Cargo.toml` for gzip decompression (if not already present) + +## 2. cpio Extraction Implementation + +- [x] 2.1 Implement `extract_cpio()` method using `gzip -dc | cpio -iu` pipeline +- [x] 2.2 Ensure cpio extraction runs in the correct destination directory (use `-D` flag or change working directory) + +## 3. Refactor Extraction Flow + +- [x] 3.1 Rename existing `tar()` method to `extract_tar()` for clarity +- [x] 3.2 Create new `extract_payload()` method that calls `detect_payload_format()` and dispatches to appropriate extractor +- [x] 3.3 Update `untar()` method to call `extract_payload()` instead of `tar()` directly + +## 4. Testing + +- [x] 4.1 Test installation with Unity 6000.3.10f1 (cpio payload) to verify fix works +- [x] 4.2 Test installation with an older Unity version (tar payload) to verify backwards compatibility +- [x] 4.3 Run `cargo test -p uvm_install` to ensure no regressions diff --git a/openspec/specs/payload-format-detection/spec.md b/openspec/specs/payload-format-detection/spec.md new file mode 100644 index 00000000..4a474eda --- /dev/null +++ b/openspec/specs/payload-format-detection/spec.md @@ -0,0 +1,47 @@ +## ADDED Requirements + +### Requirement: Detect gzipped tar payload format + +The macOS PKG installer SHALL detect when a payload is a gzipped tar archive and extract it using tar. + +#### Scenario: Payload is gzipped tar + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content does NOT start with cpio magic (070707, 070701, 070702) +- **THEN** the installer SHALL extract the payload using `tar -zmxf` + +### Requirement: Detect gzipped cpio payload format + +The macOS PKG installer SHALL detect when a payload is a gzipped cpio archive and extract it using cpio. + +#### Scenario: Payload is gzipped cpio ODC format + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content starts with `070707` (cpio ODC magic) +- **THEN** the installer SHALL extract the payload using `gzip -dc | cpio -iu` + +#### Scenario: Payload is gzipped cpio newc format + +- **WHEN** the payload file starts with gzip magic bytes (1f 8b) AND the decompressed content starts with `070701` or `070702` (cpio newc magic) +- **THEN** the installer SHALL extract the payload using `gzip -dc | cpio -iu` + +### Requirement: Extract to correct destination directory + +The macOS PKG installer SHALL extract payload contents to the specified destination directory regardless of payload format. + +#### Scenario: cpio extraction respects destination + +- **WHEN** extracting a cpio payload to destination directory D +- **THEN** the cpio command SHALL run with `-D` or `--directory` set to D, OR the process SHALL change to directory D before extraction + +#### Scenario: tar extraction respects destination + +- **WHEN** extracting a tar payload to destination directory D +- **THEN** the tar command SHALL use `-C D` to extract to the destination + +### Requirement: Backwards compatibility with tar payloads + +The installer SHALL continue to successfully install Unity versions that use tar-format payloads. + +#### Scenario: Install older Unity version with tar payload + +- **WHEN** installing a Unity version that ships with a tar-format PKG payload +- **THEN** the installation SHALL complete successfully using tar extraction diff --git a/uvm_install/Cargo.toml b/uvm_install/Cargo.toml index 5bb42ba3..0203c414 100644 --- a/uvm_install/Cargo.toml +++ b/uvm_install/Cargo.toml @@ -36,6 +36,7 @@ thiserror-context = "0.1.2" cluFlock = "1.2.5" [target.'cfg(target_os="macos")'.dependencies] dmg = "0.1.1" +flate2 = "1.1.1" mach_object = "0.1.17" sysctl = "0.6.0" [target.'cfg(target_os="windows")'.dependencies] diff --git a/uvm_install/src/sys/mac/pkg.rs b/uvm_install/src/sys/mac/pkg.rs index a65eb3af..5f82c092 100644 --- a/uvm_install/src/sys/mac/pkg.rs +++ b/uvm_install/src/sys/mac/pkg.rs @@ -2,13 +2,52 @@ use crate::install::error::InstallerErrorInner::{InstallationFailed, InstallerCr use crate::install::error::{InstallerError, InstallerResult}; use crate::install::installer::{BaseInstaller, Installer, InstallerWithDestination, Pkg}; use crate::install::{InstallHandler, UnityEditor, UnityModule}; +use flate2::read::GzDecoder; use log::{debug, info, warn}; -use std::fs::DirBuilder; +use std::fs::{DirBuilder, File}; +use std::io::Read; use std::path::Path; use std::process::{Command, Stdio}; use std::{fs, io}; use thiserror_context::Context; +/// Payload archive format inside a PKG file +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PayloadFormat { + /// gzipped tar archive + Tar, + /// gzipped cpio archive (ODC or newc format) + Cpio, +} + +/// Detect the format of a gzipped payload by examining magic bytes after decompression. +/// +/// cpio archives start with: +/// - `070707` (ODC/old ASCII format) +/// - `070701` (newc format) +/// - `070702` (newc with CRC) +/// +/// If none of these patterns match, we assume tar format. +fn detect_payload_format>(path: P) -> io::Result { + let path = path.as_ref(); + let file = File::open(path)?; + let mut decoder = GzDecoder::new(file); + + let mut buffer = [0u8; 6]; + decoder.read_exact(&mut buffer)?; + + // Check for cpio magic bytes (ASCII: "070707", "070701", "070702") + let is_cpio = &buffer == b"070707" || &buffer == b"070701" || &buffer == b"070702"; + + if is_cpio { + debug!("detected cpio payload format in {}", path.display()); + Ok(PayloadFormat::Cpio) + } else { + debug!("detected tar payload format in {}", path.display()); + Ok(PayloadFormat::Tar) + } +} + pub type EditorPkgInstaller = Installer; pub type ModulePkgNativeInstaller = Installer; pub type ModulePkgInstaller = Installer; @@ -88,11 +127,82 @@ impl Installer { ) -> Result<(), InstallerError> { let base_payload_path = base_payload_path.as_ref(); let payload = self.find_payload(&base_payload_path).context("unable to find payload in package")?; - debug!("untar payload at {}", payload.display()); - self.tar(&payload, destination) + debug!("extract payload at {}", payload.display()); + self.extract_payload(&payload, destination) + } + + fn extract_payload, D: AsRef>( + &self, + source: P, + destination: D, + ) -> InstallerResult<()> { + let source = source.as_ref(); + let destination = destination.as_ref(); + + let format = detect_payload_format(source)?; + match format { + PayloadFormat::Tar => self.extract_tar(source, destination), + PayloadFormat::Cpio => self.extract_cpio(source, destination), + } + } + + fn extract_cpio, D: AsRef>( + &self, + source: P, + destination: D, + ) -> InstallerResult<()> { + let source = source.as_ref(); + let destination = destination.as_ref(); + + debug!( + "extract cpio payload {} to {}", + source.display(), + destination.display() + ); + + // Use gzip to decompress and pipe to cpio for extraction + let mut gzip_child = Command::new("gzip") + .arg("-dc") + .arg(source) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let gzip_stdout = gzip_child.stdout.take().ok_or_else(|| { + io::Error::new(io::ErrorKind::Other, "failed to capture gzip stdout") + })?; + + let cpio_child = Command::new("cpio") + .arg("-id") + .arg("--quiet") + .current_dir(destination) + .stdin(gzip_stdout) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let cpio_output = cpio_child.wait_with_output()?; + let gzip_status = gzip_child.wait()?; + + if !gzip_status.success() { + return Err(InstallerCreateFailed( + "failed to decompress payload with gzip".to_string(), + ) + .into()); + } + + if !cpio_output.status.success() { + return Err(InstallerCreateFailed(format!( + "failed to extract cpio payload:\n{}", + String::from_utf8_lossy(&cpio_output.stderr) + )) + .into()); + } + + Ok(()) } - fn tar, D: AsRef>( + fn extract_tar, D: AsRef>( &self, source: P, destination: D,