diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b55b7abe2888..2834ae53c708 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7", + "image": "ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 2bad12ab13e8..64d56bf0a743 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -26,7 +26,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" timeout-minutes: 90 diff --git a/.github/workflows/ci-pr-only.yml b/.github/workflows/ci-pr-only.yml index 5032785cd5e8..7bac0e902e07 100644 --- a/.github/workflows/ci-pr-only.yml +++ b/.github/workflows/ci-pr-only.yml @@ -32,7 +32,7 @@ jobs: runs-on: &dind-small-setup labels: dind-small container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --mount type=tmpfs,target="/home/buildifier/.local/share/containers" steps: diff --git a/.github/workflows/pocket-ic-tests-windows.yml b/.github/workflows/pocket-ic-tests-windows.yml index 8c497bdf514d..ed925fc6d5b2 100644 --- a/.github/workflows/pocket-ic-tests-windows.yml +++ b/.github/workflows/pocket-ic-tests-windows.yml @@ -45,7 +45,7 @@ jobs: bazel-build-pocket-ic: name: Bazel Build PocketIC container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" timeout-minutes: 90 diff --git a/.github/workflows/rate-limits-backend-release.yml b/.github/workflows/rate-limits-backend-release.yml index c5317ad59b9a..5d7fe34fd19a 100644 --- a/.github/workflows/rate-limits-backend-release.yml +++ b/.github/workflows/rate-limits-backend-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/release-testing.yml b/.github/workflows/release-testing.yml index 6137468ddc25..9c7e7048fe65 100644 --- a/.github/workflows/release-testing.yml +++ b/.github/workflows/release-testing.yml @@ -32,7 +32,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" timeout-minutes: 180 diff --git a/.github/workflows/rosetta-release.yml b/.github/workflows/rosetta-release.yml index b8a7ac154ce2..de729b7d1f5a 100644 --- a/.github/workflows/rosetta-release.yml +++ b/.github/workflows/rosetta-release.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" environment: DockerHub diff --git a/.github/workflows/salt-sharing-canister-release.yml b/.github/workflows/salt-sharing-canister-release.yml index bcca69e67cf7..de3fbc92991c 100644 --- a/.github/workflows/salt-sharing-canister-release.yml +++ b/.github/workflows/salt-sharing-canister-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml index f5074f95e33a..32fdf37e81b0 100644 --- a/.github/workflows/schedule-daily.yml +++ b/.github/workflows/schedule-daily.yml @@ -20,7 +20,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml index 21f90a6a6b49..f882d7687342 100644 --- a/.github/workflows/schedule-rust-bench.yml +++ b/.github/workflows/schedule-rust-bench.yml @@ -24,7 +24,7 @@ jobs: # see linux-x86-64 runner group labels: rust-benchmarks container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc # running on bare metal machine using ubuntu user options: --user ubuntu --mount type=tmpfs,target="/home/ubuntu/.local/share/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-weekly.yml b/.github/workflows/schedule-weekly.yml index 5fc4e369d9a0..e3a3c1272c52 100644 --- a/.github/workflows/schedule-weekly.yml +++ b/.github/workflows/schedule-weekly.yml @@ -10,7 +10,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --mount type=tmpfs,target="/home/buildifier/.local/share/containers" timeout-minutes: 60 # 1 hour diff --git a/.github/workflows/update-mainnet-canister-revisions.yaml b/.github/workflows/update-mainnet-canister-revisions.yaml index c91f70c8cc99..eeb59bc5c437 100644 --- a/.github/workflows/update-mainnet-canister-revisions.yaml +++ b/.github/workflows/update-mainnet-canister-revisions.yaml @@ -25,7 +25,7 @@ jobs: labels: dind-small environment: CREATE_PR container: - image: ghcr.io/dfinity/ic-build@sha256:cb3a6693a10777d16c301d98f5b67e23db405bf962d0eb8cec74082916c17bc7 + image: ghcr.io/dfinity/ic-build@sha256:3b94487620ed73c5d52fb67f6c5e98c158da7b9ce4525ca6136772befec370cc options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/home/buildifier/.local/share/containers" env: diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 2712f3adbaab..c3b010be4971 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "e9d5e9b93b2036cff0104e43b708a1a72d5337db5d7a3c6e13f62f25cb2ed4a9", + "checksum": "6a5dd2e8a496a2051bfd7461dbd9f6705336f1dc5077b7f0c2b39b33809fb508", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -7532,6 +7532,146 @@ ], "license_file": "LICENSE" }, + "bindgen 0.72.1": { + "name": "bindgen", + "version": "0.72.1", + "package_url": "https://github.com/rust-lang/rust-bindgen", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/bindgen/0.72.1/download", + "sha256": "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bindgen", + "crate_root": "lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "bindgen", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "logging", + "prettyplease", + "runtime" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "bindgen 0.72.1", + "target": "build_script_build" + }, + { + "id": "bitflags 2.10.0", + "target": "bitflags" + }, + { + "id": "cexpr 0.6.0", + "target": "cexpr" + }, + { + "id": "clang-sys 1.6.1", + "target": "clang_sys" + }, + { + "id": "itertools 0.13.0", + "target": "itertools" + }, + { + "id": "log 0.4.28", + "target": "log" + }, + { + "id": "prettyplease 0.2.15", + "target": "prettyplease" + }, + { + "id": "proc-macro2 1.0.103", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.42", + "target": "quote" + }, + { + "id": "regex 1.12.2", + "target": "regex" + }, + { + "id": "rustc-hash 2.1.1", + "target": "rustc_hash" + }, + { + "id": "shlex 1.3.0", + "target": "shlex" + }, + { + "id": "syn 2.0.110", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.72.1" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "compile_data_glob_excludes": [ + "**/*.rs" + ], + "data_glob": [ + "**" + ], + "link_deps": { + "common": [ + { + "id": "clang-sys 1.6.1", + "target": "clang_sys" + }, + { + "id": "prettyplease 0.2.15", + "target": "prettyplease" + } + ], + "selects": {} + } + }, + "license": "BSD-3-Clause", + "license_ids": [ + "BSD-3-Clause" + ], + "license_file": "LICENSE" + }, "binread 2.2.0": { "name": "binread", "version": "2.2.0", @@ -22231,6 +22371,10 @@ "id": "secp256k1 0.22.2", "target": "secp256k1" }, + { + "id": "selinux 0.5.3", + "target": "selinux" + }, { "id": "semver 1.0.27", "target": "semver" @@ -22623,6 +22767,10 @@ "id": "x509-parser 0.16.0", "target": "x509_parser" }, + { + "id": "xattr 1.6.1", + "target": "xattr" + }, { "id": "yansi 0.5.1", "target": "yansi" @@ -72331,6 +72479,194 @@ ], "license_file": null }, + "selinux 0.5.3": { + "name": "selinux", + "version": "0.5.3", + "package_url": "https://codeberg.org/koutheir/selinux.git", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/selinux/0.5.3/download", + "sha256": "8f6af114a661557df02e60c25e5cb40779d295ec2e4ae0fd903fe414578b6191" + } + }, + "targets": [ + { + "Library": { + "crate_name": "selinux", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "selinux", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "bitflags 2.10.0", + "target": "bitflags" + }, + { + "id": "errno 0.3.10", + "target": "errno" + }, + { + "id": "libc 0.2.177", + "target": "libc" + }, + { + "id": "once_cell 1.21.3", + "target": "once_cell" + }, + { + "id": "parking_lot 0.12.5", + "target": "parking_lot" + }, + { + "id": "selinux-sys 0.6.15", + "target": "selinux_sys" + }, + { + "id": "thiserror 2.0.17", + "target": "thiserror" + } + ], + "selects": {} + }, + "edition": "2024", + "version": "0.5.3" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE.txt" + }, + "selinux-sys 0.6.15": { + "name": "selinux-sys", + "version": "0.6.15", + "package_url": "https://codeberg.org/koutheir/selinux-sys.git", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/selinux-sys/0.6.15/download", + "sha256": "debaba5832b4831ffe0ba9118b526c752c960f41c46c4ef197d9a15f5179d6fd" + } + }, + "targets": [ + { + "Library": { + "crate_name": "selinux_sys", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "selinux_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "selinux-sys 0.6.15", + "target": "build_script_build" + } + ], + "selects": {} + }, + "extra_deps": { + "common": [], + "selects": { + "x86_64-unknown-linux-gnu": [ + "@@_main~_repo_rules~libselinux//:libselinux" + ] + } + }, + "edition": "2024", + "version": "0.6.15" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "compile_data_glob_excludes": [ + "**/*.rs" + ], + "data": { + "common": [], + "selects": { + "x86_64-unknown-linux-gnu": [ + "@@_main~_repo_rules~libselinux//:libselinux" + ] + } + }, + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "bindgen 0.72.1", + "target": "bindgen" + }, + { + "id": "cc 1.2.48", + "target": "cc" + }, + { + "id": "dunce 1.0.5", + "target": "dunce" + }, + { + "id": "walkdir 2.5.0", + "target": "walkdir" + } + ], + "selects": {} + }, + "build_script_env": { + "common": {}, + "selects": { + "x86_64-unknown-linux-gnu": { + "SELINUX_INCLUDE_DIR": "/usr/include", + "SELINUX_LIB_DIR": "/usr/lib/x86_64-linux-gnu" + } + } + }, + "links": "selinux" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE.txt" + }, "semver 1.0.27": { "name": "semver", "version": "1.0.27", @@ -98766,6 +99102,7 @@ "scraper 0.17.1", "scrypt 0.11.0", "secp256k1 0.22.2", + "selinux 0.5.3", "semver 1.0.27", "serde 1.0.228", "serde-bytes-repr 0.1.5", @@ -98866,6 +99203,7 @@ "wycheproof 0.6.0", "x509-cert 0.2.5", "x509-parser 0.16.0", + "xattr 1.6.1", "yansi 0.5.1", "zeroize 1.8.1", "zstd 0.13.2" diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 10112edf2518..0fdac6333991 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -1309,6 +1309,26 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.110", +] + [[package]] name = "binread" version = "2.2.0" @@ -3858,6 +3878,7 @@ dependencies = [ "scraper", "scrypt", "secp256k1 0.22.2", + "selinux", "semver", "serde", "serde-bytes-repr", @@ -3958,6 +3979,7 @@ dependencies = [ "wycheproof", "x509-cert", "x509-parser 0.16.0", + "xattr", "yansi 0.5.1", "zeroize", "zstd 0.13.2", @@ -12183,6 +12205,33 @@ dependencies = [ "smallvec", ] +[[package]] +name = "selinux" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6af114a661557df02e60c25e5cb40779d295ec2e4ae0fd903fe414578b6191" +dependencies = [ + "bitflags 2.10.0", + "errno 0.3.10", + "libc", + "once_cell", + "parking_lot 0.12.5", + "selinux-sys", + "thiserror 2.0.17", +] + +[[package]] +name = "selinux-sys" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debaba5832b4831ffe0ba9118b526c752c960f41c46c4ef197d9a15f5179d6fd" +dependencies = [ + "bindgen 0.72.1", + "cc", + "dunce", + "walkdir", +] + [[package]] name = "semver" version = "1.0.27" diff --git a/Cargo.lock b/Cargo.lock index 9427306c0e09..27fbb9c874f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,6 +1290,26 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.110", +] + [[package]] name = "binread" version = "2.2.0" @@ -1713,6 +1733,20 @@ dependencies = [ "xz2", ] +[[package]] +name = "build_filesystem" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.27", + "ic_device", + "regex", + "selinux", + "tar", + "tempfile", + "xattr", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -16145,7 +16179,6 @@ dependencies = [ "rand 0.8.5", "sys-mount", "tempfile", - "tokio", "uuid", ] @@ -22669,6 +22702,33 @@ dependencies = [ "smallvec", ] +[[package]] +name = "selinux" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6af114a661557df02e60c25e5cb40779d295ec2e4ae0fd903fe414578b6191" +dependencies = [ + "bitflags 2.10.0", + "errno 0.3.10", + "libc", + "once_cell", + "parking_lot", + "selinux-sys", + "thiserror 2.0.17", +] + +[[package]] +name = "selinux-sys" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debaba5832b4831ffe0ba9118b526c752c960f41c46c4ef197d9a15f5179d6fd" +dependencies = [ + "bindgen 0.72.1", + "cc", + "dunce", + "walkdir", +] + [[package]] name = "semver" version = "1.0.25" @@ -26760,13 +26820,12 @@ dependencies = [ [[package]] name = "xattr" -version = "1.4.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "linux-raw-sys 0.4.15", - "rustix 0.38.44", + "rustix 1.1.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0ce2391c1375..dd78f885f12a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,6 +163,7 @@ members = [ "rs/http_utils", "rs/ic_os/attestation", "rs/ic_os/attestation/testing", + "rs/ic_os/build_tools/build_filesystem", "rs/ic_os/build_tools/dflate", "rs/ic_os/build_tools/diroid", "rs/ic_os/build_tools/inject_files", @@ -823,6 +824,7 @@ serde_cbor = "0.11.2" serde_json = { version = "^1.0.107" } serde_with = "1.14.0" serde_yaml = "0.9.33" +selinux = "0.5" sev = { version = "7.1", default-features = false, features = [ "crypto_nossl", "snp", @@ -897,6 +899,7 @@ url = { version = "2.5.3", features = ["serde"] } uuid = { version = "=1.12.1", features = ["v4", "serde"] } virt = "0.4" walkdir = "2.3.3" +xattr = "1.6.1" walrus = "0.23.3" wasm-encoder = { version = "0.240.0", features = ["wasmparser"] } wasmparser = "0.240.0" diff --git a/MODULE.bazel b/MODULE.bazel index baba00a899ba..db6ad2ac970f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -670,6 +670,18 @@ http_archive( ], ) +http_archive( + name = "e2fsprogs", + build_file = "@//third_party:BUILD.e2fsprogs.bazel", + patch_args = ["-p1"], + patches = ["//bazel:e2fsprogs.patch"], + sha256 = "86360777b8e0832f3fcbfc11e333a7a0e9df15ca3b67a8072fcb93d97a2b8ab9", + strip_prefix = "e2fsprogs-da631e117dcf8797bfda0f48bdaa05ac0fbcf7af", + urls = [ + "https://github.com/tytso/e2fsprogs/archive/da631e117dcf8797bfda0f48bdaa05ac0fbcf7af.tar.gz", + ], +) + http_file( name = "bitcoin_example_canister", downloaded_file_path = "basic_bitcoin.wasm.gz", @@ -1074,6 +1086,14 @@ new_local_repository( path = "/usr", ) +# Use libselinux from the host environment +# Used for SELinux context lookups in build tools. +new_local_repository( + name = "libselinux", + build_file = "@//third_party:BUILD.selinux.bazel", + path = "/usr", +) + # Mainnet canister references canisters = use_repo_rule("//bazel:mainnet-canisters.bzl", "canisters") diff --git a/bazel/e2fsprogs.patch b/bazel/e2fsprogs.patch new file mode 100644 index 000000000000..fd5e6163d115 --- /dev/null +++ b/bazel/e2fsprogs.patch @@ -0,0 +1,19 @@ +# Add security.selinux to the xattr filter in create_inode_libarchive.c +# This ensures that security.selinux xattrs are preserved when creating +# filesystem images from tar archives. +# See: https://github.com/tytso/e2fsprogs/issues/255 +diff --git a/misc/create_inode_libarchive.c b/misc/create_inode_libarchive.c +index 1234567..abcdefg 100644 +--- a/misc/create_inode_libarchive.c ++++ b/misc/create_inode_libarchive.c +@@ -470,7 +470,7 @@ static errcode_t set_inode_xattr_tar(ext2_filsys fs, ext2_ino_t ino, + + dl_archive_entry_xattr_reset(entry); + while (dl_archive_entry_xattr_next(entry, &name, &value, &value_size) == + ARCHIVE_OK) { +- if (strcmp(name, "security.capability") != 0 && strcmp(name, "gnu.translator")) ++ if (strcmp(name, "security.capability") != 0 && strcmp(name, "gnu.translator") && strcmp(name, "security.selinux")) + continue; + + retval = ext2fs_xattr_set(handle, name, value, value_size); + diff --git a/bazel/noble.lock.json b/bazel/noble.lock.json index 7be7492f603b..9b344cdae371 100755 --- a/bazel/noble.lock.json +++ b/bazel/noble.lock.json @@ -1099,6 +1099,348 @@ "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/g/gzip/gzip_1.12-1ubuntu3.1_amd64.deb", "version": "1.12-1ubuntu3.1" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "nettle-dev_3.9.1-2.2build1.1_amd64", + "name": "nettle-dev", + "version": "3.9.1-2.2build1.1" + }, + { + "key": "libgmp-dev_2-6.3.0-p-dfsg-2ubuntu6.1_amd64", + "name": "libgmp-dev", + "version": "2:6.3.0+dfsg-2ubuntu6.1" + }, + { + "key": "libgmpxx4ldbl_2-6.3.0-p-dfsg-2ubuntu6.1_amd64", + "name": "libgmpxx4ldbl", + "version": "2:6.3.0+dfsg-2ubuntu6.1" + }, + { + "key": "libstdc-p--p-6_14.2.0-4ubuntu2_24.04_amd64", + "name": "libstdc++6", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "libgcc-s1_14.2.0-4ubuntu2_24.04_amd64", + "name": "libgcc-s1", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "libc6_2.39-0ubuntu8.6_amd64", + "name": "libc6", + "version": "2.39-0ubuntu8.6" + }, + { + "key": "gcc-14-base_14.2.0-4ubuntu2_24.04_amd64", + "name": "gcc-14-base", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "libgmp10_2-6.3.0-p-dfsg-2ubuntu6.1_amd64", + "name": "libgmp10", + "version": "2:6.3.0+dfsg-2ubuntu6.1" + }, + { + "key": "libhogweed6t64_3.9.1-2.2build1.1_amd64", + "name": "libhogweed6t64", + "version": "3.9.1-2.2build1.1" + }, + { + "key": "libnettle8t64_3.9.1-2.2build1.1_amd64", + "name": "libnettle8t64", + "version": "3.9.1-2.2build1.1" + }, + { + "key": "libext2fs-dev_1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "libext2fs-dev", + "version": "1.47.0-2.4~exp1ubuntu4.1" + }, + { + "key": "libext2fs2t64_1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "libext2fs2t64", + "version": "1.47.0-2.4~exp1ubuntu4.1" + }, + { + "key": "comerr-dev_2.1-1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "comerr-dev", + "version": "2.1-1.47.0-2.4~exp1ubuntu4.1" + }, + { + "key": "libcom-err2_1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "libcom-err2", + "version": "1.47.0-2.4~exp1ubuntu4.1" + }, + { + "key": "libacl1-dev_2.3.2-1build1.1_amd64", + "name": "libacl1-dev", + "version": "2.3.2-1build1.1" + }, + { + "key": "libattr1-dev_1-2.5.2-1build1.1_amd64", + "name": "libattr1-dev", + "version": "1:2.5.2-1build1.1" + }, + { + "key": "libattr1_1-2.5.2-1build1.1_amd64", + "name": "libattr1", + "version": "1:2.5.2-1build1.1" + }, + { + "key": "libacl1_2.3.2-1build1.1_amd64", + "name": "libacl1", + "version": "2.3.2-1build1.1" + }, + { + "key": "zlib1g-dev_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g-dev", + "version": "1:1.3.dfsg-3.1ubuntu2.1" + }, + { + "key": "zlib1g_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g", + "version": "1:1.3.dfsg-3.1ubuntu2.1" + }, + { + "key": "libzstd-dev_1.5.5-p-dfsg2-2build1.1_amd64", + "name": "libzstd-dev", + "version": "1.5.5+dfsg2-2build1.1" + }, + { + "key": "libzstd1_1.5.5-p-dfsg2-2build1.1_amd64", + "name": "libzstd1", + "version": "1.5.5+dfsg2-2build1.1" + }, + { + "key": "libxml2-dev_2.9.14-p-dfsg-1.3ubuntu3.6_amd64", + "name": "libxml2-dev", + "version": "2.9.14+dfsg-1.3ubuntu3.6" + }, + { + "key": "libxml2_2.9.14-p-dfsg-1.3ubuntu3.6_amd64", + "name": "libxml2", + "version": "2.9.14+dfsg-1.3ubuntu3.6" + }, + { + "key": "liblzma5_5.6.1-p-really5.4.5-1ubuntu0.2_amd64", + "name": "liblzma5", + "version": "5.6.1+really5.4.5-1ubuntu0.2" + }, + { + "key": "libicu74_74.2-1ubuntu3.1_amd64", + "name": "libicu74", + "version": "74.2-1ubuntu3.1" + }, + { + "key": "libicu-dev_74.2-1ubuntu3.1_amd64", + "name": "libicu-dev", + "version": "74.2-1ubuntu3.1" + }, + { + "key": "icu-devtools_74.2-1ubuntu3.1_amd64", + "name": "icu-devtools", + "version": "74.2-1ubuntu3.1" + }, + { + "key": "liblzma-dev_5.6.1-p-really5.4.5-1ubuntu0.2_amd64", + "name": "liblzma-dev", + "version": "5.6.1+really5.4.5-1ubuntu0.2" + }, + { + "key": "liblz4-dev_1.9.4-1build1.1_amd64", + "name": "liblz4-dev", + "version": "1.9.4-1build1.1" + }, + { + "key": "liblz4-1_1.9.4-1build1.1_amd64", + "name": "liblz4-1", + "version": "1.9.4-1build1.1" + }, + { + "key": "libbz2-dev_1.0.8-5.1build0.1_amd64", + "name": "libbz2-dev", + "version": "1.0.8-5.1build0.1" + }, + { + "key": "libbz2-1.0_1.0.8-5.1build0.1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5.1build0.1" + }, + { + "key": "libarchive13t64_3.7.2-2ubuntu0.5_amd64", + "name": "libarchive13t64", + "version": "3.7.2-2ubuntu0.5" + } + ], + "key": "libarchive-dev_3.7.2-2ubuntu0.5_amd64", + "name": "libarchive-dev", + "sha256": "1f585e013dfbca8fed042de1a959e5ea7738f0594b85ab3319068be4dd46a8a8", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/liba/libarchive/libarchive-dev_3.7.2-2ubuntu0.5_amd64.deb", + "version": "3.7.2-2ubuntu0.5" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "nettle-dev_3.9.1-2.2build1.1_amd64", + "name": "nettle-dev", + "sha256": "df06f798b64f1b30be477d7260ef6ad573223f66dd31f6d8b8696b27765ade23", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/n/nettle/nettle-dev_3.9.1-2.2build1.1_amd64.deb", + "version": "3.9.1-2.2build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgmp-dev_2-6.3.0-p-dfsg-2ubuntu6.1_amd64", + "name": "libgmp-dev", + "sha256": "a9847b5ecfff791a46cb198f29a06d9d23a20afdfdd434812f2b44ccfb61d46a", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/g/gmp/libgmp-dev_6.3.0+dfsg-2ubuntu6.1_amd64.deb", + "version": "2:6.3.0+dfsg-2ubuntu6.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgmpxx4ldbl_2-6.3.0-p-dfsg-2ubuntu6.1_amd64", + "name": "libgmpxx4ldbl", + "sha256": "6f59344240b6dc139ed23ef236b8aa146e5d25e23aa90e79f814e2e4cc4b5752", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/g/gmp/libgmpxx4ldbl_6.3.0+dfsg-2ubuntu6.1_amd64.deb", + "version": "2:6.3.0+dfsg-2ubuntu6.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libext2fs-dev_1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "libext2fs-dev", + "sha256": "2aa2f4694b72e11a6501964266b6d54acc2c49e6af7c0c97d2433032edfdefa8", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/e/e2fsprogs/libext2fs-dev_1.47.0-2.4~exp1ubuntu4.1_amd64.deb", + "version": "1.47.0-2.4~exp1ubuntu4.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "comerr-dev_2.1-1.47.0-2.4_exp1ubuntu4.1_amd64", + "name": "comerr-dev", + "sha256": "e457b55be696c176d88e82baff0369b1c5d57f69e5b54f8898fae44892d16adc", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/e/e2fsprogs/comerr-dev_2.1-1.47.0-2.4~exp1ubuntu4.1_amd64.deb", + "version": "2.1-1.47.0-2.4~exp1ubuntu4.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libacl1-dev_2.3.2-1build1.1_amd64", + "name": "libacl1-dev", + "sha256": "c9711e29621acc8abb01a337d147e38288f950c75e3f761dbdbbfc634d4a7bdf", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/a/acl/libacl1-dev_2.3.2-1build1.1_amd64.deb", + "version": "2.3.2-1build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libattr1-dev_1-2.5.2-1build1.1_amd64", + "name": "libattr1-dev", + "sha256": "9a46b3d570e8992ff5ee4631abb4ee42e0e26f3febb01564e795703fe2c649f2", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/a/attr/libattr1-dev_2.5.2-1build1.1_amd64.deb", + "version": "1:2.5.2-1build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g-dev_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g-dev", + "sha256": "023cbe9dbf0af87f10e54e342c67571874e412b9950d89c6cd7b010be2e67c3c", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/z/zlib/zlib1g-dev_1.3.dfsg-3.1ubuntu2.1_amd64.deb", + "version": "1:1.3.dfsg-3.1ubuntu2.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libzstd-dev_1.5.5-p-dfsg2-2build1.1_amd64", + "name": "libzstd-dev", + "sha256": "6c9d2b5677a4677e9ead62f74723bb919113621a8028ed543a4d78952d54d165", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/libz/libzstd/libzstd-dev_1.5.5+dfsg2-2build1.1_amd64.deb", + "version": "1.5.5+dfsg2-2build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libxml2-dev_2.9.14-p-dfsg-1.3ubuntu3.6_amd64", + "name": "libxml2-dev", + "sha256": "fea8461ba7b5a5857095640b5397a2673c6c1a6546392de4b950471e1be1db91", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/libx/libxml2/libxml2-dev_2.9.14+dfsg-1.3ubuntu3.6_amd64.deb", + "version": "2.9.14+dfsg-1.3ubuntu3.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libxml2_2.9.14-p-dfsg-1.3ubuntu3.6_amd64", + "name": "libxml2", + "sha256": "665861105ae5603f71607f85363d4a7b2000a55bdbe9de3f02cba6ead6401c8e", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/libx/libxml2/libxml2_2.9.14+dfsg-1.3ubuntu3.6_amd64.deb", + "version": "2.9.14+dfsg-1.3ubuntu3.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libicu74_74.2-1ubuntu3.1_amd64", + "name": "libicu74", + "sha256": "c9a70989678660eed9a1e904c74fa043da8bec8e2036856fc16e31ced79b04f8", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb", + "version": "74.2-1ubuntu3.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libicu-dev_74.2-1ubuntu3.1_amd64", + "name": "libicu-dev", + "sha256": "612b98f4fcfc6ebc57a1b21c2695174694db5a0b7ff760b5d41032076c792398", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/i/icu/libicu-dev_74.2-1ubuntu3.1_amd64.deb", + "version": "74.2-1ubuntu3.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "icu-devtools_74.2-1ubuntu3.1_amd64", + "name": "icu-devtools", + "sha256": "cfcad4370d2e0d4abdccf33cb3d0ffef24c095d76e2121e0a1fd1286ea50b404", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/i/icu/icu-devtools_74.2-1ubuntu3.1_amd64.deb", + "version": "74.2-1ubuntu3.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblzma-dev_5.6.1-p-really5.4.5-1ubuntu0.2_amd64", + "name": "liblzma-dev", + "sha256": "f32c9c79c0d9eb88a3267fea803b4ae9f295c9e2a5cc69f030a298d8a238fa79", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/x/xz-utils/liblzma-dev_5.6.1+really5.4.5-1ubuntu0.2_amd64.deb", + "version": "5.6.1+really5.4.5-1ubuntu0.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblz4-dev_1.9.4-1build1.1_amd64", + "name": "liblz4-dev", + "sha256": "92144951b88d2bb4dd6ea5b85cab6a63e86109a1a6bb0953ada7ba9714ea0042", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/l/lz4/liblz4-dev_1.9.4-1build1.1_amd64.deb", + "version": "1.9.4-1build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libbz2-dev_1.0.8-5.1build0.1_amd64", + "name": "libbz2-dev", + "sha256": "012b1118932f20ae3fa706fa44f8ebe203a21f4765893bc7a9f6861aa09fa4c5", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/b/bzip2/libbz2-dev_1.0.8-5.1build0.1_amd64.deb", + "version": "1.0.8-5.1build0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libarchive13t64_3.7.2-2ubuntu0.5_amd64", + "name": "libarchive13t64", + "sha256": "be107a1b001bab8cf4810d00f2235c14463d4c7870ad1abcc9a04ceca00f5c9f", + "url": "https://snapshot.ubuntu.com/ubuntu/20251204T000000Z/pool/main/liba/libarchive/libarchive13t64_3.7.2-2ubuntu0.5_amd64.deb", + "version": "3.7.2-2ubuntu0.5" + }, { "arch": "amd64", "dependencies": [ diff --git a/bazel/noble.yaml b/bazel/noble.yaml index b8fcdec0d373..ff01138fc46e 100644 --- a/bazel/noble.yaml +++ b/bazel/noble.yaml @@ -30,6 +30,7 @@ packages: - "e2fsprogs" # for mkfs.ext4 used in ICOS device tests - "gawk" # for build-bootstrap-config-image - "gzip" # for tar-ing up ic regsitry store in systests + - "libarchive-dev" # for testing building filesystem images from tar (mkfs dependency) - "libcryptsetup-dev" - "libssl3t64" - "libunwind8" diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 95c5c62c27ca..4bb8772787ed 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -1401,6 +1401,10 @@ crate.spec( package = "secp256k1", version = "^0.22", ) +crate.spec( + package = "selinux", + version = "^0.5", +) crate.spec( features = [ "serde", @@ -1816,6 +1820,10 @@ crate.spec( package = "walkdir", version = "^2.3.1", ) +crate.spec( + package = "xattr", + version = "^1.6.1", +) crate.spec( package = "walrus", version = "^0.23.3", @@ -2090,6 +2098,16 @@ crate.annotation( "opt-level=3", ], ) +crate.annotation_select( + build_script_data = ["@@_main~_repo_rules~libselinux//:libselinux"], + build_script_env = { + "SELINUX_INCLUDE_DIR": "/usr/include", + "SELINUX_LIB_DIR": "/usr/lib/x86_64-linux-gnu", + }, + crate = "selinux-sys", + triples = ["x86_64-unknown-linux-gnu"], + deps = ["@@_main~_repo_rules~libselinux//:libselinux"], +) crate.annotation( build_script_env = { "CFLAGS": "-fdebug-prefix-map=$${pwd}=/source", diff --git a/ci/container/TAG b/ci/container/TAG index d2da3fdf2103..172537796abc 100644 --- a/ci/container/TAG +++ b/ci/container/TAG @@ -1 +1 @@ -d88b0bd827eb97cc1638efdb41846c3dbdcad4eca43891216fef16dc72d07092 +7ec1135d78905670bd6d57e5e007ab4eee13db9ed2ef3b93a002682768e449a7 diff --git a/ci/container/files/packages.common b/ci/container/files/packages.common index d9d593e9277f..1963db045749 100644 --- a/ci/container/files/packages.common +++ b/ci/container/files/packages.common @@ -56,6 +56,8 @@ faketime grub-efi-amd64-bin iasl # to build OVMF iputils-ping + # Supports tar file handling in IC-OS build (mkfs dependency) +libarchive-dev # Linked in by IC-OS binaries for managing encrypted disks. libcryptsetup-dev # Linked in by IC-OS binaries for creating mapped devices. diff --git a/rs/ic_os/build_tools/build_filesystem/BUILD.bazel b/rs/ic_os/build_tools/build_filesystem/BUILD.bazel new file mode 100644 index 000000000000..63b4503e9f09 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") + +package(default_visibility = [ + "//rs:ic-os-pkg", + "//rs/tests/node:__subpackages__", + "//toolchains/sysimage:__pkg__", +]) + +rust_binary( + name = "build_filesystem", + srcs = glob(["src/**/*.rs"]), + target_compatible_with = [ + "@platforms//os:linux", + ], + version = "0.1.0", + deps = [ + # Keep sorted. + "@crate_index//:anyhow", + "@crate_index//:clap", + "@crate_index//:regex", + "@crate_index//:selinux", + "@crate_index//:tar", + "@crate_index//:tempfile", + ], +) + +rust_test( + name = "build_filesystem_test", + crate = ":build_filesystem", + data = ["@e2fsprogs//:mke2fs"], + env = { + "MKE2FS_BIN": "$(rootpath @e2fsprogs//:mke2fs)", + }, + # This test needs root, so we mark it manual here and expose it in //rs/tests/node:root_tests + tags = [ + "manual", + ], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + "//rs/ic_os/device", + "@crate_index//:xattr", + ], +) + +filegroup( + name = "build_filesystem_test_with_deps", + testonly = True, + srcs = [ + ":build_filesystem_test", + "@e2fsprogs//:mke2fs", + ], + tags = ["manual"], + visibility = ["//rs/tests/node:__subpackages__"], +) diff --git a/rs/ic_os/build_tools/build_filesystem/Cargo.toml b/rs/ic_os/build_tools/build_filesystem/Cargo.toml new file mode 100644 index 000000000000..28afb8aef390 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "build_filesystem" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +regex = { workspace = true } +selinux = { workspace = true } +tar = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +ic_device = { path = "../../device" } +xattr = { workspace = true } + diff --git a/rs/ic_os/build_tools/build_filesystem/src/ext4.rs b/rs/ic_os/build_tools/build_filesystem/src/ext4.rs new file mode 100644 index 000000000000..2eba84d4e53e --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/ext4.rs @@ -0,0 +1,89 @@ +use crate::fs_builder::{FileEntry, FilesystemBuilder}; +use crate::partition_size::PartitionSize; +use crate::tar::TarBuilder; +use anyhow::{Context, Result, ensure}; +use std::path::PathBuf; +use std::process::Command; +use tar::Builder; +use tempfile::NamedTempFile; + +/// Implementation of FilesystemBuilder for ext4 filesystems +/// +/// This builder creates a tar file using TarBuilder, then converts it to an ext4 +/// filesystem image using mke2fs. +pub struct Ext4Builder { + tar_builder: TarBuilder, + output_path: PathBuf, + partition_size: PartitionSize, + label: Option, + mke2fs_path: Option, +} + +impl Ext4Builder { + /// Create a new Ext4Builder + /// + /// * `output_path` - Path where the final ext4 image will be written + /// * `partition_size` - Size of the partition + /// * `label` - Optional volume label for the filesystem + /// * `mke2fs_path` - Optional path to mke2fs binary (defaults to system mke2fs) + pub fn new( + output_path: impl Into, + partition_size: PartitionSize, + label: Option, + mke2fs_path: Option, + ) -> Result { + let tar_file = NamedTempFile::new().context("Failed to create temporary tar file")?; + let tar_builder = TarBuilder::new(Builder::new(tar_file)); + + Ok(Self { + tar_builder, + output_path: output_path.into(), + partition_size, + label, + mke2fs_path, + }) + } +} + +impl FilesystemBuilder for Ext4Builder { + fn append_entry(&mut self, entry: FileEntry<'_>) -> Result<()> { + self.tar_builder.append_entry(entry) + } + + fn finish(self: Box) -> Result<()> { + let mut tar_builder = self.tar_builder.into_inner(); + tar_builder.finish()?; + let tar_file = tar_builder.into_inner()?; + + let mke2fs_binary = self + .mke2fs_path + .as_deref() + .unwrap_or_else(|| std::path::Path::new("mke2fs")); + + let output = Command::new(mke2fs_binary) + .arg("-t") + .arg("ext4") + .arg("-E") + .arg("hash_seed=c61251eb-100b-48fe-b089-57dea7368612") + .arg("-U") + .arg("clear") + .arg("-d") + .arg(tar_file.path()) + .arg("-F") + .arg(&self.output_path) + .arg(self.partition_size.as_kb()?.to_string()) + .arg("-L") + .arg(self.label.as_deref().unwrap_or("")) + .env("SOURCE_DATE_EPOCH", "0") + .output() + .context("Failed to execute mke2fs")?; + + ensure!(output.status.success(), "mke2fs failed {output:?}"); + + Ok(()) + } + + fn needs_lost_found(&self) -> bool { + true + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/fat.rs b/rs/ic_os/build_tools/build_filesystem/src/fat.rs new file mode 100644 index 000000000000..036348413fad --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/fat.rs @@ -0,0 +1,190 @@ +use crate::fs_builder::{FileEntry, FilesystemBuilder}; +use crate::partition_size::PartitionSize; +use anyhow::{Context, Result, bail, ensure}; +use std::fs::{File, FileTimes}; +use std::io::Write; +use std::path::PathBuf; +use std::process::Command; +use std::time::{Duration, SystemTime}; +use tempfile::{NamedTempFile, TempDir}; + +/// Returns the minimum time for FAT filesystems +pub fn fat_min_time() -> SystemTime { + // 1980-01-01 00:00:00 UTC + SystemTime::UNIX_EPOCH + Duration::from_secs(315532800) +} + +/// FAT filesystem type +#[derive(Debug, Clone, Copy)] +pub enum FatType { + /// VFAT filesystem + Vfat, + /// FAT32 filesystem + Fat32, +} + +/// Implementation of FilesystemBuilder for FAT filesystems (VFAT and FAT32) +/// +/// This builder creates a FAT filesystem image using mkfs.vfat and copies +/// files directly using mcopy for each entry. +pub struct FatBuilder { + output_path: PathBuf, + partition_size: PartitionSize, + fat_type: FatType, + label: Option, + initialized: bool, +} + +impl FatBuilder { + /// Create a new FatBuilder + /// + /// * `output_path` - Path where the final FAT image will be written + /// * `partition_size` - Size of the partition + /// * `fat_type` - Type of FAT filesystem (VFAT or FAT32) + /// * `label` - Optional volume label for the filesystem + pub fn new( + output_path: impl Into, + partition_size: PartitionSize, + fat_type: FatType, + label: Option, + ) -> Result { + Ok(Self { + output_path: output_path.into(), + partition_size, + fat_type, + label, + initialized: false, + }) + } + + /// Initialize the FAT filesystem if not already done + fn ensure_initialized(&mut self) -> Result<()> { + if !self.initialized { + if self.output_path.exists() { + std::fs::remove_file(&self.output_path)?; + } + + let mut cmd = Command::new("/usr/sbin/mkfs.vfat"); + + // Add FAT type flag if FAT32 + if matches!(self.fat_type, FatType::Fat32) { + cmd.arg("-F").arg("32"); + } + + // Add volume label if provided + if let Some(ref label) = self.label { + cmd.arg("-n").arg(label); + } + + cmd.arg("-C") + .arg(&self.output_path) + .arg(self.partition_size.as_kb()?.to_string()) + .env("SOURCE_DATE_EPOCH", "0"); + + let output = cmd.output().context("Failed to execute mkfs.vfat")?; + + ensure!(output.status.success(), "mkfs.vfat failed {output:?}"); + self.initialized = true; + } + Ok(()) + } +} + +impl FilesystemBuilder for FatBuilder { + fn append_entry(&mut self, entry: FileEntry<'_>) -> Result<()> { + self.ensure_initialized()?; + + let entry_type = entry.header.entry_type(); + let entry_path = entry.path; + let fat_path = format!("::{}", entry_path.as_relative_path().display()); + + // Skip root directory + if entry_path.is_root() { + return Ok(()); + } + + match entry_type { + tar::EntryType::Directory => { + let dir = TempDir::new().context("Failed to create temporary directory")?; + File::open(dir.path())?.set_times( + FileTimes::new() + .set_modified(fat_min_time()) + .set_accessed(fat_min_time()), + )?; + // Copy directory using mcopy + let output = Command::new("mcopy") + .arg("-m") + .arg("-i") + .arg(&self.output_path) + .arg(dir.path()) + .arg(&fat_path) + .output() + .with_context(|| { + format!( + "Failed to execute mcopy for {:?}", + entry_path.as_relative_path() + ) + })?; + + ensure!( + output.status.success(), + "mcopy failed for {:?}: {output:?}", + entry_path.as_relative_path() + ); + } + tar::EntryType::Regular => { + // Create a temporary file with the contents + let mut temp_file = + NamedTempFile::new().context("Failed to create temporary file")?; + std::io::copy(entry.contents, &mut temp_file).with_context(|| { + format!( + "Failed to write temporary file for {:?}", + entry_path.as_relative_path() + ) + })?; + temp_file.flush()?; + + temp_file.as_file_mut().set_times( + FileTimes::new() + .set_modified(fat_min_time()) + .set_accessed(fat_min_time()), + )?; + + // Copy file using mcopy + let output = Command::new("mcopy") + .arg("-m") + .arg("-i") + .arg(&self.output_path) + .arg(temp_file.path()) + .arg(&fat_path) + .output() + .with_context(|| { + format!( + "Failed to execute mcopy for {:?}", + entry_path.as_relative_path() + ) + })?; + + ensure!( + output.status.success(), + "mcopy failed for {:?}: {output:?}", + entry_path.as_relative_path() + ); + } + _ => { + bail!("Unsupported entry type: {:?}", entry_type); + } + } + + Ok(()) + } + + fn finish(self: Box) -> Result<()> { + // Nothing to do - all files were already copied via mcopy + Ok(()) + } + + fn needs_lost_found(&self) -> bool { + false + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/fs_builder.rs b/rs/ic_os/build_tools/build_filesystem/src/fs_builder.rs new file mode 100644 index 000000000000..635ebecb9f31 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/fs_builder.rs @@ -0,0 +1,49 @@ +use crate::path_converter::ImagePath; +use anyhow::Result; +use std::ffi::CString; +use std::io::Read; +use tar::Header; + +/// Represents a filesystem entry with all its metadata +pub struct FileEntry<'a> { + /// Path of the entry in the filesystem + pub path: ImagePath, + /// Header containing all metadata (mode, size, uid, gid, mtime, entry type, etc.) + // Reusing tar::Header is convenient because it already includes all the necessary fields and + // makes it easy to push entries from a tar file to another tar file. + pub header: Header, + /// Contents of the file (empty for directories) + pub contents: &'a mut (dyn Read + 'a), + /// SELinux security context (if specified) + pub selinux_context: Option, +} + +impl<'a> FileEntry<'a> { + /// Create a new file entry + pub fn new(path: ImagePath, header: Header, contents: &'a mut (dyn Read + 'a)) -> Self { + Self { + path, + header, + contents, + selinux_context: None, + } + } + + /// Set the SELinux context for this entry + pub fn with_selinux_context(mut self, context: Option) -> Self { + self.selinux_context = context; + self + } +} + +/// Trait for building different types of filesystems +pub trait FilesystemBuilder: Send { + /// Append a file entry to the filesystem + fn append_entry(&mut self, entry: FileEntry<'_>) -> Result<()>; + + /// Finalize the filesystem and flush any pending data + fn finish(self: Box) -> Result<()>; + + /// Whether the filesystem needs a lost+found directory + fn needs_lost_found(&self) -> bool; +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/integration_tests.rs b/rs/ic_os/build_tools/build_filesystem/src/integration_tests.rs new file mode 100644 index 000000000000..1c7e870726a0 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/integration_tests.rs @@ -0,0 +1,761 @@ +#![cfg(test)] + +use crate::fat::fat_min_time; +use crate::{Args, OutputType, build_filesystem}; +use ic_device::mount::{FileSystem, LoopDeviceMounter, MountOptions, Mounter}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::thread::sleep; +use std::time::{Duration, SystemTime}; +use tempfile::{NamedTempFile, TempDir, TempPath}; + +fn get_mke2fs_path() -> PathBuf { + PathBuf::from(std::env::var("MKE2FS_BIN").unwrap()) +} + +/// Test fixture for creating filesystem images and mounting them +struct ImageFixture { + /// The path where the image/tar is generated + output_path: TempPath, + output_type: OutputType, +} + +/// Builder for creating ImageFixture with custom arguments +struct ImageFixtureBuilder { + output_type: OutputType, + partition_size: Option, + label: Option, + subdir: Option, + file_contexts: Option, + strip_paths: Vec, + extra_files: Vec, + tar_builder: Option>>, + output_extension: &'static str, +} + +impl ImageFixtureBuilder { + fn new(output_type: OutputType) -> Self { + let output_extension = match output_type { + OutputType::Tar => "tar", + _ => "img", + }; + Self { + output_type, + partition_size: None, + label: None, + subdir: None, + file_contexts: None, + strip_paths: Vec::new(), + extra_files: Vec::new(), + tar_builder: None, + output_extension, + } + } + + fn partition_size(mut self, size: &str) -> Self { + self.partition_size = Some(size.to_string()); + self + } + + fn partition_size_if_not_tar(mut self, size: &str) -> Self { + if self.output_type != OutputType::Tar { + self.partition_size = Some(size.to_string()); + } + self + } + + fn label(mut self, label: &str) -> Self { + self.label = Some(label.to_string()); + self + } + + fn subdir(mut self, subdir: &str) -> Self { + self.subdir = Some(PathBuf::from(subdir)); + self + } + + fn file_contexts(mut self, path: PathBuf) -> Self { + self.file_contexts = Some(path); + self + } + + fn strip_path(mut self, path: &str) -> Self { + self.strip_paths.push(path.to_string()); + self + } + + fn extra_file(mut self, file: &str) -> Self { + self.extra_files.push(file.to_string()); + self + } + + fn tar_content(mut self, builder: tar::Builder>) -> Self { + self.tar_builder = Some(builder); + self + } + + fn build(self) -> ImageFixture { + let output_path = NamedTempFile::with_suffix(format!(".{}", self.output_extension)) + .unwrap() + .into_temp_path(); + + let input_tar = if let Some(builder) = self.tar_builder { + let mut tar = NamedTempFile::new().unwrap(); + let tar_data = builder.into_inner().unwrap(); + tar.write_all(&tar_data).unwrap(); + Some(tar) + } else { + None + }; + + let parsed_extra_files = self + .extra_files + .iter() + .map(|s| s.parse().unwrap_or_else(|_| panic!("Cannot parse {s}"))) + .collect(); + + build_filesystem(Args { + output: output_path.to_path_buf(), + input: input_tar.as_ref().map(|t| t.path().to_path_buf()), + output_type: self.output_type, + partition_size: self + .partition_size + .as_ref() + .map(|s| s.parse()) + .transpose() + .unwrap(), + label: self.label, + subdir: self.subdir, + file_contexts: self.file_contexts, + strip_paths: self.strip_paths, + extra_files: parsed_extra_files, + mke2fs_path: Some(get_mke2fs_path()), + }) + .unwrap(); + + ImageFixture { + output_path, + output_type: self.output_type, + } + } +} + +impl ImageFixture { + fn builder(output_type: OutputType) -> ImageFixtureBuilder { + ImageFixtureBuilder::new(output_type) + } + + fn path(&self) -> &Path { + &self.output_path + } + + /// Convert OutputType to FileSystem for mounting + fn filesystem_type(&self) -> FileSystem { + match self.output_type { + OutputType::Ext4 => FileSystem::Ext4, + OutputType::Vfat | OutputType::Fat32 => FileSystem::Vfat, + OutputType::Tar => panic!("No filesystem type for tar"), + } + } + + /// Mount the image + /// For tar files, this extracts the tar to a temporary directory + /// For filesystem images, this mounts them using a loop device + fn mount(&self) -> MountedImage { + match self.output_type { + OutputType::Tar => MountedImage::extract_tar(self.path()), + _ => MountedImage::mount_loop(self.path(), self.filesystem_type()), + } + } + + /// Extract image from zst and mount it + fn mount_from_zst(&self) -> MountedImage { + let extracted_partition_img = NamedTempFile::new().unwrap().into_temp_path(); + + let output = Command::new("zstd") + .args(["--quiet", "--force", "-d"]) + .arg(self.path()) + .arg("-o") + .arg(&extracted_partition_img) + .output() + .unwrap(); + + assert!(output.status.success(), "zstd decompression failed"); + + MountedImage::mount_loop_from_temp(extracted_partition_img, self.filesystem_type()) + } +} + +/// Helper to mount an image and verify contents +/// For tar files, this extracts to a temp directory +/// For filesystem images, this mounts using a loop device +enum MountedImage { + // A file mounted using a loop device + LoopMounted { + mount: Box, + }, + // A file that was extracted from a tar and then mounted using a loop device + LoopMountedFromTemp { + mount: Box, + // We keep the extracted and mounted disk image alive + _extracted_partition_img: TempPath, + }, + // A tar file that was extracted to a temp directory + ExtractedTar { + extracted_tar_dir: TempDir, + }, +} + +impl MountedImage { + /// Mount a filesystem image using a loop device + fn mount_loop(image_path: &Path, fs_type: FileSystem) -> Self { + assert!( + image_path.exists(), + "Image file does not exist: {}", + image_path.display() + ); + + let mount = LoopDeviceMounter + .mount_range( + image_path.to_path_buf(), + 0, + fs::metadata(image_path).unwrap().len(), + MountOptions { + file_system: fs_type, + }, + ) + .unwrap(); + + MountedImage::LoopMounted { mount } + } + + /// Mount a filesystem image using a loop device, keeping the extracted partition image alive + fn mount_loop_from_temp(extracted_partition_img: TempPath, fs_type: FileSystem) -> Self { + let mount = LoopDeviceMounter + .mount_range( + extracted_partition_img.to_path_buf(), + 0, + fs::metadata(&extracted_partition_img).unwrap().len(), + MountOptions { + file_system: fs_type, + }, + ) + .unwrap(); + + MountedImage::LoopMountedFromTemp { + mount, + _extracted_partition_img: extracted_partition_img, + } + } + + /// Extract a tar file to a temporary directory + fn extract_tar(tar_path: &Path) -> Self { + use std::process::Command; + + let temp_dir = TempDir::new().unwrap(); + + let output = Command::new("tar") + .arg("-xaf") + .arg(tar_path) + .arg("--selinux") + .arg("--same-owner") + .arg("-C") + .arg(temp_dir.path()) + .output() + .unwrap(); + + assert!(output.status.success(), "tar extraction failed"); + + MountedImage::ExtractedTar { + extracted_tar_dir: temp_dir, + } + } + + fn mount_point(&self) -> &Path { + match self { + MountedImage::LoopMounted { mount } => mount.mount_point(), + MountedImage::LoopMountedFromTemp { mount, .. } => mount.mount_point(), + MountedImage::ExtractedTar { + extracted_tar_dir: temp_dir, + } => temp_dir.path(), + } + } + + /// Assert file exists with expected content + fn assert_file_content(&self, path: &str, expected: &str) { + let file_path = self.mount_point().join(path); + let content = fs::read_to_string(&file_path).unwrap(); + assert_eq!(content, expected, "File {} has wrong content", path); + } + + /// Assert file does not exist + fn assert_file_not_exists(&self, path: &str) { + let file_path = self.mount_point().join(path); + assert!(!file_path.exists(), "File {} should not exist", path); + } + + /// Assert directory exists + fn assert_dir_exists(&self, path: &str) { + let dir_path = self.mount_point().join(path); + assert!(dir_path.is_dir(), "Directory {} does not exist", path); + } + + /// Assert file has specific permissions + fn assert_permissions(&self, path: &str, expected_mode: u32) { + use std::os::unix::fs::PermissionsExt; + let file_path = self.mount_point().join(path); + let metadata = fs::metadata(&file_path).unwrap(); + let mode = metadata.permissions().mode() & 0o777; + assert_eq!( + mode, expected_mode, + "File {} has wrong permissions: {:o} (expected {:o})", + path, mode, expected_mode + ); + } + + /// Assert file has specific ownership + fn assert_ownership(&self, path: &str, expected_uid: u32, expected_gid: u32) { + use std::os::unix::fs::MetadataExt; + let file_path = self.mount_point().join(path); + let metadata = fs::metadata(&file_path).unwrap(); + assert_eq!(metadata.uid(), expected_uid, "File {} has wrong uid", path); + assert_eq!(metadata.gid(), expected_gid, "File {} has wrong gid", path); + } +} + +fn all_types() -> [OutputType; 4] { + [ + OutputType::Tar, + OutputType::Ext4, + OutputType::Vfat, + OutputType::Fat32, + ] +} + +fn append_file(tar: &mut tar::Builder>, path: &str, content: &[u8], mode: u32) { + let mut header = tar::Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_mode(mode); + header.set_cksum(); + tar.append_data(&mut header, path, content).unwrap(); +} + +fn append_dir(tar: &mut tar::Builder>, path: &str, mode: u32) { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(mode); + header.set_cksum(); + tar.append_data(&mut header, path, &[] as &[u8]).unwrap(); +} + +fn simple_tar() -> tar::Builder> { + let mut tar = tar::Builder::new(Vec::new()); + append_file(&mut tar, "file1.txt", "test content".as_bytes(), 0o644); + tar +} + +fn simple_tar_with_subdir() -> tar::Builder> { + let mut tar = tar::Builder::new(Vec::new()); + append_file(&mut tar, "file1.txt", "test content".as_bytes(), 0o644); + append_dir(&mut tar, "subdir", 0o755); + append_file( + &mut tar, + "subdir/file2.txt", + "nested content".as_bytes(), + 0o644, + ); + tar +} + +fn simple_tar_with_empty_dir() -> tar::Builder> { + let mut tar = simple_tar_with_subdir(); + append_dir(&mut tar, "emptydir", 0o755); + tar +} + +#[test] +fn test_basic_files_and_dirs() { + for output_type in all_types() { + println!("Testing output type: {:?}", output_type); + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .tar_content(simple_tar_with_empty_dir()) + .build(); + + let mounted = image.mount(); + + mounted.assert_file_content("file1.txt", "test content"); + mounted.assert_file_content("subdir/file2.txt", "nested content"); + mounted.assert_dir_exists("emptydir"); + mounted.assert_dir_exists("subdir"); + + if output_type != OutputType::Tar { + let metadata = fs::metadata(image.path()).unwrap(); + assert_eq!( + metadata.len(), + 4 * 1024 * 1024, + "Image size mismatch for {output_type:?} with partition_size 4M", + ); + } + } +} + +#[test] +fn test_label() { + for output_type in [OutputType::Vfat, OutputType::Fat32, OutputType::Ext4] { + println!("Testing output type: {:?}", output_type); + let label = format!("LBL{output_type:?}"); + let image = ImageFixture::builder(output_type) + .partition_size("4M") + .label(&label) + .tar_content(simple_tar()) + .build(); + + let mounted = image.mount(); + mounted.assert_file_content("file1.txt", "test content"); + + assert!( + (0..20).any(|_| { + sleep(Duration::from_millis(100)); + Path::new(&format!("/dev/disk/by-label/{label}")).exists() + }), + "Label {} not found after 2 seconds", + label + ); + } +} + +#[test] +fn test_subdir_extraction() { + for output_type in all_types() { + println!("Testing output type: {:?}", output_type); + let mut tar = tar::Builder::new(Vec::new()); + + append_file(&mut tar, "file1.txt", "test content".as_bytes(), 0o644); + append_file( + &mut tar, + "subdir/file2.txt", + "nested content".as_bytes(), + 0o644, + ); + + let builder = ImageFixture::builder(output_type) + .subdir("/subdir") + .tar_content(tar) + .partition_size_if_not_tar("4M"); + + let image = builder.build(); + + let mounted = image.mount(); + mounted.assert_file_content("file2.txt", "nested content"); + mounted.assert_file_not_exists("file1.txt"); + mounted.assert_file_not_exists("subdir/file2.txt"); + } +} + +#[test] +fn test_strip_paths() { + for output_type in all_types() { + println!("Testing output type: {:?}", output_type); + let mut tar = tar::Builder::new(Vec::new()); + append_file(&mut tar, "keep1.txt", "keep this".as_bytes(), 0o644); + append_file(&mut tar, "remove1.txt", "remove this".as_bytes(), 0o644); + append_dir(&mut tar, "keepdir", 0o755); + append_file( + &mut tar, + "keepdir/keep2.txt", + "keep nested".as_bytes(), + 0o644, + ); + append_file( + &mut tar, + "keepdir/remove2.txt", + "remove nested".as_bytes(), + 0o644, + ); + append_dir(&mut tar, "removedir", 0o755); + append_file( + &mut tar, + "removedir/file.txt", + "remove entire dir".as_bytes(), + 0o644, + ); + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .strip_path("/remove1.txt") + .strip_path("/keepdir/remove2.txt") + .strip_path("/removedir/.*") + .tar_content(tar) + .build(); + + let mounted = image.mount(); + + mounted.assert_file_content("keep1.txt", "keep this"); + mounted.assert_file_content("keepdir/keep2.txt", "keep nested"); + mounted.assert_file_not_exists("remove1.txt"); + mounted.assert_file_not_exists("keepdir/remove2.txt"); + mounted.assert_file_not_exists("removedir/file.txt"); + mounted.assert_dir_exists("removedir"); + } +} + +#[test] +fn test_extra_files() { + for output_type in all_types() { + println!("Testing output type: {:?}", output_type); + let temp_dir = TempDir::new().unwrap(); + let extra_file = temp_dir.path().join("extra.txt"); + fs::write(&extra_file, "extra content").unwrap(); + + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .extra_file(&format!("{}:/extra.txt:0644", extra_file.display())) + .tar_content(simple_tar()) + .build(); + + let mounted = image.mount(); + mounted.assert_file_content("file1.txt", "test content"); + mounted.assert_file_content("extra.txt", "extra content"); + } +} + +#[test] +fn test_mtime_set() { + for output_type in all_types() { + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .tar_content(simple_tar_with_subdir()) + .build(); + let mounted = image.mount(); + + let expected_mtime = match output_type { + OutputType::Fat32 | OutputType::Vfat => fat_min_time(), + _ => SystemTime::UNIX_EPOCH, + }; + + for path in &["file1.txt", "subdir/file2.txt", "subdir"] { + let metadata = fs::metadata(mounted.mount_point().join(path)).unwrap(); + assert_eq!( + metadata.modified().unwrap(), + expected_mtime, + "{path} mtime should match", + ); + } + } +} + +#[test] +fn test_symlinks() { + for output_type in [OutputType::Ext4, OutputType::Tar] { + println!("Testing output type: {:?}", output_type); + let mut tar = tar::Builder::new(Vec::new()); + + let mut header = tar::Header::new_gnu(); + header.set_size(11); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, "target.txt", "test target".as_bytes()) + .unwrap(); + + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_cksum(); + tar.append_link(&mut header, "link.txt", "target.txt") + .unwrap(); + + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .tar_content(tar) + .build(); + + let mounted = image.mount(); + let link_path = mounted.mount_point().join("link.txt"); + assert!(link_path.exists(), "Symlink should exist"); + + let metadata = fs::symlink_metadata(&link_path).unwrap(); + assert!( + metadata.file_type().is_symlink(), + "link.txt should be a symlink" + ); + + let target = fs::read_link(&link_path).unwrap(); + assert_eq!( + target, + PathBuf::from("target.txt"), + "Symlink target should be target.txt" + ); + } +} + +#[test] +fn test_permissions_preserved() { + for output_type in [OutputType::Ext4, OutputType::Tar] { + println!("Testing output type: {:?}", output_type); + let mut tar = tar::Builder::new(Vec::new()); + + append_file(&mut tar, "script.sh", "script".as_bytes(), 0o755); + append_file(&mut tar, "readonly.txt", "readonly".as_bytes(), 0o444); + append_file(&mut tar, "writable.txt", "writable".as_bytes(), 0o644); + + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .tar_content(tar) + .build(); + + let mounted = image.mount(); + + mounted.assert_permissions("script.sh", 0o755); + mounted.assert_permissions("readonly.txt", 0o444); + mounted.assert_permissions("writable.txt", 0o644); + } +} + +#[test] +fn test_ownership_preserved() { + for output_type in [OutputType::Ext4, OutputType::Tar] { + println!("Testing output type: {:?}", output_type); + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .tar_content(simple_tar_with_subdir()) + .build(); + let mounted = image.mount(); + + mounted.assert_ownership("file1.txt", 0, 0); + mounted.assert_ownership("subdir/file2.txt", 0, 0); + mounted.assert_ownership("subdir", 0, 0); + } +} + +#[test] +fn test_selinux_labels() { + for output_type in [OutputType::Ext4, OutputType::Tar] { + println!("Testing output type: {:?}", output_type); + let temp_dir = TempDir::new().unwrap(); + let file_contexts = temp_dir.path().join("file_contexts"); + + fs::write( + &file_contexts, + "\ + / system_u:object_r:root_t:s0\n\ + /file1\\.txt system_u:object_r:user_home_t:s0\n\ + /lost\\+found system_u:object_r:lost_found_t:s0\n\ + /subdir(/.*)? system_u:object_r:var_t:s0\n", + ) + .unwrap(); + + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .file_contexts(file_contexts) + .tar_content(simple_tar_with_subdir()) + .build(); + let mounted = image.mount(); + + assert_eq!( + xattr::get(mounted.mount_point().join("file1.txt"), "security.selinux") + .unwrap() + .unwrap(), + b"system_u:object_r:user_home_t:s0\0" + ); + + assert_eq!( + xattr::get(mounted.mount_point().join("subdir"), "security.selinux") + .unwrap() + .unwrap(), + b"system_u:object_r:var_t:s0\0" + ); + + assert_eq!( + xattr::get( + mounted.mount_point().join("subdir/file2.txt"), + "security.selinux" + ) + .unwrap() + .unwrap(), + b"system_u:object_r:var_t:s0\0" + ); + + assert_eq!( + xattr::get(mounted.mount_point(), "security.selinux") + .unwrap() + .unwrap(), + b"system_u:object_r:root_t:s0\0" + ); + + // lost+found is only created for ext4 + if output_type == OutputType::Ext4 { + assert_eq!( + xattr::get(mounted.mount_point().join("lost+found"), "security.selinux") + .unwrap() + .unwrap(), + b"system_u:object_r:lost_found_t:s0\0" + ); + } + } +} + +#[test] +fn test_zst_compressed_tar() { + let mut builder = ImageFixture::builder(OutputType::Tar).tar_content(simple_tar()); + builder.output_extension = "tar.zst"; + + let image = builder.build(); + let mounted = image.mount(); + + mounted.assert_file_content("file1.txt", "test content"); +} + +#[test] +fn test_zst_compressed_images() { + for output_type in [OutputType::Ext4, OutputType::Vfat, OutputType::Fat32] { + let mut builder = ImageFixture::builder(output_type) + .partition_size("4M") + .tar_content(simple_tar()); + builder.output_extension = "img.zst"; + + let image = builder.build(); + let mounted = image.mount_from_zst(); + + mounted.assert_file_content("file1.txt", "test content"); + } +} + +#[test] +#[should_panic(expected = "Partition size is required")] +fn test_invalid_partition_size() { + ImageFixture::builder(OutputType::Ext4) + .tar_content(simple_tar()) + .build(); +} + +#[test] +fn test_no_input_tar_with_extra_files() { + for output_type in all_types() { + println!("Testing output type: {:?}", output_type); + let temp_dir = TempDir::new().unwrap(); + let extra_file1 = temp_dir.path().join("extra1.txt"); + let extra_file2 = temp_dir.path().join("extra2.txt"); + fs::write(&extra_file1, "first extra file").unwrap(); + fs::write(&extra_file2, "second extra file").unwrap(); + + let image = ImageFixture::builder(output_type) + .partition_size_if_not_tar("4M") + .extra_file(&format!("{}:/extra1.txt:0644", extra_file1.display())) + .extra_file(&format!("{}:/extra2.txt:0755", extra_file2.display())) + .build(); + + let mounted = image.mount(); + mounted.assert_file_content("extra1.txt", "first extra file"); + mounted.assert_file_content("extra2.txt", "second extra file"); + + // Fat does not support permissions + if output_type != OutputType::Fat32 && output_type != OutputType::Vfat { + mounted.assert_permissions("extra1.txt", 0o644); + mounted.assert_permissions("extra2.txt", 0o755); + } + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/main.rs b/rs/ic_os/build_tools/build_filesystem/src/main.rs new file mode 100644 index 000000000000..9a49f4615841 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/main.rs @@ -0,0 +1,249 @@ +use anyhow::{Context, Result, bail, ensure}; +use clap::{Parser, ValueEnum}; +use regex::RegexSet; +use std::fs::File; +use std::io::BufWriter; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str::FromStr; +use tempfile::NamedTempFile; + +mod ext4; +mod fat; +mod fs_builder; +mod integration_tests; +mod partition_size; +mod path_converter; +mod processor; +mod selinux; +mod tar; + +use crate::partition_size::PartitionSize; +use crate::path_converter::ImagePath; +use ext4::Ext4Builder; +use fat::{FatBuilder, FatType}; +use fs_builder::FilesystemBuilder; +use selinux::FileContexts; +use tar::TarBuilder; + +#[derive(Parser, Debug)] +#[command(about = "Build filesystem images from input tar with transformations")] +#[cfg_attr(test, derive(Clone))] +pub(crate) struct Args { + /// Output file path + #[arg(short = 'o', long)] + pub(crate) output: PathBuf, + + /// Input tar file (optional, if not provided creates empty filesystem) + #[arg(short = 'i', long)] + pub(crate) input: Option, + + /// Output type (tar, ext4, vfat, fat32) + #[arg(short = 't', long, value_enum, default_value = "tar")] + pub(crate) output_type: OutputType, + + /// Partition size (required for ext4, vfat, and fat32, e.g., "100M", "1G") + #[arg(long)] + pub(crate) partition_size: Option, + + /// Volume label (optional, for fat32 filesystems) + #[arg(long)] + pub(crate) label: Option, + + /// Path to extract from input tar (limit to subdirectory) + #[arg(short = 'p', long)] + pub(crate) subdir: Option, + + /// SELinux file_contexts file for setting security contexts + #[arg(short = 'S', long)] + pub(crate) file_contexts: Option, + + /// Paths to remove from the tree + #[arg(long = "strip-paths", num_args = 0..)] + pub(crate) strip_paths: Vec, + + /// Extra files to inject (format: source:target:mode) + #[arg(long = "extra-files", num_args = 0..)] + pub(crate) extra_files: Vec, + + /// Path to mke2fs binary (optional, defaults to system mke2fs) + #[arg(long = "mke2fs")] + pub(crate) mke2fs_path: Option, +} + +#[derive(Debug, Clone, ValueEnum, Eq, PartialEq)] +#[cfg_attr(test, derive(Copy))] +pub(crate) enum OutputType { + Tar, + Ext4, + Vfat, + Fat32, +} + +/// Extra file to inject into the filesystem image +#[derive(Debug, Clone)] +pub(crate) struct ExtraFile { + /// Source file path on the host filesystem + pub(crate) source: PathBuf, + /// Target path in the filesystem image + pub(crate) target: ImagePath, + /// File permissions mode (octal, e.g., 0o644) + pub(crate) mode: u32, +} + +impl FromStr for ExtraFile { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let parts: Vec<&str> = s.split(':').collect(); + if parts.len() != 3 { + bail!("Invalid extra file format: {s}. Expected source:target:mode"); + } + + let source = PathBuf::from(parts[0]); + let target = ImagePath::from(parts[1]); + let mode = u32::from_str_radix(parts[2], 8) + .with_context(|| format!("Invalid mode in extra file: {}", parts[2]))?; + + Ok(ExtraFile { + source, + target, + mode, + }) + } +} + +fn main() -> Result<()> { + let args = Args::parse(); + build_filesystem(args) +} + +/// Main build_filesystem logic that can be called programmatically +pub(crate) fn build_filesystem(args: Args) -> Result<()> { + let output_str = args.output.to_str().unwrap_or_default(); + + if args.output_type == OutputType::Tar { + let tar_extensions = [".tar", ".tar.zst", ".tzst"]; + ensure!( + tar_extensions.iter().any(|ext| output_str.ends_with(ext)), + "Output file for tar must have one of the following extensions: {}", + tar_extensions.join(", ") + ); + ensure!( + args.label.is_none(), + "Volume label is not allowed for tar output" + ); + ensure!( + args.partition_size.is_none(), + "Partition size is not allowed for tar output" + ); + } else { + let extensions = [".img", ".img.zst"]; + ensure!( + extensions.iter().any(|ext| output_str.ends_with(ext)), + "Output file for raw image must have one of the following extensions: {}", + extensions.join(", ") + ); + } + + if !args.strip_paths.is_empty() && args.subdir.is_some() { + // There is no real reason not to allow these options together. However, we need to + // figure out if strip_paths should contain paths relative to subdir or paths relative to + // the root of the input tar. + bail!( + "Cannot use --strip-paths and --subdir together, if you need it, please \ + implement it" + ); + } + + // Validate input exists if provided + if let Some(input) = &args.input { + ensure!(input.exists(), "Input file does not exist: {input:?}"); + } + + let file_contexts = args + .file_contexts + .map(|path| FileContexts::new(&path)) + .transpose()?; + + let strip_paths = RegexSet::new(args.strip_paths.iter().map(|s| { + assert!(s.starts_with('/'), "strip path must start with /"); + // RegexSet matches anywhere in the string, so we anchor it + format!("^{s}$") + }))?; + + let needs_compression = output_str.ends_with(".zst") || output_str.ends_with(".tzst"); + let _temp_path; + // If compression is required, create a temporary file first, then compress it later + let image_path: &Path = if needs_compression { + _temp_path = NamedTempFile::new()?.into_temp_path(); + _temp_path.as_ref() + } else { + &args.output + }; + + let mut output_builder: Box = match args.output_type { + OutputType::Tar => { + let output_file = File::create(image_path) + .with_context(|| format!("Failed to create output file {:?}", image_path))?; + let tar_builder = ::tar::Builder::new(BufWriter::new(output_file)); + Box::new(TarBuilder::new(tar_builder)) + } + OutputType::Ext4 => { + let partition_size = args + .partition_size + .context("Partition size is required for ext4")?; + Box::new(Ext4Builder::new( + image_path, + partition_size, + args.label, + args.mke2fs_path.clone(), + )?) + } + OutputType::Vfat => { + let partition_size = args + .partition_size + .context("Partition size is required for vfat")?; + Box::new(FatBuilder::new( + image_path, + partition_size, + FatType::Vfat, + args.label, + )?) + } + OutputType::Fat32 => { + let partition_size = args + .partition_size + .context("Partition size is required for fat32")?; + Box::new(FatBuilder::new( + image_path, + partition_size, + FatType::Fat32, + args.label, + )?) + } + }; + + processor::process_filesystem( + args.input.as_deref(), + output_builder.as_mut(), + args.subdir.as_deref(), + &strip_paths, + &args.extra_files, + &file_contexts, + )?; + + output_builder.finish()?; + + if needs_compression { + let output = Command::new("zstd") + .args(["-T0", "--quiet", "--force", "-o"]) + .arg(&args.output) + .arg(image_path) + .output() + .context("Failed to run zstd")?; + ensure!(output.status.success(), "compression failed: {output:?}"); + } + + Ok(()) +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/partition_size.rs b/rs/ic_os/build_tools/build_filesystem/src/partition_size.rs new file mode 100644 index 000000000000..9b51874d8a43 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/partition_size.rs @@ -0,0 +1,102 @@ +use anyhow::{Result, ensure}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct PartitionSize(u64); + +impl PartitionSize { + pub fn as_kb(&self) -> Result { + ensure!( + self.0.is_multiple_of(1024), + "Partition size must be a multiple of 1024" + ); + + Ok(self.0 / 1024) + } +} + +impl std::str::FromStr for PartitionSize { + type Err = String; + + /// Parse a size string like "50M", "1000K", "3G" and return the size in bytes + fn from_str(s: &str) -> Result { + let size = s.trim(); + if size.is_empty() { + return Err("Size string is empty".to_string()); + } + + let (number_part, suffix) = if let Some(pos) = size.find(|c: char| c.is_alphabetic()) { + (&size[..pos], &size[pos..]) + } else { + (size, "") + }; + + let number: u64 = number_part + .parse() + .map_err(|_| format!("Failed to parse number from: {size}"))?; + + let multiplier = match suffix.to_uppercase().as_str() { + "" | "B" => 1, + "K" => 1024, + "M" => 1024 * 1024, + "G" => 1024 * 1024 * 1024, + _ => return Err(format!("Unsupported size suffix: {suffix}")), + }; + + Ok(Self(number * multiplier)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_size_to_bytes() { + assert_eq!( + "50M".parse::().unwrap(), + PartitionSize(50 * 1024 * 1024) + ); + assert_eq!( + "1000K".parse::().unwrap(), + PartitionSize(1000 * 1024) + ); + assert_eq!( + "3G".parse::().unwrap(), + PartitionSize(3 * 1024 * 1024 * 1024) + ); + assert_eq!( + "50m".parse::().unwrap(), + PartitionSize(50 * 1024 * 1024) + ); + assert_eq!( + "1000k".parse::().unwrap(), + PartitionSize(1000 * 1024) + ); + assert_eq!( + "3g".parse::().unwrap(), + PartitionSize(3 * 1024 * 1024 * 1024) + ); + assert_eq!( + " 50M ".parse::().unwrap(), + PartitionSize(50 * 1024 * 1024) + ); + assert_eq!("100".parse::().unwrap(), PartitionSize(100)); + + assert!(("".parse::()).is_err()); + assert!(("50T".parse::()).is_err()); + assert!(("abc".parse::()).is_err()); + } + + #[test] + fn test_as_kb() { + assert_eq!( + "100K".parse::().unwrap().as_kb().unwrap(), + 100 + ); + assert_eq!( + "100M".parse::().unwrap().as_kb().unwrap(), + 100 * 1024 + ); + assert!(PartitionSize(100).as_kb().is_err()); + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/path_converter.rs b/rs/ic_os/build_tools/build_filesystem/src/path_converter.rs new file mode 100644 index 000000000000..ce27b466cd18 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/path_converter.rs @@ -0,0 +1,160 @@ +use std::path::{Component, Path, PathBuf}; + +/// Converts paths between the source filesystem and the target filesystem image for simplifying +/// extracting entries from a subdir. +pub struct PathConverter { + subdir: Option, +} + +impl PathConverter { + pub fn new(subdir: Option) -> Self { + assert!( + subdir.as_ref().is_none_or(|p| p.is_absolute()), + "subdir must be absolute: {subdir:?}" + ); + Self { subdir } + } + + /// Converts a path from the source filesystem to the target filesystem image. + /// (removes the subdir prefix) + /// + /// Returns `None` if the path is outside the subdir to be included in the target. + pub fn source_to_target(&self, source_path: &ImagePath) -> Option { + match &self.subdir { + Some(subdir) => match source_path.0.strip_prefix(subdir) { + Ok(stripped) => Some(ImagePath::from(stripped.to_path_buf())), + Err(_) => None, + }, + None => Some(source_path.clone()), + } + } + + /// Converts a path from the target filesystem image to the source filesystem. + /// (adds the subdir prefix) + pub fn target_to_source(&self, target_path: &ImagePath) -> ImagePath { + if let Some(subdir) = &self.subdir { + ImagePath::from(subdir.join(target_path.as_relative_path())) + } else { + target_path.clone() + } + } +} + +/// Represents a path in the filesystem image +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImagePath(PathBuf); + +impl ImagePath { + pub fn root() -> ImagePath { + Self(PathBuf::from("/")) + } + + pub fn is_root(&self) -> bool { + self.0 == PathBuf::from("/") + } + + /// Returns the path with a leading slash (e.g. for SELinux) + pub fn as_absolute_path(&self) -> &Path { + &self.0 + } + + /// Returns the path without a leading slash (e.g. for path in the tar file) + pub fn as_relative_path(&self) -> &Path { + let stripped = &self.0.strip_prefix("/").unwrap(); + if *stripped == Path::new("") { + Path::new(".") + } else { + stripped + } + } +} + +impl> From for ImagePath { + fn from(path: T) -> Self { + let path = std::iter::once(Component::RootDir) + .chain( + path.into() + .components() + // Remove leading . and / + .skip_while(|c| matches!(c, Component::CurDir | Component::RootDir)), + ) + .collect::(); + + Self(path) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_image_path_from_absolute() { + let path = ImagePath::from("/opt/ic/bin"); + assert_eq!(path.as_absolute_path(), Path::new("/opt/ic/bin")); + assert_eq!(path.as_relative_path(), Path::new("opt/ic/bin")); + } + + #[test] + fn test_image_path_from_relative() { + let path = ImagePath::from("opt/ic/bin"); + assert_eq!(path.as_absolute_path(), Path::new("/opt/ic/bin")); + assert_eq!(path.as_relative_path(), Path::new("opt/ic/bin")); + } + + #[test] + fn test_image_path_root_as_relative() { + let path = ImagePath::root(); + assert_eq!(path.as_absolute_path(), Path::new("/")); + assert_eq!(path.as_relative_path(), Path::new(".")); + } + + #[test] + fn test_path_converter_no_subdir_source_to_target() { + let converter = PathConverter::new(None); + let source = ImagePath::from("/opt/ic/bin/replica"); + let target = converter.source_to_target(&source); + assert_eq!(target, Some(ImagePath::from("/opt/ic/bin/replica"))); + } + + #[test] + fn test_path_converter_with_subdir_source_to_target_match() { + let converter = PathConverter::new(Some(PathBuf::from("/opt/ic"))); + let source = ImagePath::from("/opt/ic/bin/replica"); + let target = converter.source_to_target(&source); + assert_eq!(target, Some(ImagePath::from("/bin/replica"))); + } + + #[test] + fn test_path_converter_with_subdir_source_to_target_no_match() { + let converter = PathConverter::new(Some(PathBuf::from("/opt/ic"))); + let source = ImagePath::from("/usr/bin/replica"); + let target = converter.source_to_target(&source); + assert_eq!(target, None); + } + + #[test] + fn test_path_converter_target_to_source_no_subdir() { + let converter = PathConverter::new(None); + let target = ImagePath::from("/opt/ic/bin/replica"); + let source = converter.target_to_source(&target); + assert_eq!(source.as_absolute_path(), Path::new("/opt/ic/bin/replica")); + } + + #[test] + fn test_path_converter_target_to_source_with_subdir() { + let converter = PathConverter::new(Some(PathBuf::from("/opt/ic"))); + let target = ImagePath::from("/bin/replica"); + let source = converter.target_to_source(&target); + assert_eq!(source.as_absolute_path(), Path::new("/opt/ic/bin/replica")); + } + + #[test] + fn test_path_converter_target_to_source_root_no_subdir() { + let converter = PathConverter::new(None); + let target = ImagePath::root(); + let source = converter.target_to_source(&target); + assert_eq!(source.as_absolute_path(), Path::new("/")); + assert_eq!(source.as_relative_path(), Path::new(".")); + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/processor.rs b/rs/ic_os/build_tools/build_filesystem/src/processor.rs new file mode 100644 index 000000000000..9db02436572c --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/processor.rs @@ -0,0 +1,207 @@ +use crate::ExtraFile; +use crate::fs_builder::{FileEntry, FilesystemBuilder}; +use crate::path_converter::{ImagePath, PathConverter}; +use crate::selinux::{FileContexts, FileType}; +use anyhow::{Context, Result, bail}; +use regex::RegexSet; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use tar::{Archive, Header}; + +pub fn process_filesystem( + input_tar_path: Option<&Path>, + output_builder: &mut dyn FilesystemBuilder, + subdir: Option<&Path>, + strip_paths: &RegexSet, + extra_files: &[ExtraFile], + selinux_file_contexts: &Option, +) -> Result<()> { + let path_converter = PathConverter::new(subdir.map(PathBuf::from)); + add_root(output_builder, selinux_file_contexts, &path_converter)?; + if output_builder.needs_lost_found() { + add_lost_found(output_builder, &path_converter, selinux_file_contexts)?; + } + + if let Some(input_tar_path) = input_tar_path { + process_input_tar( + input_tar_path, + output_builder, + &path_converter, + selinux_file_contexts, + strip_paths, + )?; + } + + process_extra_files( + extra_files, + output_builder, + &path_converter, + selinux_file_contexts, + )?; + + Ok(()) +} + +fn process_input_tar( + input_tar_path: &Path, + output_builder: &mut dyn FilesystemBuilder, + path_converter: &PathConverter, + selinux_file_contexts: &Option, + strip_paths: &RegexSet, +) -> Result<()> { + let mut input_tar = Archive::new(std::io::BufReader::new( + File::open(input_tar_path) + .with_context(|| format!("Failed to open input file {:?}", input_tar_path))?, + )); + + for entry in input_tar.entries()? { + let mut entry = entry?; + let source_path = ImagePath::from(entry.path().context("Failed to read entry path")?); + + if !strip_paths.is_match( + source_path + .as_absolute_path() + .to_str() + .context("Failed to convert path to string")?, + ) && let Some(target_path) = path_converter.source_to_target(&source_path) + { + if entry.header().entry_type().is_dir() { + add_entry( + output_builder, + entry.header().clone(), + &target_path, + &mut std::io::empty(), + path_converter, + selinux_file_contexts, + )?; + } else { + add_entry( + output_builder, + entry.header().clone(), + &target_path, + &mut entry, + path_converter, + selinux_file_contexts, + )?; + } + } + } + + Ok(()) +} + +fn process_extra_files( + extra_files: &[ExtraFile], + output_builder: &mut dyn FilesystemBuilder, + path_converter: &PathConverter, + selinux_file_contexts: &Option, +) -> Result<()> { + for extra_file in extra_files { + let metadata = std::fs::metadata(&extra_file.source) + .with_context(|| format!("Failed to read metadata for {:?}", extra_file.source))?; + let mut header = Header::new_gnu(); + header.set_size(metadata.len()); + header.set_mode(extra_file.mode); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + add_entry( + output_builder, + header, + &extra_file.target, + &mut File::open(&extra_file.source)?, + path_converter, + selinux_file_contexts, + )?; + } + Ok(()) +} + +fn add_root( + output_builder: &mut dyn FilesystemBuilder, + file_contexts: &Option, + path_converter: &PathConverter, +) -> Result<()> { + let mut header = Header::new_gnu(); + header.set_mode(0o755); + header.set_size(0); + header.set_entry_type(tar::EntryType::Directory); + header.set_cksum(); + add_entry( + output_builder, + header, + &ImagePath::root(), + &mut std::io::empty(), + path_converter, + file_contexts, + ) +} + +fn add_lost_found( + output_builder: &mut dyn FilesystemBuilder, + path_converter: &PathConverter, + file_contexts: &Option, +) -> Result<()> { + let mut header = Header::new_gnu(); + header.set_mode(0o700); + header.set_entry_type(tar::EntryType::Directory); + header.set_cksum(); + add_entry( + output_builder, + header, + &ImagePath::from("lost+found"), + &mut std::io::empty(), + path_converter, + file_contexts, + ) +} + +fn add_entry( + output_builder: &mut dyn FilesystemBuilder, + mut header: Header, + target_path: &ImagePath, + data: &mut dyn Read, + path_converter: &PathConverter, + selinux_file_contexts: &Option, +) -> Result<()> { + let source_path = path_converter.target_to_source(target_path); + + assert!( + !source_path.as_absolute_path().ends_with("/") + || source_path.as_absolute_path() == Path::new("/") + ); + assert!( + !source_path.as_relative_path().ends_with("/") + || source_path.as_relative_path() == Path::new(".") + ); + + // Always set mtime to 0 for reproducibility + header.set_mtime(0); + header.set_cksum(); + + let selinux_context = if let Some(contexts) = selinux_file_contexts { + let file_type = match header.entry_type() { + t if t.is_dir() => Some(FileType::Directory), + t if t.is_symlink() => Some(FileType::Symlink), + t if t.is_file() => Some(FileType::RegularFile), + t if t.is_hard_link() => None, + _ => bail!( + "{} has unsupported entry type: {:?}", + source_path.as_absolute_path().display(), + header.entry_type() + ), + }; + if let Some(file_type) = file_type { + contexts.find_context(source_path.as_absolute_path(), file_type)? + } else { + None + } + } else { + None + }; + + let file_entry = + FileEntry::new(target_path.clone(), header, data).with_selinux_context(selinux_context); + + output_builder.append_entry(file_entry) +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/selinux.rs b/rs/ic_os/build_tools/build_filesystem/src/selinux.rs new file mode 100644 index 000000000000..4cf99e37ec3c --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/selinux.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use anyhow::{Context, ensure}; +use std::ffi::CString; +use std::io::ErrorKind; +use std::os::raw::{c_int, c_void}; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileType { + Directory, + RegularFile, + Symlink, +} + +unsafe impl Send for FileContexts {} + +unsafe impl Sync for FileContexts {} + +pub struct FileContexts { + labeler: ::selinux::label::Labeler<::selinux::label::back_end::File>, +} + +impl FileContexts { + pub fn new(file_contexts_path: &Path) -> Result { + ensure!( + file_contexts_path.exists(), + "File contexts file does not exist: {}", + file_contexts_path.display() + ); + + // SELABEL_OPT_PATH constant for libselinux that allows specifying the file_contexts file + // instead of the default policy file + const SELABEL_OPT_PATH: c_int = 3; + + let path = CString::new(file_contexts_path.as_os_str().as_bytes())?; + let options = [(SELABEL_OPT_PATH, path.as_ptr() as *const c_void)]; + + // Use raw_format=true to avoid needing access to SELinux policy for translation + let labeler = + ::selinux::label::Labeler::<::selinux::label::back_end::File>::new(&options, true) + .with_context(|| { + format!( + "Failed to create SELinux labeler with file_contexts: {}", + file_contexts_path.display() + ) + })?; + + Ok(Self { labeler }) + } + + pub fn find_context(&self, path: &Path, file_type: FileType) -> Result> { + ensure!( + path.is_absolute(), + "Path must be absolute: {}", + path.display() + ); + // File access modes from https://man7.org/linux/man-pages/man7/inode.7.html + // S_IFDIR = 0o040000, S_IFREG = 0o100000, S_IFLNK = 0o120000 + let mode_value = match file_type { + FileType::Directory => 0o040000, + FileType::RegularFile => 0o100000, + FileType::Symlink => 0o120000, + }; + + let file_mode = ::selinux::FileAccessMode::new(mode_value) + .context("Failed to create FileAccessMode")?; + + match self.labeler.look_up_by_path(path, Some(file_mode)) { + Ok(ctx) => Ok(ctx.to_c_string()?.map(|cstr| cstr.into_owned())), + // If the path cannot be found in the selinux context file, return None + Err(::selinux::errors::Error::IO { source, .. }) + if source.kind() == ErrorKind::NotFound => + { + Ok(None) + } + Err(e) => Err(e.into()), + } + } +} diff --git a/rs/ic_os/build_tools/build_filesystem/src/tar.rs b/rs/ic_os/build_tools/build_filesystem/src/tar.rs new file mode 100644 index 000000000000..cd90f49948d2 --- /dev/null +++ b/rs/ic_os/build_tools/build_filesystem/src/tar.rs @@ -0,0 +1,49 @@ +use crate::fs_builder::{FileEntry, FilesystemBuilder}; +use anyhow::Result; +use std::io::Write; +use tar::Builder; + +/// Implementation of FilesystemBuilder for tar archives +pub struct TarBuilder { + builder: Builder, +} + +impl TarBuilder { + pub fn new(builder: Builder) -> Self { + Self { builder } + } + + pub fn into_inner(self) -> Builder { + self.builder + } +} + +impl FilesystemBuilder for TarBuilder { + fn append_entry(&mut self, entry: FileEntry<'_>) -> Result<()> { + let mut header = entry.header; + + if let Some(selinux_context) = &entry.selinux_context { + self.builder.append_pax_extensions(vec![ + ( + "SCHILY.xattr.security.selinux", + selinux_context.as_bytes_with_nul(), + ), + ("RHT.security.selinux", selinux_context.as_bytes_with_nul()), + ])?; + } + + self.builder + .append_data(&mut header, entry.path.as_relative_path(), entry.contents)?; + + Ok(()) + } + + fn finish(self: Box) -> Result<()> { + self.builder.into_inner()?.flush()?; + Ok(()) + } + + fn needs_lost_found(&self) -> bool { + false + } +} diff --git a/rs/ic_os/device/Cargo.toml b/rs/ic_os/device/Cargo.toml index 2131dd85db30..b0917e651511 100644 --- a/rs/ic_os/device/Cargo.toml +++ b/rs/ic_os/device/Cargo.toml @@ -12,7 +12,6 @@ nix = { workspace = true } partition_tools = { path = "../build_tools/partition_tools" } rand = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true } uuid = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/rs/tests/BUILD.bazel b/rs/tests/BUILD.bazel index 0bb6592cc821..c548a0baf7f5 100644 --- a/rs/tests/BUILD.bazel +++ b/rs/tests/BUILD.bazel @@ -18,6 +18,7 @@ PACKAGES = [ "@noble//bash/amd64", "@noble//ca-certificates/amd64", "@noble//coreutils/amd64", + "@noble//libarchive-dev/amd64", "@noble//libcryptsetup-dev/amd64", "@noble//dmsetup/amd64", "@noble//dosfstools/amd64", diff --git a/rs/tests/node/BUILD.bazel b/rs/tests/node/BUILD.bazel index 5f55fddb58a6..df2fe05982d9 100644 --- a/rs/tests/node/BUILD.bazel +++ b/rs/tests/node/BUILD.bazel @@ -51,6 +51,7 @@ uvm_config_image( name = "root_tests_config_image", testonly = True, srcs = [ + "//rs/ic_os/build_tools/build_filesystem:build_filesystem_test_with_deps", "//rs/ic_os/device:device_test", "//rs/ic_os/os_tools/guest_disk:guest_disk_test", "//rs/ic_os/os_tools/guest_vm_runner:upgrade_device_mapper_test", diff --git a/rs/tests/node/root_tests.rs b/rs/tests/node/root_tests.rs index 5ae2254c40a8..bb6adcf49e1f 100644 --- a/rs/tests/node/root_tests.rs +++ b/rs/tests/node/root_tests.rs @@ -53,9 +53,48 @@ fn root_test(env: TestEnv, test: &str) { .expect("Failed to run {test}"); } +fn build_filesystem_test(env: TestEnv) { + let deployed_universal_vm = env + .get_deployed_universal_vm(UNIVERSAL_VM_NAME) + .expect("unable to get deployed VM."); + deployed_universal_vm + .block_on_bash_script(&indoc::formatdoc!( + r#" + set -euo pipefail + docker load -i /config/ubuntu_test_runtime.tar + + TMPDIR=$(mktemp -d) + trap "rm -rf ${{TMPDIR}}" exit + cd "${{TMPDIR}}" + + cp /config/build_filesystem_test . + cp /config/mke2fs . + chmod +x build_filesystem_test mke2fs + + cat < Dockerfile + FROM ubuntu_test_runtime:image + COPY --chmod=755 build_filesystem_test /build_filesystem_test + COPY --chmod=755 mke2fs /mke2fs + EOF + + docker build --tag final -f Dockerfile . + docker run --privileged -v /dev:/dev --rm final /usr/bin/bash -c " + /usr/lib/systemd/systemd-udevd --daemon + export MKE2FS_BIN=/mke2fs + export RUST_BACKTRACE=1 + # We have a limited number of loop devices, so it's not worth + # running too many tests in parallel. + /build_filesystem_test --test-threads=3 + " + "# + )) + .expect("Failed to run build_filesystem_unit_test"); +} + fn main() -> Result<()> { SystemTestGroup::new() .with_setup(setup) + .add_test(systest!(build_filesystem_test)) .add_test(systest!(root_test; "upgrade_device_mapper_test")) .add_test(systest!(root_test; "guest_disk_test")) .add_test(systest!(root_test; "device_test")) diff --git a/third_party/BUILD.e2fsprogs.bazel b/third_party/BUILD.e2fsprogs.bazel new file mode 100644 index 000000000000..483181d196a1 --- /dev/null +++ b/third_party/BUILD.e2fsprogs.bazel @@ -0,0 +1,29 @@ +# Build e2fsprogs for mkfs dependency which is used to build IC-OS filesystem images + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + ), +) + +configure_make( + name = "e2fsprogs", + lib_name = "e2fsprogs", + lib_source = ":all_srcs", + out_binaries = ["mke2fs"], + postfix_script = """ + mv misc/mke2fs $INSTALLDIR/bin + """, + targets = ["progs"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "mke2fs", + srcs = [":e2fsprogs"], + output_group = "mke2fs", + visibility = ["//visibility:public"], +) diff --git a/third_party/BUILD.selinux.bazel b/third_party/BUILD.selinux.bazel new file mode 100644 index 000000000000..bce38ed97b61 --- /dev/null +++ b/third_party/BUILD.selinux.bazel @@ -0,0 +1,20 @@ +# Use libselinux from the host environment + +load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_import( + name = "libselinux-internal", + hdrs = glob(["include/selinux/*.h"]), + interface_library = "lib/x86_64-linux-gnu/libselinux.so", + system_provided = True, + visibility = ["//visibility:private"], +) + +# Use an extra cc_library to hide the depth of the include folder +cc_library( + name = "libselinux", + includes = ["include"], + visibility = ["//visibility:public"], + deps = ["libselinux-internal"], +)