diff --git a/.gitignore b/.gitignore index e7b4035..c1fe408 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ coreos/*.qcow2 secret tmp/ trustee/keys +*.tar +*.tar.gz diff --git a/README.md b/README.md index 0e27da3..e8d5885 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ Build the Fedora CoreOS or Centos Stream CoreOS image with the custom initrd: ```bash cd coreos # Centos Stream CoreOS image -just os=scos build oci-archive osbuild-qemu +just os=scos build oci-archive init build-qemu # Fedora CoreOS image -just build oci-archive osbuild-qemu +just build oci-archive init build-qemu ``` ### Create local Trustee deployment diff --git a/configs/trustee.bu b/configs/trustee.bu new file mode 100644 index 0000000..14f9016 --- /dev/null +++ b/configs/trustee.bu @@ -0,0 +1,67 @@ +variant: fcos +version: 1.6.0 +passwd: + users: + - name: core + ssh_authorized_keys: + - + +systemd: + units: + - name: serial-getty@ttyS0.service + dropins: + - name: autologin-core.conf + contents: | + [Service] + # Override Execstart in main unit + ExecStart= + # Add new Execstart with `-` prefix to ignore failure` + ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM + +storage: + directories: + - path: /var/kbs/config + overwrite: true + - path: /var/srv/www + overwrite: true + files: + - path: /etc/profile.d/systemd-pager.sh + mode: 0644 + contents: + inline: | + # Tell systemd to not use a pager when printing information + export SYSTEMD_PAGER=cat + - path: /usr/local/bin/populate_kbs.sh + mode: 0755 + contents: + local: populate_kbs.sh + - path: /usr/local/bin/kbs-client + mode: 0755 + contents: + local: kbs-client + - path: /etc/containers/systemd/key-generation.container + mode: 0644 + contents: + local: containers/key-generation.container + - path: /var/kbs/config/kbs-config.toml + mode: 0644 + contents: + local: kbs-config.toml + - path: /etc/containers/systemd/kbs.container + mode: 0644 + contents: + local: containers/kbs.container + - path: /etc/containers/systemd/kbs-client.container + mode: 0644 + contents: + local: containers/kbc.container + - path: /etc/containers/systemd/nginx.container + mode: 0644 + contents: + local: containers/nginx.container + - path: /etc/containers/systemd/register-ak.container + mode: 0644 + contents: + local: containers/register-ak.container + + diff --git a/configs/trustee/containers/kbc.container b/configs/trustee/containers/kbc.container new file mode 100644 index 0000000..da9eed2 --- /dev/null +++ b/configs/trustee/containers/kbc.container @@ -0,0 +1,13 @@ +[Unit] +Description=Trustee KBS client container +After=key-generation.container + +[Container] +ContainerName=kbs-client +Image=quay.io/trusted-execution-clusters/trustee-attester:TPM-additional-dev +Network=host +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Exec=tail -f /dev/null + +[Install] +WantedBy=default.target diff --git a/configs/trustee/containers/kbs.container b/configs/trustee/containers/kbs.container new file mode 100644 index 0000000..5f0d5b3 --- /dev/null +++ b/configs/trustee/containers/kbs.container @@ -0,0 +1,21 @@ +[Unit] +Description=Trustee KBS container +After=key-generation.container + +[Container] +ContainerName=kbs +Image=quay.io/trusted-execution-clusters/key-broker-service:fix-TPM-report-data-size +Network=host +Entrypoint=/usr/local/bin/kbs +PublishPort=8080:8080 +Environment=RUST_LOG=debug +Volume=/var/kbs/config/kbs-config.toml:/opt/confidential-containers/kbs/config/kbs-config.toml:z +Volume=kbs-storage:/opt/confidential-containers/kbs/repository +Volume=nebula-ca:/opt/confidential-containers/kbs/nebula-ca +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Volume=trusted-ak-keys:/etc/tpm/trusted_ak_keys +Exec=--config-file \ + /opt/confidential-containers/kbs/config/kbs-config.toml + +[Install] +WantedBy=default.target diff --git a/configs/trustee/containers/key-generation.container b/configs/trustee/containers/key-generation.container new file mode 100644 index 0000000..d190eff --- /dev/null +++ b/configs/trustee/containers/key-generation.container @@ -0,0 +1,17 @@ +[Unit] +Description=Trustee Key Generator +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=keyprovider +Image=docker.io/alpine/openssl:latest +Entrypoint=/bin/ash +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Exec=-c "if [ ! -s /opt/confidential-containers/kbs/user-keys/private.key ]; then \ + /usr/bin/openssl genpkey -algorithm ed25519 > /opt/confidential-containers/kbs/user-keys/private.key && \ + /usr/bin/openssl pkey -in /opt/confidential-containers/kbs/user-keys/private.key -pubout \ + -out /opt/confidential-containers/kbs/user-keys/public.pub; else exit 0; fi;" + +[Install] +WantedBy=default.target diff --git a/configs/trustee/containers/nginx.container b/configs/trustee/containers/nginx.container new file mode 100644 index 0000000..6225d9d --- /dev/null +++ b/configs/trustee/containers/nginx.container @@ -0,0 +1,14 @@ +[Unit] +Description=nginx HTTP server emulating registration server +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=nginx +Image=quay.io/fedora/nginx-126:latest +PublishPort=8000:8080 +Volume=/srv/www:/opt/app-root/src:z +Exec=nginx -g "daemon off;" + +[Install] +WantedBy=default.target diff --git a/configs/trustee/containers/register-ak.container b/configs/trustee/containers/register-ak.container new file mode 100644 index 0000000..e07a173 --- /dev/null +++ b/configs/trustee/containers/register-ak.container @@ -0,0 +1,13 @@ +[Unit] +Description=server that allow to register AK +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=register-ak +Image=quay.io/trusted-execution-clusters/test-server-ak:latest +PublishPort=5001:5001 +Volume=trusted-ak-keys:/data + +[Install] +WantedBy=default.target diff --git a/configs/trustee/kbs-client b/configs/trustee/kbs-client new file mode 100755 index 0000000..5b7797f --- /dev/null +++ b/configs/trustee/kbs-client @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail +# set -x + +KEY="${KEY:=/opt/confidential-containers/kbs/user-keys/private.key}" + +sudo podman exec -ti \ + kbs-client \ + kbs-client \ + config \ + --auth-private-key "${KEY}" \ + "${@}" diff --git a/configs/trustee/kbs-config.toml b/configs/trustee/kbs-config.toml new file mode 100644 index 0000000..8303eee --- /dev/null +++ b/configs/trustee/kbs-config.toml @@ -0,0 +1,35 @@ +[http_server] +sockets = ["0.0.0.0:8080"] +insecure_http = true + +[admin] +insecure_api = true +auth_public_key = "./keys/public.pub" + + +[attestation_token] +insecure_key = true + +[attestation_service] +type = "coco_as_builtin" +work_dir = "/opt/confidential-containers/attestation-service" +policy_engine = "opa" + +[attestation_service.attestation_token_broker] +type = "Ear" +duration_min = 5 + +[attestation_service.rvps_config] +type = "BuiltIn" + +[attestation_service.rvps_config.storage] +type = "LocalFs" + +[attestation_service.verifier_config.tpm_verifier] +trusted_ak_keys_dir = "/etc/tpm/trusted_ak_keys" +max_trusted_ak_keys = 100 + +[[plugins]] +name = "resource" +type = "LocalFs" +dir_path = "/opt/confidential-containers/kbs/repository" \ No newline at end of file diff --git a/configs/trustee/populate_kbs.sh b/configs/trustee/populate_kbs.sh new file mode 100755 index 0000000..bd5d9c7 --- /dev/null +++ b/configs/trustee/populate_kbs.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -xe + +SECRET_PATH=${SECRET_PATH:=default/machine/root} +KEY=${KEY:=/opt/confidential-containers/kbs/user-keys/private.key} + + +## set reference values for TPM +for i in {7,4,14}; do + value=$(sudo tpm2_pcrread sha256:${i} | awk -F: '/0x/ {sub(/.*0x/, "", $2); gsub(/[^0-9A-Fa-f]/, "", $2); print tolower($2)}') + kbs-client set-sample-reference-value tpm_pcr${i} "${value}" +done + +# Check reference values +kbs-client get-reference-values + + +# Create attestation policy +## This policy allows access only if the system’s TPM or SNP +## hardware measurements match trusted reference values +cat << 'EOF' > A_policy.rego +package policy +import rego.v1 + +default hardware := 97 +default executables := 3 +default configuration := 2 + +##### TPM + +hardware := 2 if { + input.tpm.pcr07 in data.reference.tpm_pcr7 + input.tpm.pcr14 in data.reference.tpm_pcr14 + input.tpm.pcr04 in data.reference.tpm_pcr4 +} + +hardware := 2 if { + input.snp.reported_tcb_snp == 27 +} + + +##### Final decision +result := { + "executables": executables, + "hardware": hardware, + "configuration": configuration +} +EOF + +sudo podman cp A_policy.rego kbs-client:/A_policy.rego +kbs-client set-attestation-policy --policy-file /A_policy.rego --type rego --id default_cpu + +# Upload resource +cat > secret << EOF +{ "key_type": "oct", "key": "2b442dd5db4478367729ef8bbf2e7480" } +EOF +sudo podman cp secret kbs-client:/secret +kbs-client set-resource --resource-file /secret --path ${SECRET_PATH} + +# Create resource policy +## This policy allows access only if both CPUs report an "affirming" status +## and provide TPM and SNP attestation evidence. +cat << 'EOF' > R_policy.rego +package policy +import rego.v1 + +default allow = false + +allow if { + input["submods"]["cpu0"]["ear.status"] == "affirming" + input["submods"]["cpu1"]["ear.status"] == "affirming" + input["submods"]["cpu1"]["ear.veraison.annotated-evidence"]["tpm"] + input["submods"]["cpu0"]["ear.veraison.annotated-evidence"]["snp"] +} +EOF + +sudo podman cp R_policy.rego kbs-client:/R_policy.rego +kbs-client set-resource-policy --policy-file /R_policy.rego diff --git a/containerfiles/trustee-attester.container b/containerfiles/trustee-attester.container index 6d73647..8b51097 100644 --- a/containerfiles/trustee-attester.container +++ b/containerfiles/trustee-attester.container @@ -13,7 +13,7 @@ RUN . /etc/os-release && \ RUN dnf install -y git tss2-devel tpm2-tss-devel cargo openssl-devel perl RUN cd /usr/src/ && \ - git clone https://github.com/confidential-containers/guest-components.git && \ + git clone https://github.com/trusted-execution-clusters/guest-components.git && \ cd guest-components && git checkout ${COMMIT} RUN cd /usr/src/guest-components && \ diff --git a/coreos/Containerfile b/coreos/Containerfile index c392a03..8bb816e 100644 --- a/coreos/Containerfile +++ b/coreos/Containerfile @@ -1,7 +1,14 @@ ARG BASE -FROM quay.io/trusted-execution-clusters/trustee-attester:fedora-b13fd8a as kbc -FROM quay.io/trusted-execution-clusters/clevis-pin-trustee as clevis -FROM ghcr.io/trusted-execution-clusters/ignition:20260112-85608d6 as ignition +ARG TRUSTEE_ATTESTER=quay.io/trusted-execution-clusters/trustee-attester:fedora-b13fd8a +ARG CLEVIS_PIN_TRUSTEE_IMAGE=quay.io/trusted-execution-clusters/clevis-pin-trustee +ARG IGNITION=ghcr.io/trusted-execution-clusters/ignition:20260112-85608d6 + +FROM $TRUSTEE_ATTESTER as kbc + +FROM $CLEVIS_PIN_TRUSTEE_IMAGE as clevis + +FROM $IGNITION as ignition + FROM $BASE COPY ./usr /usr diff --git a/coreos/justfile b/coreos/justfile index 0028ab3..b0da30a 100644 --- a/coreos/justfile +++ b/coreos/justfile @@ -21,6 +21,8 @@ image := if os == "scos" { scos_img } else { fcos_img } os_name := if os == "scos" { scos_os } else { fcos_os } label := if os == "scos" { scos_label } else { fcos_label } archive := os + ".ociarchive" +platform := "qemu" + config := if os == "scos" { scos_config } else { fcos_config } full_name := if os == "scos" { "centos-stream-coreos" } else { "fedora-coreos" } @@ -80,3 +82,8 @@ azure: {{cosa_function}} cd cache cosa osbuild azure +gcp: + #!/usr/bin/env bash + {{cosa_function}} + cd cache + cosa osbuild gcp diff --git a/scripts/GCP/README.md b/scripts/GCP/README.md new file mode 100644 index 0000000..5261da1 --- /dev/null +++ b/scripts/GCP/README.md @@ -0,0 +1,60 @@ +# Remote attestation with PCRs and AMD SEV-SNP on GCP using RHCOS + +This guide provides step-by-step instructions for setting up remote attestation using PCRs and AMD SEV-SNP on Google Cloud Platform (GCP) with Red Hat CoreOS (RHCOS). It covers the deployment of a Trustee server and the creation of a custom RHCOS client image that communicates with the Trustee service to fetch encryption keys and decrypt the root image. + + +## Prerequisites + +1. Copy the pull secret from [Red Hat OpenShift](https://console.redhat.com/openshift/create/local) to `~/.config/containers/auth.json` under `auths:quay.io:auth:` +2. Install [gcloud](https://cloud.google.com/sdk/docs/install) +3. Configure a subnet on GCP for the server and client by running `./scripts/network_setup.sh` + + +## Deploy the Trustee Server (KBS) + +1. To deploy the Trustee server, run: +```bash +./scripts/GCP/deploy-trustee.sh -k -b ./configs/trustee.bu -i +``` +2. After the server is up, populate the KBS with the reference value and add the remote ignition file: +```bash +./scripts/populate-trustee-kbs.sh +``` +(The default hostname is `kbs`) + + +## Deploy the Client + +1. Build a custom RHCOS image by running: + ```bash + cd coreos + just os=scos build oci-archive init gcp + ``` + +2. Upload the image to GCP by running: + ```bash + ./scripts/GCP/upload_image_gcp.sh + ``` + +3. Deploy the client by running: + ```bash + ./scripts/GCP/deploy-vm.sh -k -b ./configs/ak.bu -n -i -h + ``` + This will create the VM, perform attestation, and decrypt the disk using clevis-pin. + + +## Information About KBS, KBS-Client, and Clevis-Pin + +These are modified versions of [guest component](https://github.com/iroykaufman/guest-components/tree/TPM-as-additional-device) to support the TPM as an additional device. + +The changes in the guest component are also included in [PR#1093](https://github.com/confidential-containers/guest-components/pull/1093). + +## Attestation Policy + +The policy only checks hardware for both SEV-SNP and TPM. + +## Resource Policy + +Verify that both devices are affirming and exist. + + diff --git a/scripts/GCP/deploy-trustee.sh b/scripts/GCP/deploy-trustee.sh new file mode 100755 index 0000000..91626e1 --- /dev/null +++ b/scripts/GCP/deploy-trustee.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +IGNITION_FILE="config.ign" +IGNITION_CONFIG="$(pwd)/configs/${IGNITION_FILE}" + + +TRUSTEE_PORT="" + +set -xe + +## Default values +VM_NAME="kbs" +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' + +while getopts "k:b:n:i:z:m:" opt; do + case $opt in + k) key=$OPTARG ;; + b) butane=$OPTARG ;; + n) VM_NAME=$OPTARG ;; + i) IMAGE=$OPTARG ;; + z) ZONE=$OPTARG ;; + m) MACHINE_TYPE=$OPTARG ;; + \?) echo "Invalid option"; exit 1 ;; + esac +done + + +if [ -z "${key}" ]; then + echo "Please, specify the public ssh key" + exit 1 +fi +if [ -z "${butane}" ]; then + echo "Please, specify the butane configuration file" + exit 1 +fi + + + +bufile=$(mktemp) + +KEY=$(cat "$key") + +sed "s||$KEY|g" "$butane" >"${bufile}" + +podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd)/configs":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/coreos/butane:release \ + --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/trustee + +chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} + + +gcloud compute instances create ${VM_NAME} \ + --image ${IMAGE} \ + --metadata-from-file "user-data=${IGNITION_CONFIG}" \ + --confidential-compute-type "SEV_SNP" \ + --machine-type "${MACHINE_TYPE}" \ + --maintenance-policy terminate \ + --zone "${ZONE}" \ + --subnet "demo-subnet-us-central1" \ + --shielded-vtpm \ + --shielded-integrity-monitoring \ + --shielded-secure-boot \ + diff --git a/scripts/GCP/deploy-vm.sh b/scripts/GCP/deploy-vm.sh new file mode 100755 index 0000000..b69eb89 --- /dev/null +++ b/scripts/GCP/deploy-vm.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +IGNITION_FILE="config.ign" +IGNITION_CONFIG="$(pwd)/${IGNITION_FILE}" + + +TRUSTEE_PORT="" + +set -xe + +## Default values +VM_NAME="vm" +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' + +while getopts "k:b:n:i:h:z:m:" opt; do + case $opt in + k) key=$OPTARG ;; + b) butane=$OPTARG ;; + n) VM_NAME=$OPTARG ;; + i) IMAGE=$OPTARG ;; + h) hostname=$OPTARG ;; + z) ZONE=$OPTARG ;; + m) MACHINE_TYPE=$OPTARG ;; + \?) echo "Invalid option"; exit 1 ;; + esac +done + + +if [ -z "${key}" ]; then + echo "Please, specify the public ssh key" + exit 1 +fi +if [ -z "${butane}" ]; then + echo "Please, specify the butane configuration file" + exit 1 +fi + + + +bufile=$(mktemp) + +if [ ! -f "$key" ]; then + echo "Error: The specified key file '$key' does not exist." + exit 1 +fi + +KEY=$(cat "$key") + +sed "s||$KEY|g" $butane | sed "s//$hostname/" > "${bufile}" + +sudo podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd)":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/trusted-execution-clusters/butane:attestation \ + --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/rh-coreos + +chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} + + + + + +gcloud compute instances create ${VM_NAME} \ + --image ${IMAGE} \ + --metadata-from-file "user-data=${IGNITION_CONFIG}" \ + --confidential-compute-type "SEV_SNP" \ + --machine-type "${MACHINE_TYPE}" \ + --maintenance-policy terminate \ + --zone "${ZONE}" \ + --subnet "demo-subnet-us-central1" \ + --shielded-vtpm \ + --shielded-integrity-monitoring \ + --shielded-secure-boot + diff --git a/scripts/GCP/network_setup.sh b/scripts/GCP/network_setup.sh new file mode 100644 index 0000000..d0975eb --- /dev/null +++ b/scripts/GCP/network_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +## create a subnet for the server and client VMs +SUBNET_NAME="demo-subnet-us-central1" +gcloud compute networks subnets create ${SUBNET_NAME} \ + --network=default \ + --region=us-central1 \ + --range=10.0.0.0/24 +## allow ssh +RULE_NAME="allow-ssh" +NETWORK_NAME="default" +gcloud compute firewall-rules create ${RULE_NAME} \ + --network=${NETWORK_NAME} \ + --allow=tcp:22 \ + --source-ranges=0.0.0.0/0 \ + --description="Allow SSH from any IP" +## allow tcp on port 8080 for the trustee service +RULE_NAME="allow-tcp" +gcloud compute firewall-rules create ${RULE_NAME} \ + --network=${NETWORK_NAME} \ + --allow=tcp:8080 \ + --source-ranges=0.0.0.0/0 \ + --description="Allow TCP from any IP" \ No newline at end of file diff --git a/scripts/GCP/upload_image_gcp.sh b/scripts/GCP/upload_image_gcp.sh new file mode 100755 index 0000000..4110117 --- /dev/null +++ b/scripts/GCP/upload_image_gcp.sh @@ -0,0 +1,17 @@ +#!/bin/bash + + +BUCKET_NAME=$1 +IMG_NAME=$2 + +## create a bucket to store the image +gsutil mb gs://${BUCKET_NAME} + +gsutil cp ./coreos/${IMG_NAME}.x86_64.tar.gz gs://${BUCKET_NAME}/ + +## create the image in GCP +gcloud compute images create ${IMG_NAME} \ + --source-uri gs://${BUCKET_NAME}/${IMG_NAME}.x86_64.tar.gz \ + --description "My custom ${IMG_NAME} image" \ + --guest-os-features UEFI_COMPATIBLE,GVNIC,VIRTIO_SCSI_MULTIQUEUE,SEV_SNP_CAPABLE,TDX_CAPABLE + diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..fbd56e4 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +create_remote_ign_config () +{ + IP=$1 + # Setup remote ignition config + BUTANE=pin-trustee.bu + IGNITION="${BUTANE%.bu}.ign" + + sed "s//$IP/" configs/remote-ign/${BUTANE} > ${BUTANE} + + podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd):/pwd" \ + --workdir /pwd \ + quay.io/trusted-execution-clusters/butane:clevis-pin-trustee \ + --pretty --strict /pwd/$BUTANE --output "/pwd/$IGNITION" + + echo "./$IGNITION" + +} diff --git a/scripts/populate-trustee-kbs.sh b/scripts/populate-trustee-kbs.sh new file mode 100755 index 0000000..68b30e1 --- /dev/null +++ b/scripts/populate-trustee-kbs.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail +# set -x +source ./scripts/common.sh + +if [[ "${#}" > 3 ]]; then + echo "Usage: $0 " + echo "Optional: $0 " + exit 1 +fi + +KEY=$1 +TRUSTEE_PORT=8080 +IP=$2 +if [[ ${IP} == "" ]]; then +# Setup reference values, policies and secrets +until IP="$(./scripts/get-ip.sh trustee)" && [ -n "$IP" ] && curl "http://${IP}:${TRUSTEE_PORT}" >/dev/null 2>&1; do + echo "Waiting for KBS to be available..." + sleep 1 +done +fi +until ssh core@$IP \ + -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + 'populate_kbs.sh'; do + echo "Waiting for KBS to be populated..." + sleep 1 +done + +# Setup remote ignition config +HOSTNAME=$3 +if [[ ${HOSTNAME} == "" ]]; then +HOSTNAME=${IP} +fi +IGNITION=$(create_remote_ign_config $HOSTNAME) +scp -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ./${IGNITION} core@$IP: +# Setup remote web server to serve the ignition file +ssh core@$IP \ + -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "sudo mv $IGNITION /srv/www && sudo systemctl restart nginx.service"