From a7f0eaaeeb4ecf3eeb26860132cf174770fa6df9 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Tue, 16 Sep 2025 14:27:40 +0300 Subject: [PATCH 1/2] Remote attestation with PCRs and AMD SEV-SNP on GCP using RHCOS Signed-off-by: Roy Kaufman --- .gitignore | 2 + README.md | 4 +- configs/trustee-gcp.bu | 59 +++++++++++ configs/trustee-gcp/containers/kbc.container | 13 +++ configs/trustee-gcp/containers/kbs.container | 20 ++++ .../containers/key-generation.container | 17 ++++ .../trustee-gcp/containers/nginx.container | 14 +++ configs/trustee-gcp/kbs-config.toml | 32 ++++++ configs/trustee-gcp/populate_kbs.sh | 98 +++++++++++++++++++ coreos/Containerfile | 6 ++ coreos/justfile | 5 +- scripts/GCP/README.md | 63 ++++++++++++ scripts/GCP/deploy-trustee.sh | 64 ++++++++++++ scripts/GCP/deploy-vm.sh | 69 +++++++++++++ scripts/GCP/network_setup.sh | 23 +++++ scripts/GCP/upload_image_gcp.sh | 17 ++++ scripts/common.sh | 20 ++++ scripts/populate-trustee-kbs.sh | 47 +++++++++ 18 files changed, 570 insertions(+), 3 deletions(-) create mode 100644 configs/trustee-gcp.bu create mode 100644 configs/trustee-gcp/containers/kbc.container create mode 100644 configs/trustee-gcp/containers/kbs.container create mode 100644 configs/trustee-gcp/containers/key-generation.container create mode 100644 configs/trustee-gcp/containers/nginx.container create mode 100644 configs/trustee-gcp/kbs-config.toml create mode 100755 configs/trustee-gcp/populate_kbs.sh create mode 100644 scripts/GCP/README.md create mode 100755 scripts/GCP/deploy-trustee.sh create mode 100755 scripts/GCP/deploy-vm.sh create mode 100644 scripts/GCP/network_setup.sh create mode 100755 scripts/GCP/upload_image_gcp.sh create mode 100644 scripts/common.sh create mode 100755 scripts/populate-trustee-kbs.sh diff --git a/.gitignore b/.gitignore index e7b4035..c1fe408 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ coreos/*.qcow2 secret tmp/ trustee/keys +*.tar +*.tar.gz diff --git a/README.md b/README.md index 0e27da3..7a43079 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ Build the Fedora CoreOS or Centos Stream CoreOS image with the custom initrd: ```bash cd coreos # Centos Stream CoreOS image -just os=scos build oci-archive osbuild-qemu +just os=scos build oci-archive osbuild # Fedora CoreOS image -just build oci-archive osbuild-qemu +just build oci-archive osbuild ``` ### Create local Trustee deployment diff --git a/configs/trustee-gcp.bu b/configs/trustee-gcp.bu new file mode 100644 index 0000000..7ff4078 --- /dev/null +++ b/configs/trustee-gcp.bu @@ -0,0 +1,59 @@ +variant: fcos +version: 1.6.0 +passwd: + users: + - name: core + ssh_authorized_keys: + - + +systemd: + units: + - name: serial-getty@ttyS0.service + dropins: + - name: autologin-core.conf + contents: | + [Service] + # Override Execstart in main unit + ExecStart= + # Add new Execstart with `-` prefix to ignore failure` + ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM + +storage: + directories: + - path: /var/kbs/config + overwrite: true + - path: /var/srv/www + overwrite: true + files: + - path: /etc/profile.d/systemd-pager.sh + mode: 0644 + contents: + inline: | + # Tell systemd to not use a pager when printing information + export SYSTEMD_PAGER=cat + - path: /usr/local/bin/populate_kbs.sh + mode: 0755 + contents: + local: populate_kbs.sh + - path: /etc/containers/systemd/key-generation.container + mode: 0644 + contents: + local: containers/key-generation.container + - path: /var/kbs/config/kbs-config.toml + mode: 0644 + contents: + local: kbs-config.toml + - path: /etc/containers/systemd/kbs.container + mode: 0644 + contents: + local: containers/kbs.container + - path: /etc/containers/systemd/kbs-client.container + mode: 0644 + contents: + local: containers/kbc.container + - path: /etc/containers/systemd/nginx.container + mode: 0644 + contents: + local: containers/nginx.container + + diff --git a/configs/trustee-gcp/containers/kbc.container b/configs/trustee-gcp/containers/kbc.container new file mode 100644 index 0000000..9d06624 --- /dev/null +++ b/configs/trustee-gcp/containers/kbc.container @@ -0,0 +1,13 @@ +[Unit] +Description=Trustee KBS client container +After=key-generation.container + +[Container] +ContainerName=kbs-client +Image=quay.io/rkaufman/kbs-tpm-snp:v1 +Network=host +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Exec=tail -f /dev/null + +[Install] +WantedBy=default.target diff --git a/configs/trustee-gcp/containers/kbs.container b/configs/trustee-gcp/containers/kbs.container new file mode 100644 index 0000000..9dc8cc7 --- /dev/null +++ b/configs/trustee-gcp/containers/kbs.container @@ -0,0 +1,20 @@ +[Unit] +Description=Trustee KBS container +After=key-generation.container + +[Container] +ContainerName=kbs +Image=quay.io/rkaufman/kbs-tpm-snp:v1 +Network=host +Entrypoint=/usr/local/bin/kbs +PublishPort=8080:8080 +Environment=RUST_LOG=debug +Volume=/var/kbs/config/kbs-config.toml:/opt/confidential-containers/kbs/config/kbs-config.toml:z +Volume=kbs-storage:/opt/confidential-containers/kbs/repository +Volume=nebula-ca:/opt/confidential-containers/kbs/nebula-ca +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Exec=--config-file \ + /opt/confidential-containers/kbs/config/kbs-config.toml + +[Install] +WantedBy=default.target diff --git a/configs/trustee-gcp/containers/key-generation.container b/configs/trustee-gcp/containers/key-generation.container new file mode 100644 index 0000000..d190eff --- /dev/null +++ b/configs/trustee-gcp/containers/key-generation.container @@ -0,0 +1,17 @@ +[Unit] +Description=Trustee Key Generator +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=keyprovider +Image=docker.io/alpine/openssl:latest +Entrypoint=/bin/ash +Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Exec=-c "if [ ! -s /opt/confidential-containers/kbs/user-keys/private.key ]; then \ + /usr/bin/openssl genpkey -algorithm ed25519 > /opt/confidential-containers/kbs/user-keys/private.key && \ + /usr/bin/openssl pkey -in /opt/confidential-containers/kbs/user-keys/private.key -pubout \ + -out /opt/confidential-containers/kbs/user-keys/public.pub; else exit 0; fi;" + +[Install] +WantedBy=default.target diff --git a/configs/trustee-gcp/containers/nginx.container b/configs/trustee-gcp/containers/nginx.container new file mode 100644 index 0000000..6225d9d --- /dev/null +++ b/configs/trustee-gcp/containers/nginx.container @@ -0,0 +1,14 @@ +[Unit] +Description=nginx HTTP server emulating registration server +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=nginx +Image=quay.io/fedora/nginx-126:latest +PublishPort=8000:8080 +Volume=/srv/www:/opt/app-root/src:z +Exec=nginx -g "daemon off;" + +[Install] +WantedBy=default.target diff --git a/configs/trustee-gcp/kbs-config.toml b/configs/trustee-gcp/kbs-config.toml new file mode 100644 index 0000000..96972c9 --- /dev/null +++ b/configs/trustee-gcp/kbs-config.toml @@ -0,0 +1,32 @@ +[http_server] +sockets = ["0.0.0.0:8080"] +insecure_http = true + +[admin] +insecure_api = true +auth_public_key = "./keys/public.pub" + + +[attestation_token] +insecure_key = true + +[attestation_service] +type = "coco_as_builtin" +work_dir = "/opt/confidential-containers/attestation-service" +policy_engine = "opa" + +[attestation_service.attestation_token_broker] +type = "Ear" +duration_min = 5 + +[attestation_service.rvps_config] +type = "BuiltIn" + +[attestation_service.rvps_config.storage] +type = "LocalFs" + + +[[plugins]] +name = "resource" +type = "LocalFs" +dir_path = "/opt/confidential-containers/kbs/repository" \ No newline at end of file diff --git a/configs/trustee-gcp/populate_kbs.sh b/configs/trustee-gcp/populate_kbs.sh new file mode 100755 index 0000000..5af9a78 --- /dev/null +++ b/configs/trustee-gcp/populate_kbs.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set -xe + +SECRET_PATH=${SECRET_PATH:=default/machine/root} +KEY=${KEY:=/opt/confidential-containers/kbs/user-keys/private.key} + + +## set reference values for TPM +for i in {7,14}; do + value=$(sudo tpm2_pcrread sha256:${i} | awk -F: '/0x/ {sub(/.*0x/, "", $2); gsub(/[^0-9A-Fa-f]/, "", $2); print tolower($2)}') + podman exec -ti kbs-client \ + kbs-client config \ + --auth-private-key ${KEY} \ + set-sample-reference-value tpm_pcr${i} "${value}" +done + +# Check reference values +podman exec -ti kbs-client \ + kbs-client config \ + --auth-private-key ${KEY} \ + get-reference-values + + +# Create attestation policy +## This policy allows access only if the system’s TPM or SNP +## hardware measurements match trusted reference values +cat << 'EOF' > A_policy.rego +package policy +import rego.v1 + +default hardware := 97 +default executables := 3 +default configuration := 2 + +##### TPM + +hardware := 2 if { + input.tpm.pcr07 in data.reference.tpm_pcr7 + input.tpm.pcr14 in data.reference.tpm_pcr14 +} + +hardware := 2 if { + input.snp.reported_tcb_snp == 27 +} + + +##### Final decision +result := { + "executables": executables, + "hardware": hardware, + "configuration": configuration +} +EOF + +podman cp A_policy.rego kbs-client:/A_policy.rego +podman exec -ti kbs-client \ + kbs-client config \ + --auth-private-key ${KEY} \ + set-attestation-policy \ + --policy-file /A_policy.rego \ + --type rego --id default_cpu + +# Upload resource +cat > secret << EOF +{ "key_type": "oct", "key": "2b442dd5db4478367729ef8bbf2e7480" } +EOF +podman cp secret kbs-client:/secret +podman exec -ti kbs-client \ + kbs-client config \ + --auth-private-key ${KEY} \ + set-resource --resource-file /secret \ + --path ${SECRET_PATH} + + +# Create resource policy +## This policy allows access only if both CPUs report an "affirming" status +## and provide TPM and SNP attestation evidence. +cat << 'EOF' > R_policy.rego +package policy +import rego.v1 + +default allow = false + +allow if { + input["submods"]["cpu0"]["ear.status"] == "affirming" + input["submods"]["cpu1"]["ear.status"] == "affirming" + input["submods"]["cpu1"]["ear.veraison.annotated-evidence"]["tpm"] + input["submods"]["cpu0"]["ear.veraison.annotated-evidence"]["snp"] +} +EOF + +podman cp R_policy.rego kbs-client:/R_policy.rego +podman exec -ti kbs-client \ + kbs-client config \ + --auth-private-key ${KEY} \ + set-resource-policy \ + --policy-file /R_policy.rego \ diff --git a/coreos/Containerfile b/coreos/Containerfile index c392a03..13407de 100644 --- a/coreos/Containerfile +++ b/coreos/Containerfile @@ -2,6 +2,12 @@ ARG BASE FROM quay.io/trusted-execution-clusters/trustee-attester:fedora-b13fd8a as kbc FROM quay.io/trusted-execution-clusters/clevis-pin-trustee as clevis FROM ghcr.io/trusted-execution-clusters/ignition:20260112-85608d6 as ignition + +ARG KBS_CLIENT_IMAGE=quay.io/confidential-clusters/trustee-attester:2025-10-21 +ARG CLEVIS_PIN_TRUSTEE_IMAGE=quay.io/confidential-clusters/clevis-pin-trustee + +FROM $KBS_CLIENT_IMAGE as kbc +FROM $CLEVIS_PIN_TRUSTEE_IMAGE as clevis FROM $BASE COPY ./usr /usr diff --git a/coreos/justfile b/coreos/justfile index 0028ab3..faffc20 100644 --- a/coreos/justfile +++ b/coreos/justfile @@ -21,12 +21,15 @@ image := if os == "scos" { scos_img } else { fcos_img } os_name := if os == "scos" { scos_os } else { fcos_os } label := if os == "scos" { scos_label } else { fcos_label } archive := os + ".ociarchive" +platform := "qemu" +kbc_image := "quay.io/afrosi_rh/kbs-client-image:latest" +clevis_pin_trustee_image := "quay.io/confidential-clusters/clevis-pin-trustee" config := if os == "scos" { scos_config } else { fcos_config } full_name := if os == "scos" { "centos-stream-coreos" } else { "fedora-coreos" } build: - sudo podman build --no-cache --build-arg BASE={{base}} --build-arg COM_COREOS_OSNAME={{label}} -t {{image}} -f Containerfile . + sudo podman build --no-cache --build-arg BASE={{base}} --build-arg COM_COREOS_OSNAME={{label}} --build-arg KBS_CLIENT_IMAGE={{kbc_image}} --build-arg CLEVIS_PIN_TRUSTEE_IMAGE={{clevis_pin_trustee_image}} -t {{image}} -f Containerfile . oci-archive: sudo skopeo copy containers-storage:{{image}} oci-archive:{{archive}} diff --git a/scripts/GCP/README.md b/scripts/GCP/README.md new file mode 100644 index 0000000..42ff537 --- /dev/null +++ b/scripts/GCP/README.md @@ -0,0 +1,63 @@ +# Remote attestation with PCRs and AMD SEV-SNP on GCP using RHCOS + +This guide provides step-by-step instructions for setting up remote attestation using PCRs and AMD SEV-SNP on Google Cloud Platform (GCP) with Red Hat CoreOS (RHCOS). It covers the deployment of a Trustee server and the creation of a custom RHCOS client image that communicates with the Trustee service to fetch encryption keys and decrypt the root image. + + +## Prerequisites + +1. Copy the pull secret from [Red Hat OpenShift](https://console.redhat.com/openshift/create/local) to `~/.config/containers/auth.json` under `auths:quay.io:auth:` +2. Install [gcloud](https://cloud.google.com/sdk/docs/install) +3. Configure a subnet on GCP for the server and client by running `./scripts/network_setup.sh` + + +## Deploy the Trustee Server (KBS) + +1. To deploy the Trustee server, run: +```bash +./scripts/GCP/deploy-trustee.sh -k -b ./configs/trustee-gcp.bu -i +``` +2. After the server is up, populate the KBS with the reference value and add the remote ignition file: +```bash +./scripts/populate-trustee-kbs.sh +``` +(The default hostname is `kbs`) + + +## Deploy the Client + +1. Build a custom RHCOS image by running: + ```bash + cd coreos + just clevis_pin_trustee_image=quay.io/rkaufman/clevis-pin-trustee:latest os=scos base=quay.io/okd/scos-content:4.20.0-okd-scos.6-stream-coreos \ + kbc_image=quay.io/rkaufman/kbs-tpm-snp:v1 platform=gcp build oci-archive osbuild + ``` + +2. Upload the image to GCP by running: + ```bash + ./scripts/GCP/upload_image_gcp.sh + ``` + +3. Deploy the client by running: + ```bash + ./scripts/GCP/deploy-vm.sh -k -b ./configs/luks.bu -n -i -h + ``` + This will create the VM, perform attestation, and decrypt the disk using clevis-pin. + + +## Information About KBS, KBS-Client, and Clevis-Pin + +These are modified versions of [trustee](https://github.com/iroykaufman/trustee/tree/addtpm) and the [guest component](https://github.com/iroykaufman/guest-components/tree/TPM-as-additional-device) to support the TPM as an additional device. + +The changes in the guest component are also included in [PR#1093](https://github.com/confidential-containers/guest-components/pull/1093), and the changes in Trustee are related to [PR#851](https://github.com/confidential-containers/trustee/pull/851), where the most significant change is the removal of the trusted Attestation Key (AK) list. + +This uses a modified version of `clevis-pin-trustee` that adds AK before performing attestation. The source code is available here: [clevis-pin-trustee](https://github.com/iroykaufman/clevis-pin-trustee/tree/create-tpm-ak) + +## Attestation Policy + +The policy only checks hardware for both SEV-SNP and TPM. + +## Resource Policy + +Verify that both devices are affirming and exist. + + diff --git a/scripts/GCP/deploy-trustee.sh b/scripts/GCP/deploy-trustee.sh new file mode 100755 index 0000000..1a44e5f --- /dev/null +++ b/scripts/GCP/deploy-trustee.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +IGNITION_FILE="config.ign" +IGNITION_CONFIG="$(pwd)/configs/${IGNITION_FILE}" + + +TRUSTEE_PORT="" + +set -xe + + +VM_NAME="kbs" + +while getopts "k:b:n:i:" opt; do + case $opt in + k) key=$OPTARG ;; + b) butane=$OPTARG ;; + n) VM_NAME=$OPTARG ;; + i) IMAGE=$OPTARG ;; + \?) echo "Invalid option"; exit 1 ;; + esac +done + + +if [ -z "${key}" ]; then + echo "Please, specify the public ssh key" + exit 1 +fi +if [ -z "${butane}" ]; then + echo "Please, specify the butane configuration file" + exit 1 +fi + + + +bufile=$(mktemp) + +KEY=$(cat "$key") + +sed "s||$KEY|g" "$butane" >"${bufile}" + +podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd)/configs":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/coreos/butane:release \ + --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/trustee-gcp + +chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} + + + +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' + +gcloud compute instances create ${VM_NAME} \ + --image ${IMAGE} \ + --metadata-from-file "user-data=${IGNITION_CONFIG}" \ + --confidential-compute-type "SEV_SNP" \ + --machine-type "${MACHINE_TYPE}" \ + --maintenance-policy terminate \ + --zone "${ZONE}" \ + --subnet "demo-subnet-us-central1" \ + --shielded-vtpm \ + --shielded-integrity-monitoring \ + --shielded-secure-boot \ + diff --git a/scripts/GCP/deploy-vm.sh b/scripts/GCP/deploy-vm.sh new file mode 100755 index 0000000..0b7ebef --- /dev/null +++ b/scripts/GCP/deploy-vm.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +IGNITION_FILE="config.ign" +IGNITION_CONFIG="$(pwd)/${IGNITION_FILE}" + + +TRUSTEE_PORT="" + +set -xe + + +force=false +while getopts "k:b:n:i:h:" opt; do + case $opt in + k) key=$OPTARG ;; + b) butane=$OPTARG ;; + n) VM_NAME=$OPTARG ;; + i) IMAGE=$OPTARG ;; + h) hostname=$OPTARG ;; + \?) echo "Invalid option"; exit 1 ;; + esac +done + + +if [ -z "${key}" ]; then + echo "Please, specify the public ssh key" + exit 1 +fi +if [ -z "${butane}" ]; then + echo "Please, specify the butane configuration file" + exit 1 +fi + + + +bufile=$(mktemp) + +if [ ! -f "$key" ]; then + echo "Error: The specified key file '$key' does not exist." + exit 1 +fi + +KEY=$(cat "$key") + +sed "s||$KEY|g" $butane | sed "s//$hostname/" > "${bufile}" + +podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd)":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/confidential-clusters/butane:clevis-pin-trustee \ + --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/rh-coreos + +chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} + + + +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' + +gcloud compute instances create ${VM_NAME} \ + --image ${IMAGE} \ + --metadata-from-file "user-data=${IGNITION_CONFIG}" \ + --confidential-compute-type "SEV_SNP" \ + --machine-type "${MACHINE_TYPE}" \ + --maintenance-policy terminate \ + --zone "${ZONE}" \ + --subnet "demo-subnet-us-central1" \ + --shielded-vtpm \ + --shielded-integrity-monitoring \ + --shielded-secure-boot + diff --git a/scripts/GCP/network_setup.sh b/scripts/GCP/network_setup.sh new file mode 100644 index 0000000..d0975eb --- /dev/null +++ b/scripts/GCP/network_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +## create a subnet for the server and client VMs +SUBNET_NAME="demo-subnet-us-central1" +gcloud compute networks subnets create ${SUBNET_NAME} \ + --network=default \ + --region=us-central1 \ + --range=10.0.0.0/24 +## allow ssh +RULE_NAME="allow-ssh" +NETWORK_NAME="default" +gcloud compute firewall-rules create ${RULE_NAME} \ + --network=${NETWORK_NAME} \ + --allow=tcp:22 \ + --source-ranges=0.0.0.0/0 \ + --description="Allow SSH from any IP" +## allow tcp on port 8080 for the trustee service +RULE_NAME="allow-tcp" +gcloud compute firewall-rules create ${RULE_NAME} \ + --network=${NETWORK_NAME} \ + --allow=tcp:8080 \ + --source-ranges=0.0.0.0/0 \ + --description="Allow TCP from any IP" \ No newline at end of file diff --git a/scripts/GCP/upload_image_gcp.sh b/scripts/GCP/upload_image_gcp.sh new file mode 100755 index 0000000..4110117 --- /dev/null +++ b/scripts/GCP/upload_image_gcp.sh @@ -0,0 +1,17 @@ +#!/bin/bash + + +BUCKET_NAME=$1 +IMG_NAME=$2 + +## create a bucket to store the image +gsutil mb gs://${BUCKET_NAME} + +gsutil cp ./coreos/${IMG_NAME}.x86_64.tar.gz gs://${BUCKET_NAME}/ + +## create the image in GCP +gcloud compute images create ${IMG_NAME} \ + --source-uri gs://${BUCKET_NAME}/${IMG_NAME}.x86_64.tar.gz \ + --description "My custom ${IMG_NAME} image" \ + --guest-os-features UEFI_COMPATIBLE,GVNIC,VIRTIO_SCSI_MULTIQUEUE,SEV_SNP_CAPABLE,TDX_CAPABLE + diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..fbd56e4 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +create_remote_ign_config () +{ + IP=$1 + # Setup remote ignition config + BUTANE=pin-trustee.bu + IGNITION="${BUTANE%.bu}.ign" + + sed "s//$IP/" configs/remote-ign/${BUTANE} > ${BUTANE} + + podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd):/pwd" \ + --workdir /pwd \ + quay.io/trusted-execution-clusters/butane:clevis-pin-trustee \ + --pretty --strict /pwd/$BUTANE --output "/pwd/$IGNITION" + + echo "./$IGNITION" + +} diff --git a/scripts/populate-trustee-kbs.sh b/scripts/populate-trustee-kbs.sh new file mode 100755 index 0000000..aa4dfd6 --- /dev/null +++ b/scripts/populate-trustee-kbs.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail +# set -x +source common.sh + +if [[ "${#}" > 3 ]]; then + echo "Usage: $0 " + echo "Optional: $0 " + exit 1 +fi + +KEY=$1 +TRUSTEE_PORT=8080 +IP=$2 +if [[ ${IP} == "" ]]; then +# Setup reference values, policies and secrets +until IP="$(./scripts/get-ip.sh trustee)" && [ -n "$IP" ] && curl "http://${IP}:${TRUSTEE_PORT}" >/dev/null 2>&1; do + echo "Waiting for KBS to be available..." + sleep 1 +done +fi +until ssh core@$IP \ + -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + 'sudo /usr/local/bin/populate_kbs.sh'; do + echo "Waiting for KBS to be populated..." + sleep 1 +done + +# Setup remote ignition config +HOSTNAME=$3 +if [[ ${HOSTNAME} == "" ]]; then +HOSTNAME=${IP} +fi +IGNITION=$(create_remote_ign_config $HOSTNAME) +scp -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ./${IGNITION} core@$IP: +# Setup remote web server to serve the ignition file +ssh core@$IP \ + -i "${KEY%.*}" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "sudo mv $IGNITION /srv/www && sudo systemctl restart nginx.service" From 1b62184083427d811a61e33ca2c6ccb8cf557ad2 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Sun, 23 Nov 2025 12:43:49 +0200 Subject: [PATCH 2/2] Update to the latest way of preforming attestation using AK registration Signed-off-by: Roy Kaufman --- README.md | 4 +- configs/{trustee-gcp.bu => trustee.bu} | 8 ++++ .../containers/kbc.container | 2 +- .../containers/kbs.container | 3 +- .../containers/key-generation.container | 0 .../containers/nginx.container | 0 .../trustee/containers/register-ak.container | 13 +++++++ configs/trustee/kbs-client | 13 +++++++ .../{trustee-gcp => trustee}/kbs-config.toml | 3 ++ .../{trustee-gcp => trustee}/populate_kbs.sh | 39 +++++-------------- containerfiles/trustee-attester.container | 2 +- coreos/Containerfile | 13 ++++--- coreos/justfile | 10 +++-- scripts/GCP/README.md | 13 +++---- scripts/GCP/deploy-trustee.sh | 14 +++---- scripts/GCP/deploy-vm.sh | 16 +++++--- scripts/populate-trustee-kbs.sh | 4 +- 17 files changed, 91 insertions(+), 66 deletions(-) rename configs/{trustee-gcp.bu => trustee.bu} (86%) rename configs/{trustee-gcp => trustee}/containers/kbc.container (76%) rename configs/{trustee-gcp => trustee}/containers/kbs.container (81%) rename configs/{trustee-gcp => trustee}/containers/key-generation.container (100%) rename configs/{trustee-gcp => trustee}/containers/nginx.container (100%) create mode 100644 configs/trustee/containers/register-ak.container create mode 100755 configs/trustee/kbs-client rename configs/{trustee-gcp => trustee}/kbs-config.toml (82%) rename configs/{trustee-gcp => trustee}/populate_kbs.sh (64%) diff --git a/README.md b/README.md index 7a43079..e8d5885 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ Build the Fedora CoreOS or Centos Stream CoreOS image with the custom initrd: ```bash cd coreos # Centos Stream CoreOS image -just os=scos build oci-archive osbuild +just os=scos build oci-archive init build-qemu # Fedora CoreOS image -just build oci-archive osbuild +just build oci-archive init build-qemu ``` ### Create local Trustee deployment diff --git a/configs/trustee-gcp.bu b/configs/trustee.bu similarity index 86% rename from configs/trustee-gcp.bu rename to configs/trustee.bu index 7ff4078..14f9016 100644 --- a/configs/trustee-gcp.bu +++ b/configs/trustee.bu @@ -35,6 +35,10 @@ storage: mode: 0755 contents: local: populate_kbs.sh + - path: /usr/local/bin/kbs-client + mode: 0755 + contents: + local: kbs-client - path: /etc/containers/systemd/key-generation.container mode: 0644 contents: @@ -55,5 +59,9 @@ storage: mode: 0644 contents: local: containers/nginx.container + - path: /etc/containers/systemd/register-ak.container + mode: 0644 + contents: + local: containers/register-ak.container diff --git a/configs/trustee-gcp/containers/kbc.container b/configs/trustee/containers/kbc.container similarity index 76% rename from configs/trustee-gcp/containers/kbc.container rename to configs/trustee/containers/kbc.container index 9d06624..da9eed2 100644 --- a/configs/trustee-gcp/containers/kbc.container +++ b/configs/trustee/containers/kbc.container @@ -4,7 +4,7 @@ After=key-generation.container [Container] ContainerName=kbs-client -Image=quay.io/rkaufman/kbs-tpm-snp:v1 +Image=quay.io/trusted-execution-clusters/trustee-attester:TPM-additional-dev Network=host Volume=user-keys:/opt/confidential-containers/kbs/user-keys Exec=tail -f /dev/null diff --git a/configs/trustee-gcp/containers/kbs.container b/configs/trustee/containers/kbs.container similarity index 81% rename from configs/trustee-gcp/containers/kbs.container rename to configs/trustee/containers/kbs.container index 9dc8cc7..5f0d5b3 100644 --- a/configs/trustee-gcp/containers/kbs.container +++ b/configs/trustee/containers/kbs.container @@ -4,7 +4,7 @@ After=key-generation.container [Container] ContainerName=kbs -Image=quay.io/rkaufman/kbs-tpm-snp:v1 +Image=quay.io/trusted-execution-clusters/key-broker-service:fix-TPM-report-data-size Network=host Entrypoint=/usr/local/bin/kbs PublishPort=8080:8080 @@ -13,6 +13,7 @@ Volume=/var/kbs/config/kbs-config.toml:/opt/confidential-containers/kbs/config/k Volume=kbs-storage:/opt/confidential-containers/kbs/repository Volume=nebula-ca:/opt/confidential-containers/kbs/nebula-ca Volume=user-keys:/opt/confidential-containers/kbs/user-keys +Volume=trusted-ak-keys:/etc/tpm/trusted_ak_keys Exec=--config-file \ /opt/confidential-containers/kbs/config/kbs-config.toml diff --git a/configs/trustee-gcp/containers/key-generation.container b/configs/trustee/containers/key-generation.container similarity index 100% rename from configs/trustee-gcp/containers/key-generation.container rename to configs/trustee/containers/key-generation.container diff --git a/configs/trustee-gcp/containers/nginx.container b/configs/trustee/containers/nginx.container similarity index 100% rename from configs/trustee-gcp/containers/nginx.container rename to configs/trustee/containers/nginx.container diff --git a/configs/trustee/containers/register-ak.container b/configs/trustee/containers/register-ak.container new file mode 100644 index 0000000..e07a173 --- /dev/null +++ b/configs/trustee/containers/register-ak.container @@ -0,0 +1,13 @@ +[Unit] +Description=server that allow to register AK +Wants=network-online.target +After=network-online.target + +[Container] +ContainerName=register-ak +Image=quay.io/trusted-execution-clusters/test-server-ak:latest +PublishPort=5001:5001 +Volume=trusted-ak-keys:/data + +[Install] +WantedBy=default.target diff --git a/configs/trustee/kbs-client b/configs/trustee/kbs-client new file mode 100755 index 0000000..5b7797f --- /dev/null +++ b/configs/trustee/kbs-client @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail +# set -x + +KEY="${KEY:=/opt/confidential-containers/kbs/user-keys/private.key}" + +sudo podman exec -ti \ + kbs-client \ + kbs-client \ + config \ + --auth-private-key "${KEY}" \ + "${@}" diff --git a/configs/trustee-gcp/kbs-config.toml b/configs/trustee/kbs-config.toml similarity index 82% rename from configs/trustee-gcp/kbs-config.toml rename to configs/trustee/kbs-config.toml index 96972c9..8303eee 100644 --- a/configs/trustee-gcp/kbs-config.toml +++ b/configs/trustee/kbs-config.toml @@ -25,6 +25,9 @@ type = "BuiltIn" [attestation_service.rvps_config.storage] type = "LocalFs" +[attestation_service.verifier_config.tpm_verifier] +trusted_ak_keys_dir = "/etc/tpm/trusted_ak_keys" +max_trusted_ak_keys = 100 [[plugins]] name = "resource" diff --git a/configs/trustee-gcp/populate_kbs.sh b/configs/trustee/populate_kbs.sh similarity index 64% rename from configs/trustee-gcp/populate_kbs.sh rename to configs/trustee/populate_kbs.sh index 5af9a78..bd5d9c7 100755 --- a/configs/trustee-gcp/populate_kbs.sh +++ b/configs/trustee/populate_kbs.sh @@ -7,19 +7,13 @@ KEY=${KEY:=/opt/confidential-containers/kbs/user-keys/private.key} ## set reference values for TPM -for i in {7,14}; do +for i in {7,4,14}; do value=$(sudo tpm2_pcrread sha256:${i} | awk -F: '/0x/ {sub(/.*0x/, "", $2); gsub(/[^0-9A-Fa-f]/, "", $2); print tolower($2)}') - podman exec -ti kbs-client \ - kbs-client config \ - --auth-private-key ${KEY} \ - set-sample-reference-value tpm_pcr${i} "${value}" + kbs-client set-sample-reference-value tpm_pcr${i} "${value}" done # Check reference values -podman exec -ti kbs-client \ - kbs-client config \ - --auth-private-key ${KEY} \ - get-reference-values +kbs-client get-reference-values # Create attestation policy @@ -38,6 +32,7 @@ default configuration := 2 hardware := 2 if { input.tpm.pcr07 in data.reference.tpm_pcr7 input.tpm.pcr14 in data.reference.tpm_pcr14 + input.tpm.pcr04 in data.reference.tpm_pcr4 } hardware := 2 if { @@ -53,25 +48,15 @@ result := { } EOF -podman cp A_policy.rego kbs-client:/A_policy.rego -podman exec -ti kbs-client \ - kbs-client config \ - --auth-private-key ${KEY} \ - set-attestation-policy \ - --policy-file /A_policy.rego \ - --type rego --id default_cpu +sudo podman cp A_policy.rego kbs-client:/A_policy.rego +kbs-client set-attestation-policy --policy-file /A_policy.rego --type rego --id default_cpu # Upload resource cat > secret << EOF { "key_type": "oct", "key": "2b442dd5db4478367729ef8bbf2e7480" } EOF -podman cp secret kbs-client:/secret -podman exec -ti kbs-client \ - kbs-client config \ - --auth-private-key ${KEY} \ - set-resource --resource-file /secret \ - --path ${SECRET_PATH} - +sudo podman cp secret kbs-client:/secret +kbs-client set-resource --resource-file /secret --path ${SECRET_PATH} # Create resource policy ## This policy allows access only if both CPUs report an "affirming" status @@ -90,9 +75,5 @@ allow if { } EOF -podman cp R_policy.rego kbs-client:/R_policy.rego -podman exec -ti kbs-client \ - kbs-client config \ - --auth-private-key ${KEY} \ - set-resource-policy \ - --policy-file /R_policy.rego \ +sudo podman cp R_policy.rego kbs-client:/R_policy.rego +kbs-client set-resource-policy --policy-file /R_policy.rego diff --git a/containerfiles/trustee-attester.container b/containerfiles/trustee-attester.container index 6d73647..8b51097 100644 --- a/containerfiles/trustee-attester.container +++ b/containerfiles/trustee-attester.container @@ -13,7 +13,7 @@ RUN . /etc/os-release && \ RUN dnf install -y git tss2-devel tpm2-tss-devel cargo openssl-devel perl RUN cd /usr/src/ && \ - git clone https://github.com/confidential-containers/guest-components.git && \ + git clone https://github.com/trusted-execution-clusters/guest-components.git && \ cd guest-components && git checkout ${COMMIT} RUN cd /usr/src/guest-components && \ diff --git a/coreos/Containerfile b/coreos/Containerfile index 13407de..8bb816e 100644 --- a/coreos/Containerfile +++ b/coreos/Containerfile @@ -1,13 +1,14 @@ ARG BASE -FROM quay.io/trusted-execution-clusters/trustee-attester:fedora-b13fd8a as kbc -FROM quay.io/trusted-execution-clusters/clevis-pin-trustee as clevis -FROM ghcr.io/trusted-execution-clusters/ignition:20260112-85608d6 as ignition +ARG TRUSTEE_ATTESTER=quay.io/trusted-execution-clusters/trustee-attester:fedora-b13fd8a +ARG CLEVIS_PIN_TRUSTEE_IMAGE=quay.io/trusted-execution-clusters/clevis-pin-trustee +ARG IGNITION=ghcr.io/trusted-execution-clusters/ignition:20260112-85608d6 -ARG KBS_CLIENT_IMAGE=quay.io/confidential-clusters/trustee-attester:2025-10-21 -ARG CLEVIS_PIN_TRUSTEE_IMAGE=quay.io/confidential-clusters/clevis-pin-trustee +FROM $TRUSTEE_ATTESTER as kbc -FROM $KBS_CLIENT_IMAGE as kbc FROM $CLEVIS_PIN_TRUSTEE_IMAGE as clevis + +FROM $IGNITION as ignition + FROM $BASE COPY ./usr /usr diff --git a/coreos/justfile b/coreos/justfile index faffc20..b0da30a 100644 --- a/coreos/justfile +++ b/coreos/justfile @@ -22,14 +22,13 @@ os_name := if os == "scos" { scos_os } else { fcos_os } label := if os == "scos" { scos_label } else { fcos_label } archive := os + ".ociarchive" platform := "qemu" -kbc_image := "quay.io/afrosi_rh/kbs-client-image:latest" -clevis_pin_trustee_image := "quay.io/confidential-clusters/clevis-pin-trustee" + config := if os == "scos" { scos_config } else { fcos_config } full_name := if os == "scos" { "centos-stream-coreos" } else { "fedora-coreos" } build: - sudo podman build --no-cache --build-arg BASE={{base}} --build-arg COM_COREOS_OSNAME={{label}} --build-arg KBS_CLIENT_IMAGE={{kbc_image}} --build-arg CLEVIS_PIN_TRUSTEE_IMAGE={{clevis_pin_trustee_image}} -t {{image}} -f Containerfile . + sudo podman build --no-cache --build-arg BASE={{base}} --build-arg COM_COREOS_OSNAME={{label}} -t {{image}} -f Containerfile . oci-archive: sudo skopeo copy containers-storage:{{image}} oci-archive:{{archive}} @@ -83,3 +82,8 @@ azure: {{cosa_function}} cd cache cosa osbuild azure +gcp: + #!/usr/bin/env bash + {{cosa_function}} + cd cache + cosa osbuild gcp diff --git a/scripts/GCP/README.md b/scripts/GCP/README.md index 42ff537..5261da1 100644 --- a/scripts/GCP/README.md +++ b/scripts/GCP/README.md @@ -14,7 +14,7 @@ This guide provides step-by-step instructions for setting up remote attestation 1. To deploy the Trustee server, run: ```bash -./scripts/GCP/deploy-trustee.sh -k -b ./configs/trustee-gcp.bu -i +./scripts/GCP/deploy-trustee.sh -k -b ./configs/trustee.bu -i ``` 2. After the server is up, populate the KBS with the reference value and add the remote ignition file: ```bash @@ -28,8 +28,7 @@ This guide provides step-by-step instructions for setting up remote attestation 1. Build a custom RHCOS image by running: ```bash cd coreos - just clevis_pin_trustee_image=quay.io/rkaufman/clevis-pin-trustee:latest os=scos base=quay.io/okd/scos-content:4.20.0-okd-scos.6-stream-coreos \ - kbc_image=quay.io/rkaufman/kbs-tpm-snp:v1 platform=gcp build oci-archive osbuild + just os=scos build oci-archive init gcp ``` 2. Upload the image to GCP by running: @@ -39,18 +38,16 @@ This guide provides step-by-step instructions for setting up remote attestation 3. Deploy the client by running: ```bash - ./scripts/GCP/deploy-vm.sh -k -b ./configs/luks.bu -n -i -h + ./scripts/GCP/deploy-vm.sh -k -b ./configs/ak.bu -n -i -h ``` This will create the VM, perform attestation, and decrypt the disk using clevis-pin. ## Information About KBS, KBS-Client, and Clevis-Pin -These are modified versions of [trustee](https://github.com/iroykaufman/trustee/tree/addtpm) and the [guest component](https://github.com/iroykaufman/guest-components/tree/TPM-as-additional-device) to support the TPM as an additional device. +These are modified versions of [guest component](https://github.com/iroykaufman/guest-components/tree/TPM-as-additional-device) to support the TPM as an additional device. -The changes in the guest component are also included in [PR#1093](https://github.com/confidential-containers/guest-components/pull/1093), and the changes in Trustee are related to [PR#851](https://github.com/confidential-containers/trustee/pull/851), where the most significant change is the removal of the trusted Attestation Key (AK) list. - -This uses a modified version of `clevis-pin-trustee` that adds AK before performing attestation. The source code is available here: [clevis-pin-trustee](https://github.com/iroykaufman/clevis-pin-trustee/tree/create-tpm-ak) +The changes in the guest component are also included in [PR#1093](https://github.com/confidential-containers/guest-components/pull/1093). ## Attestation Policy diff --git a/scripts/GCP/deploy-trustee.sh b/scripts/GCP/deploy-trustee.sh index 1a44e5f..91626e1 100755 --- a/scripts/GCP/deploy-trustee.sh +++ b/scripts/GCP/deploy-trustee.sh @@ -8,15 +8,19 @@ TRUSTEE_PORT="" set -xe - +## Default values VM_NAME="kbs" +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' -while getopts "k:b:n:i:" opt; do +while getopts "k:b:n:i:z:m:" opt; do case $opt in k) key=$OPTARG ;; b) butane=$OPTARG ;; n) VM_NAME=$OPTARG ;; i) IMAGE=$OPTARG ;; + z) ZONE=$OPTARG ;; + m) MACHINE_TYPE=$OPTARG ;; \?) echo "Invalid option"; exit 1 ;; esac done @@ -41,15 +45,11 @@ sed "s||$KEY|g" "$butane" >"${bufile}" podman run --interactive --rm --security-opt label=disable \ --volume "$(pwd)/configs":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/coreos/butane:release \ - --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/trustee-gcp + --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/trustee chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} - -ZONE='us-central1-a' -MACHINE_TYPE='n2d-standard-2' - gcloud compute instances create ${VM_NAME} \ --image ${IMAGE} \ --metadata-from-file "user-data=${IGNITION_CONFIG}" \ diff --git a/scripts/GCP/deploy-vm.sh b/scripts/GCP/deploy-vm.sh index 0b7ebef..b69eb89 100755 --- a/scripts/GCP/deploy-vm.sh +++ b/scripts/GCP/deploy-vm.sh @@ -8,15 +8,20 @@ TRUSTEE_PORT="" set -xe +## Default values +VM_NAME="vm" +ZONE='us-central1-a' +MACHINE_TYPE='n2d-standard-2' -force=false -while getopts "k:b:n:i:h:" opt; do +while getopts "k:b:n:i:h:z:m:" opt; do case $opt in k) key=$OPTARG ;; b) butane=$OPTARG ;; n) VM_NAME=$OPTARG ;; i) IMAGE=$OPTARG ;; h) hostname=$OPTARG ;; + z) ZONE=$OPTARG ;; + m) MACHINE_TYPE=$OPTARG ;; \?) echo "Invalid option"; exit 1 ;; esac done @@ -44,16 +49,15 @@ KEY=$(cat "$key") sed "s||$KEY|g" $butane | sed "s//$hostname/" > "${bufile}" -podman run --interactive --rm --security-opt label=disable \ - --volume "$(pwd)":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/confidential-clusters/butane:clevis-pin-trustee \ +sudo podman run --interactive --rm --security-opt label=disable \ + --volume "$(pwd)":/pwd -v "${bufile}":/config.bu:z --workdir /pwd quay.io/trusted-execution-clusters/butane:attestation \ --pretty --strict /config.bu --output "/pwd/${IGNITION_FILE}" -d /pwd/rh-coreos chcon --verbose --type svirt_home_t ${IGNITION_CONFIG} -ZONE='us-central1-a' -MACHINE_TYPE='n2d-standard-2' + gcloud compute instances create ${VM_NAME} \ --image ${IMAGE} \ diff --git a/scripts/populate-trustee-kbs.sh b/scripts/populate-trustee-kbs.sh index aa4dfd6..68b30e1 100755 --- a/scripts/populate-trustee-kbs.sh +++ b/scripts/populate-trustee-kbs.sh @@ -2,7 +2,7 @@ set -euo pipefail # set -x -source common.sh +source ./scripts/common.sh if [[ "${#}" > 3 ]]; then echo "Usage: $0 " @@ -24,7 +24,7 @@ until ssh core@$IP \ -i "${KEY%.*}" \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ - 'sudo /usr/local/bin/populate_kbs.sh'; do + 'populate_kbs.sh'; do echo "Waiting for KBS to be populated..." sleep 1 done