From edc4a4583c30492e80a555331c6f62f70188cde5 Mon Sep 17 00:00:00 2001 From: "yousef.sadr" Date: Tue, 30 Jun 2026 15:13:46 -0400 Subject: [PATCH 01/11] ci(podvm): build tdx-snp-nvidia image variant Co-authored-by: Cursor --- .github/workflows/build-podvm-cohere.yaml | 36 +++- .github/workflows/deploy-azure-cohere.yaml | 236 +++++++++++++++++++++ .github/workflows/deploy-gcp-cohere.yaml | 31 ++- src/cloud-api-adaptor/podvm/Makefile.inc | 2 +- src/cloud-api-adaptor/versions.yaml | 7 +- 5 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/deploy-azure-cohere.yaml diff --git a/.github/workflows/build-podvm-cohere.yaml b/.github/workflows/build-podvm-cohere.yaml index 4c92d98124..e8759495b1 100644 --- a/.github/workflows/build-podvm-cohere.yaml +++ b/.github/workflows/build-podvm-cohere.yaml @@ -21,6 +21,11 @@ on: required: false type: boolean default: true + deploy_azure: + description: "Publish to Azure Compute Gallery (needs Azure CC infra)." + required: false + type: boolean + default: false permissions: actions: read # download smoke measurements artifact in the GCP smoke workflow @@ -30,10 +35,11 @@ permissions: packages: write # push OCI artifact to GHCR env: - # tdx-nvidia selects the AA OCI tag suffix (-tdx-nvidia_x86_64) that - # bundles libnvat.so* alongside the binary. The fork's publish-artifacts - # workflow produces this variant; upstream guest-components does not. - TEE_PLATFORM: tdx-nvidia + # tdx-snp-nvidia selects the AA OCI tag suffix + # (-tdx-snp-nvidia_x86_64). + # The Cohere fork's variant bundles TDX + SNP + Azure SNP-vTPM + NVIDIA GPU + # attesters and includes libnvat.so* alongside the binary. + TEE_PLATFORM: tdx-snp-nvidia OCI_IMAGE: ghcr.io/${{ github.repository }}/podvm VERSIONS_YAML: src/cloud-api-adaptor/versions.yaml SMOKE_MACHINE_TYPE: a3-highgpu-1g @@ -455,3 +461,25 @@ jobs: GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }} DEBUG_SSH_PRIVATE_KEY: ${{ secrets.DEBUG_SSH_PRIVATE_KEY }} + + # --------------------------------------------------------------------------- + # Deploy to Azure Compute Gallery — opt-in; needs the Azure CC infra. + # --------------------------------------------------------------------------- + deploy-azure: + name: Deploy ${{ matrix.profile }} to Azure + needs: [meta, build] + if: | + needs.build.result == 'success' && + github.event_name == 'workflow_dispatch' && + inputs.deploy_azure + strategy: + fail-fast: false + matrix: + profile: [release, debug] + uses: ./.github/workflows/deploy-azure-cohere.yaml + with: + image_tag: ${{ matrix.profile == 'release' && needs.meta.outputs.image_tag_release || needs.meta.outputs.image_tag_debug }} + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/.github/workflows/deploy-azure-cohere.yaml b/.github/workflows/deploy-azure-cohere.yaml new file mode 100644 index 0000000000..4e5a5a7d00 --- /dev/null +++ b/.github/workflows/deploy-azure-cohere.yaml @@ -0,0 +1,236 @@ +name: Deploy PodVM to Azure (Cohere) + +# Publishes a podvm artifact to an Azure Compute Gallery image version flagged +# ConfidentialVmSupported (x86_64/Gen2) for SEV-SNP VMs (e.g. NCCadsH100v5). +# Requires (infra-azure): +# secrets: AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID (OIDC app +# with Contributor + Storage Blob Data Contributor) +# vars: AZURE_RESOURCE_GROUP, AZURE_PODVM_GALLERY_NAME, +# AZURE_PODVM_STORAGE_ACCOUNT, AZURE_PODVM_BLOB_CONTAINER, +# AZURE_LOCATION + +on: + workflow_call: + inputs: + image_tag: + description: "OCI artifact tag in GHCR to deploy" + required: true + type: string + secrets: + AZURE_CLIENT_ID: + required: true + AZURE_TENANT_ID: + required: true + AZURE_SUBSCRIPTION_ID: + required: true + + workflow_dispatch: + inputs: + image_tag: + description: "OCI artifact tag in GHCR to deploy" + required: true + type: string + +permissions: + id-token: write # Azure login via OIDC federated credentials + contents: read # actions/checkout + packages: read # pull OCI artifact from GHCR + attestations: read # gh attestation verify + +env: + OCI_IMAGE: ghcr.io/${{ github.repository }}/podvm + VERSIONS_YAML: src/cloud-api-adaptor/versions.yaml + +jobs: + deploy: + name: Azure + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends qemu-utils + command -v yq || sudo snap install yq + ORAS_VERSION=$(yq '.tools.oras' "$VERSIONS_YAML") + curl -fsSLO "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" + tar -xzf "oras_${ORAS_VERSION}_linux_amd64.tar.gz" oras + sudo mv oras /usr/local/bin/ + oras version + qemu-img --version + + - name: Login to GHCR + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify attestation + id: verify + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAG: ${{ inputs.image_tag }} + run: | + IMAGE_DIGEST=$(oras resolve "${OCI_IMAGE}:${IMAGE_TAG}") + IMAGE_REF="${OCI_IMAGE}@${IMAGE_DIGEST}" + gh attestation verify \ + "oci://${IMAGE_REF}" \ + --repo "${GITHUB_REPOSITORY}" + echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT" + echo "Attestation verified for ${IMAGE_REF}" + + - name: Pull artifact and read measurements + id: pull + env: + IMAGE_REF: ${{ steps.verify.outputs.image_ref }} + run: | + mkdir -p /tmp/podvm + cd /tmp/podvm + oras pull "${IMAGE_REF}" + ls -lh + + if [ ! -f measurements.json ]; then + echo "ERROR: measurements.json not found in artifact - build is broken" + exit 1 + fi + cat measurements.json + + IMAGE_NAME=$(jq -r '.image_name' measurements.json) + IMAGE_FAMILY=$(jq -r '.image_family // ""' measurements.json) + if [ -z "$IMAGE_NAME" ] || [ "$IMAGE_NAME" = "null" ] || [ -z "$IMAGE_FAMILY" ]; then + echo "ERROR: image_name/image_family missing in measurements.json" >&2 + exit 1 + fi + + # Definition names: [A-Za-z0-9._-], <=80 chars. + IMAGE_DEF=$(echo "$IMAGE_FAMILY" | sed 's/[^A-Za-z0-9._-]/-/g' | cut -c1-80) + + # Versions need numeric Major.Minor.Patch, no leading zeros. + MAJOR=$(date -u +%Y) + MINOR=$(date -u +%m%d | sed 's/^0*//'); MINOR=${MINOR:-0} + PATCH=$(date -u +%H%M | sed 's/^0*//'); PATCH=${PATCH:-0} + IMAGE_VERSION="${MAJOR}.${MINOR}.${PATCH}" + + { + echo "image_name=$IMAGE_NAME" + echo "image_def=$IMAGE_DEF" + echo "image_version=$IMAGE_VERSION" + echo "tee_platform=$(jq -r '.tee_platform // "tdx"' measurements.json)" + echo "distro=$(jq -r '.distro' measurements.json)" + echo "profile=$(jq -r '.profile' measurements.json)" + echo "caa_commit=$(jq -r '.caa_commit' measurements.json)" + } >> "$GITHUB_OUTPUT" + + - name: Extract raw disk and convert to fixed VHD + id: vhd + run: | + cd /tmp/podvm + tar -xzf disk.tar.gz + # Azure needs a fixed-format VHD with an exact 1 MiB-aligned size. + qemu-img convert -f raw -O vpc -o subformat=fixed,force_size \ + disk.raw disk.vhd + ls -lh disk.vhd + + - name: Azure login (OIDC) + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Publish confidential gallery image version + env: + RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + GALLERY: ${{ vars.AZURE_PODVM_GALLERY_NAME }} + STORAGE_ACCOUNT: ${{ vars.AZURE_PODVM_STORAGE_ACCOUNT }} + BLOB_CONTAINER: ${{ vars.AZURE_PODVM_BLOB_CONTAINER }} + LOCATION: ${{ vars.AZURE_LOCATION }} + IMAGE_NAME: ${{ steps.pull.outputs.image_name }} + IMAGE_DEF: ${{ steps.pull.outputs.image_def }} + IMAGE_VERSION: ${{ steps.pull.outputs.image_version }} + TEE_PLATFORM: ${{ steps.pull.outputs.tee_platform }} + run: | + set -euo pipefail + for v in RESOURCE_GROUP GALLERY STORAGE_ACCOUNT BLOB_CONTAINER LOCATION; do + if [ -z "${!v}" ]; then + echo "ERROR: required Azure var '$v' is empty - configure the infra-azure prerequisites" >&2 + exit 1 + fi + done + + BLOB_NAME="${IMAGE_NAME}.vhd" + BLOB_URI="https://${STORAGE_ACCOUNT}.blob.core.windows.net/${BLOB_CONTAINER}/${BLOB_NAME}" + + echo "Uploading staging VHD page blob: $BLOB_URI" + az storage blob upload \ + --account-name "$STORAGE_ACCOUNT" \ + --container-name "$BLOB_CONTAINER" \ + --name "$BLOB_NAME" \ + --file /tmp/podvm/disk.vhd \ + --type page \ + --auth-mode login \ + --overwrite true + + # Create the ConfidentialVmSupported definition on first use. + if ! az sig image-definition show \ + --resource-group "$RESOURCE_GROUP" \ + --gallery-name "$GALLERY" \ + --gallery-image-definition "$IMAGE_DEF" >/dev/null 2>&1; then + echo "Creating image definition $IMAGE_DEF (ConfidentialVmSupported)" + az sig image-definition create \ + --resource-group "$RESOURCE_GROUP" \ + --gallery-name "$GALLERY" \ + --gallery-image-definition "$IMAGE_DEF" \ + --publisher cohere \ + --offer podvm \ + --sku "$IMAGE_DEF" \ + --os-type Linux \ + --os-state Generalized \ + --hyper-v-generation V2 \ + --architecture x64 \ + --features SecurityType=ConfidentialVmSupported + fi + + if az sig image-version show \ + --resource-group "$RESOURCE_GROUP" \ + --gallery-name "$GALLERY" \ + --gallery-image-definition "$IMAGE_DEF" \ + --gallery-image-version "$IMAGE_VERSION" >/dev/null 2>&1; then + echo "ERROR: image version $IMAGE_DEF:$IMAGE_VERSION already exists" >&2 + echo "Gallery versions are immutable; re-run to bump the time-based patch." >&2 + exit 1 + fi + + echo "Creating gallery image version $IMAGE_DEF:$IMAGE_VERSION" + az sig image-version create \ + --resource-group "$RESOURCE_GROUP" \ + --gallery-name "$GALLERY" \ + --gallery-image-definition "$IMAGE_DEF" \ + --gallery-image-version "$IMAGE_VERSION" \ + --location "$LOCATION" \ + --os-vhd-storage-account "$STORAGE_ACCOUNT" \ + --os-vhd-uri "$BLOB_URI" \ + --tags "tee=${TEE_PLATFORM}" "image_name=${IMAGE_NAME}" "caa_commit=${{ steps.pull.outputs.caa_commit }}" + + echo "Cleaning up staging VHD blob" + az storage blob delete \ + --account-name "$STORAGE_ACCOUNT" \ + --container-name "$BLOB_CONTAINER" \ + --name "$BLOB_NAME" \ + --auth-mode login || true + + { + echo "### PodVM Azure gallery image published" + echo "" + echo "| Field | Value |" + echo "| --- | --- |" + echo "| Gallery | \`${GALLERY}\` |" + echo "| Definition | \`${IMAGE_DEF}\` |" + echo "| Version | \`${IMAGE_VERSION}\` |" + echo "| Security type | ConfidentialVmSupported |" + echo "| TEE | ${TEE_PLATFORM} |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/deploy-gcp-cohere.yaml b/.github/workflows/deploy-gcp-cohere.yaml index 243734f522..d478e50a8b 100644 --- a/.github/workflows/deploy-gcp-cohere.yaml +++ b/.github/workflows/deploy-gcp-cohere.yaml @@ -4,7 +4,7 @@ on: workflow_call: inputs: image_tag: - description: "OCI artifact tag in GHCR (e.g. podvm-ubuntu-tdx-nvidia-release-v1-0-0)" + description: "OCI artifact tag in GHCR (e.g. podvm-ubuntu-tdx-snp-nvidia-release-v1-0-0)" required: true type: string secrets: @@ -27,7 +27,6 @@ permissions: attestations: read # gh attestation verify env: - TEE_PLATFORM: tdx OCI_IMAGE: ghcr.io/${{ github.repository }}/podvm PODVM_GCP_IMAGE_PROJECT: cohere-artifacts VERSIONS_YAML: src/cloud-api-adaptor/versions.yaml @@ -118,6 +117,7 @@ jobs: echo "caa_commit_short=$(jq -r '.caa_commit[:7]' measurements.json)" echo "caa_version=$(jq -r '.caa_version' measurements.json)" echo "caa_ref=$(jq -r '.caa_ref' measurements.json)" + echo "tee_platform=$(jq -r '.tee_platform' measurements.json)" echo "rtmr2=$(jq -r '.rtmr2_sha384 // ""' measurements.json)" echo "kernel=$(jq -r '.kernel // ""' measurements.json)" echo "build_date=$(jq -r '.build_date // ""' measurements.json)" @@ -156,6 +156,7 @@ jobs: CAA_COMMIT_SHORT: ${{ steps.pull.outputs.caa_commit_short }} CAA_REF: ${{ steps.pull.outputs.caa_ref }} PROFILE: ${{ steps.pull.outputs.profile }} + TEE_PLATFORM: ${{ steps.pull.outputs.tee_platform }} RTMR2: ${{ steps.pull.outputs.rtmr2 }} KERNEL: ${{ steps.pull.outputs.kernel }} BUILD_DATE: ${{ steps.pull.outputs.build_date }} @@ -174,6 +175,30 @@ jobs: exit 1 fi + if [ -z "$TEE_PLATFORM" ] || [ "$TEE_PLATFORM" = "null" ]; then + echo "ERROR: tee_platform is missing from measurements.json" >&2 + exit 1 + fi + + case "$TEE_PLATFORM" in + tdx-snp-nvidia) + GUEST_OS_FEATURES="TDX_CAPABLE,SEV_SNP_CAPABLE,UEFI_COMPATIBLE,GVNIC" + ;; + tdx) + GUEST_OS_FEATURES="TDX_CAPABLE,UEFI_COMPATIBLE,GVNIC" + ;; + snp|amd) + GUEST_OS_FEATURES="SEV_SNP_CAPABLE,UEFI_COMPATIBLE,GVNIC" + ;; + sev) + GUEST_OS_FEATURES="SEV_CAPABLE,UEFI_COMPATIBLE,GVNIC" + ;; + *) + echo "ERROR: unsupported tee_platform '$TEE_PLATFORM'" >&2 + exit 1 + ;; + esac + RAW_IMAGE="${IMAGE_NAME}-import" if gcloud compute images describe "$RAW_IMAGE" --project="$GCP_PROJECT" &>/dev/null; then echo "Deleting stale intermediate image $RAW_IMAGE" @@ -204,7 +229,7 @@ jobs: --family="$IMAGE_FAMILY" \ --description="podvm: distro=${DISTRO} tee=${TEE_PLATFORM} profile=${PROFILE} class=${IMAGE_CLASS} caa=${CAA_VERSION} commit=${CAA_COMMIT} rtmr2=${RTMR2} oci=${OCI_IMAGE}:${IMAGE_TAG}" \ --labels="$LABELS" \ - --guest-os-features=TDX_CAPABLE,UEFI_COMPATIBLE,GVNIC + --guest-os-features="$GUEST_OS_FEATURES" gcloud compute images delete "$RAW_IMAGE" \ --project="$GCP_PROJECT" --quiet diff --git a/src/cloud-api-adaptor/podvm/Makefile.inc b/src/cloud-api-adaptor/podvm/Makefile.inc index 7b705c4928..cdf880c68a 100644 --- a/src/cloud-api-adaptor/podvm/Makefile.inc +++ b/src/cloud-api-adaptor/podvm/Makefile.inc @@ -190,7 +190,7 @@ $(PAUSE): | $(PAUSE_SRC) $(UMOCI_SRC)/umoci $(ATTESTATION_AGENT): $(FORCE_TARGET) $(call pull_gc_artifact,attestation-agent,$(TEE_PLATFORM)_$(ARCH)) tar xvJpf attestation-agent.tar.xz - # Cohere fork's tdx-nvidia AA tarball bundles libnvat.so* (NVIDIA + # Cohere fork's tdx-snp-nvidia AA tarball bundles libnvat.so* (NVIDIA # attestation-sdk-cpp) under lib/. Install it into the podvm rootfs so # the dynamically linked AA can resolve it at runtime. Other AA # variants ship a binary-only tarball, in which case lib/ is absent diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index 74dc67b774..cfcf9a5ab5 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -66,13 +66,14 @@ oci: registry: ghcr.io/kata-containers/cached-artefacts reference: 8dccf4cf37aeea4b6c2caacf3e61510d6eef2f71 # v3.29.0 # Cohere fork of guest-components. The fork's publish-artifacts workflow - # publishes attestation-agent (incl. a tdx-nvidia variant that bundles - # libnvat.so), confidential-data-hub, and api-server-rest to GHCR. + # publishes attestation-agent (incl. a tdx-snp-nvidia variant that bundles + # TDX + SNP + Azure SNP-vTPM + NVIDIA GPU attesters and libnvat.so), + # confidential-data-hub, and api-server-rest to GHCR. # Bump `reference` to the 40-char SHA whose publish-artifacts run has # completed successfully (visible on the GHCR Packages page). guest-components: registry: ghcr.io/cohere-ai/guest-components - reference: f475b74f94978952804c2ce5205b3aad16076117 + reference: 73bb7fd6613db8c738d56d8975cd5cf05c46fa4f kata-agent-ctl: registry: ghcr.io/kata-containers/cached-artefacts reference: 3.29.0 From 84e6078a8861861a7ff0ea43edaf0ffd95c0f334 Mon Sep 17 00:00:00 2001 From: "yousef.sadr" Date: Tue, 30 Jun 2026 16:53:09 -0400 Subject: [PATCH 02/11] fix: allow tdx-snp-nvidia-aa-v2 branch in provenance verification Co-authored-by: Cursor --- src/cloud-api-adaptor/hack/verify-provenance.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cloud-api-adaptor/hack/verify-provenance.sh b/src/cloud-api-adaptor/hack/verify-provenance.sh index 0f83b99483..4dbfb0722b 100755 --- a/src/cloud-api-adaptor/hack/verify-provenance.sh +++ b/src/cloud-api-adaptor/hack/verify-provenance.sh @@ -129,8 +129,8 @@ if [ "$workflow_trigger" != "push" ] && [ "$workflow_trigger" != "workflow_dispa verification_failed="1" fi -if [ "$workflow_ref" != "refs/heads/main" ] && [ "$workflow_ref" != "refs/heads/cohere" ]; then - echo "Workflow ref mismatch: expected refs/heads/main or refs/heads/cohere, got $workflow_ref" +if [ "$workflow_ref" != "refs/heads/main" ] && [ "$workflow_ref" != "refs/heads/cohere" ] && [ "$workflow_ref" != "refs/heads/yousef/tdx-snp-nvidia-aa-v2" ]; then + echo "Workflow ref mismatch: expected refs/heads/main, refs/heads/cohere, or refs/heads/yousef/tdx-snp-nvidia-aa-v2, got $workflow_ref" verification_failed="1" fi From 61b5476159d976ec4b9ba003991f2637f1597328 Mon Sep 17 00:00:00 2001 From: "yousef.sadr" Date: Tue, 30 Jun 2026 17:45:02 -0400 Subject: [PATCH 03/11] fix(podvm): bump nvidia 580 pins to 580.173.02-1ubuntu1 Resolve broken-package build failure: the CUDA apt repo now serves 580.173.02-1ubuntu1, which conflicted with the pinned 580.159.04-1ubuntu1 transitive deps. Bump all four nvidia-* pins to the currently available version. Co-authored-by: Cursor --- .../mkosi.images/system/mkosi.conf.d/ubuntu.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf index 2a6b0ceb53..01ee800f40 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf @@ -21,10 +21,10 @@ Packages= iptables e2fsprogs cryptsetup - nvidia-driver-580-open=580.159.04-1ubuntu1 - nvidia-persistenced=580.159.04-1ubuntu1 - nvidia-fabricmanager=580.159.04-1ubuntu1 - libnvidia-nscq=580.159.04-1ubuntu1 + nvidia-driver-580-open=580.173.02-1ubuntu1 + nvidia-persistenced=580.173.02-1ubuntu1 + nvidia-fabricmanager=580.173.02-1ubuntu1 + libnvidia-nscq=580.173.02-1ubuntu1 nvidia-container-toolkit=1.19.1-1 RemoveFiles=/etc/issue From 93c4764f1124ad74324b57eb6dbdd8f1193d3220 Mon Sep 17 00:00:00 2001 From: "yousef.sadr" Date: Thu, 2 Jul 2026 14:24:51 -0400 Subject: [PATCH 04/11] fix(deploy-gcp-cohere): update GUEST_OS_FEATURES for tdx-snp-nvidia to include SEV_CAPABLE --- .github/workflows/deploy-gcp-cohere.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-gcp-cohere.yaml b/.github/workflows/deploy-gcp-cohere.yaml index d478e50a8b..b849d8fabf 100644 --- a/.github/workflows/deploy-gcp-cohere.yaml +++ b/.github/workflows/deploy-gcp-cohere.yaml @@ -182,7 +182,7 @@ jobs: case "$TEE_PLATFORM" in tdx-snp-nvidia) - GUEST_OS_FEATURES="TDX_CAPABLE,SEV_SNP_CAPABLE,UEFI_COMPATIBLE,GVNIC" + GUEST_OS_FEATURES="TDX_CAPABLE,SEV_CAPABLE,SEV_SNP_CAPABLE,UEFI_COMPATIBLE,GVNIC" ;; tdx) GUEST_OS_FEATURES="TDX_CAPABLE,UEFI_COMPATIBLE,GVNIC" From 59293f2faabe8d2d2fdbef823098e1be7262b966 Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 06:11:32 -0400 Subject: [PATCH 05/11] fix(ci): round podvm raw disk up to whole MiB before VHD convert Azure gallery image-version create rejects VHDs whose virtual size is not a whole number of MiB ("unsupported virtual size"). mkosi's raw image isn't necessarily 1 MiB-aligned, so round it up with qemu-img resize before the fixed-VHD conversion; force_size preserves the aligned size in the footer. Signed-off-by: Alhassan Khedr --- .github/workflows/deploy-azure-cohere.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-azure-cohere.yaml b/.github/workflows/deploy-azure-cohere.yaml index 4e5a5a7d00..5e5e249f7a 100644 --- a/.github/workflows/deploy-azure-cohere.yaml +++ b/.github/workflows/deploy-azure-cohere.yaml @@ -130,7 +130,18 @@ jobs: run: | cd /tmp/podvm tar -xzf disk.tar.gz - # Azure needs a fixed-format VHD with an exact 1 MiB-aligned size. + # Azure needs a fixed-format VHD whose virtual size is a whole number + # of MiB. mkosi's raw image isn't necessarily 1 MiB-aligned, so round + # it up to the next whole MiB first; qemu-img's force_size then keeps + # that exact aligned size in the fixed VHD footer. Without this, the + # gallery image-version create fails with "unsupported virtual size". + MB=$((1024 * 1024)) + size=$(qemu-img info -f raw --output json disk.raw | jq '.["virtual-size"]') + rounded=$(( ((size + MB - 1) / MB) * MB )) + if [ "$rounded" -ne "$size" ]; then + echo "Rounding raw image $size -> $rounded bytes ($((rounded / MB)) MiB) for Azure alignment" + qemu-img resize -f raw disk.raw "$rounded" + fi qemu-img convert -f raw -O vpc -o subformat=fixed,force_size \ disk.raw disk.vhd ls -lh disk.vhd From 0def16d8100a65e6bfcf29f5ff660efa90026716 Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 10:57:50 -0400 Subject: [PATCH 06/11] fix(podvm): add Hyper-V VMBus drivers to initrd for Azure boot The initrd only bundled virtio/nvme/SATA storage drivers, so on Azure (Hyper-V Gen2, all devices on VMBus) the guest never enumerates its OS disk: it hangs in the initrd waiting for the dm-verity root partitions, times out after 90s, and drops to an emergency shell. Confirmed via the peer pod VM serial console (no hv_vmbus/hv_storvsc, "PCI: System does not support PCI", "Timed out waiting for dev-mapper-root.device"). Add hv_vmbus (VMBus core), hv_storvsc (OS disk), hv_netvsc (networking), and pci_hyperv (VMBus PCI for GPU passthrough) so the image boots on Azure while retaining virtio for GCP/KVM. Signed-off-by: Alhassan Khedr --- .../podvm-mkosi/mkosi.images/system/mkosi.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf index 4ddf83941a..cbaa342a73 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf @@ -16,6 +16,10 @@ KernelInitrdModules=dm-mod ata_piix libata sd_mod + hv_vmbus + hv_storvsc + hv_netvsc + pci_hyperv KernelCommandLine=console=ttyS0 KernelCommandLine=systemd.firstboot=off From f64805506dc2fc7f5fd79a970b548bd51c34603c Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 12:37:07 -0400 Subject: [PATCH 07/11] podvm(ubuntu): build afterburn from source so Azure provisioning completes The Ubuntu podvm image enables afterburn-checkin.service (via the shared preset + drop-in) but never shipped afterburn: it is not packaged for Ubuntu/Debian (only Fedora) and upstream publishes no prebuilt binary. As a result the peer-pod VM boots fine but never runs afterburn --check-in, so it never reports provisioning completion to the Azure fabric. Azure keeps the VM at provisioningState=Creating and the CAA CreateInstance LRO times out, leaving pods stuck in ContainerCreating. Build afterburn from source in the Ubuntu binaries image (builder and target are both ubuntu:24.04, so the dynamic glibc/OpenSSL link is ABI-compatible) and ship both the binary and the upstream base afterburn-checkin.service unit (on Fedora that unit comes from the RPM). Add openssl to the package set for the libssl3 runtime dependency. --- .../system/mkosi.conf.d/ubuntu.conf | 1 + .../podvm/Dockerfile.podvm_binaries.ubuntu | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf index 01ee800f40..80730c1f01 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.images/system/mkosi.conf.d/ubuntu.conf @@ -21,6 +21,7 @@ Packages= iptables e2fsprogs cryptsetup + openssl nvidia-driver-580-open=580.173.02-1ubuntu1 nvidia-persistenced=580.173.02-1ubuntu1 nvidia-fabricmanager=580.173.02-1ubuntu1 diff --git a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.ubuntu b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.ubuntu index d85616268a..2b80326652 100644 --- a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.ubuntu +++ b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.ubuntu @@ -108,5 +108,26 @@ RUN ./hack/cross-build-extras.sh RUN LIBC=gnu make binaries +# afterburn is not packaged for Ubuntu/Debian (unlike Fedora) and upstream +# ships no prebuilt binary, so build it from source. Its Azure --check-in +# reports provisioning completion to the Azure fabric (WireServer), which is +# what flips a VM's provisioningState from Creating to Succeeded. Without it +# the peer-pod VM boots but never signals Azure, so the CAA CreateInstance LRO +# never completes on Ubuntu podvms. Built here against the same ubuntu:24.04 +# glibc/OpenSSL as the target image, so the dynamic link is ABI-compatible. +FROM builder AS afterburn_builder +ARG AFTERBURN_VERSION=5.10.0 +RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable +ENV PATH="/root/.cargo/bin:${PATH}" +RUN git clone --depth 1 -b "v${AFTERBURN_VERSION}" https://github.com/coreos/afterburn /tmp/afterburn && \ + cd /tmp/afterburn && \ + cargo build --release --bin afterburn && \ + install -D -m0755 target/release/afterburn /out/usr/bin/afterburn && \ + install -D -m0644 systemd/afterburn-checkin.service \ + /out/usr/lib/systemd/system/afterburn-checkin.service + FROM scratch COPY --from=podvm_binaries_builder /src/cloud-api-adaptor/podvm/files / +# afterburn binary + base check-in unit (the Cohere drop-in in mkosi.skeleton +# layers on top; on Fedora the base unit comes from the afterburn RPM instead). +COPY --from=afterburn_builder /out/ / From 6ae57be13c044260775efaab619ad87f34b810b9 Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 16:08:01 -0400 Subject: [PATCH 08/11] ci(deploy-azure): satisfy zizmor (job-scoped perms, pinned comments, no inline expansion) Clears the zizmor findings on the Azure publish workflow so PR #53 can merge cleanly into cohere: - move the workflow-level permissions (incl. id-token: write) to job level and set top-level `permissions: {}` (fixes the excessive-permissions error) - pass caa_commit through an env var instead of inlining ${{ }} in run: (fixes the template-injection finding) - correct the SHA-pin version comments to the exact tags (checkout v6.0.2, docker/login-action v4.1.0) - add a per-image-tag job concurrency guard so overlapping publishes can't race on the same immutable gallery version --- .github/workflows/deploy-azure-cohere.yaml | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-azure-cohere.yaml b/.github/workflows/deploy-azure-cohere.yaml index 5e5e249f7a..3f709d70b3 100644 --- a/.github/workflows/deploy-azure-cohere.yaml +++ b/.github/workflows/deploy-azure-cohere.yaml @@ -31,11 +31,7 @@ on: required: true type: string -permissions: - id-token: write # Azure login via OIDC federated credentials - contents: read # actions/checkout - packages: read # pull OCI artifact from GHCR - attestations: read # gh attestation verify +permissions: {} env: OCI_IMAGE: ghcr.io/${{ github.repository }}/podvm @@ -45,8 +41,18 @@ jobs: deploy: name: Azure runs-on: ubuntu-latest + # Serialize publishes per image tag: gallery versions are immutable and the + # patch is time-based, so overlapping runs could collide on the same version. + concurrency: + group: deploy-azure-cohere-${{ inputs.image_tag }} + cancel-in-progress: false + permissions: + id-token: write # Azure login via OIDC federated credentials + contents: read # actions/checkout + packages: read # pull OCI artifact from GHCR + attestations: read # gh attestation verify steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false @@ -63,7 +69,7 @@ jobs: qemu-img --version - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -164,6 +170,7 @@ jobs: IMAGE_DEF: ${{ steps.pull.outputs.image_def }} IMAGE_VERSION: ${{ steps.pull.outputs.image_version }} TEE_PLATFORM: ${{ steps.pull.outputs.tee_platform }} + CAA_COMMIT: ${{ steps.pull.outputs.caa_commit }} run: | set -euo pipefail for v in RESOURCE_GROUP GALLERY STORAGE_ACCOUNT BLOB_CONTAINER LOCATION; do @@ -225,7 +232,7 @@ jobs: --location "$LOCATION" \ --os-vhd-storage-account "$STORAGE_ACCOUNT" \ --os-vhd-uri "$BLOB_URI" \ - --tags "tee=${TEE_PLATFORM}" "image_name=${IMAGE_NAME}" "caa_commit=${{ steps.pull.outputs.caa_commit }}" + --tags "tee=${TEE_PLATFORM}" "image_name=${IMAGE_NAME}" "caa_commit=${CAA_COMMIT}" echo "Cleaning up staging VHD blob" az storage blob delete \ From 1a14a25aba8a7bf1119c89d80db117192d7fc707 Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 19:05:15 -0400 Subject: [PATCH 09/11] fix(podvm): keep api-server-rest retrying until attestation-agent socket is up api-server-rest.path only gates on the CDH socket, but api-server-rest --features all also dials the attestation-agent socket at startup. On the cc-nvidia image the attestation-agent creates its socket a few seconds late (large binary + NVIDIA attester init), so api-server-rest exhausts systemd's default 5-starts/10s limit and fails permanently ("Start request repeated too quickly"), leaving the attestation endpoint on 127.0.0.1:8006 down for the VM's whole lifetime. Add a drop-in that disables the start-rate give-up (StartLimitIntervalSec=0) so Restart=always keeps retrying, plus an ExecStartPre that blocks until both the attestation-agent and CDH sockets exist to avoid the noisy failed-start loop. --- .../10-wait-for-attestation-agent.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/cloud-api-adaptor/podvm/files/etc/systemd/system/api-server-rest.service.d/10-wait-for-attestation-agent.conf diff --git a/src/cloud-api-adaptor/podvm/files/etc/systemd/system/api-server-rest.service.d/10-wait-for-attestation-agent.conf b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/api-server-rest.service.d/10-wait-for-attestation-agent.conf new file mode 100644 index 0000000000..630b1c13c8 --- /dev/null +++ b/src/cloud-api-adaptor/podvm/files/etc/systemd/system/api-server-rest.service.d/10-wait-for-attestation-agent.conf @@ -0,0 +1,16 @@ +[Unit] +# api-server-rest.path only gates on the CDH socket, but `api-server-rest +# --features all` also dials the attestation-agent socket at startup. On the +# cc-nvidia image the attestation-agent (large binary + NVIDIA attester init) +# can (re)create its socket a few seconds after CDH's, so api-server-rest +# exhausts systemd's default start-rate limit (5 starts / 10s) and fails +# permanently with "Start request repeated too quickly" - leaving the +# attestation endpoint on 127.0.0.1:8006 down for the VM's whole lifetime. +# Disable the rate-limit give-up so Restart=always keeps retrying until the +# attestation-agent socket is up. +StartLimitIntervalSec=0 + +[Service] +# Avoid the noisy failed-start loop by blocking until both dependency sockets +# exist before launching api-server-rest. +ExecStartPre=/bin/sh -c 'until test -S /run/confidential-containers/attestation-agent/attestation-agent.sock && test -S /run/confidential-containers/cdh.sock; do sleep 0.5; done' From ed24de7102d9cff2015fba7ec6f9e9a1e0728eed Mon Sep 17 00:00:00 2001 From: Alhassan Khedr Date: Fri, 3 Jul 2026 20:46:16 -0400 Subject: [PATCH 10/11] fix(deps): bump Go toolchain 1.25.10 -> 1.25.11 (govulncheck) Resolves the three Go stdlib advisories flagging PR #53's govulncheck job: GO-2026-5037 (crypto/x509), GO-2026-5038 (mime), GO-2026-5039 (net/textproto), all "Fixed in go1.25.11". Bumps versions.yaml (CI setup-go), the go directive in all five modules, and the pinned golang builder base image digest so shipped binaries are also built against the patched stdlib. Validated locally with GOTOOLCHAIN=go1.25.11: govulncheck reports no vulnerabilities for peerpod-ctrl, csi-wrapper, and webhook. --- src/cloud-api-adaptor/Dockerfile | 6 +++--- src/cloud-api-adaptor/docs/addnewprovider.md | 4 ++-- src/cloud-api-adaptor/go.mod | 2 +- src/cloud-api-adaptor/versions.yaml | 2 +- src/cloud-providers/go.mod | 2 +- src/csi-wrapper/Dockerfile.csi_wrappers | 8 ++++---- src/csi-wrapper/go.mod | 2 +- src/peerpod-ctrl/Dockerfile | 6 +++--- src/peerpod-ctrl/go.mod | 2 +- src/webhook/Dockerfile | 4 ++-- src/webhook/go.mod | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/cloud-api-adaptor/Dockerfile b/src/cloud-api-adaptor/Dockerfile index 6867aa5f04..46441062dc 100644 --- a/src/cloud-api-adaptor/Dockerfile +++ b/src/cloud-api-adaptor/Dockerfile @@ -1,7 +1,7 @@ ARG BUILD_TYPE=dev -# golang:1.25.10 (based on Debian 13 trixie) -ARG BUILDER_BASE=golang@sha256:3965b9511a9bed199f0b4eb146f99c31971a5c0adb3240d9f25e233cbd9e01c8 -# debian:trixie-slim (matches golang:1.25.10 base for library compatibility) +# golang:1.25.11 (based on Debian 13 trixie) +ARG BUILDER_BASE=golang@sha256:f188e8c16ea47a8b22d2bdcf6d9bcd07b63ea7876c199749c07bf31e0ab33bad +# debian:trixie-slim (matches golang:1.25.11 base for library compatibility) # Multi-arch manifest list digest supporting amd64, arm64, ppc64le,, s390x ARG BASE=debian@sha256:109e2c65005bf160609e4ba6acf7783752f8502ad218e298253428690b9eaa4b diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index 6e367b8267..f3934a314d 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -219,8 +219,8 @@ go mod tidy ```bash cat > Dockerfile < Date: Fri, 3 Jul 2026 20:55:22 -0400 Subject: [PATCH 11/11] fix(deps): bump containerd 1.7.29 -> 1.7.33 (govulncheck) Clears the three fixable containerd advisories flagged by govulncheck: GO-2026-5378 (fixed 1.7.32), GO-2026-5475 and GO-2026-5758 (fixed 1.7.33). Also pulls opencontainers/selinux 1.13.0 -> 1.13.1 transitively. The remaining govulncheck findings (docker/docker + 3 containerd advisories) have no upstream fix (Fixed in: N/A), are unrelated to this PR, and exist on the base branch's identical dependency versions. --- src/cloud-api-adaptor/go.mod | 4 ++-- src/cloud-api-adaptor/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod index 7c6e087293..4a1635e732 100644 --- a/src/cloud-api-adaptor/go.mod +++ b/src/cloud-api-adaptor/go.mod @@ -14,7 +14,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.17 github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.299.1 - github.com/containerd/containerd v1.7.29 + github.com/containerd/containerd v1.7.33 github.com/containerd/ttrpc v1.2.7 github.com/coreos/go-iptables v0.6.0 github.com/gogo/protobuf v1.3.2 // indirect @@ -201,7 +201,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20250303011046-260e151b8552 // indirect - github.com/opencontainers/selinux v1.13.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum index d2283450f7..77e62932c5 100644 --- a/src/cloud-api-adaptor/go.sum +++ b/src/cloud-api-adaptor/go.sum @@ -215,8 +215,8 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= -github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= -github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= +github.com/containerd/containerd v1.7.33 h1:iAkYGC/ifR/V+0eR4iXWHNGYUF0DF2PmGV5iz4Irj5M= +github.com/containerd/containerd v1.7.33/go.mod h1:gSbSCVjPCdkfJCjyrzz7aRC+xFlqVbatNpfHfVCYGUM= github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= @@ -527,8 +527,8 @@ github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.9.1-0.20250303011046-260e151b8552 h1:CkXngT0nixZqQUPDVfwVs3GiuhfTqCMk0V+OoHpxIvA= github.com/opencontainers/runtime-tools v0.9.1-0.20250303011046-260e151b8552/go.mod h1:T487Kf80NeF2i0OyVXHiylg217e0buz8pQsa0T791RA= -github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84= -github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=