From 559d849667430ebcae38f063452d8c548e84a511 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Mon, 15 Jun 2026 15:19:45 -0700 Subject: [PATCH 1/6] feat(anc): wire check-hotfix into node wrapper behind ANC_HOTFIX_ENABLED Add a default-off ANC_HOTFIX_ENABLED-gated call to the 2.1b check-hotfix subcommand in aks-node-controller-wrapper.sh, placed before the existing download-hotfix block since check-hotfix refreshes the hotfix pointer that block consumes. The call is fail-open and wrapped defensively so it can never block provisioning. When the flag is unset/non-true the wrapper behaves exactly as before (6-month VHD backward compat). Parameterize HOTFIX_JSON to match the existing path-var pattern and enable shellspec coverage of the download-hotfix branch. Add shellspec tests for flag off, flag on ordering, fail-open, and non-true value handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../artifacts/aks-node-controller-wrapper.sh | 16 +++- .../aks_node_controller_wrapper_spec.sh | 77 ++++++++++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index 310c41da47e..f0626cfeecb 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -7,7 +7,7 @@ done BIN_PATH="${BIN_PATH:-/opt/azure/containers/aks-node-controller}" HOTFIX_BIN="${BIN_PATH}-hotfix" -HOTFIX_JSON="/opt/azure/containers/aks-node-controller-hotfix.json" +HOTFIX_JSON="${HOTFIX_JSON:-/opt/azure/containers/aks-node-controller-hotfix.json}" CONFIG_PATH="${CONFIG_PATH:-/opt/azure/containers/aks-node-controller-config.json}" NBC_CMD_PATH="${NBC_CMD_PATH:-/opt/azure/containers/aks-node-controller-nbc-cmd.sh}" LOGGER_TAG="aks-node-controller-wrapper" @@ -27,6 +27,20 @@ if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then exit 0 fi +# check-hotfix reads the kube-system/anc-hotfix-version ConfigMap and refreshes +# $HOTFIX_JSON, which the download-hotfix block below consumes, so it must run first. +# Gated default-off behind ANC_HOTFIX_ENABLED so existing VHDs behave exactly as before; +# only the literal string "true" enables it. The command is fail-open (always exits 0), +# but we still wrap it defensively so it can never block provisioning. +if [ "${ANC_HOTFIX_ENABLED:-}" = "true" ]; then + log "ANC_HOTFIX_ENABLED=true; running check-hotfix to refresh hotfix pointer" + if "$BIN_PATH" check-hotfix; then + log "ANC check-hotfix completed; hotfix pointer refresh attempted" + else + log "ANC check-hotfix failed; continuing (fail-open)" + fi +fi + if [ -f "$HOTFIX_JSON" ]; then log "Found ANC hotfix config at ${HOTFIX_JSON}; running download-hotfix" if "$BIN_PATH" download-hotfix; then diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh index 2dac2046abd..efb701f33ae 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh @@ -35,11 +35,14 @@ EOF export BIN_PATH="${TEST_DIR}/aks-node-controller" export CONFIG_PATH="${TEST_DIR}/aks-node-controller-config.json" export NBC_CMD_PATH="${TEST_DIR}/aks-node-controller-nbc-cmd.sh" + # Point hotfix pointer at a test-local path (absent by default) so tests never + # touch the production /opt/azure path and can control the download-hotfix branch. + export HOTFIX_JSON="${TEST_DIR}/aks-node-controller-hotfix.json" } cleanup_wrapper_test() { rm -rf "$TEST_DIR" - unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR + unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR HOTFIX_JSON ANC_HOTFIX_ENABLED CHECK_HOTFIX_EXIT } create_fake_aks_node_controller() { @@ -51,6 +54,21 @@ EOF chmod +x "$BIN_PATH" } + # Records each subcommand (first arg) on its own line in calls log so ordering across + # multiple invocations (check-hotfix vs download-hotfix vs provision) is observable. + # CHECK_HOTFIX_EXIT controls the exit code of the check-hotfix invocation only. + create_recording_aks_node_controller() { + cat >"$BIN_PATH" <<'EOF' +#!/bin/sh +printf '%s\n' "$1" >>"${TEST_DIR}/calls" +if [ "$1" = "check-hotfix" ]; then + exit "${CHECK_HOTFIX_EXIT:-0}" +fi +exit 0 +EOF + chmod +x "$BIN_PATH" + } + BeforeEach setup_wrapper_test AfterEach cleanup_wrapper_test @@ -108,4 +126,61 @@ EOF The variable secondArg should eq "--nbc-cmd=${NBC_CMD_PATH}" The variable thirdArg should eq "" End + + It 'does not call check-hotfix when ANC_HOTFIX_ENABLED is unset' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + + When run bash "$SCRIPT" + The status should be success + The output should not include "running check-hotfix" + The path "${TEST_DIR}/calls" should be exist + # Only provision should have been recorded; no check-hotfix line. + calls=$(cat "${TEST_DIR}/calls") + The variable calls should eq "provision" + End + + It 'treats a non-true ANC_HOTFIX_ENABLED value as disabled' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + export ANC_HOTFIX_ENABLED="1" + + When run bash "$SCRIPT" + The status should be success + The output should not include "running check-hotfix" + calls=$(cat "${TEST_DIR}/calls") + The variable calls should eq "provision" + End + + It 'runs check-hotfix before download-hotfix when ANC_HOTFIX_ENABLED is true' + touch "$CONFIG_PATH" "$HOTFIX_JSON" + create_recording_aks_node_controller + export ANC_HOTFIX_ENABLED="true" + + When run bash "$SCRIPT" + The status should be success + The output should include "running check-hotfix" + The output should include "ANC check-hotfix completed" + firstCall=$(sed -n '1p' "${TEST_DIR}/calls") + secondCall=$(sed -n '2p' "${TEST_DIR}/calls") + thirdCall=$(sed -n '3p' "${TEST_DIR}/calls") + The variable firstCall should eq "check-hotfix" + The variable secondCall should eq "download-hotfix" + The variable thirdCall should eq "provision" + End + + It 'proceeds to provision when check-hotfix fails (fail-open)' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + export ANC_HOTFIX_ENABLED="true" + export CHECK_HOTFIX_EXIT="1" + + When run bash "$SCRIPT" + The status should be success + The output should include "ANC check-hotfix failed; continuing (fail-open)" + firstCall=$(sed -n '1p' "${TEST_DIR}/calls") + lastCall=$(tail -n 1 "${TEST_DIR}/calls") + The variable firstCall should eq "check-hotfix" + The variable lastCall should eq "provision" + End End From 625de299f0b0e988ea2a2596bef856d218e7a5fb Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Mon, 15 Jun 2026 15:23:13 -0700 Subject: [PATCH 2/6] test(anc): note old-VHD fail-open coverage in wrapper spec Clarify that the check-hotfix non-zero (fail-open) case also models a node whose VHD-baked binary predates 2.1b, where check-hotfix is an unknown subcommand. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cloud-init/artifacts/aks_node_controller_wrapper_spec.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh index efb701f33ae..d09c5556dd0 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh @@ -169,6 +169,9 @@ EOF The variable thirdCall should eq "provision" End + # Fail-open also covers the backward-compat case where ANC_HOTFIX_ENABLED=true reaches + # a node whose VHD-baked binary predates 2.1b: `check-hotfix` is an unknown subcommand + # there and exits non-zero, which the wrapper tolerates so provisioning still proceeds. It 'proceeds to provision when check-hotfix fails (fail-open)' touch "$CONFIG_PATH" create_recording_aks_node_controller From c24cac1120ebe3b07f4e3bc9d4fd92e1ab539933 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Mon, 15 Jun 2026 16:17:04 -0700 Subject: [PATCH 3/6] refactor(anc): rename wrapper hotfix gate to ENABLE_PROVISIONING_HOTFIX Match the design's EnableProvisioningHotfix aks-rp region toggle and AgentBaker's contract->env naming convention (EnableIMDSRestriction -> ENABLE_IMDS_RESTRICTION), so the toggle -> absvc -> ANC opt-in chain stays traceable. No behavior change; still default-off and fail-open. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../artifacts/aks-node-controller-wrapper.sh | 10 ++++++---- .../aks_node_controller_wrapper_spec.sh | 16 ++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index f0626cfeecb..05c340bdb04 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -29,11 +29,13 @@ fi # check-hotfix reads the kube-system/anc-hotfix-version ConfigMap and refreshes # $HOTFIX_JSON, which the download-hotfix block below consumes, so it must run first. -# Gated default-off behind ANC_HOTFIX_ENABLED so existing VHDs behave exactly as before; -# only the literal string "true" enables it. The command is fail-open (always exits 0), +# Gated default-off behind ENABLE_PROVISIONING_HOTFIX so existing VHDs behave exactly as +# before; only the literal string "true" enables it. This env var is the on-node terminal +# of the EnableProvisioningHotfix aks-rp region toggle (toggle -> absvc -> ANC), so regions +# where the toggle is off see no behavior change. The command is fail-open (always exits 0), # but we still wrap it defensively so it can never block provisioning. -if [ "${ANC_HOTFIX_ENABLED:-}" = "true" ]; then - log "ANC_HOTFIX_ENABLED=true; running check-hotfix to refresh hotfix pointer" +if [ "${ENABLE_PROVISIONING_HOTFIX:-}" = "true" ]; then + log "ENABLE_PROVISIONING_HOTFIX=true; running check-hotfix to refresh hotfix pointer" if "$BIN_PATH" check-hotfix; then log "ANC check-hotfix completed; hotfix pointer refresh attempted" else diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh index d09c5556dd0..825c21f2867 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh @@ -42,7 +42,7 @@ EOF cleanup_wrapper_test() { rm -rf "$TEST_DIR" - unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR HOTFIX_JSON ANC_HOTFIX_ENABLED CHECK_HOTFIX_EXIT + unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR HOTFIX_JSON ENABLE_PROVISIONING_HOTFIX CHECK_HOTFIX_EXIT } create_fake_aks_node_controller() { @@ -127,7 +127,7 @@ EOF The variable thirdArg should eq "" End - It 'does not call check-hotfix when ANC_HOTFIX_ENABLED is unset' + It 'does not call check-hotfix when ENABLE_PROVISIONING_HOTFIX is unset' touch "$CONFIG_PATH" create_recording_aks_node_controller @@ -140,10 +140,10 @@ EOF The variable calls should eq "provision" End - It 'treats a non-true ANC_HOTFIX_ENABLED value as disabled' + It 'treats a non-true ENABLE_PROVISIONING_HOTFIX value as disabled' touch "$CONFIG_PATH" create_recording_aks_node_controller - export ANC_HOTFIX_ENABLED="1" + export ENABLE_PROVISIONING_HOTFIX="1" When run bash "$SCRIPT" The status should be success @@ -152,10 +152,10 @@ EOF The variable calls should eq "provision" End - It 'runs check-hotfix before download-hotfix when ANC_HOTFIX_ENABLED is true' + It 'runs check-hotfix before download-hotfix when ENABLE_PROVISIONING_HOTFIX is true' touch "$CONFIG_PATH" "$HOTFIX_JSON" create_recording_aks_node_controller - export ANC_HOTFIX_ENABLED="true" + export ENABLE_PROVISIONING_HOTFIX="true" When run bash "$SCRIPT" The status should be success @@ -169,13 +169,13 @@ EOF The variable thirdCall should eq "provision" End - # Fail-open also covers the backward-compat case where ANC_HOTFIX_ENABLED=true reaches + # Fail-open also covers the backward-compat case where ENABLE_PROVISIONING_HOTFIX=true reaches # a node whose VHD-baked binary predates 2.1b: `check-hotfix` is an unknown subcommand # there and exits non-zero, which the wrapper tolerates so provisioning still proceeds. It 'proceeds to provision when check-hotfix fails (fail-open)' touch "$CONFIG_PATH" create_recording_aks_node_controller - export ANC_HOTFIX_ENABLED="true" + export ENABLE_PROVISIONING_HOTFIX="true" export CHECK_HOTFIX_EXIT="1" When run bash "$SCRIPT" From d492fd95d7ea3437f344caed9d62eb066b7500bb Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Fri, 19 Jun 2026 13:58:57 -0700 Subject: [PATCH 4/6] docs(anc): reword wrapper hotfix comment for LPS endpoint read channel The hotfix pointer read channel moved from the kube-system ConfigMap (apiserver + bootstrap token) to the LPS endpoint (IMDS-attested); the fetch/auth rewrite lives in 2.1b. The wrapper's check-hotfix -> download-hotfix call contract, the ENABLE_PROVISIONING_HOTFIX gate, and the fail-open semantics are unchanged - only the explanatory comment is updated to name the new read channel accurately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index 05c340bdb04..3cfef6d8de4 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -27,7 +27,7 @@ if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then exit 0 fi -# check-hotfix reads the kube-system/anc-hotfix-version ConfigMap and refreshes +# check-hotfix reads the hotfix pointer from the LPS endpoint (IMDS-attested) and refreshes # $HOTFIX_JSON, which the download-hotfix block below consumes, so it must run first. # Gated default-off behind ENABLE_PROVISIONING_HOTFIX so existing VHDs behave exactly as # before; only the literal string "true" enables it. This env var is the on-node terminal From b539a3c9f1b0611f77a816b9f563f52b5e0a3df2 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Mon, 6 Jul 2026 13:27:58 -0700 Subject: [PATCH 5/6] docs(anc): clarify HOTFIX_JSON scope and check-hotfix fail-open comment Address Copilot review comments on the wrapper: - HOTFIX_JSON is a wrapper-only gate/test seam; the check-hotfix and download-hotfix subcommands read/write their own internal default path and do not consume this variable. Document that overriding it does not change binary behavior. - check-hotfix is fail-open on its own error paths (exit 0), but an older ANC binary predating the subcommand exits non-zero; reword the comment so it no longer claims the command 'always exits 0' and explains why we still wrap it defensively. Comment-only change; no logic change. shellcheck clean, shellspec 8/0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../artifacts/aks-node-controller-wrapper.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index 3cfef6d8de4..5259eacc302 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -7,6 +7,10 @@ done BIN_PATH="${BIN_PATH:-/opt/azure/containers/aks-node-controller}" HOTFIX_BIN="${BIN_PATH}-hotfix" +# HOTFIX_JSON is only used by this wrapper for the -f gate/logs below. The check-hotfix and +# download-hotfix subcommands read/write their own internal default path and do NOT consume +# this variable, so overriding it does not change binary behavior (it exists mainly so +# shellspec can exercise the download-hotfix branch). Keep it aligned with the binary default. HOTFIX_JSON="${HOTFIX_JSON:-/opt/azure/containers/aks-node-controller-hotfix.json}" CONFIG_PATH="${CONFIG_PATH:-/opt/azure/containers/aks-node-controller-config.json}" NBC_CMD_PATH="${NBC_CMD_PATH:-/opt/azure/containers/aks-node-controller-nbc-cmd.sh}" @@ -28,12 +32,14 @@ if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then fi # check-hotfix reads the hotfix pointer from the LPS endpoint (IMDS-attested) and refreshes -# $HOTFIX_JSON, which the download-hotfix block below consumes, so it must run first. +# the on-disk hotfix pointer file (its own default path, which $HOTFIX_JSON mirrors) that the +# download-hotfix block below consumes, so it must run first. # Gated default-off behind ENABLE_PROVISIONING_HOTFIX so existing VHDs behave exactly as # before; only the literal string "true" enables it. This env var is the on-node terminal # of the EnableProvisioningHotfix aks-rp region toggle (toggle -> absvc -> ANC), so regions -# where the toggle is off see no behavior change. The command is fail-open (always exits 0), -# but we still wrap it defensively so it can never block provisioning. +# where the toggle is off see no behavior change. check-hotfix is designed to be fail-open +# (its own error paths exit 0), but an older ANC binary that predates the subcommand exits +# non-zero, so we still wrap the call defensively to guarantee it can never block provisioning. if [ "${ENABLE_PROVISIONING_HOTFIX:-}" = "true" ]; then log "ENABLE_PROVISIONING_HOTFIX=true; running check-hotfix to refresh hotfix pointer" if "$BIN_PATH" check-hotfix; then From 8fb1371f6556fa0c7bd51b0905f2b4276ce4a401 Mon Sep 17 00:00:00 2001 From: Devin Wong Date: Mon, 6 Jul 2026 14:12:35 -0700 Subject: [PATCH 6/6] docs(anc): condense check-hotfix wrapper comment Trim the block comment above the check-hotfix call from 9 lines to 5, keeping the load-bearing points (runs first to refresh the pointer download-hotfix reads; default-off gate; fail-open + defensive wrap for old binaries) and moving the full toggle chain to the PR description. Comment-only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../artifacts/aks-node-controller-wrapper.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index 5259eacc302..db3e6ea2e63 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -31,15 +31,11 @@ if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then exit 0 fi -# check-hotfix reads the hotfix pointer from the LPS endpoint (IMDS-attested) and refreshes -# the on-disk hotfix pointer file (its own default path, which $HOTFIX_JSON mirrors) that the -# download-hotfix block below consumes, so it must run first. -# Gated default-off behind ENABLE_PROVISIONING_HOTFIX so existing VHDs behave exactly as -# before; only the literal string "true" enables it. This env var is the on-node terminal -# of the EnableProvisioningHotfix aks-rp region toggle (toggle -> absvc -> ANC), so regions -# where the toggle is off see no behavior change. check-hotfix is designed to be fail-open -# (its own error paths exit 0), but an older ANC binary that predates the subcommand exits -# non-zero, so we still wrap the call defensively to guarantee it can never block provisioning. +# check-hotfix refreshes the on-disk hotfix pointer (its own default path, mirrored by +# $HOTFIX_JSON) that download-hotfix reads below, so it must run first. Gated default-off +# behind ENABLE_PROVISIONING_HOTFIX (only the literal "true" enables it) - the on-node +# terminal of the EnableProvisioningHotfix aks-rp region toggle. Wrapped defensively: it is +# fail-open, but an older ANC binary predating the subcommand exits non-zero. if [ "${ENABLE_PROVISIONING_HOTFIX:-}" = "true" ]; then log "ENABLE_PROVISIONING_HOTFIX=true; running check-hotfix to refresh hotfix pointer" if "$BIN_PATH" check-hotfix; then