diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 40d42acb73a..612ce03136e 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -43,11 +43,23 @@ type App struct { eventLogger *helpers.EventLogger // hotfixVersionPath overrides the default hotfix version file location for testing. + // It is also the path check-hotfix writes the resolved pointer to. hotfixVersionPath string // aptSourcesDir overrides the default APT sources directory for testing. aptSourcesDir string // nodeCustomDataPath overrides the default nodecustomdata path for testing. nodeCustomDataPath string + // nodeConfigPath overrides the default AKSNodeConfig path for testing. It is the + // source for check-hotfix's LPS endpoint (apiserver FQDN + cluster CA) and the + // cold-start fallback pointer. + nodeConfigPath string + // checkHotfixFetcher overrides the real LPS hotfix-pointer GET for testing, letting + // unit tests inject a canned pointer body or errors without real networking. + checkHotfixFetcher func(ctx context.Context) ([]byte, error) + // fetchAttestedToken overrides retrieval of the IMDS attested-data token used as the + // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint + // is queried. + fetchAttestedToken func(ctx context.Context) (string, error) } // provision.json values are emitted as strings by the shell jq invocation. @@ -137,6 +149,16 @@ func (a *App) Run(ctx context.Context, args []string) int { return a.runDownloadHotfixCommand(ctx) }, }, + { + Name: "check-hotfix", + Usage: "Read the hotfix pointer from the live-patching-service and stage it (fail-open)", + Action: func(ctx context.Context, cmd *cli.Command) error { + if len(cmd.Args().Slice()) > 0 { + return fmt.Errorf("unexpected check-hotfix arguments: %s", strings.Join(cmd.Args().Slice(), " ")) + } + return a.runCheckHotfixCommand(ctx) + }, + }, }, } diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go new file mode 100644 index 00000000000..ba34ceebe8f --- /dev/null +++ b/aks-node-controller/checkhotfix.go @@ -0,0 +1,514 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" +) + +// check-hotfix reads the hotfix pointer from the live-patching-service (LPS) over the +// IMDS-attested SNI path that is reachable pre-kubelet, then writes it to the same path +// download-hotfix already reads. download-hotfix then re-resolves the pointer against the +// node's baked ANC version and keeps its unchanged patch-only, strictly-higher gating. +// check-hotfix only fetches and stages the pointer; it never installs anything and never +// blocks provisioning (fail-open). +const ( + // lpsSNIHost is the live-patching-service SNI/Host that the kube-api-proxy envoy on the + // apiserver front routes to the LPS backend. The TLS handshake pins this as ServerName + // while the TCP connection is forced to the apiserver FQDN (the curl --resolve trick), + // giving the faithful end-to-end path node -> SNI(lpsSNIHost) -> envoy -> LPS. + lpsSNIHost = "aks-security-patch.data.mcr.microsoft.com" + + // lpsAPIServerPort is the HTTPS port the apiserver front (and thus the LPS path) listens on. + lpsAPIServerPort = "443" + + // lpsHotfixPath is the LPS route serving the base->hotfix pointer map. + // + // TODO(provisioning-hotfix): this route and its response schema are a planned-maintenance + // LPS-endpoint deliverable that is NOT finalized yet. The connectivity prototype only + // proved reachability of the LPS read path (/v1/packages). Replace this placeholder with + // the real route once the LPS endpoint contract is published. The expected response body + // is the {"hotfixes":{"":""}} JSON object that parses + // directly into the shared hotfixConfig type (see parseHotfixConfig). + lpsHotfixPath = "/v1/anc-hotfix" + + // imdsAttestedDocURL returns the IMDS attested-data document, whose signature is used as + // the LPS Authorization token. IMDS is reachable pre-kubelet (the same primitive Secure + // TLS Bootstrap uses), so this works before any kube credential exists. + imdsAttestedDocURL = "http://169.254.169.254/metadata/attested/document?api-version=2025-04-07" + + // lpsFetchTimeout caps the LPS round-trip so a hung/slow endpoint never delays provisioning. + lpsFetchTimeout = 10 * time.Second + // imdsTimeout bounds the IMDS attested-document fetch. + imdsTimeout = 5 * time.Second +) + +// NOTE: the IMDS attested-token fetch and the SNI-pinned/forced-dial HTTP client below are +// duplicated from the check-lps connectivity prototype. When the shared LPS client lands, +// de-duplicate these helpers (fetchIMDSAttestedToken, buildLPSHTTPClient, App.attestedToken) +// into that single client and have check-hotfix consume it. + +// checkHotfixOutcome is the telemetry taxonomy emitted under TaskName "CheckHotfix". +type checkHotfixOutcome string + +const ( + // outcomeLPSRead: LPS pointer fetched + parsed OK and a hotfix entry matched this node's base. + outcomeLPSRead checkHotfixOutcome = "lpsRead" + // outcomeNoHotfixForBase: LPS read OK but no entry matched this node's YYYYMM.DD base. + outcomeNoHotfixForBase checkHotfixOutcome = "noHotfixForBase" + // outcomeNotEnrolled: the LPS was reachable and the request was well-formed, but it has + // no hotfix for this node yet (HTTP 401/403/404). The LPS authorizes in two stages - + // IMDS attestation, then agent-pool authorization - so a node whose pool is not yet + // enrolled in live-patching gets a 401, and 403/404 likewise mean nothing is published + // for this node. This is the expected steady state on a freshly enabled cluster, so it is + // a benign no-op: no overlay is staged and it is never treated as a failure. + outcomeNotEnrolled checkHotfixOutcome = "notEnrolled" + // outcomeCustomDataFallback: LPS read failed; the embedded customdata pointer was used. + outcomeCustomDataFallback checkHotfixOutcome = "customDataFallback" + // outcomeFailed: everything failed; nothing was staged. Provisioning still proceeds (exit 0). + outcomeFailed checkHotfixOutcome = "failed" + // outcomeDisabled: the AKSNodeConfig enable_provisioning_hotfix field is not true (false/unset), + // so check-hotfix no-ops without any remote hotfix call. Provisioning proceeds (exit 0). + outcomeDisabled checkHotfixOutcome = "disabled" +) + +// lpsUnavailableError marks a benign LPS response that means "no hotfix is available for this +// node yet" rather than a failure. It is returned for HTTP 401 (agent pool not enrolled in +// live-patching), 403, and 404. On a freshly LPS-enabled cluster the enrollment map is empty, +// so 401 is the EXPECTED steady-state response until a pool is actually enrolled; check-hotfix +// must not classify it as a hard failure or retry it. +type lpsUnavailableError struct { + statusCode int +} + +func (e *lpsUnavailableError) Error() string { + return fmt.Sprintf("LPS has no hotfix available for this node (status %d)", e.statusCode) +} + +// isLPSUnavailable reports whether err is a benign "nothing for this node yet" LPS response. +func isLPSUnavailable(err error) bool { + var u *lpsUnavailableError + return errors.As(err, &u) +} + +// runCheckHotfixCommand is the cli Action for `check-hotfix`. It ALWAYS returns nil so +// provisioning is never blocked: any error (404, 403, timeout, parse failure) is logged, +// emitted as telemetry, and swallowed. Internal helpers return errors for testability only. +func (a *App) runCheckHotfixCommand(ctx context.Context) error { + slog.Info("aks-node-controller check-hotfix started") + startTime := time.Now() + + outcome, err := a.checkHotfix(ctx) + + endTime := time.Now() + level := helpersEventLevel(outcome) + message := fmt.Sprintf("check-hotfix outcome=%s", outcome) + if err != nil { + message = fmt.Sprintf("%s error=%s", message, err.Error()) + slog.Warn("check-hotfix completed with error (fail-open)", "outcome", outcome, "error", err) + } else { + slog.Info("check-hotfix completed", "outcome", outcome) + } + if a.eventLogger != nil { + a.eventLogger.LogEvent("CheckHotfix", message, level, startTime, endTime) + } + + // Fail-open: never propagate an error so the cli exit code stays 0. + return nil +} + +// checkHotfix performs the fetch/parse/stage workflow and reports a telemetry outcome. +// It is fail-open by contract: the only caller (runCheckHotfixCommand) swallows the error. +func (a *App) checkHotfix(ctx context.Context) (checkHotfixOutcome, error) { + // Single source of truth: the enable_provisioning_hotfix contract field on the AKSNodeConfig. + // When it is not true (false or unset), no-op without any remote hotfix call. The wrapper + // calls check-hotfix unconditionally, so this Go gate is what keeps disabled nodes inert. + if !a.provisioningHotfixEnabled() { + slog.Info("check-hotfix disabled: enable_provisioning_hotfix is not true; skipping hotfix pointer fetch") + return outcomeDisabled, nil + } + + hotfixPath := a.hotfixVersionPath + if hotfixPath == "" { + hotfixPath = defaultHotfixVersionPath + } + + data, fetchErr := a.fetchHotfix(ctx) + if fetchErr != nil { + if isLPSUnavailable(fetchErr) { + // The LPS is reachable but has nothing for this node yet (e.g. the agent pool is + // not enrolled in live-patching). This is the expected steady state, not a + // failure: stage no overlay (download-hotfix keeps whatever pointer it had) and + // report a benign outcome. Fail-open. + slog.Info("LPS reports no hotfix available for this node (fail-open)", "reason", fetchErr) + return outcomeNotEnrolled, nil + } + // LPS read failed to reach/talk to the endpoint: fall back to the pointer embedded + // in the node config (cold-start path). See coldStartHotfixConfig for the contract TODO. + slog.Warn("failed to read hotfix pointer from LPS, attempting cold-start fallback", + "error", fetchErr) + cfg, ok, coldErr := a.coldStartHotfixConfig() + if coldErr != nil { + return outcomeFailed, fmt.Errorf("LPS fetch failed (%v) and cold-start fallback failed: %w", fetchErr, coldErr) + } + if !ok { + return outcomeFailed, fmt.Errorf("LPS fetch failed and no cold-start pointer present: %w", fetchErr) + } + if err := writeHotfixConfig(hotfixPath, cfg); err != nil { + return outcomeFailed, fmt.Errorf("writing cold-start hotfix config: %w", err) + } + return outcomeCustomDataFallback, nil + } + + cfg, err := parseHotfixConfig(data) + if err != nil { + return outcomeFailed, fmt.Errorf("parsing LPS hotfix pointer: %w", err) + } + + if err := writeHotfixConfig(hotfixPath, cfg); err != nil { + return outcomeFailed, fmt.Errorf("writing hotfix config: %w", err) + } + + // Report whether this node's base actually has a pointer. download-hotfix still + // performs the authoritative patch-only-strictly-higher gating; this is telemetry only. + if cfg.resolveVersion(Version) == "" { + return outcomeNoHotfixForBase, nil + } + return outcomeLPSRead, nil +} + +// helpersEventLevel maps a check-hotfix outcome to a guest-agent event level. Only the +// terminal "failed" outcome is reported as an error; the rest are informational because +// the command is fail-open and provisioning continues regardless. +func helpersEventLevel(outcome checkHotfixOutcome) helpers.EventLevel { + if outcome == outcomeFailed { + return helpers.EventLevelError + } + return helpers.EventLevelInformational +} + +// fetchHotfix returns the raw LPS response body. Tests inject checkHotfixFetcher to supply +// canned pointer JSON or errors without networking. +func (a *App) fetchHotfix(ctx context.Context) ([]byte, error) { + if a.checkHotfixFetcher != nil { + return a.checkHotfixFetcher(ctx) + } + return a.fetchHotfixFromLPS(ctx) +} + +// fetchHotfixFromLPS performs the real network GET against the LPS over the IMDS-attested +// SNI path. It sources the apiserver FQDN and cluster CA from the AKSNodeConfig that ANC +// already parses, pins the TLS ServerName to lpsSNIHost while forcing the TCP dial to the +// FQDN, attaches the IMDS attested-data token as the Authorization header, and returns the +// raw response body. A 2xx returns the body; 401/403/404 are surfaced as a benign +// lpsUnavailableError ("nothing for this node yet"); any other non-2xx is a hard error so the +// caller falls back / fails open. +func (a *App) fetchHotfixFromLPS(ctx context.Context) ([]byte, error) { + fqdn, caPEM, err := a.lpsTargetFromNodeConfig() + if err != nil { + return nil, fmt.Errorf("resolving LPS endpoint from node config: %w", err) + } + + token, err := a.attestedToken(ctx) + if err != nil { + return nil, fmt.Errorf("imds attested token: %w", err) + } + + client, caSource, err := buildLPSHTTPClient(fqdn, caPEM) + if err != nil { + return nil, fmt.Errorf("building LPS http client: %w", err) + } + // caSource records whether the server cert is verified against the provision-config CA + // or (when that CA is unavailable) trust was skipped; surface it for diagnosis. + slog.Info("check-hotfix LPS TLS trust source", "caSource", caSource, "dialHost", fqdn) + + ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) + defer cancel() + + url := fmt.Sprintf("https://%s:%s%s", lpsSNIHost, lpsAPIServerPort, lpsHotfixPath) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("building request: %w", err) + } + req.Header.Set("Authorization", token) + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return body, nil + case resp.StatusCode == http.StatusUnauthorized, + resp.StatusCode == http.StatusForbidden, + resp.StatusCode == http.StatusNotFound: + // Benign: reachable LPS with nothing for this node yet (e.g. pool not enrolled). + // Surfaced as a typed error so the caller treats it as a no-op, not a failure. + return nil, &lpsUnavailableError{statusCode: resp.StatusCode} + default: + return nil, fmt.Errorf("LPS returned status %d for %s", resp.StatusCode, url) + } +} + +// lpsTargetFromNodeConfig reads the apiserver FQDN (the forced dial target) and the cluster +// CA (TLS trust) from the AKSNodeConfig. +// +// check-hotfix runs before the provisioning scripts (cse_config.sh), so the on-node decoded +// CA file (/etc/kubernetes/certs/ca.crt) does not exist yet -- it is written later during +// provisioning. The node config is the only credential source guaranteed to be present at +// this point and it carries the CA as base64-encoded PEM (the same value cse_config.sh later +// decodes into that file). +func (a *App) lpsTargetFromNodeConfig() (fqdn string, caPEM []byte, err error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + return "", nil, fmt.Errorf("reading node config %s: %w", path, err) + } + cfg, perr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if perr != nil { + // Forward-compatible parse: unknown fields are discarded, so a non-nil error here + // means some fields were unusable. Continue with whatever parsed. + slog.Info("node config parsed with errors, continuing with partial config", "error", perr) + } + if cfg == nil { + return "", nil, fmt.Errorf("node config %s could not be parsed", path) + } + + fqdn = stripScheme(strings.TrimSpace(cfg.GetApiServerConfig().GetApiServerName())) + if fqdn == "" { + return "", nil, fmt.Errorf("node config has no api_server_config.api_server_name") + } + + if caB64 := strings.TrimSpace(cfg.GetKubernetesCaCert()); caB64 != "" { + decoded, derr := base64.StdEncoding.DecodeString(caB64) + if derr != nil { + return "", nil, fmt.Errorf("decoding node config kubernetes_ca_cert: %w", derr) + } + caPEM = decoded + } + return fqdn, caPEM, nil +} + +// attestedToken returns the IMDS attested-data signature used as the LPS Authorization +// token. The fetch is overridable for testing via App.fetchAttestedToken. +func (a *App) attestedToken(ctx context.Context) (string, error) { + if a.fetchAttestedToken != nil { + return a.fetchAttestedToken(ctx) + } + return fetchIMDSAttestedToken(ctx) +} + +// fetchIMDSAttestedToken queries IMDS for the attested-data document and returns its +// signature, the same primitive Secure TLS Bootstrap and the custom-patching flow use. +func fetchIMDSAttestedToken(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, imdsTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsAttestedDocURL, nil) + if err != nil { + return "", err + } + req.Header.Set("Metadata", "true") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("imds returned status %d", resp.StatusCode) + } + var doc struct { + Signature string `json:"signature"` + } + if err := json.Unmarshal(body, &doc); err != nil { + return "", err + } + if doc.Signature == "" { + return "", fmt.Errorf("imds attested document had empty signature") + } + return doc.Signature, nil +} + +// buildLPSHTTPClient builds the HTTP client for the LPS fetch: TLS ServerName pinned to +// lpsSNIHost, the TCP dial forced to dialHost:443 (the curl --resolve equivalent), and +// RootCAs from the cluster CA. It returns the client and a string describing the TLS trust +// source ("provision-config" or "insecure-skip-verify"), with a short timeout so +// provisioning is never delayed. When the CA is unavailable the client falls back to +// skipping verification (reachability-only), mirroring the connectivity prototype; the +// staged pointer is non-authoritative anyway because download-hotfix re-resolves and gates. +func buildLPSHTTPClient(dialHost string, caPEM []byte) (*http.Client, string, error) { + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: lpsSNIHost} + caSource := "insecure-skip-verify" + if len(caPEM) > 0 { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, "", fmt.Errorf("failed to parse cluster CA PEM") + } + tlsConfig.RootCAs = pool + caSource = "provision-config" + } else { + tlsConfig.InsecureSkipVerify = true //nolint:gosec // CA unavailable pre-provision; pointer is re-resolved and gated by download-hotfix + } + + dialAddr := net.JoinHostPort(dialHost, lpsAPIServerPort) + dialer := &net.Dialer{Timeout: lpsFetchTimeout} + transport := &http.Transport{ + TLSClientConfig: tlsConfig, + // Force the connection to dialHost regardless of the SNI/Host (lpsSNIHost) in the + // request URL -- the curl --resolve equivalent. + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, dialAddr) + }, + } + return &http.Client{Timeout: lpsFetchTimeout, Transport: transport}, caSource, nil +} + +// parseHotfixConfig extracts the hotfix pointer from an LPS response body. The body is the +// {"hotfixes":{...}} JSON object that unmarshals DIRECTLY into the shared 2.1a hotfixConfig, +// so check-hotfix and download-hotfix share ONE identical data contract. +func parseHotfixConfig(data []byte) (hotfixConfig, error) { + data = bytes.TrimSpace(data) + if len(data) == 0 { + return hotfixConfig{}, fmt.Errorf("LPS response body is empty") + } + var cfg hotfixConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return hotfixConfig{}, fmt.Errorf("unmarshaling hotfix pointer JSON: %w", err) + } + return cfg, nil +} + +// coldStartHotfixConfig reads a LENIENT top-level "hotfixes" object from the AKSNodeConfig +// JSON. This is the PoC cold-start fallback used only when the LPS endpoint could not be +// reached or talked to (transport failure / 5xx). A benign 401/403/404 is NOT a cold-start: +// the LPS authoritatively has nothing for this node, so that path stages no overlay. +// +// TODO(provisioning-hotfix): There is no formalized AKSNodeConfig contract field for the +// embedded pointer yet - the control-plane side that would populate a typed field is not +// built. Once that contract exists, replace this lenient top-level read with the typed field +// and drop the permissive JSON shape. Until then we read it best-effort and never fail +// provisioning. +func (a *App) coldStartHotfixConfig() (hotfixConfig, bool, error) { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return hotfixConfig{}, false, nil + } + return hotfixConfig{}, false, fmt.Errorf("reading node config %s: %w", path, err) + } + + // Lenient parse: the AKSNodeConfig is protojson, but the cold-start pointer is an + // out-of-contract top-level object, so parse it permissively with encoding/json. + var lenient struct { + Hotfixes map[string]string `json:"hotfixes"` + } + if err := json.Unmarshal(raw, &lenient); err != nil { + return hotfixConfig{}, false, fmt.Errorf("parsing cold-start hotfixes from node config: %w", err) + } + if len(lenient.Hotfixes) == 0 { + return hotfixConfig{}, false, nil + } + return hotfixConfig{Hotfixes: lenient.Hotfixes}, true, nil +} + +// writeHotfixConfig writes the resolved config to the path download-hotfix reads, in the +// exact {"hotfixes":{...}} shape so download-hotfix re-resolves and applies its unchanged +// gating. The write is atomic (temp file + rename) so a concurrent reader never sees a +// partial file. +func writeHotfixConfig(path string, cfg hotfixConfig) error { + // Only persist the map shape; the legacy Version field is intentionally omitted so the + // on-disk contract matches what the live-patching-service publishes. + out := hotfixConfig{Hotfixes: cfg.Hotfixes} + data, err := json.Marshal(out) + if err != nil { + return fmt.Errorf("marshaling hotfix config: %w", err) + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".aks-node-controller-hotfix-*") + if err != nil { + return fmt.Errorf("creating temp file in %s: %w", dir, err) + } + tmpPath := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpPath) + return fmt.Errorf("writing temp file %s: %w", tmpPath, err) + } + if err := tmp.Close(); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("closing temp file %s: %w", tmpPath, err) + } + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("renaming %s to %s: %w", tmpPath, path, err) + } + slog.Info("staged hotfix pointer for download-hotfix", "path", path) + return nil +} + +// provisioningHotfixEnabled reports whether the AKSNodeConfig enable_provisioning_hotfix +// contract field is explicitly true. It is the single source of truth for whether +// check-hotfix does any work. Any read/parse problem yields false (default-off, fail-open): +// a node that cannot prove the feature is on is treated as off, and provisioning still +// proceeds because the caller swallows all errors. +func (a *App) provisioningHotfixEnabled() bool { + path := a.getNodeConfigPath() + raw, err := os.ReadFile(path) + if err != nil { + slog.Info("check-hotfix gate: cannot read node config, treating as disabled", "path", path, "error", err) + return false + } + cfg, err := nodeconfigutils.UnmarshalConfigurationV1(raw) + if err != nil { + // Forward-compatible parse: unknown fields are discarded. A non-nil error means the + // document was unusable, so fall back to disabled. + slog.Info("check-hotfix gate: node config parsed with errors, evaluating partial config", "error", err) + } + return cfg.GetEnableProvisioningHotfix() +} + +// getNodeConfigPath returns the injectable node-config path, defaulting to the standard +// AKSNodeConfig location that ANC already reads. +func (a *App) getNodeConfigPath() string { + if a.nodeConfigPath != "" { + return a.nodeConfigPath + } + return nodeconfigutils.AKSNodeConfigFilePath +} + +// stripScheme removes a leading https:// or http:// scheme from a server URL. +func stripScheme(server string) string { + server = strings.TrimPrefix(server, "https://") + server = strings.TrimPrefix(server, "http://") + return strings.TrimRight(server, "/") +} diff --git a/aks-node-controller/checkhotfix_test.go b/aks-node-controller/checkhotfix_test.go new file mode 100644 index 00000000000..3f116cdee59 --- /dev/null +++ b/aks-node-controller/checkhotfix_test.go @@ -0,0 +1,570 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCAPEM is a self-signed CA certificate used to exercise the provision-config TLS +// trust path in buildLPSHTTPClient. +const testCAPEM = `-----BEGIN CERTIFICATE----- +MIIBVDCB+6ADAgECAgEBMAoGCCqGSM49BAMCMBIxEDAOBgNVBAMTB3Rlc3QtY2Ew +HhcNMjYwNjE5MjEwNDM4WhcNMzYwNjE2MjEwNDM4WjASMRAwDgYDVQQDEwd0ZXN0 +LWNhMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEDEsevoDBYiQ68iPrOeDKJLfJ +EhavIoHla/EJ5jy1EeaLp5qnDttz9IQe8PiZGSat6Dc1in8pwwQJkTcCwDMlzaNC +MEAwDgYDVR0PAQH/BAQDAgIEMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI5z +oesQcLTRf96etb8XDK8w9wFRMAoGCCqGSM49BAMCA0gAMEUCIQCDOJZ8qJDAnEB1 +2LbXQPzOc3n5Pcz3lpwQnczk/UdVJAIgcFqNv0HsWdn7Img3gNsUgSaOT1M9QBAL +52RBAH6U7DI= +-----END CERTIFICATE----- +` + +// lpsPointerBody renders an LPS hotfix-pointer response body in the {"hotfixes":{...}} shape. +func lpsPointerBody(t *testing.T, hotfixes map[string]string) []byte { + t.Helper() + b, err := json.Marshal(map[string]any{"hotfixes": hotfixes}) + require.NoError(t, err) + return b +} + +// readStagedConfig reads back the hotfix config check-hotfix wrote. +func readStagedConfig(t *testing.T, path string) hotfixConfig { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + var cfg hotfixConfig + require.NoError(t, json.Unmarshal(data, &cfg)) + return cfg +} + +// enableHotfixGate writes a minimal AKSNodeConfig that turns the enable_provisioning_hotfix +// contract gate ON and points the App at it. Without this, checkHotfix self-gates to +// outcomeDisabled and never reaches the fetch path. Tests that also need a cold-start +// hotfixes pointer write their own node config inline instead. +func enableHotfixGate(t *testing.T, app *App) { + t.Helper() + p := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(p, []byte(`{"version":"v1","enableProvisioningHotfix":true}`), 0644)) + app.nodeConfigPath = p +} + +func TestParseHotfixConfig(t *testing.T) { + t.Run("parses the hotfixes object directly", func(t *testing.T) { + cfg, err := parseHotfixConfig([]byte(`{"hotfixes":{"202604.01":"202604.01.1","202605.01":"202605.01.2"}}`)) + require.NoError(t, err) + assert.Equal(t, map[string]string{"202604.01": "202604.01.1", "202605.01": "202605.01.2"}, cfg.Hotfixes) + }) + + t.Run("tolerates surrounding whitespace", func(t *testing.T) { + cfg, err := parseHotfixConfig([]byte(" \n{\"hotfixes\":{\"202604.01\":\"202604.01.1\"}}\n ")) + require.NoError(t, err) + assert.Equal(t, map[string]string{"202604.01": "202604.01.1"}, cfg.Hotfixes) + }) + + t.Run("empty body is an error", func(t *testing.T) { + _, err := parseHotfixConfig([]byte(" ")) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty") + }) + + t.Run("invalid JSON is an error", func(t *testing.T) { + _, err := parseHotfixConfig([]byte("not json")) + require.Error(t, err) + assert.Contains(t, err.Error(), "unmarshaling hotfix pointer JSON") + }) + + t.Run("shares parser shape with download-hotfix readHotfixConfig", func(t *testing.T) { + // The body served by the LPS must round-trip through the SAME shape that + // download-hotfix's readHotfixConfig consumes. + body := `{"hotfixes":{"202604.01":"202604.01.3"}}` + fromLPS, err := parseHotfixConfig([]byte(body)) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "h.json") + require.NoError(t, os.WriteFile(path, []byte(body), 0644)) + fromFile, err := readHotfixConfig(path) + require.NoError(t, err) + assert.Equal(t, fromFile, fromLPS) + }) +} + +func TestCheckHotfix_SuccessReadAndWrite(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}), nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeLPSRead, outcome) + + cfg := readStagedConfig(t, path) + assert.Equal(t, map[string]string{"202604.01": "202604.01.1"}, cfg.Hotfixes) +} + +func TestCheckHotfix_NoHotfixForBase(t *testing.T) { + origVersion := Version + Version = "202607.15.0" // base not present in the LPS pointer + defer func() { Version = origVersion }() + + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}), nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeNoHotfixForBase, outcome) + + // The full pointer is still staged so download-hotfix re-resolves authoritatively. + cfg := readStagedConfig(t, path) + assert.Equal(t, map[string]string{"202604.01": "202604.01.1"}, cfg.Hotfixes) +} + +func TestCheckHotfix_LPSUnavailableIsBenign(t *testing.T) { + // A reachable LPS that has nothing for this node (401 pool-not-enrolled, 403, 404) is the + // expected steady state. It must be a benign no-op: outcome notEnrolled, no error, nothing + // staged, and NO cold-start overlay even when the node config carries an embedded pointer. + statuses := map[string]int{ + "401 not enrolled": http.StatusUnauthorized, + "403 forbidden": http.StatusForbidden, + "404 not found": http.StatusNotFound, + } + for name, code := range statuses { + t.Run(name, func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + + // Even with a cold-start pointer present, a benign LPS answer stages no overlay. + nodeConfig := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(nodeConfig, []byte( + `{"version":"v1","enableProvisioningHotfix":true,"hotfixes":{"202604.01":"202604.01.9"}}`), 0644)) + tt.App.nodeConfigPath = nodeConfig + + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, &lpsUnavailableError{statusCode: code} + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeNotEnrolled, outcome) + + // No overlay staged. + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) + }) + } +} + +func TestCheckHotfix_FetchErrorFailsOpenWithoutFallback(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + // Gate on, but the node config carries no cold-start hotfixes pointer -> no fallback. + enableHotfixGate(t, tt.App) + + // Transport-level failures (not benign 401/403/404) with no fallback -> failed. + cases := map[string]error{ + "timeout": context.DeadlineExceeded, + "connection err": errors.New("dial tcp: connection refused"), + "server error": errors.New("LPS returned status 500"), + } + for name, fetchErr := range cases { + t.Run(name, func(t *testing.T) { + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, fetchErr + } + outcome, err := tt.App.checkHotfix(context.Background()) + assert.Equal(t, outcomeFailed, outcome) + assert.Error(t, err) + // Nothing should be staged. + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) + }) + } +} + +func TestCheckHotfix_InvalidPointerFailsOpen(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return []byte("not valid json"), nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + assert.Equal(t, outcomeFailed, outcome) + assert.Error(t, err) + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestCheckHotfix_ColdStartFallback(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + + // Node config carries a lenient top-level hotfixes pointer (PoC cold-start contract). + nodeConfig := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(nodeConfig, []byte( + `{"version":"v1","enableProvisioningHotfix":true,"hotfixes":{"202604.01":"202604.01.2"}}`), 0644)) + tt.App.nodeConfigPath = nodeConfig + + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, errors.New("dial tcp: connection refused") + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeCustomDataFallback, outcome) + + cfg := readStagedConfig(t, path) + assert.Equal(t, map[string]string{"202604.01": "202604.01.2"}, cfg.Hotfixes) +} + +func TestCheckHotfix_ColdStartNoPointerFails(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + + nodeConfig := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(nodeConfig, []byte(`{"version":"v1","enableProvisioningHotfix":true}`), 0644)) + tt.App.nodeConfigPath = nodeConfig + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, errors.New("dial tcp: connection refused") + } + + outcome, err := tt.App.checkHotfix(context.Background()) + assert.Equal(t, outcomeFailed, outcome) + assert.Error(t, err) + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) +} + +// TestRunCheckHotfixCommand_AlwaysFailOpen verifies the cli Action always returns nil +// (exit 0) and emits telemetry, regardless of the underlying outcome. +func TestRunCheckHotfixCommand_AlwaysFailOpen(t *testing.T) { + t.Run("success path emits informational event and exits 0", func(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}), nil + } + + err := tt.App.runCheckHotfixCommand(context.Background()) + require.NoError(t, err) + + events := tt.eventLogger.Events() + require.Len(t, events, 1) + assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) + assert.Equal(t, "Informational", events[0].EventLevel) + assert.Contains(t, events[0].Message, string(outcomeLPSRead)) + }) + + t.Run("failure path emits error event but still exits 0", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, errors.New("LPS returned status 500") + } + + err := tt.App.runCheckHotfixCommand(context.Background()) + require.NoError(t, err) + + events := tt.eventLogger.Events() + require.Len(t, events, 1) + assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) + assert.Equal(t, "Error", events[0].EventLevel) + assert.Contains(t, events[0].Message, string(outcomeFailed)) + }) + + t.Run("cli wiring returns exit code 0 even on fetch failure", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + enableHotfixGate(t, tt.App) + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + return nil, errors.New("boom") + } + exitCode := tt.App.Run(context.Background(), []string{"aks-node-controller", "check-hotfix"}) + assert.Equal(t, 0, exitCode) + }) +} + +func TestCheckHotfix_DefaultsToLPSFetcherWhenNoInjection(t *testing.T) { + // With no injected fetcher and no readable node config, the real path is exercised: it + // must fail-open. Point the node-config source at a nonexistent path so LPS endpoint + // resolution fails deterministically and the network is never actually dialed. + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + enableHotfixGate(t, tt.App) + // checkHotfixFetcher intentionally nil. + + err := tt.App.runCheckHotfixCommand(context.Background()) + require.NoError(t, err) +} + +func TestAttestedToken_InjectionOverridesIMDS(t *testing.T) { + t.Run("injected token is returned without networking", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchAttestedToken = func(context.Context) (string, error) { + return "injected-signature", nil + } + tok, err := tt.App.attestedToken(context.Background()) + require.NoError(t, err) + assert.Equal(t, "injected-signature", tok) + }) + + t.Run("injected error propagates", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchAttestedToken = func(context.Context) (string, error) { + return "", errors.New("imds down") + } + _, err := tt.App.attestedToken(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "imds down") + }) +} + +func TestLPSTargetFromNodeConfig(t *testing.T) { + // A minimal AKSNodeConfig (protojson) carrying the apiserver name and base64 CA. + caPEM := "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n" + caB64 := base64.StdEncoding.EncodeToString([]byte(caPEM)) + + t.Run("reads fqdn and decodes CA", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + p := filepath.Join(t.TempDir(), "config.json") + body := `{"version":"v1","apiServerConfig":{"apiServerName":"myapi.example.com"},"kubernetesCaCert":"` + caB64 + `"}` + require.NoError(t, os.WriteFile(p, []byte(body), 0644)) + tt.App.nodeConfigPath = p + + fqdn, ca, err := tt.App.lpsTargetFromNodeConfig() + require.NoError(t, err) + assert.Equal(t, "myapi.example.com", fqdn) + assert.Equal(t, []byte(caPEM), ca) + }) + + t.Run("missing apiserver name is an error", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + p := filepath.Join(t.TempDir(), "config.json") + require.NoError(t, os.WriteFile(p, []byte(`{"version":"v1"}`), 0644)) + tt.App.nodeConfigPath = p + + _, _, err := tt.App.lpsTargetFromNodeConfig() + require.Error(t, err) + assert.Contains(t, err.Error(), "api_server_name") + }) + + t.Run("missing file is an error", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nope.json") + _, _, err := tt.App.lpsTargetFromNodeConfig() + require.Error(t, err) + }) +} + +func TestCheckHotfix_GateDisabled(t *testing.T) { + // When enable_provisioning_hotfix is not true, checkHotfix must self-gate: return + // outcomeDisabled, make NO remote hotfix call (injected fetcher untouched), and exit 0. + cases := map[string]string{ + "field unset": `{"version":"v1"}`, + "field explicit false": `{"version":"v1","enableProvisioningHotfix":false}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + + nodeConfig := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(nodeConfig, []byte(body), 0644)) + tt.App.nodeConfigPath = nodeConfig + + fetched := false + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + fetched = true + return lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}), nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeDisabled, outcome) + assert.False(t, fetched, "hotfix fetcher must not be called when the gate is disabled") + // Nothing should be staged. + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) + }) + } +} + +func TestCheckHotfix_GateMissingConfigDisables(t *testing.T) { + // A missing/unreadable node config cannot prove the feature is on -> default-off, + // fail-open: outcomeDisabled, no remote hotfix call. + tt := NewTestApp(t, TestAppConfig{}) + path := filepath.Join(t.TempDir(), "hotfix.json") + tt.App.hotfixVersionPath = path + tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "does-not-exist.json") + + fetched := false + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + fetched = true + return nil, nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.Equal(t, outcomeDisabled, outcome) + assert.False(t, fetched, "hotfix fetcher must not be called when node config is unreadable") +} + +func TestCheckHotfix_GateEnabledReachesFetch(t *testing.T) { + // When the gate is on, checkHotfix proceeds to call the injected fetcher. + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + enableHotfixGate(t, tt.App) + + fetched := false + tt.App.checkHotfixFetcher = func(context.Context) ([]byte, error) { + fetched = true + return lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}), nil + } + + outcome, err := tt.App.checkHotfix(context.Background()) + require.NoError(t, err) + assert.True(t, fetched, "hotfix fetcher must be called when the gate is enabled") + assert.Equal(t, outcomeLPSRead, outcome) +} + +func TestRunCheckHotfixCommand_DisabledEmitsInformational(t *testing.T) { + // The cli Action emits an informational telemetry event (not Error) when disabled. + tt := NewTestApp(t, TestAppConfig{}) + tt.App.hotfixVersionPath = filepath.Join(t.TempDir(), "hotfix.json") + nodeConfig := filepath.Join(t.TempDir(), "aks-node-controller-config.json") + require.NoError(t, os.WriteFile(nodeConfig, []byte(`{"version":"v1"}`), 0644)) + tt.App.nodeConfigPath = nodeConfig + + err := tt.App.runCheckHotfixCommand(context.Background()) + require.NoError(t, err) + + events := tt.eventLogger.Events() + require.Len(t, events, 1) + assert.Equal(t, "AKS.AKSNodeController.CheckHotfix", events[0].TaskName) + assert.Equal(t, "Informational", events[0].EventLevel) + assert.Contains(t, events[0].Message, string(outcomeDisabled)) +} + +func TestStripScheme(t *testing.T) { + assert.Equal(t, "host:443", stripScheme("https://host:443")) + assert.Equal(t, "host:443", stripScheme("http://host:443")) + assert.Equal(t, "host:443", stripScheme("host:443")) + assert.Equal(t, "host", stripScheme("https://host/")) +} + +func TestBuildLPSHTTPClient(t *testing.T) { + t.Run("invalid CA PEM is an error", func(t *testing.T) { + _, _, err := buildLPSHTTPClient("myapi.example.com", []byte("not a pem")) + require.Error(t, err) + assert.Contains(t, err.Error(), "cluster CA PEM") + }) + + t.Run("valid CA pins ServerName and reports provision-config trust", func(t *testing.T) { + // A real (self-signed) cert PEM so AppendCertsFromPEM succeeds. + client, caSource, err := buildLPSHTTPClient("myapi.example.com", []byte(testCAPEM)) + require.NoError(t, err) + assert.Equal(t, "provision-config", caSource) + assert.Equal(t, lpsFetchTimeout, client.Timeout) + tr := client.Transport.(*http.Transport) + assert.Equal(t, lpsSNIHost, tr.TLSClientConfig.ServerName) + assert.False(t, tr.TLSClientConfig.InsecureSkipVerify) + }) + + t.Run("no CA falls back to insecure-skip-verify", func(t *testing.T) { + client, caSource, err := buildLPSHTTPClient("myapi.example.com", nil) + require.NoError(t, err) + assert.Equal(t, "insecure-skip-verify", caSource) + tr := client.Transport.(*http.Transport) + assert.Equal(t, lpsSNIHost, tr.TLSClientConfig.ServerName) + assert.True(t, tr.TLSClientConfig.InsecureSkipVerify) + }) +} + +func TestColdStartHotfixConfig(t *testing.T) { + t.Run("missing file returns not-present without error", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.nodeConfigPath = filepath.Join(t.TempDir(), "nope.json") + cfg, ok, err := tt.App.coldStartHotfixConfig() + require.NoError(t, err) + assert.False(t, ok) + assert.Nil(t, cfg.Hotfixes) + }) + + t.Run("present pointer is parsed", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + p := filepath.Join(t.TempDir(), "config.json") + require.NoError(t, os.WriteFile(p, []byte(`{"version":"v1","hotfixes":{"202604.01":"202604.01.5"}}`), 0644)) + tt.App.nodeConfigPath = p + cfg, ok, err := tt.App.coldStartHotfixConfig() + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, map[string]string{"202604.01": "202604.01.5"}, cfg.Hotfixes) + }) + + t.Run("no hotfixes key returns not-present", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + p := filepath.Join(t.TempDir(), "config.json") + require.NoError(t, os.WriteFile(p, []byte(`{"version":"v1"}`), 0644)) + tt.App.nodeConfigPath = p + _, ok, err := tt.App.coldStartHotfixConfig() + require.NoError(t, err) + assert.False(t, ok) + }) +} + +func TestWriteHotfixConfig_ShapeAndAtomicity(t *testing.T) { + path := filepath.Join(t.TempDir(), "hotfix.json") + require.NoError(t, writeHotfixConfig(path, hotfixConfig{Hotfixes: map[string]string{"202604.01": "202604.01.1"}})) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + // Must serialize in the {"hotfixes":{...}} shape with no legacy version field. + assert.JSONEq(t, `{"hotfixes":{"202604.01":"202604.01.1"}}`, string(raw)) + + // Round-trips through download-hotfix's reader. + cfg, err := readHotfixConfig(path) + require.NoError(t, err) + assert.Equal(t, "202604.01.1", cfg.resolveVersion("202604.01.0")) +} diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go index 6199471becf..4935366757a 100644 --- a/aks-node-controller/hotfix.go +++ b/aks-node-controller/hotfix.go @@ -36,13 +36,19 @@ func (a *App) downloadHotfix(ctx context.Context) error { if hotfixPath == "" { hotfixPath = defaultHotfixVersionPath } - hotfixVersion, err := readHotfixVersion(hotfixPath) + cfg, err := readHotfixConfig(hotfixPath) if err != nil { - return fmt.Errorf("read hotfix version from %s: %w", hotfixPath, err) + // Fail-open: an unreadable or malformed hotfix config must never block + // provisioning. Log and skip so the node boots on its VHD-baked binary. + slog.Warn("failed to read hotfix config, skipping hotfix download", + "path", hotfixPath, "error", err) + return nil } + hotfixVersion := cfg.resolveVersion(Version) if hotfixVersion == "" { - slog.Info("hotfix config does not request a version, skipping download", "path", hotfixPath) + slog.Info("hotfix config does not request a version for this base, skipping download", + "path", hotfixPath, "current", Version) return nil } @@ -77,27 +83,69 @@ func (a *App) downloadHotfix(ctx context.Context) error { // hotfixConfig is the JSON structure of the hotfix configuration file. // Using JSON allows future extension (e.g., adding checksum, source URL) without format changes. type hotfixConfig struct { - Version string `json:"version"` + // Version is the legacy single-version pointer. It is still honored when Hotfixes + // is empty, preserving backward compatibility with the original config shape. + Version string `json:"version,omitempty"` + + // Hotfixes maps an ANC version base ("YYYYMM.DD") to the hotfix version + // ("YYYYMM.DD.PATCH") to apply to nodes whose baked ANC version shares that base. + // A single config can thus pin hotfixes for multiple VHD bases at once; a base + // whose key is absent gets no hotfix (default deny). When non-empty, this map + // takes precedence over Version. + Hotfixes map[string]string `json:"hotfixes,omitempty"` +} + +// hotfixBaseFromVersion extracts the "YYYYMM.DD" base from an ANC version string of +// the form "YYYYMM.DD.PATCH". It splits on "." rather than parsing semver so the literal +// day segment - including any leading zero such as "01" - is preserved to match map keys +// exactly (semver parsing would drop the leading zero, e.g. "202604.01" -> minor 1). +// All three segments must be non-empty; a present-but-empty patch (e.g. "202604.01.") +// is rejected so an obviously malformed current version never selects a map entry. +func hotfixBaseFromVersion(version string) (string, error) { + parts := strings.SplitN(strings.TrimSpace(version), ".", 3) + if len(parts) < 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return "", fmt.Errorf("version %q is not in YYYYMM.DD.PATCH form", version) + } + return parts[0] + "." + parts[1], nil } -// readHotfixVersion reads and parses the JSON hotfix config from the given path. -// Returns empty string if the file doesn't exist or contains an empty version. -func readHotfixVersion(path string) (string, error) { +// resolveVersion picks the hotfix version that applies to the given current ANC version. +// When the base->version map is populated it takes precedence: the entry matching the +// current version's "YYYYMM.DD" base is returned, while an absent base (or an unparseable +// current version) yields "" so provisioning proceeds with no hotfix. When the map is +// empty it falls back to the legacy single Version field. The returned version is still +// subject to shouldUpgradeToHotfix's patch-only-strictly-higher gating in the caller. +func (cfg hotfixConfig) resolveVersion(current string) string { + if len(cfg.Hotfixes) > 0 { + base, err := hotfixBaseFromVersion(current) + if err != nil { + slog.Warn("cannot derive hotfix base from current version, skipping hotfix", + "current", current, "error", err) + return "" + } + return strings.TrimSpace(cfg.Hotfixes[base]) + } + return strings.TrimSpace(cfg.Version) +} + +// readHotfixConfig reads and parses the JSON hotfix config from the given path. +// Returns a zero-value config if the file doesn't exist or is empty. +func readHotfixConfig(path string) (hotfixConfig, error) { + var cfg hotfixConfig data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { - return "", nil + return cfg, nil } - return "", err + return cfg, err } if len(strings.TrimSpace(string(data))) == 0 { - return "", nil + return cfg, nil } - var cfg hotfixConfig if err := json.Unmarshal(data, &cfg); err != nil { - return "", fmt.Errorf("parsing hotfix config %s: %w", path, err) + return cfg, fmt.Errorf("parsing hotfix config %s: %w", path, err) } - return strings.TrimSpace(cfg.Version), nil + return cfg, nil } // packageManager represents a supported system package manager. @@ -297,10 +345,10 @@ func copyBinaryAlongside(src, dst, refPath string) error { // ANC versions use the format YYYYMM.DD.PATCH which is valid semver (Major.Minor.Patch). // // This ensures the hotfix only targets the specific VHD it was built for: -// - Older VHDs (different base) are skipped — remediated via VHD republish -// - Newer VHDs (different base) are skipped — fix is already baked in -// - Same version is skipped — already at hotfix -// - Unparseable versions (e.g. "dev") return an error — caller should skip +// - Older VHDs (different base) are skipped - remediated via VHD republish +// - Newer VHDs (different base) are skipped - fix is already baked in +// - Same version is skipped - already at hotfix +// - Unparseable versions (e.g. "dev") return an error - caller should skip func shouldUpgradeToHotfix(current, hotfix string) (bool, error) { cv, err := semver.NewVersion(strings.TrimSpace(current)) if err != nil { diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go index 612da3b44f9..8cdd61f4908 100644 --- a/aks-node-controller/hotfix_test.go +++ b/aks-node-controller/hotfix_test.go @@ -12,51 +12,125 @@ import ( "github.com/stretchr/testify/require" ) -func TestReadHotfixVersion(t *testing.T) { - t.Run("file does not exist", func(t *testing.T) { - version, err := readHotfixVersion("/nonexistent/path") +func TestReadHotfixConfig(t *testing.T) { + t.Run("file does not exist returns zero config", func(t *testing.T) { + cfg, err := readHotfixConfig("/nonexistent/path") assert.NoError(t, err) - assert.Equal(t, "", version) + assert.Equal(t, hotfixConfig{}, cfg) }) - t.Run("file is empty", func(t *testing.T) { + t.Run("empty file returns zero config", func(t *testing.T) { path := filepath.Join(t.TempDir(), "hotfix-config.json") - require.NoError(t, os.WriteFile(path, []byte(""), 0644)) - version, err := readHotfixVersion(path) + require.NoError(t, os.WriteFile(path, []byte(" \n"), 0644)) + cfg, err := readHotfixConfig(path) assert.NoError(t, err) - assert.Equal(t, "", version) + assert.Equal(t, hotfixConfig{}, cfg) }) - t.Run("file has version", func(t *testing.T) { + t.Run("parses legacy version field", func(t *testing.T) { path := filepath.Join(t.TempDir(), "hotfix-config.json") - require.NoError(t, os.WriteFile(path, []byte(`{"version": "202603.30.0-hotfix1"}`), 0644)) - version, err := readHotfixVersion(path) - assert.NoError(t, err) - assert.Equal(t, "202603.30.0-hotfix1", version) + require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1"}`), 0644)) + cfg, err := readHotfixConfig(path) + require.NoError(t, err) + assert.Equal(t, "202604.01.1", cfg.Version) + assert.Empty(t, cfg.Hotfixes) }) - t.Run("file has empty version field", func(t *testing.T) { + t.Run("parses base->version map", func(t *testing.T) { path := filepath.Join(t.TempDir(), "hotfix-config.json") - require.NoError(t, os.WriteFile(path, []byte(`{"version": ""}`), 0644)) - version, err := readHotfixVersion(path) - assert.NoError(t, err) - assert.Equal(t, "", version) + require.NoError(t, os.WriteFile(path, []byte(`{"hotfixes": {"202604.01": "202604.01.1", "202605.30": "202605.30.2"}}`), 0644)) + cfg, err := readHotfixConfig(path) + require.NoError(t, err) + assert.Equal(t, map[string]string{"202604.01": "202604.01.1", "202605.30": "202605.30.2"}, cfg.Hotfixes) }) - t.Run("file has invalid JSON", func(t *testing.T) { + t.Run("invalid JSON errors", func(t *testing.T) { path := filepath.Join(t.TempDir(), "hotfix-config.json") require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - _, err := readHotfixVersion(path) + _, err := readHotfixConfig(path) assert.Error(t, err) assert.Contains(t, err.Error(), "parsing hotfix config") }) - t.Run("file has extra fields (forward compat)", func(t *testing.T) { + t.Run("ignores extra fields (forward compat)", func(t *testing.T) { path := filepath.Join(t.TempDir(), "hotfix-config.json") - require.NoError(t, os.WriteFile(path, []byte(`{"version": "1.0.0", "sha256": "abc123"}`), 0644)) - version, err := readHotfixVersion(path) - assert.NoError(t, err) - assert.Equal(t, "1.0.0", version) + require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1", "sha256": "abc123"}`), 0644)) + cfg, err := readHotfixConfig(path) + require.NoError(t, err) + assert.Equal(t, "202604.01.1", cfg.Version) + }) +} + +func TestHotfixBaseFromVersion(t *testing.T) { + tests := []struct { + name string + version string + want string + wantErr bool + }{ + {"standard version", "202604.01.1", "202604.01", false}, + {"preserves leading zero day", "202604.01.0", "202604.01", false}, + {"two-digit day", "202605.30.2", "202605.30", false}, + {"trims whitespace", " 202604.01.1 ", "202604.01", false}, + {"too few segments", "202604.01", "", true}, + {"empty patch segment", "202604.01.", "", true}, + {"single segment", "dev", "", true}, + {"empty", "", "", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := hotfixBaseFromVersion(tc.version) + if tc.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.want, got) + } + }) + } +} + +func TestHotfixConfigResolveVersion(t *testing.T) { + t.Run("empty map falls back to legacy Version field", func(t *testing.T) { + cfg := hotfixConfig{Version: "202604.01.1"} + assert.Equal(t, "202604.01.1", cfg.resolveVersion("202604.01.0")) + }) + + t.Run("empty config resolves to empty", func(t *testing.T) { + cfg := hotfixConfig{} + assert.Equal(t, "", cfg.resolveVersion("202604.01.0")) + }) + + t.Run("map hit returns matching base entry", func(t *testing.T) { + cfg := hotfixConfig{Hotfixes: map[string]string{ + "202604.01": "202604.01.1", + "202605.30": "202605.30.2", + }} + assert.Equal(t, "202604.01.1", cfg.resolveVersion("202604.01.0")) + assert.Equal(t, "202605.30.2", cfg.resolveVersion("202605.30.0")) + }) + + t.Run("map miss returns empty (default deny for unlisted base)", func(t *testing.T) { + cfg := hotfixConfig{Hotfixes: map[string]string{"202604.01": "202604.01.1"}} + assert.Equal(t, "", cfg.resolveVersion("202606.09.0")) + }) + + t.Run("map preserves leading-zero day matching", func(t *testing.T) { + cfg := hotfixConfig{Hotfixes: map[string]string{"202604.01": "202604.01.1"}} + assert.Equal(t, "202604.01.1", cfg.resolveVersion("202604.01.0")) + }) + + t.Run("map takes precedence over legacy Version field", func(t *testing.T) { + cfg := hotfixConfig{ + Version: "202604.01.9", + Hotfixes: map[string]string{"202604.01": "202604.01.1"}, + } + assert.Equal(t, "202604.01.1", cfg.resolveVersion("202604.01.0")) + }) + + t.Run("unparseable current version with map returns empty (fail-open)", func(t *testing.T) { + cfg := hotfixConfig{Hotfixes: map[string]string{"202604.01": "202604.01.1"}} + assert.Equal(t, "", cfg.resolveVersion("dev")) }) } @@ -190,16 +264,161 @@ func TestDownloadHotfix_MatchingBaseUpgrades(t *testing.T) { assert.True(t, installCalled, "should proceed when base matches and hotfix patch is higher") } -func TestDownloadHotfix_UnreadableFile(t *testing.T) { +func TestDownloadHotfix_UnreadableFileFailsOpen(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + dir := t.TempDir() path := filepath.Join(dir, "hotfix-config.json") - require.NoError(t, os.WriteFile(path, []byte(`{"version": "1.0.0"}`), 0o644)) + require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1"}`), 0o644)) require.NoError(t, os.Chmod(path, 0o000)) t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) + if _, err := os.ReadFile(path); err == nil { + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o755)) + } - tt := NewTestApp(t, TestAppConfig{}) + aptDir := filepath.Join(dir, "sources.list.d") + require.NoError(t, os.MkdirAll(aptDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte("deb ..."), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(*exec.Cmd) error { + installCalled = true + return nil + }, + }) + tt.App.hotfixVersionPath = path + tt.App.aptSourcesDir = aptDir + // Fail-open: an unreadable config must skip the hotfix without erroring, + // so download-hotfix never blocks provisioning. + require.NoError(t, tt.App.downloadHotfix(context.Background())) + assert.False(t, installCalled, "should skip install when the config cannot be read") +} + +func TestDownloadHotfix_InvalidJSONFailsOpen(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + dir := t.TempDir() + path := filepath.Join(dir, "hotfix-config.json") + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + + aptDir := filepath.Join(dir, "sources.list.d") + require.NoError(t, os.MkdirAll(aptDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte("deb ..."), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(*exec.Cmd) error { + installCalled = true + return nil + }, + }) tt.App.hotfixVersionPath = path - require.Error(t, tt.App.downloadHotfix(context.Background())) + tt.App.aptSourcesDir = aptDir + // Fail-open: malformed JSON must skip the hotfix without erroring. + require.NoError(t, tt.App.downloadHotfix(context.Background())) + assert.False(t, installCalled, "should skip install when the config is invalid JSON") +} + +func TestDownloadHotfix_MapBaseNotPresentSkips(t *testing.T) { + origVersion := Version + Version = "202606.09.0" + defer func() { Version = origVersion }() + + dir := t.TempDir() + path := filepath.Join(dir, "hotfix-config.json") + require.NoError(t, os.WriteFile(path, []byte(`{"hotfixes": {"202604.01": "202604.01.1"}}`), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(cmd *exec.Cmd) error { + installCalled = true + return nil + }, + }) + tt.App.hotfixVersionPath = path + require.NoError(t, tt.App.downloadHotfix(context.Background())) + assert.False(t, installCalled, "should skip when VHD base is not a key in the hotfixes map") +} + +func TestDownloadHotfix_MapMatchingBaseUpgrades(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + dir := t.TempDir() + path := filepath.Join(dir, "hotfix-config.json") + require.NoError(t, os.WriteFile(path, []byte(`{"hotfixes": {"202604.01": "202604.01.1", "202605.30": "202605.30.2"}}`), 0o644)) + + aptDir := filepath.Join(dir, "sources.list.d") + require.NoError(t, os.MkdirAll(aptDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte("deb ..."), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(cmd *exec.Cmd) error { + installCalled = true + return nil + }, + }) + tt.App.hotfixVersionPath = path + tt.App.aptSourcesDir = aptDir + // Will fail at copyBinaryAlongside since pkgBinaryPath doesn't exist in test, + // but install should have been called for the matching base entry. + err := tt.App.downloadHotfix(context.Background()) + require.Error(t, err) + assert.True(t, installCalled, "should install the hotfix for the matching base entry") +} + +func TestDownloadHotfix_MapPatchNotHigherSkips(t *testing.T) { + origVersion := Version + Version = "202604.01.2" + defer func() { Version = origVersion }() + + dir := t.TempDir() + path := filepath.Join(dir, "hotfix-config.json") + // Map entry resolves for this base, but patch (1) is not strictly higher than baked (2). + require.NoError(t, os.WriteFile(path, []byte(`{"hotfixes": {"202604.01": "202604.01.1"}}`), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(cmd *exec.Cmd) error { + installCalled = true + return nil + }, + }) + tt.App.hotfixVersionPath = path + require.NoError(t, tt.App.downloadHotfix(context.Background())) + assert.False(t, installCalled, "should skip when resolved hotfix patch is not strictly higher") +} + +func TestDownloadHotfix_MapMisconfiguredValueBaseSkips(t *testing.T) { + origVersion := Version + Version = "202604.01.0" + defer func() { Version = origVersion }() + + dir := t.TempDir() + path := filepath.Join(dir, "hotfix-config.json") + // Misconfigured entry: the value's base (202605.30) does not match its key (202604.01). + // resolveVersion selects it by key for a node on base 202604.01, but shouldUpgradeToHotfix + // must reject it because the YYYYMM.DD bases differ, so no wrong-base binary is installed. + require.NoError(t, os.WriteFile(path, []byte(`{"hotfixes": {"202604.01": "202605.30.2"}}`), 0o644)) + + installCalled := false + tt := NewTestApp(t, TestAppConfig{ + RunFunc: func(cmd *exec.Cmd) error { + installCalled = true + return nil + }, + }) + tt.App.hotfixVersionPath = path + require.NoError(t, tt.App.downloadHotfix(context.Background())) + assert.False(t, installCalled, "should skip when the map value's base does not match the node's base") } func TestRetryCommand_SuccessOnFirstAttempt(t *testing.T) { @@ -335,9 +554,9 @@ func TestShouldUpgradeToHotfix(t *testing.T) { wantErr bool }{ // Positive: same base, hotfix has higher patch - {"base .0 → hotfix .1", "202604.01.0", "202604.01.1", true, false}, - {"base .0 → hotfix .2", "202604.01.0", "202604.01.2", true, false}, - {"hotfix .1 → hotfix .2", "202604.01.1", "202604.01.2", true, false}, + {"base .0 -> hotfix .1", "202604.01.0", "202604.01.1", true, false}, + {"base .0 -> hotfix .2", "202604.01.0", "202604.01.2", true, false}, + {"hotfix .1 -> hotfix .2", "202604.01.1", "202604.01.2", true, false}, // Negative: same version {"same version .0", "202604.01.0", "202604.01.0", false, false}, diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/config.pb.go index 77d87a35bf5..65f02267048 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/config.pb.go @@ -167,6 +167,10 @@ type Configuration struct { ServiceAccountImagePullProfile *ServiceAccountImagePullProfile `protobuf:"bytes,43,opt,name=service_account_image_pull_profile,json=serviceAccountImagePullProfile,proto3" json:"service_account_image_pull_profile,omitempty"` // CSE timeout override in seconds. If not specified, defaults to 15 minutes with a maximum of 360 minutes (6 hours). CseTimeout *int32 `protobuf:"varint,44,opt,name=cse_timeout,json=cseTimeout,proto3,oneof" json:"cse_timeout,omitempty"` + // Enables provisioning-time hotfix checking. When true, aks-node-controller's + // check-hotfix reads the hotfix pointer from the LPS (live-patching-service) + // endpoint; when false or unset it no-ops without any remote call. Default-off, fail-open. + EnableProvisioningHotfix bool `protobuf:"varint,45,opt,name=enable_provisioning_hotfix,json=enableProvisioningHotfix,proto3" json:"enable_provisioning_hotfix,omitempty"` } func (x *Configuration) Reset() { @@ -500,6 +504,13 @@ func (x *Configuration) GetCseTimeout() int32 { return 0 } +func (x *Configuration) GetEnableProvisioningHotfix() bool { + if x != nil { + return x.EnableProvisioningHotfix + } + return false +} + var File_aksnodeconfig_v1_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_config_proto_rawDesc = []byte{ @@ -551,7 +562,7 @@ var file_aksnodeconfig_v1_config_proto_rawDesc = []byte{ 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x70, 0x72, 0x6f, 0x66, - 0x69, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdd, 0x15, 0x0a, 0x0d, 0x43, 0x6f, + 0x69, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9b, 0x16, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x12, 0x6b, 0x75, 0x62, 0x65, 0x5f, 0x62, 0x69, @@ -718,26 +729,30 @@ var file_aksnodeconfig_v1_config_proto_rawDesc = []byte{ 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x24, 0x0a, 0x0b, 0x63, 0x73, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x05, 0x48, 0x04, 0x52, 0x0a, - 0x63, 0x73, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, - 0x07, 0x5f, 0x69, 0x73, 0x5f, 0x76, 0x68, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x73, 0x68, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6e, 0x65, 0x65, 0x64, - 0x73, 0x5f, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x76, 0x32, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x64, - 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x5f, 0x61, 0x75, - 0x74, 0x68, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, 0x73, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x0f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2a, 0x57, 0x0a, 0x0f, 0x57, 0x6f, 0x72, - 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x1c, + 0x63, 0x73, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a, + 0x1a, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x68, 0x6f, 0x74, 0x66, 0x69, 0x78, 0x18, 0x2d, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x18, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, + 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x48, 0x6f, 0x74, 0x66, 0x69, 0x78, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x69, 0x73, 0x5f, 0x76, 0x68, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x73, 0x73, 0x68, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x6e, 0x65, 0x65, 0x64, 0x73, 0x5f, + 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x76, 0x32, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x64, 0x69, 0x73, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x5f, 0x61, 0x75, 0x74, 0x68, + 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x63, 0x73, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x0f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2a, 0x57, 0x0a, 0x0f, 0x57, 0x6f, 0x72, 0x6b, 0x6c, + 0x6f, 0x61, 0x64, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x1c, 0x57, 0x4f, + 0x52, 0x4b, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, 0x4d, 0x45, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x22, 0x0a, 0x1e, 0x57, 0x4f, 0x52, 0x4b, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, 0x4d, 0x45, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x22, - 0x0a, 0x1e, 0x57, 0x4f, 0x52, 0x4b, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, - 0x4d, 0x45, 0x5f, 0x4f, 0x43, 0x49, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, - 0x10, 0x01, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, - 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, - 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, - 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x5f, 0x4f, 0x43, 0x49, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x10, 0x01, + 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, + 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, + 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, + 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/aks-node-controller/proto/aksnodeconfig/v1/config.proto b/aks-node-controller/proto/aksnodeconfig/v1/config.proto index 943608acdda..cf0470d0b28 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/config.proto @@ -169,4 +169,9 @@ message Configuration { // CSE timeout override in seconds. If not specified, defaults to 15 minutes with a maximum of 360 minutes (6 hours). optional int32 cse_timeout = 44; + + // Enables provisioning-time hotfix checking. When true, aks-node-controller's + // check-hotfix reads the hotfix pointer from the LPS (live-patching-service) + // endpoint; when false or unset it no-ops without any remote call. Default-off, fail-open. + bool enable_provisioning_hotfix = 45; } diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh index 310c41da47e..51b6fd9c681 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-wrapper.sh @@ -7,7 +7,7 @@ done BIN_PATH="${BIN_PATH:-/opt/azure/containers/aks-node-controller}" HOTFIX_BIN="${BIN_PATH}-hotfix" -HOTFIX_JSON="/opt/azure/containers/aks-node-controller-hotfix.json" +HOTFIX_JSON="${HOTFIX_JSON:-/opt/azure/containers/aks-node-controller-hotfix.json}" CONFIG_PATH="${CONFIG_PATH:-/opt/azure/containers/aks-node-controller-config.json}" NBC_CMD_PATH="${NBC_CMD_PATH:-/opt/azure/containers/aks-node-controller-nbc-cmd.sh}" LOGGER_TAG="aks-node-controller-wrapper" @@ -27,6 +27,21 @@ if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then exit 0 fi +# check-hotfix reads the hotfix pointer from the LPS endpoint (IMDS-attested) and refreshes +# $HOTFIX_JSON, which the download-hotfix block below consumes, so it must run first. +# It is called unconditionally and is fail-open (always exits 0); the aks-node-controller +# binary self-gates on the enable_provisioning_hotfix AKSNodeConfig contract field (single +# source of truth), no-opping without any remote hotfix call when the feature is off. This +# replaces the earlier ENABLE_PROVISIONING_HOTFIX env gate so absvc sets one contract field +# instead of an env var plus a field. We still wrap it defensively so it can never block +# provisioning. +log "running check-hotfix to refresh hotfix pointer (self-gated on the node config contract field)" +if "$BIN_PATH" check-hotfix; then + log "ANC check-hotfix completed; hotfix pointer refresh attempted" +else + log "ANC check-hotfix failed; continuing (fail-open)" +fi + if [ -f "$HOTFIX_JSON" ]; then log "Found ANC hotfix config at ${HOTFIX_JSON}; running download-hotfix" if "$BIN_PATH" download-hotfix; then diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh index 2dac2046abd..a383f85956f 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_wrapper_spec.sh @@ -35,11 +35,14 @@ EOF export BIN_PATH="${TEST_DIR}/aks-node-controller" export CONFIG_PATH="${TEST_DIR}/aks-node-controller-config.json" export NBC_CMD_PATH="${TEST_DIR}/aks-node-controller-nbc-cmd.sh" + # Point hotfix pointer at a test-local path (absent by default) so tests never + # touch the production /opt/azure path and can control the download-hotfix branch. + export HOTFIX_JSON="${TEST_DIR}/aks-node-controller-hotfix.json" } cleanup_wrapper_test() { rm -rf "$TEST_DIR" - unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR + unset BIN_PATH CONFIG_PATH NBC_CMD_PATH TEST_DIR BIN_DIR HOTFIX_JSON CHECK_HOTFIX_EXIT } create_fake_aks_node_controller() { @@ -51,6 +54,21 @@ EOF chmod +x "$BIN_PATH" } + # Records each subcommand (first arg) on its own line in calls log so ordering across + # multiple invocations (check-hotfix vs download-hotfix vs provision) is observable. + # CHECK_HOTFIX_EXIT controls the exit code of the check-hotfix invocation only. + create_recording_aks_node_controller() { + cat >"$BIN_PATH" <<'EOF' +#!/bin/sh +printf '%s\n' "$1" >>"${TEST_DIR}/calls" +if [ "$1" = "check-hotfix" ]; then + exit "${CHECK_HOTFIX_EXIT:-0}" +fi +exit 0 +EOF + chmod +x "$BIN_PATH" + } + BeforeEach setup_wrapper_test AfterEach cleanup_wrapper_test @@ -108,4 +126,54 @@ EOF The variable secondArg should eq "--nbc-cmd=${NBC_CMD_PATH}" The variable thirdArg should eq "" End + + # check-hotfix is now called UNCONDITIONALLY (2.1d). The wrapper no longer reads the + # ENABLE_PROVISIONING_HOTFIX env gate; the aks-node-controller binary self-gates on the + # enable_provisioning_hotfix AKSNodeConfig contract field (single source of truth) and + # no-ops when the feature is off. So the wrapper always invokes it and stays fail-open. + It 'always runs check-hotfix before download-hotfix and provision' + touch "$CONFIG_PATH" "$HOTFIX_JSON" + create_recording_aks_node_controller + + When run bash "$SCRIPT" + The status should be success + The output should include "running check-hotfix" + The output should include "ANC check-hotfix completed" + firstCall=$(sed -n '1p' "${TEST_DIR}/calls") + secondCall=$(sed -n '2p' "${TEST_DIR}/calls") + thirdCall=$(sed -n '3p' "${TEST_DIR}/calls") + The variable firstCall should eq "check-hotfix" + The variable secondCall should eq "download-hotfix" + The variable thirdCall should eq "provision" + End + + It 'runs check-hotfix even with no hotfix pointer present (binary self-gates)' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + + When run bash "$SCRIPT" + The status should be success + The output should include "running check-hotfix" + firstCall=$(sed -n '1p' "${TEST_DIR}/calls") + lastCall=$(tail -n 1 "${TEST_DIR}/calls") + The variable firstCall should eq "check-hotfix" + The variable lastCall should eq "provision" + End + + # Fail-open also covers the backward-compat case where check-hotfix reaches a node whose + # VHD-baked binary predates 2.1b: `check-hotfix` is an unknown subcommand there and exits + # non-zero, which the wrapper tolerates so provisioning still proceeds. + It 'proceeds to provision when check-hotfix fails (fail-open)' + touch "$CONFIG_PATH" + create_recording_aks_node_controller + export CHECK_HOTFIX_EXIT="1" + + When run bash "$SCRIPT" + The status should be success + The output should include "ANC check-hotfix failed; continuing (fail-open)" + firstCall=$(sed -n '1p' "${TEST_DIR}/calls") + lastCall=$(tail -n 1 "${TEST_DIR}/calls") + The variable firstCall should eq "check-hotfix" + The variable lastCall should eq "provision" + End End