From 919d3ebf96310aefc3c7535e0b44c491828686d0 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:07:47 -0400
Subject: [PATCH 1/7] vsphere: short-circuit reconciliation of stable Running
 machines

Machines that are fully stable (Phase=Running, ProviderID set, NodeRef
present, addresses populated, not deleting) and whose provider spec has
not changed since the last full reconciliation are short-circuited in
both Exists() and Update(), avoiding all vCenter API calls. A SHA-256
hash of the provider spec is stored in an annotation to detect spec
changes, and an RFC3339 timestamp annotation enforces a 1-hour TTL to
guarantee periodic drift detection regardless of spec changes.

For a 200-machine stable cluster this eliminates ~30,000 unnecessary
vCenter API calls per hour, reducing new-machine queue wait from 10+
minutes to near-zero.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 pkg/controller/vsphere/actuator.go            | 102 ++++++++
 pkg/controller/vsphere/reconcile_skip_test.go | 238 ++++++++++++++++++
 2 files changed, 340 insertions(+)
 create mode 100644 pkg/controller/vsphere/reconcile_skip_test.go

diff --git a/pkg/controller/vsphere/actuator.go b/pkg/controller/vsphere/actuator.go
index 6b9aad746..bb5bb75bc 100644
--- a/pkg/controller/vsphere/actuator.go
+++ b/pkg/controller/vsphere/actuator.go
@@ -4,6 +4,8 @@ package vsphere
 // The lifetime of scope and reconciler is a machine actuator operation.
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"time"
 
@@ -14,6 +16,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/events"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -25,8 +28,91 @@ const (
 	deleteEventAction   = "Delete"
 	noEventAction       = ""
 	requeueAfterSeconds = 20
+
+	// lastReconciledProviderSpecHashAnnotation records a hash of the provider spec that was
+	// in effect the last time the machine went through a full reconciliation. It is used,
+	// together with lastFullReconcileTimestampAnnotation, to short-circuit reconciliation of
+	// stable Running machines and avoid unnecessary vCenter API load.
+	lastReconciledProviderSpecHashAnnotation = "machine.openshift.io/last-reconciled-provider-spec-hash"
+
+	// lastFullReconcileTimestampAnnotation records the RFC3339 timestamp of the last full
+	// reconciliation. Once it is older than fullReconcileTTL, the short-circuit no longer
+	// applies and a full reconciliation is forced, guaranteeing drift detection at least
+	// once per TTL window regardless of spec changes.
+	lastFullReconcileTimestampAnnotation = "machine.openshift.io/last-full-reconcile-timestamp"
+
+	// fullReconcileTTL bounds how long a machine can go without a full reconciliation.
+	fullReconcileTTL = time.Hour
 )
 
+// hashProviderSpec returns a short, deterministic hex-encoded SHA-256 hash of the given
+// raw provider spec bytes, used to detect provider spec changes cheaply.
+func hashProviderSpec(raw []byte) string {
+	sum := sha256.Sum256(raw)
+	return hex.EncodeToString(sum[:])[:16]
+}
+
+// canSkipFullReconcile returns true if machine is a stable, Running machine whose provider
+// spec has not changed since the last full reconciliation and whose last full reconciliation
+// happened within fullReconcileTTL. When true, callers may skip vCenter API calls entirely.
+func canSkipFullReconcile(machine *machinev1.Machine) bool {
+	if ptr.Deref(machine.Status.Phase, "") != machinev1.PhaseRunning {
+		return false
+	}
+	if ptr.Deref(machine.Spec.ProviderID, "") == "" {
+		return false
+	}
+	if machine.Status.NodeRef == nil {
+		return false
+	}
+	if len(machine.Status.Addresses) == 0 {
+		return false
+	}
+	if !machine.GetDeletionTimestamp().IsZero() {
+		return false
+	}
+
+	annotations := machine.GetAnnotations()
+
+	raw := providerSpecRawBytes(machine)
+	if raw == nil {
+		return false
+	}
+
+	lastHash, ok := annotations[lastReconciledProviderSpecHashAnnotation]
+	if !ok || lastHash != hashProviderSpec(raw) {
+		return false
+	}
+
+	lastFullReconcile, ok := annotations[lastFullReconcileTimestampAnnotation]
+	if !ok {
+		return false
+	}
+	lastFullReconcileTime, err := time.Parse(time.RFC3339, lastFullReconcile)
+	if err != nil {
+		return false
+	}
+
+	return time.Since(lastFullReconcileTime) < fullReconcileTTL
+}
+
+// markFullReconcileComplete records the current provider spec hash and timestamp on the
+// machine so that subsequent reconciliations can short-circuit via canSkipFullReconcile.
+func markFullReconcileComplete(machine *machinev1.Machine) {
+	if machine.Annotations == nil {
+		machine.Annotations = map[string]string{}
+	}
+	machine.Annotations[lastReconciledProviderSpecHashAnnotation] = hashProviderSpec(providerSpecRawBytes(machine))
+	machine.Annotations[lastFullReconcileTimestampAnnotation] = time.Now().UTC().Format(time.RFC3339)
+}
+
+func providerSpecRawBytes(machine *machinev1.Machine) []byte {
+	if machine.Spec.ProviderSpec.Value == nil {
+		return nil
+	}
+	return machine.Spec.ProviderSpec.Value.Raw
+}
+
 // Actuator is responsible for performing machine reconciliation.
 type Actuator struct {
 	client                   runtimeclient.Client
@@ -116,6 +202,11 @@ func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error
 }
 
 func (a *Actuator) Exists(ctx context.Context, machine *machinev1.Machine) (bool, error) {
+	if canSkipFullReconcile(machine) {
+		klog.V(3).Infof("%s: machine is stable, skipping full reconcile in Exists()", machine.GetName())
+		return true, nil
+	}
+
 	klog.Infof("%s: actuator checking if machine exists", machine.GetName())
 	scope, err := newMachineScope(machineScopeParams{
 		Context:                  ctx,
@@ -132,6 +223,11 @@ func (a *Actuator) Exists(ctx context.Context, machine *machinev1.Machine) (bool
 }
 
 func (a *Actuator) Update(ctx context.Context, machine *machinev1.Machine) error {
+	if canSkipFullReconcile(machine) {
+		klog.V(3).Infof("%s: machine is stable, skipping full reconcile in Update()", machine.GetName())
+		return nil
+	}
+
 	klog.Infof("%s: actuator updating machine", machine.GetName())
 	// Cleanup TaskIDCache so we don't continually grow
 	delete(a.TaskIDCache, machine.Name)
@@ -156,6 +252,12 @@ func (a *Actuator) Update(ctx context.Context, machine *machinev1.Machine) error
 		fmtErr := fmt.Errorf(reconcilerFailFmt, machine.GetName(), updateEventAction, err)
 		return a.handleMachineError(machine, fmtErr, updateEventAction)
 	}
+
+	// A full reconciliation just completed successfully: record the provider spec hash and
+	// timestamp so that future stable reconciliations can short-circuit via
+	// canSkipFullReconcile, avoiding unnecessary vCenter API calls.
+	markFullReconcileComplete(scope.machine)
+
 	previousResourceVersion := scope.machine.ResourceVersion
 
 	if err := scope.PatchMachine(); err != nil {
diff --git a/pkg/controller/vsphere/reconcile_skip_test.go b/pkg/controller/vsphere/reconcile_skip_test.go
new file mode 100644
index 000000000..b1a8301e7
--- /dev/null
+++ b/pkg/controller/vsphere/reconcile_skip_test.go
@@ -0,0 +1,238 @@
+package vsphere
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/onsi/gomega"
+	machinev1 "github.com/openshift/api/machine/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
+)
+
+func TestHashProviderSpec(t *testing.T) {
+	g := NewWithT(t)
+
+	g.Expect(hashProviderSpec(nil)).To(HaveLen(16))
+	g.Expect(hashProviderSpec([]byte("foo"))).To(Equal(hashProviderSpec([]byte("foo"))))
+	g.Expect(hashProviderSpec([]byte("foo"))).ToNot(Equal(hashProviderSpec([]byte("bar"))))
+	g.Expect(hashProviderSpec(nil)).To(Equal(hashProviderSpec([]byte{})))
+}
+
+func stableMachine() *machinev1.Machine {
+	raw := []byte(`{"foo":"bar"}`)
+	m := &machinev1.Machine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			Annotations: map[string]string{
+				lastReconciledProviderSpecHashAnnotation: hashProviderSpec(raw),
+				lastFullReconcileTimestampAnnotation:     time.Now().Format(time.RFC3339),
+			},
+		},
+		Spec: machinev1.MachineSpec{
+			ProviderID: ptr.To("vsphere://12345"),
+			ProviderSpec: machinev1.ProviderSpec{
+				Value: &runtime.RawExtension{Raw: raw},
+			},
+		},
+		Status: machinev1.MachineStatus{
+			Phase:   ptr.To(machinev1.PhaseRunning),
+			NodeRef: &corev1.ObjectReference{Name: "test-node"},
+			Addresses: []corev1.NodeAddress{
+				{Type: corev1.NodeInternalIP, Address: "10.0.0.1"},
+			},
+		},
+	}
+	return m
+}
+
+func TestCanSkipFullReconcile(t *testing.T) {
+	cases := []struct {
+		name     string
+		mutate   func(m *machinev1.Machine)
+		expected bool
+	}{
+		{
+			name:     "fully stable machine",
+			mutate:   func(m *machinev1.Machine) {},
+			expected: true,
+		},
+		{
+			name: "missing phase",
+			mutate: func(m *machinev1.Machine) {
+				m.Status.Phase = nil
+			},
+			expected: false,
+		},
+		{
+			name: "wrong phase",
+			mutate: func(m *machinev1.Machine) {
+				m.Status.Phase = ptr.To(machinev1.PhaseProvisioned)
+			},
+			expected: false,
+		},
+		{
+			name: "missing providerID",
+			mutate: func(m *machinev1.Machine) {
+				m.Spec.ProviderID = nil
+			},
+			expected: false,
+		},
+		{
+			name: "empty providerID",
+			mutate: func(m *machinev1.Machine) {
+				m.Spec.ProviderID = ptr.To("")
+			},
+			expected: false,
+		},
+		{
+			name: "missing nodeRef",
+			mutate: func(m *machinev1.Machine) {
+				m.Status.NodeRef = nil
+			},
+			expected: false,
+		},
+		{
+			name: "missing addresses",
+			mutate: func(m *machinev1.Machine) {
+				m.Status.Addresses = nil
+			},
+			expected: false,
+		},
+		{
+			name: "deleting",
+			mutate: func(m *machinev1.Machine) {
+				now := metav1.Now()
+				m.DeletionTimestamp = &now
+			},
+			expected: false,
+		},
+		{
+			name: "nil provider spec value",
+			mutate: func(m *machinev1.Machine) {
+				m.Spec.ProviderSpec.Value = nil
+			},
+			expected: false,
+		},
+		{
+			name: "missing annotations (upgrade path)",
+			mutate: func(m *machinev1.Machine) {
+				m.Annotations = nil
+			},
+			expected: false,
+		},
+		{
+			name: "missing hash annotation only",
+			mutate: func(m *machinev1.Machine) {
+				delete(m.Annotations, lastReconciledProviderSpecHashAnnotation)
+			},
+			expected: false,
+		},
+		{
+			name: "missing timestamp annotation only",
+			mutate: func(m *machinev1.Machine) {
+				delete(m.Annotations, lastFullReconcileTimestampAnnotation)
+			},
+			expected: false,
+		},
+		{
+			name: "hash mismatch",
+			mutate: func(m *machinev1.Machine) {
+				m.Annotations[lastReconciledProviderSpecHashAnnotation] = "deadbeefdeadbeef"
+			},
+			expected: false,
+		},
+		{
+			name:     "hash match",
+			mutate:   func(m *machinev1.Machine) {},
+			expected: true,
+		},
+		{
+			name: "expired timestamp",
+			mutate: func(m *machinev1.Machine) {
+				m.Annotations[lastFullReconcileTimestampAnnotation] = time.Now().Add(-2 * time.Hour).Format(time.RFC3339)
+			},
+			expected: false,
+		},
+		{
+			name: "timestamp just under TTL",
+			mutate: func(m *machinev1.Machine) {
+				m.Annotations[lastFullReconcileTimestampAnnotation] = time.Now().Add(-59 * time.Minute).Format(time.RFC3339)
+			},
+			expected: true,
+		},
+		{
+			name: "unparseable timestamp",
+			mutate: func(m *machinev1.Machine) {
+				m.Annotations[lastFullReconcileTimestampAnnotation] = "not-a-timestamp"
+			},
+			expected: false,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			m := stableMachine()
+			tc.mutate(m)
+			g.Expect(canSkipFullReconcile(m)).To(Equal(tc.expected))
+		})
+	}
+}
+
+// TestActuatorShortCircuitsStableMachine verifies that a stable, Running machine causes
+// Exists() and Update() to return immediately without touching the Actuator's client,
+// apiReader, or event recorder (i.e. without making any vCenter or Kubernetes API calls).
+// A bare Actuator with nil fields is used deliberately: if the short-circuit did not fire,
+// any attempt to use those nil fields would panic.
+func TestActuatorShortCircuitsStableMachine(t *testing.T) {
+	g := NewWithT(t)
+
+	a := &Actuator{}
+	m := stableMachine()
+
+	exists, err := a.Exists(t.Context(), m)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(exists).To(BeTrue())
+
+	g.Expect(a.Update(t.Context(), m)).To(Succeed())
+}
+
+// TestActuatorUpgradePathDoesNotShortCircuit verifies that a machine missing the
+// short-circuit annotations (e.g. right after an upgrade from a version that didn't set
+// them) does not short-circuit, since canSkipFullReconcile requires them to be present.
+func TestActuatorUpgradePathDoesNotShortCircuit(t *testing.T) {
+	g := NewWithT(t)
+
+	m := stableMachine()
+	m.Annotations = nil
+
+	g.Expect(canSkipFullReconcile(m)).To(BeFalse())
+}
+
+func TestMarkFullReconcileComplete(t *testing.T) {
+	g := NewWithT(t)
+
+	raw := []byte(`{"foo":"bar"}`)
+	m := &machinev1.Machine{
+		Spec: machinev1.MachineSpec{
+			ProviderSpec: machinev1.ProviderSpec{
+				Value: &runtime.RawExtension{Raw: raw},
+			},
+		},
+	}
+
+	before := time.Now()
+	markFullReconcileComplete(m)
+	after := time.Now()
+
+	g.Expect(m.Annotations).To(HaveKeyWithValue(lastReconciledProviderSpecHashAnnotation, hashProviderSpec(raw)))
+	g.Expect(m.Annotations).To(HaveKey(lastFullReconcileTimestampAnnotation))
+
+	parsed, err := time.Parse(time.RFC3339, m.Annotations[lastFullReconcileTimestampAnnotation])
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(parsed).To(BeTemporally(">=", before.Truncate(time.Second)))
+	g.Expect(parsed).To(BeTemporally("<=", after.Add(time.Second)))
+}

From 78f4f221f7b3838e18cdb3ef2308b0aaec8286a1 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:08:39 -0400
Subject: [PATCH 2/7] vsphere: increase SyncPeriod to 60 minutes and add
 --sync-period flag

Change the informer resync interval from 10 minutes to 60 minutes,
reducing the frequency of resync-driven reconciliation 6x. Combined
with the per-machine 1-hour TTL from the short-circuit annotation,
stable machines get exactly 1 full reconciliation per hour.

A --sync-period flag is added for operational flexibility. Watch-driven
reconciliation (spec changes, status updates) still fires immediately
regardless of this setting.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 cmd/vsphere/main.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go
index 06f6b7f5c..947089ee2 100644
--- a/cmd/vsphere/main.go
+++ b/cmd/vsphere/main.go
@@ -34,7 +34,7 @@ import (
 	"github.com/openshift/machine-api-operator/pkg/version"
 )
 
-const timeout = 10 * time.Minute
+const timeout = 60 * time.Minute
 
 func main() {
 	var printVersion bool
@@ -99,6 +99,14 @@ func main() {
 		"The address for health checking.",
 	)
 
+	syncPeriodFlag := flag.Duration(
+		"sync-period",
+		timeout,
+		"Minimum interval at which cached resources are re-reconciled. This is the only "+
+			"backstop for drift detection once a machine's per-machine reconciliation TTL "+
+			"annotation is honored, so it should not be disabled.",
+	)
+
 	majorVersion := version.Version.Major
 
 	if majorVersion == 0 {
@@ -123,7 +131,7 @@ func main() {
 	}
 
 	cfg := config.GetConfigOrDie()
-	syncPeriod := timeout
+	syncPeriod := *syncPeriodFlag
 
 	le := util.GetLeaderElectionConfig(cfg, configv1.LeaderElection{
 		Disable:       !*leaderElect,

From 779fd473f8107768b195fe0c1d30625934ae02f7 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:12:24 -0400
Subject: [PATCH 3/7] vsphere: eliminate redundant vCenter calls within a
 reconciliation

Cache machineScope between Exists() and Update() via a sync.Map-based
scopeCache on the Actuator, avoiding redundant session validation and
credential reads. Cache the VM reference from exists() for reuse in
update(), eliminating a duplicate FindByUuid call. Cache UUID() and
getPowerState() results on the virtualMachine struct to avoid duplicate
property collector calls within reconcileMachineWithCloudState.

Convert TaskIDCache from map[string]string to *sync.Map to make it safe
for concurrent reconciliation.

Combined savings: ~5 SOAP + 2 REST calls per full reconciliation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 cmd/vsphere/main.go                           |   6 +-
 pkg/controller/vsphere/actuator.go            |  73 ++++++--
 pkg/controller/vsphere/actuator_cache_test.go |  97 ++++++++++
 .../vsphere/actuator_concurrency_test.go      | 170 ++++++++++++++++++
 pkg/controller/vsphere/actuator_test.go       |   3 +-
 pkg/controller/vsphere/machine_scope.go       |   5 +
 pkg/controller/vsphere/reconciler.go          |  50 +++++-
 .../vsphere/virtualmachine_cache_test.go      |  48 +++++
 8 files changed, 427 insertions(+), 25 deletions(-)
 create mode 100644 pkg/controller/vsphere/actuator_cache_test.go
 create mode 100644 pkg/controller/vsphere/actuator_concurrency_test.go
 create mode 100644 pkg/controller/vsphere/virtualmachine_cache_test.go

diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go
index 947089ee2..db68e5bb1 100644
--- a/cmd/vsphere/main.go
+++ b/cmd/vsphere/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -185,8 +186,9 @@ func main() {
 	}
 
 	// Create a taskIDCache for create task IDs in case they are lost due to
-	// network error or stale cache.
-	taskIDCache := make(map[string]string)
+	// network error or stale cache. sync.Map is used since concurrent reconciliation is
+	// enabled via --max-concurrent-reconciles.
+	taskIDCache := &sync.Map{}
 
 	// Initialize machine actuator.
 	machineActuator := machine.NewActuator(machine.ActuatorParams{
diff --git a/pkg/controller/vsphere/actuator.go b/pkg/controller/vsphere/actuator.go
index bb5bb75bc..11e677170 100644
--- a/pkg/controller/vsphere/actuator.go
+++ b/pkg/controller/vsphere/actuator.go
@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"sync"
 	"time"
 
 	"k8s.io/component-base/featuregate"
@@ -118,9 +119,16 @@ type Actuator struct {
 	client                   runtimeclient.Client
 	apiReader                runtimeclient.Reader
 	eventRecorder            events.EventRecorder
-	TaskIDCache              map[string]string
+	TaskIDCache              *sync.Map
 	FeatureGates             featuregate.MutableFeatureGate
 	openshiftConfigNamespace string
+	// scopeCache holds machineScopes created by Exists(), keyed by machine UID, so that a
+	// subsequent Update() call within the same reconciliation can reuse the scope instead of
+	// creating a new one (which would validate the vCenter session and read the credentials
+	// ConfigMap/Secret again). Entries are removed by Update(), Create(), and Delete() to
+	// avoid leaking scopes. Uses sync.Map (not a plain map) because concurrent reconciliation
+	// of distinct machines is supported.
+	scopeCache sync.Map
 }
 
 // ActuatorParams holds parameter information for Actuator.
@@ -128,23 +136,49 @@ type ActuatorParams struct {
 	Client                   runtimeclient.Client
 	APIReader                runtimeclient.Reader
 	EventRecorder            events.EventRecorder
-	TaskIDCache              map[string]string
+	TaskIDCache              *sync.Map
 	FeatureGates             featuregate.MutableFeatureGate
 	OpenshiftConfigNamespace string
 }
 
 // NewActuator returns an actuator.
 func NewActuator(params ActuatorParams) *Actuator {
+	taskIDCache := params.TaskIDCache
+	if taskIDCache == nil {
+		taskIDCache = &sync.Map{}
+	}
 	return &Actuator{
 		client:                   params.Client,
 		apiReader:                params.APIReader,
 		eventRecorder:            params.EventRecorder,
-		TaskIDCache:              params.TaskIDCache,
+		TaskIDCache:              taskIDCache,
 		FeatureGates:             params.FeatureGates,
 		openshiftConfigNamespace: params.OpenshiftConfigNamespace,
 	}
 }
 
+// getOrCreateScope returns the machineScope cached by a preceding Exists() call for this
+// machine's UID, refreshing its machine and context references, or creates a new one via
+// newMachineScope if no cache entry exists.
+func (a *Actuator) getOrCreateScope(ctx context.Context, machine *machinev1.Machine) (*machineScope, error) {
+	if cached, ok := a.scopeCache.Load(machine.GetUID()); ok {
+		scope := cached.(*machineScope)
+		scope.Context = ctx
+		scope.machine = machine
+		scope.machineToBePatched = runtimeclient.MergeFrom(machine.DeepCopy())
+		return scope, nil
+	}
+
+	return newMachineScope(machineScopeParams{
+		Context:                  ctx,
+		client:                   a.client,
+		machine:                  machine,
+		apiReader:                a.apiReader,
+		featureGates:             a.FeatureGates,
+		openshiftConfigNameSpace: a.openshiftConfigNamespace,
+	})
+}
+
 // Set corresponding event based on error. It also returns the original error
 // for convenience, so callers can do "return handleMachineError(...)".
 func (a *Actuator) handleMachineError(machine *machinev1.Machine, err error, eventAction string) error {
@@ -158,6 +192,9 @@ func (a *Actuator) handleMachineError(machine *machinev1.Machine, err error, eve
 // Create creates a machine and is invoked by the machine controller.
 func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error {
 	klog.Infof("%s: actuator creating machine", machine.GetName())
+	// Create() should not normally follow an Exists() call for the same machine, but clear
+	// any stale scope cache entry defensively to avoid a leak if it does.
+	defer a.scopeCache.Delete(machine.GetUID())
 
 	scope, err := newMachineScope(machineScopeParams{
 		Context:                  ctx,
@@ -174,8 +211,8 @@ func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error
 
 	// Ensure we're not reconciling a stale machine by checking our task-id.
 	// This is a workaround for a cache race condition.
-	if val, ok := a.TaskIDCache[machine.Name]; ok {
-		if val != scope.providerStatus.TaskRef {
+	if val, ok := a.TaskIDCache.Load(machine.Name); ok {
+		if val.(string) != scope.providerStatus.TaskRef {
 			klog.Errorf("%s: machine object missing expected provider task ID, requeue", machine.GetName())
 			return &machinecontroller.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
 		}
@@ -185,7 +222,7 @@ func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error
 	err = newReconciler(scope).create()
 	// save the taskRef in our cache in case of any error with patch.
 	if scope.providerStatus.TaskRef != "" {
-		a.TaskIDCache[machine.Name] = scope.providerStatus.TaskRef
+		a.TaskIDCache.Store(machine.Name, scope.providerStatus.TaskRef)
 	}
 	if err != nil {
 		fmtErr := fmt.Errorf(reconcilerFailFmt, machine.GetName(), createEventAction, err)
@@ -219,6 +256,11 @@ func (a *Actuator) Exists(ctx context.Context, machine *machinev1.Machine) (bool
 	if err != nil {
 		return false, fmt.Errorf(scopeFailFmt, machine.GetName(), err)
 	}
+
+	// Cache the scope so a subsequent Update() call within the same reconciliation can reuse
+	// it instead of re-validating the vCenter session and re-reading credentials.
+	a.scopeCache.Store(machine.GetUID(), scope)
+
 	return newReconciler(scope).exists()
 }
 
@@ -230,20 +272,16 @@ func (a *Actuator) Update(ctx context.Context, machine *machinev1.Machine) error
 
 	klog.Infof("%s: actuator updating machine", machine.GetName())
 	// Cleanup TaskIDCache so we don't continually grow
-	delete(a.TaskIDCache, machine.Name)
+	a.TaskIDCache.Delete(machine.Name)
 
-	scope, err := newMachineScope(machineScopeParams{
-		Context:                  ctx,
-		client:                   a.client,
-		machine:                  machine,
-		apiReader:                a.apiReader,
-		featureGates:             a.FeatureGates,
-		openshiftConfigNameSpace: a.openshiftConfigNamespace,
-	})
+	scope, err := a.getOrCreateScope(ctx, machine)
 	if err != nil {
 		fmtErr := fmt.Errorf(scopeFailFmt, machine.GetName(), err)
 		return a.handleMachineError(machine, fmtErr, updateEventAction)
 	}
+	// The scope (if reused from Exists()) is only valid for this single reconciliation.
+	defer a.scopeCache.Delete(machine.GetUID())
+
 	if err := newReconciler(scope).update(); err != nil {
 		// Update machine and machine status in case it was modified
 		if err := scope.PatchMachine(); err != nil {
@@ -278,7 +316,10 @@ func (a *Actuator) Delete(ctx context.Context, machine *machinev1.Machine) error
 	klog.Infof("%s: actuator deleting machine", machine.GetName())
 	// Cleanup TaskIDCache so we don't continually grow
 	// Cleanup here as well in case Update() was never successfully called.
-	delete(a.TaskIDCache, machine.Name)
+	a.TaskIDCache.Delete(machine.Name)
+	// Defensive cleanup: an Exists() call for a machine that is now being deleted may have
+	// cached a scope that Update() never got a chance to consume.
+	defer a.scopeCache.Delete(machine.GetUID())
 
 	scope, err := newMachineScope(machineScopeParams{
 		Context:                  ctx,
diff --git a/pkg/controller/vsphere/actuator_cache_test.go b/pkg/controller/vsphere/actuator_cache_test.go
new file mode 100644
index 000000000..8807e3a72
--- /dev/null
+++ b/pkg/controller/vsphere/actuator_cache_test.go
@@ -0,0 +1,97 @@
+package vsphere
+
+import (
+	"sync"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	machinev1 "github.com/openshift/api/machine/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apimachinerytypes "k8s.io/apimachinery/pkg/types"
+)
+
+// TestGetOrCreateScopeReusesCachedScope verifies that getOrCreateScope returns the exact
+// same *machineScope instance that was previously cached for a machine's UID (as would
+// happen after Exists() populates the cache), updating its machine and Context references
+// rather than constructing a brand new scope (which would incur vCenter API calls).
+func TestGetOrCreateScopeReusesCachedScope(t *testing.T) {
+	g := NewWithT(t)
+
+	machineUID := apimachinerytypes.UID("test-uid")
+	cachedScope := &machineScope{}
+
+	a := &Actuator{}
+	a.scopeCache.Store(machineUID, cachedScope)
+
+	newMachine := &machinev1.Machine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  machineUID,
+		},
+	}
+
+	ctx := t.Context()
+	scope, err := a.getOrCreateScope(ctx, newMachine)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(scope).To(BeIdenticalTo(cachedScope))
+	g.Expect(scope.machine).To(BeIdenticalTo(newMachine))
+	g.Expect(scope.Context).To(Equal(ctx))
+}
+
+// TestGetOrCreateScopeCreatesNewScopeWhenNotCached verifies that, absent a cache entry,
+// getOrCreateScope falls back to newMachineScope. A nil context is used to force a cheap,
+// deterministic failure from newMachineScope without needing a real vCenter/K8s client.
+func TestGetOrCreateScopeCreatesNewScopeWhenNotCached(t *testing.T) {
+	g := NewWithT(t)
+
+	a := &Actuator{}
+	machine := &machinev1.Machine{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  apimachinerytypes.UID("some-other-uid"),
+		},
+	}
+
+	_, err := a.getOrCreateScope(nil, machine) //nolint:staticcheck
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(ContainSubstring("machine scope require a context"))
+}
+
+// TestTaskIDCacheConcurrencySafety exercises the sync.Map-based TaskIDCache concurrently to
+// guard against regressions reintroducing an unsynchronized map (see Change 4's concurrency
+// requirements). Run with `go test -race` to catch data races.
+func TestTaskIDCacheConcurrencySafety(t *testing.T) {
+	a := &Actuator{TaskIDCache: &sync.Map{}}
+
+	var wg sync.WaitGroup
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			name := "machine"
+			a.TaskIDCache.Store(name, "task-id")
+			a.TaskIDCache.Load(name)
+			a.TaskIDCache.Delete(name)
+		}(i)
+	}
+	wg.Wait()
+}
+
+// TestScopeCacheConcurrencySafety exercises the sync.Map-based scopeCache concurrently to
+// guard against regressions reintroducing an unsynchronized map.
+func TestScopeCacheConcurrencySafety(t *testing.T) {
+	a := &Actuator{}
+
+	var wg sync.WaitGroup
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			uid := apimachinerytypes.UID("uid")
+			a.scopeCache.Store(uid, &machineScope{})
+			a.scopeCache.Load(uid)
+			a.scopeCache.Delete(uid)
+		}(i)
+	}
+	wg.Wait()
+}
diff --git a/pkg/controller/vsphere/actuator_concurrency_test.go b/pkg/controller/vsphere/actuator_concurrency_test.go
new file mode 100644
index 000000000..55898b3f4
--- /dev/null
+++ b/pkg/controller/vsphere/actuator_concurrency_test.go
@@ -0,0 +1,170 @@
+package vsphere
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	machinev1 "github.com/openshift/api/machine/v1beta1"
+	"github.com/vmware/govmomi/simulator"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apimachinerytypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/events"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	testutils "github.com/openshift/machine-api-operator/pkg/util/testing"
+)
+
+// TestActuatorConcurrentReconciliation drives Exists()+Update() for several distinct
+// machines concurrently, as Change 4's MaxConcurrentReconciles enables. It exists primarily
+// to be run with `-race`, guarding against regressions that reintroduce unsynchronized maps
+// on the Actuator (TaskIDCache, scopeCache) now that reconciliation is no longer guaranteed
+// to be serialized.
+func TestActuatorConcurrentReconciliation(t *testing.T) {
+	g := NewWithT(t)
+
+	const numMachines = 5
+
+	model, server := initSimulatorCustom(t, func(m *simulator.Model) {
+		m.Machine = numMachines
+	})
+	defer model.Remove()
+	defer server.Close()
+
+	simSession := getSimulatorSession(t, server)
+
+	host, port, err := net.SplitHostPort(server.URL.Host)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	namespace := "test"
+	credentialsSecretUsername := fmt.Sprintf("%s.username", host)
+	credentialsSecretPassword := fmt.Sprintf("%s.password", host)
+	password, _ := server.URL.User.Password()
+
+	credentialsSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			credentialsSecretUsername: []byte(server.URL.User.Username()),
+			credentialsSecretPassword: []byte(password),
+		},
+	}
+
+	testConfig := fmt.Sprintf(testConfigFmt, port, "test", namespace)
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      OpenshiftConfigManagedConfigMap,
+			Namespace: openshiftConfigNamespaceForTest,
+		},
+		Data: map[string]string{
+			OpenshiftConfigManagedCloudConfigKey: testConfig,
+		},
+	}
+
+	_, err = createTagAndCategory(simSession, tagToCategoryName("CLUSTERID"), "CLUSTERID")
+	g.Expect(err).ToNot(HaveOccurred())
+
+	vms := model.Map().All("VirtualMachine")
+	g.Expect(len(vms)).To(BeNumerically(">=", numMachines))
+
+	machines := make([]*machinev1.Machine, numMachines)
+	for i := 0; i < numMachines; i++ {
+		vm := vms[i].(*simulator.VirtualMachine)
+		instanceUUID := fmt.Sprintf("aaaaaaaa-bbbb-cccc-dddd-%012d", i)
+		vm.Config.InstanceUuid = instanceUUID
+
+		providerSpec, err := RawExtensionFromProviderSpec(&machinev1.VSphereMachineProviderSpec{
+			Workspace: &machinev1.Workspace{
+				Server: host,
+			},
+			CredentialsSecret: &corev1.LocalObjectReference{
+				Name: "test",
+			},
+			Template: vm.Name,
+			Network: machinev1.NetworkSpec{
+				Devices: []machinev1.NetworkDeviceSpec{
+					{NetworkName: "test"},
+				},
+			},
+		})
+		g.Expect(err).ToNot(HaveOccurred())
+
+		machines[i] = &machinev1.Machine{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("test-%d", i),
+				Namespace: namespace,
+				UID:       apimachinerytypes.UID(instanceUUID),
+				Labels: map[string]string{
+					machinev1.MachineClusterIDLabel: "CLUSTERID",
+				},
+			},
+			Spec: machinev1.MachineSpec{
+				ProviderSpec: machinev1.ProviderSpec{
+					Value: providerSpec,
+				},
+			},
+		}
+	}
+
+	objs := []runtimeclient.Object{credentialsSecret, configMap}
+	for _, machine := range machines {
+		objs = append(objs, machine)
+	}
+
+	client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(objs...).WithStatusSubresource(&machinev1.Machine{}).Build()
+
+	gate, err := testutils.NewDefaultMutableFeatureGate()
+	g.Expect(err).ToNot(HaveOccurred())
+
+	a := NewActuator(ActuatorParams{
+		Client:                   client,
+		APIReader:                client,
+		EventRecorder:            events.NewFakeRecorder(numMachines * 4),
+		FeatureGates:             gate,
+		OpenshiftConfigNamespace: openshiftConfigNamespaceForTest,
+	})
+
+	var wg sync.WaitGroup
+	errs := make([]error, numMachines)
+	for i := 0; i < numMachines; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			machine := machines[i]
+			ctx := context.Background()
+
+			exists, err := a.Exists(ctx, machine)
+			if err != nil {
+				errs[i] = fmt.Errorf("exists: %w", err)
+				return
+			}
+			if !exists {
+				errs[i] = fmt.Errorf("expected machine %d to exist", i)
+				return
+			}
+			if err := a.Update(ctx, machine); err != nil {
+				errs[i] = fmt.Errorf("update: %w", err)
+				return
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	for i, err := range errs {
+		g.Expect(err).ToNot(HaveOccurred(), "machine %d", i)
+	}
+
+	// scopeCache must not leak entries once every machine's Update() has completed.
+	for _, machine := range machines {
+		_, ok := a.scopeCache.Load(machine.GetUID())
+		g.Expect(ok).To(BeFalse(), "expected scope cache entry for %s to be cleared", machine.GetName())
+	}
+}
diff --git a/pkg/controller/vsphere/actuator_test.go b/pkg/controller/vsphere/actuator_test.go
index 38118eed5..2465298f4 100644
--- a/pkg/controller/vsphere/actuator_test.go
+++ b/pkg/controller/vsphere/actuator_test.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
@@ -364,7 +365,7 @@ func TestMachineEvents(t *testing.T) {
 				t.Errorf("Unexpected error setting up feature gates: %v", err)
 			}
 
-			taskIDCache := make(map[string]string)
+			taskIDCache := &sync.Map{}
 			params := ActuatorParams{
 				Client:                   k8sClient,
 				EventRecorder:            eventRecorder,
diff --git a/pkg/controller/vsphere/machine_scope.go b/pkg/controller/vsphere/machine_scope.go
index 141a19fa2..1b73431d4 100644
--- a/pkg/controller/vsphere/machine_scope.go
+++ b/pkg/controller/vsphere/machine_scope.go
@@ -10,6 +10,7 @@ import (
 	machinev1 "github.com/openshift/api/machine/v1beta1"
 	machinecontroller "github.com/openshift/machine-api-operator/pkg/controller/machine"
 	"github.com/openshift/machine-api-operator/pkg/controller/vsphere/session"
+	"github.com/vmware/govmomi/vim25/types"
 	apicorev1 "k8s.io/api/core/v1"
 	apimachineryerrors "k8s.io/apimachinery/pkg/api/errors"
 	apimachineryutilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -49,6 +50,10 @@ type machineScope struct {
 	providerStatus     *machinev1.VSphereMachineProviderStatus
 	machineToBePatched runtimeclient.Patch
 	featureGates       featuregate.MutableFeatureGate
+	// cachedVMRef caches the result of findVM() from a preceding exists() call within the
+	// same reconciliation (see Actuator.scopeCache), so that update() can avoid a redundant
+	// vCenter FindByUuid/finder call.
+	cachedVMRef *types.ManagedObjectReference
 }
 
 // newMachineScope creates a new machineScope from the supplied parameters.
diff --git a/pkg/controller/vsphere/reconciler.go b/pkg/controller/vsphere/reconciler.go
index 7075de326..2acc71dcc 100644
--- a/pkg/controller/vsphere/reconciler.go
+++ b/pkg/controller/vsphere/reconciler.go
@@ -287,7 +287,7 @@ func (r *Reconciler) update() error {
 		}
 	}
 
-	vmRef, err := findVM(r.machineScope)
+	vmRef, err := r.getVMRef()
 	if err != nil {
 		metrics.RegisterFailedInstanceUpdate(&metrics.MachineLabels{
 			Name:      r.machine.Name,
@@ -327,6 +327,16 @@ func (r *Reconciler) update() error {
 	return nil
 }
 
+// getVMRef returns the VM reference cached by a preceding exists() call on the same
+// machineScope (see Actuator.scopeCache), if present, avoiding a redundant vCenter lookup.
+// Otherwise it falls back to looking the VM up via findVM().
+func (r *Reconciler) getVMRef() (types.ManagedObjectReference, error) {
+	if r.machineScope.cachedVMRef != nil {
+		return *r.machineScope.cachedVMRef, nil
+	}
+	return findVM(r.machineScope)
+}
+
 // exists returns true if machine exists.
 func (r *Reconciler) exists() (bool, error) {
 	if err := validateMachine(*r.machine); err != nil {
@@ -342,6 +352,11 @@ func (r *Reconciler) exists() (bool, error) {
 		return false, nil
 	}
 
+	// Cache the VM reference on the scope so that, if this scope is reused for a subsequent
+	// update() call within the same reconciliation (see Actuator.scopeCache), it does not
+	// need to perform a redundant findVM() call.
+	r.machineScope.cachedVMRef = &vmRef
+
 	// Check if machine was powered on after clone.
 	// If it is powered off and in "Provisioning" phase, treat machine as non-existed yet and proceed with creation procedure.
 	powerState := types.VirtualMachinePowerState(ptr.Deref(r.machineScope.providerStatus.InstanceState, ""))
@@ -613,7 +628,7 @@ func (r *Reconciler) reconcileRegionAndZoneLabels(vm *virtualMachine) error {
 }
 
 func (r *Reconciler) reconcileProviderID(vm *virtualMachine) error {
-	providerID, err := convertUUIDToProviderID(vm.Obj.UUID(vm.Context))
+	providerID, err := convertUUIDToProviderID(vm.getUUID())
 	if err != nil {
 		return err
 	}
@@ -1444,7 +1459,7 @@ func setProviderStatus(taskRef string, condition metav1.Condition, scope *machin
 	klog.Infof("%s: Updating provider status", scope.machine.Name)
 
 	if vm != nil {
-		id := vm.Obj.UUID(scope.Context)
+		id := vm.getUUID()
 		scope.providerStatus.InstanceID = &id
 
 		// This can return an error if machine is being deleted
@@ -1484,6 +1499,21 @@ type virtualMachine struct {
 	context.Context
 	Ref types.ManagedObjectReference
 	Obj *object.VirtualMachine
+
+	// cachedUUID and cachedPowerState memoize getUUID() and getPowerState() for the
+	// lifetime of this *virtualMachine, since both are fetched more than once per full
+	// reconciliation (reconcileProviderID/reconcilePowerStateAnnontation and
+	// setProviderStatus) via separate vCenter property collector calls.
+	cachedUUID       string
+	cachedPowerState *types.VirtualMachinePowerState
+}
+
+// getUUID returns the VM's instance UUID, caching the result for the lifetime of vm.
+func (vm *virtualMachine) getUUID() string {
+	if vm.cachedUUID == "" {
+		vm.cachedUUID = vm.Obj.UUID(vm.Context)
+	}
+	return vm.cachedUUID
 }
 
 // getHostSystemAncestors looks up and returns vm's host system ancestors, such as "Cluster" and "Datacenter".
@@ -1574,21 +1604,29 @@ func (vm *virtualMachine) powerOffVM() (string, error) {
 }
 
 func (vm *virtualMachine) getPowerState() (types.VirtualMachinePowerState, error) {
+	if vm.cachedPowerState != nil {
+		return *vm.cachedPowerState, nil
+	}
+
 	powerState, err := vm.Obj.PowerState(vm.Context)
 	if err != nil {
 		return "", err
 	}
 
+	var result types.VirtualMachinePowerState
 	switch powerState {
 	case types.VirtualMachinePowerStatePoweredOn:
-		return types.VirtualMachinePowerStatePoweredOn, nil
+		result = types.VirtualMachinePowerStatePoweredOn
 	case types.VirtualMachinePowerStatePoweredOff:
-		return types.VirtualMachinePowerStatePoweredOff, nil
+		result = types.VirtualMachinePowerStatePoweredOff
 	case types.VirtualMachinePowerStateSuspended:
-		return types.VirtualMachinePowerStateSuspended, nil
+		result = types.VirtualMachinePowerStateSuspended
 	default:
 		return "", fmt.Errorf("unexpected power state %q for vm %v", powerState, vm)
 	}
+
+	vm.cachedPowerState = &result
+	return result, nil
 }
 
 // reconcileTags ensures that the required tags are present on the virtual machine, eg the Cluster ID
diff --git a/pkg/controller/vsphere/virtualmachine_cache_test.go b/pkg/controller/vsphere/virtualmachine_cache_test.go
new file mode 100644
index 000000000..53aa92032
--- /dev/null
+++ b/pkg/controller/vsphere/virtualmachine_cache_test.go
@@ -0,0 +1,48 @@
+package vsphere
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+// TestVirtualMachineGetUUIDUsesCache verifies that getUUID() returns the cached UUID without
+// touching vm.Obj. A nil vm.Obj is used deliberately: if the cache were bypassed, calling
+// vm.Obj.UUID() would panic, proving the cached value was returned instead.
+func TestVirtualMachineGetUUIDUsesCache(t *testing.T) {
+	g := NewWithT(t)
+
+	vm := &virtualMachine{cachedUUID: "cached-uuid-1234"}
+
+	g.Expect(vm.getUUID()).To(Equal("cached-uuid-1234"))
+}
+
+// TestVirtualMachineGetPowerStateUsesCache verifies that getPowerState() returns the cached
+// power state without touching vm.Obj. A nil vm.Obj is used deliberately: if the cache were
+// bypassed, calling vm.Obj.PowerState() would panic, proving the cached value was returned.
+func TestVirtualMachineGetPowerStateUsesCache(t *testing.T) {
+	g := NewWithT(t)
+
+	cached := types.VirtualMachinePowerStatePoweredOn
+	vm := &virtualMachine{cachedPowerState: &cached}
+
+	state, err := vm.getPowerState()
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn))
+}
+
+// TestReconcilerGetVMRefUsesCachedRef verifies that getVMRef() returns the VM reference
+// cached by a preceding exists() call without performing a vCenter lookup. A machineScope
+// with a nil session is used deliberately: if the cache were bypassed, findVM() would panic
+// on the nil session, proving the cached value was returned instead.
+func TestReconcilerGetVMRefUsesCachedRef(t *testing.T) {
+	g := NewWithT(t)
+
+	ref := types.ManagedObjectReference{Type: "VirtualMachine", Value: "vm-123"}
+	r := &Reconciler{machineScope: &machineScope{cachedVMRef: &ref}}
+
+	got, err := r.getVMRef()
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(got).To(Equal(ref))
+}

From 1dd39f7834c93cee5a8a1ad13090385da656c803 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:13:59 -0400
Subject: [PATCH 4/7] vsphere: wire through --max-concurrent-reconciles flag

Add a --max-concurrent-reconciles flag to the vSphere machine controller
binary, defaulting to 1, and pass it through to controller-runtime via
AddWithActuatorOpts. The operator (sync.go) sets it to 3 for vSphere,
conservative compared to Azure/GCP's 10 since vCenter is a shared
appliance with lower API throughput.

Combined with the short-circuit from commit 1, stable machines complete
reconciliation in microseconds, so 3 workers handles concurrent creation
bursts without multiplying vCenter API load.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 cmd/vsphere/main.go       | 11 ++++++-
 pkg/operator/sync.go      |  6 ++++
 pkg/operator/sync_test.go | 66 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go
index db68e5bb1..f1eca7b19 100644
--- a/cmd/vsphere/main.go
+++ b/cmd/vsphere/main.go
@@ -108,6 +108,13 @@ func main() {
 			"annotation is honored, so it should not be disabled.",
 	)
 
+	maxConcurrentReconciles := flag.Int(
+		"max-concurrent-reconciles",
+		1,
+		"The number of concurrent machine reconciles allowed. Increasing this allows new "+
+			"machines to be processed without waiting behind stable machines in the queue.",
+	)
+
 	majorVersion := version.Version.Major
 
 	if majorVersion == 0 {
@@ -216,7 +223,9 @@ func main() {
 		klog.Fatalf("unable to add ipamv1beta1 to scheme: %v", err)
 	}
 
-	if err := capimachine.AddWithActuator(mgr, machineActuator, defaultMutableGate); err != nil {
+	if err := capimachine.AddWithActuatorOpts(mgr, machineActuator, controller.Options{
+		MaxConcurrentReconciles: *maxConcurrentReconciles,
+	}, defaultMutableGate); err != nil {
 		klog.Fatal(err)
 	}
 
diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go
index 631468f8d..4030eccc5 100644
--- a/pkg/operator/sync.go
+++ b/pkg/operator/sync.go
@@ -752,6 +752,12 @@ func newContainers(config *OperatorConfig, features map[string]bool, tlsArgs []s
 	switch config.PlatformType {
 	case configv1.AzurePlatformType, configv1.GCPPlatformType:
 		machineControllerArgs = append(machineControllerArgs, "--max-concurrent-reconciles=10")
+	case configv1.VSpherePlatformType:
+		// vCenter is a shared appliance with lower API throughput than public cloud APIs, so
+		// a smaller worker count is used than Azure/GCP. Combined with the vSphere
+		// controller's stable-machine short-circuit, only new/scaling/changed machines hit
+		// vCenter, so 3 workers comfortably handles typical concurrent creation bursts.
+		machineControllerArgs = append(machineControllerArgs, "--max-concurrent-reconciles=3")
 	case configv1.BareMetalPlatformType:
 		machineControllerArgs = append(machineControllerArgs, tlsArgs...)
 	}
diff --git a/pkg/operator/sync_test.go b/pkg/operator/sync_test.go
index 3066eeb22..6ba205415 100644
--- a/pkg/operator/sync_test.go
+++ b/pkg/operator/sync_test.go
@@ -968,6 +968,72 @@ func TestNewPodTemplateSpecTLSArgs(t *testing.T) {
 	}
 }
 
+func TestMachineControllerConcurrencyArgs(t *testing.T) {
+	testCases := []struct {
+		name         string
+		platformType configv1.PlatformType
+		expectedArg  string
+	}{
+		{
+			name:         "Azure gets concurrency 10",
+			platformType: configv1.AzurePlatformType,
+			expectedArg:  "--max-concurrent-reconciles=10",
+		},
+		{
+			name:         "GCP gets concurrency 10",
+			platformType: configv1.GCPPlatformType,
+			expectedArg:  "--max-concurrent-reconciles=10",
+		},
+		{
+			name:         "VSphere gets concurrency 3",
+			platformType: configv1.VSpherePlatformType,
+			expectedArg:  "--max-concurrent-reconciles=3",
+		},
+		{
+			name:         "BareMetal gets no concurrency flag",
+			platformType: configv1.BareMetalPlatformType,
+		},
+		{
+			name:         "unrecognized platform gets no concurrency flag",
+			platformType: configv1.NonePlatformType,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			config := &OperatorConfig{
+				TargetNamespace: targetNamespace,
+				PlatformType:    tc.platformType,
+				Controllers: Controllers{
+					Provider:      "provider-image:latest",
+					MachineSet:    "machineset-image:latest",
+					NodeLink:      "nodelink-image:latest",
+					KubeRBACProxy: "kube-rbac-proxy-image:latest",
+				},
+			}
+
+			podTemplate := newPodTemplateSpec(config, map[string]bool{})
+
+			var machineControllerArgs []string
+			for _, container := range podTemplate.Spec.Containers {
+				if container.Name == "machine-controller" {
+					machineControllerArgs = container.Args
+				}
+			}
+			g.Expect(machineControllerArgs).ToNot(BeNil())
+
+			joined := strings.Join(machineControllerArgs, " ")
+			if tc.expectedArg == "" {
+				g.Expect(joined).ToNot(ContainSubstring("--max-concurrent-reconciles"))
+			} else {
+				g.Expect(joined).To(ContainSubstring(tc.expectedArg))
+			}
+		})
+	}
+}
+
 func TestResolveTLSProfile(t *testing.T) {
 	g := NewWithT(t)
 

From dccbccb096ec33113da56dfa5571c3e862d6dd0f Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:15:17 -0400
Subject: [PATCH 5/7] TEMPORARY: default --max-concurrent-reconciles to 3

Change the default value of --max-concurrent-reconciles from 1 to 3 so
the concurrency improvement takes effect without requiring the operator
to pass the flag. This commit should be dropped before merge.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 cmd/vsphere/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go
index f1eca7b19..55e5728dc 100644
--- a/cmd/vsphere/main.go
+++ b/cmd/vsphere/main.go
@@ -110,7 +110,7 @@ func main() {
 
 	maxConcurrentReconciles := flag.Int(
 		"max-concurrent-reconciles",
-		1,
+		3,
 		"The number of concurrent machine reconciles allowed. Increasing this allows new "+
 			"machines to be processed without waiting behind stable machines in the queue.",
 	)

From 66146f403fefb17acff0328cfbf8ee34193790b0 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Tue, 7 Jul 2026 18:16:46 -0400
Subject: [PATCH 6/7] vsphere: address review nitpicks

Rename the generic 'timeout' constant to 'defaultSyncPeriod' to clarify
its purpose as the informer resync interval default.

Change cachedUUID from string to *string so that a legitimately empty
UUID is cached and not re-fetched, matching the cachedPowerState pattern.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 cmd/vsphere/main.go                                 | 4 ++--
 pkg/controller/vsphere/reconciler.go                | 9 +++++----
 pkg/controller/vsphere/virtualmachine_cache_test.go | 3 ++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go
index 55e5728dc..e569236ca 100644
--- a/cmd/vsphere/main.go
+++ b/cmd/vsphere/main.go
@@ -35,7 +35,7 @@ import (
 	"github.com/openshift/machine-api-operator/pkg/version"
 )
 
-const timeout = 60 * time.Minute
+const defaultSyncPeriod = 60 * time.Minute
 
 func main() {
 	var printVersion bool
@@ -102,7 +102,7 @@ func main() {
 
 	syncPeriodFlag := flag.Duration(
 		"sync-period",
-		timeout,
+		defaultSyncPeriod,
 		"Minimum interval at which cached resources are re-reconciled. This is the only "+
 			"backstop for drift detection once a machine's per-machine reconciliation TTL "+
 			"annotation is honored, so it should not be disabled.",
diff --git a/pkg/controller/vsphere/reconciler.go b/pkg/controller/vsphere/reconciler.go
index 2acc71dcc..b34388256 100644
--- a/pkg/controller/vsphere/reconciler.go
+++ b/pkg/controller/vsphere/reconciler.go
@@ -1504,16 +1504,17 @@ type virtualMachine struct {
 	// lifetime of this *virtualMachine, since both are fetched more than once per full
 	// reconciliation (reconcileProviderID/reconcilePowerStateAnnontation and
 	// setProviderStatus) via separate vCenter property collector calls.
-	cachedUUID       string
+	cachedUUID       *string
 	cachedPowerState *types.VirtualMachinePowerState
 }
 
 // getUUID returns the VM's instance UUID, caching the result for the lifetime of vm.
 func (vm *virtualMachine) getUUID() string {
-	if vm.cachedUUID == "" {
-		vm.cachedUUID = vm.Obj.UUID(vm.Context)
+	if vm.cachedUUID == nil {
+		uuid := vm.Obj.UUID(vm.Context)
+		vm.cachedUUID = &uuid
 	}
-	return vm.cachedUUID
+	return *vm.cachedUUID
 }
 
 // getHostSystemAncestors looks up and returns vm's host system ancestors, such as "Cluster" and "Datacenter".
diff --git a/pkg/controller/vsphere/virtualmachine_cache_test.go b/pkg/controller/vsphere/virtualmachine_cache_test.go
index 53aa92032..06129fa8c 100644
--- a/pkg/controller/vsphere/virtualmachine_cache_test.go
+++ b/pkg/controller/vsphere/virtualmachine_cache_test.go
@@ -13,7 +13,8 @@ import (
 func TestVirtualMachineGetUUIDUsesCache(t *testing.T) {
 	g := NewWithT(t)
 
-	vm := &virtualMachine{cachedUUID: "cached-uuid-1234"}
+	cachedUUID := "cached-uuid-1234"
+	vm := &virtualMachine{cachedUUID: &cachedUUID}
 
 	g.Expect(vm.getUUID()).To(Equal("cached-uuid-1234"))
 }

From 666b13f407c170340a40b9a119dccc5a478dc516 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Fri, 10 Jul 2026 13:53:30 -0400
Subject: [PATCH 7/7] vsphere: return RequeueAfterError for in-progress vCenter
 tasks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When a vCenter task (clone, power-on, delete) is still running,
reconciler.go returned a plain error, triggering controller-runtime's
5ms exponential backoff. This produced rapid-fire GetTask() API calls
and spurious FailedCreate/Update/Delete Warning events for normal
task-in-progress polling.

Return RequeueAfterError (20s fixed interval) instead, and short-circuit
before handleMachineError in the Actuator to suppress Warning events and
Error logs for expected intermediate state — mirroring the existing
TaskIDCache pattern.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 pkg/controller/vsphere/actuator.go        |  16 +++
 pkg/controller/vsphere/actuator_test.go   |  27 +++++
 pkg/controller/vsphere/reconciler.go      |  13 ++-
 pkg/controller/vsphere/reconciler_test.go | 135 ++++++++++++++++++++++
 4 files changed, 188 insertions(+), 3 deletions(-)

diff --git a/pkg/controller/vsphere/actuator.go b/pkg/controller/vsphere/actuator.go
index 11e677170..07ad7ca97 100644
--- a/pkg/controller/vsphere/actuator.go
+++ b/pkg/controller/vsphere/actuator.go
@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -226,6 +227,13 @@ func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error
 	}
 	if err != nil {
 		fmtErr := fmt.Errorf(reconcilerFailFmt, machine.GetName(), createEventAction, err)
+		var requeueErr *machinecontroller.RequeueAfterError
+		if errors.As(fmtErr, &requeueErr) {
+			if err := scope.PatchMachine(); err != nil {
+				return err
+			}
+			return fmtErr
+		}
 		retErr = a.handleMachineError(machine, fmtErr, createEventAction)
 	} else {
 		a.eventRecorder.Eventf(machine, nil, corev1.EventTypeNormal, createEventAction, createEventAction, "Created Machine %v", machine.GetName())
@@ -288,6 +296,10 @@ func (a *Actuator) Update(ctx context.Context, machine *machinev1.Machine) error
 			return err
 		}
 		fmtErr := fmt.Errorf(reconcilerFailFmt, machine.GetName(), updateEventAction, err)
+		var requeueErr *machinecontroller.RequeueAfterError
+		if errors.As(fmtErr, &requeueErr) {
+			return fmtErr
+		}
 		return a.handleMachineError(machine, fmtErr, updateEventAction)
 	}
 
@@ -338,6 +350,10 @@ func (a *Actuator) Delete(ctx context.Context, machine *machinev1.Machine) error
 			return err
 		}
 		fmtErr := fmt.Errorf(reconcilerFailFmt, machine.GetName(), deleteEventAction, err)
+		var requeueErr *machinecontroller.RequeueAfterError
+		if errors.As(fmtErr, &requeueErr) {
+			return fmtErr
+		}
 		return a.handleMachineError(machine, fmtErr, deleteEventAction)
 	}
 	a.eventRecorder.Eventf(machine, nil, corev1.EventTypeNormal, deleteEventAction, deleteEventAction, "Deleted machine %v", machine.GetName())
diff --git a/pkg/controller/vsphere/actuator_test.go b/pkg/controller/vsphere/actuator_test.go
index 2465298f4..427b335c2 100644
--- a/pkg/controller/vsphere/actuator_test.go
+++ b/pkg/controller/vsphere/actuator_test.go
@@ -2,6 +2,7 @@ package vsphere
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	machinecontroller "github.com/openshift/machine-api-operator/pkg/controller/machine"
 	testutils "github.com/openshift/machine-api-operator/pkg/util/testing"
 
 	. "github.com/onsi/gomega"
@@ -423,3 +425,28 @@ func TestMachineEvents(t *testing.T) {
 		})
 	}
 }
+
+func TestRequeueAfterErrorShortCircuitsHandleMachineError(t *testing.T) {
+	t.Run("RequeueAfterError unwraps through reconcilerFailFmt", func(t *testing.T) {
+		g := NewWithT(t)
+
+		requeueErr := &machinecontroller.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
+		wrappedErr := fmt.Errorf(reconcilerFailFmt, "test-machine", createEventAction, requeueErr)
+
+		var unwrapped *machinecontroller.RequeueAfterError
+		g.Expect(errors.As(wrappedErr, &unwrapped)).To(BeTrue(),
+			"errors.As should unwrap RequeueAfterError through reconcilerFailFmt (%%w)")
+		g.Expect(unwrapped.RequeueAfter).To(Equal(requeueAfterSeconds * time.Second))
+	})
+
+	t.Run("plain error does not match RequeueAfterError", func(t *testing.T) {
+		g := NewWithT(t)
+
+		plainErr := fmt.Errorf("something went wrong")
+		wrappedErr := fmt.Errorf(reconcilerFailFmt, "test-machine", createEventAction, plainErr)
+
+		var unwrapped *machinecontroller.RequeueAfterError
+		g.Expect(errors.As(wrappedErr, &unwrapped)).To(BeFalse(),
+			"plain error should not match RequeueAfterError")
+	})
+}
diff --git a/pkg/controller/vsphere/reconciler.go b/pkg/controller/vsphere/reconciler.go
index b34388256..be079c257 100644
--- a/pkg/controller/vsphere/reconciler.go
+++ b/pkg/controller/vsphere/reconciler.go
@@ -12,6 +12,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/vmware/govmomi/task"
 
@@ -212,7 +213,9 @@ func (r *Reconciler) create() error {
 		if taskIsFinished {
 			klog.V(4).Infof("%v task %v has completed", moTask.Info.DescriptionId, moTask.Reference().Value)
 		} else {
-			return fmt.Errorf("%v task %v has not finished", moTask.Info.DescriptionId, moTask.Reference().Value)
+			klog.V(3).Infof("%s: %v task %v has not finished, requeueing",
+				r.machine.GetName(), moTask.Info.DescriptionId, moTask.Reference().Value)
+			return &machinecontroller.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
 		}
 	}
 
@@ -282,7 +285,9 @@ func (r *Reconciler) update() error {
 				})
 				return fmt.Errorf("%v task %v finished with error: %w", moTask.Info.DescriptionId, moTask.Reference().Value, err)
 			} else if !taskIsFinished {
-				return fmt.Errorf("%v task %v has not finished", moTask.Info.DescriptionId, moTask.Reference().Value)
+				klog.V(3).Infof("%s: %v task %v has not finished, requeueing",
+					r.machine.GetName(), moTask.Info.DescriptionId, moTask.Reference().Value)
+				return &machinecontroller.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
 			}
 		}
 	}
@@ -412,7 +417,9 @@ func (r *Reconciler) delete() error {
 					)
 				}
 			} else if !taskIsFinished {
-				return fmt.Errorf("%v task %v has not finished", moTask.Info.DescriptionId, moTask.Reference().Value)
+				klog.V(3).Infof("%s: %v task %v has not finished, requeueing",
+					r.machine.GetName(), moTask.Info.DescriptionId, moTask.Reference().Value)
+				return &machinecontroller.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
 			}
 		}
 	}
diff --git a/pkg/controller/vsphere/reconciler_test.go b/pkg/controller/vsphere/reconciler_test.go
index 3fc67d993..103aa6e47 100644
--- a/pkg/controller/vsphere/reconciler_test.go
+++ b/pkg/controller/vsphere/reconciler_test.go
@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 
@@ -3584,4 +3585,138 @@ func TestReconcilePowerStateAnnontation(t *testing.T) {
 	}
 }
 
+func TestTaskRunningReturnsRequeueAfterError(t *testing.T) {
+	// Add a delay to clone tasks so we can observe the "task running" state.
+	// The clone task runs asynchronously in the simulator, so the second create() call
+	// will find it still running and return RequeueAfterError.
+	simulator.TaskDelay.MethodDelay = map[string]int{
+		"CloneVM": 500,
+	}
+
+	model, session, server := initSimulator(t)
+	defer model.Remove()
+	defer server.Close()
+	host, port, err := net.SplitHostPort(server.URL.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	credentialsSecretUsername := fmt.Sprintf("%s.username", host)
+	credentialsSecretPassword := fmt.Sprintf("%s.password", host)
+	password, _ := server.URL.User.Password()
+	namespace := "test"
+	vmName := "testName"
+	vm := model.Map().Any("VirtualMachine").(*simulator.VirtualMachine)
+	vm.Name = vmName
+	vm.Config.Version = minimumHWVersionString
+
+	credentialsSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			credentialsSecretUsername: []byte(server.URL.User.Username()),
+			credentialsSecretPassword: []byte(password),
+		},
+	}
+
+	testConfig := fmt.Sprintf(testConfigFmt, port, "test", namespace)
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      OpenshiftConfigManagedConfigMap,
+			Namespace: openshiftConfigNamespaceForTest,
+		},
+		Data: map[string]string{
+			OpenshiftConfigManagedCloudConfigKey: testConfig,
+		},
+	}
+
+	userDataSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "vsphere-ignition",
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			userDataSecretKey: []byte("{}"),
+		},
+	}
+
+	providerSpec := machinev1.VSphereMachineProviderSpec{
+		Template: vmName,
+		Workspace: &machinev1.Workspace{
+			Server: host,
+		},
+		CredentialsSecret: &corev1.LocalObjectReference{
+			Name: "test",
+		},
+		DiskGiB: 10,
+		UserDataSecret: &corev1.LocalObjectReference{
+			Name: "vsphere-ignition",
+		},
+	}
+
+	t.Run("create returns RequeueAfterError when clone task is running", func(t *testing.T) {
+		g := NewWithT(t)
+
+		machineName := "test-create-requeue"
+		rawProviderSpec, err := RawExtensionFromProviderSpec(&providerSpec)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		machine := &machinev1.Machine{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      machineName,
+				Namespace: namespace,
+				Labels: map[string]string{
+					machinev1.MachineClusterIDLabel: "CLUSTERID",
+				},
+			},
+			Spec: machinev1.MachineSpec{
+				ProviderSpec: machinev1.ProviderSpec{
+					Value: rawProviderSpec,
+				},
+			},
+		}
+
+		client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(
+			credentialsSecret, configMap, userDataSecret, machine).Build()
+
+		gates, err := testutils.NewDefaultMutableFeatureGate()
+		g.Expect(err).NotTo(HaveOccurred())
+
+		machineScope, err := newMachineScope(machineScopeParams{
+			client:                   client,
+			Context:                  context.Background(),
+			machine:                  machine,
+			apiReader:                client,
+			openshiftConfigNameSpace: openshiftConfigNamespaceForTest,
+			featureGates:             gates,
+		})
+		g.Expect(err).NotTo(HaveOccurred())
+
+		reconciler := newReconciler(machineScope)
+
+		// First create() starts the clone and sets TaskRef
+		err = reconciler.create()
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(reconciler.providerStatus.TaskRef).NotTo(BeEmpty(), "TaskRef should be set after clone")
+
+		// Second create() should find the task still running and return RequeueAfterError
+		err = reconciler.create()
+		g.Expect(err).To(HaveOccurred())
+
+		var requeueErr *machinecontroller.RequeueAfterError
+		g.Expect(errors.As(err, &requeueErr)).To(BeTrue(), "expected RequeueAfterError, got: %v", err)
+		g.Expect(requeueErr.RequeueAfter).To(Equal(time.Duration(requeueAfterSeconds) * time.Second))
+
+		// Wait for the clone task to complete before exiting, so the simulator
+		// goroutine finishes and doesn't race with clearing TaskDelay.
+		g.Expect(waitForTaskToComplete(session, reconciler)).To(Succeed())
+	})
+
+	// Clear TaskDelay after the clone task has completed to avoid data races
+	// with simulator goroutines.
+	simulator.TaskDelay.MethodDelay = nil
+}
+
 // See https://github.com/vmware/govmomi/blob/master/simulator/example_extend_test.go#L33:6 for extending behaviour example