diff --git a/api/core/v1alpha2/events.go b/api/core/v1alpha2/events.go index 7262ad7390..6f5279ac7a 100644 --- a/api/core/v1alpha2/events.go +++ b/api/core/v1alpha2/events.go @@ -58,6 +58,15 @@ const ( // ReasonErrRestartAwaitingChanges is event reason indicating that the vm has pending changes requiring a restart. ReasonErrRestartAwaitingChanges = "RestartAwaitingChanges" + // ReasonVMCPUResizing is event reason that the vm cpu is resizing. + ReasonVMCPUResizing = "CPUResizing" + + // ReasonVMMemoryResizing is event reason that the vm memory is resizing. + ReasonVMMemoryResizing = "MemoryResizing" + + // ReasonVMCPUAndMemoryResizing is event reason that the vm cpu and memory are resizing. + ReasonVMCPUAndMemoryResizing = "CPUAndMemoryResizing" + // ReasonErrVMOPFailed is event reason that operation is failed ReasonErrVMOPFailed = "VirtualMachineOperationFailed" diff --git a/build/components/versions.yml b/build/components/versions.yml index 379d07f0ea..5abfc7a494 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.44 + 3p-kubevirt: feat/inplace-resize # v1.6.2-v12n.44 3p-containerized-data-importer: v1.60.3-v12n.19 distribution: 2.8.3 package: diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 542d241165..e6e6758c19 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,6 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src +fromCacheVersion: "015" secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 10ab494bbd..f20e2b37c5 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -231,6 +231,9 @@ const ( DefaultUSBDeviceGroup = "64535" // DefaultUSBDeviceUser is the default device user ID for USB devices. DefaultUSBDeviceUser = "64535" + + AnnVirtualMachineInstanceInPlaceResizeInProgress = "kubevirt.io/in-place-resize-in-progress" + AnnVirtualMachineInstanceDisableInPlaceResize = "kubevirt.io/disable-in-place-resize" ) // AddAnnotation adds an annotation to an object diff --git a/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go new file mode 100644 index 0000000000..821f27dd68 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go @@ -0,0 +1,88 @@ +package inplaceresize + +import ( + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "k8s.io/component-base/featuregate" + virtv1 "kubevirt.io/api/core/v1" + "context" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "fmt" + "github.com/deckhouse/virtualization-controller/pkg/common/kvvm" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "errors" +) + +func New(featureGates featuregate.FeatureGate, client client.Client) *Service { + return &Service{ + featureGates: featureGates, + client: client, + } +} + +type Service struct { + featureGates featuregate.FeatureGate + client client.Client +} + +func (s *Service) InProgress(kvvmi *virtv1.VirtualMachineInstance) bool { + if s.featureGates.Enabled(featuregates.HotplugCPUInPlaceResize) || s.featureGates.Enabled(featuregates.HotplugMemoryInPlaceResize) { + return kvvmi.GetAnnotations()[annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress] == "true" + } + + return false +} + +func (s *Service) IsCompleted(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + return cond.Reason == "PodResizeCompleted" +} + +var ConditionNotFoundErr = errors.New("condition not found") + +func (s *Service) IsPossible(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + cond, exists := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + if !exists { + return false, fmt.Errorf("failed to get PodResourceResizeInProgress condition: %w", ConditionNotFoundErr) + } + + switch cond.Reason { + case "PodResizeCompleted": + return false, nil + case "PodResizePending", "PodResizeInProgress": + default: + return false, fmt.Errorf("unexpected PodResourceResizeInProgress condition reason: %s", cond.Reason) + } + + pod, err := kvvm.FindPodByKVVMI(ctx, s.client, kvvmi) + if err != nil { + return false, err + } + + podResizePending, _ := conditions.GetPodCondition(corev1.PodResizePending, pod.Status.Conditions) + if podResizePending.Reason == corev1.PodReasonDeferred || podResizePending.Reason == corev1.PodReasonInfeasible { + return false, nil + } + podResizeInProgress, _ := conditions.GetPodCondition(corev1.PodResizeInProgress, pod.Status.Conditions) + if podResizeInProgress.Reason == corev1.PodReasonError { + return false, nil + } + + return true, nil +} + +func (s *Service) ResizeCondition(kvvmi *virtv1.VirtualMachineInstance) virtv1.VirtualMachineInstanceCondition { + cond, _ := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + return cond +} + +func (s *Service) CPUChange(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceVCPUChange, kvvmi.Status.Conditions) + return cond.Status == corev1.ConditionTrue +} + +func (s *Service) MemoryChange(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) + return cond.Status == corev1.ConditionTrue +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index a4401040f4..f3b5c3241f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -44,6 +44,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" "github.com/deckhouse/virtualization-controller/pkg/controller/vmchange" "github.com/deckhouse/virtualization-controller/pkg/dvcr" @@ -74,6 +75,7 @@ func NewSyncKvvmHandler( recorder: recorder, featureGate: featureGate, syncVolumesService: syncVolumesService, + inplaceResize: inplaceresize.New(featureGate, client), } } @@ -83,6 +85,7 @@ type SyncKvvmHandler struct { dvcrSettings *dvcr.Settings featureGate featuregate.FeatureGate syncVolumesService syncVolumesService + inplaceResize *inplaceresize.Service } func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { @@ -228,6 +231,13 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat changed.Status.RestartAwaitingChanges = nil } + kvvmi, err := s.KVVMI(ctx) + if err != nil { + return reconcile.Result{}, err + } + + inplaceResizeInProgress := kvvmi != nil && h.inplaceResize.InProgress(kvvmi) + // 4. Set ConfigurationApplied condition. switch { case waitForNetwork: @@ -266,6 +276,10 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat Status(metav1.ConditionTrue). Reason(vmcondition.ReasonChangesPendingRestart). Message("VirtualMachineClass.spec has been modified. Waiting for the user to restart in order to apply the configuration changes.") + case inplaceResizeInProgress: + msg := h.buildInProgressInplaceResizeMsg(kvvmi) + h.recorder.Event(current, corev1.EventTypeNormal, h.resizingEventReason(kvvmi), msg) + cbConfApplied.Status(metav1.ConditionFalse).Reason(vmcondition.ReasonConfigurationNotApplied).Message(msg) case synced: h.recorder.Event(current, corev1.EventTypeNormal, v1alpha2.ReasonErrVmSynced, "The virtual machine configuration successfully synced") cbConfApplied.Status(metav1.ConditionTrue).Reason(vmcondition.ReasonConfigurationApplied) @@ -1132,3 +1146,48 @@ func (h *SyncKvvmHandler) isPlacementPolicyChanged(allChanges vmchange.SpecChang return false } + +func (h *SyncKvvmHandler) buildInProgressInplaceResizeMsg(kvvmi *virtv1.VirtualMachineInstance) string { + msg := strings.Builder{} + + switch { + case h.inplaceResize.CPUChange(kvvmi): + msg.WriteString("CPU hotplug is in progress.") + case h.inplaceResize.MemoryChange(kvvmi): + msg.WriteString("Memory hotplug is in progress.") + default: + msg.WriteString("Hotplug is in progress.") + } + + cond := h.inplaceResize.ResizeCondition(kvvmi) + switch cond.Reason { + case "PodResizePending", "PodResizeInProgress": + msg.WriteString(" ") + msg.WriteString(strings.ReplaceAll(cond.Message, "Pod", "VM")) + case "PodResizeCompleted": + msg.WriteString(" Waiting when cpu and memory will be hotplugged on virtual machine.") + default: + msg.WriteString(" reason: ") + msg.WriteString(cond.Reason) + msg.WriteString(", message: ") + msg.WriteString(cond.Message) + } + + return msg.String() +} + +func (h *SyncKvvmHandler) resizingEventReason(kvvmi *virtv1.VirtualMachineInstance) string { + cpuChange := h.inplaceResize.CPUChange(kvvmi) + memoryChange := h.inplaceResize.MemoryChange(kvvmi) + + switch { + case cpuChange && memoryChange: + return v1alpha2.ReasonVMCPUAndMemoryResizing + case cpuChange: + return v1alpha2.ReasonVMCPUResizing + case memoryChange: + return v1alpha2.ReasonVMMemoryResizing + default: + return "" + } +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 7c2b292069..6455fbdbe7 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" @@ -174,6 +175,29 @@ var _ = Describe("SyncKvvmHandler", func() { return kvvmi } + makeResizingKVVMI := func(reason, message string, conditionType virtv1.VirtualMachineInstanceConditionType) *virtv1.VirtualMachineInstance { + kvvmi := makeKVVMI() + kvvmi.Annotations = map[string]string{ + annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress: "true", + } + kvvmi.Status.Conditions = []virtv1.VirtualMachineInstanceCondition{ + { + Type: "PodResourceResizeInProgress", + Status: corev1.ConditionTrue, + Reason: reason, + Message: message, + }, + } + if conditionType != "" { + kvvmi.Status.Conditions = append(kvvmi.Status.Conditions, virtv1.VirtualMachineInstanceCondition{ + Type: conditionType, + Status: corev1.ConditionTrue, + }) + } + + return kvvmi + } + makeVMIP := func() *v1alpha2.VirtualMachineIPAddress { return &v1alpha2.VirtualMachineIPAddress{ ObjectMeta: metav1.ObjectMeta{ @@ -471,6 +495,54 @@ var _ = Describe("SyncKvvmHandler", func() { Entry("Pending phase with changes not applied, condition should not exist", v1alpha2.MachinePending, true, metav1.ConditionUnknown, false), ) + DescribeTable("ConfigurationApplied Condition for in-place resize", + func(featureGate featuregate.FeatureGate, kvvmi *virtv1.VirtualMachineInstance, expectedMessage string) { + ip := makeVMIP() + vmClass := makeVMClass() + vm := makeVM(v1alpha2.MachineRunning) + kvvm := makeKVVM(vm) + + fakeClient, reconcileObj, vmState = setupEnvironment(vm, kvvm, kvvmi, ip, vmClass) + featureGates = featureGate + + reconcile() + + newVM := &v1alpha2.VirtualMachine{} + err := fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), newVM) + Expect(err).NotTo(HaveOccurred()) + + confAppliedCond, confAppliedExists := conditions.GetCondition(vmcondition.TypeConfigurationApplied, newVM.Status.Conditions) + Expect(confAppliedExists).To(BeTrue()) + Expect(confAppliedCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(confAppliedCond.Reason).To(Equal(vmcondition.ReasonConfigurationNotApplied.String())) + Expect(confAppliedCond.Message).To(Equal(expectedMessage)) + }, + Entry( + "cpu hotplug pending", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI(string(corev1.PodResizePending), "Waiting for kubelet", virtv1.VirtualMachineInstanceVCPUChange), + "CPU hotplug is in progress. Waiting for kubelet", + ), + Entry( + "memory hotplug in progress", + newFeatureGateEnableMemoryInPlaceResize(), + makeResizingKVVMI(string(corev1.PodResizeInProgress), "Resizing pod resources", virtv1.VirtualMachineInstanceMemoryChange), + "Memory hotplug is in progress. Resizing pod resources", + ), + Entry( + "resize completed", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI("PodResizeCompleted", "Completed", virtv1.VirtualMachineInstanceVCPUChange), + "CPU hotplug is in progress. Waiting when cpu and memory will be hotplugged on virtual machine.", + ), + Entry( + "unexpected resize reason", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI("UnexpectedReason", "unexpected", ""), + "Hotplug is in progress. reason: UnexpectedReason, message: unexpected", + ), + ) + It("keeps ConfigurationApplied False and requeues while SDN is not ready", func() { ip := &v1alpha2.VirtualMachineIPAddress{ ObjectMeta: metav1.ObjectMeta{Name: "test-ip", Namespace: namespace}, @@ -550,6 +622,14 @@ func newFeatureGateEnableResourceHotplug() featuregate.FeatureGate { return newFeatureGate(featuregates.HotplugCPUWithLiveMigration, featuregates.HotplugMemoryWithLiveMigration) } +func newFeatureGateEnableCPUInPlaceResize() featuregate.FeatureGate { + return newFeatureGate(featuregates.HotplugCPUInPlaceResize) +} + +func newFeatureGateEnableMemoryInPlaceResize() featuregate.FeatureGate { + return newFeatureGate(featuregates.HotplugMemoryInPlaceResize) +} + func newResourceQuota(cpuHard, memoryHard, cpuUsed, memoryUsed resource.Quantity) *corev1.ResourceQuota { const quotaNamespace = "default" diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go index 89d81be3a0..a2f8c07cf1 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go @@ -55,7 +55,7 @@ func (c *comparatorCPU) Compare(current, desired *v1alpha2.VirtualMachineSpec) [ fractionChangedAction := ActionApplyImmediate // Require reboot if CPU hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugCPUInPlaceResize) { coresChangedAction = ActionRestart fractionChangedAction = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go index c442f57f3c..a0616a1d7b 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go @@ -59,7 +59,7 @@ func (c *comparatorMemory) Compare(current, desired *v1alpha2.VirtualMachineSpec } // Require reboot if memory hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugMemoryInPlaceResize) { actionType = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go index 5389f8340b..e3b2837be1 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go @@ -32,20 +32,25 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" + "errors" + "time" ) const hotplugHandler = "HotplugHandler" -func NewHotplugHandler(client client.Client, migration OneShotMigration) *HotplugHandler { +func NewHotplugHandler(client client.Client, migration OneShotMigration, inplaceResize *inplaceresize.Service) *HotplugHandler { return &HotplugHandler{ client: client, oneShotMigration: migration, + inplaceResize: inplaceResize, } } type HotplugHandler struct { client client.Client oneShotMigration OneShotMigration + inplaceResize *inplaceresize.Service } func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine) (reconcile.Result, error) { @@ -62,6 +67,24 @@ func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine return reconcile.Result{}, client.IgnoreNotFound(err) } + if h.inplaceResize.InProgress(kvvmi) { + completed := h.inplaceResize.IsCompleted(kvvmi) + possible, err := h.inplaceResize.IsPossible(ctx, kvvmi) + if err != nil { + if errors.Is(err, inplaceresize.ConditionNotFoundErr) { + logger.FromContext(ctx).Info("Waiting for inplace resize condition, requeue after 1 second") + return reconcile.Result{RequeueAfter: 1 * time.Second}, nil + } + return reconcile.Result{}, err + } + + if possible || completed { + return reconcile.Result{}, nil + } + // inplace resize is not possible, but it is not complete + // switch to resize via live migration + } + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) isMemoryHotplug := cond.Status == corev1.ConditionTrue diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go index e1d721c824..13a0ff13f2 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,6 +32,9 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) var _ = Describe("TestHotplugResourcesHandler", func() { @@ -49,7 +53,14 @@ var _ = Describe("TestHotplugResourcesHandler", func() { fakeClient = nil }) - newVMAndKVVMI := func(hasHotMemoryChange bool) (*v1alpha2.VirtualMachine, *virtv1.VirtualMachineInstance) { + type inPlaceResizeState struct { + inProgress bool + conditionReason string + podResizePendingReason string + podResizeInProgressReason string + } + + newVMAndKVVMI := func(hasHotMemoryChange bool, resizeState inPlaceResizeState) (*v1alpha2.VirtualMachine, *virtv1.VirtualMachineInstance) { vm := vmbuilder.NewEmpty(name, namespace) kvvmi := newEmptyKVVMI(name, namespace) @@ -59,9 +70,75 @@ var _ = Describe("TestHotplugResourcesHandler", func() { Status: corev1.ConditionTrue, }) } + + if resizeState.inProgress { + if kvvmi.Annotations == nil { + kvvmi.Annotations = make(map[string]string) + } + kvvmi.Annotations[annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress] = "true" + kvvmi.Status.Conditions = append(kvvmi.Status.Conditions, virtv1.VirtualMachineInstanceCondition{ + Type: "PodResourceResizeInProgress", + Status: corev1.ConditionTrue, + Reason: resizeState.conditionReason, + }) + } + return vm, kvvmi } + newLauncherPod := func(kvvmi *virtv1.VirtualMachineInstance, resizeState inPlaceResizeState) *corev1.Pod { + if !resizeState.inProgress || resizeState.conditionReason == "PodResizeCompleted" { + return nil + } + + const ( + nodeName = "test-node" + podName = "virt-launcher-test" + ) + + podUID := types.UID("virt-launcher-test-uid") + kvvmi.UID = types.UID("kvvmi-test-uid") + kvvmi.Status.NodeName = nodeName + kvvmi.Status.ActivePods = map[types.UID]string{ + podUID: nodeName, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: kvvmi.Namespace, + UID: podUID, + Labels: map[string]string{ + virtv1.AppLabel: "virt-launcher", + virtv1.CreatedByLabel: string(kvvmi.UID), + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + if resizeState.podResizePendingReason != "" { + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodResizePending, + Status: corev1.ConditionTrue, + Reason: resizeState.podResizePendingReason, + }) + } + if resizeState.podResizeInProgressReason != "" { + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodResizeInProgress, + Status: corev1.ConditionTrue, + Reason: resizeState.podResizeInProgressReason, + }) + } + + return pod + } + newOnceMigrationMock := func(shouldMigrate bool) *OneShotMigrationMock { return &OneShotMigrationMock{ OnceMigrateFunc: func(ctx context.Context, vm *v1alpha2.VirtualMachine, annotationKey, annotationExpectedValue string) (bool, error) { @@ -78,11 +155,13 @@ var _ = Describe("TestHotplugResourcesHandler", func() { awaitingRestart bool shouldMigrate bool expectedMigrationCalls int + expectedErr error + resizeState inPlaceResizeState } DescribeTable("HotplugResourcesHandler should return serviceCompleteErr if migration executed", func(settings testResourcesSettings) { - vm, kvvmi := newVMAndKVVMI(settings.hasHotMemoryChangeCondition) + vm, kvvmi := newVMAndKVVMI(settings.hasHotMemoryChangeCondition, settings.resizeState) if settings.awaitingRestart { vm.Status.Conditions = append(vm.Status.Conditions, metav1.Condition{ Type: vmcondition.TypeAwaitingRestartToApplyConfiguration.String(), @@ -90,20 +169,32 @@ var _ = Describe("TestHotplugResourcesHandler", func() { Reason: vmcondition.ReasonChangesPendingRestart.String(), }) } - fakeClient = setupEnvironment(vm, kvvmi) + pod := newLauncherPod(kvvmi, settings.resizeState) + if pod != nil { + fakeClient = setupEnvironment(vm, kvvmi, pod) + } else { + fakeClient = setupEnvironment(vm, kvvmi) + } mockMigration := newOnceMigrationMock(settings.shouldMigrate) - h := NewHotplugHandler(fakeClient, mockMigration) - _, err := h.Handle(ctx, vm) + gate, setFromMap, err := featuregates.NewUnlocked() + Expect(err).NotTo(HaveOccurred()) + + err = setFromMap(map[string]bool{ + string(featuregates.HotplugMemoryWithLiveMigration): true, + string(featuregates.HotplugCPUWithLiveMigration): true, + string(featuregates.HotplugMemoryInPlaceResize): true, + string(featuregates.HotplugCPUInPlaceResize): true}) + Expect(err).NotTo(HaveOccurred()) + + h := NewHotplugHandler(fakeClient, mockMigration, inplaceresize.New(gate, fakeClient)) + _, err = h.Handle(ctx, vm) Expect(mockMigration.OnceMigrateCalls()).To(HaveLen(settings.expectedMigrationCalls)) - if settings.hasHotMemoryChangeCondition && !settings.shouldMigrate { - Expect(err).ToNot(HaveOccurred()) - } else if settings.hasHotMemoryChangeCondition { - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(serviceCompleteErr)) + if settings.expectedErr != nil { + Expect(err).To(MatchError(settings.expectedErr)) } else { Expect(err).NotTo(HaveOccurred()) } @@ -114,10 +205,11 @@ var _ = Describe("TestHotplugResourcesHandler", func() { hasHotMemoryChangeCondition: true, shouldMigrate: true, expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, }, ), Entry( - "Migration should not be executed the second time", + "Migration should not return an error when one-shot migration reports no action", testResourcesSettings{ hasHotMemoryChangeCondition: true, shouldMigrate: false, @@ -139,5 +231,69 @@ var _ = Describe("TestHotplugResourcesHandler", func() { expectedMigrationCalls: 0, }, ), + Entry( + "Migration should not be executed when in-place resize is already completed", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + expectedMigrationCalls: 0, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeCompleted", + }, + }, + ), + Entry( + "Migration should not be executed when in-place resize is still possible", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + expectedMigrationCalls: 0, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizePending", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize is deferred", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizePending", + podResizePendingReason: "ResizeDeferred", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize is infeasible", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeInProgress", + podResizePendingReason: "ResizeInfeasible", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize ended with error", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeInProgress", + podResizeInProgressReason: "ResizeError", + }, + }, + ), ) }) diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go index 150aa322a1..2893ca6d91 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go @@ -29,6 +29,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/workload-updater/internal/service" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) const ( @@ -52,7 +53,7 @@ func SetupController( isMemoryHotplug := featuregates.Default().Enabled(featuregates.HotplugMemoryWithLiveMigration) isCPUHotplug := featuregates.Default().Enabled(featuregates.HotplugCPUWithLiveMigration) if isMemoryHotplug || isCPUHotplug { - hotplugHandler := handler.NewHotplugHandler(client, service.NewOneShotMigrationService(client, "hotplug-resources-")) + hotplugHandler := handler.NewHotplugHandler(client, service.NewOneShotMigrationService(client, "hotplug-resources-"), inplaceresize.New(featuregates.Default(), client)) handlers = append(handlers, hotplugHandler) } r := NewReconciler(client, handlers) diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 3358b0a85a..0e6c2c9337 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -32,6 +32,8 @@ const ( USB featuregate.Feature = "USB" HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" + HotplugCPUInPlaceResize featuregate.Feature = "HotplugCPUInPlaceResize" + HotplugMemoryInPlaceResize featuregate.Feature = "HotplugMemoryInPlaceResize" ) var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -69,6 +71,16 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: version.GetEdition() == version.EditionCE, PreRelease: featuregate.Alpha, }, + HotplugCPUInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, + HotplugMemoryInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, } var ( diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 5d3afcd436..3b5846cc71 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -223,10 +223,15 @@ properties: description: | Enable experimental or early access features. - - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number. (Not available in CE); - - `HotplugMemoryWithLiveMigration` — enable live changing of memory size. (Not available in CE); + - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); + - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); + - `HotplugCPUInPlaceResize` - enable live changing of cpu cores number via InPlaceResize. (Not available in CE); + - `HotplugMemoryInPlaceResize` - enable live changing of memory size via InPlaceResize. (Not available in CE); items: type: string enum: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" + - "HotplugCPUInPlaceResize" + - "HotplugMemoryInPlaceResize" + diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 8579d3e3ee..5542973c8b 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -153,7 +153,9 @@ properties: description: | Включение экспериментальных или недостаточно обкатанных возможностей. - - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки. (Не доступно в CE); - - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки. (Не доступно в CE); + - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugCPUInPlaceResize` - включить изменение количества ядер процессора без перезагрузки через InPlaceResize (Не доступно в CE) + - `HotplugMemoryInPlaceResize` - включить изменение размера памяти без перезагрузки через InPlaceResie (Не доступно в CE)) items: type: string diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 535aca4f75..ea6a778309 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -71,6 +71,7 @@ spec: - HostDevicesWithDRA - HostDevices - HotplugHostDevicesWithDRA # custom feature gate - added in our KubeVirt fork, not present in upstream + - InPlaceResize # custom feature gate - added in our KubeVirt fork, not present in upstream virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index c3c7b0119a..38213700da 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -514,6 +514,12 @@ rules: - pods/status verbs: - patch +- apiGroups: + - "" + resources: + - pods/resize + verbs: + - update - apiGroups: - "" resources: diff --git a/test/e2e/internal/precheck/featuregate.go b/test/e2e/internal/precheck/featuregate.go new file mode 100644 index 0000000000..0cdcd095ab --- /dev/null +++ b/test/e2e/internal/precheck/featuregate.go @@ -0,0 +1,99 @@ +package precheck + +import ( + "context" + "fmt" + "slices" + + . "github.com/onsi/ginkgo/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + dv1alpha1 "github.com/deckhouse/virtualization/test/e2e/internal/api/deckhouse/v1alpha1" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" +) + +type featureGatePrecheck struct { + label string + env string + gates []string +} + +func (p featureGatePrecheck) Label() string { + return p.label +} + +func (p featureGatePrecheck) Run(ctx context.Context, f *framework.Framework) error { + if !isCheckEnabled(p.env) { + _, _ = fmt.Fprintf(GinkgoWriter, "%s check is disabled", p.label) + return nil + } + + virtualizationModuleConfig := &dv1alpha1.ModuleConfig{} + err := f.GenericClient().Get(ctx, client.ObjectKey{Name: virtualizationModuleName}, virtualizationModuleConfig) + if err != nil { + return fmt.Errorf("%s=no to disable this precheck: failed to get virtualization module config spec: %w", p.env, err) + } + + gates, err := getFeatureGates(virtualizationModuleConfig) + if err != nil { + return fmt.Errorf("%s=no to disable this precheck: %w", p.env, err) + } + + for _, gate := range p.gates { + if !slices.Contains(gates, gate) { + return fmt.Errorf("%s=no to disable this precheck: feature gate %s is not enabled in virtualization module config spec", p.env, gate) + } + } + + return nil +} + +const ( + hotplugCPUWithLiveMigrationCheckEnvName = "HOTPLUG_CPU_LIVE_MIGRATION_PRECHECK" + hotplugMemoryWithLiveMigrationCheckEnvName = "HOTPLUG_MEMORY_LIVE_MIGRATION_PRECHECK" + hotplugCPUInPlaceCheckEnvName = "HOTPLUG_CPU_IN_PLACE_PRECHECK" + hotplugMemoryInPlaceCheckEnvName = "HOTPLUG_MEMORY_IN_PLACE_PRECHECK" +) + +func init() { + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugCPUWithLiveMigrationPrecheck, + env: hotplugCPUWithLiveMigrationCheckEnvName, + gates: []string{"HotplugCPUWithLiveMigration"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugMemoryWithLiveMigrationPrecheck, + env: hotplugMemoryWithLiveMigrationCheckEnvName, + gates: []string{"HotplugMemoryWithLiveMigration"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugCPUInPlaceResizePrecheck, + env: hotplugCPUInPlaceCheckEnvName, + gates: []string{"HotplugCPUInPlaceResize"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugMemoryInPlaceResizePrecheck, + env: hotplugMemoryInPlaceCheckEnvName, + gates: []string{"HotplugMemoryInPlaceResize"}, + }, false) +} + +func getFeatureGates(mc *dv1alpha1.ModuleConfig) ([]string, error) { + gates, ok := mc.Spec.Settings["featureGates"].([]interface{}) + if !ok { + return nil, fmt.Errorf("failed to get feature gates from virtualization module config spec: %T", gates) + } + var result []string + for _, g := range gates { + if s, ok := g.(string); ok { + result = append(result, s) + } else { + return nil, fmt.Errorf("failed to get feature gates from virtualization module config spec: %T", gates) + } + } + + return result, nil +} diff --git a/test/e2e/internal/precheck/labels.go b/test/e2e/internal/precheck/labels.go index 5d5bf6dbf4..a6bb08aa09 100644 --- a/test/e2e/internal/precheck/labels.go +++ b/test/e2e/internal/precheck/labels.go @@ -58,6 +58,18 @@ const ( // NoPrecheck - test doesn't require any prechecks. // Use this label for tests that don't depend on cluster configuration. NoPrecheck = "no-precheck" + + // HotplugCPUWithLiveMigrationPrecheck - test requires HotplugCPUWithLiveMigration feature gate to be enabled. + HotplugCPUWithLiveMigrationPrecheck = "hotplugcpuwithlivemigration-precheck" + + // HotplugCPUInPlaceResizePrecheck - test requires HotplugCPUInPlaceResize feature gate to be enabled. + HotplugCPUInPlaceResizePrecheck = "hotplugcpuinplaceresize-precheck" + + // HotplugMemoryWithLiveMigrationPrecheck - test requires HotplugMemoryWithLiveMigration feature gate to be enabled. + HotplugMemoryWithLiveMigrationPrecheck = "hotplugmemorywithlivemigration-precheck" + + // HotplugMemoryInPlaceResizePrecheck - test requires HotplugMemoryInPlaceResize feature gate to be enabled. + HotplugMemoryInPlaceResizePrecheck = "hotplugmemoryinplaceresize-precheck" ) // KnownPrecheckLabels returns all known precheck label constants. @@ -75,6 +87,10 @@ func KnownPrecheckLabels() []string { PrecheckPostCleanup, PrecheckPrecreatedCVI, NoPrecheck, + HotplugCPUWithLiveMigrationPrecheck, + HotplugCPUInPlaceResizePrecheck, + HotplugMemoryWithLiveMigrationPrecheck, + HotplugMemoryInPlaceResizePrecheck, } } diff --git a/test/e2e/internal/util/vm.go b/test/e2e/internal/util/vm.go index f8e8a3218b..20afe97bb3 100644 --- a/test/e2e/internal/util/vm.go +++ b/test/e2e/internal/util/vm.go @@ -211,6 +211,41 @@ func shellArgs(args []string) string { return strings.Join(quoted, " ") } +func GetVMNode(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine) (string, error) { + GinkgoHelper() + + err := f.GenericClient().Get(ctx, client.ObjectKeyFromObject(vm), vm) + if err != nil { + return "", err + } + if vm.Status.Node == "" { + return "", fmt.Errorf("vm %s/%s has empty status.node", vm.Namespace, vm.Name) + } + + return vm.Status.Node, nil +} + +func ExpectNoVMOperationsForVirtualMachine(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + vmops, err := f.VirtClient().VirtualMachineOperations(vm.Namespace).List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + + for _, vmop := range vmops.Items { + if vmop.Spec.VirtualMachine == vm.Name { + Fail(fmt.Sprintf("unexpected VMOP %q for VM %q", vmop.Name, vm.Name)) + } + } +} + +func ExpectVMOnNode(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, expectedNode string) { + GinkgoHelper() + + node, err := GetVMNode(ctx, f, vm) + Expect(err).NotTo(HaveOccurred()) + Expect(node).To(Equal(expectedNode)) +} + func UntilVMMigrationSucceeded(key client.ObjectKey, timeout time.Duration) { GinkgoHelper() diff --git a/test/e2e/vm/hotplug_cpu.go b/test/e2e/vm/hotplug_cpu.go index 426b484ce4..ea005d4f22 100644 --- a/test/e2e/vm/hotplug_cpu.go +++ b/test/e2e/vm/hotplug_cpu.go @@ -21,82 +21,77 @@ import ( "encoding/json" "fmt" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/precheck" "github.com/deckhouse/virtualization/test/e2e/internal/util" + "os" ) -var _ = Describe("HotplugCPU", Label(precheck.NoPrecheck), func() { +func decoratorsForCPUHotplugWithLiveMigration() []interface{} { + if os.Getenv("PARALLEL_CPU_HOTPLUG_MIGRATIONS") != "true" { + return nil + } + return []interface{}{Ordered, ContinueOnFailure} +} + +var _ = Describe("HotplugCPU", func() { var ( f *framework.Framework t *cpuHotplugTest ) BeforeEach(func() { - Skip("Hotplug CPU requires enabled feature gates in ModuleConfig. Skip until prechecks are implemented.") f = framework.NewFramework("cpu-hotplug") DeferCleanup(f.After) f.Before() t = newCPUHotplugTest(f) }) - DescribeTable("should apply cpu core changes without restart", func(initialCores, changedCores int) { - By("Environment preparation") - vmName := fmt.Sprintf("vm-%d-%d", initialCores, changedCores) - t.generateResources(vmName, initialCores) - err := f.CreateWithDeferredDeletion(context.Background(), t.VM, t.VD) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for VM agent to be ready") - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking initial CPU configuration") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(initialCores)) - - guestCPUCount, err := t.getGuestCPUCount() - Expect(err).NotTo(HaveOccurred()) - Expect(guestCPUCount).To(Equal(initialCores)) - - By("Applying CPU core changes") - patch, err := json.Marshal([]map[string]interface{}{{ - "op": "replace", - "path": "/spec/cpu/cores", - "value": changedCores, - }}) - Expect(err).NotTo(HaveOccurred()) - err = f.GenericClient().Patch(context.Background(), t.VM, crclient.RawPatch(types.JSONPatchType, patch)) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting until CPU configuration is applied without restart") - util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking changed CPU configuration") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(changedCores)) - - guestCPUCount, err = t.getGuestCPUCount() - Expect(err).NotTo(HaveOccurred()) - Expect(guestCPUCount).To(Equal(changedCores)) - }, - Entry("one socket topology, change cores from 1 to 3", 1, 3), - Entry("one socket topology, change cores to maximum 16", 4, 16), - ) + Describe("InPlaceResize", Label(precheck.HotplugCPUInPlaceResizePrecheck), func() { + DescribeTable("should apply cpu core changes in-place without restart", + func(initialCores, changedCores int) { + t.applyCPUCoreChangeInPlace(initialCores, changedCores) + }, + Entry("one socket topology, change cores from 1 to 2", 1, 2), + Entry("one socket topology, change cores from 1 to 4", 1, 4), + Entry("one socket topology, change cores from 4 to 3", 4, 3), + ) + }) + + Describe("LiveMigration", decoratorsForCPUHotplugWithLiveMigration(), Label(precheck.HotplugCPUWithLiveMigrationPrecheck), func() { + DescribeTable("should apply cpu core changes via live migration without restart", + func(initialCores, changedCores int) { + t.applyCPUCoreChangeWithLiveMigration(initialCores, changedCores) + }, + Entry("one socket topology, change cores from 1 to 2", 1, 2), + Entry("one socket topology, change cores from 1 to 4", 1, 4), + Entry("one socket topology, change cores from 4 to 3", 4, 3), + ) + }) + + Describe("QuotaBlockedMigration", + Label(precheck.HotplugCPUInPlaceResizePrecheck), + Label(precheck.HotplugCPUWithLiveMigrationPrecheck), func() { + It("should wait for quota removal and then migrate to apply cpu hotplug", func() { + t.applyCPUCoreChangeWithQuotaBlockedMigration(1, 4, resource.MustParse("2")) + }) + }) }) type cpuHotplugTest struct { @@ -110,13 +105,163 @@ func newCPUHotplugTest(f *framework.Framework) *cpuHotplugTest { return &cpuHotplugTest{Framework: f} } -func (t *cpuHotplugTest) generateResources(vmName string, cores int) { +func (t *cpuHotplugTest) applyCPUCoreChangeInPlace(initialCores, changedCores int) { + t.applyCPUCoreChange(initialCores, changedCores, false) +} + +func (t *cpuHotplugTest) applyCPUCoreChangeWithLiveMigration(initialCores, changedCores int) { + t.applyCPUCoreChange(initialCores, changedCores, true) +} + +func (t *cpuHotplugTest) applyCPUCoreChangeWithQuotaBlockedMigration(initialCores, changedCores int, cpuLimitQuota resource.Quantity) { + ctx := context.Background() + + By("Environment preparation") + vmName := fmt.Sprintf("vm-%d-%d-quota-migrate", initialCores, changedCores) + t.generateResources(vmName, initialCores, true) + + quota := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "project-quota", + Namespace: t.Framework.Namespace().Name, + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: cpuLimitQuota, + }, + }, + } + + err := t.Framework.CreateWithDeferredDeletion(ctx, quota, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + By("Checking initial CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(initialCores)) + + guestCPUCount, err := t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(initialCores)) + + By("Applying CPU core changes") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/cpu/cores", + "value": changedCores, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for workload updater to create migration VMOP") + vmop := untilHotplugMigrationVMOPCreated(ctx, t.Framework, t.VM, framework.MaxTimeout) + util.UntilObjectPhase(ctx, string(v1alpha2.VMOPPhasePending), framework.LongTimeout, vmop) + + By("Checking CPU configuration is not applied before migration can proceed") + guestCPUCount, err = t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(initialCores)) + + By("Removing resource quota") + err = t.Framework.GenericClient().Delete(ctx, quota) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting until CPU configuration is applied via live migration") + util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) + util.UntilObjectPhase(ctx, string(v1alpha2.VMOPPhaseCompleted), framework.LongTimeout, vmop) + + util.UntilSSHReady(t.Framework, t.VM, framework.MiddleTimeout) + + By("Checking changed CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(changedCores)) + + guestCPUCount, err = t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(changedCores)) +} + +func (t *cpuHotplugTest) applyCPUCoreChange(initialCores, changedCores int, liveMigration bool) { + ctx := context.Background() + + By("Environment preparation") + vmName := fmt.Sprintf("vm-%d-%d", initialCores, changedCores) + if liveMigration { + vmName += "-migrate" + } + t.generateResources(vmName, initialCores, liveMigration) + err := t.Framework.CreateWithDeferredDeletion(ctx, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + By("Checking initial CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(initialCores)) + + guestCPUCount, err := t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(initialCores)) + + initialNode, err := util.GetVMNode(ctx, t.Framework, t.VM) + Expect(err).NotTo(HaveOccurred()) + + By("Applying CPU core changes") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/cpu/cores", + "value": changedCores, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + if liveMigration { + By("Waiting until CPU configuration is applied via live migration") + util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) + } else { + By("Waiting until CPU configuration is applied in-place") + untilVMCPUCoresApplied(crclient.ObjectKeyFromObject(t.VM), changedCores, framework.MaxTimeout) + util.ExpectNoVMOperationsForVirtualMachine(ctx, t.Framework, t.VM) + util.ExpectVMOnNode(ctx, t.Framework, t.VM, initialNode) + } + + util.UntilSSHReady(t.Framework, t.VM, framework.MiddleTimeout) + + By("Checking changed CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(changedCores)) + + guestCPUCount, err = t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(changedCores)) +} + +func (t *cpuHotplugTest) generateResources(vmName string, cores int, disableInPlaceResize bool) { + t.generateResourcesWithRestartApproval(vmName, cores, disableInPlaceResize, v1alpha2.Automatic) +} + +func (t *cpuHotplugTest) generateResourcesWithRestartApproval(vmName string, cores int, disableInPlaceResize bool, restartApprovalMode v1alpha2.RestartApprovalMode) { vdName := fmt.Sprintf("vd-%s-root", vmName) t.VD = object.NewVDFromCVI(vdName, t.Framework.Namespace().Name, object.PrecreatedCVIAlpineBIOS, vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), ) - t.VM = vmbuilder.New( + opts := []vmbuilder.Option{ vmbuilder.WithName(vmName), vmbuilder.WithNamespace(t.Framework.Namespace().Name), vmbuilder.WithCPU(cores, ptr.To("10%")), @@ -129,8 +274,13 @@ func (t *cpuHotplugTest) generateResources(vmName string, cores int) { Name: t.VD.Name, }, ), - vmbuilder.WithRestartApprovalMode(v1alpha2.Automatic), - ) + vmbuilder.WithRestartApprovalMode(restartApprovalMode), + } + if disableInPlaceResize { + opts = append(opts, vmbuilder.WithAnnotation("kubevirt.internal.virtualization.deckhouse.io/disable-in-place-resize", "true")) + } + + t.VM = vmbuilder.New(opts...) } func (t *cpuHotplugTest) getGuestCPUCount() (int, error) { @@ -147,3 +297,44 @@ func (t *cpuHotplugTest) getGuestCPUCount() (int, error) { return cpuCount, nil } + +func untilVMCPUCoresApplied(key crclient.ObjectKey, expectedCores int, timeout time.Duration) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := framework.GetClients().VirtClient().VirtualMachines(key.Namespace).Get(context.Background(), key.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.Resources.CPU.Cores).To(Equal(expectedCores)) + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +} + +func untilHotplugMigrationVMOPCreated(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, timeout time.Duration) *v1alpha2.VirtualMachineOperation { + GinkgoHelper() + + var createdVMOP *v1alpha2.VirtualMachineOperation + + Eventually(func(g Gomega) { + vmops, err := f.VirtClient().VirtualMachineOperations(vm.Namespace).List(ctx, metav1.ListOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + for i := range vmops.Items { + vmop := &vmops.Items[i] + if vmop.Spec.VirtualMachine != vm.Name { + continue + } + if vmop.Spec.Type != v1alpha2.VMOPTypeEvict { + continue + } + if vmop.Annotations[annotations.AnnVMOPWorkloadUpdate] != "true" { + continue + } + + createdVMOP = vmop.DeepCopy() + return + } + + g.Expect(createdVMOP).NotTo(BeNil()) + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) + + return createdVMOP +} diff --git a/test/e2e/vm/hotplug_memory.go b/test/e2e/vm/hotplug_memory.go index 190b7637d1..86c03c98d4 100644 --- a/test/e2e/vm/hotplug_memory.go +++ b/test/e2e/vm/hotplug_memory.go @@ -23,84 +23,82 @@ import ( "regexp" "strconv" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/precheck" "github.com/deckhouse/virtualization/test/e2e/internal/util" + "os" ) -var _ = Describe("HotplugMemory", Label(precheck.NoPrecheck), func() { +func decoratorsForMemoryHotplugWithLiveMigration() []interface{} { + if os.Getenv("PARALLEL_MEMORY_HOTPLUG_MIGRATIONS") != "true" { + return nil + } + return []interface{}{Ordered, ContinueOnFailure} +} + +var _ = Describe("HotplugMemory", func() { var ( f *framework.Framework t *memoryHotplugTest ) BeforeEach(func() { - Skip("Hotplug memory requires enabling feature gate 'HotplugMemoryWithLiveMigration' in ModuleConfig. Skip until prechecks are implemented.") f = framework.NewFramework("memory-hotplug") DeferCleanup(f.After) f.Before() t = newMemoryHotplugTest(f) }) - DescribeTable("should apply memory changes with live migration", func(initialMemory, changedMemory string) { - initialQuantity := resource.MustParse(initialMemory) - changedQuantity := resource.MustParse(changedMemory) - - By("Environment preparation") - vmName := strings.ToLower(fmt.Sprintf("vm-%s-%s", initialMemory, changedMemory)) - t.generateResources(vmName, initialMemory) - err := f.CreateWithDeferredDeletion(context.Background(), t.VM, t.VD) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for VM agent to be ready") - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking initial memory size") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.Memory.Size).To(Equal(initialQuantity)) - - guestMemorySize, err := t.getGuestMemorySize() - Expect(err).NotTo(HaveOccurred()) - Expect(guestMemorySize).To(Equal(int(initialQuantity.Value()))) - - By("Applying memory size changes") - patch, err := json.Marshal([]map[string]interface{}{{ - "op": "replace", - "path": "/spec/memory/size", - "value": changedMemory, - }}) - Expect(err).NotTo(HaveOccurred()) - err = f.GenericClient().Patch(context.Background(), t.VM, crclient.RawPatch(types.JSONPatchType, patch)) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting until memory size is applied without restart") - util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking changed memory size") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.Memory.Size).To(Equal(changedQuantity)) - - guestMemorySize, err = t.getGuestMemorySize() - Expect(err).NotTo(HaveOccurred()) - Expect(guestMemorySize).To(Equal(int(changedQuantity.Value()))) - }, - Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), - ) + Describe("InPlaceResize", Label(precheck.HotplugMemoryInPlaceResizePrecheck), func() { + DescribeTable("should apply memory changes in-place without restart", + func(initialMemory, changedMemory string) { + t.applyMemoryChangeInPlace(initialMemory, changedMemory) + }, + Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), + Entry("change memory from 1Gi to 4Gi", "1Gi", "4Gi"), + ) + + DescribeTable("should require restart to decrease memory", + func(initialMemory, changedMemory string) { + t.requireRestartToDecreaseMemory(initialMemory, changedMemory, false) + }, + Entry("decrease memory from 2Gi to 1Gi", "2Gi", "1Gi"), + Entry("decrease memory from 4Gi to 1Gi", "4Gi", "1Gi"), + ) + }) + + Describe("LiveMigration", decoratorsForMemoryHotplugWithLiveMigration(), Label(precheck.HotplugMemoryWithLiveMigrationPrecheck), func() { + DescribeTable("should apply memory changes via live migration without restart", + func(initialMemory, changedMemory string) { + t.applyMemoryChangeWithLiveMigration(initialMemory, changedMemory) + }, + Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), + Entry("change memory from 1Gi to 4Gi", "1Gi", "4Gi"), + ) + + DescribeTable("should require restart to decrease memory", + func(initialMemory, changedMemory string) { + t.requireRestartToDecreaseMemory(initialMemory, changedMemory, true) + }, + Entry("decrease memory from 2Gi to 1Gi", "2Gi", "1Gi"), + Entry("decrease memory from 4Gi to 1Gi", "4Gi", "1Gi"), + ) + }) }) type memoryHotplugTest struct { @@ -114,7 +112,134 @@ func newMemoryHotplugTest(f *framework.Framework) *memoryHotplugTest { return &memoryHotplugTest{Framework: f} } -func (t *memoryHotplugTest) generateResources(vmName, memSize string) { +func (t *memoryHotplugTest) applyMemoryChangeInPlace(initialMemory, changedMemory string) { + t.applyMemoryChange(initialMemory, changedMemory, false) +} + +func (t *memoryHotplugTest) applyMemoryChangeWithLiveMigration(initialMemory, changedMemory string) { + t.applyMemoryChange(initialMemory, changedMemory, true) +} + +func (t *memoryHotplugTest) requireRestartToDecreaseMemory(initialMemory, changedMemory string, liveMigration bool) { + ctx := context.Background() + initialQuantity := resource.MustParse(initialMemory) + + By("Environment preparation") + vmName := strings.ToLower(fmt.Sprintf("vm-%s-%s-decrease", initialMemory, changedMemory)) + if liveMigration { + vmName += "-migrate" + } + t.generateResourcesWithRestartApproval(vmName, initialMemory, liveMigration, v1alpha2.Manual) + err := t.Framework.CreateWithDeferredDeletion(ctx, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + initialNode, err := util.GetVMNode(ctx, t.Framework, t.VM) + Expect(err).NotTo(HaveOccurred()) + + initialGuestMemorySize, err := t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(initialGuestMemorySize).To(Equal(int(initialQuantity.Value()))) + + By("Applying memory decrease") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/memory/size", + "value": changedMemory, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting until restart is required") + Expect(util.IsRestartRequired(t.VM, framework.MaxTimeout)).To(BeTrue()) + util.ExpectNoVMOperationsForVirtualMachine(ctx, t.Framework, t.VM) + util.ExpectVMOnNode(ctx, t.Framework, t.VM, initialNode) + + By("Checking memory size is not applied without restart") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.Memory.Size).To(Equal(initialQuantity)) + + guestMemorySize, err := t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(guestMemorySize).To(Equal(initialGuestMemorySize)) +} + +func (t *memoryHotplugTest) applyMemoryChange(initialMemory, changedMemory string, liveMigration bool) { + ctx := context.Background() + initialQuantity := resource.MustParse(initialMemory) + changedQuantity := resource.MustParse(changedMemory) + + By("Environment preparation") + vmName := strings.ToLower(fmt.Sprintf("vm-%s-%s", initialMemory, changedMemory)) + if liveMigration { + vmName += "-migrate" + } + t.generateResources(vmName, initialMemory, liveMigration) + err := t.Framework.CreateWithDeferredDeletion(ctx, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + By("Checking initial memory size") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.Memory.Size).To(Equal(initialQuantity)) + + guestMemorySize, err := t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(guestMemorySize).To(Equal(int(initialQuantity.Value()))) + + initialNode, err := util.GetVMNode(ctx, t.Framework, t.VM) + Expect(err).NotTo(HaveOccurred()) + + By("Applying memory size changes") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/memory/size", + "value": changedMemory, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + if liveMigration { + By("Waiting until memory size is applied via live migration") + util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) + } else { + By("Waiting until memory size is applied in-place") + untilVMMemorySizeApplied(crclient.ObjectKeyFromObject(t.VM), changedQuantity, framework.MaxTimeout) + util.ExpectNoVMOperationsForVirtualMachine(ctx, t.Framework, t.VM) + util.ExpectVMOnNode(ctx, t.Framework, t.VM, initialNode) + } + + util.UntilSSHReady(t.Framework, t.VM, framework.MiddleTimeout) + + By("Checking changed memory size") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.Memory.Size).To(Equal(changedQuantity)) + + guestMemorySize, err = t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(guestMemorySize).To(Equal(int(changedQuantity.Value()))) +} + +func (t *memoryHotplugTest) generateResources(vmName, memSize string, disableInPlaceResize bool) { + t.generateResourcesWithRestartApproval(vmName, memSize, disableInPlaceResize, v1alpha2.Automatic) +} + +func (t *memoryHotplugTest) generateResourcesWithRestartApproval(vmName, memSize string, disableInPlaceResize bool, restartApprovalMode v1alpha2.RestartApprovalMode) { memSizeQuantity := resource.MustParse(memSize) vdName := fmt.Sprintf("vd-%s-root", vmName) @@ -122,7 +247,7 @@ func (t *memoryHotplugTest) generateResources(vmName, memSize string) { vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), ) - t.VM = vmbuilder.New( + opts := []vmbuilder.Option{ vmbuilder.WithName(vmName), vmbuilder.WithNamespace(t.Framework.Namespace().Name), vmbuilder.WithCPU(2, ptr.To("10%")), @@ -135,8 +260,13 @@ func (t *memoryHotplugTest) generateResources(vmName, memSize string) { Name: t.VD.Name, }, ), - vmbuilder.WithRestartApprovalMode(v1alpha2.Automatic), - ) + vmbuilder.WithRestartApprovalMode(restartApprovalMode), + } + if disableInPlaceResize { + opts = append(opts, vmbuilder.WithAnnotation(annotations.AnnVirtualMachineInstanceDisableInPlaceResize, "true")) + } + + t.VM = vmbuilder.New(opts...) } var totalOnlineMemRe = regexp.MustCompile(`^Total online memory:\s+(\d+)$`) @@ -158,3 +288,13 @@ func (t *memoryHotplugTest) getGuestMemorySize() (int, error) { return 0, fmt.Errorf("failed to find total online memory in lsmem output: %v", cmdOut) } + +func untilVMMemorySizeApplied(key crclient.ObjectKey, expectedSize resource.Quantity, timeout time.Duration) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := framework.GetClients().VirtClient().VirtualMachines(key.Namespace).Get(context.Background(), key.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.Resources.Memory.Size).To(Equal(expectedSize)) + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +}