From 2bc1542eb7d15bf070a0f2e889df052882959178 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 20 Apr 2026 23:21:36 +0300 Subject: [PATCH 1/3] impl Signed-off-by: Yaroslav Borbat --- api/core/v1alpha2/vmcondition/condition.go | 13 ++ build/components/versions.yml | 2 +- images/virt-artifact/werf.inc.yaml | 1 + .../pkg/common/annotations/annotations.go | 3 + .../inplaceresize/inplaceresize_service.go | 75 +++++++ .../pkg/controller/vm/internal/resizing.go | 66 ++++++ .../controller/vm/internal/resizing_test.go | 195 ++++++++++++++++++ .../pkg/controller/vm/vm_controller.go | 2 + .../pkg/controller/vmchange/comparator_cpu.go | 2 +- .../controller/vmchange/comparator_memory.go | 2 +- .../internal/handler/hotplug.go | 19 +- .../internal/handler/hotplug_test.go | 178 +++++++++++++++- .../workload_updater_controller.go | 3 +- .../pkg/featuregates/featuregate.go | 12 ++ openapi/config-values.yaml | 9 +- openapi/doc-ru-config-values.yaml | 6 +- templates/kubevirt/kubevirt.yaml | 1 + .../kubevirt/virt-operator/rbac-for-us.yaml | 6 + test/e2e/internal/precheck/featuregate.go | 99 +++++++++ test/e2e/internal/precheck/labels.go | 16 ++ test/e2e/internal/util/vm.go | 35 ++++ test/e2e/vm/hotplug_cpu.go | 153 +++++++++----- test/e2e/vm/hotplug_memory.go | 156 +++++++++----- 23 files changed, 932 insertions(+), 122 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/resizing.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go create mode 100644 test/e2e/internal/precheck/featuregate.go diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 2320fd5da9..7db351c763 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -50,6 +50,8 @@ const ( // TypeMaintenance indicates that the VirtualMachine is in maintenance mode. // During this condition, the VM remains stopped and no changes are allowed. TypeMaintenance Type = "Maintenance" + + TypeResizing Type = "Resizing" ) type AgentReadyReason string @@ -283,3 +285,14 @@ func (r MaintenanceReason) String() string { const ( ReasonMaintenanceRestore MaintenanceReason = "RestoreInProgress" ) + +type ResizingReason string + +func (r ResizingReason) String() string { + return string(r) +} + +const ( + ReasonResizingPending ResizingReason = "Pending" + ReasonResizingInProgress ResizingReason = "InProgress" +) diff --git a/build/components/versions.yml b/build/components/versions.yml index 379d07f0ea..5abfc7a494 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.44 + 3p-kubevirt: feat/inplace-resize # v1.6.2-v12n.44 3p-containerized-data-importer: v1.60.3-v12n.19 distribution: 2.8.3 package: diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 542d241165..65165a97bb 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,6 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src +fromCacheVersion: "014" secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 10ab494bbd..f20e2b37c5 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -231,6 +231,9 @@ const ( DefaultUSBDeviceGroup = "64535" // DefaultUSBDeviceUser is the default device user ID for USB devices. DefaultUSBDeviceUser = "64535" + + AnnVirtualMachineInstanceInPlaceResizeInProgress = "kubevirt.io/in-place-resize-in-progress" + AnnVirtualMachineInstanceDisableInPlaceResize = "kubevirt.io/disable-in-place-resize" ) // AddAnnotation adds an annotation to an object diff --git a/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go new file mode 100644 index 0000000000..f4eb98d3d2 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go @@ -0,0 +1,75 @@ +package inplaceresize + +import ( + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "k8s.io/component-base/featuregate" + virtv1 "kubevirt.io/api/core/v1" + "context" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "fmt" + "github.com/deckhouse/virtualization-controller/pkg/common/kvvm" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func New(featureGates featuregate.FeatureGate, client client.Client) *Service { + return &Service{ + featureGates: featureGates, + client: client, + } +} + +type Service struct { + featureGates featuregate.FeatureGate + client client.Client +} + +func (s *Service) InProgress(kvvmi *virtv1.VirtualMachineInstance) bool { + if s.featureGates.Enabled(featuregates.HotplugCPUInPlaceResize) || s.featureGates.Enabled(featuregates.HotplugMemoryInPlaceResize) { + return kvvmi.GetAnnotations()[annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress] == "true" + } + + return false +} + +func (s *Service) IsCompleted(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + return cond.Reason == "PodResizeCompleted" +} + +func (s *Service) IsPossible(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + cond, exists := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + if !exists { + return false, fmt.Errorf("failed to get PodResourceResizeInProgress condition") + } + + switch cond.Reason { + case "PodResizeCompleted": + return false, nil + case "PodResizePending", "PodResizeInProgress": + default: + return false, fmt.Errorf("unexpected PodResourceResizeInProgress condition reason: %s", cond.Reason) + } + + pod, err := kvvm.FindPodByKVVMI(ctx, s.client, kvvmi) + if err != nil { + return false, err + } + + podResizePending, _ := conditions.GetPodCondition(corev1.PodResizePending, pod.Status.Conditions) + if podResizePending.Reason == corev1.PodReasonDeferred || podResizePending.Reason == corev1.PodReasonInfeasible { + return false, nil + } + podResizeInProgress, _ := conditions.GetPodCondition(corev1.PodResizeInProgress, pod.Status.Conditions) + if podResizeInProgress.Reason == corev1.PodReasonError { + return false, nil + } + + return true, nil +} + +func (s *Service) ResizeCondition(kvvmi *virtv1.VirtualMachineInstance) virtv1.VirtualMachineInstanceCondition { + cond, _ := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) + return cond +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go b/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go new file mode 100644 index 0000000000..5c89847f71 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go @@ -0,0 +1,66 @@ +package internal + +import ( + "context" + "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const resizingHandlerName = "ResizingHandler" + +type ResizingHandler struct { + inplaceResize *inplaceresize.Service +} + +func NewResizingHandler(svc *inplaceresize.Service) *ResizingHandler { + return &ResizingHandler{ + inplaceResize: svc, + } +} + +func (h *ResizingHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { + vm := s.VirtualMachine().Changed() + + if isDeletion(vm) { + return reconcile.Result{}, nil + } + + kvvmi, err := s.KVVMI(ctx) + if err != nil { + return reconcile.Result{}, err + } + + if kvvmi == nil || !h.inplaceResize.InProgress(kvvmi) { + conditions.RemoveCondition(vmcondition.TypeResizing, &vm.Status.Conditions) + return reconcile.Result{}, nil + } + + cb := conditions.NewConditionBuilder(vmcondition.TypeResizing). + Generation(vm.GetGeneration()). + Status(metav1.ConditionTrue) + + cond := h.inplaceResize.ResizeCondition(kvvmi) + switch cond.Reason { + case "PodResizePending": + cb.Reason(vmcondition.ReasonResizingPending).Message(cond.Message) + case "PodResizeInProgress": + cb.Reason(vmcondition.ReasonResizingInProgress).Message(cond.Message) + case "PodResizeCompleted": + cb.Reason(vmcondition.ReasonResizingInProgress).Message("Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine") + default: + conditions.RemoveCondition(vmcondition.TypeResizing, &vm.Status.Conditions) + return reconcile.Result{}, nil + } + + conditions.SetCondition(cb, &vm.Status.Conditions) + + return reconcile.Result{}, nil +} + +func (h *ResizingHandler) Name() string { + return resizingHandlerName +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go new file mode 100644 index 0000000000..b86abf2720 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/common/testutil" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization-controller/pkg/controller/reconciler" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" + "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" +) + +var _ = Describe("ResizingHandler", func() { + const ( + name = "vm-resizing" + namespace = "default" + nodeName = "test-node" + ) + + var ( + ctx = testutil.ContextBackgroundWithNoOpLogger() + fakeClient client.WithWatch + resource *reconciler.Resource[*v1alpha2.VirtualMachine, v1alpha2.VirtualMachineStatus] + vmState state.VirtualMachineState + ) + + AfterEach(func() { + fakeClient = nil + resource = nil + vmState = nil + }) + + newVM := func() *v1alpha2.VirtualMachine { + return vmbuilder.NewEmpty(name, namespace) + } + + newHandler := func() *ResizingHandler { + gate, setFeatureMap, err := featuregates.NewUnlocked() + Expect(err).NotTo(HaveOccurred()) + err = setFeatureMap(map[string]bool{ + string(featuregates.HotplugCPUInPlaceResize): true, + string(featuregates.HotplugMemoryInPlaceResize): true, + }) + Expect(err).NotTo(HaveOccurred()) + + return NewResizingHandler(inplaceresize.New(gate, fakeClient)) + } + + newResizingKVVMI := func(reason, message string) *virtv1.VirtualMachineInstance { + kvvmi := newEmptyKVVMI(name, namespace) + kvvmi.UID = types.UID("kvvmi-uid") + kvvmi.Annotations = map[string]string{ + annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress: "true", + } + kvvmi.Status.NodeName = nodeName + kvvmi.Status.Conditions = []virtv1.VirtualMachineInstanceCondition{ + { + Type: "PodResourceResizeInProgress", + Status: corev1.ConditionTrue, + Reason: reason, + Message: message, + }, + } + + if reason != "PodResizeCompleted" { + kvvmi.Status.ActivePods = map[types.UID]string{ + types.UID("virt-launcher-uid"): nodeName, + } + } + + return kvvmi + } + + reconcile := func() error { + h := newHandler() + _, err := h.Handle(ctx, vmState) + if err != nil { + return err + } + + return resource.Update(context.Background()) + } + + Describe("Condition presence and absence scenarios", func() { + It("Should remove condition when KVVMI is absent", func() { + vm := newVM() + vm.Status.Conditions = []metav1.Condition{{ + Type: vmcondition.TypeResizing.String(), + Status: metav1.ConditionTrue, + }} + + fakeClient, resource, vmState = setupEnvironment(vm) + err := reconcile() + Expect(err).NotTo(HaveOccurred()) + + actualVM := &v1alpha2.VirtualMachine{} + err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) + Expect(err).NotTo(HaveOccurred()) + + _, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) + Expect(exists).To(BeFalse()) + }) + + DescribeTable("Should set resizing condition according to resize state", + func(reason, message, expectedReason, expectedMessage string) { + vm := newVM() + kvvmi := newResizingKVVMI(reason, message) + + fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) + err := reconcile() + Expect(err).NotTo(HaveOccurred()) + + actualVM := &v1alpha2.VirtualMachine{} + err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) + Expect(err).NotTo(HaveOccurred()) + + cond, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) + Expect(exists).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(expectedReason)) + Expect(cond.Message).To(Equal(expectedMessage)) + }, + Entry("pending resize", string(corev1.PodResizePending), "Waiting for kubelet", vmcondition.ReasonResizingPending.String(), "Waiting for kubelet"), + Entry("in progress resize", string(corev1.PodResizeInProgress), "Resizing pod resources", vmcondition.ReasonResizingInProgress.String(), "Resizing pod resources"), + ) + + It("Should keep in-progress reason after pod resize completion", func() { + vm := newVM() + kvvmi := newResizingKVVMI("PodResizeCompleted", "Completed") + + fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) + err := reconcile() + Expect(err).NotTo(HaveOccurred()) + + actualVM := &v1alpha2.VirtualMachine{} + err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) + Expect(err).NotTo(HaveOccurred()) + + cond, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) + Expect(exists).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(vmcondition.ReasonResizingInProgress.String())) + Expect(cond.Message).To(Equal("Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine")) + }) + }) + + It("Should remove condition when resize reason is unexpected", func() { + vm := newVM() + vm.Status.Conditions = []metav1.Condition{{ + Type: vmcondition.TypeResizing.String(), + Status: metav1.ConditionTrue, + }} + kvvmi := newResizingKVVMI("UnexpectedReason", "unexpected") + + fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) + err := reconcile() + Expect(err).NotTo(HaveOccurred()) + + actualVM := &v1alpha2.VirtualMachine{} + err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) + Expect(err).NotTo(HaveOccurred()) + + _, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) + Expect(exists).To(BeFalse()) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index ebc466e99e..08ddcffc89 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -38,6 +38,7 @@ import ( vmmetrics "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics/virtualmachine" "github.com/deckhouse/virtualization/api/client/kubeclient" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) const ( @@ -83,6 +84,7 @@ func SetupController( internal.NewSyncPowerStateHandler(client, recorder), internal.NewSyncMetadataHandler(client), internal.NewLifeCycleHandler(client, recorder), + internal.NewResizingHandler(inplaceresize.New(featuregates.Default(), client)), internal.NewMigratingHandler(migrateVolumesService), internal.NewFirmwareHandler(firmwareImage), internal.NewEvictHandler(), diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go index 89d81be3a0..a2f8c07cf1 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_cpu.go @@ -55,7 +55,7 @@ func (c *comparatorCPU) Compare(current, desired *v1alpha2.VirtualMachineSpec) [ fractionChangedAction := ActionApplyImmediate // Require reboot if CPU hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugCPUWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugCPUInPlaceResize) { coresChangedAction = ActionRestart fractionChangedAction = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go index c442f57f3c..a0616a1d7b 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/comparator_memory.go @@ -59,7 +59,7 @@ func (c *comparatorMemory) Compare(current, desired *v1alpha2.VirtualMachineSpec } // Require reboot if memory hotplug is not enabled. - if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) { + if !c.featureGate.Enabled(featuregates.HotplugMemoryWithLiveMigration) && !c.featureGate.Enabled(featuregates.HotplugMemoryInPlaceResize) { actionType = ActionRestart } diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go index 5389f8340b..b6f471c857 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug.go @@ -32,20 +32,23 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) const hotplugHandler = "HotplugHandler" -func NewHotplugHandler(client client.Client, migration OneShotMigration) *HotplugHandler { +func NewHotplugHandler(client client.Client, migration OneShotMigration, inplaceResize *inplaceresize.Service) *HotplugHandler { return &HotplugHandler{ client: client, oneShotMigration: migration, + inplaceResize: inplaceResize, } } type HotplugHandler struct { client client.Client oneShotMigration OneShotMigration + inplaceResize *inplaceresize.Service } func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine) (reconcile.Result, error) { @@ -62,6 +65,20 @@ func (h *HotplugHandler) Handle(ctx context.Context, vm *v1alpha2.VirtualMachine return reconcile.Result{}, client.IgnoreNotFound(err) } + if h.inplaceResize.InProgress(kvvmi) { + completed := h.inplaceResize.IsCompleted(kvvmi) + possible, err := h.inplaceResize.IsPossible(ctx, kvvmi) + if err != nil { + return reconcile.Result{}, err + } + + if possible || completed { + return reconcile.Result{}, nil + } + // inplace resize is not possible, but it is not complete + // switch to resize via live migration + } + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) isMemoryHotplug := cond.Status == corev1.ConditionTrue diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go index e1d721c824..13a0ff13f2 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/internal/handler/hotplug_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,6 +32,9 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) var _ = Describe("TestHotplugResourcesHandler", func() { @@ -49,7 +53,14 @@ var _ = Describe("TestHotplugResourcesHandler", func() { fakeClient = nil }) - newVMAndKVVMI := func(hasHotMemoryChange bool) (*v1alpha2.VirtualMachine, *virtv1.VirtualMachineInstance) { + type inPlaceResizeState struct { + inProgress bool + conditionReason string + podResizePendingReason string + podResizeInProgressReason string + } + + newVMAndKVVMI := func(hasHotMemoryChange bool, resizeState inPlaceResizeState) (*v1alpha2.VirtualMachine, *virtv1.VirtualMachineInstance) { vm := vmbuilder.NewEmpty(name, namespace) kvvmi := newEmptyKVVMI(name, namespace) @@ -59,9 +70,75 @@ var _ = Describe("TestHotplugResourcesHandler", func() { Status: corev1.ConditionTrue, }) } + + if resizeState.inProgress { + if kvvmi.Annotations == nil { + kvvmi.Annotations = make(map[string]string) + } + kvvmi.Annotations[annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress] = "true" + kvvmi.Status.Conditions = append(kvvmi.Status.Conditions, virtv1.VirtualMachineInstanceCondition{ + Type: "PodResourceResizeInProgress", + Status: corev1.ConditionTrue, + Reason: resizeState.conditionReason, + }) + } + return vm, kvvmi } + newLauncherPod := func(kvvmi *virtv1.VirtualMachineInstance, resizeState inPlaceResizeState) *corev1.Pod { + if !resizeState.inProgress || resizeState.conditionReason == "PodResizeCompleted" { + return nil + } + + const ( + nodeName = "test-node" + podName = "virt-launcher-test" + ) + + podUID := types.UID("virt-launcher-test-uid") + kvvmi.UID = types.UID("kvvmi-test-uid") + kvvmi.Status.NodeName = nodeName + kvvmi.Status.ActivePods = map[types.UID]string{ + podUID: nodeName, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: kvvmi.Namespace, + UID: podUID, + Labels: map[string]string{ + virtv1.AppLabel: "virt-launcher", + virtv1.CreatedByLabel: string(kvvmi.UID), + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + if resizeState.podResizePendingReason != "" { + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodResizePending, + Status: corev1.ConditionTrue, + Reason: resizeState.podResizePendingReason, + }) + } + if resizeState.podResizeInProgressReason != "" { + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodResizeInProgress, + Status: corev1.ConditionTrue, + Reason: resizeState.podResizeInProgressReason, + }) + } + + return pod + } + newOnceMigrationMock := func(shouldMigrate bool) *OneShotMigrationMock { return &OneShotMigrationMock{ OnceMigrateFunc: func(ctx context.Context, vm *v1alpha2.VirtualMachine, annotationKey, annotationExpectedValue string) (bool, error) { @@ -78,11 +155,13 @@ var _ = Describe("TestHotplugResourcesHandler", func() { awaitingRestart bool shouldMigrate bool expectedMigrationCalls int + expectedErr error + resizeState inPlaceResizeState } DescribeTable("HotplugResourcesHandler should return serviceCompleteErr if migration executed", func(settings testResourcesSettings) { - vm, kvvmi := newVMAndKVVMI(settings.hasHotMemoryChangeCondition) + vm, kvvmi := newVMAndKVVMI(settings.hasHotMemoryChangeCondition, settings.resizeState) if settings.awaitingRestart { vm.Status.Conditions = append(vm.Status.Conditions, metav1.Condition{ Type: vmcondition.TypeAwaitingRestartToApplyConfiguration.String(), @@ -90,20 +169,32 @@ var _ = Describe("TestHotplugResourcesHandler", func() { Reason: vmcondition.ReasonChangesPendingRestart.String(), }) } - fakeClient = setupEnvironment(vm, kvvmi) + pod := newLauncherPod(kvvmi, settings.resizeState) + if pod != nil { + fakeClient = setupEnvironment(vm, kvvmi, pod) + } else { + fakeClient = setupEnvironment(vm, kvvmi) + } mockMigration := newOnceMigrationMock(settings.shouldMigrate) - h := NewHotplugHandler(fakeClient, mockMigration) - _, err := h.Handle(ctx, vm) + gate, setFromMap, err := featuregates.NewUnlocked() + Expect(err).NotTo(HaveOccurred()) + + err = setFromMap(map[string]bool{ + string(featuregates.HotplugMemoryWithLiveMigration): true, + string(featuregates.HotplugCPUWithLiveMigration): true, + string(featuregates.HotplugMemoryInPlaceResize): true, + string(featuregates.HotplugCPUInPlaceResize): true}) + Expect(err).NotTo(HaveOccurred()) + + h := NewHotplugHandler(fakeClient, mockMigration, inplaceresize.New(gate, fakeClient)) + _, err = h.Handle(ctx, vm) Expect(mockMigration.OnceMigrateCalls()).To(HaveLen(settings.expectedMigrationCalls)) - if settings.hasHotMemoryChangeCondition && !settings.shouldMigrate { - Expect(err).ToNot(HaveOccurred()) - } else if settings.hasHotMemoryChangeCondition { - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(serviceCompleteErr)) + if settings.expectedErr != nil { + Expect(err).To(MatchError(settings.expectedErr)) } else { Expect(err).NotTo(HaveOccurred()) } @@ -114,10 +205,11 @@ var _ = Describe("TestHotplugResourcesHandler", func() { hasHotMemoryChangeCondition: true, shouldMigrate: true, expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, }, ), Entry( - "Migration should not be executed the second time", + "Migration should not return an error when one-shot migration reports no action", testResourcesSettings{ hasHotMemoryChangeCondition: true, shouldMigrate: false, @@ -139,5 +231,69 @@ var _ = Describe("TestHotplugResourcesHandler", func() { expectedMigrationCalls: 0, }, ), + Entry( + "Migration should not be executed when in-place resize is already completed", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + expectedMigrationCalls: 0, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeCompleted", + }, + }, + ), + Entry( + "Migration should not be executed when in-place resize is still possible", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + expectedMigrationCalls: 0, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizePending", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize is deferred", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizePending", + podResizePendingReason: "ResizeDeferred", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize is infeasible", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeInProgress", + podResizePendingReason: "ResizeInfeasible", + }, + }, + ), + Entry( + "Migration should be executed when in-place resize ended with error", + testResourcesSettings{ + hasHotMemoryChangeCondition: true, + shouldMigrate: true, + expectedMigrationCalls: 1, + expectedErr: serviceCompleteErr, + resizeState: inPlaceResizeState{ + inProgress: true, + conditionReason: "PodResizeInProgress", + podResizeInProgressReason: "ResizeError", + }, + }, + ), ) }) diff --git a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go index 150aa322a1..2893ca6d91 100644 --- a/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go +++ b/images/virtualization-artifact/pkg/controller/workload-updater/workload_updater_controller.go @@ -29,6 +29,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/workload-updater/internal/service" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) const ( @@ -52,7 +53,7 @@ func SetupController( isMemoryHotplug := featuregates.Default().Enabled(featuregates.HotplugMemoryWithLiveMigration) isCPUHotplug := featuregates.Default().Enabled(featuregates.HotplugCPUWithLiveMigration) if isMemoryHotplug || isCPUHotplug { - hotplugHandler := handler.NewHotplugHandler(client, service.NewOneShotMigrationService(client, "hotplug-resources-")) + hotplugHandler := handler.NewHotplugHandler(client, service.NewOneShotMigrationService(client, "hotplug-resources-"), inplaceresize.New(featuregates.Default(), client)) handlers = append(handlers, hotplugHandler) } r := NewReconciler(client, handlers) diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 3358b0a85a..0e6c2c9337 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -32,6 +32,8 @@ const ( USB featuregate.Feature = "USB" HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" + HotplugCPUInPlaceResize featuregate.Feature = "HotplugCPUInPlaceResize" + HotplugMemoryInPlaceResize featuregate.Feature = "HotplugMemoryInPlaceResize" ) var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -69,6 +71,16 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: version.GetEdition() == version.EditionCE, PreRelease: featuregate.Alpha, }, + HotplugCPUInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, + HotplugMemoryInPlaceResize: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, } var ( diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 5d3afcd436..3b5846cc71 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -223,10 +223,15 @@ properties: description: | Enable experimental or early access features. - - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number. (Not available in CE); - - `HotplugMemoryWithLiveMigration` — enable live changing of memory size. (Not available in CE); + - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); + - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); + - `HotplugCPUInPlaceResize` - enable live changing of cpu cores number via InPlaceResize. (Not available in CE); + - `HotplugMemoryInPlaceResize` - enable live changing of memory size via InPlaceResize. (Not available in CE); items: type: string enum: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" + - "HotplugCPUInPlaceResize" + - "HotplugMemoryInPlaceResize" + diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 8579d3e3ee..5542973c8b 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -153,7 +153,9 @@ properties: description: | Включение экспериментальных или недостаточно обкатанных возможностей. - - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки. (Не доступно в CE); - - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки. (Не доступно в CE); + - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); + - `HotplugCPUInPlaceResize` - включить изменение количества ядер процессора без перезагрузки через InPlaceResize (Не доступно в CE) + - `HotplugMemoryInPlaceResize` - включить изменение размера памяти без перезагрузки через InPlaceResie (Не доступно в CE)) items: type: string diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 535aca4f75..ea6a778309 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -71,6 +71,7 @@ spec: - HostDevicesWithDRA - HostDevices - HotplugHostDevicesWithDRA # custom feature gate - added in our KubeVirt fork, not present in upstream + - InPlaceResize # custom feature gate - added in our KubeVirt fork, not present in upstream virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index c3c7b0119a..38213700da 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -514,6 +514,12 @@ rules: - pods/status verbs: - patch +- apiGroups: + - "" + resources: + - pods/resize + verbs: + - update - apiGroups: - "" resources: diff --git a/test/e2e/internal/precheck/featuregate.go b/test/e2e/internal/precheck/featuregate.go new file mode 100644 index 0000000000..0cdcd095ab --- /dev/null +++ b/test/e2e/internal/precheck/featuregate.go @@ -0,0 +1,99 @@ +package precheck + +import ( + "context" + "fmt" + "slices" + + . "github.com/onsi/ginkgo/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + dv1alpha1 "github.com/deckhouse/virtualization/test/e2e/internal/api/deckhouse/v1alpha1" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" +) + +type featureGatePrecheck struct { + label string + env string + gates []string +} + +func (p featureGatePrecheck) Label() string { + return p.label +} + +func (p featureGatePrecheck) Run(ctx context.Context, f *framework.Framework) error { + if !isCheckEnabled(p.env) { + _, _ = fmt.Fprintf(GinkgoWriter, "%s check is disabled", p.label) + return nil + } + + virtualizationModuleConfig := &dv1alpha1.ModuleConfig{} + err := f.GenericClient().Get(ctx, client.ObjectKey{Name: virtualizationModuleName}, virtualizationModuleConfig) + if err != nil { + return fmt.Errorf("%s=no to disable this precheck: failed to get virtualization module config spec: %w", p.env, err) + } + + gates, err := getFeatureGates(virtualizationModuleConfig) + if err != nil { + return fmt.Errorf("%s=no to disable this precheck: %w", p.env, err) + } + + for _, gate := range p.gates { + if !slices.Contains(gates, gate) { + return fmt.Errorf("%s=no to disable this precheck: feature gate %s is not enabled in virtualization module config spec", p.env, gate) + } + } + + return nil +} + +const ( + hotplugCPUWithLiveMigrationCheckEnvName = "HOTPLUG_CPU_LIVE_MIGRATION_PRECHECK" + hotplugMemoryWithLiveMigrationCheckEnvName = "HOTPLUG_MEMORY_LIVE_MIGRATION_PRECHECK" + hotplugCPUInPlaceCheckEnvName = "HOTPLUG_CPU_IN_PLACE_PRECHECK" + hotplugMemoryInPlaceCheckEnvName = "HOTPLUG_MEMORY_IN_PLACE_PRECHECK" +) + +func init() { + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugCPUWithLiveMigrationPrecheck, + env: hotplugCPUWithLiveMigrationCheckEnvName, + gates: []string{"HotplugCPUWithLiveMigration"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugMemoryWithLiveMigrationPrecheck, + env: hotplugMemoryWithLiveMigrationCheckEnvName, + gates: []string{"HotplugMemoryWithLiveMigration"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugCPUInPlaceResizePrecheck, + env: hotplugCPUInPlaceCheckEnvName, + gates: []string{"HotplugCPUInPlaceResize"}, + }, false) + + RegisterPrecheck(&featureGatePrecheck{ + label: HotplugMemoryInPlaceResizePrecheck, + env: hotplugMemoryInPlaceCheckEnvName, + gates: []string{"HotplugMemoryInPlaceResize"}, + }, false) +} + +func getFeatureGates(mc *dv1alpha1.ModuleConfig) ([]string, error) { + gates, ok := mc.Spec.Settings["featureGates"].([]interface{}) + if !ok { + return nil, fmt.Errorf("failed to get feature gates from virtualization module config spec: %T", gates) + } + var result []string + for _, g := range gates { + if s, ok := g.(string); ok { + result = append(result, s) + } else { + return nil, fmt.Errorf("failed to get feature gates from virtualization module config spec: %T", gates) + } + } + + return result, nil +} diff --git a/test/e2e/internal/precheck/labels.go b/test/e2e/internal/precheck/labels.go index 5d5bf6dbf4..a6bb08aa09 100644 --- a/test/e2e/internal/precheck/labels.go +++ b/test/e2e/internal/precheck/labels.go @@ -58,6 +58,18 @@ const ( // NoPrecheck - test doesn't require any prechecks. // Use this label for tests that don't depend on cluster configuration. NoPrecheck = "no-precheck" + + // HotplugCPUWithLiveMigrationPrecheck - test requires HotplugCPUWithLiveMigration feature gate to be enabled. + HotplugCPUWithLiveMigrationPrecheck = "hotplugcpuwithlivemigration-precheck" + + // HotplugCPUInPlaceResizePrecheck - test requires HotplugCPUInPlaceResize feature gate to be enabled. + HotplugCPUInPlaceResizePrecheck = "hotplugcpuinplaceresize-precheck" + + // HotplugMemoryWithLiveMigrationPrecheck - test requires HotplugMemoryWithLiveMigration feature gate to be enabled. + HotplugMemoryWithLiveMigrationPrecheck = "hotplugmemorywithlivemigration-precheck" + + // HotplugMemoryInPlaceResizePrecheck - test requires HotplugMemoryInPlaceResize feature gate to be enabled. + HotplugMemoryInPlaceResizePrecheck = "hotplugmemoryinplaceresize-precheck" ) // KnownPrecheckLabels returns all known precheck label constants. @@ -75,6 +87,10 @@ func KnownPrecheckLabels() []string { PrecheckPostCleanup, PrecheckPrecreatedCVI, NoPrecheck, + HotplugCPUWithLiveMigrationPrecheck, + HotplugCPUInPlaceResizePrecheck, + HotplugMemoryWithLiveMigrationPrecheck, + HotplugMemoryInPlaceResizePrecheck, } } diff --git a/test/e2e/internal/util/vm.go b/test/e2e/internal/util/vm.go index f8e8a3218b..20afe97bb3 100644 --- a/test/e2e/internal/util/vm.go +++ b/test/e2e/internal/util/vm.go @@ -211,6 +211,41 @@ func shellArgs(args []string) string { return strings.Join(quoted, " ") } +func GetVMNode(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine) (string, error) { + GinkgoHelper() + + err := f.GenericClient().Get(ctx, client.ObjectKeyFromObject(vm), vm) + if err != nil { + return "", err + } + if vm.Status.Node == "" { + return "", fmt.Errorf("vm %s/%s has empty status.node", vm.Namespace, vm.Name) + } + + return vm.Status.Node, nil +} + +func ExpectNoVMOperationsForVirtualMachine(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + vmops, err := f.VirtClient().VirtualMachineOperations(vm.Namespace).List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + + for _, vmop := range vmops.Items { + if vmop.Spec.VirtualMachine == vm.Name { + Fail(fmt.Sprintf("unexpected VMOP %q for VM %q", vmop.Name, vm.Name)) + } + } +} + +func ExpectVMOnNode(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, expectedNode string) { + GinkgoHelper() + + node, err := GetVMNode(ctx, f, vm) + Expect(err).NotTo(HaveOccurred()) + Expect(node).To(Equal(expectedNode)) +} + func UntilVMMigrationSucceeded(key client.ObjectKey, timeout time.Duration) { GinkgoHelper() diff --git a/test/e2e/vm/hotplug_cpu.go b/test/e2e/vm/hotplug_cpu.go index 426b484ce4..ab389ee732 100644 --- a/test/e2e/vm/hotplug_cpu.go +++ b/test/e2e/vm/hotplug_cpu.go @@ -21,16 +21,19 @@ import ( "encoding/json" "fmt" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" @@ -38,65 +41,38 @@ import ( "github.com/deckhouse/virtualization/test/e2e/internal/util" ) -var _ = Describe("HotplugCPU", Label(precheck.NoPrecheck), func() { +var _ = Describe("HotplugCPU", func() { var ( f *framework.Framework t *cpuHotplugTest ) BeforeEach(func() { - Skip("Hotplug CPU requires enabled feature gates in ModuleConfig. Skip until prechecks are implemented.") f = framework.NewFramework("cpu-hotplug") DeferCleanup(f.After) f.Before() t = newCPUHotplugTest(f) }) - DescribeTable("should apply cpu core changes without restart", func(initialCores, changedCores int) { - By("Environment preparation") - vmName := fmt.Sprintf("vm-%d-%d", initialCores, changedCores) - t.generateResources(vmName, initialCores) - err := f.CreateWithDeferredDeletion(context.Background(), t.VM, t.VD) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for VM agent to be ready") - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking initial CPU configuration") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(initialCores)) - - guestCPUCount, err := t.getGuestCPUCount() - Expect(err).NotTo(HaveOccurred()) - Expect(guestCPUCount).To(Equal(initialCores)) - - By("Applying CPU core changes") - patch, err := json.Marshal([]map[string]interface{}{{ - "op": "replace", - "path": "/spec/cpu/cores", - "value": changedCores, - }}) - Expect(err).NotTo(HaveOccurred()) - err = f.GenericClient().Patch(context.Background(), t.VM, crclient.RawPatch(types.JSONPatchType, patch)) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting until CPU configuration is applied without restart") - util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking changed CPU configuration") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(changedCores)) - - guestCPUCount, err = t.getGuestCPUCount() - Expect(err).NotTo(HaveOccurred()) - Expect(guestCPUCount).To(Equal(changedCores)) - }, - Entry("one socket topology, change cores from 1 to 3", 1, 3), - Entry("one socket topology, change cores to maximum 16", 4, 16), - ) + Describe("InPlaceResize", Label(precheck.HotplugCPUInPlaceResizePrecheck), func() { + DescribeTable("should apply cpu core changes in-place without restart", + func(initialCores, changedCores int) { + t.applyCPUCoreChange(initialCores, changedCores, false) + }, + Entry("one socket topology, change cores from 1 to 3", 1, 3), + Entry("one socket topology, change cores to maximum 16", 4, 16), + ) + }) + + Describe("LiveMigration", Label(precheck.HotplugCPUWithLiveMigrationPrecheck), func() { + DescribeTable("should apply cpu core changes via live migration without restart", + func(initialCores, changedCores int) { + t.applyCPUCoreChange(initialCores, changedCores, true) + }, + Entry("one socket topology, change cores from 1 to 3", 1, 3), + Entry("one socket topology, change cores to maximum 16", 4, 16), + ) + }) }) type cpuHotplugTest struct { @@ -110,13 +86,75 @@ func newCPUHotplugTest(f *framework.Framework) *cpuHotplugTest { return &cpuHotplugTest{Framework: f} } -func (t *cpuHotplugTest) generateResources(vmName string, cores int) { +func (t *cpuHotplugTest) applyCPUCoreChange(initialCores, changedCores int, forceMigration bool) { + ctx := context.Background() + + By("Environment preparation") + vmName := fmt.Sprintf("vm-%d-%d", initialCores, changedCores) + if forceMigration { + vmName += "-migrate" + } + t.generateResources(vmName, initialCores, forceMigration) + err := t.Framework.CreateWithDeferredDeletion(ctx, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + By("Checking initial CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(initialCores)) + + guestCPUCount, err := t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(initialCores)) + + initialNode, err := util.GetVMNode(ctx, t.Framework, t.VM) + Expect(err).NotTo(HaveOccurred()) + + By("Applying CPU core changes") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/cpu/cores", + "value": changedCores, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + if forceMigration { + By("Waiting until CPU configuration is applied via live migration") + util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) + } else { + By("Waiting until CPU configuration is applied in-place") + untilVMCPUCoresApplied(crclient.ObjectKeyFromObject(t.VM), changedCores, framework.MaxTimeout) + util.ExpectNoVMOperationsForVirtualMachine(ctx, t.Framework, t.VM) + util.ExpectVMOnNode(ctx, t.Framework, t.VM, initialNode) + } + + util.UntilSSHReady(t.Framework, t.VM, framework.MiddleTimeout) + + By("Checking changed CPU configuration") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.CPU.Cores).To(Equal(changedCores)) + + guestCPUCount, err = t.getGuestCPUCount() + Expect(err).NotTo(HaveOccurred()) + Expect(guestCPUCount).To(Equal(changedCores)) +} + +func (t *cpuHotplugTest) generateResources(vmName string, cores int, disableInPlaceResize bool) { vdName := fmt.Sprintf("vd-%s-root", vmName) t.VD = object.NewVDFromCVI(vdName, t.Framework.Namespace().Name, object.PrecreatedCVIAlpineBIOS, vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), ) - t.VM = vmbuilder.New( + opts := []vmbuilder.Option{ vmbuilder.WithName(vmName), vmbuilder.WithNamespace(t.Framework.Namespace().Name), vmbuilder.WithCPU(cores, ptr.To("10%")), @@ -130,7 +168,12 @@ func (t *cpuHotplugTest) generateResources(vmName string, cores int) { }, ), vmbuilder.WithRestartApprovalMode(v1alpha2.Automatic), - ) + } + if disableInPlaceResize { + opts = append(opts, vmbuilder.WithAnnotation(annotations.AnnVirtualMachineInstanceDisableInPlaceResize, "true")) + } + + t.VM = vmbuilder.New(opts...) } func (t *cpuHotplugTest) getGuestCPUCount() (int, error) { @@ -147,3 +190,13 @@ func (t *cpuHotplugTest) getGuestCPUCount() (int, error) { return cpuCount, nil } + +func untilVMCPUCoresApplied(key crclient.ObjectKey, expectedCores int, timeout time.Duration) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := framework.GetClients().VirtClient().VirtualMachines(key.Namespace).Get(context.Background(), key.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.Resources.CPU.Cores).To(Equal(expectedCores)) + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +} diff --git a/test/e2e/vm/hotplug_memory.go b/test/e2e/vm/hotplug_memory.go index 190b7637d1..5fb1f1d5d9 100644 --- a/test/e2e/vm/hotplug_memory.go +++ b/test/e2e/vm/hotplug_memory.go @@ -23,16 +23,19 @@ import ( "regexp" "strconv" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" @@ -40,67 +43,37 @@ import ( "github.com/deckhouse/virtualization/test/e2e/internal/util" ) -var _ = Describe("HotplugMemory", Label(precheck.NoPrecheck), func() { +var _ = Describe("HotplugMemory", func() { var ( f *framework.Framework t *memoryHotplugTest ) BeforeEach(func() { - Skip("Hotplug memory requires enabling feature gate 'HotplugMemoryWithLiveMigration' in ModuleConfig. Skip until prechecks are implemented.") + Skip("Hotplug memory requires enabled feature gates in ModuleConfig. Skip until prechecks are implemented.") f = framework.NewFramework("memory-hotplug") DeferCleanup(f.After) f.Before() t = newMemoryHotplugTest(f) }) - DescribeTable("should apply memory changes with live migration", func(initialMemory, changedMemory string) { - initialQuantity := resource.MustParse(initialMemory) - changedQuantity := resource.MustParse(changedMemory) - - By("Environment preparation") - vmName := strings.ToLower(fmt.Sprintf("vm-%s-%s", initialMemory, changedMemory)) - t.generateResources(vmName, initialMemory) - err := f.CreateWithDeferredDeletion(context.Background(), t.VM, t.VD) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for VM agent to be ready") - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking initial memory size") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.Memory.Size).To(Equal(initialQuantity)) - - guestMemorySize, err := t.getGuestMemorySize() - Expect(err).NotTo(HaveOccurred()) - Expect(guestMemorySize).To(Equal(int(initialQuantity.Value()))) - - By("Applying memory size changes") - patch, err := json.Marshal([]map[string]interface{}{{ - "op": "replace", - "path": "/spec/memory/size", - "value": changedMemory, - }}) - Expect(err).NotTo(HaveOccurred()) - err = f.GenericClient().Patch(context.Background(), t.VM, crclient.RawPatch(types.JSONPatchType, patch)) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting until memory size is applied without restart") - util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) - util.UntilSSHReady(f, t.VM, framework.MiddleTimeout) - - By("Checking changed memory size") - err = f.Clients.GenericClient().Get(context.Background(), crclient.ObjectKeyFromObject(t.VM), t.VM) - Expect(err).NotTo(HaveOccurred()) - Expect(t.VM.Status.Resources.Memory.Size).To(Equal(changedQuantity)) - - guestMemorySize, err = t.getGuestMemorySize() - Expect(err).NotTo(HaveOccurred()) - Expect(guestMemorySize).To(Equal(int(changedQuantity.Value()))) - }, - Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), - ) + Describe("InPlaceResize", Label(precheck.HotplugMemoryInPlaceResizePrecheck), func() { + DescribeTable("should apply memory changes in-place without restart", + func(initialMemory, changedMemory string) { + t.applyMemoryChange(initialMemory, changedMemory, false) + }, + Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), + ) + }) + + Describe("LiveMigration", Label(precheck.HotplugMemoryWithLiveMigrationPrecheck), func() { + DescribeTable("should apply memory changes via live migration without restart", + func(initialMemory, changedMemory string) { + t.applyMemoryChange(initialMemory, changedMemory, true) + }, + Entry("change memory from 1Gi to 2Gi", "1Gi", "2Gi"), + ) + }) }) type memoryHotplugTest struct { @@ -114,7 +87,71 @@ func newMemoryHotplugTest(f *framework.Framework) *memoryHotplugTest { return &memoryHotplugTest{Framework: f} } -func (t *memoryHotplugTest) generateResources(vmName, memSize string) { +func (t *memoryHotplugTest) applyMemoryChange(initialMemory, changedMemory string, forceMigration bool) { + ctx := context.Background() + initialQuantity := resource.MustParse(initialMemory) + changedQuantity := resource.MustParse(changedMemory) + + By("Environment preparation") + vmName := strings.ToLower(fmt.Sprintf("vm-%s-%s", initialMemory, changedMemory)) + if forceMigration { + vmName += "-migrate" + } + t.generateResources(vmName, initialMemory, forceMigration) + err := t.Framework.CreateWithDeferredDeletion(ctx, t.VM, t.VD) + Expect(err).NotTo(HaveOccurred()) + + By("Wait until VM agent is ready") + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(t.VM), framework.LongTimeout) + + By("Waiting for VM agent to be ready") + util.UntilSSHReady(t.Framework, t.VM, framework.ShortTimeout) + + By("Checking initial memory size") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.Memory.Size).To(Equal(initialQuantity)) + + guestMemorySize, err := t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(guestMemorySize).To(Equal(int(initialQuantity.Value()))) + + initialNode, err := util.GetVMNode(ctx, t.Framework, t.VM) + Expect(err).NotTo(HaveOccurred()) + + By("Applying memory size changes") + patch, err := json.Marshal([]map[string]interface{}{{ + "op": "replace", + "path": "/spec/memory/size", + "value": changedMemory, + }}) + Expect(err).NotTo(HaveOccurred()) + err = t.Framework.GenericClient().Patch(ctx, t.VM, crclient.RawPatch(types.JSONPatchType, patch)) + Expect(err).NotTo(HaveOccurred()) + + if forceMigration { + By("Waiting until memory size is applied via live migration") + util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(t.VM), framework.MaxTimeout) + } else { + By("Waiting until memory size is applied in-place") + untilVMMemorySizeApplied(crclient.ObjectKeyFromObject(t.VM), changedQuantity, framework.MaxTimeout) + util.ExpectNoVMOperationsForVirtualMachine(ctx, t.Framework, t.VM) + util.ExpectVMOnNode(ctx, t.Framework, t.VM, initialNode) + } + + util.UntilSSHReady(t.Framework, t.VM, framework.MiddleTimeout) + + By("Checking changed memory size") + err = t.Framework.Clients.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(t.VM), t.VM) + Expect(err).NotTo(HaveOccurred()) + Expect(t.VM.Status.Resources.Memory.Size).To(Equal(changedQuantity)) + + guestMemorySize, err = t.getGuestMemorySize() + Expect(err).NotTo(HaveOccurred()) + Expect(guestMemorySize).To(Equal(int(changedQuantity.Value()))) +} + +func (t *memoryHotplugTest) generateResources(vmName, memSize string, disableInPlaceResize bool) { memSizeQuantity := resource.MustParse(memSize) vdName := fmt.Sprintf("vd-%s-root", vmName) @@ -122,7 +159,7 @@ func (t *memoryHotplugTest) generateResources(vmName, memSize string) { vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), ) - t.VM = vmbuilder.New( + opts := []vmbuilder.Option{ vmbuilder.WithName(vmName), vmbuilder.WithNamespace(t.Framework.Namespace().Name), vmbuilder.WithCPU(2, ptr.To("10%")), @@ -136,7 +173,12 @@ func (t *memoryHotplugTest) generateResources(vmName, memSize string) { }, ), vmbuilder.WithRestartApprovalMode(v1alpha2.Automatic), - ) + } + if disableInPlaceResize { + opts = append(opts, vmbuilder.WithAnnotation(annotations.AnnVirtualMachineInstanceDisableInPlaceResize, "true")) + } + + t.VM = vmbuilder.New(opts...) } var totalOnlineMemRe = regexp.MustCompile(`^Total online memory:\s+(\d+)$`) @@ -158,3 +200,13 @@ func (t *memoryHotplugTest) getGuestMemorySize() (int, error) { return 0, fmt.Errorf("failed to find total online memory in lsmem output: %v", cmdOut) } + +func untilVMMemorySizeApplied(key crclient.ObjectKey, expectedSize resource.Quantity, timeout time.Duration) { + GinkgoHelper() + + Eventually(func(g Gomega) { + vm, err := framework.GetClients().VirtClient().VirtualMachines(key.Namespace).Get(context.Background(), key.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(vm.Status.Resources.Memory.Size).To(Equal(expectedSize)) + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +} From 5971c9b35d7720e26345c6f5e4d0f6d59b0100a9 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 15 Jun 2026 14:29:12 +0300 Subject: [PATCH 2/3] bump Signed-off-by: Yaroslav Borbat --- images/virt-artifact/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 65165a97bb..e6e6758c19 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -9,7 +9,7 @@ image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact final: false fromImage: builder/src -fromCacheVersion: "014" +fromCacheVersion: "015" secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} From 7951945766824b5ac4e8a79de0dd93fc5e1d7a19 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 15 Jun 2026 17:08:46 +0300 Subject: [PATCH 3/3] fix Signed-off-by: Yaroslav Borbat --- api/core/v1alpha2/events.go | 3 + api/core/v1alpha2/vmcondition/condition.go | 13 -- .../inplaceresize/inplaceresize_service.go | 10 + .../pkg/controller/vm/internal/resizing.go | 66 ------ .../controller/vm/internal/resizing_test.go | 195 ------------------ .../pkg/controller/vm/internal/sync_kvvm.go | 43 ++++ .../controller/vm/internal/sync_kvvm_test.go | 83 ++++++++ .../pkg/controller/vm/vm_controller.go | 2 - 8 files changed, 139 insertions(+), 276 deletions(-) delete mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/resizing.go delete mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go diff --git a/api/core/v1alpha2/events.go b/api/core/v1alpha2/events.go index 7262ad7390..da6a8c33de 100644 --- a/api/core/v1alpha2/events.go +++ b/api/core/v1alpha2/events.go @@ -58,6 +58,9 @@ const ( // ReasonErrRestartAwaitingChanges is event reason indicating that the vm has pending changes requiring a restart. ReasonErrRestartAwaitingChanges = "RestartAwaitingChanges" + // ReasonVMResizing is event reason that the vm is resizing. + ReasonVMResizing = "Resizing" + // ReasonErrVMOPFailed is event reason that operation is failed ReasonErrVMOPFailed = "VirtualMachineOperationFailed" diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 7db351c763..2320fd5da9 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -50,8 +50,6 @@ const ( // TypeMaintenance indicates that the VirtualMachine is in maintenance mode. // During this condition, the VM remains stopped and no changes are allowed. TypeMaintenance Type = "Maintenance" - - TypeResizing Type = "Resizing" ) type AgentReadyReason string @@ -285,14 +283,3 @@ func (r MaintenanceReason) String() string { const ( ReasonMaintenanceRestore MaintenanceReason = "RestoreInProgress" ) - -type ResizingReason string - -func (r ResizingReason) String() string { - return string(r) -} - -const ( - ReasonResizingPending ResizingReason = "Pending" - ReasonResizingInProgress ResizingReason = "InProgress" -) diff --git a/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go index f4eb98d3d2..8fdb15295d 100644 --- a/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go +++ b/images/virtualization-artifact/pkg/controller/service/inplaceresize/inplaceresize_service.go @@ -73,3 +73,13 @@ func (s *Service) ResizeCondition(kvvmi *virtv1.VirtualMachineInstance) virtv1.V cond, _ := conditions.GetKVVMICondition("PodResourceResizeInProgress", kvvmi.Status.Conditions) return cond } + +func (s *Service) CPUChange(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceVCPUChange, kvvmi.Status.Conditions) + return cond.Status == corev1.ConditionTrue +} + +func (s *Service) MemoryChange(kvvmi *virtv1.VirtualMachineInstance) bool { + cond, _ := conditions.GetKVVMICondition(virtv1.VirtualMachineInstanceMemoryChange, kvvmi.Status.Conditions) + return cond.Status == corev1.ConditionTrue +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go b/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go deleted file mode 100644 index 5c89847f71..0000000000 --- a/images/virtualization-artifact/pkg/controller/vm/internal/resizing.go +++ /dev/null @@ -1,66 +0,0 @@ -package internal - -import ( - "context" - "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" - "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" - "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const resizingHandlerName = "ResizingHandler" - -type ResizingHandler struct { - inplaceResize *inplaceresize.Service -} - -func NewResizingHandler(svc *inplaceresize.Service) *ResizingHandler { - return &ResizingHandler{ - inplaceResize: svc, - } -} - -func (h *ResizingHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { - vm := s.VirtualMachine().Changed() - - if isDeletion(vm) { - return reconcile.Result{}, nil - } - - kvvmi, err := s.KVVMI(ctx) - if err != nil { - return reconcile.Result{}, err - } - - if kvvmi == nil || !h.inplaceResize.InProgress(kvvmi) { - conditions.RemoveCondition(vmcondition.TypeResizing, &vm.Status.Conditions) - return reconcile.Result{}, nil - } - - cb := conditions.NewConditionBuilder(vmcondition.TypeResizing). - Generation(vm.GetGeneration()). - Status(metav1.ConditionTrue) - - cond := h.inplaceResize.ResizeCondition(kvvmi) - switch cond.Reason { - case "PodResizePending": - cb.Reason(vmcondition.ReasonResizingPending).Message(cond.Message) - case "PodResizeInProgress": - cb.Reason(vmcondition.ReasonResizingInProgress).Message(cond.Message) - case "PodResizeCompleted": - cb.Reason(vmcondition.ReasonResizingInProgress).Message("Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine") - default: - conditions.RemoveCondition(vmcondition.TypeResizing, &vm.Status.Conditions) - return reconcile.Result{}, nil - } - - conditions.SetCondition(cb, &vm.Status.Conditions) - - return reconcile.Result{}, nil -} - -func (h *ResizingHandler) Name() string { - return resizingHandlerName -} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go deleted file mode 100644 index b86abf2720..0000000000 --- a/images/virtualization-artifact/pkg/controller/vm/internal/resizing_test.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2026 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - virtv1 "kubevirt.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" - "github.com/deckhouse/virtualization-controller/pkg/common/annotations" - "github.com/deckhouse/virtualization-controller/pkg/common/testutil" - "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" - "github.com/deckhouse/virtualization-controller/pkg/controller/reconciler" - "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" - "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" - "github.com/deckhouse/virtualization-controller/pkg/featuregates" - "github.com/deckhouse/virtualization/api/core/v1alpha2" - "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" -) - -var _ = Describe("ResizingHandler", func() { - const ( - name = "vm-resizing" - namespace = "default" - nodeName = "test-node" - ) - - var ( - ctx = testutil.ContextBackgroundWithNoOpLogger() - fakeClient client.WithWatch - resource *reconciler.Resource[*v1alpha2.VirtualMachine, v1alpha2.VirtualMachineStatus] - vmState state.VirtualMachineState - ) - - AfterEach(func() { - fakeClient = nil - resource = nil - vmState = nil - }) - - newVM := func() *v1alpha2.VirtualMachine { - return vmbuilder.NewEmpty(name, namespace) - } - - newHandler := func() *ResizingHandler { - gate, setFeatureMap, err := featuregates.NewUnlocked() - Expect(err).NotTo(HaveOccurred()) - err = setFeatureMap(map[string]bool{ - string(featuregates.HotplugCPUInPlaceResize): true, - string(featuregates.HotplugMemoryInPlaceResize): true, - }) - Expect(err).NotTo(HaveOccurred()) - - return NewResizingHandler(inplaceresize.New(gate, fakeClient)) - } - - newResizingKVVMI := func(reason, message string) *virtv1.VirtualMachineInstance { - kvvmi := newEmptyKVVMI(name, namespace) - kvvmi.UID = types.UID("kvvmi-uid") - kvvmi.Annotations = map[string]string{ - annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress: "true", - } - kvvmi.Status.NodeName = nodeName - kvvmi.Status.Conditions = []virtv1.VirtualMachineInstanceCondition{ - { - Type: "PodResourceResizeInProgress", - Status: corev1.ConditionTrue, - Reason: reason, - Message: message, - }, - } - - if reason != "PodResizeCompleted" { - kvvmi.Status.ActivePods = map[types.UID]string{ - types.UID("virt-launcher-uid"): nodeName, - } - } - - return kvvmi - } - - reconcile := func() error { - h := newHandler() - _, err := h.Handle(ctx, vmState) - if err != nil { - return err - } - - return resource.Update(context.Background()) - } - - Describe("Condition presence and absence scenarios", func() { - It("Should remove condition when KVVMI is absent", func() { - vm := newVM() - vm.Status.Conditions = []metav1.Condition{{ - Type: vmcondition.TypeResizing.String(), - Status: metav1.ConditionTrue, - }} - - fakeClient, resource, vmState = setupEnvironment(vm) - err := reconcile() - Expect(err).NotTo(HaveOccurred()) - - actualVM := &v1alpha2.VirtualMachine{} - err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) - Expect(err).NotTo(HaveOccurred()) - - _, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) - Expect(exists).To(BeFalse()) - }) - - DescribeTable("Should set resizing condition according to resize state", - func(reason, message, expectedReason, expectedMessage string) { - vm := newVM() - kvvmi := newResizingKVVMI(reason, message) - - fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) - err := reconcile() - Expect(err).NotTo(HaveOccurred()) - - actualVM := &v1alpha2.VirtualMachine{} - err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) - Expect(err).NotTo(HaveOccurred()) - - cond, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) - Expect(exists).To(BeTrue()) - Expect(cond.Status).To(Equal(metav1.ConditionTrue)) - Expect(cond.Reason).To(Equal(expectedReason)) - Expect(cond.Message).To(Equal(expectedMessage)) - }, - Entry("pending resize", string(corev1.PodResizePending), "Waiting for kubelet", vmcondition.ReasonResizingPending.String(), "Waiting for kubelet"), - Entry("in progress resize", string(corev1.PodResizeInProgress), "Resizing pod resources", vmcondition.ReasonResizingInProgress.String(), "Resizing pod resources"), - ) - - It("Should keep in-progress reason after pod resize completion", func() { - vm := newVM() - kvvmi := newResizingKVVMI("PodResizeCompleted", "Completed") - - fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) - err := reconcile() - Expect(err).NotTo(HaveOccurred()) - - actualVM := &v1alpha2.VirtualMachine{} - err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) - Expect(err).NotTo(HaveOccurred()) - - cond, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) - Expect(exists).To(BeTrue()) - Expect(cond.Status).To(Equal(metav1.ConditionTrue)) - Expect(cond.Reason).To(Equal(vmcondition.ReasonResizingInProgress.String())) - Expect(cond.Message).To(Equal("Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine")) - }) - }) - - It("Should remove condition when resize reason is unexpected", func() { - vm := newVM() - vm.Status.Conditions = []metav1.Condition{{ - Type: vmcondition.TypeResizing.String(), - Status: metav1.ConditionTrue, - }} - kvvmi := newResizingKVVMI("UnexpectedReason", "unexpected") - - fakeClient, resource, vmState = setupEnvironment(vm, kvvmi) - err := reconcile() - Expect(err).NotTo(HaveOccurred()) - - actualVM := &v1alpha2.VirtualMachine{} - err = fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), actualVM) - Expect(err).NotTo(HaveOccurred()) - - _, exists := conditions.GetCondition(vmcondition.TypeResizing, actualVM.Status.Conditions) - Expect(exists).To(BeFalse()) - }) -}) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index a4401040f4..41283f49b3 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -44,6 +44,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" + "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" "github.com/deckhouse/virtualization-controller/pkg/controller/vmchange" "github.com/deckhouse/virtualization-controller/pkg/dvcr" @@ -74,6 +75,7 @@ func NewSyncKvvmHandler( recorder: recorder, featureGate: featureGate, syncVolumesService: syncVolumesService, + inplaceResize: inplaceresize.New(featureGate, client), } } @@ -83,6 +85,7 @@ type SyncKvvmHandler struct { dvcrSettings *dvcr.Settings featureGate featuregate.FeatureGate syncVolumesService syncVolumesService + inplaceResize *inplaceresize.Service } func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { @@ -228,6 +231,13 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat changed.Status.RestartAwaitingChanges = nil } + kvvmi, err := s.KVVMI(ctx) + if err != nil { + return reconcile.Result{}, err + } + + inplaceResizeInProgress := kvvmi != nil && h.inplaceResize.InProgress(kvvmi) + // 4. Set ConfigurationApplied condition. switch { case waitForNetwork: @@ -266,6 +276,10 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat Status(metav1.ConditionTrue). Reason(vmcondition.ReasonChangesPendingRestart). Message("VirtualMachineClass.spec has been modified. Waiting for the user to restart in order to apply the configuration changes.") + case inplaceResizeInProgress: + msg := h.buildInProgressInplaceResizeMsg(kvvmi) + h.recorder.Event(current, corev1.EventTypeNormal, v1alpha2.ReasonVMResizing, msg) + cbConfApplied.Status(metav1.ConditionFalse).Reason(vmcondition.ReasonConfigurationNotApplied).Message(msg) case synced: h.recorder.Event(current, corev1.EventTypeNormal, v1alpha2.ReasonErrVmSynced, "The virtual machine configuration successfully synced") cbConfApplied.Status(metav1.ConditionTrue).Reason(vmcondition.ReasonConfigurationApplied) @@ -1132,3 +1146,32 @@ func (h *SyncKvvmHandler) isPlacementPolicyChanged(allChanges vmchange.SpecChang return false } + +func (h *SyncKvvmHandler) buildInProgressInplaceResizeMsg(kvvmi *virtv1.VirtualMachineInstance) string { + msg := strings.Builder{} + + switch { + case h.inplaceResize.CPUChange(kvvmi): + msg.WriteString("CPU hotplug is in progress.") + case h.inplaceResize.MemoryChange(kvvmi): + msg.WriteString("Memory hotplug is in progress.") + default: + msg.WriteString("Hotplug is in progress.") + } + + cond := h.inplaceResize.ResizeCondition(kvvmi) + switch cond.Reason { + case "PodResizePending", "PodResizeInProgress": + msg.WriteString(" ") + msg.WriteString(cond.Message) + case "PodResizeCompleted": + msg.WriteString(" Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine.") + default: + msg.WriteString(" reason: ") + msg.WriteString(cond.Reason) + msg.WriteString(", message: ") + msg.WriteString(cond.Message) + } + + return msg.String() +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 7c2b292069..57f7b460b0 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" @@ -174,6 +175,29 @@ var _ = Describe("SyncKvvmHandler", func() { return kvvmi } + makeResizingKVVMI := func(reason, message string, conditionType virtv1.VirtualMachineInstanceConditionType) *virtv1.VirtualMachineInstance { + kvvmi := makeKVVMI() + kvvmi.Annotations = map[string]string{ + annotations.AnnVirtualMachineInstanceInPlaceResizeInProgress: "true", + } + kvvmi.Status.Conditions = []virtv1.VirtualMachineInstanceCondition{ + { + Type: "PodResourceResizeInProgress", + Status: corev1.ConditionTrue, + Reason: reason, + Message: message, + }, + } + if conditionType != "" { + kvvmi.Status.Conditions = append(kvvmi.Status.Conditions, virtv1.VirtualMachineInstanceCondition{ + Type: conditionType, + Status: corev1.ConditionTrue, + }) + } + + return kvvmi + } + makeVMIP := func() *v1alpha2.VirtualMachineIPAddress { return &v1alpha2.VirtualMachineIPAddress{ ObjectMeta: metav1.ObjectMeta{ @@ -471,6 +495,57 @@ var _ = Describe("SyncKvvmHandler", func() { Entry("Pending phase with changes not applied, condition should not exist", v1alpha2.MachinePending, true, metav1.ConditionUnknown, false), ) + DescribeTable("ConfigurationApplied Condition for in-place resize", + func(featureGate featuregate.FeatureGate, kvvmi *virtv1.VirtualMachineInstance, expectedMessage string) { + ip := makeVMIP() + vmClass := makeVMClass() + vm := makeVM(v1alpha2.MachineRunning) + kvvm := makeKVVM(vm) + + fakeClient, reconcileObj, vmState = setupEnvironment(vm, kvvm, kvvmi, ip, vmClass) + featureGates = featureGate + + reconcile() + + newVM := &v1alpha2.VirtualMachine{} + err := fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), newVM) + Expect(err).NotTo(HaveOccurred()) + + confAppliedCond, confAppliedExists := conditions.GetCondition(vmcondition.TypeConfigurationApplied, newVM.Status.Conditions) + Expect(confAppliedExists).To(BeTrue()) + Expect(confAppliedCond.Status).To(Equal(metav1.ConditionFalse)) + Expect(confAppliedCond.Reason).To(Equal(vmcondition.ReasonConfigurationNotApplied.String())) + Expect(confAppliedCond.Message).To(Equal(expectedMessage)) + + _, resizingExists := conditions.GetCondition(vmcondition.TypeResizing, newVM.Status.Conditions) + Expect(resizingExists).To(BeFalse()) + }, + Entry( + "cpu hotplug pending", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI(string(corev1.PodResizePending), "Waiting for kubelet", virtv1.VirtualMachineInstanceVCPUChange), + "CPU hotplug is in progress. Waiting for kubelet", + ), + Entry( + "memory hotplug in progress", + newFeatureGateEnableMemoryInPlaceResize(), + makeResizingKVVMI(string(corev1.PodResizeInProgress), "Resizing pod resources", virtv1.VirtualMachineInstanceMemoryChange), + "Memory hotplug is in progress. Resizing pod resources", + ), + Entry( + "resize completed", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI("PodResizeCompleted", "Completed", virtv1.VirtualMachineInstanceVCPUChange), + "CPU hotplug is in progress. Pod resize completed, waiting when cpu and memory will be hotplugged on virtual machine.", + ), + Entry( + "unexpected resize reason", + newFeatureGateEnableCPUInPlaceResize(), + makeResizingKVVMI("UnexpectedReason", "unexpected", ""), + "Hotplug is in progress. reason: UnexpectedReason, message: unexpected", + ), + ) + It("keeps ConfigurationApplied False and requeues while SDN is not ready", func() { ip := &v1alpha2.VirtualMachineIPAddress{ ObjectMeta: metav1.ObjectMeta{Name: "test-ip", Namespace: namespace}, @@ -550,6 +625,14 @@ func newFeatureGateEnableResourceHotplug() featuregate.FeatureGate { return newFeatureGate(featuregates.HotplugCPUWithLiveMigration, featuregates.HotplugMemoryWithLiveMigration) } +func newFeatureGateEnableCPUInPlaceResize() featuregate.FeatureGate { + return newFeatureGate(featuregates.HotplugCPUInPlaceResize) +} + +func newFeatureGateEnableMemoryInPlaceResize() featuregate.FeatureGate { + return newFeatureGate(featuregates.HotplugMemoryInPlaceResize) +} + func newResourceQuota(cpuHard, memoryHard, cpuUsed, memoryUsed resource.Quantity) *corev1.ResourceQuota { const quotaNamespace = "default" diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index 08ddcffc89..ebc466e99e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -38,7 +38,6 @@ import ( vmmetrics "github.com/deckhouse/virtualization-controller/pkg/monitoring/metrics/virtualmachine" "github.com/deckhouse/virtualization/api/client/kubeclient" "github.com/deckhouse/virtualization/api/core/v1alpha2" - "github.com/deckhouse/virtualization-controller/pkg/controller/service/inplaceresize" ) const ( @@ -84,7 +83,6 @@ func SetupController( internal.NewSyncPowerStateHandler(client, recorder), internal.NewSyncMetadataHandler(client), internal.NewLifeCycleHandler(client, recorder), - internal.NewResizingHandler(inplaceresize.New(featuregates.Default(), client)), internal.NewMigratingHandler(migrateVolumesService), internal.NewFirmwareHandler(firmwareImage), internal.NewEvictHandler(),