Skip to content

Commit 4a491b7

Browse files
weizhouapacheweizhouapache
committed
controller: set resource states to Removing when remove an application
1 parent 63ff842 commit 4a491b7

4 files changed

Lines changed: 166 additions & 24 deletions

File tree

‎apis/v1/types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ type VirtualMachine struct {
9999
ObservedSpec VirtualMachineSpec `json:"observedSpec,omitempty" yaml:"observedSpec,omitempty" gorm:"serializer:json"`
100100
// ApplicationID links this VM to a parent Application (store application name or id)
101101
ApplicationID string `json:"applicationId,omitempty" yaml:"applicationId,omitempty" gorm:"column:application_id"`
102+
// ComponentID links this VM to a parent Component (store component name or id)
103+
ComponentID string `json:"componentId,omitempty" yaml:"componentId,omitempty" gorm:"column:component_id"`
102104
// CloudStackID is the external provider ID for this VM in CloudStack
103105
CloudStackID string `json:"cloudStackId,omitempty" yaml:"cloudStackId,omitempty" gorm:"column:cloudstack_id"`
104106
}

‎controller/controller.go‎

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ func (c *Controller) Start() {
4646
}
4747
}()
4848

49-
// Run reconciliation every 30 seconds
50-
ticker := time.NewTicker(30 * time.Second)
49+
// Run reconciliation every 10 seconds
50+
ticker := time.NewTicker(10 * time.Second)
5151
defer ticker.Stop()
5252

5353
for range ticker.C {
@@ -544,17 +544,34 @@ func (c *Controller) handleDelete(w http.ResponseWriter, r *http.Request) {
544544
http.Error(w, "application not found", http.StatusNotFound)
545545
return
546546
}
547-
for _, cref := range app.Spec.Components {
548-
db.DB.Where("name = ?", cref.Name).Delete(&v1.Component{})
547+
// Mark the application as removing; actual deletion will be processed
548+
// by the reconciliation loop in the order: VMs -> Components -> Applications
549+
app.Status.ObservedState = "Removing"
550+
app.Status.Ready = false
551+
app.Status.LastChecked = time.Now()
552+
if err := db.DB.Save(&app).Error; err != nil {
553+
http.Error(w, "failed to mark application for removal", http.StatusInternalServerError)
554+
return
549555
}
550-
db.DB.Delete(&app)
551556
w.WriteHeader(http.StatusOK)
552-
w.Write([]byte(`{"status":"deleted"}`))
557+
w.Write([]byte(`{"status":"Removing"}`))
553558
return
554559
case "Component":
555-
db.DB.Where("name = ?", name).Delete(&v1.Component{})
560+
var comp v1.Component
561+
if db.DB == nil || db.DB.Where("name = ?", name).First(&comp).Error != nil {
562+
http.Error(w, "component not found", http.StatusNotFound)
563+
return
564+
}
565+
// Mark component as removing; reconciler will delete when VMs are gone
566+
comp.Status.ObservedState = "Removing"
567+
comp.Status.Ready = false
568+
comp.Status.LastChecked = time.Now()
569+
if err := db.DB.Save(&comp).Error; err != nil {
570+
http.Error(w, "failed to mark component for removal", http.StatusInternalServerError)
571+
return
572+
}
556573
w.WriteHeader(http.StatusOK)
557-
w.Write([]byte(`{"status":"deleted"}`))
574+
w.Write([]byte(`{"status":"Removing"}`))
558575
return
559576
case "VirtualMachineSpec":
560577
var spec v1.VirtualMachineSpecResource
@@ -580,19 +597,23 @@ func (c *Controller) handleDelete(w http.ResponseWriter, r *http.Request) {
580597
dp.SetExpunge(true)
581598
c.csClient.VirtualMachine.DestroyVirtualMachine(dp)
582599
w.WriteHeader(http.StatusOK)
583-
w.Write([]byte(`{"status":"deleted"}`))
600+
w.Write([]byte(`{"status":"Deleted"}`))
584601
return
585602
}
586603
http.Error(w, "virtualmachine not found", http.StatusNotFound)
587604
return
588605
}
589-
if vm.CloudStackID != "" {
590-
dp := c.csClient.VirtualMachine.NewDestroyVirtualMachineParams(vm.CloudStackID)
591-
c.csClient.VirtualMachine.DestroyVirtualMachine(dp)
606+
// Instead of deleting immediately, mark VM as Removing so the reconciler
607+
// will perform deletion in the proper order (VMs -> Components -> Applications).
608+
vm.Status.ObservedState = "Removing"
609+
vm.Status.Ready = false
610+
vm.Status.LastChecked = time.Now()
611+
if err := db.DB.Save(&vm).Error; err != nil {
612+
http.Error(w, "failed to mark virtualmachine for removal", http.StatusInternalServerError)
613+
return
592614
}
593-
db.DB.Delete(&vm)
594615
w.WriteHeader(http.StatusOK)
595-
w.Write([]byte(`{"status":"deleted"}`))
616+
w.Write([]byte(`{"status":"Removing"}`))
596617
return
597618
case "Network", "Volume", "SSHKey", "SecurityGroup", "AffinityGroup", "UserData", "Template", "Snapshot":
598619
// Delegate deletes of unmanaged CloudStack resources to handlers
@@ -672,17 +693,28 @@ func (c *Controller) Apply(resource interface{}) error {
672693

673694
// applyApplication creates/updates an Application resource
674695
func (c *Controller) applyApplication(app *v1.Application) error {
675-
// Save desired state to database
696+
// Save desired state to database and mark as Starting.
697+
if app.Status.ObservedState == "" {
698+
app.Status.ObservedState = "Starting"
699+
app.Status.Ready = false
700+
app.Status.LastChecked = time.Now()
701+
}
676702
if err := db.DB.Save(app).Error; err != nil {
677703
return err
678704
}
679705

680-
// Resolve component dependencies and create resources
681-
return c.ResolveComponentDependencies(app)
706+
// Actual creation of component VMs is handled by the reconciler.
707+
return nil
682708
}
683709

684710
// applyComponent creates/updates a Component resource
685711
func (c *Controller) applyComponent(comp *v1.Component) error {
712+
if comp.Status.ObservedState == "" {
713+
comp.Status.ObservedState = "Created"
714+
comp.Status.Ready = false
715+
comp.Status.LastChecked = time.Now()
716+
}
717+
// Persist desired component with effective spec (if present)
686718
return db.DB.Save(comp).Error
687719
}
688720

@@ -697,6 +729,11 @@ func (c *Controller) applyVMSpec(vs *v1.VirtualMachineSpecResource) error {
697729
}
698730
return fmt.Errorf("VirtualMachineSpec %s already exists and cannot be updated", vs.Metadata.Name)
699731
}
732+
if vs.Status.ObservedState == "" {
733+
vs.Status.ObservedState = "Created"
734+
vs.Status.Ready = false
735+
vs.Status.LastChecked = time.Now()
736+
}
700737
return db.DB.Create(vs).Error
701738
}
702739

‎controller/dependency.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func (c *Controller) ResolveComponentDependencies(app *v1.Application) error {
3232

3333
// Create component VMs
3434
log.Printf("Creating component %s (replicas: %d)", compRef.Name, compRef.Replicas)
35-
if err := c.createComponentVMs(&component, compRef); err != nil {
35+
if err := c.createComponentVMs(app.Metadata.Name, &component, compRef); err != nil {
3636
return err
3737
}
3838
}

‎controller/reconcile.go‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ import (
1515
func (c *Controller) ReconcileAll() {
1616
log.Println("Starting reconciliation loop")
1717

18-
// Reconcile Applications
18+
// Reconcile Applications (ignore those marked Removing)
1919
var apps []v1.Application
20-
if err := db.DB.Where("deleted_at IS NULL").Find(&apps).Error; err != nil {
20+
if err := db.DB.Where("deleted_at IS NULL AND observed_state <> ?", "Removing").Find(&apps).Error; err != nil {
2121
log.Printf("Failed to list applications: %v", err)
2222
return
2323
}
@@ -27,9 +27,9 @@ func (c *Controller) ReconcileAll() {
2727
log.Printf("Failed to reconcile application %s: %v", app.Metadata.Name, err)
2828
}
2929
}
30-
// Reconcile VMs
30+
// Reconcile VMs (ignore those marked Removing)
3131
var vms []v1.VirtualMachine
32-
if err := db.DB.Where("deleted_at IS NULL").Find(&vms).Error; err != nil {
32+
if err := db.DB.Where("deleted_at IS NULL AND observed_state <> ?", "Removing").Find(&vms).Error; err != nil {
3333
log.Printf("Failed to list VMs: %v", err)
3434
return
3535
}
@@ -39,16 +39,107 @@ func (c *Controller) ReconcileAll() {
3939
log.Printf("Failed to reconcile VM %s: %v", vm.Metadata.Name, err)
4040
}
4141
}
42+
43+
// After normal reconciliation, process removals in order: VMs -> Components -> Applications
44+
// 1) VMs marked Removing
45+
var removingVMs []v1.VirtualMachine
46+
if err := db.DB.Where("observed_state = ?", "Removing").Find(&removingVMs).Error; err != nil {
47+
log.Printf("Failed to list removing VMs: %v", err)
48+
} else {
49+
for _, vm := range removingVMs {
50+
log.Printf("Removing VM: %s", vm.Metadata.Name)
51+
if vm.CloudStackID != "" {
52+
dp := c.csClient.VirtualMachine.NewDestroyVirtualMachineParams(vm.CloudStackID)
53+
dp.SetExpunge(true)
54+
c.csClient.VirtualMachine.DestroyVirtualMachine(dp)
55+
}
56+
db.DB.Delete(&vm)
57+
}
58+
}
59+
60+
// 2) Components marked Removing
61+
var removingComps []v1.Component
62+
if err := db.DB.Where("observed_state = ?", "Removing").Find(&removingComps).Error; err != nil {
63+
log.Printf("Failed to list removing components: %v", err)
64+
} else {
65+
for _, comp := range removingComps {
66+
// Check if any VMs still reference this component
67+
var vmCount int64
68+
if err := db.DB.Model(&v1.VirtualMachine{}).Where("component_id = ?", comp.Metadata.Name).Count(&vmCount).Error; err != nil {
69+
log.Printf("Failed to count VMs for component %s: %v", comp.Metadata.Name, err)
70+
continue
71+
}
72+
if vmCount > 0 {
73+
log.Printf("Skipping deletion of Component %s: %d VMs still exist", comp.Metadata.Name, vmCount)
74+
continue
75+
}
76+
log.Printf("Removing Component: %s", comp.Metadata.Name)
77+
db.DB.Delete(&comp)
78+
}
79+
}
80+
81+
// Process applications marked for removal: ensure VMs -> Components -> Applications
82+
var removingApps []v1.Application
83+
if err := db.DB.Where("observed_state = ?", "Removing").Find(&removingApps).Error; err != nil {
84+
log.Printf("Failed to list removing applications: %v", err)
85+
return
86+
}
87+
for _, app := range removingApps {
88+
log.Printf("Processing removal for application: %s", app.Metadata.Name)
89+
90+
// Delete VMs that reference this application
91+
var appVMs []v1.VirtualMachine
92+
if err := db.DB.Where("application_id = ?", app.Metadata.Name).Find(&appVMs).Error; err == nil {
93+
for _, vm := range appVMs {
94+
if vm.CloudStackID != "" {
95+
dp := c.csClient.VirtualMachine.NewDestroyVirtualMachineParams(vm.CloudStackID)
96+
dp.SetExpunge(true)
97+
c.csClient.VirtualMachine.DestroyVirtualMachine(dp)
98+
}
99+
db.DB.Delete(&vm)
100+
}
101+
}
102+
103+
// Delete component records referenced by application only if no components remain
104+
var compNames []string
105+
for _, cref := range app.Spec.Components {
106+
compNames = append(compNames, cref.Name)
107+
}
108+
var remaining int64
109+
if len(compNames) > 0 {
110+
if err := db.DB.Model(&v1.Component{}).Where("name IN ?", compNames).Count(&remaining).Error; err != nil {
111+
log.Printf("Failed to count components for application %s: %v", app.Metadata.Name, err)
112+
continue
113+
}
114+
}
115+
if remaining > 0 {
116+
log.Printf("Skipping deletion of Application %s: %d components still exist", app.Metadata.Name, remaining)
117+
continue
118+
}
119+
// Safe to delete application record
120+
db.DB.Delete(&app)
121+
log.Printf("Application %s removed", app.Metadata.Name)
122+
}
42123
}
43124

44125
// ReconcileApplication ensures application state matches desired state
45126
func (c *Controller) ReconcileApplication(app *v1.Application) error {
127+
// Skip applications that are marked for removal
128+
if app.Status.ObservedState == "Removing" {
129+
return nil
130+
}
131+
46132
// Resolve component dependencies (health enforcement)
47133
return c.ResolveComponentDependencies(app)
48134
}
49135

50136
// ReconcileComponent ensures component state matches desired state
51137
func (c *Controller) ReconcileComponent(comp *v1.Component) error {
138+
// Skip components that are marked for removal
139+
if comp.Status.ObservedState == "Removing" {
140+
return nil
141+
}
142+
52143
// Check component health
53144
healthy, err := c.CheckComponentHealth(comp)
54145
if err != nil {
@@ -76,6 +167,10 @@ func (c *Controller) ReconcileComponent(comp *v1.Component) error {
76167

77168
// ReconcileVM ensures VM state matches desired state
78169
func (c *Controller) ReconcileVM(vm *v1.VirtualMachine) error {
170+
// Skip VMs marked for removal
171+
if vm.Status.ObservedState == "Removing" {
172+
return nil
173+
}
79174
// Populate observed spec from CloudStack (if possible)
80175
if err := c.populateObservedSpec(vm); err != nil {
81176
return err
@@ -238,7 +333,7 @@ func (c *Controller) populateObservedSpec(vm *v1.VirtualMachine) error {
238333
}
239334

240335
// createComponentVMs creates VM replicas for a component
241-
func (c *Controller) createComponentVMs(comp *v1.Component, compRef v1.ComponentRef) error {
336+
func (c *Controller) createComponentVMs(appName string, comp *v1.Component, compRef v1.ComponentRef) error {
242337
// Load the referenced reusable VM spec
243338
var vsr v1.VirtualMachineSpecResource
244339
if err := db.DB.Where("name = ?", compRef.VirtualMachineSpec).First(&vsr).Error; err != nil {
@@ -271,6 +366,14 @@ func (c *Controller) createComponentVMs(comp *v1.Component, compRef v1.Component
271366
Spec: effective,
272367
}
273368

369+
// Link VM to owning application if provided
370+
if appName != "" {
371+
vm.ApplicationID = appName
372+
}
373+
374+
// Link VM to owning component
375+
vm.ComponentID = comp.Metadata.Name
376+
274377
// Persist desired VM record and create in CloudStack
275378
if err := db.DB.Save(vm).Error; err != nil {
276379
return err
@@ -306,7 +409,7 @@ func (c *Controller) recreateComponentVMs(comp *v1.Component) error {
306409
VirtualMachineSpec: comp.Spec.VirtualMachineSpec,
307410
Replicas: comp.Spec.Replicas,
308411
}
309-
return c.createComponentVMs(comp, compRef)
412+
return c.createComponentVMs("", comp, compRef)
310413
}
311414

312415
// mergeVMSpec merges allowed overrides into a base VirtualMachineSpec

0 commit comments

Comments
 (0)