controller: create/delete vms with 2 seconds interval

Author: weizhouapache

Development.md

@@ -8,7 +8,7 @@ Kubernetes-style declarative orchestration tool for Apache CloudStack
 go build -o cloudstackctl main.go
 
 # Build binary for Controller (if run locally)
-go build -o cloudstackctl-controller cmd/controller
+go build -o cloudstackctl-controller cmd/controller/main.go
 
 # Install to PATH
 sudo mv cloudstackctl /usr/local/bin/

controller/controller.go

@@ -127,12 +127,17 @@ func (c *Controller) stopAppWorker(appName string) {
 			vmsByComp[comp] = append(vmsByComp[comp], vm)
 		}
 
-		// Process components in deterministic order, deleting each component's VMs in parallel
+		// Process components in deterministic order, deleting each component's VMs in parallel.
+		// Launches are staggered 2 seconds apart to avoid concurrent CloudStack
+		// API contention; goroutines themselves run concurrently after their delay.
 		for _, comp := range compOrder {
 			vms := vmsByComp[comp]
 			log.Printf("stopAppWorker: removing %d VMs for component '%s'", len(vms), comp)
 			var wg sync.WaitGroup
-			for _, vm := range vms {
+			for i, vm := range vms {
+				if i > 0 {
+					time.Sleep(2 * time.Second)
+				}
 				wg.Add(1)
 				vmCopy := vm
 				go func(v v1.VirtualMachine) {

controller/reconcile.go

@@ -431,16 +431,23 @@ func (c *Controller) createComponentVMs(appName string, comp *v1.Component, comp
 		vms = append(vms, vm)
 	}
 
-	// Create VMs in parallel (limit concurrency with a semaphore)
+	// Create VMs in parallel (limit concurrency with a semaphore).
+	// Launches are staggered 2 seconds apart to avoid concurrent CloudStack
+	// API contention; goroutines themselves run concurrently after their delay.
 	var wg sync.WaitGroup
 	errCh := make(chan error, len(vms))
 	sem := make(chan struct{}, 5) // max 5 concurrent creations
 
+	launchIdx := 0
 	for _, vm := range vms {
 		// if VM already has CloudStackID, skip
 		if vm.CloudStackID != "" {
 			continue
 		}
+		if launchIdx > 0 {
+			time.Sleep(2 * time.Second)
+		}
+		launchIdx++
 		wg.Add(1)
 		sem <- struct{}{}
 		go func(vmName string) {