When processing events, look back to the owning deployment to understand

if the pod's replica count is being changed to avoid excessive requests

Author: kommendorkapten

deploy/manifest.yaml

@@ -17,6 +17,9 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

internal/controller/controller.go

@@ -13,6 +13,8 @@ import (
 	"github.com/github/deployment-tracker/pkg/metrics"
 
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
@@ -93,8 +95,9 @@ func New(clientset kubernetes.Interface, namespace string, cfg *Config) (*Contro
 				return
 			}
 
-			// Only process pods that are running
-			if pod.Status.Phase == corev1.PodRunning {
+			// Only process pods that are running and belong
+			// to a deployment
+			if pod.Status.Phase == corev1.PodRunning && getDeploymentName(pod) != "" {
 				key, err := cache.MetaNamespaceKeyFunc(obj)
 
 				// For our purposes, there are in practice
@@ -124,8 +127,9 @@ func New(clientset kubernetes.Interface, namespace string, cfg *Config) (*Contro
 				return
 			}
 
-			// Skip if pod is being deleted
-			if newPod.DeletionTimestamp != nil {
+			// Skip if pod is being deleted or doesn't belong
+			// to a deployment
+			if newPod.DeletionTimestamp != nil || getDeploymentName(newPod) == "" {
 				return
 			}
 
@@ -162,8 +166,13 @@ func New(clientset kubernetes.Interface, namespace string, cfg *Config) (*Contro
 					return
 				}
 			}
-			key, err := cache.MetaNamespaceKeyFunc(obj)
 
+			// Only process pods that belong to a deployment
+			if getDeploymentName(pod) == "" {
+				return
+			}
+
+			key, err := cache.MetaNamespaceKeyFunc(obj)
 			// For our purposes, there are in practice
 			// no error event we care about, so don't
 			// bother with handling it.
@@ -176,7 +185,6 @@ func New(clientset kubernetes.Interface, namespace string, cfg *Config) (*Contro
 			}
 		},
 	})
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to add event handlers: %w", err)
 	}
@@ -261,16 +269,27 @@ func (c *Controller) processEvent(ctx context.Context, event PodEvent) error {
 
 	if event.EventType == "DELETED" {
 		// For delete events, use the pod captured at deletion time
-		// since it's already been removed from the cache
 		pod = event.DeletedPod
 		if pod == nil {
 			slog.Error("Delete event missing pod data",
 				"key", event.Key,
 			)
 			return nil
 		}
+
+		// Check if the parent deployment still exists
+		// If it does, this is just a scale-down event, skip it
+		deploymentName := getDeploymentName(pod)
+		if deploymentName != "" && c.deploymentExists(ctx, pod.Namespace, deploymentName) {
+			slog.Debug("Deployment still exists, skipping pod delete (scale down)",
+				"namespace", pod.Namespace,
+				"deployment", deploymentName,
+				"pod", pod.Name,
+			)
+			return nil
+		}
 	} else {
-		// For other events, get the pod from the informer's cache
+		// For create events, get the pod from the informer's cache
 		obj, exists, err := c.podInformer.GetIndexer().GetByKey(event.Key)
 		if err != nil {
 			slog.Error("Failed to get pod from cache",
@@ -318,6 +337,25 @@ func (c *Controller) processEvent(ctx context.Context, event PodEvent) error {
 	return lastErr
 }
 
+// deploymentExists checks if a deployment exists in the cluster.
+func (c *Controller) deploymentExists(ctx context.Context, namespace, name string) bool {
+	_, err := c.clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return false
+		}
+		// On error, assume it exists to be safe
+		// (avoid false decommissions)
+		slog.Warn("Failed to check if deployment exists, assuming it does",
+			"namespace", namespace,
+			"deployment", name,
+			"error", err,
+		)
+		return true
+	}
+	return true
+}
+
 // recordContainer records a single container's deployment info.
 func (c *Controller) recordContainer(ctx context.Context, pod *corev1.Pod, container corev1.Container, status, eventType string) error {
 	dn := getARDeploymentName(pod, container, c.cfg.Template)