Use optimistic concurrency and log retries

The Kubernetes clients provided by controller-runtime Manager fetch from
a cache. When fetching then writing back a single object, one should use
the object's resourceVersion to avoid races and lost updates.

Issue: [sc-16285]
See: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency

Author: cbandy

internal/bridge/installation.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	corev1apply "k8s.io/client-go/applyconfigurations/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -34,6 +35,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/crunchydata/postgres-operator/internal/controller/runtime"
+	"github.com/crunchydata/postgres-operator/internal/logging"
 	"github.com/crunchydata/postgres-operator/internal/naming"
 )
 
@@ -123,6 +125,12 @@ func (r *InstallationReconciler) Reconcile(
 	// TODO: Check for corev1.NamespaceTerminatingCause after
 	// k8s.io/apimachinery@v0.25; see https://issue.k8s.io/108528.
 
+	// Write conflicts are returned as errors; log and retry with backoff.
+	if err != nil && apierrors.IsConflict(err) {
+		logging.FromContext(ctx).Info("Requeue", "reason", err)
+		err, result.Requeue, result.RequeueAfter = nil, true, 0
+	}
+
 	return result, err
 }
 
@@ -132,6 +140,12 @@ func (r *InstallationReconciler) reconcile(ctx context.Context, read *corev1.Sec
 		return err
 	}
 
+	// We GET-extract-PATCH the Secret and do not build it up from scratch.
+	// Send the ResourceVersion from the GET in the body of every PATCH.
+	if len(read.ResourceVersion) != 0 {
+		write.WithResourceVersion(read.ResourceVersion)
+	}
+
 	// Read the Installation from the Secret, if any.
 	var installation Installation
 	if yaml.Unmarshal(read.Data[KeyBridgeToken], &installation) != nil {

internal/bridge/installation_test.go

@@ -64,6 +64,17 @@ func TestExtractSecretContract(t *testing.T) {
 			assert.Equal(t, *extracted.Name, "s2")
 		}
 	})
+
+	t.Run("ResourceVersion", func(t *testing.T) {
+		versioned := &corev1.Secret{}
+		versioned.ResourceVersion = "asdf"
+
+		extracted, err := corev1apply.ExtractSecret(versioned, "")
+		assert.NilError(t, err)
+
+		// ResourceVersion is not copied from the original.
+		assert.Assert(t, extracted.ResourceVersion == nil)
+	})
 }
 
 func TestInstallationReconcile(t *testing.T) {