diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index d0c7b9627..b1acea67a 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -6,6 +6,7 @@ package main import ( + "cmp" "context" "encoding/json" "errors" @@ -464,6 +465,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl applyHealthzProbes(proxyClass, ss) } + if replaced, err := r.maybeReplaceStatefulSet(ctx, ss, logger); err != nil { + return r.notReadyErrf(pg, logger, "failed to replace StatefulSet: %w", err) + } else if replaced { + return notReady(reasonProxyGroupCreating, "waiting for StatefulSet to be replaced to update its Pod management policy") + } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels @@ -554,6 +561,36 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za return nil } +// maybeReplaceStatefulSet deletes the ProxyGroup's existing StatefulSet if its +// Pod management policy differs from the one in ss. Kubernetes doesn't allow +// that field to change on an existing StatefulSet, so it has to be deleted and +// created again. The Pods are orphaned rather than deleted, so they keep +// serving until a later reconcile creates the new StatefulSet, which adopts +// them. It reports whether the StatefulSet is being replaced. +func (r *ProxyGroupReconciler) maybeReplaceStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, logger *zap.SugaredLogger) (bool, error) { + existing := new(appsv1.StatefulSet) + err := r.Get(ctx, client.ObjectKeyFromObject(ss), existing) + switch { + case apierrors.IsNotFound(err): + return false, nil + case err != nil: + return false, fmt.Errorf("failed to get StatefulSet: %w", err) + case !existing.DeletionTimestamp.IsZero(): + return true, nil + } + // The API server defaults an unset policy to OrderedReady. + current := cmp.Or(existing.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement) + desired := cmp.Or(ss.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement) + if current == desired { + return false, nil + } + logger.Infof("Deleting StatefulSet %q, keeping its Pods, to change its Pod management policy from %q to %q; it is recreated on the next reconcile", existing.Name, current, desired) + if err := r.Delete(ctx, existing, client.PropagationPolicy(metav1.DeletePropagationOrphan)); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete StatefulSet: %w", err) + } + return true, nil +} + func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) { return staticendpoints.EnsureNodePortServices(ctx, r.Client, staticendpoints.Config{ Namespace: r.tsNamespace, diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 04d68ff39..4c1ff311b 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -89,6 +89,9 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string OwnerReferences: pgOwnerReference(pg), } ss.Spec.Replicas = new(pgReplicas(pg)) + // Start Pods in parallel so one that isn't ready yet, e.g. while its + // device awaits approval, doesn't hold back the others. + ss.Spec.PodManagementPolicy = appsv1.ParallelPodManagement ss.Spec.Selector = &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), } @@ -326,7 +329,8 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por OwnerReferences: pgOwnerReference(pg), }, Spec: appsv1.StatefulSetSpec{ - Replicas: new(pgReplicas(pg)), + Replicas: new(pgReplicas(pg)), + PodManagementPolicy: appsv1.ParallelPodManagement, Selector: &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), },