From 1af2fa9eeb472003e574abf841a04e440a6b1c8a Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Fri, 9 Oct 2026 16:09:03 +0000 Subject: [PATCH] cmd/k8s-operator: start ProxyGroup Pods in parallel With a readiness probe and the default OrderedReady policy, one unready ProxyGroup Pod stops the rest from starting, so ProxyGroup StatefulSets now use Parallel. The policy can't change in place, so an existing StatefulSet is deleted with its Pods orphaned and recreated, adopting them. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/proxygroup.go | 37 ++++++++++++++++++++++++++++ cmd/k8s-operator/proxygroup_specs.go | 6 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index d0c7b9627..b1acea67a 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -6,6 +6,7 @@ package main import ( + "cmp" "context" "encoding/json" "errors" @@ -464,6 +465,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl applyHealthzProbes(proxyClass, ss) } + if replaced, err := r.maybeReplaceStatefulSet(ctx, ss, logger); err != nil { + return r.notReadyErrf(pg, logger, "failed to replace StatefulSet: %w", err) + } else if replaced { + return notReady(reasonProxyGroupCreating, "waiting for StatefulSet to be replaced to update its Pod management policy") + } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels @@ -554,6 +561,36 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za return nil } +// maybeReplaceStatefulSet deletes the ProxyGroup's existing StatefulSet if its +// Pod management policy differs from the one in ss. Kubernetes doesn't allow +// that field to change on an existing StatefulSet, so it has to be deleted and +// created again. The Pods are orphaned rather than deleted, so they keep +// serving until a later reconcile creates the new StatefulSet, which adopts +// them. It reports whether the StatefulSet is being replaced. +func (r *ProxyGroupReconciler) maybeReplaceStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, logger *zap.SugaredLogger) (bool, error) { + existing := new(appsv1.StatefulSet) + err := r.Get(ctx, client.ObjectKeyFromObject(ss), existing) + switch { + case apierrors.IsNotFound(err): + return false, nil + case err != nil: + return false, fmt.Errorf("failed to get StatefulSet: %w", err) + case !existing.DeletionTimestamp.IsZero(): + return true, nil + } + // The API server defaults an unset policy to OrderedReady. + current := cmp.Or(existing.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement) + desired := cmp.Or(ss.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement) + if current == desired { + return false, nil + } + logger.Infof("Deleting StatefulSet %q, keeping its Pods, to change its Pod management policy from %q to %q; it is recreated on the next reconcile", existing.Name, current, desired) + if err := r.Delete(ctx, existing, client.PropagationPolicy(metav1.DeletePropagationOrphan)); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete StatefulSet: %w", err) + } + return true, nil +} + func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) { return staticendpoints.EnsureNodePortServices(ctx, r.Client, staticendpoints.Config{ Namespace: r.tsNamespace, diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 04d68ff39..4c1ff311b 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -89,6 +89,9 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string OwnerReferences: pgOwnerReference(pg), } ss.Spec.Replicas = new(pgReplicas(pg)) + // Start Pods in parallel so one that isn't ready yet, e.g. while its + // device awaits approval, doesn't hold back the others. + ss.Spec.PodManagementPolicy = appsv1.ParallelPodManagement ss.Spec.Selector = &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), } @@ -326,7 +329,8 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por OwnerReferences: pgOwnerReference(pg), }, Spec: appsv1.StatefulSetSpec{ - Replicas: new(pgReplicas(pg)), + Replicas: new(pgReplicas(pg)), + PodManagementPolicy: appsv1.ParallelPodManagement, Selector: &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), },