mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
cmd/k8s-operator: start ProxyGroup Pods in parallel
With a readiness probe and the default OrderedReady policy, one unready ProxyGroup Pod stops the rest from starting, so ProxyGroup StatefulSets now use Parallel. The policy can't change in place, so an existing StatefulSet is deleted with its Pods orphaned and recreated, adopting them. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd <[email protected]>
This commit is contained in:
1 parent
b76d891d98
commit
1af2fa9eeb
2 files changed
+42
-1
No files matched your search
@@ -6,6 +6,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -464,6 +465,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl
|
||||
applyHealthzProbes(proxyClass, ss)
|
||||
}
|
||||
|
||||
if replaced, err := r.maybeReplaceStatefulSet(ctx, ss, logger); err != nil {
|
||||
return r.notReadyErrf(pg, logger, "failed to replace StatefulSet: %w", err)
|
||||
} else if replaced {
|
||||
return notReady(reasonProxyGroupCreating, "waiting for StatefulSet to be replaced to update its Pod management policy")
|
||||
}
|
||||
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
|
||||
s.Spec = ss.Spec
|
||||
s.ObjectMeta.Labels = ss.ObjectMeta.Labels
|
||||
@@ -554,6 +561,36 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybeReplaceStatefulSet deletes the ProxyGroup's existing StatefulSet if its
|
||||
// Pod management policy differs from the one in ss. Kubernetes doesn't allow
|
||||
// that field to change on an existing StatefulSet, so it has to be deleted and
|
||||
// created again. The Pods are orphaned rather than deleted, so they keep
|
||||
// serving until a later reconcile creates the new StatefulSet, which adopts
|
||||
// them. It reports whether the StatefulSet is being replaced.
|
||||
func (r *ProxyGroupReconciler) maybeReplaceStatefulSet(ctx context.Context, ss *appsv1.StatefulSet, logger *zap.SugaredLogger) (bool, error) {
|
||||
existing := new(appsv1.StatefulSet)
|
||||
err := r.Get(ctx, client.ObjectKeyFromObject(ss), existing)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, fmt.Errorf("failed to get StatefulSet: %w", err)
|
||||
case !existing.DeletionTimestamp.IsZero():
|
||||
return true, nil
|
||||
}
|
||||
// The API server defaults an unset policy to OrderedReady.
|
||||
current := cmp.Or(existing.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement)
|
||||
desired := cmp.Or(ss.Spec.PodManagementPolicy, appsv1.OrderedReadyPodManagement)
|
||||
if current == desired {
|
||||
return false, nil
|
||||
}
|
||||
logger.Infof("Deleting StatefulSet %q, keeping its Pods, to change its Pod management policy from %q to %q; it is recreated on the next reconcile", existing.Name, current, desired)
|
||||
if err := r.Delete(ctx, existing, client.PropagationPolicy(metav1.DeletePropagationOrphan)); err != nil && !apierrors.IsNotFound(err) {
|
||||
return false, fmt.Errorf("failed to delete StatefulSet: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) {
|
||||
return staticendpoints.EnsureNodePortServices(ctx, r.Client, staticendpoints.Config{
|
||||
Namespace: r.tsNamespace,
|
||||
|
||||
@@ -89,6 +89,9 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
|
||||
OwnerReferences: pgOwnerReference(pg),
|
||||
}
|
||||
ss.Spec.Replicas = new(pgReplicas(pg))
|
||||
// Start Pods in parallel so one that isn't ready yet, e.g. while its
|
||||
// device awaits approval, doesn't hold back the others.
|
||||
ss.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
|
||||
ss.Spec.Selector = &metav1.LabelSelector{
|
||||
MatchLabels: pgLabels(pg.Name, nil),
|
||||
}
|
||||
@@ -326,7 +329,8 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
|
||||
OwnerReferences: pgOwnerReference(pg),
|
||||
},
|
||||
Spec: appsv1.StatefulSetSpec{
|
||||
Replicas: new(pgReplicas(pg)),
|
||||
Replicas: new(pgReplicas(pg)),
|
||||
PodManagementPolicy: appsv1.ParallelPodManagement,
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: pgLabels(pg.Name, nil),
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user