From b76d891d987cb157399ddfbefa4946b6ee188f6f Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Fri, 9 Oct 2026 16:09:03 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator: test proxy liveness and readiness probes Covers the ProxyClass probe defaults and overrides, the env cases that skip or move the probes, probes on ProxyGroup proxies, and rejecting probes on the init container or with an env that leaves the endpoints unreachable. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/proxygroup_test.go | 65 +++++++++++++++ cmd/k8s-operator/sts_test.go | 80 +++++++++++++++++++ .../reconciler/proxyclass/proxyclass_test.go | 20 +++++ 3 files changed, 165 insertions(+) diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 1a6372733..9b9164f59 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1333,6 +1333,31 @@ func TestProxyGroupTypes(t *testing.T) { }) { t.Errorf("expected egress readiness gate %q to be set, got %v", tsEgressReadinessGate, sts.Spec.Template.Spec.ReadinessGates) } + + // Unlike legacy egress proxies, egress ProxyGroups only DNAT their mapped ports, so they get probes. + mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { + p.Spec.StatefulSet = &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ + LivenessProbe: &tsapi.Probe{}, + ReadinessProbe: &tsapi.Probe{}, + }, + }, + } + }) + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.Spec.ProxyClass = pc.Name + }) + expectReconciled(t, reconciler, "", pg.Name) + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + if diff := cmp.Diff(httpProbe("/livez", defaultLocalAddrPort, &tsapi.Probe{}, 5), sts.Spec.Template.Spec.Containers[0].LivenessProbe); diff != "" { + t.Errorf("unexpected liveness probe (-want +got):\n%s", diff) + } + if diff := cmp.Diff(httpProbe("/readyz", defaultLocalAddrPort, &tsapi.Probe{}, 1), sts.Spec.Template.Spec.Containers[0].ReadinessProbe); diff != "" { + t.Errorf("unexpected readiness probe (-want +got):\n%s", diff) + } }) t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) { pg := &tsapi.ProxyGroup{ @@ -1805,6 +1830,8 @@ func TestValidateProxyGroup(t *testing.T) { image string noauth bool initContainer bool + probes bool + env []tsapi.Env staticSAExists bool expectedErrs int } @@ -1883,6 +1910,35 @@ func TestValidateProxyGroup(t *testing.T) { typ: tsapi.ProxyGroupTypeEgress, initContainer: true, }, + "probes_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}}, + }, + "probes_with_loopback_local_addr_port": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}, + expectedErrs: 1, + }, + "probes_with_health_check_disabled": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}}, + expectedErrs: 1, + }, + "probes_with_deprecated_healthcheck": { + typ: tsapi.ProxyGroupTypeEgress, + probes: true, + env: []tsapi.Env{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}}, + expectedErrs: 1, + }, + "probes_ignored_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}, + }, } { t.Run(name, func(t *testing.T) { pc := &tsapi.ProxyClass{ @@ -1900,6 +1956,12 @@ func TestValidateProxyGroup(t *testing.T) { Image: tc.image, } } + if tc.probes { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Env: tc.env, + ReadinessProbe: &tsapi.Probe{}, + } + } if tc.initContainer { pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} } @@ -2232,6 +2294,9 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + applyHealthzProbes(proxyClass, statefulSet) + } if shouldExist { expectEqual(t, fc, role) diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 35166d491..8b5409caf 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -316,6 +316,86 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) } +func Test_applyHealthzProbes(t *testing.T) { + withProbes := func(liveness, readiness *tsapi.Probe) *tsapi.ProxyClass { + return &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{TailscaleContainer: &tsapi.Container{LivenessProbe: liveness, ReadinessProbe: readiness}}, + }, + }, + } + } + both := withProbes(&tsapi.Probe{}, &tsapi.Probe{}) + for name, tt := range map[string]struct { + pc *tsapi.ProxyClass + env []corev1.EnvVar + wantLiveness bool + wantReadiness bool + wantPeriod int32 + wantPort int + }{ + "nil_proxyclass_defaults_off": {}, + "unset_probes_default_off": {pc: withProbes(nil, nil)}, + "liveness_set": {pc: withProbes(&tsapi.Probe{}, nil), wantLiveness: true, wantPort: defaultLocalAddrPort}, + "readiness_set": {pc: withProbes(nil, &tsapi.Probe{}), wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "both_set": {pc: both, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "custom_readiness_period": {pc: withProbes(nil, &tsapi.Probe{PeriodSeconds: 30}), wantReadiness: true, wantPeriod: 30, wantPort: defaultLocalAddrPort}, + "custom_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: 9010}, + "loopback_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}}, + "localhost_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "localhost:9002"}}}, + "health_check_disabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}}}, + "health_check_enabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "true"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "deprecated_healthcheck": {pc: both, env: []corev1.EnvVar{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}}}, + } { + t.Run(name, func(t *testing.T) { + var ss appsv1.StatefulSet + if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { + t.Fatalf("unmarshaling proxy template: %v", err) + } + ss.Spec.Template.Spec.Containers[0].Env = append(ss.Spec.Template.Spec.Containers[0].Env, tt.env...) + applyHealthzProbes(tt.pc, &ss) + c := ss.Spec.Template.Spec.Containers[0] + + if got := c.LivenessProbe != nil; got != tt.wantLiveness { + t.Errorf("liveness probe present = %v, want %v", got, tt.wantLiveness) + } + if got := c.ReadinessProbe != nil; got != tt.wantReadiness { + t.Errorf("readiness probe present = %v, want %v", got, tt.wantReadiness) + } + if c.LivenessProbe != nil { + if c.LivenessProbe.HTTPGet.Path != "/livez" { + t.Errorf("liveness path = %q, want /livez", c.LivenessProbe.HTTPGet.Path) + } + if got := c.LivenessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort { + t.Errorf("liveness port = %d, want %d", got, tt.wantPort) + } + if c.LivenessProbe.TimeoutSeconds != 5 { + t.Errorf("liveness timeout = %d, want 5", c.LivenessProbe.TimeoutSeconds) + } + } + if c.ReadinessProbe != nil { + if c.ReadinessProbe.HTTPGet.Path != "/readyz" { + t.Errorf("readiness path = %q, want /readyz", c.ReadinessProbe.HTTPGet.Path) + } + if got := c.ReadinessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort { + t.Errorf("readiness port = %d, want %d", got, tt.wantPort) + } + if c.ReadinessProbe.PeriodSeconds != tt.wantPeriod { + t.Errorf("readiness period = %d, want %d", c.ReadinessProbe.PeriodSeconds, tt.wantPeriod) + } + if c.ReadinessProbe.TimeoutSeconds != 1 { + t.Errorf("readiness timeout = %d, want 1", c.ReadinessProbe.TimeoutSeconds) + } + } + v, _ := lookupEnv(c.Env, "TS_ENABLE_HEALTH_CHECK") + if got, want := v == "true", tt.wantLiveness || tt.wantReadiness; got != want { + t.Errorf("TS_ENABLE_HEALTH_CHECK = %q, want enabled %v", v, want) + } + }) + } +} + func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { tests := []struct { name string diff --git a/k8s-operator/reconciler/proxyclass/proxyclass_test.go b/k8s-operator/reconciler/proxyclass/proxyclass_test.go index db0b10bd5..e1523a8de 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass_test.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass_test.go @@ -274,6 +274,26 @@ func TestProxyClassValidation(t *testing.T) { }, valid: false, }, + "probes_for_main_container": { + spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}}, + }, + }, + }, + valid: true, + }, + "probes_for_init_container": { + spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleInitContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}}, + }, + }, + }, + valid: false, + }, "static_endpoints_valid_specific_ports": { spec: tsapi.ProxyClassSpec{StaticEndpoints: nodePortWith([]tsapi.PortRange{{Port: 3001}, {Port: 3005}})}, valid: true,