diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 1a6372733..9b9164f59 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1333,6 +1333,31 @@ func TestProxyGroupTypes(t *testing.T) { }) { t.Errorf("expected egress readiness gate %q to be set, got %v", tsEgressReadinessGate, sts.Spec.Template.Spec.ReadinessGates) } + + // Unlike legacy egress proxies, egress ProxyGroups only DNAT their mapped ports, so they get probes. + mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { + p.Spec.StatefulSet = &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ + LivenessProbe: &tsapi.Probe{}, + ReadinessProbe: &tsapi.Probe{}, + }, + }, + } + }) + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.Spec.ProxyClass = pc.Name + }) + expectReconciled(t, reconciler, "", pg.Name) + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + if diff := cmp.Diff(httpProbe("/livez", defaultLocalAddrPort, &tsapi.Probe{}, 5), sts.Spec.Template.Spec.Containers[0].LivenessProbe); diff != "" { + t.Errorf("unexpected liveness probe (-want +got):\n%s", diff) + } + if diff := cmp.Diff(httpProbe("/readyz", defaultLocalAddrPort, &tsapi.Probe{}, 1), sts.Spec.Template.Spec.Containers[0].ReadinessProbe); diff != "" { + t.Errorf("unexpected readiness probe (-want +got):\n%s", diff) + } }) t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) { pg := &tsapi.ProxyGroup{ @@ -1805,6 +1830,8 @@ func TestValidateProxyGroup(t *testing.T) { image string noauth bool initContainer bool + probes bool + env []tsapi.Env staticSAExists bool expectedErrs int } @@ -1883,6 +1910,35 @@ func TestValidateProxyGroup(t *testing.T) { typ: tsapi.ProxyGroupTypeEgress, initContainer: true, }, + "probes_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}}, + }, + "probes_with_loopback_local_addr_port": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}, + expectedErrs: 1, + }, + "probes_with_health_check_disabled": { + typ: tsapi.ProxyGroupTypeIngress, + probes: true, + env: []tsapi.Env{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}}, + expectedErrs: 1, + }, + "probes_with_deprecated_healthcheck": { + typ: tsapi.ProxyGroupTypeEgress, + probes: true, + env: []tsapi.Env{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}}, + expectedErrs: 1, + }, + "probes_ignored_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + probes: true, + env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}, + }, } { t.Run(name, func(t *testing.T) { pc := &tsapi.ProxyClass{ @@ -1900,6 +1956,12 @@ func TestValidateProxyGroup(t *testing.T) { Image: tc.image, } } + if tc.probes { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Env: tc.env, + ReadinessProbe: &tsapi.Probe{}, + } + } if tc.initContainer { pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} } @@ -2232,6 +2294,9 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + applyHealthzProbes(proxyClass, statefulSet) + } if shouldExist { expectEqual(t, fc, role) diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 35166d491..8b5409caf 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -316,6 +316,86 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) } +func Test_applyHealthzProbes(t *testing.T) { + withProbes := func(liveness, readiness *tsapi.Probe) *tsapi.ProxyClass { + return &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{TailscaleContainer: &tsapi.Container{LivenessProbe: liveness, ReadinessProbe: readiness}}, + }, + }, + } + } + both := withProbes(&tsapi.Probe{}, &tsapi.Probe{}) + for name, tt := range map[string]struct { + pc *tsapi.ProxyClass + env []corev1.EnvVar + wantLiveness bool + wantReadiness bool + wantPeriod int32 + wantPort int + }{ + "nil_proxyclass_defaults_off": {}, + "unset_probes_default_off": {pc: withProbes(nil, nil)}, + "liveness_set": {pc: withProbes(&tsapi.Probe{}, nil), wantLiveness: true, wantPort: defaultLocalAddrPort}, + "readiness_set": {pc: withProbes(nil, &tsapi.Probe{}), wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "both_set": {pc: both, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "custom_readiness_period": {pc: withProbes(nil, &tsapi.Probe{PeriodSeconds: 30}), wantReadiness: true, wantPeriod: 30, wantPort: defaultLocalAddrPort}, + "custom_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: 9010}, + "loopback_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}}, + "localhost_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "localhost:9002"}}}, + "health_check_disabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}}}, + "health_check_enabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "true"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort}, + "deprecated_healthcheck": {pc: both, env: []corev1.EnvVar{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}}}, + } { + t.Run(name, func(t *testing.T) { + var ss appsv1.StatefulSet + if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { + t.Fatalf("unmarshaling proxy template: %v", err) + } + ss.Spec.Template.Spec.Containers[0].Env = append(ss.Spec.Template.Spec.Containers[0].Env, tt.env...) + applyHealthzProbes(tt.pc, &ss) + c := ss.Spec.Template.Spec.Containers[0] + + if got := c.LivenessProbe != nil; got != tt.wantLiveness { + t.Errorf("liveness probe present = %v, want %v", got, tt.wantLiveness) + } + if got := c.ReadinessProbe != nil; got != tt.wantReadiness { + t.Errorf("readiness probe present = %v, want %v", got, tt.wantReadiness) + } + if c.LivenessProbe != nil { + if c.LivenessProbe.HTTPGet.Path != "/livez" { + t.Errorf("liveness path = %q, want /livez", c.LivenessProbe.HTTPGet.Path) + } + if got := c.LivenessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort { + t.Errorf("liveness port = %d, want %d", got, tt.wantPort) + } + if c.LivenessProbe.TimeoutSeconds != 5 { + t.Errorf("liveness timeout = %d, want 5", c.LivenessProbe.TimeoutSeconds) + } + } + if c.ReadinessProbe != nil { + if c.ReadinessProbe.HTTPGet.Path != "/readyz" { + t.Errorf("readiness path = %q, want /readyz", c.ReadinessProbe.HTTPGet.Path) + } + if got := c.ReadinessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort { + t.Errorf("readiness port = %d, want %d", got, tt.wantPort) + } + if c.ReadinessProbe.PeriodSeconds != tt.wantPeriod { + t.Errorf("readiness period = %d, want %d", c.ReadinessProbe.PeriodSeconds, tt.wantPeriod) + } + if c.ReadinessProbe.TimeoutSeconds != 1 { + t.Errorf("readiness timeout = %d, want 1", c.ReadinessProbe.TimeoutSeconds) + } + } + v, _ := lookupEnv(c.Env, "TS_ENABLE_HEALTH_CHECK") + if got, want := v == "true", tt.wantLiveness || tt.wantReadiness; got != want { + t.Errorf("TS_ENABLE_HEALTH_CHECK = %q, want enabled %v", v, want) + } + }) + } +} + func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { tests := []struct { name string diff --git a/k8s-operator/reconciler/proxyclass/proxyclass_test.go b/k8s-operator/reconciler/proxyclass/proxyclass_test.go index db0b10bd5..e1523a8de 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass_test.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass_test.go @@ -274,6 +274,26 @@ func TestProxyClassValidation(t *testing.T) { }, valid: false, }, + "probes_for_main_container": { + spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}}, + }, + }, + }, + valid: true, + }, + "probes_for_init_container": { + spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleInitContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}}, + }, + }, + }, + valid: false, + }, "static_endpoints_valid_specific_ports": { spec: tsapi.ProxyClassSpec{StaticEndpoints: nodePortWith([]tsapi.PortRange{{Port: 3001}, {Port: 3005}})}, valid: true,