mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
cmd/k8s-operator,k8s-operator: test proxy liveness and readiness probes
Covers the ProxyClass probe defaults and overrides, the env cases that skip or move the probes, probes on ProxyGroup proxies, and rejecting probes on the init container or with an env that leaves the endpoints unreachable. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd <[email protected]>
This commit is contained in:
1 parent
6308edfdce
commit
b76d891d98
3 files changed
+165
No files matched your search
@@ -1333,6 +1333,31 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
}) {
|
||||
t.Errorf("expected egress readiness gate %q to be set, got %v", tsEgressReadinessGate, sts.Spec.Template.Spec.ReadinessGates)
|
||||
}
|
||||
|
||||
// Unlike legacy egress proxies, egress ProxyGroups only DNAT their mapped ports, so they get probes.
|
||||
mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
|
||||
p.Spec.StatefulSet = &tsapi.StatefulSet{
|
||||
Pod: &tsapi.Pod{
|
||||
TailscaleContainer: &tsapi.Container{
|
||||
LivenessProbe: &tsapi.Probe{},
|
||||
ReadinessProbe: &tsapi.Probe{},
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
|
||||
p.Spec.ProxyClass = pc.Name
|
||||
})
|
||||
expectReconciled(t, reconciler, "", pg.Name)
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
|
||||
t.Fatalf("failed to get StatefulSet: %v", err)
|
||||
}
|
||||
if diff := cmp.Diff(httpProbe("/livez", defaultLocalAddrPort, &tsapi.Probe{}, 5), sts.Spec.Template.Spec.Containers[0].LivenessProbe); diff != "" {
|
||||
t.Errorf("unexpected liveness probe (-want +got):\n%s", diff)
|
||||
}
|
||||
if diff := cmp.Diff(httpProbe("/readyz", defaultLocalAddrPort, &tsapi.Probe{}, 1), sts.Spec.Template.Spec.Containers[0].ReadinessProbe); diff != "" {
|
||||
t.Errorf("unexpected readiness probe (-want +got):\n%s", diff)
|
||||
}
|
||||
})
|
||||
t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) {
|
||||
pg := &tsapi.ProxyGroup{
|
||||
@@ -1805,6 +1830,8 @@ func TestValidateProxyGroup(t *testing.T) {
|
||||
image string
|
||||
noauth bool
|
||||
initContainer bool
|
||||
probes bool
|
||||
env []tsapi.Env
|
||||
staticSAExists bool
|
||||
expectedErrs int
|
||||
}
|
||||
@@ -1883,6 +1910,35 @@ func TestValidateProxyGroup(t *testing.T) {
|
||||
typ: tsapi.ProxyGroupTypeEgress,
|
||||
initContainer: true,
|
||||
},
|
||||
"probes_for_ingress_pg": {
|
||||
typ: tsapi.ProxyGroupTypeIngress,
|
||||
probes: true,
|
||||
env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}},
|
||||
},
|
||||
"probes_with_loopback_local_addr_port": {
|
||||
typ: tsapi.ProxyGroupTypeIngress,
|
||||
probes: true,
|
||||
env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}},
|
||||
expectedErrs: 1,
|
||||
},
|
||||
"probes_with_health_check_disabled": {
|
||||
typ: tsapi.ProxyGroupTypeIngress,
|
||||
probes: true,
|
||||
env: []tsapi.Env{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}},
|
||||
expectedErrs: 1,
|
||||
},
|
||||
"probes_with_deprecated_healthcheck": {
|
||||
typ: tsapi.ProxyGroupTypeEgress,
|
||||
probes: true,
|
||||
env: []tsapi.Env{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}},
|
||||
expectedErrs: 1,
|
||||
},
|
||||
"probes_ignored_for_kube_pg": {
|
||||
typ: tsapi.ProxyGroupTypeKubernetesAPIServer,
|
||||
staticSAExists: true,
|
||||
probes: true,
|
||||
env: []tsapi.Env{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}},
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
pc := &tsapi.ProxyClass{
|
||||
@@ -1900,6 +1956,12 @@ func TestValidateProxyGroup(t *testing.T) {
|
||||
Image: tc.image,
|
||||
}
|
||||
}
|
||||
if tc.probes {
|
||||
pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{
|
||||
Env: tc.env,
|
||||
ReadinessProbe: &tsapi.Probe{},
|
||||
}
|
||||
}
|
||||
if tc.initContainer {
|
||||
pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{}
|
||||
}
|
||||
@@ -2232,6 +2294,9 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
|
||||
t.Fatal(err)
|
||||
}
|
||||
statefulSet.Annotations = defaultProxyClassAnnotations
|
||||
if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer {
|
||||
applyHealthzProbes(proxyClass, statefulSet)
|
||||
}
|
||||
|
||||
if shouldExist {
|
||||
expectEqual(t, fc, role)
|
||||
|
||||
@@ -316,6 +316,86 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
|
||||
verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
|
||||
}
|
||||
|
||||
func Test_applyHealthzProbes(t *testing.T) {
|
||||
withProbes := func(liveness, readiness *tsapi.Probe) *tsapi.ProxyClass {
|
||||
return &tsapi.ProxyClass{
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Pod: &tsapi.Pod{TailscaleContainer: &tsapi.Container{LivenessProbe: liveness, ReadinessProbe: readiness}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
both := withProbes(&tsapi.Probe{}, &tsapi.Probe{})
|
||||
for name, tt := range map[string]struct {
|
||||
pc *tsapi.ProxyClass
|
||||
env []corev1.EnvVar
|
||||
wantLiveness bool
|
||||
wantReadiness bool
|
||||
wantPeriod int32
|
||||
wantPort int
|
||||
}{
|
||||
"nil_proxyclass_defaults_off": {},
|
||||
"unset_probes_default_off": {pc: withProbes(nil, nil)},
|
||||
"liveness_set": {pc: withProbes(&tsapi.Probe{}, nil), wantLiveness: true, wantPort: defaultLocalAddrPort},
|
||||
"readiness_set": {pc: withProbes(nil, &tsapi.Probe{}), wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort},
|
||||
"both_set": {pc: both, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort},
|
||||
"custom_readiness_period": {pc: withProbes(nil, &tsapi.Probe{PeriodSeconds: 30}), wantReadiness: true, wantPeriod: 30, wantPort: defaultLocalAddrPort},
|
||||
"custom_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9010"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: 9010},
|
||||
"loopback_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "127.0.0.1:9002"}}},
|
||||
"localhost_local_addr_port": {pc: both, env: []corev1.EnvVar{{Name: "TS_LOCAL_ADDR_PORT", Value: "localhost:9002"}}},
|
||||
"health_check_disabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "false"}}},
|
||||
"health_check_enabled": {pc: both, env: []corev1.EnvVar{{Name: "TS_ENABLE_HEALTH_CHECK", Value: "true"}}, wantLiveness: true, wantReadiness: true, wantPeriod: 10, wantPort: defaultLocalAddrPort},
|
||||
"deprecated_healthcheck": {pc: both, env: []corev1.EnvVar{{Name: "TS_HEALTHCHECK_ADDR_PORT", Value: "[::]:8080"}}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var ss appsv1.StatefulSet
|
||||
if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
|
||||
t.Fatalf("unmarshaling proxy template: %v", err)
|
||||
}
|
||||
ss.Spec.Template.Spec.Containers[0].Env = append(ss.Spec.Template.Spec.Containers[0].Env, tt.env...)
|
||||
applyHealthzProbes(tt.pc, &ss)
|
||||
c := ss.Spec.Template.Spec.Containers[0]
|
||||
|
||||
if got := c.LivenessProbe != nil; got != tt.wantLiveness {
|
||||
t.Errorf("liveness probe present = %v, want %v", got, tt.wantLiveness)
|
||||
}
|
||||
if got := c.ReadinessProbe != nil; got != tt.wantReadiness {
|
||||
t.Errorf("readiness probe present = %v, want %v", got, tt.wantReadiness)
|
||||
}
|
||||
if c.LivenessProbe != nil {
|
||||
if c.LivenessProbe.HTTPGet.Path != "/livez" {
|
||||
t.Errorf("liveness path = %q, want /livez", c.LivenessProbe.HTTPGet.Path)
|
||||
}
|
||||
if got := c.LivenessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort {
|
||||
t.Errorf("liveness port = %d, want %d", got, tt.wantPort)
|
||||
}
|
||||
if c.LivenessProbe.TimeoutSeconds != 5 {
|
||||
t.Errorf("liveness timeout = %d, want 5", c.LivenessProbe.TimeoutSeconds)
|
||||
}
|
||||
}
|
||||
if c.ReadinessProbe != nil {
|
||||
if c.ReadinessProbe.HTTPGet.Path != "/readyz" {
|
||||
t.Errorf("readiness path = %q, want /readyz", c.ReadinessProbe.HTTPGet.Path)
|
||||
}
|
||||
if got := c.ReadinessProbe.HTTPGet.Port.IntValue(); got != tt.wantPort {
|
||||
t.Errorf("readiness port = %d, want %d", got, tt.wantPort)
|
||||
}
|
||||
if c.ReadinessProbe.PeriodSeconds != tt.wantPeriod {
|
||||
t.Errorf("readiness period = %d, want %d", c.ReadinessProbe.PeriodSeconds, tt.wantPeriod)
|
||||
}
|
||||
if c.ReadinessProbe.TimeoutSeconds != 1 {
|
||||
t.Errorf("readiness timeout = %d, want 1", c.ReadinessProbe.TimeoutSeconds)
|
||||
}
|
||||
}
|
||||
v, _ := lookupEnv(c.Env, "TS_ENABLE_HEALTH_CHECK")
|
||||
if got, want := v == "true", tt.wantLiveness || tt.wantReadiness; got != want {
|
||||
t.Errorf("TS_ENABLE_HEALTH_CHECK = %q, want enabled %v", v, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -274,6 +274,26 @@ func TestProxyClassValidation(t *testing.T) {
|
||||
},
|
||||
valid: false,
|
||||
},
|
||||
"probes_for_main_container": {
|
||||
spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Pod: &tsapi.Pod{
|
||||
TailscaleContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}},
|
||||
},
|
||||
},
|
||||
},
|
||||
valid: true,
|
||||
},
|
||||
"probes_for_init_container": {
|
||||
spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Pod: &tsapi.Pod{
|
||||
TailscaleInitContainer: &tsapi.Container{ReadinessProbe: &tsapi.Probe{}},
|
||||
},
|
||||
},
|
||||
},
|
||||
valid: false,
|
||||
},
|
||||
"static_endpoints_valid_specific_ports": {
|
||||
spec: tsapi.ProxyClassSpec{StaticEndpoints: nodePortWith([]tsapi.PortRange{{Port: 3001}, {Port: 3005}})},
|
||||
valid: true,
|
||||
|
||||
Reference in new issue
Block a user