cmd/containerboot,kube/{certs,health}: test the /livez and /readyz checks

Covers the Probe handlers and conditions, AllCertsPresent with present,
missing, expired and unreadable certs, and the containerboot checks for
tailscaled failing or timing out, certs for advertised HTTPS-only Services and
a serve config file that doesn't exist.

Updates tailscale/corp#38175

Signed-off-by: chaosinthecrd <[email protected]>
This commit is contained in:
chaosinthecrd authored and Tom Meadows committed 2026-11-19 13:17:44 +00:00
1 parent b5fa6773ce
commit 6e8f0f40cc
4 files changed
+261

No files matched your search

+113
View File
@@ -0,0 +1,113 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"errors"
"strings"
"testing"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
"tailscale.com/kube/certs"
"tailscale.com/kube/localclient"
"tailscale.com/tailcfg"
)
type fakeHealthLocalClient struct {
status *ipnstate.Status
prefs *ipn.Prefs
serveConfig *ipn.ServeConfig
err error
}
func (f *fakeHealthLocalClient) StatusWithoutPeers(context.Context) (*ipnstate.Status, error) {
return f.status, f.err
}
func (f *fakeHealthLocalClient) GetPrefs(context.Context) (*ipn.Prefs, error) {
return f.prefs, f.err
}
func (f *fakeHealthLocalClient) GetServeConfig(context.Context) (*ipn.ServeConfig, error) {
return f.serveConfig, f.err
}
func TestTailscaledResponding(t *testing.T) {
for name, tt := range map[string]struct {
err error
wantErr string
}{
"responding": {},
"not_responding": {err: errors.New("connection refused"), wantErr: "failed to get status"},
"timed_out": {err: context.DeadlineExceeded, wantErr: "failed to get status"},
} {
t.Run(name, func(t *testing.T) {
lc := &fakeHealthLocalClient{status: &ipnstate.Status{}, err: tt.err}
checkErr(t, tailscaledResponding(lc)(t.Context()), tt.wantErr)
})
}
}
func TestCertsPresent(t *testing.T) {
const domain = "my-app.tailnetxyz.ts.net"
withHTTPS := &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
ipn.HostPort(domain + ":443"): {},
},
},
},
}
withHTTPAndHTTPS := &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
TCP: map[uint16]*ipn.TCPPortHandler{
80: {HTTP: true},
443: {HTTPS: true},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
ipn.HostPort(domain + ":80"): {},
ipn.HostPort(domain + ":443"): {},
},
},
},
}
validCert := map[string]*tailcfg.C2NTLSCertInfo{domain: {Valid: true}}
for name, tt := range map[string]struct {
serveConfig *ipn.ServeConfig
advertised []string
certs map[string]*tailcfg.C2NTLSCertInfo
wantErr string
}{
"advertised_no_cert": {serveConfig: withHTTPS, advertised: []string{"svc:my-app"}, wantErr: reasonCertsNotPresent},
"advertised_cert": {serveConfig: withHTTPS, advertised: []string{"svc:my-app"}, certs: validCert},
"not_advertised_no_cert": {serveConfig: withHTTPS},
"http_and_https_no_cert": {serveConfig: withHTTPAndHTTPS, advertised: []string{"svc:my-app"}},
"no_services": {serveConfig: &ipn.ServeConfig{}},
} {
t.Run(name, func(t *testing.T) {
cm := certs.NewCertManager(&localclient.FakeLocalClient{CertStatusResults: tt.certs}, t.Logf)
lc := &fakeHealthLocalClient{prefs: &ipn.Prefs{AdvertiseServices: tt.advertised}, serveConfig: tt.serveConfig}
checkErr(t, certsPresent(cm, lc)(t.Context()), tt.wantErr)
})
}
}
// checkErr checks that err is nil if wantErr is empty, or otherwise contains
// wantErr.
func checkErr(t *testing.T, err error, wantErr string) {
t.Helper()
switch {
case wantErr == "" && err != nil:
t.Errorf("unexpected error: %v", err)
case wantErr != "" && (err == nil || !strings.Contains(err.Error(), wantErr)):
t.Errorf("error = %v, want it to contain %q", err, wantErr)
}
}
+28
View File
@@ -66,6 +66,9 @@ func TestContainerBoot(t *testing.T) {
healthURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
}
readyzURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d/readyz", port)
}
egressSvcTerminateURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP)
}
@@ -1004,6 +1007,31 @@ func TestContainerBoot(t *testing.T) {
},
}
},
"readyz_without_serve_config_file": func(env *testEnv) testCase {
return testCase{
Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
"TS_ENABLE_HEALTH_CHECK": "true",
"TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/no-serve-config.json"),
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
EndpointStatuses: map[string]int{
readyzURL(env.localAddrPort): 503, // Doesn't start passing until the next phase.
},
}, {
Notify: runningNotify,
EndpointStatuses: map[string]int{
readyzURL(env.localAddrPort): 200,
},
},
},
}
},
"metrics_and_health_on_same_port": func(env *testEnv) testCase {
return testCase{
Env: map[string]string{
+46
View File
@@ -341,3 +341,49 @@ func TestNextRetryInterval(t *testing.T) {
})
}
}
// TestAllCertsPresent checks that AllCertsPresent only reports true when every
// HTTPS endpoint in the serve config has a cached cert, expired or not.
func TestAllCertsPresent(t *testing.T) {
webConfig := func(domains ...string) *ipn.ServeConfig {
web := make(map[ipn.HostPort]*ipn.WebServerConfig)
for _, d := range domains {
web[ipn.HostPort(d+":443")] = &ipn.WebServerConfig{}
}
return &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {Web: web},
},
}
}
valid := &tailcfg.C2NTLSCertInfo{Valid: true}
expired := &tailcfg.C2NTLSCertInfo{Expired: true, Error: "cert expired"}
unreadable := &tailcfg.C2NTLSCertInfo{Error: "failed to read Secret"}
for name, tt := range map[string]struct {
sc *ipn.ServeConfig
present map[string]*tailcfg.C2NTLSCertInfo
want bool
}{
"nil_config": {nil, nil, true},
"no_https_endpoints": {&ipn.ServeConfig{}, nil, true},
"single_present": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid}, true},
"single_missing": {webConfig("a.ts.net"), nil, false},
"one_of_two_missing": {webConfig("a.ts.net", "b.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid}, false},
"all_present": {webConfig("a.ts.net", "b.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid, "b.ts.net": valid}, true},
"expired": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": expired}, true},
"unreadable": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": unreadable}, true},
} {
t.Run(name, func(t *testing.T) {
cm := &CertManager{
lc: &localclient.FakeLocalClient{CertStatusResults: tt.present},
}
got, err := cm.AllCertsPresent(context.Background(), tt.sc)
if err != nil {
t.Fatalf("AllCertsPresent() error = %v", err)
}
if got != tt.want {
t.Errorf("AllCertsPresent() = %v, want %v", got, tt.want)
}
})
}
}
+74
View File
@@ -0,0 +1,74 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package health
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestProbe(t *testing.T) {
for name, tt := range map[string]struct {
conditions map[string]bool // name -> passing
checkErr error
wantCode int
wantBody string
}{
"no_checks": {wantCode: http.StatusOK},
"condition_not_set": {conditions: map[string]bool{"netmap": false}, wantCode: http.StatusServiceUnavailable, wantBody: "netmap: not ready"},
"condition_set": {conditions: map[string]bool{"netmap": true}, wantCode: http.StatusOK},
"one_condition_unset": {conditions: map[string]bool{"netmap": true, "serve": false}, wantCode: http.StatusServiceUnavailable, wantBody: "serve: not ready"},
"check_fails": {conditions: map[string]bool{"netmap": true}, checkErr: errors.New("no certs"), wantCode: http.StatusServiceUnavailable, wantBody: "certs: no certs"},
} {
t.Run(name, func(t *testing.T) {
p := &Probe{name: "test", logf: t.Logf}
for name, pass := range tt.conditions {
p.AddCondition(name, "not ready").Set(pass)
}
p.AddCheck("certs", func(context.Context) error { return tt.checkErr })
rec := httptest.NewRecorder()
p.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil))
if rec.Code != tt.wantCode {
t.Errorf("status = %d, want %d", rec.Code, tt.wantCode)
}
if !strings.Contains(rec.Body.String(), tt.wantBody) {
t.Errorf("body = %q, want it to contain %q", rec.Body.String(), tt.wantBody)
}
})
}
}
// TestProbeNil checks that a nil Probe and the nil Condition it returns can be
// used when health checks are disabled.
func TestProbeNil(t *testing.T) {
var p *Probe
p.AddCheck("certs", func(context.Context) error { return nil })
p.AddCondition("netmap", "not ready").Set(true)
}
func TestRegisterHandlers(t *testing.T) {
mux := http.NewServeMux()
lz := RegisterLivezHandlers(mux, t.Logf)
rz := RegisterReadyzHandlers(mux, t.Logf)
lz.AddCondition("tailscaled", "not live")
rz.AddCondition("netmap", "not ready")
for path, want := range map[string]string{
"/livez": "tailscaled: not live",
"/readyz": "netmap: not ready",
} {
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest("GET", path, nil))
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), want) {
t.Errorf("%s = %d %q, want 503 containing %q", path, rec.Code, rec.Body.String(), want)
}
}
}