mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
cmd/containerboot,kube/{certs,health}: test the /livez and /readyz checks
Covers the Probe handlers and conditions, AllCertsPresent with present, missing, expired and unreadable certs, and the containerboot checks for tailscaled failing or timing out, certs for advertised HTTPS-only Services and a serve config file that doesn't exist. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd <[email protected]>
This commit is contained in:
1 parent
b5fa6773ce
commit
6e8f0f40cc
4 files changed
+261
No files matched your search
@@ -0,0 +1,113 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"tailscale.com/ipn"
|
||||
"tailscale.com/ipn/ipnstate"
|
||||
"tailscale.com/kube/certs"
|
||||
"tailscale.com/kube/localclient"
|
||||
"tailscale.com/tailcfg"
|
||||
)
|
||||
|
||||
type fakeHealthLocalClient struct {
|
||||
status *ipnstate.Status
|
||||
prefs *ipn.Prefs
|
||||
serveConfig *ipn.ServeConfig
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeHealthLocalClient) StatusWithoutPeers(context.Context) (*ipnstate.Status, error) {
|
||||
return f.status, f.err
|
||||
}
|
||||
|
||||
func (f *fakeHealthLocalClient) GetPrefs(context.Context) (*ipn.Prefs, error) {
|
||||
return f.prefs, f.err
|
||||
}
|
||||
|
||||
func (f *fakeHealthLocalClient) GetServeConfig(context.Context) (*ipn.ServeConfig, error) {
|
||||
return f.serveConfig, f.err
|
||||
}
|
||||
|
||||
func TestTailscaledResponding(t *testing.T) {
|
||||
for name, tt := range map[string]struct {
|
||||
err error
|
||||
wantErr string
|
||||
}{
|
||||
"responding": {},
|
||||
"not_responding": {err: errors.New("connection refused"), wantErr: "failed to get status"},
|
||||
"timed_out": {err: context.DeadlineExceeded, wantErr: "failed to get status"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
lc := &fakeHealthLocalClient{status: &ipnstate.Status{}, err: tt.err}
|
||||
checkErr(t, tailscaledResponding(lc)(t.Context()), tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertsPresent(t *testing.T) {
|
||||
const domain = "my-app.tailnetxyz.ts.net"
|
||||
withHTTPS := &ipn.ServeConfig{
|
||||
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
|
||||
"svc:my-app": {
|
||||
Web: map[ipn.HostPort]*ipn.WebServerConfig{
|
||||
ipn.HostPort(domain + ":443"): {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
withHTTPAndHTTPS := &ipn.ServeConfig{
|
||||
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
|
||||
"svc:my-app": {
|
||||
TCP: map[uint16]*ipn.TCPPortHandler{
|
||||
80: {HTTP: true},
|
||||
443: {HTTPS: true},
|
||||
},
|
||||
Web: map[ipn.HostPort]*ipn.WebServerConfig{
|
||||
ipn.HostPort(domain + ":80"): {},
|
||||
ipn.HostPort(domain + ":443"): {},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
validCert := map[string]*tailcfg.C2NTLSCertInfo{domain: {Valid: true}}
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
serveConfig *ipn.ServeConfig
|
||||
advertised []string
|
||||
certs map[string]*tailcfg.C2NTLSCertInfo
|
||||
wantErr string
|
||||
}{
|
||||
"advertised_no_cert": {serveConfig: withHTTPS, advertised: []string{"svc:my-app"}, wantErr: reasonCertsNotPresent},
|
||||
"advertised_cert": {serveConfig: withHTTPS, advertised: []string{"svc:my-app"}, certs: validCert},
|
||||
"not_advertised_no_cert": {serveConfig: withHTTPS},
|
||||
"http_and_https_no_cert": {serveConfig: withHTTPAndHTTPS, advertised: []string{"svc:my-app"}},
|
||||
"no_services": {serveConfig: &ipn.ServeConfig{}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cm := certs.NewCertManager(&localclient.FakeLocalClient{CertStatusResults: tt.certs}, t.Logf)
|
||||
lc := &fakeHealthLocalClient{prefs: &ipn.Prefs{AdvertiseServices: tt.advertised}, serveConfig: tt.serveConfig}
|
||||
checkErr(t, certsPresent(cm, lc)(t.Context()), tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// checkErr checks that err is nil if wantErr is empty, or otherwise contains
|
||||
// wantErr.
|
||||
func checkErr(t *testing.T, err error, wantErr string) {
|
||||
t.Helper()
|
||||
switch {
|
||||
case wantErr == "" && err != nil:
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
case wantErr != "" && (err == nil || !strings.Contains(err.Error(), wantErr)):
|
||||
t.Errorf("error = %v, want it to contain %q", err, wantErr)
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,9 @@ func TestContainerBoot(t *testing.T) {
|
||||
healthURL := func(port int) string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
|
||||
}
|
||||
readyzURL := func(port int) string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d/readyz", port)
|
||||
}
|
||||
egressSvcTerminateURL := func(port int) string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d%s", port, kubetypes.EgessServicesPreshutdownEP)
|
||||
}
|
||||
@@ -1004,6 +1007,31 @@ func TestContainerBoot(t *testing.T) {
|
||||
},
|
||||
}
|
||||
},
|
||||
"readyz_without_serve_config_file": func(env *testEnv) testCase {
|
||||
return testCase{
|
||||
Env: map[string]string{
|
||||
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", env.localAddrPort),
|
||||
"TS_ENABLE_HEALTH_CHECK": "true",
|
||||
"TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/no-serve-config.json"),
|
||||
},
|
||||
Phases: []phase{
|
||||
{
|
||||
WantCmds: []string{
|
||||
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
|
||||
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
|
||||
},
|
||||
EndpointStatuses: map[string]int{
|
||||
readyzURL(env.localAddrPort): 503, // Doesn't start passing until the next phase.
|
||||
},
|
||||
}, {
|
||||
Notify: runningNotify,
|
||||
EndpointStatuses: map[string]int{
|
||||
readyzURL(env.localAddrPort): 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
"metrics_and_health_on_same_port": func(env *testEnv) testCase {
|
||||
return testCase{
|
||||
Env: map[string]string{
|
||||
|
||||
@@ -341,3 +341,49 @@ func TestNextRetryInterval(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllCertsPresent checks that AllCertsPresent only reports true when every
|
||||
// HTTPS endpoint in the serve config has a cached cert, expired or not.
|
||||
func TestAllCertsPresent(t *testing.T) {
|
||||
webConfig := func(domains ...string) *ipn.ServeConfig {
|
||||
web := make(map[ipn.HostPort]*ipn.WebServerConfig)
|
||||
for _, d := range domains {
|
||||
web[ipn.HostPort(d+":443")] = &ipn.WebServerConfig{}
|
||||
}
|
||||
return &ipn.ServeConfig{
|
||||
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
|
||||
"svc:my-app": {Web: web},
|
||||
},
|
||||
}
|
||||
}
|
||||
valid := &tailcfg.C2NTLSCertInfo{Valid: true}
|
||||
expired := &tailcfg.C2NTLSCertInfo{Expired: true, Error: "cert expired"}
|
||||
unreadable := &tailcfg.C2NTLSCertInfo{Error: "failed to read Secret"}
|
||||
for name, tt := range map[string]struct {
|
||||
sc *ipn.ServeConfig
|
||||
present map[string]*tailcfg.C2NTLSCertInfo
|
||||
want bool
|
||||
}{
|
||||
"nil_config": {nil, nil, true},
|
||||
"no_https_endpoints": {&ipn.ServeConfig{}, nil, true},
|
||||
"single_present": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid}, true},
|
||||
"single_missing": {webConfig("a.ts.net"), nil, false},
|
||||
"one_of_two_missing": {webConfig("a.ts.net", "b.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid}, false},
|
||||
"all_present": {webConfig("a.ts.net", "b.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": valid, "b.ts.net": valid}, true},
|
||||
"expired": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": expired}, true},
|
||||
"unreadable": {webConfig("a.ts.net"), map[string]*tailcfg.C2NTLSCertInfo{"a.ts.net": unreadable}, true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cm := &CertManager{
|
||||
lc: &localclient.FakeLocalClient{CertStatusResults: tt.present},
|
||||
}
|
||||
got, err := cm.AllCertsPresent(context.Background(), tt.sc)
|
||||
if err != nil {
|
||||
t.Fatalf("AllCertsPresent() error = %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("AllCertsPresent() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProbe(t *testing.T) {
|
||||
for name, tt := range map[string]struct {
|
||||
conditions map[string]bool // name -> passing
|
||||
checkErr error
|
||||
wantCode int
|
||||
wantBody string
|
||||
}{
|
||||
"no_checks": {wantCode: http.StatusOK},
|
||||
"condition_not_set": {conditions: map[string]bool{"netmap": false}, wantCode: http.StatusServiceUnavailable, wantBody: "netmap: not ready"},
|
||||
"condition_set": {conditions: map[string]bool{"netmap": true}, wantCode: http.StatusOK},
|
||||
"one_condition_unset": {conditions: map[string]bool{"netmap": true, "serve": false}, wantCode: http.StatusServiceUnavailable, wantBody: "serve: not ready"},
|
||||
"check_fails": {conditions: map[string]bool{"netmap": true}, checkErr: errors.New("no certs"), wantCode: http.StatusServiceUnavailable, wantBody: "certs: no certs"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
p := &Probe{name: "test", logf: t.Logf}
|
||||
for name, pass := range tt.conditions {
|
||||
p.AddCondition(name, "not ready").Set(pass)
|
||||
}
|
||||
p.AddCheck("certs", func(context.Context) error { return tt.checkErr })
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil))
|
||||
if rec.Code != tt.wantCode {
|
||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantCode)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), tt.wantBody) {
|
||||
t.Errorf("body = %q, want it to contain %q", rec.Body.String(), tt.wantBody)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestProbeNil checks that a nil Probe and the nil Condition it returns can be
|
||||
// used when health checks are disabled.
|
||||
func TestProbeNil(t *testing.T) {
|
||||
var p *Probe
|
||||
p.AddCheck("certs", func(context.Context) error { return nil })
|
||||
p.AddCondition("netmap", "not ready").Set(true)
|
||||
}
|
||||
|
||||
func TestRegisterHandlers(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
lz := RegisterLivezHandlers(mux, t.Logf)
|
||||
rz := RegisterReadyzHandlers(mux, t.Logf)
|
||||
lz.AddCondition("tailscaled", "not live")
|
||||
rz.AddCondition("netmap", "not ready")
|
||||
|
||||
for path, want := range map[string]string{
|
||||
"/livez": "tailscaled: not live",
|
||||
"/readyz": "netmap: not ready",
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("GET", path, nil))
|
||||
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), want) {
|
||||
t.Errorf("%s = %d %q, want 503 containing %q", path, rec.Code, rec.Body.String(), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user