diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml index 84432cd72..3dab7c25d 100644 --- a/.github/workflows/ssh-integrationtest.yml +++ b/.github/workflows/ssh-integrationtest.yml @@ -7,11 +7,23 @@ concurrency: cancel-in-progress: true on: + workflow_dispatch: pull_request: paths: - "ssh/**" - - "tempfork/gliderlabs/ssh/**" - - ".github/workflows/ssh-integrationtest" + # gliderssh and x/crypto/ssh bumps. + - "go.mod" + - "go.sum" + - "Makefile" + - ".github/workflows/ssh-integrationtest.yml" + schedule: + # Changes outside ssh/** (ipn/ipnlocal, util/osuser, envknob, + # cmd/tailscaled) can break Tailscale SSH without triggering the + # pull_request paths above, so also run on main daily around + # midnight UK time (cron is UTC; 00:13 GMT, 01:13 BST), so results + # are ready the next morning. Off the hour to avoid GitHub's :00 + # cron backlog. + - cron: "13 0 * * *" jobs: ssh-integrationtest: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 73a82d009..df7c99d68 100644 --- a/Makefile +++ b/Makefile @@ -135,14 +135,19 @@ publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to locati .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers - @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ - echo "Testing on ubuntu:focal, ubuntu:jammy, ubuntu:noble, alpine:latest (in parallel)" && \ - docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers & \ - docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers & \ - docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers & \ - docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers & \ - wait + @set -e; \ + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test; \ + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled; \ + bases="ubuntu:focal ubuntu:jammy ubuntu:noble alpine:latest"; \ + echo "Testing on $$bases (in parallel)"; \ + failed=$$(mktemp); \ + for base in $$bases; do \ + { docker build --build-arg="BASE=$$base" -t "ssh-$$(echo $$base | tr : -)" ssh/tailssh/testcontainers || echo $$base >>$$failed; } & \ + done; \ + wait; \ + if [ -s $$failed ]; then echo "FAILED on:" $$(cat $$failed); rm -f $$failed; exit 1; fi; \ + rm -f $$failed; \ + echo "PASS on $$bases" .PHONY: generate generate: ## Generate code diff --git a/cmd/tailscale/cli/ssh_test.go b/cmd/tailscale/cli/ssh_test.go new file mode 100644 index 000000000..5ac218e1d --- /dev/null +++ b/cmd/tailscale/cli/ssh_test.go @@ -0,0 +1,273 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "crypto/ed25519" + "crypto/rand" + "net" + "net/netip" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" + "tailscale.com/ipn/ipnstate" + "tailscale.com/types/key" +) + +// newSSHTestKey returns a new ed25519 SSH public key and its +// authorized_keys form, without the trailing newline. +func newSSHTestKey(t *testing.T) (ssh.PublicKey, string) { + t.Helper() + pub, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + t.Fatal(err) + } + return sshPub, strings.TrimSpace(string(ssh.MarshalAuthorizedKey(sshPub))) +} + +// sshTestStatus returns a Status containing peers, each under a new +// random node key. +func sshTestStatus(peers ...*ipnstate.PeerStatus) *ipnstate.Status { + st := &ipnstate.Status{Peer: map[key.NodePublic]*ipnstate.PeerStatus{}} + for _, ps := range peers { + st.Peer[key.NewNode().Public()] = ps + } + return st +} + +func TestGenKnownHosts(t *testing.T) { + k1, k1Str := newSSHTestKey(t) + k2, k2Str := newSSHTestKey(t) + _, k3Str := newSSHTestKey(t) + + foo := &ipnstate.PeerStatus{ + DNSName: "foo.example.ts.net.", + TailscaleIPs: []netip.Addr{ + netip.MustParseAddr("100.64.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0::1"), + }, + // Surrounding whitespace is trimmed, and a key containing a + // newline is dropped rather than injecting an extra line. + SSH_HostKeys: []string{" " + k1Str + "\n", k3Str + "\nevil.example.com " + k3Str}, + } + bar := &ipnstate.PeerStatus{ + DNSName: "bar.example.ts.net.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2")}, + SSH_HostKeys: []string{k2Str}, + } + noKeys := &ipnstate.PeerStatus{ + DNSName: "nokeys.example.ts.net.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.3")}, + } + st := sshTestStatus(foo, bar, noKeys) + + got := string(genKnownHosts(st)) + if got != string(genKnownHosts(st)) { + t.Fatal("genKnownHosts output is not deterministic") + } + if strings.Contains(got, "evil.example.com") || strings.Contains(got, k3Str) { + t.Errorf("genKnownHosts included a host key containing a newline:\n%s", got) + } + if strings.Contains(got, "nokeys") || strings.Contains(got, "100.64.0.3") { + t.Errorf("genKnownHosts included a peer without host keys:\n%s", got) + } + wantLines := []string{ + "foo.example.ts.net. " + k1Str, + "100.64.0.1 " + k1Str, + "fd7a:115c:a1e0::1 " + k1Str, + "bar.example.ts.net. " + k2Str, + "100.64.0.2 " + k2Str, + } + gotLines := strings.Split(strings.TrimSuffix(got, "\n"), "\n") + if len(gotLines) != len(wantLines) { + t.Fatalf("got %d lines, want %d:\n%s", len(gotLines), len(wantLines), got) + } + for _, want := range wantLines { + if !strings.Contains(got, want+"\n") { + t.Errorf("missing line %q in:\n%s", want, got) + } + } + + // Verify that OpenSSH-format parsing accepts the file and matches + // each name and address to the right key, using the same host + // strings runSSH passes to ssh(1). + khFile := filepath.Join(t.TempDir(), "known_hosts") + if err := os.WriteFile(khFile, []byte(got), 0600); err != nil { + t.Fatal(err) + } + check, err := knownhosts.New(khFile) + if err != nil { + t.Fatalf("knownhosts.New: %v", err) + } + fakeAddr := &net.TCPAddr{IP: net.IPv4(100, 64, 0, 1), Port: 22} + for _, tt := range []struct { + host string + key ssh.PublicKey + wantErr bool + }{ + {"foo.example.ts.net.", k1, false}, + {"100.64.0.1", k1, false}, + {"fd7a:115c:a1e0::1", k1, false}, + {"bar.example.ts.net.", k2, false}, + {"100.64.0.2", k2, false}, + {"foo.example.ts.net.", k2, true}, // wrong key + {"100.64.0.3", k1, true}, // unknown host + } { + err := check(net.JoinHostPort(tt.host, "22"), fakeAddr, tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("known_hosts check of %s: err=%v, wantErr=%v", tt.host, err, tt.wantErr) + } + } +} + +func TestWriteKnownHosts(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("relies on XDG_CONFIG_HOME controlling os.UserConfigDir") + } + confDir := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", confDir) + + _, k1Str := newSSHTestKey(t) + ps := &ipnstate.PeerStatus{ + DNSName: "foo.example.ts.net.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1")}, + SSH_HostKeys: []string{k1Str}, + } + st := sshTestStatus(ps) + + path, err := writeKnownHosts(st) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(confDir, "tailscale", "ssh_known_hosts"); path != want { + t.Errorf("path = %q, want %q", path, want) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != string(genKnownHosts(st)) { + t.Errorf("file contents = %q, want %q", got, genKnownHosts(st)) + } + + // A changed netmap host key replaces the stale file contents. + _, k2Str := newSSHTestKey(t) + ps.SSH_HostKeys = []string{k2Str} + if _, err := writeKnownHosts(st); err != nil { + t.Fatal(err) + } + got, err = os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(got), k1Str) || !strings.Contains(string(got), k2Str) { + t.Errorf("known_hosts not updated after host key change:\n%s", got) + } +} + +func TestPeerStatusFromArg(t *testing.T) { + foo := &ipnstate.PeerStatus{ + DNSName: "foo.example.ts.net.", + TailscaleIPs: []netip.Addr{ + netip.MustParseAddr("100.64.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0::1"), + }, + } + bar := &ipnstate.PeerStatus{ + DNSName: "bar.example.ts.net.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2")}, + } + st := sshTestStatus(foo, bar) + + tests := []struct { + arg string + want *ipnstate.PeerStatus // or nil for no match + }{ + {"foo", foo}, + {"FOO", foo}, + {"foo.example.ts.net", foo}, + {"foo.example.ts.net.", foo}, + {"Foo.Example.TS.net", foo}, + {"100.64.0.1", foo}, + {"fd7a:115c:a1e0::1", foo}, + {"bar", bar}, + {"100.64.0.2", bar}, + {"", nil}, + {"baz", nil}, + {"foo.example", nil}, + {"100.64.0.9", nil}, + {"example.ts.net", nil}, + } + for _, tt := range tests { + got, ok := peerStatusFromArg(st, tt.arg) + if ok != (tt.want != nil) || got != tt.want { + t.Errorf("peerStatusFromArg(%q) = %v, %v; want %v", tt.arg, got, ok, tt.want) + } + wantName := "" + if tt.want != nil { + wantName = tt.want.DNSName + } + if name, ok := nodeDNSNameFromArg(st, tt.arg); name != wantName || ok != (tt.want != nil) { + t.Errorf("nodeDNSNameFromArg(%q) = %q, %v; want %q", tt.arg, name, ok, wantName) + } + } +} + +func TestIPFromPeerStatus(t *testing.T) { + tests := []struct { + name string + ips []string + want string + wantOK bool + }{ + {"none", nil, "", false}, + {"v4_only", []string{"100.64.0.1"}, "100.64.0.1", true}, + {"v4_after_v6", []string{"fd7a:115c:a1e0::1", "100.64.0.1"}, "100.64.0.1", true}, + {"v6_only", []string{"fd7a:115c:a1e0::1"}, "fd7a:115c:a1e0::1", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ps := &ipnstate.PeerStatus{} + for _, s := range tt.ips { + ps.TailscaleIPs = append(ps.TailscaleIPs, netip.MustParseAddr(s)) + } + got, ok := ipFromPeerStatus(ps) + if got != tt.want || ok != tt.wantOK { + t.Errorf("got %q, %v; want %q, %v", got, ok, tt.want, tt.wantOK) + } + }) + } +} + +func TestIsSSHOverTailscale(t *testing.T) { + old := getSSHClientEnvVar + t.Cleanup(func() { getSSHClientEnvVar = old }) + + tests := []struct { + sshClient string + want bool + }{ + {"", false}, + {"100.64.0.1 51234 22", true}, + {"fd7a:115c:a1e0::1 51234 22", true}, + {"192.168.1.2 51234 22", false}, + {"100.64.0.1", false}, // no port fields + {"not-an-ip 51234 22", false}, + } + for _, tt := range tests { + getSSHClientEnvVar = func() string { return tt.sshClient } + if got := isSSHOverTailscale(); got != tt.want { + t.Errorf("SSH_CLIENT=%q: isSSHOverTailscale = %v, want %v", tt.sshClient, got, tt.want) + } + } +} diff --git a/ssh/tailssh/tailssh_integration_fwd_test.go b/ssh/tailssh/tailssh_integration_fwd_test.go new file mode 100644 index 000000000..bf6bb6f06 --- /dev/null +++ b/ssh/tailssh/tailssh_integration_fwd_test.go @@ -0,0 +1,903 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build integrationtest + +package tailssh + +import ( + "bufio" + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" + "tailscale.com/envknob" + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// This file tests TCP, Unix socket, and agent forwarding end to end +// through the in-process server and its per-connection ssh-monitor. + +// fwdSessionUser is the local user every forwarding test logs in as. +const fwdSessionUser = "testuser" + +// fwdTestTimeout bounds each blocking step of the forwarding tests. +const fwdTestTimeout = 15 * time.Second + +// fwdTestClient starts a server configured by opts and returns a client +// connected to it as [fwdSessionUser]. The client is closed after a minute +// so that a stuck forwarding op fails the test instead of hanging it. +func fwdTestClient(t *testing.T, opts testServerOpts) *ssh.Client { + t.Helper() + opts.username = fwdSessionUser + addr := testServerWithOpts(t, opts) + cl, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{ + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: fwdTestTimeout, + }) + if err != nil { + t.Fatal(err) + } + watchdog := time.AfterFunc(time.Minute, func() { cl.Close() }) + t.Cleanup(func() { + watchdog.Stop() + cl.Close() + }) + return cl +} + +// fwdSessionUserIDs returns the uid and gid of [fwdSessionUser]. +func fwdSessionUserIDs(t *testing.T) (uid, gid int) { + t.Helper() + u, err := user.Lookup(fwdSessionUser) + if err != nil { + t.Fatal(err) + } + uid, err = strconv.Atoi(u.Uid) + if err != nil { + t.Fatal(err) + } + gid, err = strconv.Atoi(u.Gid) + if err != nil { + t.Fatal(err) + } + return uid, gid +} + +// fwdSocketDir returns a new directory under /tmp, one of the allowed +// forwarding socket directories, that [fwdSessionUser] can create and +// remove sockets in. +func fwdSocketDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "tailssh-fwd-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + if err := os.Chmod(dir, 0o777); err != nil { + t.Fatal(err) + } + return dir +} + +// serveEcho accepts connections on ln until it is closed and echoes +// back everything each connection sends. +func serveEcho(ln net.Listener) { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + io.Copy(c, c) + }() + } +} + +// startTCPEcho starts a loopback TCP echo server in the test process, +// standing in for a service on the SSH server's host. +func startTCPEcho(t *testing.T) net.Listener { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go serveEcho(ln) + return ln +} + +// checkEcho sends n random bytes on c, which must lead to an echo +// server, while concurrently reading them back. It closes c when done. +func checkEcho(t *testing.T, c net.Conn, n int) { + t.Helper() + defer c.Close() + want := make([]byte, n) + rand.Read(want) + writeErr := make(chan error, 1) + go func() { + _, err := c.Write(want) + writeErr <- err + }() + got := make([]byte, n) + readErr := make(chan error, 1) + go func() { + _, err := io.ReadFull(c, got) + readErr <- err + }() + for range 2 { + select { + case err := <-writeErr: + if err != nil { + t.Fatalf("write: %v", err) + } + case err := <-readErr: + if err != nil { + t.Fatalf("read: %v", err) + } + case <-time.After(fwdTestTimeout): + t.Fatal("timed out exchanging data through the forward") + } + } + if !bytes.Equal(got, want) { + t.Fatalf("echoed data differs from sent data (%d bytes)", n) + } +} + +// readAllTimeout reads c to EOF, bounded by [fwdTestTimeout]. SSH +// channel conns don't support deadlines, so the read runs in a +// goroutine. +func readAllTimeout(t *testing.T, c io.Reader) string { + t.Helper() + type result struct { + b []byte + err error + } + done := make(chan result, 1) + go func() { + b, err := io.ReadAll(c) + done <- result{b, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("read: %v", r.err) + } + return string(r.b) + case <-time.After(fwdTestTimeout): + t.Fatal("timed out reading to EOF") + return "" + } +} + +// waitConnRefused waits for network address addr to stop accepting +// connections. The listener's last descriptor copies are released +// asynchronously by the monitor and its one-shot child, so this polls. +func waitConnRefused(t *testing.T, network, addr string) { + t.Helper() + err := tstest.WaitFor(fwdTestTimeout, func() error { + c, err := net.DialTimeout(network, addr, time.Second) + if err != nil { + return nil + } + c.Close() + return fmt.Errorf("%s %s still accepts connections", network, addr) + }) + if err != nil { + t.Fatal(err) + } +} + +// TestIntegrationTCPLocalForwarding tests direct-tcpip (ssh -L): the +// client opens a channel to a TCP service on the server's host, and the +// server dials it and proxies data in both directions. +func TestIntegrationTCPLocalForwarding(t *testing.T) { + echo := startTCPEcho(t) + cl := fwdTestClient(t, testServerOpts{allowLocalPortForwarding: true}) + + for _, n := range []int{1, 256 << 10} { + c, err := cl.Dial("tcp", echo.Addr().String()) + if err != nil { + t.Fatalf("direct-tcpip to %v: %v", echo.Addr(), err) + } + checkEcho(t, c, n) + } +} + +// TestIntegrationTCPRemoteForwarding tests tcpip-forward (ssh -R) with +// an allocated port, connections from the server's host back to the +// client, cancel-tcpip-forward, and the loopback-only bind policy. +func TestIntegrationTCPRemoteForwarding(t *testing.T) { + cl := fwdTestClient(t, testServerOpts{allowRemotePortForwarding: true}) + + // Port 0 asks the server to allocate a port and report it back. + ln, err := cl.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("tcpip-forward: %v", err) + } + defer ln.Close() + addr := ln.Addr().String() + if port := ln.Addr().(*net.TCPAddr).Port; port == 0 { + t.Fatalf("tcpip-forward returned port 0, want the allocated port") + } + go serveEcho(ln) + + // A local process on the server's host connects to the forwarded + // port and reaches the client's echo service. + for _, n := range []int{1, 256 << 10} { + c, err := net.DialTimeout("tcp", addr, fwdTestTimeout) + if err != nil { + t.Fatalf("dialing forwarded port %s: %v", addr, err) + } + checkEcho(t, c, n) + } + + // Closing the client's listener sends cancel-tcpip-forward, after + // which the server must stop listening on the port. + if err := ln.Close(); err != nil { + t.Fatalf("cancel-tcpip-forward: %v", err) + } + waitConnRefused(t, "tcp", addr) + + // Without TS_SSH_FORWARD_GATEWAY_PORTS, only loopback binds are + // permitted. + if ln, err := cl.Listen("tcp", "0.0.0.0:0"); err == nil { + ln.Close() + t.Errorf("tcpip-forward on 0.0.0.0 succeeded; want refusal of non-loopback bind") + } +} + +// TestIntegrationTCPForwardingDenied checks that the policy's +// AllowLocalPortForwarding and AllowRemotePortForwarding bits each +// gate only their own direction of TCP forwarding. +func TestIntegrationTCPForwardingDenied(t *testing.T) { + echo := startTCPEcho(t) + tests := []struct { + name string + local, remote bool + }{ + {"none", false, false}, + {"local_only", true, false}, + {"remote_only", false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cl := fwdTestClient(t, testServerOpts{ + allowLocalPortForwarding: tt.local, + allowRemotePortForwarding: tt.remote, + }) + + c, err := cl.Dial("tcp", echo.Addr().String()) + if tt.local { + if err != nil { + t.Fatalf("permitted direct-tcpip failed: %v", err) + } + checkEcho(t, c, 1) + } else { + if err == nil { + c.Close() + t.Fatal("direct-tcpip succeeded; want rejection") + } + var oce *ssh.OpenChannelError + if !errors.As(err, &oce) || oce.Reason != ssh.Prohibited { + t.Errorf("direct-tcpip error = %v; want administratively prohibited", err) + } + } + + ln, err := cl.Listen("tcp", "127.0.0.1:0") + if tt.remote { + if err != nil { + t.Fatalf("permitted tcpip-forward failed: %v", err) + } + ln.Close() + } else if err == nil { + ln.Close() + t.Fatal("tcpip-forward succeeded; want rejection") + } + }) + } +} + +// halfCloseRequest is what the requesting side of a half-close exchange +// sends before closing its write side. +const halfCloseRequest = "request-then-close-write" + +// halfCloseResponse is what the responding side sends only after it has +// read the request to EOF. +const halfCloseResponse = "response-after-eof" + +// respondAfterEOF is the responding side of a half-close exchange on c: +// it reads c to EOF, reports what it read to got, then writes +// [halfCloseResponse] and closes c. The response can only reach the +// requester if the forward propagated the requester's EOF without +// tearing down the other direction. +func respondAfterEOF(c net.Conn, got chan<- string) { + defer c.Close() + b, _ := io.ReadAll(c) + got <- string(b) + io.WriteString(c, halfCloseResponse) +} + +// requestWithHalfClose is the requesting side of a half-close exchange +// on c: it writes [halfCloseRequest], closes its write side, and checks +// that [halfCloseResponse] arrives afterward. It also checks the +// request the responder reported to got. +func requestWithHalfClose(t *testing.T, c net.Conn, got <-chan string) { + t.Helper() + defer c.Close() + if _, err := io.WriteString(c, halfCloseRequest); err != nil { + t.Fatalf("write: %v", err) + } + cw, ok := c.(interface{ CloseWrite() error }) + if !ok { + t.Fatalf("%T does not implement CloseWrite", c) + } + if err := cw.CloseWrite(); err != nil { + t.Fatalf("CloseWrite: %v", err) + } + if resp := readAllTimeout(t, c); resp != halfCloseResponse { + t.Errorf("response = %q; want %q", resp, halfCloseResponse) + } + select { + case req := <-got: + if req != halfCloseRequest { + t.Errorf("responder read %q; want %q", req, halfCloseRequest) + } + case <-time.After(fwdTestTimeout): + t.Fatal("responder never reported its request") + } +} + +// TestIntegrationTCPForwardingHalfClose checks that an EOF from one end +// of a TCP forward reaches the other end while the reverse direction +// keeps flowing, for both ssh -L and ssh -R. +func TestIntegrationTCPForwardingHalfClose(t *testing.T) { + t.Run("local", func(t *testing.T) { + svc, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer svc.Close() + got := make(chan string, 1) + go func() { + c, err := svc.Accept() + if err == nil { + respondAfterEOF(c, got) + } + }() + + cl := fwdTestClient(t, testServerOpts{allowLocalPortForwarding: true}) + c, err := cl.Dial("tcp", svc.Addr().String()) + if err != nil { + t.Fatalf("direct-tcpip: %v", err) + } + requestWithHalfClose(t, c, got) + }) + + t.Run("remote", func(t *testing.T) { + cl := fwdTestClient(t, testServerOpts{allowRemotePortForwarding: true}) + ln, err := cl.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("tcpip-forward: %v", err) + } + defer ln.Close() + got := make(chan string, 1) + go func() { + c, err := ln.Accept() + if err == nil { + respondAfterEOF(c, got) + } + }() + + c, err := net.DialTimeout("tcp", ln.Addr().String(), fwdTestTimeout) + if err != nil { + t.Fatalf("dialing forwarded port: %v", err) + } + requestWithHalfClose(t, c, got) + }) +} + +// TestIntegrationUnixReverseForwardingCancel tests +// cancel-streamlocal-forward@openssh.com: after the client closes its +// listener, the server's socket stops accepting connections, and the +// same path can be forwarded again. +func TestIntegrationUnixReverseForwardingCancel(t *testing.T) { + cl := fwdTestClient(t, testServerOpts{allowRemotePortForwarding: true}) + sockPath := filepath.Join(fwdSocketDir(t), "cancel.sock") + + for i := range 2 { + ln, err := cl.ListenUnix(sockPath) + if err != nil { + t.Fatalf("streamlocal-forward #%d: %v", i, err) + } + go serveEcho(ln) + c, err := net.DialTimeout("unix", sockPath, fwdTestTimeout) + if err != nil { + t.Fatalf("dialing forwarded socket #%d: %v", i, err) + } + checkEcho(t, c, 1<<10) + + // Closing the client's listener sends + // cancel-streamlocal-forward@openssh.com. The server leaves the + // socket file in place (see [conn.fwdListenUnix]), but nothing + // listens on it. The next iteration's bind reclaims it. + if err := ln.Close(); err != nil { + t.Fatalf("cancel-streamlocal-forward #%d: %v", i, err) + } + waitConnRefused(t, "unix", sockPath) + } +} + +// TestIntegrationUnixReverseForwardingStaleSocket tests the server's +// unlink-if-owned BindUnlink policy for streamlocal-forward: an existing +// socket file owned by the session user is replaced, while a socket +// owned by someone else, or a non-socket file, is left untouched and +// the forward fails. +func TestIntegrationUnixReverseForwardingStaleSocket(t *testing.T) { + uid, gid := fwdSessionUserIDs(t) + + // makeStaleSocket leaves a socket file with no listener at path. + makeStaleSocket := func(t *testing.T, path string) { + t.Helper() + ln, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) + if err != nil { + t.Fatal(err) + } + ln.SetUnlinkOnClose(false) + ln.Close() + } + // chownToSessionUser makes path owned by [fwdSessionUser], which it + // already is when the test runs as that user. + chownToSessionUser := func(t *testing.T, path string) { + t.Helper() + if os.Getuid() == uid { + return + } + if err := os.Lchown(path, uid, gid); err != nil { + t.Fatal(err) + } + } + + tests := []struct { + name string + setup func(t *testing.T, path string) + wantBind bool + wantType fs.FileMode // file type that must remain after a refused bind + }{ + { + name: "own_stale_socket", + setup: func(t *testing.T, path string) { + makeStaleSocket(t, path) + chownToSessionUser(t, path) + }, + wantBind: true, + }, + { + name: "foreign_stale_socket", + setup: func(t *testing.T, path string) { + if os.Getuid() != 0 { + t.Skip("creating a socket owned by another user requires root") + } + makeStaleSocket(t, path) // owned by root + }, + wantType: fs.ModeSocket, + }, + { + name: "regular_file", + setup: func(t *testing.T, path string) { + if err := os.WriteFile(path, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + chownToSessionUser(t, path) + }, + wantType: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sockPath := filepath.Join(fwdSocketDir(t), "stale.sock") + tt.setup(t, sockPath) + before, err := os.Lstat(sockPath) + if err != nil { + t.Fatal(err) + } + + cl := fwdTestClient(t, testServerOpts{allowRemotePortForwarding: true}) + ln, err := cl.ListenUnix(sockPath) + if !tt.wantBind { + if err == nil { + ln.Close() + t.Fatal("streamlocal-forward over an existing file succeeded; want refusal") + } + after, err := os.Lstat(sockPath) + if err != nil { + t.Fatalf("existing file removed by refused forward: %v", err) + } + if after.Mode().Type() != tt.wantType || !os.SameFile(before, after) { + t.Errorf("existing file replaced by refused forward: mode %v -> %v", before.Mode(), after.Mode()) + } + return + } + if err != nil { + t.Fatalf("streamlocal-forward over own stale socket: %v", err) + } + defer ln.Close() + // A connection reaching the client proves the stale socket was + // replaced, since nothing listens on the old one. Inode numbers + // are reused too quickly to compare. + go serveEcho(ln) + c, err := net.DialTimeout("unix", sockPath, fwdTestTimeout) + if err != nil { + t.Fatalf("dialing forwarded socket: %v", err) + } + checkEcho(t, c, 1<<10) + }) + } +} + +// startAgentSession forwards keyring over cl, starts a session that +// requests agent forwarding and stays open until the returned end func +// is called, and returns the session's SSH_AUTH_SOCK ("" if unset). +func startAgentSession(t *testing.T, cl *ssh.Client, keyring agent.Agent) (sock string, end func()) { + t.Helper() + if err := agent.ForwardToAgent(cl, keyring); err != nil { + t.Fatal(err) + } + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + // The server accepts the request even when it won't forward the + // agent; the policy is enforced by not creating the socket. + if err := agent.RequestAgentForwarding(s); err != nil { + t.Fatalf("requesting agent forwarding: %v", err) + } + stdin, err := s.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := s.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := s.Start(`echo "SSH_AUTH_SOCK=${SSH_AUTH_SOCK:-}"; read x`); err != nil { + t.Fatal(err) + } + line := make(chan string, 1) + go func() { + sc := bufio.NewScanner(stdout) + for sc.Scan() { + if v, ok := strings.CutPrefix(sc.Text(), "SSH_AUTH_SOCK="); ok { + line <- v + break + } + } + io.Copy(io.Discard, stdout) + }() + select { + case sock = <-line: + case <-time.After(fwdTestTimeout): + t.Fatal("timed out waiting for the session's SSH_AUTH_SOCK") + } + return sock, func() { + stdin.Close() + s.Wait() + } +} + +// newTestKeyring returns an agent keyring holding one new key. +func newTestKeyring(t *testing.T) agent.Agent { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + keyring := agent.NewKeyring() + if err := keyring.Add(agent.AddedKey{PrivateKey: priv}); err != nil { + t.Fatal(err) + } + return keyring +} + +// TestIntegrationAgentForwardingSocket checks the agent socket the +// server creates for ssh -A: it is a live connection to the client's +// agent, owned by the session user with mode 0600 inside a 0700 +// user-owned directory, and the directory is removed when the session +// ends. With AllowAgentForwarding off, the session gets no socket. +func TestIntegrationAgentForwardingSocket(t *testing.T) { + t.Run("allowed", func(t *testing.T) { + uid, _ := fwdSessionUserIDs(t) + cl := fwdTestClient(t, testServerOpts{}) + sock, end := startAgentSession(t, cl, newTestKeyring(t)) + if sock == "" { + t.Fatal("SSH_AUTH_SOCK unset in session with agent forwarding allowed") + } + dir := filepath.Dir(sock) + for _, f := range []struct { + path string + wantType fs.FileMode + wantPerm fs.FileMode + }{ + {sock, fs.ModeSocket, 0o600}, + {dir, fs.ModeDir, 0o700}, + } { + fi, err := os.Lstat(f.path) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Type(); got != f.wantType { + t.Errorf("%s: type %v; want %v", f.path, got, f.wantType) + } + if got := fi.Mode().Perm(); got != f.wantPerm { + t.Errorf("%s: mode %04o; want %04o", f.path, got, f.wantPerm) + } + if got := fi.Sys().(*syscall.Stat_t).Uid; int(got) != uid { + t.Errorf("%s: owned by uid %d; want %s's uid %d", f.path, got, fwdSessionUser, uid) + } + } + + c, err := net.DialTimeout("unix", sock, fwdTestTimeout) + if err != nil { + t.Fatalf("dialing agent socket: %v", err) + } + keys, err := agent.NewClient(c).List() + c.Close() + if err != nil { + t.Fatalf("listing keys through forwarded agent: %v", err) + } + if len(keys) != 1 { + t.Errorf("forwarded agent has %d keys; want 1", len(keys)) + } + + end() + err = tstest.WaitFor(fwdTestTimeout, func() error { + if _, err := os.Lstat(dir); !os.IsNotExist(err) { + return fmt.Errorf("agent dir %s still present after session end (err=%v)", dir, err) + } + return nil + }) + if err != nil { + t.Error(err) + } + }) + + t.Run("denied", func(t *testing.T) { + cl := fwdTestClient(t, testServerOpts{ + modifyRule: func(r *tailcfg.SSHRule) { r.Action.AllowAgentForwarding = false }, + }) + sock, end := startAgentSession(t, cl, newTestKeyring(t)) + defer end() + if sock != "" { + t.Errorf("SSH_AUTH_SOCK = %q with AllowAgentForwarding=false; want unset", sock) + } + }) +} + +// TestIntegrationForwardingDisabledByEnvknob checks that +// TS_SSH_DISABLE_FORWARDING refuses TCP, Unix socket, and agent +// forwarding even when the policy allows all of them. +func TestIntegrationForwardingDisabledByEnvknob(t *testing.T) { + envknob.SetenvForTest(t, "TS_SSH_DISABLE_FORWARDING", "true") + + echo := startTCPEcho(t) + dir := fwdSocketDir(t) + unixSvcPath := filepath.Join(dir, "svc.sock") + unixSvc, err := net.Listen("unix", unixSvcPath) + if err != nil { + t.Fatal(err) + } + defer unixSvc.Close() + if err := os.Chmod(unixSvcPath, 0o666); err != nil { + t.Fatal(err) + } + go serveEcho(unixSvc) + + cl := fwdTestClient(t, testServerOpts{ + allowLocalPortForwarding: true, + allowRemotePortForwarding: true, + }) + + if c, err := cl.Dial("tcp", echo.Addr().String()); err == nil { + c.Close() + t.Error("TCP local forward succeeded") + } + if ln, err := cl.Listen("tcp", "127.0.0.1:0"); err == nil { + ln.Close() + t.Error("TCP remote forward succeeded") + } + if c, err := cl.Dial("unix", unixSvcPath); err == nil { + c.Close() + t.Error("Unix local forward succeeded") + } + if ln, err := cl.ListenUnix(filepath.Join(dir, "reverse.sock")); err == nil { + ln.Close() + t.Error("Unix remote forward succeeded") + } + sock, end := startAgentSession(t, cl, newTestKeyring(t)) + defer end() + if sock != "" { + t.Errorf("SSH_AUTH_SOCK = %q; want unset", sock) + } +} + +// openSSHProc is a system ssh client started by [startOpenSSH]. +type openSSHProc struct { + lines chan string // stderr lines; sends are dropped when full + done chan struct{} // closed when the client has exited + err error // exit error, set before done is closed + stderr lockedBuffer // all stderr output +} + +// startOpenSSH starts the system ssh client at sshPath with -N against +// the server at addr, logging in as [fwdSessionUser] with "none" auth, +// with the given extra arguments. The client is killed at test cleanup. +func startOpenSSH(t *testing.T, sshPath, addr string, args ...string) *openSSHProc { + t.Helper() + host, port, err := net.SplitHostPort(addr) + if err != nil { + t.Fatal(err) + } + argv := []string{ + "-F", "/dev/null", + "-N", "-T", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=15", + "-o", "ExitOnForwardFailure=yes", + "-o", "GSSAPIAuthentication=no", + "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "HostbasedAuthentication=no", + "-o", "IdentityAgent=none", + "-o", "KbdInteractiveAuthentication=no", + "-o", "NumberOfPasswordPrompts=0", + "-o", "PasswordAuthentication=no", + "-o", "PreferredAuthentications=none", + "-o", "PubkeyAuthentication=no", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-p", port, + } + argv = append(argv, args...) + argv = append(argv, fwdSessionUser+"@"+host) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + cmd := exec.CommandContext(ctx, sshPath, argv...) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + p := &openSSHProc{ + lines: make(chan string, 64), + done: make(chan struct{}), + } + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + fmt.Fprintln(&p.stderr, sc.Text()) + select { + case p.lines <- sc.Text(): + default: + } + } + p.err = cmd.Wait() + close(p.done) + }() + t.Cleanup(func() { + cancel() + <-p.done + if t.Failed() { + t.Logf("ssh %q stderr:\n%s", argv, p.stderr.String()) + } + }) + return p +} + +// allocatedPortRx matches OpenSSH's report of the port a "-R 0:..." +// forward was given. +var allocatedPortRx = regexp.MustCompile(`Allocated port (\d+) for remote forward`) + +// TestIntegrationOpenSSHForwarding runs the system OpenSSH client with +// -L and -R against the server, with ExitOnForwardFailure so that a +// refused forward ends the client. +func TestIntegrationOpenSSHForwarding(t *testing.T) { + sshPath, err := exec.LookPath("ssh") + if err != nil { + t.Skipf("skipping without OpenSSH client: %v", err) + } + echo := startTCPEcho(t) + echoPort := strconv.Itoa(echo.Addr().(*net.TCPAddr).Port) + allowAll := testServerOpts{ + username: fwdSessionUser, + allowLocalPortForwarding: true, + allowRemotePortForwarding: true, + } + + t.Run("local", func(t *testing.T) { + // The client listens on a Unix socket so there is no local port + // to pick, and forwards it over direct-tcpip to the echo service. + localSock := filepath.Join(t.TempDir(), "L.sock") + addr := testServerWithOpts(t, allowAll) + p := startOpenSSH(t, sshPath, addr, "-L", localSock+":127.0.0.1:"+echoPort) + + var c net.Conn + err := tstest.WaitFor(fwdTestTimeout, func() error { + select { + case <-p.done: + return fmt.Errorf("ssh exited: %v", p.err) + default: + } + var err error + c, err = net.Dial("unix", localSock) + return err + }) + if err != nil { + t.Fatalf("ssh -L listener never became ready: %v", err) + } + checkEcho(t, c, 64<<10) + }) + + t.Run("remote", func(t *testing.T) { + addr := testServerWithOpts(t, allowAll) + p := startOpenSSH(t, sshPath, addr, "-R", "127.0.0.1:0:127.0.0.1:"+echoPort) + + var port string + timeout := time.After(fwdTestTimeout) + for port == "" { + select { + case line := <-p.lines: + if m := allocatedPortRx.FindStringSubmatch(line); m != nil { + port = m[1] + } + case <-p.done: + t.Fatalf("ssh exited before reporting its allocated port: %v", p.err) + case <-timeout: + t.Fatal("timed out waiting for ssh -R to report its allocated port") + } + } + c, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", port), fwdTestTimeout) + if err != nil { + t.Fatalf("dialing forwarded port: %v", err) + } + checkEcho(t, c, 64<<10) + }) + + t.Run("remote_denied", func(t *testing.T) { + addr := testServerWithOpts(t, testServerOpts{username: fwdSessionUser}) + p := startOpenSSH(t, sshPath, addr, "-R", "127.0.0.1:0:127.0.0.1:"+echoPort) + select { + case <-p.done: + case <-time.After(fwdTestTimeout): + t.Fatal("ssh with a refused -R and ExitOnForwardFailure=yes did not exit") + } + var ee *exec.ExitError + if !errors.As(p.err, &ee) || ee.ExitCode() != 255 { + t.Errorf("ssh exit = %v; want exit status 255", p.err) + } + if out := p.stderr.String(); !strings.Contains(out, "remote port forwarding failed") { + t.Errorf("ssh stderr lacks the forwarding failure:\n%s", out) + } + }) +} diff --git a/ssh/tailssh/tailssh_integration_lifetime_test.go b/ssh/tailssh/tailssh_integration_lifetime_test.go new file mode 100644 index 000000000..5c3ebb406 --- /dev/null +++ b/ssh/tailssh/tailssh_integration_lifetime_test.go @@ -0,0 +1,999 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build integrationtest && linux + +package tailssh + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "golang.org/x/crypto/ssh" + "tailscale.com/envknob" + "tailscale.com/hostinfo" + "tailscale.com/tailcfg" +) + +// This file contains integration tests of session lifetime: what ends a +// running session (client disconnect, SessionDuration, policy changes, +// RuleExpires, server shutdown) and which of the session's processes, +// including the incubator and the per-connection ssh-monitor, go away +// when it does. A termination (revocation, expiry, SessionDuration) kills +// the whole process tree. A disconnect or shutdown hangs up a pty +// session but, as with OpenSSH's sshd, leaves a non-pty session's +// background jobs running. It also covers host key selection, utmp +// registration via login, and the TAILSCALE_SSH_DEFAULT_PATH envknob. + +// procInfo identifies a process. The start time distinguishes it from a +// later process that reuses its pid. +type procInfo struct { + pid int + ppid int + pgid int + sid int + start string // field 22 of /proc//stat + comm string +} + +func (p procInfo) String() string { + cmdline, _ := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", p.pid)) + return fmt.Sprintf("pid %d pgid %d sid %d (%s) %q", p.pid, p.pgid, p.sid, p.comm, strings.TrimSpace(string(bytes.ReplaceAll(cmdline, []byte{0}, []byte{' '})))) +} + +// readProc returns the process with the given pid. It reports false if +// there is no such process or it is a zombie, which counts as gone: it has +// exited and only awaits reaping by its parent. +func readProc(pid int) (procInfo, bool) { + b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)) + if err != nil { + return procInfo{}, false + } + s := string(b) + open, closeParen := strings.IndexByte(s, '('), strings.LastIndexByte(s, ')') + if open < 0 || closeParen < open { + return procInfo{}, false + } + // The fields after the comm start at field 3 (state). + f := strings.Fields(s[closeParen+1:]) + if len(f) < 20 { + return procInfo{}, false + } + if f[0] == "Z" || f[0] == "X" { + return procInfo{}, false + } + ppid, _ := strconv.Atoi(f[1]) + pgid, _ := strconv.Atoi(f[2]) + sid, _ := strconv.Atoi(f[3]) + return procInfo{ + pid: pid, + ppid: ppid, + pgid: pgid, + sid: sid, + start: f[19], + comm: s[open+1 : closeParen], + }, true +} + +// alive reports whether p is still running. +func (p procInfo) alive() bool { + cur, ok := readProc(p.pid) + return ok && cur.start == p.start +} + +// sessionProcs returns the processes with the given pids plus every +// ancestor of the first pid below the ssh-monitor: the session's shell +// wrappers (login, su) and the incubator. It also returns the +// per-connection ssh-monitor, which must be the test process's child. +func sessionProcs(t *testing.T, pids ...int) (tree []procInfo, monitor procInfo) { + t.Helper() + for _, pid := range pids { + p, ok := readProc(pid) + if !ok { + t.Fatalf("session process %d not running", pid) + } + tree = append(tree, p) + } + self := os.Getpid() + for p := tree[0]; ; { + if p.ppid <= 1 { + t.Fatalf("session process %v does not descend from the test process (pid %d); chain: %v", tree[0], self, tree) + } + parent, ok := readProc(p.ppid) + if !ok { + t.Fatalf("ancestor %d of %v not running", p.ppid, p) + } + if parent.ppid == self { + if !strings.Contains(parent.String(), "ssh-monitor") { + t.Fatalf("session's top ancestor %v is not an ssh-monitor", parent) + } + return tree, parent + } + tree = append(tree, parent) + p = parent + } +} + +// procTeardownTimeout bounds how long session processes may take to exit +// after their session ends. Teardown includes the 5s SIGHUP to SIGKILL +// escalation ([hangupEscalation]) and the ssh-monitor's two bounded reap +// sweeps ([monTeardownReap]). +const procTeardownTimeout = 12 * time.Second + +// waitProcsGone waits up to timeout for every process in procs to exit +// and returns the ones still running at the deadline. +func waitProcsGone(procs []procInfo, timeout time.Duration) (alive []procInfo) { + deadline := time.Now().Add(timeout) + for { + alive = alive[:0] + for _, p := range procs { + if p.alive() { + alive = append(alive, p) + } + } + if len(alive) == 0 || time.Now().After(deadline) { + return alive + } + time.Sleep(25 * time.Millisecond) + } +} + +// killProcs SIGKILLs every process in procs that is still running. +func killProcs(procs []procInfo) { + for _, p := range procs { + if p.alive() { + syscall.Kill(p.pid, syscall.SIGKILL) + } + } +} + +// sessionMode is how a [lifetimeSession] runs its command. +type sessionMode int + +const ( + modeExec sessionMode = iota // exec request, no pty + modePtyExec // pty-req, then exec request + modePtyShell // pty-req, then shell request with the command typed in +) + +var sessionModes = []struct { + name string + mode sessionMode +}{ + {"exec", modeExec}, + {"pty_exec", modePtyExec}, + {"pty_shell", modePtyShell}, +} + +// lifetimeScript starts two background sleeps, prints its own and their +// pids, and then echoes stdin lines back so the test can check that the +// session is still alive. It is run through sh -c so that a pty shell's +// job control does not move the sleeps into a process group of their own. +const lifetimeScript = `sh -c 'sleep 1000 & a=$!; sleep 1000 & b=$!; echo "PIDS $$ $a $b"; while read x; do echo "echo:$x"; done'` + +var pidsRx = regexp.MustCompile(`PIDS (\d+) (\d+) (\d+)`) + +// lifetimeSession is a running [lifetimeScript] session on its own +// connection. +type lifetimeSession struct { + nc net.Conn // raw client transport, for abrupt disconnects + cl *ssh.Client + stdin io.WriteCloser + stdout *lockedBuffer // stdout, plus stderr in pty modes + stderr *lockedBuffer + done chan error // receives the result of [ssh.Session.Wait] + procs []procInfo // the session's processes below the ssh-monitor, the incubator last + monitor procInfo // the connection's ssh-monitor + pings int +} + +// dialLifetimeClient connects to the server at addr. Unlike [ssh.Dial], +// it bounds the SSH handshake, not just the TCP dial, and it returns the +// raw transport so tests can drop the connection abruptly. +func dialLifetimeClient(addr string, cfg *ssh.ClientConfig) (net.Conn, *ssh.Client, error) { + nc, err := net.DialTimeout("tcp", addr, 10*time.Second) + if err != nil { + return nil, nil, err + } + nc.SetDeadline(time.Now().Add(15 * time.Second)) + c, chans, reqs, err := ssh.NewClientConn(nc, addr, cfg) + if err != nil { + nc.Close() + return nil, nil, err + } + nc.SetDeadline(time.Time{}) + return nc, ssh.NewClient(c, chans, reqs), nil +} + +// insecureClientConfig returns a client config for testuser that accepts +// any host key. +func insecureClientConfig() *ssh.ClientConfig { + return &ssh.ClientConfig{ + User: "testuser", + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } +} + +// startLifetimeSession opens a connection to addr, runs [lifetimeScript] +// in the given mode, and waits for it to report its pids. +func startLifetimeSession(t *testing.T, addr string, mode sessionMode) *lifetimeSession { + t.Helper() + nc, cl, err := dialLifetimeClient(addr, insecureClientConfig()) + if err != nil { + t.Fatalf("dial: %v", err) + } + t.Cleanup(func() { cl.Close() }) + sess, err := cl.NewSession() + if err != nil { + t.Fatalf("NewSession: %v", err) + } + ls := &lifetimeSession{ + nc: nc, + cl: cl, + stdout: new(lockedBuffer), + stderr: new(lockedBuffer), + done: make(chan error, 1), + } + sess.Stdout = ls.stdout + sess.Stderr = ls.stderr + ls.stdin, err = sess.StdinPipe() + if err != nil { + t.Fatal(err) + } + if mode != modeExec { + if err := sess.RequestPty("xterm", 40, 200, ssh.TerminalModes{ssh.ECHO: 0}); err != nil { + t.Fatalf("RequestPty: %v", err) + } + } + if mode == modePtyShell { + err = sess.Shell() + } else { + err = sess.Start(lifetimeScript) + } + if err != nil { + t.Fatalf("starting session: %v", err) + } + go func() { ls.done <- sess.Wait() }() + if mode == modePtyShell { + // Wait for the prompt. Input typed before it can be lost, as + // with busybox login on Alpine. + ls.waitOutput(t, func(out string) bool { + return strings.HasSuffix(out, "$ ") || strings.HasSuffix(out, "# ") + }) + if _, err := io.WriteString(ls.stdin, lifetimeScript+"\n"); err != nil { + t.Fatalf("typing command: %v", err) + } + } + + var m []string + ls.waitOutput(t, func(out string) bool { + m = pidsRx.FindStringSubmatch(out) + return m != nil + }) + var pids []int + for _, s := range m[1:] { + pid, _ := strconv.Atoi(s) + pids = append(pids, pid) + } + ls.procs, ls.monitor = sessionProcs(t, pids...) + t.Cleanup(func() { killProcs(ls.procs) }) + t.Logf("session processes: %v; monitor: %v", ls.procs, ls.monitor) + return ls +} + +// waitOutput waits until cond holds for the session's stdout so far. It +// fails the test if the session ends first or on timeout. +func (ls *lifetimeSession) waitOutput(t *testing.T, cond func(string) bool) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + for !cond(ls.stdout.String()) { + select { + case err := <-ls.done: + ls.done <- err + t.Fatalf("session ended (%v) before expected output; stdout: %q; stderr: %q", err, ls.stdout.String(), ls.stderr.String()) + default: + } + if time.Now().After(deadline) { + t.Fatalf("timeout waiting for session output; stdout: %q; stderr: %q", ls.stdout.String(), ls.stderr.String()) + } + time.Sleep(10 * time.Millisecond) + } +} + +// ping checks that the session's script still echoes stdin. +func (ls *lifetimeSession) ping(t *testing.T) { + t.Helper() + ls.pings++ + want := fmt.Sprintf("echo:ping%d", ls.pings) + if _, err := fmt.Fprintf(ls.stdin, "ping%d\n", ls.pings); err != nil { + t.Fatalf("writing ping: %v", err) + } + ls.waitOutput(t, func(out string) bool { return strings.Contains(out, want) }) +} + +// jobs returns the session's two background jobs, the sleeps started by +// [lifetimeScript], and its other processes. +func (ls *lifetimeSession) jobs() (jobs, others []procInfo) { + return ls.procs[1:3], append([]procInfo{ls.procs[0]}, ls.procs[3:]...) +} + +// waitEndedKeepingJobs waits for a non-pty session that ended without a +// termination (a disconnect or [server.Shutdown]) to wind down, and +// checks that its background jobs survive. OpenSSH's sshd signals +// nothing when a non-pty session ends, and clients such as VS Code +// Remote-SSH rely on that; see #22076. The script's shell exits on stdin +// EOF, and with it the wrappers above it and the ssh-monitor. +func (ls *lifetimeSession) waitEndedKeepingJobs(t *testing.T) (leak string) { + t.Helper() + jobs, others := ls.jobs() + leak = ls.waitProcsGone(t, append(others, ls.monitor)...) + for _, p := range jobs { + if !p.alive() { + t.Errorf("background job %v did not outlive its non-pty session", p) + } + } + return leak +} + +// waitEnded waits for a session that ended without a termination to +// wind down: a pty session's whole process tree is hung up, and a +// non-pty session keeps its background jobs, as checked by +// [lifetimeSession.waitEndedKeepingJobs]. +func (ls *lifetimeSession) waitEnded(t *testing.T, mode sessionMode) (leak string) { + t.Helper() + if mode == modeExec { + return ls.waitEndedKeepingJobs(t) + } + return ls.waitAllGone(t) +} + +// waitDone waits for the session to end and returns the result of +// [ssh.Session.Wait]. +func (ls *lifetimeSession) waitDone(t *testing.T, timeout time.Duration) error { + t.Helper() + select { + case err := <-ls.done: + ls.done <- err + return err + case <-time.After(timeout): + t.Fatalf("session still running after %v; stdout: %q; stderr: %q", timeout, ls.stdout.String(), ls.stderr.String()) + return nil + } +} + +// escapedGroupLimitation describes the known limitation that +// [lifetimeSession.waitProcsGone] tolerates. It matches OpenSSH's sshd, +// which also only signals the session's own process group, but unlike +// sshd, tailssh ends sessions on revocation, rule expiry, and +// SessionDuration, where survivors outlive the user's access. +const escapedGroupLimitation = "KNOWN LIMITATION: tailssh ends a session by signaling only the incubator's process group, " + + "so processes moved out of it outlived their SSH session " + + "(util-linux su -c starts the command in a new session; the in-process incubator starts a pty shell " + + "in a new foreground group; an interactive shell gives each job its own group)" + +// waitProcsGone waits for procs, some of the session's processes, to +// exit. Survivors fail the test, with one exception for a known limitation: if +// every survivor has left the incubator's process group, it kills them +// and returns a description of the leak for [skipIfLeaked], so that the +// rest of the test can run. +func (ls *lifetimeSession) waitProcsGone(t *testing.T, procs ...procInfo) (leak string) { + t.Helper() + start := time.Now() + alive := waitProcsGone(procs, procTeardownTimeout) + if len(alive) == 0 { + t.Logf("%d session processes exited within %v", len(procs), time.Since(start).Round(time.Millisecond)) + return "" + } + desc := fmt.Sprint(alive) + incubator := ls.procs[len(ls.procs)-1] + if slices.ContainsFunc(alive, func(p procInfo) bool { return p.pgid == incubator.pgid || p == ls.monitor }) { + t.Fatalf("%d session processes still running after %v: %s", len(alive), procTeardownTimeout, desc) + } + killProcs(alive) + if still := waitProcsGone(alive, procTeardownTimeout); len(still) > 0 { + t.Fatalf("leaked session processes survived SIGKILL: %v", still) + } + return escapedGroupLimitation + ": " + desc +} + +// waitAllGone waits for all of the session's processes, including the +// ssh-monitor, to exit, as [lifetimeSession.waitProcsGone] does. +func (ls *lifetimeSession) waitAllGone(t *testing.T) (leak string) { + t.Helper() + return ls.waitProcsGone(t, append(slices.Clone(ls.procs), ls.monitor)...) +} + +// skipIfLeaked skips t if any leak from [lifetimeSession.waitProcsGone] +// is non-empty. Failures recorded earlier still fail the test. +func skipIfLeaked(t *testing.T, leaks ...string) { + t.Helper() + var all []string + for _, leak := range leaks { + if leak != "" { + all = append(all, leak) + } + } + if len(all) > 0 { + t.Skip(strings.Join(all, "; ")) + } +} + +// exitDesc describes a [ssh.Session.Wait] result for logs. +func exitDesc(err error) string { + if err == nil { + return "exit status 0" + } + if ee, ok := errors.AsType[*ssh.ExitError](err); ok { + return fmt.Sprintf("exit status %d (signal %q)", ee.ExitStatus(), ee.Signal()) + } + return fmt.Sprintf("%T: %v", err, err) +} + +// TestIntegrationClientDisconnect checks what dropping the client's +// transport without any SSH-level close leaves behind: nothing of a pty +// session, and only the background jobs of a non-pty session. The +// incubator and the ssh-monitor exit either way. +func TestIntegrationClientDisconnect(t *testing.T) { + for _, v1 := range []bool{false, true} { + addr := testServerWithOpts(t, testServerOpts{username: "testuser", forceV1Behavior: v1}) + for _, m := range sessionModes { + t.Run(fmt.Sprintf("%s_v1=%v", m.name, v1), func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + ls := startLifetimeSession(t, addr, m.mode) + ls.ping(t) + ls.nc.Close() + skipIfLeaked(t, ls.waitEnded(t, m.mode)) + }) + } + } +} + +// TestIntegrationSessionDuration checks that [tailcfg.SSHAction.SessionDuration] +// ends a running session with a message and a failure exit, and that the +// session's processes are killed. +func TestIntegrationSessionDuration(t *testing.T) { + const dur = 3 * time.Second + addr := testServerWithOpts(t, testServerOpts{ + username: "testuser", + modifyRule: func(r *tailcfg.SSHRule) { + r.Action.SessionDuration = dur + }, + }) + for _, m := range []sessionMode{modeExec, modePtyShell} { + t.Run(sessionModes[m].name, func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + start := time.Now() + ls := startLifetimeSession(t, addr, m) + ls.ping(t) + // The session's output streams stay open while any of its + // processes do, so wait for those first. The connection, and + // with it the ssh-monitor, stays up. + leak := ls.waitProcsGone(t, ls.procs...) + err := ls.waitDone(t, 20*time.Second) + elapsed := time.Since(start) + t.Logf("session ended after %v with %s", elapsed.Round(time.Millisecond), exitDesc(err)) + if elapsed < dur { + t.Errorf("session ended after %v, before its %v duration", elapsed, dur) + } + if err == nil { + t.Errorf("session exited 0, want a failure") + } + wantMsg := fmt.Sprintf("Session timeout of %v elapsed.", dur) + // In pty mode, stderr is merged into the pty's output. + if got := ls.stderr.String() + ls.stdout.String(); !strings.Contains(got, wantMsg) { + t.Errorf("client output lacks %q; stdout: %q; stderr: %q", wantMsg, ls.stdout.String(), ls.stderr.String()) + } + ls.cl.Close() + ls.waitProcsGone(t, ls.monitor) + skipIfLeaked(t, leak) + }) + } +} + +// recheckConnsNow synchronously re-evaluates the policy for every +// authenticated connection of srv and reports whether all of them still +// hold their grant. Unlike [server.OnPolicyChange], it does not return +// before the evaluation is done. +func recheckConnsNow(srv *server) bool { + srv.mu.Lock() + var conns []*conn + for c := range srv.activeConns { + if c.authCompleted.Load() { + conns = append(conns, c) + } + } + srv.mu.Unlock() + ok := true + for _, c := range conns { + if !c.recheckPolicy() { + ok = false + } + } + return ok +} + +// TestIntegrationPolicyChangeKillsSession checks that a policy change +// that revokes or narrows the grant ends an active session and its +// processes, and that a change that only widens it does not. +func TestIntegrationPolicyChangeKillsSession(t *testing.T) { + // All revocations share one teardown path, so only the first is + // run in every session mode. + allModes := []sessionMode{modeExec, modePtyShell} + revocations := []struct { + name string + modes []sessionMode + modify func(*tailcfg.SSHRule) + }{ + {"reject", allModes, func(r *tailcfg.SSHRule) { + r.Action = &tailcfg.SSHAction{Reject: true} + }}, + {"user_remapped", []sessionMode{modePtyShell}, func(r *tailcfg.SSHRule) { + r.SSHUsers = map[string]string{"*": "root"} + }}, + {"principal_removed", []sessionMode{modePtyShell}, func(r *tailcfg.SSHRule) { + r.Principals = []*tailcfg.SSHPrincipal{{NodeIP: "100.64.99.99"}} + }}, + {"agent_forwarding_revoked", []sessionMode{modePtyShell}, func(r *tailcfg.SSHRule) { + r.Action.AllowAgentForwarding = false + }}, + } + for _, tt := range revocations { + for _, m := range tt.modes { + t.Run(tt.name+"_"+sessionModes[m].name, func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + srv, tb, addr := startTestServer(t, testServerOpts{username: "testuser"}) + ls := startLifetimeSession(t, addr, m) + ls.ping(t) + tb.setModifyRule(tt.modify) + srv.OnPolicyChange() + // A revocation closes the connection, so the client sees + // the session end even if processes outlive it. + err := ls.waitDone(t, 20*time.Second) + t.Logf("session ended with %s; stderr: %q", exitDesc(err), ls.stderr.String()) + if err == nil { + t.Errorf("revoked session exited 0, want a failure") + } + skipIfLeaked(t, ls.waitAllGone(t)) + }) + } + } + + t.Run("non_material", func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + srv, tb, addr := startTestServer(t, testServerOpts{username: "testuser"}) + ls := startLifetimeSession(t, addr, modeExec) + ls.ping(t) + // Widening changes: more AcceptEnv patterns, newly granted + // forwarding, a later RuleExpires, and a message. None of + // them narrows what the connection was granted at auth. + later := time.Now().Add(time.Hour) + tb.setModifyRule(func(r *tailcfg.SSHRule) { + r.AcceptEnv = append(r.AcceptEnv, "EXTRA_*") + r.Action.AllowLocalPortForwarding = true + r.Action.AllowRemotePortForwarding = true + r.Action.Message = "policy updated\n" + r.RuleExpires = &later + }) + srv.OnPolicyChange() + // OnPolicyChange rechecks asynchronously. A synchronous + // recheck of the same policy reaches the same verdict. + if !recheckConnsNow(srv) { + t.Fatalf("non-material policy change revoked the connection") + } + ls.ping(t) + for _, p := range ls.procs { + if !p.alive() { + t.Errorf("session process %v died after a non-material policy change", p) + } + } + // A new session on the same connection is also still allowed. + s, err := ls.cl.NewSession() + if err != nil { + t.Fatalf("NewSession after non-material change: %v", err) + } + defer s.Close() + if out, err := s.Output("echo second"); err != nil || strings.TrimSpace(string(out)) != "second" { + t.Errorf("second session after non-material change: %q, %v", out, err) + } + }) +} + +// TestIntegrationRuleExpiresKillsSession checks that an active session is +// ended when the rule that granted it reaches its +// [tailcfg.SSHRule.RuleExpires], with no policy change notification, and +// that new connections are then refused. +func TestIntegrationRuleExpiresKillsSession(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + expires := time.Now().Add(4 * time.Second) + addr := testServerWithOpts(t, testServerOpts{ + username: "testuser", + modifyRule: func(r *tailcfg.SSHRule) { + r.RuleExpires = &expires + }, + }) + ls := startLifetimeSession(t, addr, modeExec) + ls.ping(t) + err := ls.waitDone(t, time.Until(expires)+20*time.Second) + t.Logf("session ended with %s", exitDesc(err)) + if now := time.Now(); now.Before(expires) { + t.Errorf("session ended %v before the rule expired", expires.Sub(now)) + } + if err == nil { + t.Errorf("expired session exited 0, want a failure") + } + leak := ls.waitAllGone(t) + + if _, cl, err := dialLifetimeClient(addr, insecureClientConfig()); err == nil { + cl.Close() + t.Errorf("connection after rule expiry was accepted") + } + skipIfLeaked(t, leak) +} + +// TestIntegrationShutdown checks that [server.Shutdown] ends active +// sessions, leaving only a non-pty session's background jobs running, +// that it returns, and that the server refuses connections afterwards. +func TestIntegrationShutdown(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + srv, _, addr := startTestServer(t, testServerOpts{username: "testuser"}) + modes := []sessionMode{modeExec, modePtyShell} + var sessions []*lifetimeSession + for _, m := range modes { + ls := startLifetimeSession(t, addr, m) + ls.ping(t) + sessions = append(sessions, ls) + } + + shutdownDone := make(chan struct{}) + go func() { + defer close(shutdownDone) + srv.Shutdown() + }() + var leaks []string + for i, ls := range sessions { + leaks = append(leaks, ls.waitEnded(t, modes[i])) + } + select { + case <-shutdownDone: + case <-time.After(20 * time.Second): + t.Fatalf("Shutdown did not return") + } + if n := srv.NumActiveConns(); n != 0 { + t.Errorf("NumActiveConns after Shutdown = %d, want 0", n) + } + for i, ls := range sessions { + err := ls.waitDone(t, 20*time.Second) + t.Logf("session %d ended with %s", i, exitDesc(err)) + if err == nil { + t.Errorf("session %d exited 0 after Shutdown, want a failure", i) + } + } + + c1, c2 := net.Pipe() + defer c2.Close() + defer c1.Close() + if err := srv.HandleSSHConn(&addressFakingConn{c1}); err == nil || !strings.Contains(err.Error(), "shutting down") { + t.Errorf("HandleSSHConn after Shutdown = %v, want a shutting-down error", err) + } + // HandleSSHConn closes a connection it refuses. + c2.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := c2.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("read from refused conn = %v, want io.EOF", err) + } + + // A client is refused promptly rather than left waiting for its + // handshake deadline. + nc, err := net.DialTimeout("tcp", addr, 10*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + nc.SetDeadline(time.Now().Add(10 * time.Second)) + if c, _, _, err := ssh.NewClientConn(nc, addr, insecureClientConfig()); err == nil { + c.Close() + t.Errorf("connection after Shutdown was accepted") + } else if errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("connection after Shutdown was left open until the client's deadline: %v", err) + } else { + t.Logf("connection after Shutdown failed as expected: %v", err) + } + skipIfLeaked(t, leaks...) +} + +// hostKeyAlgos maps the host key types tailssh generates to the client +// algorithm that selects each. +var hostKeyAlgos = map[string]string{ + "rsa": ssh.KeyAlgoRSASHA256, + "ecdsa": ssh.KeyAlgoECDSA256, + "ed25519": ssh.KeyAlgoED25519, +} + +// presentedHostKey connects to addr offering only the host key algorithm +// algo and returns the host key the server presents. +func presentedHostKey(t *testing.T, addr, algo string) ssh.PublicKey { + t.Helper() + var got ssh.PublicKey + cfg := &ssh.ClientConfig{ + User: "testuser", + HostKeyAlgorithms: []string{algo}, + HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error { + got = key + return nil + }, + } + _, cl, err := dialLifetimeClient(addr, cfg) + if err != nil { + t.Fatalf("dial with host key algorithm %s: %v", algo, err) + } + cl.Close() + return got +} + +// systemHostKeyExists reports whether tailssh would consider the system +// OpenSSH host key of type typ, per [systemHostKeyFile]. +func systemHostKeyExists(typ string) bool { + f := systemHostKeyFile(typ) + if f == "" { + return false + } + _, err := os.Stat(f) + return err == nil +} + +// TestIntegrationHostKeys checks which host keys the server presents. +func TestIntegrationHostKeys(t *testing.T) { + t.Run("generated_stable_and_private", func(t *testing.T) { + varRoot := t.TempDir() + addr1 := testServerWithOpts(t, testServerOpts{username: "testuser", varRoot: varRoot}) + addr2 := testServerWithOpts(t, testServerOpts{username: "testuser", varRoot: varRoot}) + for typ, algo := range hostKeyAlgos { + k1 := presentedHostKey(t, addr1, algo) + k2 := presentedHostKey(t, addr2, algo) + if !bytes.Equal(k1.Marshal(), k2.Marshal()) { + t.Errorf("%s: servers sharing a var root presented different host keys", typ) + } + if systemHostKeyExists(typ) { + t.Logf("%s: system host key in use; not checking the generated file", typ) + continue + } + path := filepath.Join(varRoot, "ssh", "ssh_host_"+typ+"_key") + fi, err := os.Stat(path) + if err != nil { + t.Errorf("%s: %v", typ, err) + continue + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("%s: %s has mode %v, want 0600", typ, path, perm) + } + pem, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.ParsePrivateKey(pem) + if err != nil { + t.Fatalf("%s: parsing %s: %v", typ, path, err) + } + if !bytes.Equal(signer.PublicKey().Marshal(), k1.Marshal()) { + t.Errorf("%s: presented host key does not match %s", typ, path) + } + } + fi, err := os.Stat(filepath.Join(varRoot, "ssh")) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != 0o700 { + t.Errorf("key directory has mode %v, want 0700", perm) + } + }) + + t.Run("system_key_reused", func(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("system host keys in /etc/ssh are only read when tailscaled runs as root; see systemHostKeyFile") + } + keygen, err := exec.LookPath("ssh-keygen") + if err != nil { + t.Skipf("no ssh-keygen: %v", err) + } + const typ = "ed25519" + keyFile := systemHostKeyFile(typ) + if _, err := os.Stat(keyFile); err == nil { + t.Logf("using existing %s", keyFile) + } else { + if _, err := os.Stat(filepath.Dir(keyFile)); os.IsNotExist(err) { + if err := os.Mkdir(filepath.Dir(keyFile), 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Remove(filepath.Dir(keyFile)) }) + } + t.Cleanup(func() { + os.Remove(keyFile) + os.Remove(keyFile + ".pub") + }) + if out, err := exec.Command(keygen, "-q", "-t", typ, "-N", "", "-f", keyFile).CombinedOutput(); err != nil { + t.Fatalf("ssh-keygen: %v, %s", err, out) + } + } + pub, err := os.ReadFile(keyFile + ".pub") + if err != nil { + t.Fatal(err) + } + want, _, _, _, err := ssh.ParseAuthorizedKey(pub) + if err != nil { + t.Fatal(err) + } + + varRoot := t.TempDir() + addr := testServerWithOpts(t, testServerOpts{username: "testuser", varRoot: varRoot}) + got := presentedHostKey(t, addr, hostKeyAlgos[typ]) + if !bytes.Equal(got.Marshal(), want.Marshal()) { + t.Errorf("server presented %s, want system key %s", ssh.FingerprintSHA256(got), ssh.FingerprintSHA256(want)) + } + if _, err := os.Stat(filepath.Join(varRoot, "ssh", "ssh_host_"+typ+"_key")); !os.IsNotExist(err) { + t.Errorf("tailssh generated its own %s key despite the system key (stat err %v)", typ, err) + } + // Types without a system key still get generated keys. + if !systemHostKeyExists("rsa") { + presentedHostKey(t, addr, hostKeyAlgos["rsa"]) + if _, err := os.Stat(filepath.Join(varRoot, "ssh", "ssh_host_rsa_key")); err != nil { + t.Errorf("no generated rsa key: %v", err) + } + } + }) +} + +// utmpRecordSize and the offsets below describe glibc's struct utmp on +// 64-bit Linux, the record format of utmp and wtmp. +const ( + utmpRecordSize = 384 + utmpUserOff = 44 + utmpHostOff = 76 + utmpUserLen = 32 + utmpHostLen = 256 + utmpUserProc = 7 // USER_PROCESS +) + +// utmpRecord is the part of a utmp record that the tests check. +type utmpRecord struct { + typ int16 + user string + host string +} + +// parseUtmp parses the complete utmp records in b. +func parseUtmp(b []byte) []utmpRecord { + var recs []utmpRecord + for rec := range slices.Chunk(b, utmpRecordSize) { + if len(rec) < utmpRecordSize { + break + } + recs = append(recs, utmpRecord{ + typ: int16(binary.NativeEndian.Uint16(rec[0:2])), + user: cString(rec[utmpUserOff : utmpUserOff+utmpUserLen]), + host: cString(rec[utmpHostOff : utmpHostOff+utmpHostLen]), + }) + } + return recs +} + +// TestIntegrationLoginRecordsUtmp checks that a pty shell session started +// through login(1) is registered in utmp, as who(1) shows it, with the +// client's Tailscale IP as the remote host, which tailssh passes as +// login -h. +func TestIntegrationLoginRecordsUtmp(t *testing.T) { + const utmp = "/run/utmp" + switch { + case os.Geteuid() != 0: + t.Skip("login is only used when tailscaled runs as root") + case hostinfo.IsSELinuxEnforcing(): + t.Skip("login is not used when SELinux is enforcing") + } + if _, err := exec.LookPath("login"); err != nil { + t.Skip("no login command") + } + if musl, _ := filepath.Glob("/lib/ld-musl-*"); len(musl) > 0 { + t.Skip("musl libc's utmp functions are no-ops, so login records nothing") + } + // Containers usually lack utmp, and login only updates an existing + // file. Create it for the test if needed. + if _, err := os.Stat(utmp); os.IsNotExist(err) { + if err := os.WriteFile(utmp, nil, 0o664); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Remove(utmp) }) + } + defer dumpIncubatorLogOnFail(t) + + addr := testServerWithOpts(t, testServerOpts{username: "testuser"}) + ls := startLifetimeSession(t, addr, modePtyShell) + ls.ping(t) + if !slices.Contains(procComms(ls.procs), "login") { + t.Fatalf("pty shell session was not started through login; processes: %v", ls.procs) + } + + b, err := os.ReadFile(utmp) + if err != nil { + t.Fatal(err) + } + recs := parseUtmp(b) + want := utmpRecord{typ: utmpUserProc, user: "testuser", host: "100.100.100.102"} + if !slices.Contains(recs, want) { + t.Errorf("utmp has no %+v record; records: %+v", want, recs) + } +} + +func procComms(procs []procInfo) []string { + var comms []string + for _, p := range procs { + comms = append(comms, p.comm) + } + return comms +} + +func cString(b []byte) string { + if i := bytes.IndexByte(b, 0); i >= 0 { + b = b[:i] + } + return string(b) +} + +// TestIntegrationDefaultPath checks the PATH that sessions get from +// [defaultPathForUser], including the TAILSCALE_SSH_DEFAULT_PATH +// template. It uses the in-process incubator path, since login and su +// reset PATH themselves: as root that means v1 behavior, which never +// uses su for commands. +func TestIntegrationDefaultPath(t *testing.T) { + u, err := userLookup("testuser") + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + tmpl string + want string + }{ + {"default", "", defaultPathForUser(u)}, + {"template", "/opt/@{PAM_USER}/bin:@{HOME}/bin:/usr/bin:/bin", "/opt/testuser/bin:" + u.HomeDir + "/bin:/usr/bin:/bin"}, + {"unknown_expansion", "/opt/@{NOPE}/bin:/usr/bin", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + envknob.SetenvForTest(t, "TAILSCALE_SSH_DEFAULT_PATH", tt.tmpl) + addr := testServerWithOpts(t, testServerOpts{ + username: "testuser", + forceV1Behavior: true, + }) + _, cl, err := dialLifetimeClient(addr, insecureClientConfig()) + if err != nil { + t.Fatal(err) + } + defer cl.Close() + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + out, err := s.Output(`echo "PATH=[$PATH]"`) + if err != nil { + t.Fatalf("%v; output: %q", err, out) + } + if want := "PATH=[" + tt.want + "]"; strings.TrimSpace(string(out)) != want { + t.Errorf("got %q, want %q", strings.TrimSpace(string(out)), want) + } + }) + } +} diff --git a/ssh/tailssh/tailssh_integration_session_test.go b/ssh/tailssh/tailssh_integration_session_test.go new file mode 100644 index 000000000..bf8168903 --- /dev/null +++ b/ssh/tailssh/tailssh_integration_session_test.go @@ -0,0 +1,712 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build integrationtest + +package tailssh + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + "time" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + "tailscale.com/envknob" + "tailscale.com/tstest" +) + +// This file contains integration tests of per-session behavior: pty +// setup and resizing, terminal modes, the session environment, exit +// statuses, and the TS_SSH_DISABLE_* envknobs. + +// sessionTestTimeout bounds each test's SSH client. When it elapses the +// client is closed, which unblocks any pending session reads or waits. +const sessionTestTimeout = 60 * time.Second + +// forEachBehavior runs f as a "v2" subtest and as a "v1" subtest with +// forceV1Behavior set. +func forEachBehavior(t *testing.T, f func(t *testing.T, forceV1Behavior bool)) { + for _, forceV1Behavior := range []bool{false, true} { + name := "v2" + if forceV1Behavior { + name = "v1" + } + t.Run(name, func(t *testing.T) { + defer dumpIncubatorLogOnFail(t) + f(t, forceV1Behavior) + }) + } +} + +// dialSessionTestClient starts a test server for testuser configured by +// opts and returns a client connected to it. The client is closed at +// the end of the test or after [sessionTestTimeout], whichever is first. +func dialSessionTestClient(t *testing.T, opts testServerOpts) *ssh.Client { + t.Helper() + if opts.username == "" { + opts.username = "testuser" + } + addr := testServerWithOpts(t, opts) + cl, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{ + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 15 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + timer := time.AfterFunc(sessionTestTimeout, func() { cl.Close() }) + t.Cleanup(func() { + timer.Stop() + cl.Close() + }) + return cl +} + +// ptyRequest describes the pty-req a test session sends. +type ptyRequest struct { + term string + rows, cols int + modes ssh.TerminalModes +} + +// defaultPTY is a pty request with echo disabled, so that input sent to +// an interactive shell does not appear in its output. +var defaultPTY = ptyRequest{term: "xterm", rows: 24, cols: 80, modes: ssh.TerminalModes{ssh.ECHO: 0}} + +// runPTY runs cmd in a new session of cl with a pty allocated per req. +// It returns the session's stdout and stderr, with carriage returns +// removed, and the error from [ssh.Session.Run]. +func runPTY(t *testing.T, cl *ssh.Client, req ptyRequest, cmd string) (stdout, stderr string, err error) { + t.Helper() + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.RequestPty(req.term, req.rows, req.cols, req.modes); err != nil { + t.Fatalf("RequestPty: %v", err) + } + var outBuf, errBuf bytes.Buffer + s.Stdout = &outBuf + s.Stderr = &errBuf + err = s.Run(cmd) + return stripCR(outBuf.String()), stripCR(errBuf.String()), err +} + +func stripCR(s string) string { + return strings.ReplaceAll(s, "\r", "") +} + +// exitStatus returns the exit status and exit signal carried by err, the +// result of [ssh.Session.Run] or [ssh.Session.Wait]. A nil err is status 0. +// It fails the test if err is not an [*ssh.ExitError]. +func exitStatus(t *testing.T, err error) (code int, signal string) { + t.Helper() + if err == nil { + return 0, "" + } + ee, ok := errors.AsType[*ssh.ExitError](err) + if !ok { + t.Fatalf("got %T (%v), want *ssh.ExitError", err, err) + } + return ee.ExitStatus(), ee.Signal() +} + +// ptyShell is an interactive login shell session with a pty. +type ptyShell struct { + s *ssh.Session + stdin io.WriteCloser + out lockedBuffer // stdout and stderr + waitc chan error // receives the result of [ssh.Session.Wait] +} + +// startPTYShell starts an interactive shell on cl with a pty allocated +// per req. +func startPTYShell(t *testing.T, cl *ssh.Client, req ptyRequest) *ptyShell { + t.Helper() + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + p := &ptyShell{s: s, waitc: make(chan error, 1)} + if p.stdin, err = s.StdinPipe(); err != nil { + t.Fatal(err) + } + s.Stdout = &p.out + s.Stderr = &p.out + if err := s.RequestPty(req.term, req.rows, req.cols, req.modes); err != nil { + t.Fatalf("RequestPty: %v", err) + } + if err := s.Shell(); err != nil { + t.Fatalf("Shell: %v", err) + } + go func() { p.waitc <- s.Wait() }() + return p +} + +// send writes line and a newline to the shell's stdin. +func (p *ptyShell) send(t *testing.T, line string) { + t.Helper() + if _, err := io.WriteString(p.stdin, line+"\n"); err != nil { + t.Fatalf("writing %q to shell: %v", line, err) + } +} + +// query repeatedly sends cmd to the shell until its output contains want. +// Sending more than once tolerates input discarded while login sets up +// the terminal, and state changes that the server applies asynchronously. +func (p *ptyShell) query(t *testing.T, cmd, want string) { + t.Helper() + err := tstest.WaitFor(20*time.Second, func() error { + if strings.Contains(p.output(), want) { + return nil + } + p.send(t, cmd) + return fmt.Errorf("output does not contain %q", want) + }) + if err != nil { + t.Fatalf("%v; output so far:\n%s", err, p.output()) + } +} + +// output returns the shell's output so far, with carriage returns removed. +func (p *ptyShell) output() string { + return stripCR(p.out.String()) +} + +// queryValue sends cmd to the shell until it prints a line of the form +// "key=value", and returns the value. The value must not contain spaces. +func (p *ptyShell) queryValue(t *testing.T, cmd, key string) string { + t.Helper() + re := regexp.MustCompile(`(?m)(?:^|\$ )` + regexp.QuoteMeta(key) + `=(\S*)$`) + var val string + err := tstest.WaitFor(20*time.Second, func() error { + if m := re.FindStringSubmatch(p.output()); m != nil { + val = m[1] + return nil + } + p.send(t, cmd) + return fmt.Errorf("output has no %s= line", key) + }) + if err != nil { + t.Fatalf("%v; output so far:\n%s", err, p.output()) + } + return val +} + +// parentCmd prints the command name of the shell's parent process. +const parentCmd = `printf 'parent=%s\n' "$(cat /proc/$PPID/comm)"` + +// loginWaitsForShell reports whether the shell's parent process is +// login(1). That is the case when login forks the shell and waits for +// it, as the shadow-utils login on Ubuntu and the BusyBox login on +// Alpine do, rather than exec'ing it. Both exit 0 whatever the shell's +// exit status. +func (p *ptyShell) loginWaitsForShell(t *testing.T) bool { + t.Helper() + return p.queryValue(t, parentCmd, "parent") == "login" +} + +// isBusyBoxLogin reports whether the login command is BusyBox's. Unlike +// the shadow-utils login, it keeps the terminal modes it starts with. +func isBusyBoxLogin() bool { + path, err := exec.LookPath("login") + if err != nil { + return false + } + resolved, err := filepath.EvalSymlinks(path) + return err == nil && filepath.Base(resolved) == "busybox" +} + +// wait waits for the shell session to end and returns its exit status. +func (p *ptyShell) wait(t *testing.T) (code int, signal string) { + t.Helper() + select { + case err := <-p.waitc: + return exitStatus(t, err) + case <-time.After(20 * time.Second): + t.Fatalf("timeout waiting for shell to exit; output:\n%s", p.out.String()) + } + panic("unreachable") +} + +// sttySizeCmd prints the pty size prefixed by "size=", so that the +// expected output never matches the command text if it is echoed. +const sttySizeCmd = `printf 'size=%s\n' "$(stty size)"` + +func TestIntegrationPTYInitialSize(t *testing.T) { + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + for _, size := range [][2]int{{24, 80}, {53, 191}} { + req := ptyRequest{term: "xterm", rows: size[0], cols: size[1]} + out, _, err := runPTY(t, cl, req, sttySizeCmd) + if err != nil { + t.Fatalf("stty size: %v; output: %q", err, out) + } + if want := fmt.Sprintf("size=%d %d\n", size[0], size[1]); !strings.Contains(out, want) { + t.Errorf("output = %q, want %q", out, want) + } + } + }) +} + +func TestIntegrationPTYWindowResize(t *testing.T) { + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + p := startPTYShell(t, cl, defaultPTY) + p.query(t, sttySizeCmd, "size=24 80\n") + for _, size := range [][2]int{{50, 132}, {30, 100}} { + if err := p.s.WindowChange(size[0], size[1]); err != nil { + t.Fatalf("WindowChange: %v", err) + } + p.query(t, sttySizeCmd, fmt.Sprintf("size=%d %d\n", size[0], size[1])) + } + p.send(t, "exit 0") + if code, sig := p.wait(t); code != 0 || sig != "" { + t.Errorf("exit status = %d, signal %q; want 0", code, sig) + } + }) +} + +func TestIntegrationPTYTerminalModes(t *testing.T) { + // sttyFlags returns the set of boolean flags (like "echo" or + // "-echo") in the output of "stty -a". + sttyFlags := func(out string) []string { + var flags []string + for f := range strings.FieldsSeq(out) { + if !strings.ContainsAny(f, "=;") { + flags = append(flags, f) + } + } + return flags + } + tests := []struct { + modes ssh.TerminalModes + want []string + }{ + {ssh.TerminalModes{ssh.ECHO: 0, ssh.ICRNL: 0, ssh.ISIG: 1, ssh.IUTF8: 1}, []string{"-echo", "-icrnl", "isig", "iutf8"}}, + {ssh.TerminalModes{ssh.ECHO: 1, ssh.ICRNL: 1, ssh.ISIG: 0, ssh.IUTF8: 0}, []string{"echo", "icrnl", "-isig", "-iutf8"}}, + } + check := func(t *testing.T, modes ssh.TerminalModes, want []string, out string) { + t.Helper() + flags := sttyFlags(out) + for _, w := range want { + if !slices.Contains(flags, w) { + t.Errorf("modes %v: stty -a output lacks %q:\n%s", modes, w, out) + } + } + } + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + t.Run("exec", func(t *testing.T) { + for _, tt := range tests { + out, _, err := runPTY(t, cl, ptyRequest{term: "xterm", rows: 24, cols: 80, modes: tt.modes}, "stty -a") + if err != nil { + t.Fatalf("stty -a: %v; output: %q", err, out) + } + check(t, tt.modes, tt.want, out) + } + }) + t.Run("shell", func(t *testing.T) { + for _, tt := range tests { + p := startPTYShell(t, cl, ptyRequest{term: "xterm", rows: 24, cols: 80, modes: tt.modes}) + if p.loginWaitsForShell(t) && !isBusyBoxLogin() { + t.Skip("KNOWN BUG: shadow-utils login resets the client's terminal modes") + } + out := p.queryValue(t, `printf 'modes=%s\n' "$(stty -a | tr ' \n' ',,')"`, "modes") + p.send(t, "exit") + check(t, tt.modes, tt.want, strings.ReplaceAll(out, ",", " ")) + } + }) + }) +} + +// TestIntegrationPTYExec tests running a command with a pty, like +// "ssh -t host cmd". +func TestIntegrationPTYExec(t *testing.T) { + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + stdout, stderr, err := runPTY(t, cl, defaultPTY, + `[ -t 0 ] && [ -t 1 ] && [ -t 2 ] && echo all-tty; echo to-stdout; echo to-stderr >&2; exit 3`) + if code, sig := exitStatus(t, err); code != 3 || sig != "" { + t.Errorf("exit status = %d, signal %q; want 3", code, sig) + } + for _, want := range []string{"all-tty\n", "to-stdout\n", "to-stderr\n"} { + if !strings.Contains(stdout, want) { + t.Errorf("stdout = %q, want it to contain %q", stdout, want) + } + } + if stderr != "" { + t.Errorf("stderr = %q, want empty: the pty carries stderr", stderr) + } + }) +} + +// TestIntegrationPTYTerm tests that the pty request's terminal type is +// the session's $TERM, both for commands and for login shells. +func TestIntegrationPTYTerm(t *testing.T) { + const cmd = `printf 'term=%s\n' "$TERM"` + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + t.Run("exec", func(t *testing.T) { + req := defaultPTY + req.term = "vt220" + out, _, err := runPTY(t, cl, req, cmd) + if err != nil { + t.Fatalf("%v; output: %q", err, out) + } + if !strings.Contains(out, "term=vt220\n") { + t.Errorf("output = %q, want term=vt220", out) + } + }) + t.Run("shell", func(t *testing.T) { + req := defaultPTY + req.term = "screen-256color" + p := startPTYShell(t, cl, req) + p.query(t, cmd, "term=screen-256color\n") + p.send(t, "exit") + p.wait(t) + }) + }) +} + +// TestIntegrationSessionEnv tests the environment of a non-pty session. +// It replaces the unit test TestSSH/env, which is skipped in CI (#4051). +func TestIntegrationSessionEnv(t *testing.T) { + um, err := userLookup("testuser") + if err != nil { + t.Fatal(err) + } + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + // LANG and LC_* are accepted by default, like OpenSSH's + // AcceptEnv. Other variables need the policy's AcceptEnv. + for k, v := range map[string]string{"LANG": "C.UTF-8", "LC_TIME": "C", "NOT_ACCEPTED": "x"} { + if err := s.Setenv(k, v); err != nil { + t.Fatalf("Setenv(%q): %v", k, err) + } + } + out, err := s.CombinedOutput(`env; printf 'cwd=%s\n' "$(pwd)"; ` + parentCmd) + if err != nil { + t.Fatalf("%v; output:\n%s", err, out) + } + env := map[string]string{} + for line := range strings.Lines(string(out)) { + if k, v, ok := strings.Cut(strings.TrimSuffix(line, "\n"), "="); ok { + env[k] = v + } + } + + wantDir := um.HomeDir + if os.Getenv("SKIP_FILE_OPS") == "1" { + wantDir = "/" + } else if _, err := os.Stat(um.HomeDir); err != nil { + // The session starts in / if the home directory doesn't + // exist. On Ubuntu, pam_mkhomedir creates it during su + // sessions, but the in-process path doesn't run PAM. + wantDir = "/" + } + wantEnv := map[string]string{ + "USER": "testuser", + "LOGNAME": "testuser", + "HOME": um.HomeDir, + "SHELL": um.LoginShell(), + "LANG": "C.UTF-8", + "LC_TIME": "C", + "cwd": wantDir, + } + for k, want := range wantEnv { + if got := env[k]; got != want { + t.Errorf("%s = %q, want %q", k, got, want) + } + } + if got, ok := env["PWD"]; ok && got != env["cwd"] { + t.Errorf("PWD = %q, but working directory is %q", got, env["cwd"]) + } + if path := strings.Split(env["PATH"], ":"); !slices.Contains(path, "/usr/bin") || !slices.Contains(path, "/bin") { + t.Errorf("PATH = %q, want it to contain /usr/bin and /bin", env["PATH"]) + } + if v, ok := env["NOT_ACCEPTED"]; ok { + t.Errorf("NOT_ACCEPTED = %q, want unset", v) + } + if t.Failed() { + t.Logf("session output:\n%s", out) + } + }) +} + +func TestIntegrationDisablePTYEnvknob(t *testing.T) { + envknob.SetenvForTest(t, "TS_SSH_DISABLE_PTY", "true") + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + + // The pty request itself is accepted, but the session refuses to + // start anything and exits with status 1. + out, _, err := runPTY(t, cl, defaultPTY, "echo ran-with-pty") + if strings.Contains(out, "ran-with-pty") { + t.Errorf("command ran with a pty despite TS_SSH_DISABLE_PTY; output: %q", out) + } + if code, sig := exitStatus(t, err); code != 1 || sig != "" { + t.Errorf("exit status = %d, signal %q; want 1", code, sig) + } + + // Sessions without a pty still work, on the same connection. + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + if out, err := s.CombinedOutput("echo no-pty-ok"); err != nil || string(out) != "no-pty-ok\n" { + t.Errorf("non-pty session: output %q, err %v; want %q", out, err, "no-pty-ok\n") + } + }) +} + +func TestIntegrationDisableSFTPEnvknob(t *testing.T) { + envknob.SetenvForTest(t, "TS_SSH_DISABLE_SFTP", "true") + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + if scl, err := sftp.NewClient(cl); err == nil { + scl.Close() + t.Fatal("sftp.NewClient succeeded despite TS_SSH_DISABLE_SFTP") + } + + // The server accepts the subsystem request, then explains the + // refusal on stderr and closes the channel. + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + stderr, err := s.StderrPipe() + if err != nil { + t.Fatal(err) + } + if err := s.RequestSubsystem("sftp"); err != nil { + t.Fatalf("RequestSubsystem: %v", err) + } + if got, err := io.ReadAll(stderr); err != nil || !strings.Contains(string(got), "sftp disabled") { + t.Errorf("sftp subsystem stderr = %q, %v; want it to contain %q", got, err, "sftp disabled") + } + + s2, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s2.Close() + if out, err := s2.CombinedOutput("echo exec-ok"); err != nil || string(out) != "exec-ok\n" { + t.Errorf("exec session: output %q, err %v; want %q", out, err, "exec-ok\n") + } + }) +} + +// TestIntegrationX11Rejected tests that X11 forwarding, which Tailscale +// SSH does not support, is refused without breaking the session. +func TestIntegrationX11Rejected(t *testing.T) { + // x11Request is the "x11-req" payload from RFC 4254, section 6.3.1. + type x11Request struct { + SingleConnection bool + AuthProtocol string + AuthCookie string + ScreenNumber uint32 + } + payload := ssh.Marshal(x11Request{ + AuthProtocol: "MIT-MAGIC-COOKIE-1", + AuthCookie: "0123456789abcdef0123456789abcdef", + }) + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + for _, withPTY := range []bool{false, true} { + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + if withPTY { + if err := s.RequestPty("xterm", 24, 80, ssh.TerminalModes{}); err != nil { + t.Fatalf("RequestPty: %v", err) + } + } + ok, err := s.SendRequest("x11-req", true, payload) + if err != nil { + t.Fatalf("x11-req: %v", err) + } + if ok { + t.Errorf("x11-req accepted (pty=%v), want rejection", withPTY) + } + out, err := s.CombinedOutput(`printf 'display=%s\n' "${DISPLAY:-unset}"`) + if err != nil { + t.Fatalf("command after x11-req (pty=%v): %v; output: %q", withPTY, err, out) + } + if got := stripCR(string(out)); !strings.Contains(got, "display=unset\n") { + t.Errorf("output after x11-req (pty=%v) = %q, want display=unset", withPTY, got) + } + } + }) +} + +// TestIntegrationOpenSSHInteractivePTY drives an interactive login shell +// with the system OpenSSH client and a forced pty ("ssh -tt"). +func TestIntegrationOpenSSHInteractivePTY(t *testing.T) { + sshPath, err := exec.LookPath("ssh") + if err != nil { + t.Skipf("skipping without OpenSSH client: %v", err) + } + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + addr := testServerWithOpts(t, testServerOpts{username: "testuser", forceV1Behavior: forceV1Behavior}) + host, port, err := net.SplitHostPort(addr) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + // The options match TestOpenSSHExitCodes, but with -tt instead + // of -T. Auth is pinned to "none". + cmd := exec.CommandContext(ctx, sshPath, + "-F", "/dev/null", + "-tt", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=15", + "-o", "GSSAPIAuthentication=no", + "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "HostbasedAuthentication=no", + "-o", "IdentityAgent=none", + "-o", "KbdInteractiveAuthentication=no", + "-o", "NumberOfPasswordPrompts=0", + "-o", "PasswordAuthentication=no", + "-o", "PreferredAuthentications=none", + "-o", "PubkeyAuthentication=no", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-p", port, + "testuser@"+host, + ) + p := &ptyShell{} + if p.stdin, err = cmd.StdinPipe(); err != nil { + t.Fatal(err) + } + cmd.Stdout = &p.out + cmd.Stderr = &p.out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitc := make(chan error, 1) + go func() { waitc <- cmd.Wait() }() + + if got := p.queryValue(t, `printf 'user=%s\n' "$(id -un)"`, "user"); got != "testuser" { + t.Errorf("id -un = %q, want testuser", got) + } + loginWaits := p.loginWaitsForShell(t) + p.send(t, "exit 5") + + select { + case err = <-waitc: + case <-ctx.Done(): + t.Fatalf("timeout waiting for ssh to exit; output:\n%s", p.output()) + } + code := 0 + if ee, ok := errors.AsType[*exec.ExitError](err); ok { + code = ee.ExitCode() + } else if err != nil { + t.Fatalf("ssh: %v", err) + } + if loginWaits { + // See TestIntegrationPTYExitStatus. + t.Logf("KNOWN BUG: login discards the shell's exit status; ssh exit code = %d", code) + } else if code != 5 { + t.Errorf("ssh exit code = %d, want 5; output:\n%s", code, p.output()) + } + }) +} + +// TestIntegrationPTYExitStatus tests the exit status the server reports +// when a session's shell exits with a code or is killed by a signal. +// +// The server never sends an "exit-signal" message. When the session's +// process dies from a signal, it reports the shell convention of 128 +// plus the signal number as the exit status, except on the in-process +// path, which reports 1 with a "[tailscale-ssh: process died: ...]" +// message on stderr. +// +// KNOWN BUG: this pins the current behavior. OpenSSH's sshd sends +// "exit-signal" for a signal death, and the two paths should agree. +func TestIntegrationPTYExitStatus(t *testing.T) { + // checkSignaled checks the exit status of a session whose shell + // killed itself with signal number signum. + checkSignaled := func(t *testing.T, err error, output string, signum int) { + t.Helper() + code, sig := exitStatus(t, err) + if sig != "" { + t.Errorf("got exit-signal %q; the server is not expected to send one", sig) + } + want := 128 + signum + if strings.Contains(output, "[tailscale-ssh: process died:") { + want = 1 + } + if code != want { + t.Errorf("exit status = %d, want %d; output: %q", code, want, output) + } + } + forEachBehavior(t, func(t *testing.T, forceV1Behavior bool) { + cl := dialSessionTestClient(t, testServerOpts{forceV1Behavior: forceV1Behavior}) + // startShell starts an interactive shell. It skips the test if + // the shell runs under a login that exits 0 regardless + // of how the shell exited. + startShell := func(t *testing.T) *ptyShell { + p := startPTYShell(t, cl, defaultPTY) + if p.loginWaitsForShell(t) { + t.Skip("KNOWN BUG: login discards the shell's exit status") + } + return p + } + t.Run("shell_exit_code", func(t *testing.T) { + p := startShell(t) + p.send(t, "exit 7") + if code, sig := p.wait(t); code != 7 || sig != "" { + t.Errorf("exit status = %d, signal %q; want 7; output:\n%s", code, sig, p.output()) + } + }) + t.Run("shell_signal", func(t *testing.T) { + p := startShell(t) + // Interactive shells ignore SIGTERM, so use SIGKILL. + p.send(t, "kill -KILL $$") + select { + case err := <-p.waitc: + checkSignaled(t, err, p.output(), 9) + case <-time.After(20 * time.Second): + t.Fatalf("timeout waiting for shell to exit; output:\n%s", p.output()) + } + }) + t.Run("exec_pty_signal", func(t *testing.T) { + out, _, err := runPTY(t, cl, defaultPTY, "kill -TERM $$") + checkSignaled(t, err, out, 15) + }) + t.Run("exec_signal", func(t *testing.T) { + s, err := cl.NewSession() + if err != nil { + t.Fatal(err) + } + defer s.Close() + out, err := s.CombinedOutput("kill -TERM $$") + checkSignaled(t, err, string(out), 15) + }) + }) +} diff --git a/ssh/tailssh/tailssh_integration_test.go b/ssh/tailssh/tailssh_integration_test.go index f68605de1..41ca753bf 100644 --- a/ssh/tailssh/tailssh_integration_test.go +++ b/ssh/tailssh/tailssh_integration_test.go @@ -38,7 +38,9 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" "tailscale.com/net/tsdial" + "tailscale.com/syncs" "tailscale.com/tailcfg" + "tailscale.com/tailcfg/nodecap" "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/netmap" @@ -936,29 +938,12 @@ func testClient(t *testing.T, forceV1Behavior bool, allowSendEnv bool, authMetho } func testServer(t *testing.T, username string, forceV1Behavior bool, allowSendEnv bool) string { - srv := &server{ - lb: &testBackend{localUser: username, forceV1Behavior: forceV1Behavior, allowSendEnv: allowSendEnv}, - logf: log.Printf, - tailscaledPath: os.Getenv("TAILSCALED_PATH"), - timeNow: time.Now, - } - - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { l.Close() }) - - go func() { - for { - conn, err := l.Accept() - if err == nil { - go srv.HandleSSHConn(&addressFakingConn{conn}) - } - } - }() - - return l.Addr().String() + t.Helper() + return testServerWithOpts(t, testServerOpts{ + username: username, + forceV1Behavior: forceV1Behavior, + allowSendEnv: allowSendEnv, + }) } type testServerOpts struct { @@ -968,22 +953,40 @@ type testServerOpts struct { allowLocalPortForwarding bool allowRemotePortForwarding bool logf logger.Logf // defaults to log.Printf + varRoot string // TailscaleVarRoot; defaults to testVarRoot + + // modifyRule, if non-nil, adjusts the single SSH rule the test + // backend's netmap returns. It can be replaced later with + // [testBackend.setModifyRule]. + modifyRule func(*tailcfg.SSHRule) } func testServerWithOpts(t *testing.T, opts testServerOpts) string { + t.Helper() + _, _, addr := startTestServer(t, opts) + return addr +} + +// startTestServer starts a Tailscale SSH server configured by opts, +// listening on a random localhost port. It returns the server, its +// backend, and the address it listens on. +func startTestServer(t *testing.T, opts testServerOpts) (*server, *testBackend, string) { t.Helper() logf := opts.logf if logf == nil { logf = log.Printf } + tb := &testBackend{ + localUser: opts.username, + forceV1Behavior: opts.forceV1Behavior, + allowSendEnv: opts.allowSendEnv, + allowLocalPortForwarding: opts.allowLocalPortForwarding, + allowRemotePortForwarding: opts.allowRemotePortForwarding, + varRoot: opts.varRoot, + } + tb.setModifyRule(opts.modifyRule) srv := &server{ - lb: &testBackend{ - localUser: opts.username, - forceV1Behavior: opts.forceV1Behavior, - allowSendEnv: opts.allowSendEnv, - allowLocalPortForwarding: opts.allowLocalPortForwarding, - allowRemotePortForwarding: opts.allowRemotePortForwarding, - }, + lb: tb, logf: logf, tailscaledPath: os.Getenv("TAILSCALED_PATH"), timeNow: time.Now, @@ -998,13 +1001,14 @@ func testServerWithOpts(t *testing.T, opts testServerOpts) string { go func() { for { conn, err := l.Accept() - if err == nil { - go srv.HandleSSHConn(&addressFakingConn{conn}) + if err != nil { + return } + go srv.HandleSSHConn(&addressFakingConn{conn}) } }() - return l.Addr().String() + return srv, tb, l.Addr().String() } func testSession(t *testing.T, forceV1Behavior bool, allowSendEnv bool, sendEnv map[string]string) *session { @@ -1069,6 +1073,16 @@ type testBackend struct { allowSendEnv bool allowLocalPortForwarding bool allowRemotePortForwarding bool + varRoot string // or empty for testVarRoot + + modifyRule syncs.AtomicValue[func(*tailcfg.SSHRule)] // or nil +} + +// setModifyRule sets the func that adjusts the SSH rule returned by +// [testBackend.NetMap]. Callers that want existing connections to +// observe the change must also call [server.OnPolicyChange]. +func (tb *testBackend) setModifyRule(f func(*tailcfg.SSHRule)) { + tb.modifyRule.Store(f) } func (tb *testBackend) ShouldRunSSH() bool { @@ -1076,30 +1090,30 @@ func (tb *testBackend) ShouldRunSSH() bool { } func (tb *testBackend) NetMap() *netmap.NetworkMap { - capMap := make(set.Set[tailcfg.NodeCapability]) + capMap := make(set.Set[nodecap.Cap]) if tb.forceV1Behavior { - capMap[tailcfg.NodeAttrSSHBehaviorV1] = struct{}{} + capMap[nodecap.SSHBehaviorV1] = struct{}{} } if tb.allowSendEnv { - capMap[tailcfg.NodeAttrSSHEnvironmentVariables] = struct{}{} + capMap[nodecap.SSHEnvironmentVariables] = struct{}{} + } + rule := &tailcfg.SSHRule{ + Principals: []*tailcfg.SSHPrincipal{{Any: true}}, + Action: &tailcfg.SSHAction{ + Accept: true, + AllowAgentForwarding: true, + AllowLocalPortForwarding: tb.allowLocalPortForwarding, + AllowRemotePortForwarding: tb.allowRemotePortForwarding, + }, + SSHUsers: map[string]string{"*": tb.localUser}, + AcceptEnv: []string{"GIT_*", "EXACT_MATCH", "TEST?NG"}, + } + if f := tb.modifyRule.Load(); f != nil { + f(rule) } return &netmap.NetworkMap{ - SSHPolicy: &tailcfg.SSHPolicy{ - Rules: []*tailcfg.SSHRule{ - { - Principals: []*tailcfg.SSHPrincipal{{Any: true}}, - Action: &tailcfg.SSHAction{ - Accept: true, - AllowAgentForwarding: true, - AllowLocalPortForwarding: tb.allowLocalPortForwarding, - AllowRemotePortForwarding: tb.allowRemotePortForwarding, - }, - SSHUsers: map[string]string{"*": tb.localUser}, - AcceptEnv: []string{"GIT_*", "EXACT_MATCH", "TEST?NG"}, - }, - }, - }, - AllCaps: capMap, + SSHPolicy: &tailcfg.SSHPolicy{Rules: []*tailcfg.SSHRule{rule}}, + AllCaps: capMap, } } @@ -1120,7 +1134,7 @@ func (tb *testBackend) Dialer() *tsdial.Dialer { } func (tb *testBackend) TailscaleVarRoot() string { - return testVarRoot + return cmp.Or(tb.varRoot, testVarRoot) } func (tb *testBackend) NodeKey() key.NodePublic { diff --git a/ssh/tailssh/testcontainers/Dockerfile b/ssh/tailssh/testcontainers/Dockerfile index 9d662ca1a..9c57273d5 100644 --- a/ssh/tailssh/testcontainers/Dockerfile +++ b/ssh/tailssh/testcontainers/Dockerfile @@ -45,7 +45,11 @@ RUN set -e && \ TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run TestLocalUnixForwarding && \ TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run TestReverseUnixForwarding && \ TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run TestUnixForwardingDenied && \ - TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run TestUnixForwardingPathRestriction + TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run TestUnixForwardingPathRestriction && \ + echo "Run every other integration test, skipping the ones already run above." && \ + TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v \ + -test.run 'TestIntegration|TestOpenSSHExitCodes|TestLocalUnixForwardingHalfClose' \ + -test.skip '^(TestIntegrationSFTP|TestIntegrationSCP|TestIntegrationSSH|TestIntegrationSSHNoLeakedPTY|TestIntegrationParamiko)$' # Run tests as non-root user testuser and make sure tests still pass. RUN set -e && \ @@ -53,7 +57,7 @@ RUN set -e && \ chown testuser:groupone /tmp/tailscalessh.log && \ export TAILSCALED_PATH=$(pwd)/tailscaled && \ eval $(su -m testuser -c "ssh-agent -s") && \ - su -m testuser -c "./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration|TestDoDropPrivileges'" && \ + su -m testuser -c "./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration|TestOpenSSHExitCodes|TestLocalUnixForwardingHalfClose|TestDoDropPrivileges'" && \ echo "Also, deny everyone access to the user's home directory and make sure non file-related tests still pass." && \ mkdir -p /home/testuser && chown testuser:groupone /home/testuser && chmod 0000 /home/testuser && \ SKIP_FILE_OPS=1 su -m testuser -c "./tailssh.test -test.v -test.run TestIntegrationSSH" && \ @@ -70,7 +74,7 @@ RUN if echo "$BASE" | grep -q "ubuntu:"; then \ printf "#!/bin/bash\necho 'Enforcing'" > /usr/bin/getenforce && \ chmod 755 /usr/bin/getenforce && \ eval $(ssh-agent -s) && \ - TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration' && \ + TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration|TestOpenSSHExitCodes|TestLocalUnixForwardingHalfClose' && \ mv /tmp/login_orig /usr/bin/login && \ rm /usr/bin/getenforce \ ; fi @@ -92,4 +96,4 @@ RUN set -e && \ chown root:root /tmp/tailscalessh.log && \ rm $(which su) && \ eval $(ssh-agent -s) && \ - TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration|TestDoDropPrivileges' + TAILSCALED_PATH=$(pwd)/tailscaled ./tailssh.test -test.v -test.run 'TestSSHAgentForwarding|TestIntegration|TestOpenSSHExitCodes|TestLocalUnixForwardingHalfClose|TestDoDropPrivileges' diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index 5872df1bf..16afe64cb 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -200,6 +200,10 @@ type Server struct { msgToSend map[key.NodePublic][]any // FIFO queue per node; values are *tailcfg.PingRequest or *tailcfg.MapResponse allExpired bool // All nodes will be told their node key is expired. + // machineSSHHandler, if non-nil, serves Noise requests under + // /machine/ssh/. See [Server.SetMachineSSHHandler]. + machineSSHHandler http.Handler + // tkaStorage records the Tailnet Lock state, if any. // If nil, Tailnet Lock is not enabled in the Tailnet. tkaStorage tka.CompactableChonk @@ -422,6 +426,7 @@ func (s *Server) initMux() { s.mux.HandleFunc("/machine/tka/", s.serveTKA) s.mux.HandleFunc("/machine/webclient/", s.serveWebClient) s.mux.HandleFunc("/machine/", s.serveMachine) + s.mux.HandleFunc("/machine/ssh/", s.serveMachineSSH) s.mux.HandleFunc("/ts2021", s.serveNoiseUpgrade) s.mux.HandleFunc("/c2n/", s.serveC2N) } @@ -577,6 +582,33 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) { } } +// SetMachineSSHHandler sets the handler for Noise requests from nodes to +// paths under /machine/ssh/. Tests can point a [tailcfg.SSHAction]'s +// HoldAndDelegate URL (or a [tailcfg.SSHRecorderFailureAction]'s NotifyURL) +// at such a path, with any host, to model the control plane's check mode +// and SSH event endpoints. A nil h, the default, makes those requests fail +// with 404 Not Found. +func (s *Server) SetMachineSSHHandler(h http.Handler) { + s.mu.Lock() + defer s.mu.Unlock() + s.machineSSHHandler = h +} + +func (s *Server) serveMachineSSH(w http.ResponseWriter, r *http.Request) { + if _, ok := r.Context().Value(peerMachinePublicContextKey{}).(key.MachinePublic); !ok { + http.Error(w, "Noise transport required", http.StatusForbidden) + return + } + s.mu.Lock() + h := s.machineSSHHandler + s.mu.Unlock() + if h == nil { + http.NotFound(w, r) + return + } + h.ServeHTTP(w, r) +} + // serveWebClient handles the Noise-protected web client auth flow endpoints // posted to /machine/webclient/init//to/ and // /machine/webclient/wait//to//. It is the test-control diff --git a/tstest/natlab/vmtest/ssh_features_test.go b/tstest/natlab/vmtest/ssh_features_test.go new file mode 100644 index 000000000..00cc06f9f --- /dev/null +++ b/tstest/natlab/vmtest/ssh_features_test.go @@ -0,0 +1,788 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/netip" + "strconv" + "strings" + "testing" + "time" + + "github.com/creachadair/mds/shell" + "tailscale.com/tailcfg" + "tailscale.com/tailcfg/nodecap" + "tailscale.com/tstest" +) + +// This file has the subtests of TestTailscaleSSH_Ubuntu (and some of +// TestTailscaleSSH_FreeBSD) that drive the OpenSSH client on the client VM +// through the tailnet to the server's Tailscale SSH server. + +// anyPrincipal is an SSH policy principal list matching any connection. +var anyPrincipal = []*tailcfg.SSHPrincipal{{Any: true}} + +// featurePolicy returns an SSH policy like the default one installed by +// [vmtest.TailscaleSSH], but which also permits agent forwarding and TCP port +// forwarding, and accepts the VMTEST_ACCEPTED environment variable. +func featurePolicy() *tailcfg.SSHPolicy { + return &tailcfg.SSHPolicy{ + Rules: []*tailcfg.SSHRule{{ + Principals: anyPrincipal, + SSHUsers: map[string]string{"*": "=", "root": "root"}, + Action: &tailcfg.SSHAction{ + Accept: true, + AllowAgentForwarding: true, + AllowLocalPortForwarding: true, + AllowRemotePortForwarding: true, + }, + AcceptEnv: []string{"VMTEST_ACCEPTED"}, + }}, + } +} + +// acceptPolicy returns an SSH policy with a single rule mapping users per +// sshUsers and applying action. +func acceptPolicy(sshUsers map[string]string, action *tailcfg.SSHAction) *tailcfg.SSHPolicy { + return &tailcfg.SSHPolicy{ + Rules: []*tailcfg.SSHRule{{ + Principals: anyPrincipal, + SSHUsers: sshUsers, + Action: action, + }}, + } +} + +// enableSSHEnvironmentVariables grants the server the +// [nodecap.SSHEnvironmentVariables] node attribute, without which +// Tailscale SSH ignores [tailcfg.SSHRule.AcceptEnv]. +func (st *sshTest) enableSSHEnvironmentVariables(t *testing.T) { + t.Helper() + nodeKey := st.env.Status(st.server).Self.PublicKey + st.env.ControlServer().SetNodeCapMap(nodeKey, tailcfg.NodeCapMap{ + nodecap.SSHEnvironmentVariables: nil, + }) + if err := tstest.WaitFor(30*time.Second, func() error { + if !st.env.Status(st.server).Self.HasCap(nodecap.SSHEnvironmentVariables) { + return fmt.Errorf("server lacks %v", nodecap.SSHEnvironmentVariables) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +// checkContains reports a test error for each of wants that out doesn't +// contain. +func checkContains(t *testing.T, desc, out string, wants ...string) { + t.Helper() + for _, want := range wants { + if !strings.Contains(out, want) { + t.Errorf("%s: output does not contain %q:\n%s", desc, want, out) + } + } +} + +// bgPrefix is the path prefix of the files that hold the output and exit +// code of a script started by [sshTest.startBackground]. +const bgPrefix = "/tmp/vmtest-bg-" + +// startBackground starts script in the background on [sshTest.client], +// recording its combined output and its exit code for +// [sshTest.backgroundOutput] and [sshTest.waitBackground]. +func (st *sshTest) startBackground(t *testing.T, name, script string) { + t.Helper() + p := bgPrefix + name + st.mustRun(t, fmt.Sprintf("rm -f %[1]s.out %[1]s.exit\n"+ + "( ( %[2]s ) >%[1]s.out 2>&1 %[1]s.exit ) >/dev/null 2>&1 deadline: + proc.kill() + sys.stdout.buffer.write(out) + sys.exit(124) + if not ready and now >= next_probe: + proc.stdin.write(b"echo vmtest-ready-$((40+2))\n") + proc.stdin.flush() + next_probe = now + 1 + if select.select([proc.stdout], [], [], 0.1)[0]: + data = os.read(proc.stdout.fileno(), 4096) + if not data: + break + out += data + if not ready and b"vmtest-ready-42" in out: + ready = True + proc.stdin.write(os.environ["VMTEST_PTY_INPUT"].encode() + b"\n") + proc.stdin.flush() +sys.stdout.buffer.write(out) +sys.exit(proc.wait()) +` + +// testPTY tests sessions with a pty, both an interactive login shell typed +// into via stdin and a command run with a pty. +func (st *sshTest) testPTY(t *testing.T) { + // The arithmetic makes the command's output differ from the pty's echo + // of the typed command. + out, code := st.run(t, "cat >/tmp/vmtest-pty.py <<'EOF'"+ptyDriverPy+"EOF\n"+ + `VMTEST_PTY_INPUT=$(printf 'tty\necho pty-$((6*7))\nexit 3\n') python3 /tmp/vmtest-pty.py `+st.sshCmd("tsuser", "-tt")) + // TODO: require exit status 3. Tailscale SSH runs interactive shells + // with a pty under login(1), and both util-linux's and FreeBSD's login + // exit 0 regardless of the shell's exit status, whereas OpenSSH's sshd + // passes it through. + switch code { + case 3: + case 124: + t.Fatalf("interactive shell timed out; output:\n%s", out) + default: + t.Logf("KNOWN BUG: interactive shell exited %d, want 3", code) + } + checkContains(t, "interactive shell", out, "/dev/pts/", "pty-42") + + // With a pty, the command's stderr goes to the pty, which ssh writes to + // its stdout, so discard ssh's own stderr. + out, code = st.run(t, "timeout 60 "+st.sshCmd("tsuser", "-tt")+" "+shell.Quote("tty; echo pty-stderr-$((6*7)) >&2; exit 5")+" 2>/dev/null") + if code != 5 { + t.Errorf("command with pty exited %d, want 5; output:\n%s", code, out) + } + checkContains(t, "command with pty", out, "/dev/pts/", "pty-stderr-42") +} + +// testExitCodes tests that the remote command's exit code is the ssh client's. +func (st *sshTest) testExitCodes(t *testing.T) { + for _, tc := range []struct { + cmd string + want int + }{ + {"true", 0}, + {"exit 42", 42}, + {"vmtest-no-such-command", 127}, + } { + if out, code := st.ssh(t, "tsuser", tc.cmd); code != tc.want { + t.Errorf("ssh %q exited %d, want %d; output:\n%s", tc.cmd, code, tc.want, out) + } + } +} + +// lastLines returns the last n lines of s, failing the test if it has fewer. +func lastLines(t *testing.T, s string, n int) []string { + t.Helper() + lines := strings.Split(s, "\n") + if len(lines) < n { + t.Fatalf("want at least %d lines of output; got:\n%s", n, s) + } + return lines[len(lines)-n:] +} + +// testSFTP tests uploading and downloading with the OpenSSH sftp client +// in batch mode, as a non-root user. +func (st *sshTest) testSFTP(t *testing.T) { + dlSum := st.mustSSH(t, "tsuser", "head -c 300000 /dev/urandom >sftp-dl.bin && cksum up.bin +printf 'put up.bin sftp-up.bin\nget sftp-dl.bin dl.bin\n' >batch +sftp -b batch `+strings.TrimSpace(sshOpts)+" "+st.dest("tsuser")+` >/dev/null +cksum scp-dl.bin && cksum up.bin +`+scp+" up.bin "+st.dest("tsuser")+`:scp-up.bin +`+scp+" "+st.dest("tsuser")+`:scp-dl.bin dl.bin +cksum /dev/null +trap 'ssh-agent -k >/dev/null' EXIT +ssh-add -q /tmp/vmtest-agentkey +echo "fingerprint=$(ssh-keygen -lf /tmp/vmtest-agentkey.pub | cut -d' ' -f2)" +set +e +` + st.sshCmd("tsuser", "-A") + " 'ssh-add -l'" + + out, code := st.run(t, script) + if code != 0 { + t.Fatalf("ssh -A ... ssh-add -l exited %d:\n%s", code, out) + } + _, fp, ok := strings.Cut(out, "fingerprint=") + if !ok { + t.Fatalf("no fingerprint in output:\n%s", out) + } + fp, _, _ = strings.Cut(fp, "\n") + checkContains(t, "ssh-add -l on server", out, fp+" vmtest-agent-key") + + t.Run("NotPermitted", func(t *testing.T) { + st.withPolicy(t, acceptPolicy(map[string]string{"*": "="}, &tailcfg.SSHAction{Accept: true})) + // ssh-add exits 2 if it can't connect to an agent. + if out, code := st.run(t, script); code != 2 { + t.Errorf("ssh -A ... ssh-add -l with agent forwarding not permitted exited %d, want 2:\n%s", code, out) + } + }) +} + +// testAcceptEnv tests that "ssh -o SendEnv" delivers variables the policy's +// AcceptEnv permits, and only those. +func (st *sshTest) testAcceptEnv(t *testing.T) { + out := st.mustRun(t, "VMTEST_ACCEPTED=yes VMTEST_REJECTED=no "+ + st.sshCmd("tsuser", "-o SendEnv=VMTEST_ACCEPTED", "-o SendEnv=VMTEST_REJECTED")+ + " "+shell.Quote(`echo "accepted=${VMTEST_ACCEPTED-unset} rejected=${VMTEST_REJECTED-unset}"`)) + if want := "accepted=yes rejected=unset"; out != want { + t.Errorf("got %q, want %q", out, want) + } +} + +// largeTransferTimeout bounds each direction of [sshTest.testLargeTransfer]. +// The transfers normally take a few seconds; the bound is generous to avoid +// flakes on loaded CI machines while still catching stalls. +const largeTransferTimeout = 90 * time.Second + +// testLargeTransfer pipes 64 MiB through an SSH session in each direction. +func (st *sshTest) testLargeTransfer(t *testing.T) { + const size = 64 << 20 + for _, tc := range []struct { + name string + script string + }{ + {"Upload", fmt.Sprintf("head -c %d /dev/zero | %s 'wc -c'", size, st.sshCmd("tsuser"))}, + {"Download", fmt.Sprintf("%s 'head -c %d /dev/zero' | wc -c", st.sshCmd("tsuser"), size)}, + } { + start := time.Now() + out := st.mustRun(t, tc.script) + d := time.Since(start) + if got := strings.TrimSpace(out); got != strconv.Itoa(size) { + t.Errorf("%s: got %q bytes, want %d", tc.name, got, size) + } + t.Logf("%s: %d bytes in %v (%.1f MB/s)", tc.name, size, d.Round(time.Millisecond), float64(size)/1e6/d.Seconds()) + if d > largeTransferTimeout { + t.Errorf("%s took %v, want at most %v", tc.name, d, largeTransferTimeout) + } + } +} + +// testControlMaster tests multiple concurrent session channels on one SSH +// connection, using OpenSSH connection multiplexing. +func (st *sshTest) testControlMaster(t *testing.T) { + const ctl = "-o ControlPath=/tmp/vmtest-mux.sock" + st.mustRun(t, "rm -f /tmp/vmtest-mux.sock\n"+ + st.sshCmd("tsuser", ctl, "-o ControlMaster=yes", "-o ControlPersist=120", "-f", "-N")+" /dev/null 2>&1") + t.Cleanup(func() { st.run(t, st.sshCmd("tsuser", ctl, "-O exit")) }) + + out := st.mustRun(t, st.sshCmd("tsuser", ctl, "-O check")) + checkContains(t, "ssh -O check", out, "Master running") + + // Each session creates a file and then waits for all sessions' files to + // exist, so they only all succeed if they run concurrently. + const numSessions = 4 + st.mustSSH(t, "tsuser", "rm -rf /tmp/vmtest-mux && mkdir /tmp/vmtest-mux") + var script strings.Builder + for i := range numSessions { + remote := fmt.Sprintf("touch /tmp/vmtest-mux/%d; n=0; "+ + "while [ $(ls /tmp/vmtest-mux | wc -l) -lt %d ]; do "+ + "n=$((n+1)); if [ $n -gt 300 ]; then echo mux-%d-timeout; exit 1; fi; sleep 0.1; "+ + "done; echo mux-%d-ok", i, numSessions, i, i) + fmt.Fprintf(&script, "%s %s >/tmp/vmtest-mux.%d.out 2>&1 &\n", + st.sshCmd("tsuser", ctl, "-o ControlMaster=no"), shell.Quote(remote), i) + } + script.WriteString("wait\ncat /tmp/vmtest-mux.*.out\n") + out = st.mustRun(t, script.String()) + for i := range numSessions { + checkContains(t, "multiplexed sessions", out, fmt.Sprintf("mux-%d-ok", i)) + } + + // All the sessions went over the master's one TCP connection. + out = st.mustRun(t, "ss -Htn state established dst "+st.serverIP.String()+":22 | wc -l") + if out != "1" { + t.Errorf("%s TCP connections to the server's SSH port, want 1", out) + } +} + +// pollCurl is a shell fragment that fetches $url, retrying for up to about +// 10 seconds while the forwarding it goes through is set up. +const pollCurl = `ok=1 +for i in $(seq 100); do + if curl -sf "$url"; then ok=0; break; fi + sleep 0.1 +done` + +// testLocalPortForwarding tests "ssh -L" to a TCP listener on the server's +// loopback interface, and that "ssh -W" (which uses the same channel type) +// is refused when the policy doesn't permit local port forwarding. +func (st *sshTest) testLocalPortForwarding(t *testing.T) { + st.serverExec(t, "mkdir -p /tmp/vmtest-lfwd && echo local-forward-ok >/tmp/vmtest-lfwd/fwd.txt && "+ + "(setsid python3 -m http.server 18123 --bind 127.0.0.1 --directory /tmp/vmtest-lfwd /dev/null 2>&1 &)") + t.Cleanup(func() { st.env.SSHExec(st.server, "pkill -f '[h]ttp.server 18123'") }) + + out := st.mustRun(t, st.sshCmd("tsuser", "-N", "-o ExitOnForwardFailure=yes", "-L 127.0.0.1:19123:127.0.0.1:18123")+ + " /dev/null 2>&1 &\n"+ + "pid=$!\n"+ + "url=http://127.0.0.1:19123/fwd.txt\n"+ + pollCurl+"\n"+ + "kill $pid\n"+ + "exit $ok") + if out != "local-forward-ok" { + t.Errorf("fetch through ssh -L = %q, want %q", out, "local-forward-ok") + } + + t.Run("NotPermitted", func(t *testing.T) { + st.withPolicy(t, acceptPolicy(map[string]string{"*": "="}, &tailcfg.SSHAction{Accept: true})) + out, code := st.run(t, `printf 'GET /fwd.txt HTTP/1.0\r\n\r\n' | `+st.sshCmd("tsuser", "-W 127.0.0.1:18123")) + if code == 0 || strings.Contains(out, "local-forward-ok") { + t.Errorf("ssh -W with local port forwarding not permitted: exit %d, want failure; output:\n%s", code, out) + } + }) +} + +// testRemotePortForwarding tests "ssh -R" from the server's loopback +// interface to a TCP listener on the client's, and that it's refused when the +// policy doesn't permit remote port forwarding. +func (st *sshTest) testRemotePortForwarding(t *testing.T) { + st.mustRun(t, "mkdir -p /tmp/vmtest-rfwd && echo remote-forward-ok >/tmp/vmtest-rfwd/fwd.txt && "+ + "(setsid python3 -m http.server 18124 --bind 127.0.0.1 --directory /tmp/vmtest-rfwd /dev/null 2>&1 &)") + t.Cleanup(func() { st.run(t, "pkill -f '[h]ttp.server 18124'") }) + + remote := "url=http://127.0.0.1:19124/fwd.txt\n" + pollCurl + "\nexit $ok" + fwd := []string{"-o ExitOnForwardFailure=yes", "-R 127.0.0.1:19124:127.0.0.1:18124"} + out := st.mustRun(t, st.sshCmd("tsuser", fwd...)+" "+shell.Quote(remote)) + if out != "remote-forward-ok" { + t.Errorf("fetch through ssh -R = %q, want %q", out, "remote-forward-ok") + } + + t.Run("NotPermitted", func(t *testing.T) { + st.withPolicy(t, acceptPolicy(map[string]string{"*": "="}, &tailcfg.SSHAction{Accept: true})) + out, code := st.run(t, st.sshCmd("tsuser", fwd...)+" 'sleep 10; echo session-finished'") + if code != 255 || strings.Contains(out, "session-finished") { + t.Errorf("ssh -R with remote port forwarding not permitted: exit %d, want 255 before the session finishes; output:\n%s", code, out) + } + checkContains(t, "ssh -R not permitted", out, "remote port forwarding failed") + }) +} + +// testTailscaleSSHCommand tests the "tailscale ssh" CLI wrapper on the client, +// which runs OpenSSH with a ProxyCommand of "tailscale nc" and strict host key +// checking against a known_hosts file it generates from the netmap. +func (st *sshTest) testTailscaleSSHCommand(t *testing.T) { + shortName, _, _ := strings.Cut(st.serverDNSName, ".") + for _, host := range []string{shortName, strings.TrimSuffix(st.serverDNSName, ".")} { + out := st.mustRun(t, "tailscale ssh tsuser@"+host+" id -un") + if out != "tsuser" { + t.Errorf("tailscale ssh tsuser@%s id -un = %q, want %q", host, out, "tsuser") + } + } + + // Tailscale SSH on Ubuntu serves the system's host keys, which the + // server advertised in its Hostinfo for "tailscale ssh" to trust. + hostKey := st.serverExec(t, "cut -d' ' -f1,2 /etc/ssh/ssh_host_ed25519_key.pub") + knownHosts := st.mustRun(t, "cat /root/.config/tailscale/ssh_known_hosts") + checkContains(t, "tailscale ssh known_hosts", knownHosts, + st.serverDNSName+" "+hostKey+"\n", + st.serverIP.String()+" "+hostKey+"\n") +} + +// testBanners tests that the OpenSSH client shows the Message of the policy's +// Accept and Reject actions. +func (st *sshTest) testBanners(t *testing.T) { + // OpenSSH only shows auth banners at LogLevel INFO or above. + const logInfo = "-o LogLevel=INFO" + everyone := map[string]string{"*": "="} + + t.Run("Accept", func(t *testing.T) { + st.withPolicy(t, acceptPolicy(everyone, &tailcfg.SSHAction{Accept: true, Message: "vmtest-accept-banner\n"})) + out, code := st.run(t, st.sshCmd("tsuser", logInfo)+" 'echo session-ran'") + if code != 0 { + t.Fatalf("ssh exited %d:\n%s", code, out) + } + checkContains(t, "accept banner", out, "vmtest-accept-banner", "session-ran") + }) + t.Run("Reject", func(t *testing.T) { + st.withPolicy(t, acceptPolicy(everyone, &tailcfg.SSHAction{Reject: true, Message: "vmtest-reject-banner\n"})) + out, code := st.run(t, st.sshCmd("tsuser", logInfo)+" 'echo session-ran'") + if code != 255 || strings.Contains(out, "session-ran") { + t.Errorf("ssh exited %d, want 255 without running the session; output:\n%s", code, out) + } + checkContains(t, "reject banner", out, "vmtest-reject-banner") + }) +} + +// testUserMapping tests the policy's mapping of SSH user names to local users. +func (st *sshTest) testUserMapping(t *testing.T) { + st.withPolicy(t, acceptPolicy(map[string]string{ + "alice": "tsuser", + "tsuser": "=", + }, &tailcfg.SSHAction{Accept: true})) + + st.check(t, "alice maps to tsuser", "alice", "id -un && pwd", "tsuser\n/home/tsuser") + st.check(t, "tsuser maps to itself", "tsuser", "id -un", "tsuser") + for _, user := range []string{"carol", "root"} { + out, code := st.run(t, st.sshCmd(user, "-o LogLevel=INFO")+" 'echo session-ran'") + if code != 255 || strings.Contains(out, "session-ran") { + t.Errorf("unmapped user %q: ssh exited %d, want 255 without running the session; output:\n%s", user, code, out) + } + checkContains(t, "unmapped user "+user, out, "tailnet policy does not permit you to SSH") + } +} + +// testRevokeActiveSession tests that a policy change revoking a user's +// access terminates that user's established sessions, with and without a pty, +// killing their processes. +func (st *sshTest) testRevokeActiveSession(t *testing.T) { + for _, tc := range []struct { + name string + flags []string + sleep string // distinctive, so pgrep only finds this session's process + }{ + {"NoPTY", nil, "sleep 607"}, + {"PTY", []string{"-tt"}, "sleep 608"}, + } { + t.Run(tc.name, func(t *testing.T) { + // The brackets keep pkill from matching the shell running it. + pattern := "'[" + tc.sleep[:1] + "]" + tc.sleep[1:] + "'" + t.Cleanup(func() { + st.run(t, "pkill -f "+pattern) + st.env.SSHExec(st.server, "pkill -f "+pattern) + }) + pgrep := "pgrep -u tsuser -f " + pattern + bgName := "revoke-" + tc.name + + st.startBackground(t, bgName, st.sshCmd("tsuser", tc.flags...)+" '"+tc.sleep+"'") + if err := tstest.WaitFor(30*time.Second, func() error { + _, err := st.env.SSHExec(st.server, pgrep) + return err + }); err != nil { + out, _ := st.backgroundOutput(t, bgName) + t.Fatalf("remote %q never started: %v; ssh output:\n%s", tc.sleep, err, out) + } + + st.withPolicy(t, acceptPolicy(map[string]string{"root": "root"}, &tailcfg.SSHAction{Accept: true})) + + out, code := st.waitBackground(t, bgName, 30*time.Second) + if code != 255 { + t.Errorf("ssh exited %d after its access was revoked, want 255; output:\n%s", code, out) + } + // TODO: require the "Access revoked." message. tailssh writes it + // to the session's stderr asynchronously and closes the + // connection right away, so the client rarely gets it. + if !strings.Contains(out, "Access revoked") { + t.Logf("KNOWN BUG: no revocation message in ssh output:\n%s", out) + } + + // TODO: make this an error. The session's processes survive + // revocation: su -c runs the command in a new session + // (setsid), out of reach of the SIGHUP and SIGKILL that + // tailssh sends the session's process group, and on a pty + // it's no longer in the pty's session either. + if err := tstest.WaitFor(10*time.Second, func() error { + if out, err := st.env.SSHExec(st.server, pgrep); err == nil { + return fmt.Errorf("remote process still running: pid %s", strings.TrimSpace(out)) + } + return nil + }); err != nil { + t.Logf("KNOWN BUG: after revocation: %v", err) + } + }) + } +} + +// testCheckMode tests the HoldAndDelegate ("check mode") flow end to end: +// tailssh shows the action's Message, fetches the delegate URL (with its +// variables expanded) from the control server over Noise, and then applies +// the action the control server returns. +func (st *sshTest) testCheckMode(t *testing.T) { + cs := st.env.ControlServer() + + // hold is a delegate URL fetch awaiting the test's verdict. + type hold struct { + uri string + verdict chan<- *tailcfg.SSHAction + } + holds := make(chan hold) + done := make(chan struct{}) + cs.SetMachineSSHHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + verdict := make(chan *tailcfg.SSHAction, 1) + select { + case holds <- hold{r.URL.RequestURI(), verdict}: + case <-done: + http.Error(w, "test over", http.StatusServiceUnavailable) + return + case <-r.Context().Done(): + return + } + select { + case a := <-verdict: + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(a) + case <-done: + http.Error(w, "test over", http.StatusServiceUnavailable) + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { + close(done) + cs.SetMachineSSHHandler(nil) + st.run(t, "pkill -f '[v]mtest-check-mode'") + }) + + st.withPolicy(t, acceptPolicy(map[string]string{"*": "="}, &tailcfg.SSHAction{ + Message: "vmtest-check-pending\n", + HoldAndDelegate: "https://unused/machine/ssh/action/$SRC_NODE_ID/to/$DST_NODE_ID?ssh_user=$SSH_USER&local_user=$LOCAL_USER&src_ip=$SRC_NODE_IP", + })) + + // awaitHold starts ssh in the background and waits for its delegate URL + // fetch, checking that the client is shown the hold message and is + // still waiting. + awaitHold := func(t *testing.T, name string) hold { + t.Helper() + st.startBackground(t, name, st.sshCmd("tsuser", "-o LogLevel=INFO")+" 'echo vmtest-check-mode; id -un'") + var h hold + select { + case h = <-holds: + case <-time.After(30 * time.Second): + out, _ := st.backgroundOutput(t, name) + t.Fatalf("no delegate URL fetch; ssh output:\n%s", out) + } + t.Logf("delegate URL fetch: %s", h.uri) + if !strings.HasPrefix(h.uri, "/machine/ssh/action/") { + t.Errorf("delegate URL path = %q, want /machine/ssh/action/ prefix", h.uri) + } + checkContains(t, "delegate URL", h.uri, "ssh_user=tsuser", "local_user=tsuser", "src_ip="+st.clientIP.String()) + + if err := tstest.WaitFor(30*time.Second, func() error { + out, exited := st.backgroundOutput(t, name) + if exited { + return fmt.Errorf("ssh exited while held; output:\n%s", out) + } + if !strings.Contains(out, "vmtest-check-pending") { + return fmt.Errorf("hold message not shown; output:\n%s", out) + } + return nil + }); err != nil { + t.Fatal(err) + } + return h + } + + t.Run("Approve", func(t *testing.T) { + h := awaitHold(t, "check-approve") + h.verdict <- &tailcfg.SSHAction{Accept: true} + out, code := st.waitBackground(t, "check-approve", 30*time.Second) + if code != 0 { + t.Fatalf("ssh exited %d after approval:\n%s", code, out) + } + checkContains(t, "approved session", out, "vmtest-check-mode\ntsuser") + }) + t.Run("Deny", func(t *testing.T) { + h := awaitHold(t, "check-deny") + h.verdict <- &tailcfg.SSHAction{Reject: true, Message: "vmtest-check-denied\n"} + out, code := st.waitBackground(t, "check-deny", 30*time.Second) + if code != 255 || strings.Contains(out, "tsuser") { + t.Errorf("ssh exited %d after denial, want 255 without running the session; output:\n%s", code, out) + } + checkContains(t, "denied session", out, "vmtest-check-denied") + }) +} + +// recorderPy is a minimal session recorder speaking the legacy /record +// protocol of [tailscale.com/sessionrecording]: it accepts a chunked POST +// after a 100 Continue response, writing the body to a new file in the output +// directory as it arrives. It doesn't speak HTTP/2, so tailssh's probe for the +// /v2/record endpoint fails and it falls back to /record. +const recorderPy = ` +import http.server, itertools, os, sys + +port, outdir = int(sys.argv[1]), sys.argv[2] +os.makedirs(outdir, exist_ok=True) +seq = itertools.count() + +class Recorder(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): + if self.path != "/record": + self.send_error(404) + return + with open(os.path.join(outdir, "%d.cast" % next(seq)), "wb") as f: + while True: + size = int(self.rfile.readline().split(b";")[0], 16) + if size == 0: + self.rfile.readline() + break + f.write(self.rfile.read(size)) + f.flush() + self.rfile.readline() + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + +http.server.ThreadingHTTPServer(("0.0.0.0", port), Recorder).serve_forever() +` + +// testRecording tests session recording to a recorder on the client node, +// and the OnRecordingFailure behavior when no recorder is reachable. +func (st *sshTest) testRecording(t *testing.T) { + const port = 8199 + st.mustRun(t, "cat >/tmp/vmtest-recorder.py <<'EOF'"+recorderPy+"EOF\n"+ + "rm -rf /tmp/vmtest-rec\n"+ + fmt.Sprintf("(setsid python3 /tmp/vmtest-recorder.py %d /tmp/vmtest-rec /tmp/vmtest-recorder.log 2>&1 &)", port)) + t.Cleanup(func() { st.run(t, "pkill -f '[v]mtest-recorder.py'") }) + if err := tstest.WaitFor(30*time.Second, func() error { + // The recorder answers GET with an error, which curl without -f + // doesn't mind. + if out, code := st.run(t, fmt.Sprintf("curl -s -o /dev/null http://127.0.0.1:%d/", port)); code != 0 { + return fmt.Errorf("recorder not up: exit %d: %s", code, out) + } + return nil + }); err != nil { + t.Fatal(err) + } + + recorder := netip.AddrPortFrom(st.clientIP, port) + noRecorder := netip.AddrPortFrom(st.clientIP, port-1) // nothing listens here + const rejectMsg = "vmtest-recording-unavailable" + recordingPolicy := func(rec netip.AddrPort, onFailure *tailcfg.SSHRecorderFailureAction) *tailcfg.SSHPolicy { + return acceptPolicy(map[string]string{"*": "="}, &tailcfg.SSHAction{ + Accept: true, + Recorders: []netip.AddrPort{rec}, + OnRecordingFailure: onFailure, + }) + } + + t.Run("Recorded", func(t *testing.T) { + st.withPolicy(t, recordingPolicy(recorder, &tailcfg.SSHRecorderFailureAction{RejectSessionWithMessage: rejectMsg})) + // The arithmetic makes the session's output differ from its + // command, which the recording's header also includes. + if out := st.mustSSH(t, "tsuser", "echo rec-$((40+2))"); out != "rec-42" { + t.Errorf("recorded session output = %q, want %q", out, "rec-42") + } + if err := tstest.WaitFor(30*time.Second, func() error { + out, _ := st.run(t, "cat /tmp/vmtest-rec/*.cast") + if !strings.Contains(out, "rec-42") { + return fmt.Errorf("session output not in recordings:\n%s", out) + } + return nil + }); err != nil { + t.Error(err) + } + }) + t.Run("RejectWithoutRecorder", func(t *testing.T) { + st.withPolicy(t, recordingPolicy(noRecorder, &tailcfg.SSHRecorderFailureAction{RejectSessionWithMessage: rejectMsg})) + out, code := st.ssh(t, "tsuser", "echo session-ran") + // tailssh exits 254 when it refuses a session for want of a recorder. + if code != 254 || strings.Contains(out, "session-ran") { + t.Errorf("ssh exited %d, want 254 without running the session; output:\n%s", code, out) + } + checkContains(t, "rejected session", out, rejectMsg) + }) + t.Run("FailOpenWithoutRecorder", func(t *testing.T) { + st.withPolicy(t, recordingPolicy(noRecorder, nil)) + if out := st.mustSSH(t, "tsuser", "echo session-ran"); out != "session-ran" { + t.Errorf("fail-open session output = %q, want %q", out, "session-ran") + } + }) +} diff --git a/tstest/natlab/vmtest/ssh_test.go b/tstest/natlab/vmtest/ssh_test.go index 8616c7a32..24f0fd1b4 100644 --- a/tstest/natlab/vmtest/ssh_test.go +++ b/tstest/natlab/vmtest/ssh_test.go @@ -4,6 +4,8 @@ package vmtest_test import ( + "bytes" + "encoding/json" "fmt" "net/netip" "strconv" @@ -27,7 +29,9 @@ func TestTailscaleSSH_Ubuntu(t *testing.T) { // testTailscaleSSHForOS exercises the Tailscale SSH server ("tailscale up // --ssh", not a system sshd) on a Linux node of the given distribution, with // an Ubuntu node as the SSH client. It tests logging in as both root and a -// non-root user, which exercises the incubator's su-based login path. +// non-root user, which exercises the incubator's su-based login path, and +// then runs subtests covering the SSH features that the real OpenSSH client +// uses across the real tailnet. func testTailscaleSSHForOS(t *testing.T, serverOS vmtest.OSImageLinux) { testSuite := newTestSuite(t, serverOS.Name, serverOS) server := testSuite.server @@ -45,6 +49,26 @@ func testTailscaleSSHForOS(t *testing.T, serverOS vmtest.OSImageLinux) { t.Errorf("%s nonexistent user: ssh succeeded, want failure:\n%s", serverOS.Name, out) } + testSuite.enableSSHEnvironmentVariables(t) + testSuite.setPolicy(t, featurePolicy()) + + t.Run("PTY", testSuite.testPTY) + t.Run("ExitCodes", testSuite.testExitCodes) + t.Run("SFTP", testSuite.testSFTP) + t.Run("SCP", testSuite.testSCP) + t.Run("AgentForwarding", testSuite.testAgentForwarding) + t.Run("AcceptEnv", testSuite.testAcceptEnv) + t.Run("LargeTransfer", testSuite.testLargeTransfer) + t.Run("ControlMaster", testSuite.testControlMaster) + t.Run("LocalPortForwarding", testSuite.testLocalPortForwarding) + t.Run("RemotePortForwarding", testSuite.testRemotePortForwarding) + t.Run("TailscaleSSHCommand", testSuite.testTailscaleSSHCommand) + t.Run("Banners", testSuite.testBanners) + t.Run("UserMapping", testSuite.testUserMapping) + t.Run("RevokeActiveSession", testSuite.testRevokeActiveSession) + t.Run("CheckMode", testSuite.testCheckMode) + t.Run("Recording", testSuite.testRecording) + testSuite.checkAutogroupNonroot(t, serverOS.Name) } @@ -70,6 +94,9 @@ func TestTailscaleSSH_FreeBSD(t *testing.T) { t.Errorf("freebsd nonexistent user: ssh succeeded, want failure:\n%s", out) } + t.Run("PTY", testSuite.testPTY) + t.Run("SFTP", testSuite.testSFTP) + testSuite.checkAutogroupNonroot(t, "freebsd") } @@ -93,48 +120,108 @@ func TestTailscaleSSH_Gokrazy(t *testing.T) { testSuite.check(t, "gokrazy nonexistent user maps to root", "nosuchuser", "pwd", "/") } -// ssh runs cmd as user on the node at ip over Tailscale SSH, from the -// client node, returning the combined output and the ssh client's exit -// code. The command run via Env.SSHExec always exits 0 so that its -// transport-error (exit 255) retry loop doesn't kick in when a -// Tailscale SSH connection is expected to fail. +// exitMarker precedes the exit code of a script run by [sshTest.run]. The +// script's wrapper always exits 0 so that [vmtest.Env.SSHExec]'s +// transport-error (exit 255) retry loop doesn't kick in when a Tailscale SSH +// connection is expected to fail. const exitMarker = "tailscale-ssh-exit=" +// sshOpts are the OpenSSH client options used for every connection from +// [sshTest.client] to [sshTest.server]. +const sshOpts = " -o StrictHostKeyChecking=no" + + " -o UserKnownHostsFile=/dev/null" + + " -o BatchMode=yes" + + " -o ConnectTimeout=10" + + " -o LogLevel=ERROR" + type sshTest struct { client *vmtest.Node server *vmtest.Node - serverIP netip.Addr + clientIP netip.Addr // client's Tailscale IPv4 address + serverIP netip.Addr // server's Tailscale IPv4 address env *vmtest.Env + + // serverDNSName is the server's MagicDNS name, with a trailing dot. + serverDNSName string } -// ssh runs a command as the specified user on the [sshTest.server] via its -// debug SSH NIC. -func (st *sshTest) ssh(t *testing.T, user string, cmd string) (string, int) { - t.Helper() +// dest returns the "user@host" argument for connecting as user to +// [sshTest.server]. +func (st *sshTest) dest(user string) string { + return user + "@" + st.serverIP.String() +} - inner := "ssh" + - " -o StrictHostKeyChecking=no" + - " -o UserKnownHostsFile=/dev/null" + - " -o BatchMode=yes" + - " -o ConnectTimeout=10" + - " -o LogLevel=ERROR" + - " " + user + "@" + st.serverIP.String() + - " " + shell.Quote(cmd) - out, err := st.env.SSHExec(st.client, inner+" 2>&1; echo "+exitMarker+"$?") +// sshCmd returns the shell command line that runs the OpenSSH client on +// [sshTest.client] to connect as user to [sshTest.server] over Tailscale SSH. +// The extra arguments come before [sshOpts], so "-o" options in extra take +// precedence, as ssh uses the first value it sees for each option. The caller +// appends the remote command, if any. +func (st *sshTest) sshCmd(user string, extra ...string) string { + args := append([]string{"ssh"}, extra...) + args = append(args, strings.TrimSpace(sshOpts), st.dest(user)) + return strings.Join(args, " ") +} + +// run runs the shell script as root on [sshTest.client] via its debug SSH +// NIC, returning the script's combined stdout and stderr (trimmed of +// surrounding whitespace) and its exit code. +func (st *sshTest) run(t *testing.T, script string) (string, int) { + t.Helper() + out, err := st.env.SSHExec(st.client, "(\n"+script+"\n) 2>&1; echo "+exitMarker+"$?") if err != nil { - t.Fatalf("ssh %s@%s: %v\n%s", user, st.serverIP, err, out) + t.Fatalf("running on client: %v\nscript:\n%s\noutput:\n%s", err, script, out) } i := strings.LastIndex(out, exitMarker) if i == -1 { - t.Fatalf("ssh %s@%s: no exit marker in output:\n%s", user, st.serverIP, out) + t.Fatalf("no exit marker in output of script:\n%s\noutput:\n%s", script, out) } code, err := strconv.Atoi(strings.TrimSpace(out[i+len(exitMarker):])) if err != nil { - t.Fatalf("ssh %s@%s: bad exit marker in output:\n%s", user, st.serverIP, out) + t.Fatalf("bad exit marker in output of script:\n%s\noutput:\n%s", script, out) } return strings.TrimSpace(out[:i]), code } +// mustRun is like [sshTest.run] but fails the test if the script exits +// non-zero. +func (st *sshTest) mustRun(t *testing.T, script string) string { + t.Helper() + out, code := st.run(t, script) + if code != 0 { + t.Fatalf("script exited %d:\n%s\noutput:\n%s", code, script, out) + } + return out +} + +// ssh runs cmd as user on [sshTest.server] over Tailscale SSH, from the +// client node, returning the combined output and the ssh client's exit +// code. +func (st *sshTest) ssh(t *testing.T, user string, cmd string) (string, int) { + t.Helper() + return st.run(t, st.sshCmd(user)+" "+shell.Quote(cmd)) +} + +// mustSSH is like [sshTest.ssh] but fails the test if ssh exits non-zero. +func (st *sshTest) mustSSH(t *testing.T, user string, cmd string) string { + t.Helper() + out, code := st.ssh(t, user, cmd) + if code != 0 { + t.Fatalf("ssh %s %q exited %d:\n%s", st.dest(user), cmd, code, out) + } + return out +} + +// serverExec runs cmd as root on [sshTest.server] via its debug SSH NIC, +// not over Tailscale SSH, failing the test if it fails. +func (st *sshTest) serverExec(t *testing.T, cmd string) string { + t.Helper() + out, err := st.env.SSHExec(st.server, cmd) + if err != nil { + t.Fatalf("running %q on server: %v\n%s", cmd, err, out) + } + return strings.TrimSpace(out) +} + // waitSSH waits for the node's Tailscale SSH server to accept a // connection; the first one can race tailscaled's SSH server startup // and WireGuard path setup. @@ -168,6 +255,49 @@ func (st *sshTest) check(t *testing.T, desc, user string, cmd, want string) { } } +// setPolicy sets the tailnet's SSH policy and waits for it to reach +// [sshTest.server]'s netmap, so that new connections are evaluated against +// it. +func (st *sshTest) setPolicy(t *testing.T, policy *tailcfg.SSHPolicy) { + t.Helper() + st.env.ControlServer().SetSSHPolicy(policy) + want, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + if err := tstest.WaitFor(30*time.Second, func() error { + out, err := st.env.Tailscale(st.server, "debug", "netmap") + if err != nil { + return err + } + var nm struct { + SSHPolicy *tailcfg.SSHPolicy + } + if err := json.Unmarshal([]byte(out), &nm); err != nil { + return err + } + got, err := json.Marshal(nm.SSHPolicy) + if err != nil { + return err + } + if !bytes.Equal(got, want) { + return fmt.Errorf("server has SSH policy %s; want %s", got, want) + } + return nil + }); err != nil { + t.Fatalf("SSH policy never reached server: %v", err) + } +} + +// withPolicy sets the tailnet's SSH policy, like [sshTest.setPolicy], and +// restores the previous policy when t finishes. +func (st *sshTest) withPolicy(t *testing.T, policy *tailcfg.SSHPolicy) { + t.Helper() + old := st.env.ControlServer().SSHPolicy + st.setPolicy(t, policy) + t.Cleanup(func() { st.setPolicy(t, old) }) +} + // checkAutogroupNonroot adjusts the server's SSH policy to be equivalent to "autogroup:nonroot" // and verifies that attempting to SSH as root fails. func (st *sshTest) checkAutogroupNonroot(t *testing.T, name string) { @@ -203,13 +333,13 @@ func newTestSuite(t *testing.T, serverName string, serverOS vmtest.OSImage) *ssh server := sshTestNode(env, serverName, serverOS, vmtest.TailscaleSSH()) env.Start() - serverIP := tailscaleIP4(t, env, server) - return &sshTest{ - client: client, - server: server, - serverIP: serverIP, - env: env, + client: client, + server: server, + clientIP: tailscaleIP4(t, env, client), + serverIP: tailscaleIP4(t, env, server), + env: env, + serverDNSName: env.Status(server).Self.DNSName, } }