Files
tailscale/ssh/tailssh/sessmonitor_test.go
James Tucker 1334899864 ssh/tailssh: stop killing session processes on graceful disconnects
Monitor teardown SIGHUPed and SIGKILLed every remembered child group
when the transport closed. sshd signals nothing at a non-PTY session
end; background jobs with redirected stdio survive and reparent to
init. Clients rely on that: VS Code Remote-SSH starts its remote
server as a plain background job in the session's group and expects
it to outlive connection drops, and tailscale upgrades run as
non-PTY execs whose work should survive a daemon restart.

Use an explicit graceful monitor close to preserve session jobs on
ordinary disconnect and daemon shutdown. Close PTY masters normally,
letting the terminal driver signal the foreground group, but do not
signal non-PTY jobs. Remove Linux Pdeathsig from session children;
forwarding children retain it and always die with the connection.

Commit duration and recording-failure termination to the monitor
before publishing cancellation or writing the termination message.
The monitor owns per-session HUP-to-KILL escalation even after the
leader exits or the connection closes. Revocation arms connection-wide
cleanup. Pin every termination target's leader until after SIGKILL,
including during teardown, so group signaling cannot hit reused PIDs.
Select teardown targets explicitly and leave graceful survivors alone.

Unexpected RPC loss fails closed. If a termination mandate cannot be
delivered, close the RPC rather than subsequently requesting graceful
teardown; this may also kill sibling jobs on a broken channel. Keep
both the escalation and monitor shutdown waits bounded.

Add coverage for default-HUP graceful survivors, actual duration and
recording-failure termination followed by immediate disconnect, sibling
survival, HUP-exiting leaders with immune group members, broken RPCs,
and leader pinning through teardown's SIGKILL pass.

Fixes #22076

Signed-off-by: James Tucker <[email protected]>

Change-Id: I70b4ef40035330dd542d256ba79d889a12426a74
2026-10-30 16:39:06 -07:00

1058 lines
32 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build (linux && !android) || (darwin && !ios)
// Session-through-the-monitor tests: a real gliderssh session served by
// a real tailssh conn whose child is spawned by a real ssh-monitor
// process. Both are re-exec'd through the self-exec seams. The tests
// pin the exit-status wire ordering (exit-status before output before EOF) and
// the exit-notification plumbing on the new path.
package tailssh
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"os/exec"
"os/user"
"runtime"
"slices"
"strconv"
"strings"
"sync"
"testing"
"time"
gliderssh "github.com/tailscale/gliderssh"
gossh "golang.org/x/crypto/ssh"
"golang.org/x/sys/unix"
"tailscale.com/tailcfg"
"tailscale.com/tailcfg/nodecap"
"tailscale.com/tstest"
)
// testIncubatorPrefix re-execs the test binary, selecting only the helper
// test, with the incubator's argv following "--" (the testing framework's
// flag parsing stops there). -test.timeout=0: the incubator child serves a
// full session, not a test with its own deadline. In tailscaled the argv
// prefix is "be-child ssh" (childproc dispatch); the monitor execs the
// prefix after the executable path.
var testIncubatorPrefix = []string{"-test.run=^TestSSHIncubatorHelperProcess$", "-test.timeout=0", "--", "ssh"}
// incubFdsDumpMarker gates the incubator's fd-table dump: only the
// session whose argv carries this marker, as INCUB-FDS-DUMP=<path>,
// dumps to <path>, so a concurrently live pty session is not confused
// with the assertion target. The path travels in the exec request's
// argv.
const incubFdsDumpMarker = "INCUB-FDS-DUMP"
// incubatorChildArgv shapes a test child's argv the way launchViaMonitor
// sends it: an incubator invocation, carrying the identity flags the
// monitor requires and rewrites to its spawn-bound identity. The values
// are placeholders; the monitor overwrites them, so these test children
// need not carry the real connection identity.
func incubatorChildArgv(argv []string) []string {
return append(slices.Clone(argv),
"--uid=1", "--gid=1", "--groups=1", "--local-user=test-user")
}
// incubDumpArg extracts the INCUB-FDS-DUMP=<path> dump target from the
// incubator's argv, or returns "".
func incubDumpArg(args []string) string {
prefix := incubFdsDumpMarker + "="
joined := strings.Join(args, " ")
i := strings.Index(joined, prefix)
if i < 0 {
return ""
}
rest := joined[i+len(prefix):]
if j := strings.IndexAny(rest, " \"'\t\r\n"); j >= 0 {
rest = rest[:j]
}
return rest
}
// TestSSHIncubatorHelperProcess is not a test; it is the entry point
// of the re-exec'd session child, the incubator. It acts only when the
// binary is exec'd with the prefix above, and returns immediately
// otherwise.
func TestSSHIncubatorHelperProcess(t *testing.T) {
for i, a := range os.Args {
if a == "--" && i+2 < len(os.Args) && os.Args[i+1] == "ssh" {
// Declared-fds pin for the session child: when this
// child's argv carries the marker, dump the exec-inherited
// fd table (stdio, the --env-fd pipe at 3) before the
// incubator starts.
if path := incubDumpArg(os.Args[i+2:]); path != "" {
dumpChildFdsTo(path, "incubator")
}
if err := beIncubator(os.Args[i+2:]); err != nil {
fmt.Fprintln(os.Stderr, "ssh incubator child:", err)
os.Exit(1)
}
// beIncubator exec'd away (login/su) or ran the command
// in-process; handleSSHInProcess exits with the child's code.
os.Exit(0)
}
}
}
// monitorSessionSeams routes the monitor and incubator spawns to the
// self-exec helper tests and returns nothing. Cleanup restores both.
func monitorSessionSeams(t *testing.T) {
t.Helper()
oldMon, oldInc := monitorSpawnPrefix, incubatorSpawnPrefix
monitorSpawnPrefix = testMonitorPrefix
incubatorSpawnPrefix = testIncubatorPrefix
t.Cleanup(func() { monitorSpawnPrefix, incubatorSpawnPrefix = oldMon, oldInc })
}
// newMonitorSessionServer is newMonitorSessionServerFor with the
// existing defaults: the current local user, the SSHBehaviorV1 cap, no
// policy grant (sessions only; the Handler bypasses the recheck gates),
// and no forwarding seams.
func newMonitorSessionServer(t *testing.T, handler func(c *conn, s gliderssh.Session)) string {
t.Helper()
return newMonitorSessionServerFor(t, nil, nil, nil, false, handler)
}
// newMonitorSessionServerFor is newMonitorSessionServer with the conn's
// bound identity, node caps, policy grant, and forwarding seams
// parametrized, for the fd-isolation tests (the PTY-master leak
// regression) and the fwd-child declared-fds pin:
//
// - lu: the conn's local user; nil = the current user (the default
// above). Under root runs the fd-isolation tests bind the conn to the
// non-root sshfwda test user, so the session children run as a
// different uid than the pty-holding test process, the cross-user
// shape of the original injection scenario.
// - caps: node caps; nil = [SSHBehaviorV1] (the V1 pin also keeps the
// incubator on the in-process branch: no login/su, no PAM).
// - rule: lb.matchingRule, the policy grant whose action and accept-env
// latching the conn uses for the forwarding gates' recheckPolicy
// (AllowLocalPortForwarding for a unix -L dial through this conn);
// nil = none (sessions only).
// - fwdSeams: route the conn's one-shot fwd children through the
// re-exec'd helper (the seam env crossing the exec boundary into
// the monitor process).
//
// handler serves each session channel; it receives the owning conn so it
// can call conn.newSSHSession(...).run().
func newMonitorSessionServerFor(
t *testing.T,
lu *userMeta,
caps []nodecap.Cap,
rule *tailcfg.SSHRule,
fwdSeams bool,
handler func(c *conn, s gliderssh.Session),
) string {
t.Helper()
if lu == nil {
u, err := user.Current()
if err != nil {
t.Fatal(err)
}
lu, err = userLookup(u.Username)
if err != nil {
t.Fatal(err)
}
}
if caps == nil {
caps = []nodecap.Cap{nodecap.SSHBehaviorV1}
}
monitorSessionSeams(t)
// The session children are exec'd from tailscaledPath by the root
// monitor, and the fwd children (fwdSeams) run as the bound user:
// both need the reachable copy of the test binary under a root run.
exe := makeExecutableReachable(t)
if fwdSeams {
t.Setenv("TS_TEST_SSH_FWD_PREFIX", strings.Join(testFwdPrefix, " "))
}
if rule != nil && rule.SSHUsers["test"] == "" {
rule.SSHUsers["test"] = lu.Username
}
logf := tstest.WhileTestRunningLogger(t)
srv := &server{
lb: &localState{
sshEnabled: true,
varRoot: t.TempDir(),
caps: caps,
matchingRule: rule,
},
logf: logf,
tailscaledPath: exe,
}
sc, err := srv.newConn()
if err != nil {
t.Fatal(err)
}
// Tear the conn's monitor down with the conn (HandleConn is driven
// directly below, so HandleSSHConn's teardown does not run).
t.Cleanup(sc.closeConnMonitor)
sc.insecureSkipTailscaleAuth = true
um, err := userLookup(lu.Username)
if err != nil {
t.Fatal(err)
}
// /bin/sh: a predictable direct child of the incubator.
um.loginShellCached = "/bin/sh"
sc.localUser = um
// Latch the auth-time policy spelling for this synthetic conn.
sc.mappedLocalUser = lu.Username
if rule != nil {
sc.mappedLocalUser = rule.SSHUsers["test"]
}
// The bound user's groups, like clientAuth would have captured them
// (the incubator argv's --groups must be non-empty): for a conn bound
// to another uid use that user's own primary group, not the test
// process's (root-run) groups.
if um.Uid == strconv.Itoa(os.Getuid()) {
for _, g := range testGroups(t) {
sc.userGroupIDs = append(sc.userGroupIDs, strconv.Itoa(g))
}
} else {
sc.userGroupIDs = []string{um.Gid}
}
sc.info = &sshConnInfo{
sshUser: "test",
src: netip.MustParseAddrPort("1.2.3.4:32342"),
dst: netip.MustParseAddrPort("1.2.3.5:22"),
node: (&tailcfg.Node{}).View(),
uprof: tailcfg.UserProfile{},
}
action := &tailcfg.SSHAction{Accept: true}
if rule != nil {
action = rule.Action
sc.acceptEnv = rule.AcceptEnv
// The fwd gates' recheckPolicy paths need Principals to match the
// synthetic conn info (any principal, like the fwd conn tests).
if len(rule.Principals) == 0 {
rule.Principals = []*tailcfg.SSHPrincipal{{Any: true}}
}
}
sc.action0 = action
sc.finalAction = action
sc.authCompleted.Store(true)
sc.Handler = func(s gliderssh.Session) {
handler(sc, s)
}
ln, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go sc.HandleConn(c)
}
}()
return ln.Addr().String()
}
// TestSessionMonitorExitStatusOrder is the monitor-path version of
// TestExitStatusPrecedesEOF: a session whose child is exec'd by the
// per-connection ssh-monitor must keep the teardown wire order
// (exit-status, remaining output, EOF, CHANNEL_CLOSE) even though the
// child's exit arrives as an async monitor notification instead of
// cmd.Wait. The child produces output on both streams and exits 7; the
// client must see the output and the exit status, in that order.
func TestSessionMonitorExitStatusOrder(t *testing.T) {
recCh := make(chan *frameRecorder, 1)
addr := newMonitorSessionServer(t, func(c *conn, s gliderssh.Session) {
rec := &frameRecorder{Session: s}
select {
case recCh <- rec:
default:
}
c.newSSHSession(rec).run()
})
cl, err := gossh.Dial("tcp", addr, &gossh.ClientConfig{
HostKeyCallback: gossh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
})
if err != nil {
t.Fatal(err)
}
defer cl.Close()
s, err := cl.NewSession()
if err != nil {
t.Fatal(err)
}
// Separate buffers: the client serves stdout and stderr from distinct
// goroutines, so they must not share an unsynchronized writer.
var stdout, stderr bytes.Buffer
s.Stdout = &stdout
s.Stderr = &stderr
err = s.Run("echo out; echo err 1>&2; exit 7")
var ee *gossh.ExitError
if !errors.As(err, &ee) {
t.Fatalf("want *gossh.ExitError, got %T: %v", err, err)
}
if got := ee.ExitStatus(); got != 7 {
t.Fatalf("exit status = %d, want 7", got)
}
// The session's output must have arrived before the teardown frames.
if got, want := stdout.String(), "out\n"; got != want {
t.Errorf("session stdout = %q; want %q", got, want)
}
if got, want := stderr.String(), "err\n"; got != want {
t.Errorf("session stderr = %q; want %q", got, want)
}
var rec *frameRecorder
select {
case rec = <-recCh:
case <-time.After(10 * time.Second):
t.Fatal("session never started")
}
var got []string
if err := tstest.WaitFor(5*time.Second, func() error {
got = rec.got()
if len(got) < 3 {
return errNotAllFrames
}
return nil
}); err != nil {
t.Fatalf("only saw frames %v, want 3", got)
}
want := []string{"exit-status", "eof", "close"}
if !slices.Equal(got, want) {
t.Errorf("frame order = %v, want %v", got, want)
}
}
// TestSessionMonitorPTY drives a pty session through the monitor: the
// PTY child descriptors travel to the monitor via SCM_RIGHTS, which applies
// Setsid+Setctty with the PTY child endpoint as fd 0. Window-change requests
// are applied by the resizeWindow goroutine's own master descriptor. The
// client requests a pty at 10x40, the child reports its size, then the
// window is changed to 24x80 and the child reports the new size, pinning
// the resize path end to end, and the exit status is delivered.
func TestSessionMonitorPTY(t *testing.T) {
addr := newMonitorSessionServer(t, func(c *conn, s gliderssh.Session) {
c.newSSHSession(s).run()
})
cl, err := gossh.Dial("tcp", addr, &gossh.ClientConfig{
HostKeyCallback: gossh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
})
if err != nil {
t.Fatal(err)
}
defer cl.Close()
s, err := cl.NewSession()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.RequestPty("xterm", 10, 40, gossh.TerminalModes{gossh.ECHO: 0}); err != nil {
t.Fatal(err)
}
stdin, err := s.StdinPipe()
if err != nil {
t.Fatal(err)
}
deadlineTimer := time.AfterFunc(15*time.Second, func() { cl.Close() })
defer deadlineTimer.Stop()
// stdout is served by a client goroutine; guard the buffer.
var out sessTestBuffer
s.Stdout = &out
s.Stderr = &out
const command = `
stty size
read ready
while [ "$(stty size)" != "24 80" ]; do
sleep 0.05
done
stty size
exit 3
`
if err := s.Start(command); err != nil {
t.Fatal(err)
}
// Wait until the first stty ran: the pty is up and resizeWindow is
// draining window changes (gliderssh's winch channel is buffered
// at one).
deadline := time.Now().Add(10 * time.Second)
for !strings.Contains(out.String(), "10 40") {
if time.Now().After(deadline) {
t.Fatalf("no initial stty output; got %q", out.String())
}
time.Sleep(50 * time.Millisecond)
}
// The window change must be applied to THIS session's master (the
// resize goroutine's own descriptor) and be observable
// to the child.
if err := s.WindowChange(24, 80); err != nil {
t.Fatal(err)
}
if _, err := io.WriteString(stdin, "ready\n"); err != nil {
t.Fatal(err)
}
err = s.Wait()
var ee *gossh.ExitError
if !errors.As(err, &ee) {
t.Fatalf("want *gossh.ExitError, got %T: %v", err, err)
}
if got := ee.ExitStatus(); got != 3 {
t.Fatalf("exit status = %d, want 3", got)
}
// "rows cols" per stty(1): first the requested 10x40, then 24x80
// after the window change.
if got, want := strings.Fields(out.String()), []string{"10", "40", "24", "80"}; !slices.Equal(got, want) {
t.Errorf("stty size outputs = %q; want %q", got, want)
}
}
// TestSessionMonitorConnCloseKillsHupImmuneChild checks that connection close
// kills mandated sessions and preserves other sessions, regardless of HUP handling.
func TestSessionMonitorConnCloseKillsHupImmuneChild(t *testing.T) {
for _, mode := range []string{"mandated", "graceful", "graceful-default-hup"} {
t.Run(mode, func(t *testing.T) {
lu := fwdTestIdentity(t)
c := newFwdTestConn(t, lu, &tailcfg.SSHAction{Accept: true})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ss := &sshSession{
ctx: ctx,
conn: c,
logf: t.Logf,
}
// Mandated termination must kill a HUP-immune child.
command := "trap '' HUP; echo ready; while :; do sleep 1; done"
if mode == "graceful-default-hup" {
command = "echo ready; exec sleep 300"
}
cmd := exec.Command("/bin/sh", "-c", command)
cmd.Args = incubatorChildArgv(cmd.Args)
stdin, stdinKeep, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdinKeep.Close()
stdout, stdoutW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdout.Close()
stderr, stderrW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stderr.Close()
if err := ss.launchViaMonitor(cmd, nil, []*os.File{stdin, stdoutW, stderrW}, false); err != nil {
t.Fatalf("launchViaMonitor: %v", err)
}
stdoutW.Close()
stderrW.Close()
if err := stdout.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
t.Fatal(err)
}
marker := make([]byte, len("ready\n"))
if _, err := io.ReadFull(stdout, marker); err != nil {
t.Fatalf("reading ready marker: %v", err)
}
if string(marker) != "ready\n" {
t.Fatalf("child output = %q; want ready (the HUP trap is installed)", marker)
}
pid := ss.monChild.PID
if mode == "mandated" {
// Terminate this session without setting the connection-wide flag.
ss.hangupMonitorProcess(true)
}
// Wait for monitor teardown to finish.
c.closeConnMonitor()
if strings.HasPrefix(mode, "graceful") {
// Allow time for a parent-death signal before checking liveness.
// pidGone treats Linux zombies as dead.
time.Sleep(200 * time.Millisecond)
if pidGone(pid) {
t.Fatalf("child %d did not survive a graceful conn close", pid)
}
// Kill the survivor and wait for exit.
_ = unix.Kill(-pid, unix.SIGKILL)
_ = unix.Kill(pid, unix.SIGKILL)
deadline := time.Now().Add(10 * time.Second)
for !pidGone(pid) {
if time.Now().After(deadline) {
t.Fatalf("failed to clean up surviving child %d", pid)
}
time.Sleep(100 * time.Millisecond)
}
return
}
// Teardown must kill and reap the child within its shutdown bound.
deadline := time.Now().Add(monTeardownReap*2 + 5*time.Second)
for {
if err := unix.Kill(pid, 0); err != nil {
return // reaped and gone
}
if time.Now().After(deadline) {
t.Fatalf("HUP-immune child %d survived monitor teardown", pid)
}
time.Sleep(100 * time.Millisecond)
}
})
}
}
// TestSessionMonitorTeardownKillsSamePgidStraggler checks that teardown kills
// HUP-immune group members after their leader exits.
func TestSessionMonitorTeardownKillsSamePgidStraggler(t *testing.T) {
testSessionMonitorTerminatedStraggler(t, true)
}
func TestSessionMonitorEscalationKillsSamePgidStraggler(t *testing.T) {
testSessionMonitorTerminatedStraggler(t, false)
}
func testSessionMonitorTerminatedStraggler(t *testing.T, closeMonitor bool) {
t.Helper()
lu := fwdTestIdentity(t)
c := newFwdTestConn(t, lu, &tailcfg.SSHAction{Accept: true})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ss := &sshSession{
ctx: ctx,
conn: c,
logf: t.Logf,
}
// The leader exits on HUP. Its background child ignores HUP and reports its PID.
cmd := exec.Command("/bin/sh", "-c",
"nohup sh -c 'echo $$; exec sleep 300' & echo ready; wait")
cmd.Args = incubatorChildArgv(cmd.Args)
stdin, stdinKeep, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdinKeep.Close()
stdout, stdoutW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdout.Close()
stderr, stderrW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stderr.Close()
if err := ss.launchViaMonitor(cmd, nil, []*os.File{stdin, stdoutW, stderrW}, false); err != nil {
t.Fatalf("launchViaMonitor: %v", err)
}
stdoutW.Close()
stderrW.Close()
if err := stdout.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {
t.Fatal(err)
}
// Read until both markers arrived: "ready" (the leader is running)
// and the straggler's pid line (the nohup'd member is up).
straggler := 0
sawReady := false
var buf []byte
deadline := time.Now().Add(10 * time.Second)
for !(sawReady && straggler != 0) {
if time.Now().After(deadline) {
t.Fatalf("child markers never arrived; got %q", buf)
}
chunk := make([]byte, 256)
n, err := stdout.Read(chunk)
if n > 0 {
buf = append(buf, chunk[:n]...)
}
if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) {
t.Fatalf("reading child output: %v", err)
}
for _, line := range strings.Fields(string(buf)) {
if line == "ready" {
sawReady = true
}
if p, err := strconv.Atoi(line); err == nil && p > 1 {
straggler = p
}
}
}
t.Cleanup(func() { _ = unix.Kill(-ss.monChild.PID, unix.SIGKILL) })
ss.hangupMonitorProcess(true)
if closeMonitor {
c.closeConnMonitor()
} else {
// Check leader exit before timer escalation.
if err := tstest.WaitFor(time.Second, func() error {
if !pidGone(ss.monChild.PID) {
return errors.New("leader still alive")
}
return nil
}); err != nil {
t.Fatal(err)
}
if pidGone(straggler) {
t.Fatal("straggler did not survive HUP")
}
}
// Allow escalation and reap time. Orphaned Linux zombies count as dead.
deadline = time.Now().Add(monTeardownReap*2 + 5*time.Second)
for {
if pidGone(straggler) {
return // reaped and gone (or a reaped-pending zombie)
}
if time.Now().After(deadline) {
t.Fatalf("HUP-immune group member %d survived monitor teardown", straggler)
}
time.Sleep(100 * time.Millisecond)
}
}
func TestPIDGoneWithUnreapedChild(t *testing.T) {
cmd := exec.Command("/bin/sleep", "300")
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cmd.Process.Kill()
cmd.Wait()
})
pid := cmd.Process.Pid
if pidGone(pid) {
t.Fatal("live child reported dead")
}
if err := cmd.Process.Kill(); err != nil {
t.Fatal(err)
}
// Keep the child unreaped until after the liveness checks.
if err := tstest.WaitFor(5*time.Second, func() error {
if !pidGone(pid) {
return errors.New("unreaped child reported alive")
}
return nil
}); err != nil {
t.Fatal(err)
}
if err := unix.Kill(pid, 0); err != nil {
t.Fatalf("child was reaped before cleanup: %v", err)
}
}
// TestWaitMonitorProcessExitNotificationWins checks priority and repeatable
// reads when exit and monitor death occur before or during the wait.
func TestWaitMonitorProcessExitNotificationWins(t *testing.T) {
for i := range 100 {
dead := make(chan struct{})
exit := newMonExitStatus()
notify := func() {
exit.complete(42)
close(dead)
}
if i%2 == 0 {
notify()
} else {
go notify()
}
ss := &sshSession{monExit: exit, monDead: dead}
for range 2 {
code, ok, err := ss.waitMonitorProcess()
if !ok || err != nil || code != 42 {
t.Fatalf("waitMonitorProcess = (%d, %v, %v); want (42, true, nil)", code, ok, err)
}
}
}
}
// TestSessionPtyTermWiring pins the pty TERM wiring: a pty request
// whose TERM fails acceptEnvPair's value rule is dropped, and a valid
// environ-sourced TERM survives the pty request.
func TestSessionPtyTermWiring(t *testing.T) {
addr := newMonitorSessionServer(t, func(c *conn, s gliderssh.Session) {
c.newSSHSession(s).run()
})
// The echo rides behind a leading sleep and is read through a poll
// loop (the shape TestSessionMonitorPTY uses): output written while
// the pty's termios is still settling races the capture.
check := func(setTermEnv bool) string {
t.Helper()
cl, err := gossh.Dial("tcp", addr, &gossh.ClientConfig{
HostKeyCallback: gossh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
})
if err != nil {
t.Fatal(err)
}
defer cl.Close()
s, err := cl.NewSession()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if setTermEnv {
// A valid environ-sourced TERM: accepted by acceptEnvPair's
// value rule and placed in the incubator env by incubatorEnv.
if err := s.Setenv("TERM", "dumb"); err != nil {
t.Fatal(err)
}
}
// A pty TERM that is a path: rejected by the value rule, dropped.
if err := s.RequestPty("x/bad-term", 24, 80, gossh.TerminalModes{}); err != nil {
t.Fatal(err)
}
var out sessTestBuffer
s.Stdout = &out
s.Stderr = &out
if err := s.Start(`sleep 1; echo "TERM=${TERM-unset}"; exit 3`); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(10 * time.Second)
for !strings.Contains(out.String(), "TERM=") {
if time.Now().After(deadline) {
t.Fatalf("no TERM line from the child; got %q", out.String())
}
time.Sleep(50 * time.Millisecond)
}
err = s.Wait()
var ee *gossh.ExitError
if !errors.As(err, &ee) || ee.ExitStatus() != 3 {
t.Fatalf("session Wait = %v; want exit status 3", err)
}
return out.String()
}
// No environ TERM: the bad pty TERM is dropped, so the child has no
// TERM from us (not the bad value, not a sanitized one). What the
// child's shell then does with an absent TERM is the shell's own
// business: macOS's /bin/sh (bash 3.2) initializes TERM=dumb itself
// when the environment lacks one, while other shells leave it unset.
// The pinned invariant is the absence of the rejected value.
wantDropped := "x/bad-term"
wantLine := "TERM=unset\r"
if runtime.GOOS == "darwin" {
wantLine = "TERM=dumb\r" // /bin/sh initializes it; see above
}
if got := check(false); strings.Contains(got, wantDropped) || !strings.Contains(got, wantLine) {
t.Errorf("child TERM after rejecting PTY TERM = %q; want %q", got, wantLine)
}
// A valid environ-sourced TERM is not overridden by the bad pty TERM
// (the old last-wins inversion).
if got := check(true); !strings.Contains(got, "TERM=dumb\r") {
t.Errorf("child TERM after rejecting PTY TERM = %q; want TERM=dumb", got)
}
}
// sessTestBuffer is a concurrency-safe bytes.Buffer: the client serves
// stdout and stderr from distinct goroutines (the integration-tagged suite
// has its own lockedBuffer; this one is local to the monitor session tests).
type sessTestBuffer struct {
mu sync.Mutex
b bytes.Buffer
}
func (out *sessTestBuffer) Write(p []byte) (int, error) {
out.mu.Lock()
defer out.mu.Unlock()
return out.b.Write(p)
}
func (out *sessTestBuffer) String() string {
out.mu.Lock()
defer out.mu.Unlock()
return out.b.String()
}
// TestSessionMonitorHangupEscalation pins the teardown escalation: a
// session child group that survives the teardown SIGHUP is SIGKILLed
// after hangupEscalation, and its exit code is 128+SIGKILL, the shell
// convention.
func TestSessionMonitorHangupEscalation(t *testing.T) {
lu := fwdTestIdentity(t)
c := newFwdTestConn(t, lu, &tailcfg.SSHAction{Accept: true})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ss := &sshSession{
ctx: ctx,
conn: c,
logf: t.Logf,
}
// An HUP-immune child: sh traps (ignores) SIGHUP and its own children
// re-spawn, so only the group SIGKILL ends it. The marker gates the
// hangup on the trap being installed.
cmd := exec.Command("/bin/sh", "-c", "trap '' HUP; echo ready; while :; do sleep 1; done")
cmd.Args = incubatorChildArgv(cmd.Args)
stdin, stdinKeep, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdinKeep.Close()
stdout, stdoutW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdout.Close()
stderr, stderrW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stderr.Close()
if err := ss.launchViaMonitor(cmd, nil, []*os.File{stdin, stdoutW, stderrW}, false); err != nil {
t.Fatalf("launchViaMonitor: %v", err)
}
stdoutW.Close()
stderrW.Close()
// Read just the marker (the child never exits, so no EOF read here).
if err := stdout.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
t.Fatal(err)
}
marker := make([]byte, len("ready\n"))
if _, err := io.ReadFull(stdout, marker); err != nil {
t.Fatalf("reading ready marker: %v", err)
}
if string(marker) != "ready\n" {
t.Fatalf("child output = %q; want ready (the HUP trap is installed)", marker)
}
ss.hangupMonitorProcess(true)
// The child must die via the SIGKILL escalation (128+9) within the
// escalation window plus slack.
deadline := time.Now().Add(hangupEscalation + 5*time.Second)
for {
select {
case <-ss.monExit.done:
code := ss.monExit.code
if want := 128 + int(unix.SIGKILL); code != want {
t.Fatalf("exit code = %d, want %d (128+SIGKILL)", code, want)
}
return
case <-time.After(100 * time.Millisecond):
if time.Now().After(deadline) {
t.Fatalf("child survived SIGHUP by %v", hangupEscalation+5*time.Second)
}
}
}
}
// TestLaunchViaMonitorRefusesCanceledContext pins the pre-flight ctx
// refusal at the top of launchViaMonitor: a session whose ctx is
// already canceled must refuse the launch before the exec op is
// written, without spawning a monitor to do it.
func TestLaunchViaMonitorRefusesCanceledContext(t *testing.T) {
lu := fwdTestIdentity(t)
c := newFwdTestConn(t, lu, &tailcfg.SSHAction{Accept: true})
ctx, cancel := context.WithCancel(context.Background())
cancel()
ss := &sshSession{
ctx: ctx,
conn: c,
logf: t.Logf,
}
stdin, stdinKeep, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdinKeep.Close()
stdout, stdoutW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stdout.Close()
stderr, stderrW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
defer stderr.Close()
cmd := exec.Command("/bin/sh", "-c", "echo doomed; sleep 30")
cmd.Args = incubatorChildArgv(cmd.Args)
err = ss.launchViaMonitor(cmd, nil, []*os.File{stdin, stdoutW, stderrW}, false)
stdoutW.Close()
stderrW.Close()
if !errors.Is(err, context.Canceled) {
t.Fatalf("launchViaMonitor = %v; want context.Canceled", err)
}
if mc := c.connMonitorIfAny(); mc != nil {
t.Fatal("launchViaMonitor spawned a monitor for a canceled session")
}
}
// TestSessionRevokedDuringRecorderStallNoMonitorLeak pins the
// monitor-respawn leak end to end. A revocation landing while a
// session is between attach and launch, here stalled in recorder setup,
// the same window that contains the agent-forwarding policy recheck,
// must not, once the recorder setup completes, spawn the session child
// through a freshly respawned monitor.
func TestSessionRevokedDuringRecorderStallNoMonitorLeak(t *testing.T) {
// A local recorder that accepts the recording POST but stalls before
// the 100-continue that unblocks startNewRecording, pinning run() in
// its recorder setup while the revocation lands.
postArrived := make(chan struct{}, 1)
release := make(chan struct{})
releaseRecorder := sync.OnceFunc(func() { close(release) })
recSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == "HEAD" && r.URL.Path == "/v2/record":
// No V2 support is advertised. That forces the V1 /record path.
w.WriteHeader(http.StatusNotFound)
case r.Method == "POST" && r.URL.Path == "/record":
select {
case postArrived <- struct{}{}:
default:
}
<-release
// The first body read triggers the 100-continue.
io.Copy(io.Discard, r.Body)
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(recSrv.Close)
// Unblock the handler before Close waits for it, even on a Fatal path.
t.Cleanup(releaseRecorder)
recIPPort, err := netip.ParseAddrPort(recSrv.Listener.Addr().String())
if err != nil {
t.Fatal(err)
}
rule := &tailcfg.SSHRule{
SSHUsers: map[string]string{"test": ""},
Action: &tailcfg.SSHAction{
Accept: true,
Recorders: []netip.AddrPort{recIPPort},
},
}
connCh := make(chan *conn, 1)
addr := newMonitorSessionServerFor(t, nil, nil, rule, false, func(c *conn, s gliderssh.Session) {
select {
case connCh <- c:
default:
}
c.newSSHSession(s).run()
})
cl, err := gossh.Dial("tcp", addr, &gossh.ClientConfig{
HostKeyCallback: gossh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
})
if err != nil {
t.Fatal(err)
}
defer cl.Close()
s, err := cl.NewSession()
if err != nil {
t.Fatal(err)
}
if err := s.Start("sleep 60"); err != nil {
t.Fatal(err)
}
c := <-connCh
select {
case <-postArrived:
case <-time.After(10 * time.Second):
t.Fatal("recorder POST never arrived; the session never reached its recorder setup")
}
// The revocation lands mid-stall. Every session ctx is canceled and
// the transport is closed.
c.revokeConn("test revocation")
// This is the conn teardown that HandleSSHConn's deferred
// closeConnMonitor performs once the conn is done.
c.closeConnMonitor()
// Release the recorder. startNewRecording completes and run() walks
// into launchProcess on the revoked conn, the spawn window. The
// canceled-ctx refusal and the closed monitor must each do their
// job, so no child and no respawned monitor appear.
releaseRecorder()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if mc := c.connMonitorIfAny(); mc != nil {
t.Fatal("a monitor was spawned on the torn-down conn; nobody would close it (respawn leak)")
}
time.Sleep(50 * time.Millisecond)
}
// And the closed flag must refuse any late spawn outright.
if _, err := c.connMonitor(); err == nil {
t.Fatal("connMonitor respawned a monitor on the torn-down conn")
}
}
// TestHangupMonitorProcessAfterMonitorClosed checks that teardown neither
// recreates the monitor nor signals an unrelated process with a reused PID.
func TestHangupMonitorProcessAfterMonitorClosed(t *testing.T) {
lu := fwdTestIdentity(t)
c := newFwdTestConn(t, lu, &tailcfg.SSHAction{Accept: true})
if _, err := c.connMonitor(); err != nil {
t.Fatal(err)
}
c.closeConnMonitor()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "cat")
stdin, err := cmd.StdinPipe()
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
cmd.Stdout = &output
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() {
cmd.Process.Kill()
cmd.Wait()
}()
ss := &sshSession{
ctx: ctx,
conn: c,
logf: t.Logf,
monChild: monitorProcess{PID: cmd.Process.Pid, ID: 1},
}
if !ss.hangupMonitorProcess(true) {
t.Fatal("monitor session took the direct hangup path")
}
if got := c.connMonitorIfAny(); got != nil {
t.Fatal("hangup recreated a closed monitor")
}
if _, err := io.WriteString(stdin, "alive\n"); err != nil {
t.Fatal(err)
}
stdin.Close()
if err := cmd.Wait(); err != nil {
t.Fatalf("unrelated process was signaled: %v", err)
}
if output.String() != "alive\n" {
t.Fatalf("unrelated process output = %q", output.String())
}
}