ssh/tailssh: fix exit-status frame ordering

Send exit-status before EOF before CHANNEL_CLOSE per RFC 4254 §6.10.
The old code raced CloseWrite against Exit; macOS OpenSSH often saw
EOF first and dropped the exit code.

Replace atomic.Bool/atomic.Int32/channel sync with a sync.WaitGroup
over the output goroutines and emit:

    cmd.Wait -> ss.Exit -> closeAll(childPipes) -> wg.Wait -> ss.Close

The stdin copier stays out of wg: it blocks reading from the SSH
channel until the client half-closes, and many clients (go-scp,
plain `ssh host cmd`, scp, sftp) don't half-close before they
receive CHANNEL_CLOSE from us. Including it deadlocks. It self-
cleans via deferred ss.Close.

Align exit codes with shell convention:
  255 SSH-internal error (OpenSSH ssh.c:1693)
  254 recording-infrastructure failure
  127 command-not-found (POSIX)

Switch session termination from SIGKILL to SIGHUP, matching
OpenSSH PTY-master-close (session.c:2246). Process-group delivery
follows in a later commit.

Relies on the gliderssh wire-order change (tailscale/gliderssh#8),
which go.mod already pins since 3f5eb3199.

Based on tailscale/tailscale#18331.

Updates #18256

Signed-off-by: James Tucker <[email protected]>
Signed-off-by: Kristoffer Dalby <[email protected]>
This commit is contained in:
James Tucker authored and Kristoffer Dalby committed 2026-07-27 12:55:15 +00:00
1 parent a29c4b4c2a
commit f65dc0e3f4
2 files changed
+97 -65

No files matched your search

+96 -62
View File
@@ -886,11 +886,19 @@ func (ss *sshSession) killProcessOnContextDone() {
// We don't need to Process.Wait here, sshSession.run() does
// the waiting regardless of termination reason.
// TODO(maisem): should this be a SIGTERM followed by a SIGKILL?
ss.cmd.Process.Kill()
// SIGHUP = POSIX terminal-disconnect semantics; OpenSSH gets it
// implicitly via PTY-master close (session.c:2246), we send it
// explicitly because non-PTY sessions use pipes.
ss.cmd.Process.Signal(syscall.SIGHUP)
})
}
// isNotFoundOrExecutable reports whether err is a launchProcess
// failure caused by the command not existing on disk.
func isNotFoundOrExecutable(err error) bool {
return errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist)
}
// attachSession registers ss as an active session.
func (c *conn) attachSession(ss *sshSession) {
c.srv.sessionWaitGroup.Add(1)
@@ -976,10 +984,21 @@ func (ss *sshSession) run() {
metricActiveSessions.Add(1)
defer metricActiveSessions.Add(-1)
defer ss.cancelCtx(errSessionDone)
defer ss.Close() // CHANNEL_CLOSE, last on the wire; see ss.Exit below.
if attached := ss.conn.srv.attachSessionToConnIfNotShutdown(ss); !attached {
fmt.Fprintf(ss, "Tailscale SSH is shutting down\r\n")
ss.Exit(1)
// 255 signals an SSH-layer failure (the session never reached the
// user's command), distinct from any exit status the remote
// program might produce. The ssh(1) EXIT STATUS section
// documents this as the value the client reports "if an error
// occurred", and OpenSSH's own ssh.c uses exit(255) for every
// transport/protocol fatal path. Wrappers and CI use the 255
// boundary to tell "connection broke" from "command exited N".
// See:
// https://man.openbsd.org/ssh#EXIT_STATUS
// https://github.com/openssh/openssh-portable/blob/V_10_2_P1/ssh.c#L1693
ss.Exit(255)
return
}
defer ss.conn.detachSession(ss)
@@ -1001,7 +1020,9 @@ func (ss *sshSession) run() {
if lu.Uid != fmt.Sprint(euid) {
ss.logf("can't switch to user %q from process euid %v", lu.Username, euid)
fmt.Fprintf(ss, "can't switch user\r\n")
ss.Exit(1)
// 255: SSH-layer failure, no user command ever ran. See the
// attachSession branch above for the full citation.
ss.Exit(255)
return
}
}
@@ -1029,7 +1050,27 @@ func (ss *sshSession) run() {
fmt.Fprintf(ss, "can't start new recording\r\n")
}
ss.logf("startNewRecording: %v", err)
ss.Exit(1)
// 254 is Tailscale-specific: the SSH transport is fine
// and the user's command is well-formed, but the session
// recording policy could not be satisfied (recorder
// unreachable, upload denied, etc.) so we refuse to run
// the command at all. We need a code that operators can
// alert on without collapsing it into the generic buckets:
// - 1 is overloaded; any program can produce it
// (Bash exit-code conventions, codes 1-2)
// - 127 means "command not found" (POSIX 2018, sh
// §2.8.2)
// - 130 is Ctrl-C (128 + SIGINT)
// - 255 means "SSH itself failed" (ssh(1) EXIT STATUS)
// 254 sits in the reserved >128 region but is not claimed
// by any of the above, so it is unambiguous for
// "recording-required session denied".
// Refs:
// https://man.openbsd.org/ssh#EXIT_STATUS
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
// https://tldp.org/LDP/abs/html/exitcodes.html
// https://github.com/tailscale/tailscale/issues/18256
ss.Exit(254)
return
}
ss.logf("startNewRecording: <nil>")
@@ -1042,6 +1083,7 @@ func (ss *sshSession) run() {
err := ss.launchProcess()
if err != nil {
logf("start failed: %v", err.Error())
exitCode := 1
if errors.Is(err, context.Canceled) {
cause := context.Cause(ss.ctx)
if serr, ok := cause.(SSHTerminationError); ok {
@@ -1049,14 +1091,35 @@ func (ss *sshSession) run() {
io.WriteString(ss.Stderr(), "\r\n\r\n"+msg+"\r\n\r\n")
}
}
} else if isNotFoundOrExecutable(err) {
// 127 = "command not found" per POSIX 2018 sh §2.8.2
// ("if a command is not found, the exit status shall be
// 127"). 126 is reserved for "found but not executable";
// we collapse both into 127 because exec.ErrNotFound and
// os.ErrNotExist don't distinguish the underlying cause
// reliably across platforms, and 127 is what users
// recognise (it's what bash / dash / zsh report).
// Refs:
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
// https://tldp.org/LDP/abs/html/exitcodes.html
exitCode = 127
}
ss.Exit(1)
ss.Exit(exitCode)
return
}
ss.exitHandled = make(chan struct{})
go ss.killProcessOnContextDone()
var processDone atomic.Bool
// wg covers stdout and stderr only: their close ordering is
// load-bearing for the wire-level frame sequence below.
//
// The stdin copier is deliberately NOT in wg. It blocks reading from
// the SSH channel until the client half-closes its write side; many
// clients (go-scp, non-SFTP exec) don't half-close before they
// receive CHANNEL_CLOSE from us. Including it would deadlock. It
// self-cleans when deferred ss.Close runs.
var wg sync.WaitGroup
go func() {
defer ss.wrStdin.Close()
if _, err := io.Copy(rec.writer("i", ss.wrStdin), ss); err != nil {
@@ -1064,52 +1127,31 @@ func (ss *sshSession) run() {
ss.cancelCtx(err)
}
}()
outputDone := make(chan struct{})
var openOutputStreams atomic.Int32
if ss.rdStderr != nil {
openOutputStreams.Store(2)
} else {
openOutputStreams.Store(1)
}
go func() {
wg.Go(func() {
defer ss.rdStdout.Close()
_, err := io.Copy(rec.writer("o", ss), ss.rdStdout)
if err != nil && !errors.Is(err, io.EOF) {
isErrBecauseProcessExited := processDone.Load() && errors.Is(err, syscall.EIO)
if !isErrBecauseProcessExited {
logf("stdout copy: %v", err)
ss.cancelCtx(err)
}
if _, err := io.Copy(rec.writer("o", ss), ss.rdStdout); err != nil && !errors.Is(err, io.EOF) {
logf("stdout copy: %v", err)
}
if openOutputStreams.Add(-1) == 0 {
ss.CloseWrite()
close(outputDone)
}
}()
ss.CloseWrite() // CHANNEL_EOF; channel stays open for exit-status (RFC 4254 §5.3).
})
// rdStderr is nil for ptys.
if ss.rdStderr != nil {
go func() {
wg.Go(func() {
defer ss.rdStderr.Close()
_, err := io.Copy(ss.Stderr(), ss.rdStderr)
if err != nil {
if _, err := io.Copy(ss.Stderr(), ss.rdStderr); err != nil {
logf("stderr copy: %v", err)
}
if openOutputStreams.Add(-1) == 0 {
ss.CloseWrite()
close(outputDone)
}
}()
})
}
err = ss.cmd.Wait()
processDone.Store(true)
if ss.ctx.Err() != nil {
// Context was canceled (e.g., recording upload failure).
// Wait for killProcessOnContextDone to finish writing any
// termination message before we proceed. This must happen
// before closeAll and CloseWrite so the SSH channel is
// still writable.
// Cancellation (e.g. recording upload failure) wrote a
// termination message via killProcessOnContextDone; wait
// for it before Exit closes the channel for writes.
<-ss.exitHandled
}
@@ -1118,31 +1160,23 @@ func (ss *sshSession) run() {
// aforementioned goroutine.
ss.exitOnce.Do(func() {})
// Close the process-side of all pipes to signal the asynchronous
// io.Copy routines reading/writing from the pipes to terminate.
// Block for the io.Copy to finish before calling ss.Exit below.
closeAll(ss.childPipes...)
select {
case <-outputDone:
case <-ss.ctx.Done():
<-ss.exitHandled
}
var exitCode int
if err == nil {
ss.logf("Session complete")
ss.Exit(0)
return
}
if ee, ok := err.(*exec.ExitError); ok {
code := ee.ProcessState.ExitCode()
ss.logf("Wait: code=%v", code)
ss.Exit(code)
return
exitCode = 0
} else if ee, ok := err.(*exec.ExitError); ok {
exitCode = ee.ProcessState.ExitCode()
ss.logf("Wait: code=%v", exitCode)
} else {
ss.logf("Wait: %v", err)
exitCode = 1
}
ss.logf("Wait: %v", err)
ss.Exit(1)
return
// Order on the wire: exit-status, EOF (CloseWrite in stdout copier),
// CHANNEL_CLOSE (deferred ss.Close). RFC 4254 §6.10.
ss.Exit(exitCode)
closeAll(ss.childPipes...)
wg.Wait()
}
// recordSSHToLocalDisk is a deprecated dev knob to allow recording SSH sessions
+1 -3
View File
@@ -527,9 +527,7 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) {
},
sshCommand: "echo hello",
wantClientOutput: "session rejected\r\n",
// 254 once recording failures get their own exit code; the
// current code exits 1 here.
wantExitCode: 0,
wantExitCode: 254,
clientOutputMustNotContain: []string{"hello"},
},