From 92f43003fc6caee00f4cb752b3435904f14ec098 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 6 May 2026 09:37:48 +0000 Subject: [PATCH] ssh/tailssh: fix exit-status frame ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Send exit-status before EOF before CHANNEL_CLOSE per RFC 4254 §6.10. The old code raced CloseWrite against Exit; macOS OpenSSH often saw EOF first and dropped the exit code. Replace atomic.Bool/atomic.Int32/channel sync with a sync.WaitGroup over the output goroutines and emit: cmd.Wait -> ss.Exit -> closeAll(childPipes) -> wg.Wait -> ss.Close The stdin copier stays out of wg: it blocks reading from the SSH channel until the client half-closes, and many clients (go-scp, plain `ssh host cmd`, scp, sftp) don't half-close before they receive CHANNEL_CLOSE from us. Including it deadlocks. It self- cleans via deferred ss.Close. Align exit codes with shell convention: 255 SSH-internal error (OpenSSH ssh.c:1693) 254 recording-infrastructure failure 127 command-not-found (POSIX) Switch session termination from SIGKILL to SIGHUP, matching OpenSSH PTY-master-close (session.c:2246). Process-group delivery follows in a later commit. Relies on the gliderssh wire-order change (tailscale/gliderssh#8), which go.mod already pins since 3f5eb3199. Based on tailscale/tailscale#18331. Updates #18256 Signed-off-by: James Tucker Signed-off-by: Kristoffer Dalby --- ssh/tailssh/tailssh.go | 158 ++++++++++++++++++++++-------------- ssh/tailssh/tailssh_test.go | 4 +- 2 files changed, 97 insertions(+), 65 deletions(-) diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index 4408683eb..e338e239f 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -880,11 +880,19 @@ func (ss *sshSession) killProcessOnContextDone() { // We don't need to Process.Wait here, sshSession.run() does // the waiting regardless of termination reason. - // TODO(maisem): should this be a SIGTERM followed by a SIGKILL? - ss.cmd.Process.Kill() + // SIGHUP = POSIX terminal-disconnect semantics; OpenSSH gets it + // implicitly via PTY-master close (session.c:2246), we send it + // explicitly because non-PTY sessions use pipes. + ss.cmd.Process.Signal(syscall.SIGHUP) }) } +// isNotFoundOrExecutable reports whether err is a launchProcess +// failure caused by the command not existing on disk. +func isNotFoundOrExecutable(err error) bool { + return errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist) +} + // attachSession registers ss as an active session. func (c *conn) attachSession(ss *sshSession) { c.srv.sessionWaitGroup.Add(1) @@ -970,10 +978,21 @@ func (ss *sshSession) run() { metricActiveSessions.Add(1) defer metricActiveSessions.Add(-1) defer ss.cancelCtx(errSessionDone) + defer ss.Close() // CHANNEL_CLOSE, last on the wire; see ss.Exit below. if attached := ss.conn.srv.attachSessionToConnIfNotShutdown(ss); !attached { fmt.Fprintf(ss, "Tailscale SSH is shutting down\r\n") - ss.Exit(1) + // 255 signals an SSH-layer failure (the session never reached the + // user's command), distinct from any exit status the remote + // program might produce. The ssh(1) EXIT STATUS section + // documents this as the value the client reports "if an error + // occurred", and OpenSSH's own ssh.c uses exit(255) for every + // transport/protocol fatal path. Wrappers and CI use the 255 + // boundary to tell "connection broke" from "command exited N". + // See: + // https://man.openbsd.org/ssh#EXIT_STATUS + // https://github.com/openssh/openssh-portable/blob/V_10_2_P1/ssh.c#L1693 + ss.Exit(255) return } defer ss.conn.detachSession(ss) @@ -995,7 +1014,9 @@ func (ss *sshSession) run() { if lu.Uid != fmt.Sprint(euid) { ss.logf("can't switch to user %q from process euid %v", lu.Username, euid) fmt.Fprintf(ss, "can't switch user\r\n") - ss.Exit(1) + // 255: SSH-layer failure, no user command ever ran. See the + // attachSession branch above for the full citation. + ss.Exit(255) return } } @@ -1023,7 +1044,27 @@ func (ss *sshSession) run() { fmt.Fprintf(ss, "can't start new recording\r\n") } ss.logf("startNewRecording: %v", err) - ss.Exit(1) + // 254 is Tailscale-specific: the SSH transport is fine + // and the user's command is well-formed, but the session + // recording policy could not be satisfied (recorder + // unreachable, upload denied, etc.) so we refuse to run + // the command at all. We need a code that operators can + // alert on without collapsing it into the generic buckets: + // - 1 is overloaded; any program can produce it + // (Bash exit-code conventions, codes 1-2) + // - 127 means "command not found" (POSIX 2018, sh + // §2.8.2) + // - 130 is Ctrl-C (128 + SIGINT) + // - 255 means "SSH itself failed" (ssh(1) EXIT STATUS) + // 254 sits in the reserved >128 region but is not claimed + // by any of the above, so it is unambiguous for + // "recording-required session denied". + // Refs: + // https://man.openbsd.org/ssh#EXIT_STATUS + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02 + // https://tldp.org/LDP/abs/html/exitcodes.html + // https://github.com/tailscale/tailscale/issues/18256 + ss.Exit(254) return } ss.logf("startNewRecording: ") @@ -1036,6 +1077,7 @@ func (ss *sshSession) run() { err := ss.launchProcess() if err != nil { logf("start failed: %v", err.Error()) + exitCode := 1 if errors.Is(err, context.Canceled) { cause := context.Cause(ss.ctx) if serr, ok := cause.(SSHTerminationError); ok { @@ -1043,14 +1085,35 @@ func (ss *sshSession) run() { io.WriteString(ss.Stderr(), "\r\n\r\n"+msg+"\r\n\r\n") } } + } else if isNotFoundOrExecutable(err) { + // 127 = "command not found" per POSIX 2018 sh §2.8.2 + // ("if a command is not found, the exit status shall be + // 127"). 126 is reserved for "found but not executable"; + // we collapse both into 127 because exec.ErrNotFound and + // os.ErrNotExist don't distinguish the underlying cause + // reliably across platforms, and 127 is what users + // recognise (it's what bash / dash / zsh report). + // Refs: + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02 + // https://tldp.org/LDP/abs/html/exitcodes.html + exitCode = 127 } - ss.Exit(1) + ss.Exit(exitCode) return } ss.exitHandled = make(chan struct{}) go ss.killProcessOnContextDone() - var processDone atomic.Bool + // wg covers stdout and stderr only: their close ordering is + // load-bearing for the wire-level frame sequence below. + // + // The stdin copier is deliberately NOT in wg. It blocks reading from + // the SSH channel until the client half-closes its write side; many + // clients (go-scp, non-SFTP exec) don't half-close before they + // receive CHANNEL_CLOSE from us. Including it would deadlock. It + // self-cleans when deferred ss.Close runs. + var wg sync.WaitGroup + go func() { defer ss.wrStdin.Close() if _, err := io.Copy(rec.writer("i", ss.wrStdin), ss); err != nil { @@ -1058,52 +1121,31 @@ func (ss *sshSession) run() { ss.cancelCtx(err) } }() - outputDone := make(chan struct{}) - var openOutputStreams atomic.Int32 - if ss.rdStderr != nil { - openOutputStreams.Store(2) - } else { - openOutputStreams.Store(1) - } - go func() { + + wg.Go(func() { defer ss.rdStdout.Close() - _, err := io.Copy(rec.writer("o", ss), ss.rdStdout) - if err != nil && !errors.Is(err, io.EOF) { - isErrBecauseProcessExited := processDone.Load() && errors.Is(err, syscall.EIO) - if !isErrBecauseProcessExited { - logf("stdout copy: %v", err) - ss.cancelCtx(err) - } + if _, err := io.Copy(rec.writer("o", ss), ss.rdStdout); err != nil && !errors.Is(err, io.EOF) { + logf("stdout copy: %v", err) } - if openOutputStreams.Add(-1) == 0 { - ss.CloseWrite() - close(outputDone) - } - }() + ss.CloseWrite() // CHANNEL_EOF; channel stays open for exit-status (RFC 4254 §5.3). + }) + // rdStderr is nil for ptys. if ss.rdStderr != nil { - go func() { + wg.Go(func() { defer ss.rdStderr.Close() - _, err := io.Copy(ss.Stderr(), ss.rdStderr) - if err != nil { + if _, err := io.Copy(ss.Stderr(), ss.rdStderr); err != nil { logf("stderr copy: %v", err) } - if openOutputStreams.Add(-1) == 0 { - ss.CloseWrite() - close(outputDone) - } - }() + }) } err = ss.cmd.Wait() - processDone.Store(true) if ss.ctx.Err() != nil { - // Context was canceled (e.g., recording upload failure). - // Wait for killProcessOnContextDone to finish writing any - // termination message before we proceed. This must happen - // before closeAll and CloseWrite so the SSH channel is - // still writable. + // Cancellation (e.g. recording upload failure) wrote a + // termination message via killProcessOnContextDone; wait + // for it before Exit closes the channel for writes. <-ss.exitHandled } @@ -1112,31 +1154,23 @@ func (ss *sshSession) run() { // aforementioned goroutine. ss.exitOnce.Do(func() {}) - // Close the process-side of all pipes to signal the asynchronous - // io.Copy routines reading/writing from the pipes to terminate. - // Block for the io.Copy to finish before calling ss.Exit below. - closeAll(ss.childPipes...) - select { - case <-outputDone: - case <-ss.ctx.Done(): - <-ss.exitHandled - } - + var exitCode int if err == nil { ss.logf("Session complete") - ss.Exit(0) - return - } - if ee, ok := err.(*exec.ExitError); ok { - code := ee.ProcessState.ExitCode() - ss.logf("Wait: code=%v", code) - ss.Exit(code) - return + exitCode = 0 + } else if ee, ok := err.(*exec.ExitError); ok { + exitCode = ee.ProcessState.ExitCode() + ss.logf("Wait: code=%v", exitCode) + } else { + ss.logf("Wait: %v", err) + exitCode = 1 } - ss.logf("Wait: %v", err) - ss.Exit(1) - return + // Order on the wire: exit-status, EOF (CloseWrite in stdout copier), + // CHANNEL_CLOSE (deferred ss.Close). RFC 4254 §6.10. + ss.Exit(exitCode) + closeAll(ss.childPipes...) + wg.Wait() } // recordSSHToLocalDisk is a deprecated dev knob to allow recording SSH sessions diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index 94a8549d6..1b5fdd2d7 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -566,9 +566,7 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) { }, sshCommand: "echo hello", wantClientOutput: "session rejected\r\n", - // 254 once recording failures get their own exit code; the - // current code exits 1 here. - wantExitCode: 0, + wantExitCode: 254, clientOutputMustNotContain: []string{"hello"}, },