mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-21 09:11:18 -08:00
ssh/tailssh: fix exit-status frame ordering
Send exit-status before EOF before CHANNEL_CLOSE per RFC 4254 §6.10.
The old code raced CloseWrite against Exit; macOS OpenSSH often saw
EOF first and dropped the exit code.
Replace atomic.Bool/atomic.Int32/channel sync with a sync.WaitGroup
over the output goroutines and emit:
cmd.Wait -> ss.Exit -> closeAll(childPipes) -> wg.Wait -> ss.Close
The stdin copier stays out of wg: it blocks reading from the SSH
channel until the client half-closes, and many clients (go-scp,
plain `ssh host cmd`, scp, sftp) don't half-close before they
receive CHANNEL_CLOSE from us. Including it deadlocks. It self-
cleans via deferred ss.Close.
Align exit codes with shell convention:
255 SSH-internal error (OpenSSH ssh.c:1693)
254 recording-infrastructure failure
127 command-not-found (POSIX)
Switch session termination from SIGKILL to SIGHUP, matching
OpenSSH PTY-master-close (session.c:2246). Process-group delivery
follows in a later commit.
Relies on the gliderssh wire-order change (tailscale/gliderssh#8),
which go.mod already pins since 3f5eb3199.
Based on tailscale/tailscale#18331.
Updates #18256
Signed-off-by: James Tucker <[email protected]>
Signed-off-by: Kristoffer Dalby <[email protected]>
This commit is contained in:
1 parent
12fcb2217b
commit
92f43003fc
2 files changed
+97
-65
No files matched your search
+96
-62
@@ -880,11 +880,19 @@ func (ss *sshSession) killProcessOnContextDone() {
|
||||
// We don't need to Process.Wait here, sshSession.run() does
|
||||
// the waiting regardless of termination reason.
|
||||
|
||||
// TODO(maisem): should this be a SIGTERM followed by a SIGKILL?
|
||||
ss.cmd.Process.Kill()
|
||||
// SIGHUP = POSIX terminal-disconnect semantics; OpenSSH gets it
|
||||
// implicitly via PTY-master close (session.c:2246), we send it
|
||||
// explicitly because non-PTY sessions use pipes.
|
||||
ss.cmd.Process.Signal(syscall.SIGHUP)
|
||||
})
|
||||
}
|
||||
|
||||
// isNotFoundOrExecutable reports whether err is a launchProcess
|
||||
// failure caused by the command not existing on disk.
|
||||
func isNotFoundOrExecutable(err error) bool {
|
||||
return errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
// attachSession registers ss as an active session.
|
||||
func (c *conn) attachSession(ss *sshSession) {
|
||||
c.srv.sessionWaitGroup.Add(1)
|
||||
@@ -970,10 +978,21 @@ func (ss *sshSession) run() {
|
||||
metricActiveSessions.Add(1)
|
||||
defer metricActiveSessions.Add(-1)
|
||||
defer ss.cancelCtx(errSessionDone)
|
||||
defer ss.Close() // CHANNEL_CLOSE, last on the wire; see ss.Exit below.
|
||||
|
||||
if attached := ss.conn.srv.attachSessionToConnIfNotShutdown(ss); !attached {
|
||||
fmt.Fprintf(ss, "Tailscale SSH is shutting down\r\n")
|
||||
ss.Exit(1)
|
||||
// 255 signals an SSH-layer failure (the session never reached the
|
||||
// user's command), distinct from any exit status the remote
|
||||
// program might produce. The ssh(1) EXIT STATUS section
|
||||
// documents this as the value the client reports "if an error
|
||||
// occurred", and OpenSSH's own ssh.c uses exit(255) for every
|
||||
// transport/protocol fatal path. Wrappers and CI use the 255
|
||||
// boundary to tell "connection broke" from "command exited N".
|
||||
// See:
|
||||
// https://man.openbsd.org/ssh#EXIT_STATUS
|
||||
// https://github.com/openssh/openssh-portable/blob/V_10_2_P1/ssh.c#L1693
|
||||
ss.Exit(255)
|
||||
return
|
||||
}
|
||||
defer ss.conn.detachSession(ss)
|
||||
@@ -995,7 +1014,9 @@ func (ss *sshSession) run() {
|
||||
if lu.Uid != fmt.Sprint(euid) {
|
||||
ss.logf("can't switch to user %q from process euid %v", lu.Username, euid)
|
||||
fmt.Fprintf(ss, "can't switch user\r\n")
|
||||
ss.Exit(1)
|
||||
// 255: SSH-layer failure, no user command ever ran. See the
|
||||
// attachSession branch above for the full citation.
|
||||
ss.Exit(255)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1023,7 +1044,27 @@ func (ss *sshSession) run() {
|
||||
fmt.Fprintf(ss, "can't start new recording\r\n")
|
||||
}
|
||||
ss.logf("startNewRecording: %v", err)
|
||||
ss.Exit(1)
|
||||
// 254 is Tailscale-specific: the SSH transport is fine
|
||||
// and the user's command is well-formed, but the session
|
||||
// recording policy could not be satisfied (recorder
|
||||
// unreachable, upload denied, etc.) so we refuse to run
|
||||
// the command at all. We need a code that operators can
|
||||
// alert on without collapsing it into the generic buckets:
|
||||
// - 1 is overloaded; any program can produce it
|
||||
// (Bash exit-code conventions, codes 1-2)
|
||||
// - 127 means "command not found" (POSIX 2018, sh
|
||||
// §2.8.2)
|
||||
// - 130 is Ctrl-C (128 + SIGINT)
|
||||
// - 255 means "SSH itself failed" (ssh(1) EXIT STATUS)
|
||||
// 254 sits in the reserved >128 region but is not claimed
|
||||
// by any of the above, so it is unambiguous for
|
||||
// "recording-required session denied".
|
||||
// Refs:
|
||||
// https://man.openbsd.org/ssh#EXIT_STATUS
|
||||
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
|
||||
// https://tldp.org/LDP/abs/html/exitcodes.html
|
||||
// https://github.com/tailscale/tailscale/issues/18256
|
||||
ss.Exit(254)
|
||||
return
|
||||
}
|
||||
ss.logf("startNewRecording: <nil>")
|
||||
@@ -1036,6 +1077,7 @@ func (ss *sshSession) run() {
|
||||
err := ss.launchProcess()
|
||||
if err != nil {
|
||||
logf("start failed: %v", err.Error())
|
||||
exitCode := 1
|
||||
if errors.Is(err, context.Canceled) {
|
||||
cause := context.Cause(ss.ctx)
|
||||
if serr, ok := cause.(SSHTerminationError); ok {
|
||||
@@ -1043,14 +1085,35 @@ func (ss *sshSession) run() {
|
||||
io.WriteString(ss.Stderr(), "\r\n\r\n"+msg+"\r\n\r\n")
|
||||
}
|
||||
}
|
||||
} else if isNotFoundOrExecutable(err) {
|
||||
// 127 = "command not found" per POSIX 2018 sh §2.8.2
|
||||
// ("if a command is not found, the exit status shall be
|
||||
// 127"). 126 is reserved for "found but not executable";
|
||||
// we collapse both into 127 because exec.ErrNotFound and
|
||||
// os.ErrNotExist don't distinguish the underlying cause
|
||||
// reliably across platforms, and 127 is what users
|
||||
// recognise (it's what bash / dash / zsh report).
|
||||
// Refs:
|
||||
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
|
||||
// https://tldp.org/LDP/abs/html/exitcodes.html
|
||||
exitCode = 127
|
||||
}
|
||||
ss.Exit(1)
|
||||
ss.Exit(exitCode)
|
||||
return
|
||||
}
|
||||
ss.exitHandled = make(chan struct{})
|
||||
go ss.killProcessOnContextDone()
|
||||
|
||||
var processDone atomic.Bool
|
||||
// wg covers stdout and stderr only: their close ordering is
|
||||
// load-bearing for the wire-level frame sequence below.
|
||||
//
|
||||
// The stdin copier is deliberately NOT in wg. It blocks reading from
|
||||
// the SSH channel until the client half-closes its write side; many
|
||||
// clients (go-scp, non-SFTP exec) don't half-close before they
|
||||
// receive CHANNEL_CLOSE from us. Including it would deadlock. It
|
||||
// self-cleans when deferred ss.Close runs.
|
||||
var wg sync.WaitGroup
|
||||
|
||||
go func() {
|
||||
defer ss.wrStdin.Close()
|
||||
if _, err := io.Copy(rec.writer("i", ss.wrStdin), ss); err != nil {
|
||||
@@ -1058,52 +1121,31 @@ func (ss *sshSession) run() {
|
||||
ss.cancelCtx(err)
|
||||
}
|
||||
}()
|
||||
outputDone := make(chan struct{})
|
||||
var openOutputStreams atomic.Int32
|
||||
if ss.rdStderr != nil {
|
||||
openOutputStreams.Store(2)
|
||||
} else {
|
||||
openOutputStreams.Store(1)
|
||||
}
|
||||
go func() {
|
||||
|
||||
wg.Go(func() {
|
||||
defer ss.rdStdout.Close()
|
||||
_, err := io.Copy(rec.writer("o", ss), ss.rdStdout)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
isErrBecauseProcessExited := processDone.Load() && errors.Is(err, syscall.EIO)
|
||||
if !isErrBecauseProcessExited {
|
||||
logf("stdout copy: %v", err)
|
||||
ss.cancelCtx(err)
|
||||
}
|
||||
if _, err := io.Copy(rec.writer("o", ss), ss.rdStdout); err != nil && !errors.Is(err, io.EOF) {
|
||||
logf("stdout copy: %v", err)
|
||||
}
|
||||
if openOutputStreams.Add(-1) == 0 {
|
||||
ss.CloseWrite()
|
||||
close(outputDone)
|
||||
}
|
||||
}()
|
||||
ss.CloseWrite() // CHANNEL_EOF; channel stays open for exit-status (RFC 4254 §5.3).
|
||||
})
|
||||
|
||||
// rdStderr is nil for ptys.
|
||||
if ss.rdStderr != nil {
|
||||
go func() {
|
||||
wg.Go(func() {
|
||||
defer ss.rdStderr.Close()
|
||||
_, err := io.Copy(ss.Stderr(), ss.rdStderr)
|
||||
if err != nil {
|
||||
if _, err := io.Copy(ss.Stderr(), ss.rdStderr); err != nil {
|
||||
logf("stderr copy: %v", err)
|
||||
}
|
||||
if openOutputStreams.Add(-1) == 0 {
|
||||
ss.CloseWrite()
|
||||
close(outputDone)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
err = ss.cmd.Wait()
|
||||
processDone.Store(true)
|
||||
|
||||
if ss.ctx.Err() != nil {
|
||||
// Context was canceled (e.g., recording upload failure).
|
||||
// Wait for killProcessOnContextDone to finish writing any
|
||||
// termination message before we proceed. This must happen
|
||||
// before closeAll and CloseWrite so the SSH channel is
|
||||
// still writable.
|
||||
// Cancellation (e.g. recording upload failure) wrote a
|
||||
// termination message via killProcessOnContextDone; wait
|
||||
// for it before Exit closes the channel for writes.
|
||||
<-ss.exitHandled
|
||||
}
|
||||
|
||||
@@ -1112,31 +1154,23 @@ func (ss *sshSession) run() {
|
||||
// aforementioned goroutine.
|
||||
ss.exitOnce.Do(func() {})
|
||||
|
||||
// Close the process-side of all pipes to signal the asynchronous
|
||||
// io.Copy routines reading/writing from the pipes to terminate.
|
||||
// Block for the io.Copy to finish before calling ss.Exit below.
|
||||
closeAll(ss.childPipes...)
|
||||
select {
|
||||
case <-outputDone:
|
||||
case <-ss.ctx.Done():
|
||||
<-ss.exitHandled
|
||||
}
|
||||
|
||||
var exitCode int
|
||||
if err == nil {
|
||||
ss.logf("Session complete")
|
||||
ss.Exit(0)
|
||||
return
|
||||
}
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
code := ee.ProcessState.ExitCode()
|
||||
ss.logf("Wait: code=%v", code)
|
||||
ss.Exit(code)
|
||||
return
|
||||
exitCode = 0
|
||||
} else if ee, ok := err.(*exec.ExitError); ok {
|
||||
exitCode = ee.ProcessState.ExitCode()
|
||||
ss.logf("Wait: code=%v", exitCode)
|
||||
} else {
|
||||
ss.logf("Wait: %v", err)
|
||||
exitCode = 1
|
||||
}
|
||||
|
||||
ss.logf("Wait: %v", err)
|
||||
ss.Exit(1)
|
||||
return
|
||||
// Order on the wire: exit-status, EOF (CloseWrite in stdout copier),
|
||||
// CHANNEL_CLOSE (deferred ss.Close). RFC 4254 §6.10.
|
||||
ss.Exit(exitCode)
|
||||
closeAll(ss.childPipes...)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// recordSSHToLocalDisk is a deprecated dev knob to allow recording SSH sessions
|
||||
|
||||
@@ -566,9 +566,7 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) {
|
||||
},
|
||||
sshCommand: "echo hello",
|
||||
wantClientOutput: "session rejected\r\n",
|
||||
// 254 once recording failures get their own exit code; the
|
||||
// current code exits 1 here.
|
||||
wantExitCode: 0,
|
||||
wantExitCode: 254,
|
||||
|
||||
clientOutputMustNotContain: []string{"hello"},
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user