Files
James Tucker 1334899864 ssh/tailssh: stop killing session processes on graceful disconnects
Monitor teardown SIGHUPed and SIGKILLed every remembered child group
when the transport closed. sshd signals nothing at a non-PTY session
end; background jobs with redirected stdio survive and reparent to
init. Clients rely on that: VS Code Remote-SSH starts its remote
server as a plain background job in the session's group and expects
it to outlive connection drops, and tailscale upgrades run as
non-PTY execs whose work should survive a daemon restart.

Use an explicit graceful monitor close to preserve session jobs on
ordinary disconnect and daemon shutdown. Close PTY masters normally,
letting the terminal driver signal the foreground group, but do not
signal non-PTY jobs. Remove Linux Pdeathsig from session children;
forwarding children retain it and always die with the connection.

Commit duration and recording-failure termination to the monitor
before publishing cancellation or writing the termination message.
The monitor owns per-session HUP-to-KILL escalation even after the
leader exits or the connection closes. Revocation arms connection-wide
cleanup. Pin every termination target's leader until after SIGKILL,
including during teardown, so group signaling cannot hit reused PIDs.
Select teardown targets explicitly and leave graceful survivors alone.

Unexpected RPC loss fails closed. If a termination mandate cannot be
delivered, close the RPC rather than subsequently requesting graceful
teardown; this may also kill sibling jobs on a broken channel. Keep
both the escalation and monitor shutdown waits bounded.

Add coverage for default-HUP graceful survivors, actual duration and
recording-failure termination followed by immediate disconnect, sibling
survival, HUP-exiting leaders with immune group members, broken RPCs,
and leader pinning through teardown's SIGKILL pass.

Fixes #22076

Signed-off-by: James Tucker <[email protected]>

Change-Id: I70b4ef40035330dd542d256ba79d889a12426a74
2026-10-30 16:39:06 -07:00

462 lines
15 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
// This file contains the plan9-specific version of the incubator. Tailscaled
// launches the incubator as the same user as it was launched as. The
// incubator then registers a new session with the OS, sets its UID
// and groups to the specified `--uid`, `--gid` and `--groups`, and
// then launches the requested `--cmd`.
package tailssh
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"os"
"os/exec"
"runtime"
"slices"
"strconv"
"strings"
"sync/atomic"
"github.com/go4org/plan9netshell"
"tailscale.com/cmd/tailscaled/childproc"
"tailscale.com/tailcfg"
"tailscale.com/types/logger"
)
// registerIncubator registers the incubator child process handlers.
// It is called from [Register], which is called from the init of
// tailscale.com/feature/ssh.
func registerIncubator() {
childproc.Add("ssh", beIncubator)
childproc.Add("sftp", beSFTP)
childproc.Add("plan9-netshell", beNetshell)
}
// newIncubatorCommand returns a new exec.Cmd configured with
// `tailscaled be-child ssh` as the entrypoint.
//
// If ss.srv.tailscaledPath is empty, this method is equivalent to
// exec.CommandContext.
//
// It also returns forwardedEnv, the client-forwarded environment variables
// accepted by the acceptEnv policy. These may contain secrets, so the caller
// passes them to the child via an inherited file rather than the command line
// or environment.
//
// The returned Cmd.Env is guaranteed to be nil; the caller must populate it.
func (ss *sshSession) newIncubatorCommand(logf logger.Logf) (cmd *exec.Cmd, forwardedEnv []string, err error) {
defer func() {
if cmd != nil && cmd.Env != nil {
panic("internal error")
}
}()
var isSFTP, isShell bool
switch ss.Subsystem() {
case "sftp":
isSFTP = true
case "":
isShell = ss.RawCommand() == ""
default:
panic(fmt.Sprintf("unexpected subsystem: %v", ss.Subsystem()))
}
if ss.conn.srv.tailscaledPath == "" {
if isSFTP {
// SFTP relies on the embedded Go-based SFTP server in tailscaled,
// so without tailscaled, we can't serve SFTP.
return nil, nil, errors.New("no tailscaled found on path, can't serve SFTP")
}
loginShell := ss.conn.localUser.LoginShell()
logf("directly running /bin/rc -c %q", ss.RawCommand())
return exec.CommandContext(ss.ctx, loginShell, "-c", ss.RawCommand()), nil, nil
}
lu := ss.conn.localUser
ci := ss.conn.info
remoteUser := ci.uprof.LoginName
if ci.node.IsTagged() {
remoteUser = strings.Join(ci.node.Tags().AsSlice(), ",")
}
incubatorArgs := []string{
"be-child",
"ssh",
// TODO: "--uid=" + lu.Uid,
// TODO: "--gid=" + lu.Gid,
"--local-user=" + lu.Username,
"--home-dir=" + lu.HomeDir,
"--remote-user=" + remoteUser,
"--remote-ip=" + ci.src.Addr().String(),
"--has-tty=false", // updated in-place by startWithPTY
"--tty-name=", // updated in-place by startWithPTY
}
nm := ss.conn.srv.lb.NetMapNoPeers()
forceV1Behavior := nm.HasCap(tailcfg.NodeAttrSSHBehaviorV1) && !nm.HasCap(tailcfg.NodeAttrSSHBehaviorV2)
if forceV1Behavior {
incubatorArgs = append(incubatorArgs, "--force-v1-behavior")
}
if debugTest.Load() {
incubatorArgs = append(incubatorArgs, "--debug-test")
}
switch {
case isSFTP:
// Note that we include both the `--sftp` flag and a command to launch
// tailscaled as `be-child sftp`. If login or su is available, and
// we're not running with tailcfg.NodeAttrSSHBehaviorV1, this will
// result in serving SFTP within a login shell, with full PAM
// integration. Otherwise, we'll serve SFTP in the incubator process
// with no PAM integration.
incubatorArgs = append(incubatorArgs, "--sftp", fmt.Sprintf("--cmd=%s be-child sftp", ss.conn.srv.tailscaledPath))
case isShell:
incubatorArgs = append(incubatorArgs, "--shell")
default:
incubatorArgs = append(incubatorArgs, "--cmd="+ss.RawCommand())
}
allowSendEnv := nm.HasCap(tailcfg.NodeAttrSSHEnvironmentVariables)
if allowSendEnv {
env, err := filterEnv(ss.conn.acceptEnv, ss.Session.Environ())
if err != nil {
return nil, nil, err
}
if len(env) > 0 {
// The accepted environment may contain secrets, so it travels via an
// inherited file (created by the caller). The fd number is not sensitive.
incubatorArgs = append(incubatorArgs, fmt.Sprintf("--env-fd=%d", forwardedEnvChildFD))
forwardedEnv = env
}
}
return exec.CommandContext(ss.ctx, ss.conn.srv.tailscaledPath, incubatorArgs...), forwardedEnv, nil
}
var debugTest atomic.Bool
type incubatorArgs struct {
localUser string
homeDir string
remoteUser string
remoteIP string
ttyName string
hasTTY bool
cmd string
isSFTP bool
isShell bool
forceV1Behavior bool
debugTest bool
isSELinuxEnforcing bool
// Deprecated: encodedEnv is deprecated and must not be used by new code.
// It is parsed only so this child keeps working when exec'd by an
// outdated parent tailscaled that still passes it.
encodedEnv string
// envFD is the file descriptor to read the forwarded environment from
// (a JSON array of KEY=VALUE pairs), or -1 if none.
envFD int
// forwardedEnv holds the pairs loaded by loadForwardedEnv.
forwardedEnv []string
}
func parseIncubatorArgs(args []string) (incubatorArgs, error) {
var ia incubatorArgs
flags := flag.NewFlagSet("", flag.ExitOnError)
flags.StringVar(&ia.localUser, "local-user", "", "the user to run as")
flags.StringVar(&ia.homeDir, "home-dir", "/", "the user's home directory")
flags.StringVar(&ia.remoteUser, "remote-user", "", "the remote user/tags")
flags.StringVar(&ia.remoteIP, "remote-ip", "", "the remote Tailscale IP")
flags.StringVar(&ia.ttyName, "tty-name", "", "the tty name (pts/3)")
flags.BoolVar(&ia.hasTTY, "has-tty", false, "is the output attached to a tty")
flags.StringVar(&ia.cmd, "cmd", "", "the cmd to launch, including all arguments (ignored in sftp mode)")
flags.BoolVar(&ia.isShell, "shell", false, "is launching a shell (with no cmds)")
flags.BoolVar(&ia.isSFTP, "sftp", false, "run sftp server (cmd is ignored)")
flags.BoolVar(&ia.forceV1Behavior, "force-v1-behavior", false, "allow falling back to the su command if login is unavailable")
flags.BoolVar(&ia.debugTest, "debug-test", false, "should debug in test mode")
flags.BoolVar(&ia.isSELinuxEnforcing, "is-selinux-enforcing", false, "whether SELinux is in enforcing mode")
// DEPRECATED: retained for version-skew compatibility only. DO NOT USE.
flags.StringVar(&ia.encodedEnv, "encoded-env", "", "deprecated; do not use")
flags.IntVar(&ia.envFD, "env-fd", -1, "file descriptor to read the forwarded environment from (JSON array of KEY=VALUE pairs)")
flags.Parse(args)
// envFD comes from an ExtraFiles entry, so it must never name stdin/out/err
if ia.envFD >= 0 && ia.envFD < 3 {
return ia, fmt.Errorf("invalid --env-fd %d: must be >= 3", ia.envFD)
}
return ia, nil
}
// loadForwardedEnv reads the client-forwarded environment pairs into ia.forwardedEnv, from the
// inherited file named by --env-fd. The pairs only enter the su/login/shell environment,
// never this process's own environment.
func (ia *incubatorArgs) loadForwardedEnv() error {
var pairs []string
switch {
case ia.envFD >= 0:
if ia.envFD < 3 {
return fmt.Errorf("invalid --env-fd=%d: must be >= 3", ia.envFD)
}
f := os.NewFile(uintptr(ia.envFD), "forwarded-env")
defer f.Close()
if err := json.NewDecoder(f).Decode(&pairs); err != nil {
return fmt.Errorf("unable to read forwarded environment: %w", err)
}
case ia.encodedEnv != "": // Legacy path to support an outdated parent tailscaled
if unquoted, err := strconv.Unquote(ia.encodedEnv); err != nil {
return fmt.Errorf("unable to parse encodedEnv %q: %w", ia.encodedEnv, err)
} else if err := json.Unmarshal([]byte(unquoted), &pairs); err != nil {
return fmt.Errorf("unable to parse encodedEnv %q: %w", ia.encodedEnv, err)
}
}
// Enforce "su -w" integrity and the dangerous-env-var denylist
// child-side: old parents may not have filtered these
pairs = slices.DeleteFunc(pairs, func(kv string) bool {
k, _, ok := strings.Cut(kv, "=")
return !ok || isDangerousEnvVar(k) || forbiddenEnvKey(k) || strings.Contains(kv, "\x00")
})
ia.forwardedEnv = pairs
return nil
}
// forwardedEnviron returns the environment to hand to the user's process (the
// current environment plus the client-forwarded pairs) and the comma-separated
// allowlist of forwarded key names, plus SSH_AUTH_SOCK.
func (ia *incubatorArgs) forwardedEnviron() ([]string, string) {
allowListKeys := []string{"SSH_AUTH_SOCK"}
environ := append(os.Environ(), ia.forwardedEnv...)
for _, kv := range ia.forwardedEnv {
if k, _, ok := strings.Cut(kv, "="); ok {
allowListKeys = append(allowListKeys, k)
}
}
return environ, strings.Join(allowListKeys, ",")
}
func beNetshell(args []string) error {
plan9netshell.Main()
return nil
}
// beIncubator is the entrypoint to the `tailscaled be-child ssh` subcommand.
// It is responsible for informing the system of a new login session for the
// user. This is sometimes necessary for mounting home directories and
// decrypting file systems.
//
// Tailscaled launches the incubator as the same user as it was launched as.
func beIncubator(args []string) error {
// To defend against issues like https://golang.org/issue/1435,
// defensively lock our current goroutine's thread to the current
// system thread before we start making any UID/GID/group changes.
//
// This shouldn't matter on Linux because syscall.AllThreadsSyscall is
// used to invoke syscalls on all OS threads, but (as of 2023-03-23)
// that function is not implemented on all platforms.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
ia, err := parseIncubatorArgs(args)
if err != nil {
return err
}
if ia.encodedEnv != "" {
log.Printf("WARNING: tailscaled be-child: accepted SSH environment variables were passed via the deprecated --encoded-env flag; " +
"the running tailscaled is outdated. Update tailscaled to the latest version and restart for the latest security fixes.")
}
if err := ia.loadForwardedEnv(); err != nil {
return err
}
if ia.isSFTP && ia.isShell {
return fmt.Errorf("--sftp and --shell are mutually exclusive")
}
if ia.isShell {
plan9netshell.Main()
return nil
}
dlogf := logger.Discard
if ia.debugTest {
// In testing, we don't always have syslog, so log to a temp file.
if logFile, err := os.OpenFile("/tmp/tailscalessh.log", os.O_APPEND|os.O_WRONLY, 0666); err == nil {
lf := log.New(logFile, "", 0)
dlogf = func(msg string, args ...any) {
lf.Printf(msg, args...)
logFile.Sync()
}
defer logFile.Close()
}
}
return handleInProcess(dlogf, ia)
}
func handleInProcess(dlogf logger.Logf, ia incubatorArgs) error {
if ia.isSFTP {
return handleSFTPInProcess(dlogf, ia)
}
return handleSSHInProcess(dlogf, ia)
}
func handleSFTPInProcess(dlogf logger.Logf, ia incubatorArgs) error {
dlogf("handling sftp")
return serveSFTP()
}
// handleSSHInProcess is a last resort if we couldn't use login or su. It
// registers a new session with the OS, sets its UID, GID and groups to the
// specified values, and then launches the requested `--cmd` in the user's
// login shell.
func handleSSHInProcess(dlogf logger.Logf, ia incubatorArgs) error {
environ, _ := ia.forwardedEnviron()
dlogf("running /bin/rc -c %q", ia.cmd)
cmd := newCommand("/bin/rc", environ, []string{"-c", ia.cmd})
err := cmd.Run()
if ee, ok := err.(*exec.ExitError); ok {
ps := ee.ProcessState
code := ps.ExitCode()
if code < 0 {
// TODO(bradfitz): do we need to also check the syscall.WaitStatus
// and make our process look like it also died by signal/same signal
// as our child process? For now we just do the exit code.
fmt.Fprintf(os.Stderr, "[tailscale-ssh: process died: %v]\n", ps.String())
code = 1 // for now. so we don't exit with negative
}
os.Exit(code)
}
return err
}
func newCommand(cmdPath string, cmdEnviron []string, cmdArgs []string) *exec.Cmd {
cmd := exec.Command(cmdPath, cmdArgs...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = cmdEnviron
return cmd
}
// launchProcess launches an incubator process for the provided session.
// It is responsible for configuring the process execution environment.
// The caller can wait for the process to exit by calling cmd.Wait().
//
// It sets ss.cmd, stdin, stdout, and stderr.
func (ss *sshSession) launchProcess() error {
var err error
var forwardedEnv []string
ss.cmd, forwardedEnv, err = ss.newIncubatorCommand(ss.logf)
if err != nil {
return err
}
cmd := ss.cmd
cmd.Dir = "/"
cmd.Env = append(os.Environ(), envForUser(ss.conn.localUser)...)
for _, kv := range ss.Environ() {
if acceptEnvPair(kv) {
cmd.Env = append(cmd.Env, kv)
}
}
ci := ss.conn.info
cmd.Env = append(cmd.Env,
fmt.Sprintf("SSH_CLIENT=%s %d %d", ci.src.Addr(), ci.src.Port(), ci.dst.Port()),
fmt.Sprintf("SSH_CONNECTION=%s %d %s %d", ci.src.Addr(), ci.src.Port(), ci.dst.Addr(), ci.dst.Port()),
)
if ss.agentListener != nil {
cmd.Env = append(cmd.Env, fmt.Sprintf("SSH_AUTH_SOCK=%s", ss.agentListener.Addr()))
}
// Client-forwarded environment variables may contain secrets, so they
// are passed to the child via an inherited pipe.
if len(forwardedEnv) > 0 {
envFile, err := forwardedEnvFile(forwardedEnv)
if err != nil {
return err
}
defer envFile.Close()
cmd.ExtraFiles = []*os.File{envFile}
}
return ss.startWithStdPipes()
}
// startWithStdPipes starts cmd with os.Pipe for Stdin, Stdout and Stderr.
func (ss *sshSession) startWithStdPipes() (err error) {
var rdStdin, wrStdout, wrStderr io.ReadWriteCloser
defer func() {
if err != nil {
closeAll(rdStdin, ss.wrStdin, ss.rdStdout, wrStdout, ss.rdStderr, wrStderr)
}
}()
if ss.cmd == nil {
return errors.New("nil cmd")
}
if rdStdin, ss.wrStdin, err = os.Pipe(); err != nil {
return err
}
if ss.rdStdout, wrStdout, err = os.Pipe(); err != nil {
return err
}
if ss.rdStderr, wrStderr, err = os.Pipe(); err != nil {
return err
}
ss.cmd.Stdin = rdStdin
ss.cmd.Stdout = wrStdout
ss.cmd.Stderr = wrStderr
ss.childPipes = []io.Closer{rdStdin, wrStdout, wrStderr}
return ss.cmd.Start()
}
// waitMonitorProcess is the plan9 seam of the monitor session path.
// plan9 has no monitor, so it always reports the direct path.
func (ss *sshSession) waitMonitorProcess() (code int, ok bool, err error) {
return 0, false, nil
}
// hangupMonitorProcess returns false on Plan 9, which launches children directly.
// See [sshSession.hangupProcess].
func (ss *sshSession) hangupMonitorProcess(mandated bool) bool {
return false
}
func envForUser(u *userMeta) []string {
return []string{
fmt.Sprintf("user=%s", u.Username),
fmt.Sprintf("home=%s", u.HomeDir),
fmt.Sprintf("path=%s", defaultPathForUser(u)),
}
}
// acceptEnvPair reports whether the environment variable key=value pair
// should be accepted from the client. It uses the same default as OpenSSH
// AcceptEnv.
func acceptEnvPair(kv string) bool {
k, _, ok := strings.Cut(kv, "=")
if !ok {
return false
}
// Never forward names reserved for our own parent->child bookkeeping or unsafe for the child,
// even during bringup, so a client cannot spoof the incubator's env.
if forbiddenEnvKey(k) || isDangerousEnvVar(k) {
return false
}
return true // permit anything else on plan9 during bringup, for debugging at least
}