mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
Cancel fail-closed sessions before sending recording upload failure notifications. Bound each notifyControl request to five seconds, including recorder connection failures on the synchronous session startup path. An unresponsive control request must not keep an unrecorded session running or indefinitely delay rejection or fail-open startup. Keep the timeout local to each notification, independent of the session and upload lifetime, without adding another goroutine. Test blocked startup notifications in both rejection and fail-open modes, and use virtual time to cover earlier and simultaneous cancellation/deadlines. Send SFTP recording rejection diagnostics to stderr, preserving the binary protocol on stdout and exit status 254. Add real SSH/recorder tests for fail-closed and fail-open behavior with blocked notifications, and an SFTP client regression test. Updates tailscale/corp#50380 Signed-off-by: James Tucker <[email protected]> Change-Id: I270d624975a6a9115502a4915ee5ff28a67d714b
244 lines
8.1 KiB
Go
244 lines
8.1 KiB
Go
// Copyright (c) Tailscale Inc & contributors
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build linux || darwin
|
|
|
|
package tailssh
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/netip"
|
|
"runtime"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"tailscale.com/net/memnet"
|
|
"tailscale.com/tailcfg"
|
|
testssh "tailscale.com/tempfork/sshtest/ssh"
|
|
"tailscale.com/tstest"
|
|
"tailscale.com/types/key"
|
|
)
|
|
|
|
// recordingNotifyCaptureState observes a notification while control is blocked
|
|
// and records why the request eventually returns.
|
|
type recordingNotifyCaptureState struct {
|
|
*blockingRecordingNotifyState
|
|
requests chan *http.Request
|
|
results chan error
|
|
}
|
|
|
|
func (b *recordingNotifyCaptureState) DoNoiseRequest(req *http.Request) (*http.Response, error) {
|
|
if req.URL.Path != "/recording-failed" {
|
|
return b.localState.DoNoiseRequest(req)
|
|
}
|
|
b.requests <- req
|
|
resp, err := b.blockingRecordingNotifyState.DoNoiseRequest(req)
|
|
b.results <- err
|
|
return resp, err
|
|
}
|
|
|
|
func newRecordingNotifyCaptureState(ls *localState) *recordingNotifyCaptureState {
|
|
return &recordingNotifyCaptureState{
|
|
blockingRecordingNotifyState: &blockingRecordingNotifyState{
|
|
localState: ls,
|
|
entered: make(chan time.Time, 1),
|
|
release: make(chan struct{}),
|
|
},
|
|
requests: make(chan *http.Request, 1),
|
|
results: make(chan error, 1),
|
|
}
|
|
}
|
|
|
|
// TestRecordingNotifyTimeout tests the timeout with virtual time, including
|
|
// session cancellation at exactly the notification deadline. A notification
|
|
// using a background context must survive session cancellation, but it must
|
|
// honor cancellation or an earlier deadline on its own parent context.
|
|
func TestRecordingNotifyTimeout(t *testing.T) {
|
|
for _, mode := range []string{"timeout", "parent-deadline", "parent-cancel", "simultaneous-parent-cancel", "session-cancel", "simultaneous-session-cancel"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
lb := newRecordingNotifyCaptureState(new(localState))
|
|
sessionCtx, cancelSession := context.WithCancel(context.Background())
|
|
defer cancelSession()
|
|
ss := &sshSession{
|
|
ctx: sessionCtx,
|
|
logf: t.Logf,
|
|
conn: &conn{
|
|
srv: &server{lb: lb},
|
|
localUser: new(userMeta),
|
|
info: &sshConnInfo{node: (&tailcfg.Node{ID: 1}).View()},
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
wantDuration := 5 * time.Second
|
|
var wantErr error = context.DeadlineExceeded
|
|
switch mode {
|
|
case "parent-deadline":
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, time.Second)
|
|
defer cancel()
|
|
wantDuration = time.Second
|
|
case "parent-cancel", "simultaneous-parent-cancel":
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithCancel(ctx)
|
|
defer cancel()
|
|
if mode == "parent-cancel" {
|
|
wantDuration, wantErr = time.Second, context.Canceled
|
|
}
|
|
defer time.AfterFunc(wantDuration, cancel).Stop()
|
|
case "session-cancel":
|
|
defer time.AfterFunc(time.Second, cancelSession).Stop()
|
|
case "simultaneous-session-cancel":
|
|
defer time.AfterFunc(5*time.Second, cancelSession).Stop()
|
|
}
|
|
start := time.Now()
|
|
ss.notifyControl(ctx, key.NodePublic{}, tailcfg.SSHSessionRecordingFailed, nil, "https://unused/recording-failed")
|
|
synctest.Wait()
|
|
if elapsed := time.Since(start); elapsed != wantDuration {
|
|
t.Errorf("notification duration = %v; want %v", elapsed, wantDuration)
|
|
}
|
|
err := <-lb.results
|
|
if mode == "simultaneous-parent-cancel" {
|
|
// Either timer can win; both must end the request at 5s.
|
|
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Errorf("notification error = %v; want cancellation or deadline", err)
|
|
}
|
|
} else if !errors.Is(err, wantErr) {
|
|
t.Errorf("notification error = %v; want %v", err, wantErr)
|
|
}
|
|
if mode == "timeout" && sessionCtx.Err() != nil {
|
|
t.Errorf("notification timeout canceled session: %v", sessionCtx.Err())
|
|
}
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestSSHRecordingConnectFailureNotifyTimeout exercises the synchronous startup
|
|
// path. Rejection and fail-open startup must both finish when a stuck control
|
|
// notification times out, with no upload watcher or duplicate notification.
|
|
func TestSSHRecordingConnectFailureNotifyTimeout(t *testing.T) {
|
|
if runtime.GOOS != "linux" {
|
|
t.Skip("requires the Linux direct-exec test path")
|
|
}
|
|
for _, reject := range []bool{true, false} {
|
|
name := "fail-open"
|
|
if reject {
|
|
name = "reject"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
recorder := mockRecordingServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
})
|
|
onFailure := &tailcfg.SSHRecorderFailureAction{NotifyURL: "https://unused/recording-failed"}
|
|
if reject {
|
|
onFailure.RejectSessionWithMessage = "session rejected"
|
|
}
|
|
lb := newRecordingNotifyCaptureState(&localState{
|
|
sshEnabled: true,
|
|
varRoot: t.TempDir(),
|
|
matchingRule: newSSHRule(&tailcfg.SSHAction{
|
|
Accept: true,
|
|
Recorders: []netip.AddrPort{netip.MustParseAddrPort(recorder.Listener.Addr().String())},
|
|
OnRecordingFailure: onFailure,
|
|
}),
|
|
})
|
|
s := &server{logf: tstest.WhileTestRunningLogger(t), lb: lb}
|
|
defer func() {
|
|
close(lb.release)
|
|
s.Shutdown()
|
|
}()
|
|
sc, dc := memnet.NewTCPConn(netip.MustParseAddrPort("100.100.100.101:2231"), netip.MustParseAddrPort("100.100.100.102:22"), 1024)
|
|
defer sc.Close()
|
|
defer dc.Close()
|
|
if err := sc.SetDeadline(time.Now().Add(20 * time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
type result struct {
|
|
out []byte
|
|
err error
|
|
}
|
|
clientDone := make(chan result, 1)
|
|
go func() {
|
|
c, chans, reqs, err := testssh.NewClientConn(sc, sc.RemoteAddr().String(), &testssh.ClientConfig{
|
|
User: "alice", HostKeyCallback: testssh.InsecureIgnoreHostKey(),
|
|
})
|
|
if err != nil {
|
|
clientDone <- result{err: err}
|
|
return
|
|
}
|
|
client := testssh.NewClient(c, chans, reqs)
|
|
defer client.Close()
|
|
session, err := client.NewSession()
|
|
if err != nil {
|
|
clientDone <- result{err: err}
|
|
return
|
|
}
|
|
defer session.Close()
|
|
out, err := session.CombinedOutput("echo ran")
|
|
clientDone <- result{out, err}
|
|
}()
|
|
handleDone := make(chan error, 1)
|
|
go func() { handleDone <- s.HandleSSHConn(dc) }()
|
|
var req *http.Request
|
|
select {
|
|
case req = <-lb.requests:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("recording failure notification was not attempted")
|
|
}
|
|
if _, ok := req.Context().Deadline(); !ok {
|
|
t.Fatal("connect-failure notification has no deadline")
|
|
}
|
|
var event tailcfg.SSHEventNotifyRequest
|
|
if err := json.NewDecoder(req.Body).Decode(&event); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantEvent := tailcfg.SSHSessionRecordingFailed
|
|
if reject {
|
|
wantEvent = tailcfg.SSHSessionRecordingRejected
|
|
}
|
|
if event.EventType != wantEvent || len(event.RecordingAttempts) != 1 || event.RecordingAttempts[0].FailureMessage == "" {
|
|
t.Fatalf("unexpected connect-failure notification: %+v", event)
|
|
}
|
|
// Allow margin beyond the five-second request deadline so a test
|
|
// timeout cannot race the behavior it is checking.
|
|
select {
|
|
case res := <-clientDone:
|
|
if reject {
|
|
ee, ok := res.err.(*testssh.ExitError)
|
|
if !ok || ee.ExitStatus() != 254 || string(res.out) != "session rejected\r\n" {
|
|
t.Errorf("rejected session = %q, %v; want rejection and exit 254", res.out, res.err)
|
|
}
|
|
} else if res.err != nil || string(res.out) != "ran\n" {
|
|
t.Errorf("fail-open session = %q, %v; want command output and success", res.out, res.err)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("session remained blocked on control notification")
|
|
}
|
|
select {
|
|
case err := <-lb.results:
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Errorf("notification error = %v; want deadline exceeded", err)
|
|
}
|
|
default:
|
|
t.Fatal("session completed before notification returned")
|
|
}
|
|
select {
|
|
case err := <-handleDone:
|
|
if err != nil {
|
|
t.Errorf("HandleSSHConn: %v", err)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("SSH handler did not finish")
|
|
}
|
|
if len(lb.requests) != 0 {
|
|
t.Error("unexpected second recording failure notification")
|
|
}
|
|
})
|
|
}
|
|
}
|