diff --git a/net/tstun/wrap.go b/net/tstun/wrap.go index 749efa416..702c36046 100644 --- a/net/tstun/wrap.go +++ b/net/tstun/wrap.go @@ -208,6 +208,11 @@ type Wrapper struct { captureHook syncs.AtomicValue[packet.CaptureCallback] + // outboundInjectHook, if set, receives packets injected outbound + // (InjectOutbound and friends) instead of the queue that + // wireguard-go reads; see SetOutboundInjectHook. + outboundInjectHook syncs.AtomicValue[func([]byte)] + metrics *metrics eventClient *eventbus.Client @@ -505,6 +510,10 @@ func (t *Wrapper) injectOutbound(ctx context.Context, r tunInjectedRead) (err er if t.isClosed() { return ErrClosed } + if hook := t.outboundInjectHook.Load(); hook != nil { + t.sendToOutboundInjectHook(hook, r) + return nil + } // Avoid a cancellation select when the queue has space. select { case t.injectionQueue.ch <- r: @@ -537,6 +546,10 @@ func (t *Wrapper) tryInjectOutbound(data []byte) error { if t.isClosed() { return ErrClosed } + if hook := t.outboundInjectHook.Load(); hook != nil { + hook(data) + return nil + } select { case t.injectionQueue.ch <- tunInjectedRead{data: data}: t.drainIfClosed() @@ -547,6 +560,31 @@ func (t *Wrapper) tryInjectOutbound(data []byte) error { return ErrInjectionQueueFull } +// SetOutboundInjectHook makes packets injected outbound (InjectOutbound, +// TryInjectOutbound, InjectOutboundPacketBuffer and so on) go to fn, which +// takes ownership of them, instead of being queued for wireguard-go to read +// and send to peers. It's for the tswg kernel data plane, which sends them +// into the kernel's WireGuard device. fn must not block for long. +func (t *Wrapper) SetOutboundInjectHook(fn func(pkt []byte)) { + t.outboundInjectHook.Store(fn) +} + +// sendToOutboundInjectHook passes the injected packet r to hook. +func (t *Wrapper) sendToOutboundInjectHook(hook func([]byte), r tunInjectedRead) { + if r.packet == nil { + hook(r.data) + return + } + if !buildfeatures.HasNetstack { + panic("unreachable") + } + b := r.packet.ToBuffer() + data := b.Flatten() + b.Release() + r.packet.DecRef() + hook(data) +} + // snat does SNAT on p if the destination address requires a different source address. // snat never mutates packet length or IP address family. func (pc *peerConfigTable) snat(p *packet.Parsed) { diff --git a/tstest/natlab/vmtest/tswg_tailscaled_test.go b/tstest/natlab/vmtest/tswg_tailscaled_test.go index 2e77caaa2..172e702b8 100644 --- a/tstest/natlab/vmtest/tswg_tailscaled_test.go +++ b/tstest/natlab/vmtest/tswg_tailscaled_test.go @@ -10,94 +10,127 @@ import ( "fmt" "io" "net/http" + "net/netip" + "strings" "testing" "time" + "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/tstest/natlab/vmtest" "tailscale.com/tstest/natlab/vnet" "tailscale.com/wgengine/tswg" ) +// tswgTailscaled is a test fixture of two gokrazy VMs, each behind its own +// NAT, running tailscaled with the tswg kernel data plane. +type tswgTailscaled struct { + t *testing.T + env *vmtest.Env + nodes []*vmtest.Node +} + +// newTSWGTailscaled boots two gokrazy VMs behind NATs of type nat with +// TS_DEBUG_TSWG=1, loads tswg, restarts tailscaled so that it switches +// from TUN (its fallback while the module wasn't loaded) to the kernel +// data plane, and waits for the kernel device to be set up. +func newTSWGTailscaled(t *testing.T, nat vnet.NAT) *tswgTailscaled { + dir := tswgDir(t) + ko := buildTSWGForGokrazy(t, dir) + + f := &tswgTailscaled{t: t, env: vmtest.New(t)} + for i := range 2 { + f.nodes = append(f.nodes, f.env.AddNode(fmt.Sprintf("tswg%d", i+1), + f.env.AddNetwork( + fmt.Sprintf("2.%d.%d.%d", i+1, i+1, i+1), + fmt.Sprintf("192.168.%d.1/24", i+1), nat), + vmtest.OS(vmtest.Gokrazy), + vnet.TailscaledEnv{Key: "TS_DEBUG_TSWG", Value: "1"})) + } + f.env.Start() + + for _, n := range f.nodes { + tta(t, n, "POST", "/tswg/load", ko) + f.env.RestartTailscaled(n) + } + for _, n := range f.nodes { + f.waitKernelReady(n) + } + return f +} + +// show returns the state of n's kernel tailscale0 device. +func (f *tswgTailscaled) show(n *vmtest.Node) (*tswg.Device, error) { + res, err := ttaErr(n, "GET", "/tswg/show?dev=tailscale0", nil) + if err != nil { + return nil, err + } + var d tswg.Device + if err := json.Unmarshal([]byte(res), &d); err != nil { + return nil, err + } + return &d, nil +} + +// waitKernelReady waits for n's tailscaled to have set up its kernel +// device: a shared socket and the other node as a peer. +func (f *tswgTailscaled) waitKernelReady(n *vmtest.Node) { + f.t.Helper() + if err := tstest.WaitFor(30*time.Second, func() error { + d, err := f.show(n) + if err != nil { + return err + } + if !d.UserSocket4 { + return fmt.Errorf("tailscale0 has no shared IPv4 socket") + } + if len(d.Peers) != 1 { + return fmt.Errorf("tailscale0 has %d peers, want 1", len(d.Peers)) + } + return nil + }); err != nil { + f.t.Fatalf("[%s] kernel data plane not set up: %v", n.Name(), err) + } +} + +// ip returns n's Tailscale IPv4 address. +func (f *tswgTailscaled) ip(n *vmtest.Node) netip.Addr { + return f.env.Status(n).Self.TailscaleIPs[0] +} + +// metric returns the value of n's client metric name. +func (f *tswgTailscaled) metric(n *vmtest.Node, name string) int64 { + return f.env.ClientMetrics(n)[name].Value +} + // TestTSWGTailscaledDirect runs tailscaled with the tswg kernel data plane -// (TS_DEBUG_TSWG=1) on two gokrazy VMs behind separate easy NATs, so that -// disco's NAT traversal over the socket that magicsock shares with the -// kernel finds a direct path, and checks that the kernel then sends to the -// peer directly. +// on two gokrazy VMs behind separate easy NATs, so that disco's NAT +// traversal over the socket that magicsock shares with the kernel finds a +// direct path, and checks that the kernel then sends to the peer directly. func TestTSWGTailscaledDirect(t *testing.T) { - testTSWGTailscaled(t, vnet.EasyNAT, true) + testTSWGTailscaledPath(t, vnet.EasyNAT, true) } // TestTSWGTailscaledDERP is like TestTSWGTailscaledDirect, but behind hard // NATs on both sides, so that there's no direct path and all traffic goes // from the kernel through tailscaled's loopback bridge to DERP and back. func TestTSWGTailscaledDERP(t *testing.T) { - testTSWGTailscaled(t, vnet.HardNAT, false) + testTSWGTailscaledPath(t, vnet.HardNAT, false) } -func testTSWGTailscaled(t *testing.T, nat vnet.NAT, wantDirect bool) { - dir := tswgDir(t) - ko := buildTSWGForGokrazy(t, dir) - - env := vmtest.New(t) - var nodes []*vmtest.Node - for i := range 2 { - nodes = append(nodes, env.AddNode(fmt.Sprintf("tswg%d", i+1), - env.AddNetwork( - fmt.Sprintf("2.%d.%d.%d", i+1, i+1, i+1), - fmt.Sprintf("192.168.%d.1/24", i+1), nat), - vmtest.OS(vmtest.Gokrazy), - vnet.TailscaledEnv{Key: "TS_DEBUG_TSWG", Value: "1"})) - } - env.Start() - - // tailscaled started before the module was loaded, so it fell back - // to TUN. Load the module and restart tailscaled to switch to the - // kernel data plane. - for _, n := range nodes { - tta(t, n, "POST", "/tswg/load", ko) - env.RestartTailscaled(n) - } - - show := func(n *vmtest.Node) (*tswg.Device, error) { - var d tswg.Device - res, err := ttaErr(n, "GET", "/tswg/show?dev=tailscale0", nil) - if err != nil { - return nil, err - } - if err := json.Unmarshal([]byte(res), &d); err != nil { - return nil, err - } - return &d, nil - } - for _, n := range nodes { - if err := tstest.WaitFor(30*time.Second, func() error { - d, err := show(n) - if err != nil { - return err - } - if !d.UserSocket4 { - return fmt.Errorf("tailscale0 has no shared IPv4 socket") - } - if len(d.Peers) != 1 { - return fmt.Errorf("tailscale0 has %d peers, want 1", len(d.Peers)) - } - return nil - }); err != nil { - t.Fatalf("[%s] kernel data plane not set up: %v", n.Name(), err) - } - } - - ip2 := env.Status(nodes[1]).Self.TailscaleIPs[0] - env.LANPing(nodes[0], ip2) +func testTSWGTailscaledPath(t *testing.T, nat vnet.NAT, wantDirect bool) { + f := newTSWGTailscaled(t, nat) + n1, n2 := f.nodes[0], f.nodes[1] + ip2 := f.ip(n2) + f.env.LANPing(n1, ip2) // Wait for the kernel's endpoint for the peer to settle: direct, or // tailscaled's loopback bridge. var d *tswg.Device err := tstest.WaitFor(60*time.Second, func() error { - pingOnce(t, nodes[0], ip2) // keep the session active + pingOnce(t, n1, ip2) // keep the session active var err error - d, err = show(nodes[0]) + d, err = f.show(n1) if err != nil { return err } @@ -115,7 +148,7 @@ func testTSWGTailscaled(t *testing.T, nat vnet.NAT, wantDirect bool) { } p := d.Peers[0] t.Logf("[%s] kernel peer %v: endpoint=%v rx=%d tx=%d handshake=%v no-roam=%v", - nodes[0].Name(), p.PublicKey.ShortString(), p.Endpoint, p.RxBytes, p.TxBytes, p.LastHandshake, p.NoRoam) + n1.Name(), p.PublicKey.ShortString(), p.Endpoint, p.RxBytes, p.TxBytes, p.LastHandshake, p.NoRoam) if !p.NoRoam { t.Errorf("kernel peer has NoRoam = false, want true") } @@ -124,8 +157,82 @@ func testTSWGTailscaled(t *testing.T, nat vnet.NAT, wantDirect bool) { } // And the tunnel still works, both ways, on the settled path. - env.LANPing(nodes[0], ip2) - env.LANPing(nodes[1], env.Status(nodes[0]).Self.TailscaleIPs[0]) + f.env.LANPing(n1, ip2) + f.env.LANPing(n2, f.ip(n1)) +} + +// TestTSWGTailscaledTSMP checks TSMP in kernel mode: TSMP pings (and their +// pongs), which tailscaled sends and answers through raw sockets on the +// kernel device, and the TSMP disco key advertisements that tailscaled +// sends when the kernel establishes a session, as wireguard-go does. +func TestTSWGTailscaledTSMP(t *testing.T) { + f := newTSWGTailscaled(t, vnet.EasyNAT) + n1, n2 := f.nodes[0], f.nodes[1] + + // Establishing the session makes each side advertise its disco key + // to the other over TSMP. + f.env.LANPing(n1, f.ip(n2)) + const recvMetric = "magicsock_tsmp_disco_key_advertisement_received" + for _, n := range f.nodes { + if err := tstest.WaitFor(15*time.Second, func() error { + if v := f.metric(n, recvMetric); v == 0 { + return fmt.Errorf("%s = 0", recvMetric) + } + return nil + }); err != nil { + t.Errorf("[%s] no TSMP disco key advertisement received: %v", n.Name(), err) + } + } + + for _, pair := range [][2]*vmtest.Node{{n1, n2}, {n2, n1}} { + if err := f.env.Ping(pair[0], pair[1], tailcfg.PingTSMP, 30*time.Second); err != nil { + t.Errorf("TSMP ping %s -> %s: %v", pair[0].Name(), pair[1].Name(), err) + } + } +} + +// TestTSWGTailscaledPeerRestart checks that when a peer restarts (and so +// loses its WireGuard session), the kernel forgets its stale session as +// soon as tailscaled learns the peer's new disco key, so that the next +// packet starts a new handshake. Without that, the kernel would keep +// sending with dead keys until WireGuard's own timeouts start a new +// handshake about 15 seconds later. +func TestTSWGTailscaledPeerRestart(t *testing.T) { + f := newTSWGTailscaled(t, vnet.EasyNAT) + n1, n2 := f.nodes[0], f.nodes[1] + ip2 := f.ip(n2) + f.env.LANPing(n1, ip2) + + n2Key := f.env.Status(n2).Self.PublicKey + oldDisco, ok, err := f.env.PeerDiscoKey(n1, n2Key) + if err != nil || !ok { + t.Fatalf("getting %s's disco key for %s: ok=%v, %v", n1.Name(), n2.Name(), ok, err) + } + + // Restarting tailscaled on n2 recreates its kernel device, losing the + // session, and gives it a new disco key. + f.env.RestartTailscaled(n2) + f.waitKernelReady(n2) + if err := tstest.WaitFor(30*time.Second, func() error { + k, ok, err := f.env.PeerDiscoKey(n1, n2Key) + if err != nil { + return err + } + if !ok || k == oldDisco { + return fmt.Errorf("still has the old disco key") + } + return nil + }); err != nil { + t.Fatalf("%s never learned %s's new disco key: %v", n1.Name(), n2.Name(), err) + } + + // The first round of pings from n1 must get through. + if !pingOnce(t, n1, ip2) { + t.Errorf("first ping after %s restarted failed; the stale session wasn't cleared", n2.Name()) + } + if logs := f.env.NodeLogs(n1); !strings.Contains(logs, "tswg: cleared session with") { + t.Errorf("%s's logs don't mention clearing the session", n1.Name()) + } } // ttaErr is like tta but returns errors instead of failing the test. diff --git a/wgengine/kerneldp_linux.go b/wgengine/kerneldp_linux.go index 27de7659f..a7cb227f5 100644 --- a/wgengine/kerneldp_linux.go +++ b/wgengine/kerneldp_linux.go @@ -20,6 +20,7 @@ import ( "github.com/tailscale/netlink" "golang.org/x/sys/unix" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/tstun" "tailscale.com/tsconst" "tailscale.com/types/key" "tailscale.com/types/logger" @@ -52,11 +53,17 @@ type kernelDataplane struct { peerConfigFn *atomic.Pointer[func(key.NodePublic) (wgcfg.PeerConfig, bool)] kick chan struct{} // buffered(1); wakes the reconcile loop - ms *magicsock.Conn // set by start - bridge *net.UDPConn // set by start - ctx context.Context - cancel context.CancelFunc - done chan struct{} // closed when the reconcile loop exits + ms *magicsock.Conn // set by start + bridge *net.UDPConn // set by start + ipSocks *ipSockets // set by start; see kerneldp_tsmp_linux.go + + // priorityMsgFn, if set, returns the message (a TSMP disco key + // advertisement) to send to a peer when the kernel establishes a new + // session with it, as wireguard-go does with its priority messages. + priorityMsgFn atomic.Pointer[func(key.NodePublic) []byte] + ctx context.Context + cancel context.CancelFunc + done chan struct{} // closed when the reconcile loop exits // kernelPort is the kernel device's UDP listen port, which the bridge // sends injected packets to. @@ -73,6 +80,7 @@ type kernelDataplane struct { loopPeer map[netip.Addr]key.NodePublic // inverse of loopAddr nextLoop uint32 // next index for loopAddr counters map[key.NodePublic][2]uint64 // last seen rx, tx bytes + handshakes map[key.NodePublic]time.Time // last seen handshake times status []ipnstate.PeerStatusLite // from the last dump warnedHybrid map[key.NodePublic]bool } @@ -108,13 +116,15 @@ func newKernelDataplane(logf logger.Logf, ifname string, peerConfigFn *atomic.Po loopAddr: map[key.NodePublic]netip.Addr{}, loopPeer: map[netip.Addr]key.NodePublic{}, counters: map[key.NodePublic][2]uint64{}, + handshakes: map[key.NodePublic]time.Time{}, warnedHybrid: map[key.NodePublic]bool{}, } } -// start opens the bridge, shares magicsock's sockets with the kernel, and -// starts the bridge and reconcile loops. -func (k *kernelDataplane) start(ms *magicsock.Conn) error { +// start opens the bridge, shares magicsock's sockets with the kernel, +// connects tun's injected and TSMP packets to the kernel device, and +// starts the bridge, TSMP and reconcile loops. +func (k *kernelDataplane) start(ms *magicsock.Conn, tun *tstun.Wrapper) error { k.ms = ms nl, err := tswg.Open() if err != nil { @@ -164,8 +174,20 @@ func (k *kernelDataplane) start(ms *magicsock.Conn) error { return fmt.Errorf("bringing up %s: %w", k.ifname, err) } + k.ipSocks, err = k.openIPSockets() + if err != nil { + k.bridge.Close() + nl.Close() + return err + } + tun.SetOutboundInjectHook(k.sendIPPacket) + k.ctx, k.cancel = context.WithCancel(context.Background()) go k.readBridge() + go k.readTSMP4(tun) + if k.ipSocks.recv6 != nil { + go k.readTSMP6(tun) + } go k.reconcileLoop() k.logf("kernel data plane on %s; loopback bridge on %v", k.ifname, k.bridge.LocalAddr()) return nil @@ -178,6 +200,7 @@ func (k *kernelDataplane) Close() error { } k.cancel() k.bridge.Close() + k.ipSocks.close() <-k.done k.nlMu.Lock() defer k.nlMu.Unlock() @@ -214,6 +237,16 @@ func (k *kernelDataplane) setSocket(network string, fd uintptr) error { return nil } +// setPriorityMessageFunc sets the function that returns the message to send +// to a peer when the kernel establishes a new session with it. +func (k *kernelDataplane) setPriorityMessageFunc(fn func(key.NodePublic) []byte) { + if fn == nil { + k.priorityMsgFn.Store(nil) + return + } + k.priorityMsgFn.Store(&fn) +} + // setPrivateKey sets the device's private key. func (k *kernelDataplane) setPrivateKey(pk key.NodePrivate) { k.mu.Lock() @@ -222,6 +255,39 @@ func (k *kernelDataplane) setPrivateKey(pk key.NodePrivate) { k.kickReconcile() } +// clearSession makes the kernel forget its session with the peer with key +// pk, so that the next packet to the peer starts a new handshake. It's the +// kernel equivalent of wireguard-go's ScheduleHandshakeOnUserSend, for when +// the peer restarted and lost its keys. +// +// It also updates the peer's kernel endpoint in the same step: the peer's +// path has likely just changed too (a restarted peer usually has a new +// port), and the new handshake should go out on the new path rather than +// waiting for the next reconcile. +func (k *kernelDataplane) clearSession(pk key.NodePublic) { + path, _ := k.ms.KernelPeerPath(pk) + k.mu.Lock() + st, ok := k.pushed[pk] + if !ok { + k.mu.Unlock() + return // not in the kernel; nothing to clear + } + st.endpoint = k.endpointLocked(pk, path.Direct) + k.pushed[pk] = st + k.mu.Unlock() + err := k.configure(tswg.Config{Peers: []tswg.PeerConfig{{ + PublicKey: pk, + UpdateOnly: true, + ClearSession: true, + Endpoint: st.endpoint, + }}}) + if err != nil { + k.logf("clearing session with %v: %v", pk.ShortString(), err) + return + } + k.logf("cleared session with %v", pk.ShortString()) +} + // kickReconcile makes the reconcile loop run soon. func (k *kernelDataplane) kickReconcile() { select { @@ -230,6 +296,16 @@ func (k *kernelDataplane) kickReconcile() { } } +// endpointLocked returns the kernel endpoint for the peer with key pk: +// direct, if valid, else the peer's address on the loopback bridge. k.mu +// must be held. +func (k *kernelDataplane) endpointLocked(pk key.NodePublic, direct netip.AddrPort) netip.AddrPort { + if direct.IsValid() { + return direct + } + return netip.AddrPortFrom(k.loopAddrLocked(pk), uint16(k.bridge.LocalAddr().(*net.UDPAddr).Port)) +} + // loopAddrLocked returns the bridge address of the peer with key pk, // assigning one if needed. k.mu must be held. func (k *kernelDataplane) loopAddrLocked(pk key.NodePublic) netip.Addr { @@ -377,9 +453,7 @@ func (k *kernelDataplane) reconcile() error { var peers []tswg.PeerConfig noRoam := true for _, w := range wants { - if !w.st.endpoint.IsValid() { - w.st.endpoint = netip.AddrPortFrom(k.loopAddrLocked(w.pk), uint16(k.bridge.LocalAddr().(*net.UDPAddr).Port)) - } + w.st.endpoint = k.endpointLocked(w.pk, w.st.endpoint) if w.conf.Hybrid && !k.warnedHybrid[w.pk] { k.warnedHybrid[w.pk] = true k.logf("[unexpected] peer %v wants a hybrid handshake, which tswg doesn't support; using a classic handshake", w.pk.ShortString()) @@ -449,9 +523,14 @@ func (k *kernelDataplane) readCounters() error { tx, rx bool } var acts []activity + var newSessions []key.NodePublic status := make([]ipnstate.PeerStatusLite, 0, len(d.Peers)) k.mu.Lock() for _, p := range d.Peers { + if p.LastHandshake.After(k.handshakes[p.PublicKey]) { + k.handshakes[p.PublicKey] = p.LastHandshake + newSessions = append(newSessions, p.PublicKey) + } status = append(status, ipnstate.PeerStatusLite{ NodeKey: p.PublicKey, RxBytes: int64(p.RxBytes), @@ -472,6 +551,13 @@ func (k *kernelDataplane) readCounters() error { for _, a := range acts { k.ms.NoteKernelActivity(a.pk, a.tx, a.rx) } + if fn := k.priorityMsgFn.Load(); fn != nil { + for _, pk := range newSessions { + if msg := (*fn)(pk); len(msg) > 0 { + k.sendIPPacket(msg) + } + } + } return nil } diff --git a/wgengine/kerneldp_other.go b/wgengine/kerneldp_other.go index 5c7da8585..399024eb7 100644 --- a/wgengine/kerneldp_other.go +++ b/wgengine/kerneldp_other.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/tstun" "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/wgengine/magicsock" @@ -24,12 +25,14 @@ func newKernelDataplane(logger.Logf, string, *atomic.Pointer[func(key.NodePublic return &kernelDataplane{} } -func (*kernelDataplane) start(*magicsock.Conn) error { +func (*kernelDataplane) start(*magicsock.Conn, *tstun.Wrapper) error { return errors.New("tswg kernel data plane not supported on this platform or build") } -func (*kernelDataplane) Close() error { return nil } -func (*kernelDataplane) setPrivateKey(key.NodePrivate) {} -func (*kernelDataplane) kickReconcile() {} -func (*kernelDataplane) injectFromMagicsock(key.NodePublic, []byte) {} -func (*kernelDataplane) peerStatuses() []ipnstate.PeerStatusLite { return nil } +func (*kernelDataplane) Close() error { return nil } +func (*kernelDataplane) setPrivateKey(key.NodePrivate) {} +func (*kernelDataplane) kickReconcile() {} +func (*kernelDataplane) clearSession(key.NodePublic) {} +func (*kernelDataplane) injectFromMagicsock(key.NodePublic, []byte) {} +func (*kernelDataplane) peerStatuses() []ipnstate.PeerStatusLite { return nil } +func (*kernelDataplane) setPriorityMessageFunc(func(key.NodePublic) []byte) {} diff --git a/wgengine/kerneldp_tsmp_linux.go b/wgengine/kerneldp_tsmp_linux.go new file mode 100644 index 000000000..c83c3455b --- /dev/null +++ b/wgengine/kerneldp_tsmp_linux.go @@ -0,0 +1,250 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !ts_omit_tswg + +package wgengine + +import ( + "encoding/binary" + "errors" + "fmt" + "net/netip" + "os" + + "golang.org/x/sys/unix" + "tailscale.com/net/packet" + "tailscale.com/net/tstun" + "tailscale.com/types/ipproto" +) + +// This file has the kernel data plane's IP-level plumbing between the +// kernel's tswg device and tstun: packets that tailscaled itself sends to +// peers (TSMP pings, pongs and disco key advertisements, ICMP echo +// replies, netstack) and the TSMP packets that peers send to it. +// +// With wireguard-go, all of those pass through the tstun.Wrapper between +// the TUN device and wireguard-go. In kernel mode, packets don't pass +// through tailscaled, so: +// +// - tstun's injected outbound packets go to sendIPPacket (via +// tstun.Wrapper.SetOutboundInjectHook), which writes them, IP header +// and all, to a raw IP socket bound to the tswg device, so the kernel +// routes them into the device, which encrypts them to the peer; +// - TSMP packets (IP protocol 99) that arrive on the tswg device are +// read from raw sockets for that protocol, bound to the device, and +// fed to tstun as if they came from wireguard-go, which handles them +// as usual (answering pings, delivering pongs, learning disco keys). +// With such a raw socket open, the kernel doesn't send ICMP protocol +// unreachable errors for TSMP. + +// ipSockets are the kernel data plane's raw IP sockets. +type ipSockets struct { + send4, send6 int // raw IPPROTO_RAW sockets, header included; or -1 + recv4, recv6 *os.File // raw IPPROTO_TSMP sockets; or nil +} + +// rawSocket returns a raw IP socket for family and proto, bound to the +// network device ifname. +func rawSocket(family, proto int, ifname string) (int, error) { + fd, err := unix.Socket(family, unix.SOCK_RAW|unix.SOCK_CLOEXEC, proto) + if err != nil { + return -1, err + } + if err := unix.SetsockoptString(fd, unix.SOL_SOCKET, unix.SO_BINDTODEVICE, ifname); err != nil { + unix.Close(fd) + return -1, err + } + return fd, nil +} + +// openIPSockets opens the raw sockets on k's device. The IPv6 sockets are +// optional, since IPv6 might be disabled. +func (k *kernelDataplane) openIPSockets() (*ipSockets, error) { + s := &ipSockets{send4: -1, send6: -1} + var err error + if s.send4, err = rawSocket(unix.AF_INET, unix.IPPROTO_RAW, k.ifname); err != nil { + return nil, fmt.Errorf("raw IPv4 send socket: %w", err) + } + fd, err := rawSocket(unix.AF_INET, int(ipproto.TSMP), k.ifname) + if err != nil { + s.close() + return nil, fmt.Errorf("raw IPv4 TSMP socket: %w", err) + } + s.recv4, err = pollableFile(fd, "tswg-tsmp4") + if err != nil { + s.close() + return nil, err + } + + if fd, err := rawSocket(unix.AF_INET6, unix.IPPROTO_RAW, k.ifname); err == nil { + s.send6 = fd + } else { + k.logf("no raw IPv6 send socket: %v", err) + } + if fd, err := rawSocket(unix.AF_INET6, int(ipproto.TSMP), k.ifname); err == nil { + // IPv6 raw sockets don't return the IP header, so ask for the + // destination address and hop limit to rebuild it. + err := errors.Join( + unix.SetsockoptInt(fd, unix.IPPROTO_IPV6, unix.IPV6_RECVPKTINFO, 1), + unix.SetsockoptInt(fd, unix.IPPROTO_IPV6, unix.IPV6_RECVHOPLIMIT, 1)) + if err == nil { + s.recv6, err = pollableFile(fd, "tswg-tsmp6") + } else { + unix.Close(fd) + } + if err != nil { + k.logf("no raw IPv6 TSMP socket: %v", err) + } + } else { + k.logf("no raw IPv6 TSMP socket: %v", err) + } + return s, nil +} + +// pollableFile wraps fd in an *os.File that uses the Go netpoller, so that +// reads block in the poller (and closing the file unblocks them). +func pollableFile(fd int, name string) (*os.File, error) { + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return nil, err + } + return os.NewFile(uintptr(fd), name), nil +} + +func (s *ipSockets) close() { + for _, fd := range []int{s.send4, s.send6} { + if fd >= 0 { + unix.Close(fd) + } + } + for _, f := range []*os.File{s.recv4, s.recv6} { + if f != nil { + f.Close() + } + } +} + +// sendIPPacket is the tstun outbound inject hook: it sends pkt, a complete +// IPv4 or IPv6 packet that tailscaled generated for a peer, into the +// kernel's tswg device. +func (k *kernelDataplane) sendIPPacket(pkt []byte) { + var p packet.Parsed + p.Decode(pkt) + dst := p.Dst.Addr() + var err error + switch { + case p.IPVersion == 4 && k.ipSocks.send4 >= 0: + err = unix.Sendto(k.ipSocks.send4, pkt, 0, &unix.SockaddrInet4{Addr: dst.As4()}) + case p.IPVersion == 6 && k.ipSocks.send6 >= 0: + err = unix.Sendto(k.ipSocks.send6, pkt, 0, &unix.SockaddrInet6{Addr: dst.As16()}) + default: + err = fmt.Errorf("can't send IPv%d packet", p.IPVersion) + } + if err != nil { + k.logf("sending %d-byte %v packet to %v: %v", len(pkt), p.IPProto, dst, err) + } +} + +// readTSMP4 reads the TSMP packets that arrive on the device over IPv4 and +// passes them to tstun. IPv4 raw sockets return the whole IP packet. +func (k *kernelDataplane) readTSMP4(tun *tstun.Wrapper) { + buf := make([]byte, tstun.WritePacketStartOffset+tstun.MaxPacketSize) + for { + n, err := k.ipSocks.recv4.Read(buf[tstun.WritePacketStartOffset:]) + if err != nil { + if k.ctx.Err() == nil { + k.logf("reading IPv4 TSMP: %v", err) + } + return + } + k.deliverTSMP(tun, buf[:tstun.WritePacketStartOffset+n]) + } +} + +// readTSMP6 is like readTSMP4 for IPv6, whose raw sockets return only the +// payload, so it rebuilds the IPv6 header from the source address and the +// destination address and hop limit control messages. +func (k *kernelDataplane) readTSMP6(tun *tstun.Wrapper) { + const hdrLen = 40 + buf := make([]byte, tstun.WritePacketStartOffset+hdrLen+tstun.MaxPacketSize) + oob := make([]byte, unix.CmsgSpace(unix.SizeofInet6Pktinfo)+unix.CmsgSpace(4)) + rc, err := k.ipSocks.recv6.SyscallConn() + if err != nil { + k.logf("reading IPv6 TSMP: %v", err) + return + } + payload := buf[tstun.WritePacketStartOffset+hdrLen:] + for { + var n, oobn int + var from unix.Sockaddr + var rerr error + err := rc.Read(func(fd uintptr) bool { + n, oobn, _, from, rerr = unix.Recvmsg(int(fd), payload, oob, 0) + return rerr != unix.EAGAIN + }) + if err == nil { + err = rerr + } + if err != nil { + if k.ctx.Err() == nil { + k.logf("reading IPv6 TSMP: %v", err) + } + return + } + src, ok := from.(*unix.SockaddrInet6) + if !ok { + continue + } + dst, hopLimit, ok := parseIPv6RecvCmsgs(oob[:oobn]) + if !ok { + continue + } + h := buf[tstun.WritePacketStartOffset:][:hdrLen] + clear(h) + h[0] = 6 << 4 + binary.BigEndian.PutUint16(h[4:6], uint16(n)) + h[6] = byte(ipproto.TSMP) + h[7] = hopLimit + copy(h[8:24], src.Addr[:]) + dst16 := dst.As16() + copy(h[24:40], dst16[:]) + k.deliverTSMP(tun, buf[:tstun.WritePacketStartOffset+hdrLen+n]) + } +} + +// parseIPv6RecvCmsgs returns the destination address (IPV6_PKTINFO) and hop +// limit (IPV6_HOPLIMIT) from an IPv6 raw socket's control messages. +func parseIPv6RecvCmsgs(oob []byte) (dst netip.Addr, hopLimit uint8, ok bool) { + msgs, err := unix.ParseSocketControlMessage(oob) + if err != nil { + return netip.Addr{}, 0, false + } + hopLimit = 64 + for _, m := range msgs { + if m.Header.Level != unix.IPPROTO_IPV6 { + continue + } + switch m.Header.Type { + case unix.IPV6_PKTINFO: + if len(m.Data) >= unix.SizeofInet6Pktinfo { + // struct in6_pktinfo { in6_addr addr; int ifindex; } + dst = netip.AddrFrom16([16]byte(m.Data[:16])) + ok = true + } + case unix.IPV6_HOPLIMIT: + if len(m.Data) >= 4 { + hopLimit = uint8(binary.NativeEndian.Uint32(m.Data)) + } + } + } + return dst, hopLimit, ok +} + +// deliverTSMP passes buf, a TSMP packet from a peer starting at offset +// tstun.WritePacketStartOffset, to tstun, as wireguard-go would. +func (k *kernelDataplane) deliverTSMP(tun *tstun.Wrapper, buf []byte) { + if _, err := tun.Write([][]byte{buf}, tstun.WritePacketStartOffset); err != nil { + k.logf("delivering TSMP packet: %v", err) + } +} diff --git a/wgengine/magicsock/kernel.go b/wgengine/magicsock/kernel.go index 3f6733232..08bc937de 100644 --- a/wgengine/magicsock/kernel.go +++ b/wgengine/magicsock/kernel.go @@ -102,20 +102,36 @@ func (c *Conn) KernelPeerPaths() []KernelPeerPath { defer c.mu.Unlock() paths := make([]KernelPeerPath, 0, c.peerMap.nodeCount()) c.peerMap.forEachEndpoint(func(de *endpoint) { - p := KernelPeerPath{Key: de.publicKey} - de.mu.Lock() - udpAddr, derpAddr, _ := de.addrForSendLocked(now) - de.mu.Unlock() - // A DERP address alongside the UDP one means the UDP path isn't - // trusted (yet), so magicsock sends to both. - if udpAddr.isDirect() && !derpAddr.IsValid() { - p.Direct = udpAddr.ap - } - paths = append(paths, p) + paths = append(paths, de.kernelPath(now)) }) return paths } +// KernelPeerPath is like KernelPeerPaths, for the single peer with key k. +func (c *Conn) KernelPeerPath(k key.NodePublic) (_ KernelPeerPath, ok bool) { + c.mu.Lock() + de, ok := c.peerMap.endpointForNodeKey(k) + c.mu.Unlock() + if !ok { + return KernelPeerPath{}, false + } + return de.kernelPath(mono.Now()), true +} + +// kernelPath returns the path the tswg kernel module should use for de. +func (de *endpoint) kernelPath(now mono.Time) KernelPeerPath { + p := KernelPeerPath{Key: de.publicKey} + de.mu.Lock() + udpAddr, derpAddr, _ := de.addrForSendLocked(now) + de.mu.Unlock() + // A DERP address alongside the UDP one means the UDP path isn't + // trusted (yet), so magicsock sends to both. + if udpAddr.isDirect() && !derpAddr.IsValid() { + p.Direct = udpAddr.ap + } + return p +} + // NoteKernelActivity tells magicsock that the tswg kernel module sent (tx) // or received (rx) WireGuard packets for the peer with key k on its own, // over a direct path, without going through magicsock. magicsock uses diff --git a/wgengine/tswg/tswg.go b/wgengine/tswg/tswg.go index aa9b63a19..3ed7c1aca 100644 --- a/wgengine/tswg/tswg.go +++ b/wgengine/tswg/tswg.go @@ -63,6 +63,13 @@ type PeerConfig struct { // arriving via DERP would steal the endpoint from a good direct path. // The flag is applied before Endpoint, so setting both is one step. NoRoam *bool + + // ClearSession, if true, makes the kernel forget the peer's session + // keys and any handshake in progress, so that the next packet to the + // peer starts a new handshake right away. tailscaled sets it when a + // peer's disco key changes in a way that means it restarted (and so + // lost its keys). + ClearSession bool } // Device is the current state of a tswg device. diff --git a/wgengine/tswg/tswg_linux.go b/wgengine/tswg/tswg_linux.go index eb40a8162..2a4db6bdf 100644 --- a/wgengine/tswg/tswg_linux.go +++ b/wgengine/tswg/tswg_linux.go @@ -71,7 +71,8 @@ const ( // tswg additions, nested in peerATSWG. peerATSWG = 64 - tswgPeerAFlags = 1 + tswgPeerAFlags = 1 + tswgPeerAClearSession = 2 tswgPeerFlagNoRoam = 1 ) @@ -175,13 +176,18 @@ func encodePeer(ae *netlink.AttributeEncoder, p PeerConfig) error { if p.PresharedKey != nil { ae.Bytes(peerAPresharedKey, p.PresharedKey[:]) } - if p.NoRoam != nil { + if p.NoRoam != nil || p.ClearSession { ae.Nested(peerATSWG, func(nae *netlink.AttributeEncoder) error { - var flags uint32 - if *p.NoRoam { - flags |= tswgPeerFlagNoRoam + if p.NoRoam != nil { + var flags uint32 + if *p.NoRoam { + flags |= tswgPeerFlagNoRoam + } + nae.Uint32(tswgPeerAFlags, flags) + } + if p.ClearSession { + nae.Flag(tswgPeerAClearSession, true) } - nae.Uint32(tswgPeerAFlags, flags) return nil }) } diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 69e405f2c..1e2c0c12d 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -466,7 +466,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) } closePool.add(e.magicConn) if e.kdp != nil { - if err := e.kdp.start(e.magicConn); err != nil { + if err := e.kdp.start(e.magicConn, e.tundev); err != nil { return nil, fmt.Errorf("wgengine: starting kernel data plane: %w", err) } closePool.add(e.kdp) @@ -711,6 +711,13 @@ func (e *userspaceEngine) SetPeerConfigFunc(fn func(key.NodePublic) (config wgcf panic("SetPeerConfigFunc: nil fn") } e.peerConfigFn.Store(&fn) + if e.kdp != nil { + // The kernel data plane reads fn itself. wireguard-go must never + // create peers in this mode, or it would handshake over magicsock + // with our keys and fight the kernel's sessions. + e.kdp.kickReconcile() + return + } e.wgdev.SetPeerLookupFunc(wgcfg.NewPeerLookupFunc(e.wgdev.Bind(), e.logf, func(pubk device.NoisePublicKey) (wgcfg.PeerConfig, bool) { return fn(key.NodePublicFromRaw32(mem.B(pubk[:]))) })) @@ -745,6 +752,9 @@ func (e *userspaceEngine) SyncDevicePeer(k key.NodePublic) { // MarkDevicePeerForHandshake implements [Engine.MarkDevicePeerForHandshake]. func (e *userspaceEngine) MarkDevicePeerForHandshake(k key.NodePublic) { + if e.kdp != nil { + e.kdp.clearSession(k) + } e.wgLock.Lock() defer e.wgLock.Unlock() e.wgLogger.Invalidate() @@ -764,6 +774,11 @@ func (e *userspaceEngine) MarkDevicePeerForHandshake(k key.NodePublic) { // in the device. Callers without a LocalBackend that need outbound packets // to lazily create peers must install their own callback. func (e *userspaceEngine) SetPeerByIPPacketFunc(fn func(netip.Addr) (_ key.NodePublic, ok bool)) { + if e.kdp != nil { + // The kernel routes by its own allowed IPs, and wireguard-go + // must not create peers; see SetPeerConfigFunc. + return + } if fn == nil { e.wgdev.SetPeerByIPPacketFunc(nil) return @@ -794,6 +809,12 @@ func (e *userspaceEngine) SetPeerSessionStateFunc(fn func(key.NodePublic, PeerWi // [github.com/tailscale/wireguard-go/device.Device]. The returned message must // not exceed [github.com/tailscale/wireguard-go/device.MaxPriorityMessageContentSize]. func (e *userspaceEngine) SetPeerPriorityMessageOnEstablishmentFunc(fn func(key.NodePublic) (msg []byte)) { + if e.kdp != nil { + // The kernel data plane sends these itself when the kernel + // establishes a session. + e.kdp.setPriorityMessageFunc(fn) + return + } if fn != nil { e.wgdev.SetPriorityMessageOnEstablishmentFunc(func(pk device.NoisePublicKey) (msg []byte) { return fn(key.NodePublicFromRaw32(mem.B(pk[:])))