Files
tailscale/net/msgx/msgx_test.go
Brad Fitzpatrick 0cd86f6b26 net/msgx, net/batching, wgengine/magicsock: batch UDP syscalls on macOS
Add net/msgx, which sends and receives multiple datagrams per system call
on macOS using the kernel's recvmsg_x and sendmsg_x entry points, the
Darwin counterparts of recvmmsg and sendmmsg. They are declared only in
xnu's private socket_private.h, so msgx invokes them as raw system calls
by number rather than through libSystem. That keeps their names out of
the binary, returns errno reliably from the trap itself, and avoids a cgo
transition per call. The calls need no privileges, but msgx.Available
limits their use: never in a sandboxed App Store process (Tailscale's own
GUI builds, detected by bundle ID, or a third-party app embedding tsnet,
detected by the App Sandbox's APP_SANDBOX_CONTAINER_ID environment
variable), and only after a loopback self-test with the running kernel
passes, since Apple does not promise these numbers or layouts stay
stable. Otherwise callers fall back to one datagram per call. Whether the
self-test ran, whether it passed, and which step failed are exported as
client metrics so a kernel change in the wild is visible.

On top of it, add a Darwin batching.Conn: reads are batched on every
socket (the main magicsock sockets included); writes are batched only on
connected sockets, since sendmsg_x takes no per-message destination,
which is what the per-peer connected sockets are for. connBind.BatchSize
now comes from batching.MaxBatchSize rather than a GOOS switch.

Measured between two M5 Pro Mac minis on a 10 Gbps LAN (iperf3 over
Tailscale, 10 second runs, tailscaled CPU seconds sender/receiver), with
per-packet utun I/O still in place:

  baseline                         1.37 Gbps   29.3s / 22.1s
  connected sockets only           1.41 Gbps   24.8s / 15.6s
  connected sockets + msgx         1.44 Gbps   25.3s /  9.4s

Throughput is bounded by the utun path (addressed separately); receiver
CPU drops by more than half here.

Three debug knobs (TS_DEBUG_MSGX, TS_DEBUG_MSGX_RX_MAIN,
TS_DEBUG_MSGX_BATCHSIZE) remain for evaluation.

Signed-off-by: Brad Fitzpatrick <[email protected]>
Change-Id: I4e9b7d2c8a1f5063b9e2d7c4a8f1e6b3d5c2a9f7
2026-09-25 19:12:47 +00:00

87 lines
2.2 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build darwin && !ios
package msgx
import (
"net"
"testing"
"time"
)
func TestAvailable(t *testing.T) {
if !Available() {
t.Skipf("msgx unavailable: %v", UnavailableReason())
}
}
// TestRoundTrip sends a burst of datagrams over loopback with sendmsg_x and
// reads them back with recvmsg_x in batches, checking counts, contents, and
// source addresses. It exercises the same path as the self-test but with more
// datagrams than fit one call.
func TestRoundTrip(t *testing.T) {
if !Available() {
t.Skipf("msgx unavailable: %v", UnavailableReason())
}
recvConn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
if err != nil {
t.Fatal(err)
}
defer recvConn.Close()
sendConn, err := net.DialUDP("udp4", nil, recvConn.LocalAddr().(*net.UDPAddr))
if err != nil {
t.Fatal(err)
}
defer sendConn.Close()
sendRC, _ := sendConn.SyscallConn()
recvRC, _ := recvConn.SyscallConn()
const count = 300 // more than MaxBatch and more than the kernel's 256 cap
payloads := make([][]byte, count)
for i := range payloads {
payloads[i] = []byte{byte(i), byte(i >> 8), 'x'}
}
for rem := payloads; len(rem) > 0; {
n, err := Send(sendRC, rem)
if err != nil {
t.Fatalf("Send: %v", err)
}
if n == 0 {
t.Fatal("Send accepted nothing")
}
rem = rem[n:]
}
recvConn.SetReadDeadline(time.Now().Add(3 * time.Second))
from := sendConn.LocalAddr().(*net.UDPAddr).AddrPort()
msgs := make([]Message, MaxBatch)
for i := range msgs {
msgs[i].Payload = make([]byte, 16)
}
got := 0
batches := 0
for got < count {
n, err := Recv(recvRC, msgs)
if err != nil {
t.Fatalf("Recv after %d datagrams: %v", got, err)
}
batches++
for _, m := range msgs[:n] {
want := payloads[got]
if m.N != len(want) || string(m.Payload[:m.N]) != string(want) {
t.Fatalf("datagram %d: got %q, want %q", got, m.Payload[:m.N], want)
}
if m.Addr != from {
t.Fatalf("datagram %d: source %v, want %v", got, m.Addr, from)
}
got++
}
}
if batches >= count {
t.Errorf("no batching: %d datagrams took %d Recv calls", count, batches)
}
t.Logf("%d datagrams in %d Recv calls", count, batches)
}