tstun,wgengine: open multiple TUN and UDP queues (#22188)

Open up to 16 parallel I/O queues, further capped at GOMAXPROCS.

TS_DEBUG_MAX_QUEUES can be used to override the above. 

Following was captured on a c8gn.24xlarge server peering with 16
c8gn.xlarge clients. Ubuntu 24.04, kernel v6.14, loaded with a single
iperf3 thread.

   TX
       peers  build         Gb/s  vs stock   cores  cores/Gbps
           1  stock         4.48        --    2.35       0.524
              branch        4.48       -0%    2.38       0.531
           4  stock        11.62        --    4.95       0.426
              branch       18.61      +60%    7.93       0.426
          16  stock        11.51        --    5.18       0.450
              branch       22.97     +100%   14.05       0.612

   RX
           1  stock         4.53        --    2.74       0.605
              branch        4.53       -0%    2.70       0.596
           4  stock         9.06        --    4.65       0.513
              branch       18.47     +104%    9.72       0.526
          16  stock         8.63        --    4.81       0.558
              branch       42.87     +397%   28.10       0.655


Updates tailscale/corp#48168
Updates tailscale/corp#37878

Change-Id: Id3db86038a5f08eb8e84a42c6622463ba50e3f3c

Signed-off-by: Alex Valiushko <[email protected]>
This commit is contained in:
Alex Valiushko authored and GitHub committed 2026-11-12 09:08:37 -07:00
1 parent ddb3fd6b43
commit 2e6fcb4f60
3 files changed
+30 -1

No files matched your search

+19
View File
@@ -705,3 +705,22 @@ func IPCVersion() string {
}
return version.Long()
}
// MaxQueues limits the amount of tun and magicsock queues to open. The value
// is capped by GOMAXPROCS and can be overridden by TS_DEBUG_MAX_QUEUES.
//
// Each extra queue means allocating a goroutine and related buffers, and brings
// diminishing returns in performance (which also only apply to nodes speaking
// with multiple concurrent peers), so we stop scaling at a sane default max:
// 16.
//
// TODO(illotum): 16 is conservative. On a subnet router peering with 256
// concurrent clients TX grows ~6x with 16 queues, and ~10x with 128. RX is
// capped by someting else and doesn't scale with more queues at all. This value
// may and should be reevaluated after other performance improvements.
func MaxQueues() int {
if n, ok := LookupInt("TS_DEBUG_MAX_QUEUES"); ok {
return max(1, n)
}
return min(16, runtime.GOMAXPROCS(0))
}
+10 -1
View File
@@ -17,6 +17,7 @@ import (
"time"
"github.com/tailscale/wireguard-go/tun"
"tailscale.com/envknob"
"tailscale.com/feature"
"tailscale.com/feature/buildfeatures"
"tailscale.com/types/logger"
@@ -58,12 +59,20 @@ func New(logf logger.Logf, tunName string) (tun.Device, string, error) {
if runtime.GOOS == "plan9" {
cleanUpPlan9Interfaces()
}
maxQueues := envknob.MaxQueues()
// Try to create the TUN device up to two times. If it fails
// the first time and we're on Linux, try a desperate
// "modprobe tun" to load the tun module and try again.
for try := range 2 {
dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU()))
dev, err = tun.CreateTUN(
tunName,
int(DefaultTUNMTU()),
tun.WithQueues(maxQueues),
)
if err == nil || !modprobeTunHook.IsSet() {
if err == nil {
logf("opened %d of %d TUN queues", len(tun.QueuesOf(dev)), maxQueues)
}
if try > 0 {
logf("created TUN device %q after doing `modprobe tun`", tunName)
}
+1
View File
@@ -430,6 +430,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
EventBus: e.eventBus,
Logf: logf,
Port: conf.ListenPort,
UDPSockets: envknob.MaxQueues(), // in sync with tun queues
EndpointsFunc: endpointsFn,
DERPActiveFunc: e.RequestStatus,
IdleFunc: e.tundev.IdleDuration,