net/netmon,net/netns: cache Windows default interface indices

Windows socket setup enumerates adapters and the routing table through
GetWindowsDefault for every netns socket. These calls can be expensive,
particularly on systems where security software adds syscall latency.

Cache nonzero default interface indices per monitor and address family,
and use them from netns controlC. Invalidate on native route, address,
and IP interface change notifications.

Coalesce concurrent lookups and retry results invalidated during lookup.
Bound lookup and wait attempts; once the budget is exhausted, fall back
to a single uncached lookup rather than failing socket setup, erring
toward a possibly stale interface index over no socket at all. Perform
fresh lookups for zero indices and errors, and retain the existing
unbound-socket fallback for a fresh zero result.

Use uncached lookups after monitor shutdown or when interface-change
notification registration fails.

Test cache reuse, independent address families, notification invalidation,
concurrent fills, invalidation during lookup, route restoration after a
zero result, churn and wait-budget fallbacks, and shutdown.

Fixes #21885

Signed-off-by: James Tucker <[email protected]>
This commit is contained in:
James Tucker authored and James Tucker committed 2026-10-23 17:50:52 -04:00
1 parent dd50731b7b
commit ae4ab7e7d5
6 files changed
+558 -37

No files matched your search

+99
View File
@@ -0,0 +1,99 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package netmon
import (
"sync"
)
// defaultInterfaceCacheAttempts is the number of lookup and wait attempts a
// single get may make, including waits for another caller's lookup, before
// falling back to one uncached lookup. Bounding them keeps network churn from
// keeping socket setup in this cache forever.
const defaultInterfaceCacheAttempts = 2
// defaultInterfaceCache caches a nonzero default interface index until an OS change
// notification invalidates it. Windows uses one cache per address family.
// It is independent of Windows APIs so its concurrency can be tested on any OS.
// The zero value is ready for use.
type defaultInterfaceCache struct {
mu sync.Mutex // guards following
inFlight chan struct{} // closed when the current lookup completes, or nil
generation uint64
index uint32
valid bool
disabled bool // notifications are no longer being delivered
}
func (c *defaultInterfaceCache) invalidate() {
c.mu.Lock()
defer c.mu.Unlock()
c.generation++
c.valid = false
}
// disable stops caching when the notification subscription is closed.
func (c *defaultInterfaceCache) disable() {
c.mu.Lock()
defer c.mu.Unlock()
c.disabled = true
c.generation++
c.valid = false
}
func (c *defaultInterfaceCache) get(lookup func() (uint32, error)) (uint32, error) {
for attempt := 0; ; attempt++ {
c.mu.Lock()
if c.disabled {
c.mu.Unlock()
return lookup()
}
if c.valid {
index := c.index
c.mu.Unlock()
return index, nil
}
if attempt == defaultInterfaceCacheAttempts {
c.mu.Unlock()
// Churn prevented a stable result. Do one uncached lookup rather
// than fail: a possibly stale interface index beats no socket.
return lookup()
}
if done := c.inFlight; done != nil {
c.mu.Unlock()
<-done
continue
}
// Only one caller performs the expensive lookup. Invalidation must
// not wait for it, since the OS calls can be slow on some systems.
done := make(chan struct{})
c.inFlight = done
generation := c.generation
c.mu.Unlock()
index, err := lookup()
c.mu.Lock()
c.inFlight = nil
close(done)
if c.disabled {
c.mu.Unlock()
return index, err
}
if generation != c.generation {
// The network changed during the lookup. Don't publish the old
// result, even if the lookup succeeded.
c.mu.Unlock()
continue
}
if err == nil && index != 0 {
// Never cache "no default interface". Zero leaves a socket
// unconstrained, possibly routing it back into Tailscale. Each
// caller must check the OS again rather than reuse that absence
// while a route-restoration notification is still pending.
c.index, c.valid = index, true
}
c.mu.Unlock()
return index, err
}
}
+275
View File
@@ -0,0 +1,275 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package netmon
import (
"errors"
"sync/atomic"
"testing"
"testing/synctest"
)
func BenchmarkDefaultInterfaceCache(b *testing.B) {
var c defaultInterfaceCache
lookup := func() (uint32, error) { return 42, nil }
c.get(lookup)
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
c.get(lookup)
}
}
func TestDefaultInterfaceCache(t *testing.T) {
var c defaultInterfaceCache
var calls int
lookup := func() (uint32, error) {
calls++
return 42, nil
}
for range 3 {
if got, err := c.get(lookup); got != 42 || err != nil {
t.Fatalf("get = %v, %v; want 42, nil", got, err)
}
}
if calls != 1 {
t.Fatalf("lookups = %v; want 1", calls)
}
c.invalidate()
if got, err := c.get(lookup); got != 42 || err != nil {
t.Fatalf("get after invalidation = %v, %v; want 42, nil", got, err)
}
if calls != 2 {
t.Fatalf("lookups after invalidation = %v; want 2", calls)
}
}
func TestDefaultInterfaceCacheNoDefault(t *testing.T) {
var c defaultInterfaceCache
var calls int
index := uint32(0)
lookup := func() (uint32, error) {
calls++
return index, nil
}
for range 2 {
if got, err := c.get(lookup); got != 0 || err != nil {
t.Fatalf("get = %v, %v; want 0, nil", got, err)
}
}
if calls != 2 {
t.Fatalf("lookups with no default = %v; want 2", calls)
}
// The route returns before its notification is delivered. Without an
// explicit invalidation, the very next caller must see the restored route.
index = 42
for range 2 {
if got, err := c.get(lookup); got != 42 || err != nil {
t.Fatalf("get after route restoration = %v, %v; want 42, nil", got, err)
}
}
if calls != 3 {
t.Fatalf("lookups after route restoration = %v; want 3", calls)
}
}
func TestDefaultInterfaceCacheChurn(t *testing.T) {
var c defaultInterfaceCache
var calls int
// Every lookup races a network change, so no result is ever stable.
got, err := c.get(func() (uint32, error) {
calls++
c.invalidate()
return uint32(40 + calls), nil
})
if got != 43 || err != nil {
t.Fatalf("get during churn = %v, %v; want 43, nil", got, err)
}
if calls != defaultInterfaceCacheAttempts+1 {
t.Fatalf("lookups = %v; want %v", calls, defaultInterfaceCacheAttempts+1)
}
// The unstable result must not be published.
if got, err := c.get(func() (uint32, error) { return 44, nil }); got != 44 || err != nil {
t.Fatalf("get after churn = %v, %v; want 44, nil", got, err)
}
}
func TestDefaultInterfaceCacheWaitBudget(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var c defaultInterfaceCache
c.inFlight = make(chan struct{})
var calls atomic.Int32
done := make(chan struct{})
go func() {
defer close(done)
// Waiting for other callers' fills consumes the wait budget too.
// After exhausting it, get must do its own uncached lookup rather
// than fail without ever looking anything up.
got, err := c.get(func() (uint32, error) {
return uint32(41 + calls.Add(1)), nil
})
if got != 42 || err != nil {
t.Errorf("get = %v, %v; want 42, nil", got, err)
}
}()
for attempt := range defaultInterfaceCacheAttempts {
synctest.Wait()
// Simulate other callers taking successive fills.
c.mu.Lock()
previous := c.inFlight
c.inFlight = nil
if attempt+1 < defaultInterfaceCacheAttempts {
c.inFlight = make(chan struct{})
}
close(previous)
c.mu.Unlock()
}
<-done
if got := calls.Load(); got != 1 {
t.Fatalf("lookups = %v; want 1", got)
}
})
}
func TestDefaultInterfaceCacheError(t *testing.T) {
var c defaultInterfaceCache
wantErr := errors.New("lookup failed")
calls := 0
lookup := func() (uint32, error) {
calls++
if calls == 1 {
return 0, wantErr
}
return 42, nil
}
if _, err := c.get(lookup); err != wantErr {
t.Fatalf("get error = %v; want %v", err, wantErr)
}
for range 2 {
if got, err := c.get(lookup); got != 42 || err != nil {
t.Fatalf("get = %v, %v; want 42, nil", got, err)
}
}
if calls != 2 {
t.Fatalf("lookups = %v; want 2", calls)
}
}
func TestDefaultInterfaceCacheConcurrent(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var c defaultInterfaceCache
var calls atomic.Int32
release := make(chan struct{})
lookup := func() (uint32, error) {
calls.Add(1)
<-release
return 42, nil
}
const n = 20
done := make(chan struct{}, n)
for range n {
go func() {
if got, err := c.get(lookup); got != 42 || err != nil {
t.Errorf("get = %v, %v; want 42, nil", got, err)
}
done <- struct{}{}
}()
}
synctest.Wait()
if got := calls.Load(); got != 1 {
t.Fatalf("concurrent lookups = %v; want 1", got)
}
close(release)
for range n {
<-done
}
if got := calls.Load(); got != 1 {
t.Fatalf("total lookups = %v; want 1", got)
}
})
}
func TestDefaultInterfaceCacheInvalidateDuringLookup(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var c defaultInterfaceCache
var calls atomic.Int32
release := make(chan struct{})
lookup := func() (uint32, error) {
if calls.Add(1) == 1 {
<-release
return 1, nil // obsolete result
}
return 2, nil
}
done := make(chan struct{})
go func() {
defer close(done)
if got, err := c.get(lookup); got != 2 || err != nil {
t.Errorf("get = %v, %v; want 2, nil", got, err)
}
}()
synctest.Wait()
// Must not block behind the expensive lookup.
c.invalidate()
close(release)
<-done
if got, err := c.get(lookup); got != 2 || err != nil {
t.Fatalf("cached get = %v, %v; want 2, nil", got, err)
}
if got := calls.Load(); got != 2 {
t.Fatalf("lookups = %v; want 2", got)
}
})
}
func TestDefaultInterfaceCacheDisableDuringLookup(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var c defaultInterfaceCache
var calls atomic.Int32
release := make(chan struct{})
lookup := func() (uint32, error) {
n := calls.Add(1)
if n == 1 {
<-release
}
return uint32(n), nil
}
done := make(chan struct{})
go func() {
defer close(done)
if got, err := c.get(lookup); got != 1 || err != nil {
t.Errorf("get = %v, %v; want 1, nil", got, err)
}
}()
synctest.Wait()
c.disable()
close(release)
<-done
// Even a later notification must not re-enable the cache.
c.invalidate()
for _, want := range []uint32{2, 3} {
if got, err := c.get(lookup); got != want || err != nil {
t.Fatalf("disabled get = %v, %v; want %v, nil", got, err, want)
}
}
})
}
func TestDefaultInterfaceCacheDisablePopulated(t *testing.T) {
var c defaultInterfaceCache
var calls uint32
lookup := func() (uint32, error) {
calls++
return calls, nil
}
if got, err := c.get(lookup); got != 1 || err != nil {
t.Fatalf("get = %v, %v; want 1, nil", got, err)
}
c.disable()
for _, want := range []uint32{2, 3} {
if got, err := c.get(lookup); got != want || err != nil {
t.Fatalf("disabled get = %v, %v; want %v, nil", got, err, want)
}
}
}
+88 -10
View File
@@ -10,6 +10,7 @@ import (
"sync"
"time"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
"tailscale.com/net/tsaddr"
"tailscale.com/types/logger"
@@ -27,13 +28,22 @@ type eventMessage struct {
func (eventMessage) ignore() bool { return false }
type winMon struct {
logf logger.Logf
ctx context.Context
cancel context.CancelFunc
isActive func() bool
messagec chan eventMessage
addressChangeCallback *winipcfg.UnicastAddressChangeCallback
routeChangeCallback *winipcfg.RouteChangeCallback
logf logger.Logf
ctx context.Context
cancel context.CancelFunc
isActive func() bool
messagec chan eventMessage
addressChangeCallback *winipcfg.UnicastAddressChangeCallback
routeChangeCallback *winipcfg.RouteChangeCallback
interfaceChangeCallback *winipcfg.InterfaceChangeCallback
// defaultInterface4 and defaultInterface6 cache the default interface
// index for each address family. They are invalidated directly by OS
// change notifications, rather than by the monitor's debounced State,
// which only records the IPv4 default route and no interface or route
// metrics.
defaultInterface4 defaultInterfaceCache
defaultInterface6 defaultInterfaceCache
mu sync.Mutex
lastLog time.Time // time we last logged about any windows change event
@@ -59,22 +69,79 @@ func newOSMon(_ *eventbus.Bus, logf logger.Logf, pm *Monitor) (osMon, error) {
m.addressChangeCallback, err = winipcfg.RegisterUnicastAddressChangeCallback(m.unicastAddressChanged)
if err != nil {
m.logf("winipcfg.RegisterUnicastAddressChangeCallback error: %v", err)
m.cancel()
m.Close()
return nil, err
}
m.routeChangeCallback, err = winipcfg.RegisterRouteChangeCallback(m.routeChanged)
if err != nil {
m.addressChangeCallback.Unregister()
m.logf("winipcfg.RegisterRouteChangeCallback error: %v", err)
m.cancel()
m.Close()
return nil, err
}
m.interfaceChangeCallback, err = winipcfg.RegisterInterfaceChangeCallback(m.interfaceChanged)
if err != nil {
m.logf("winipcfg.RegisterInterfaceChangeCallback error: %v; default interface caching disabled", err)
// The cache cannot be invalidated correctly without interface change
// notifications, so disable it. Route and address monitoring is
// unaffected.
m.defaultInterface4.disable()
m.defaultInterface6.disable()
}
return m, nil
}
// WindowsDefaultInterfaceIndex returns the index of the non-Tailscale default
// interface for family, which must be AF_INET or AF_INET6. Zero means no default
// interface was found. Nonzero results are cached until an OS network change
// notification; zero results and errors are not cached. Static and closed
// monitors, and monitors without the required notification subscriptions,
// perform uncached lookups, as do lookups that cannot stabilize under repeated
// network changes.
func (m *Monitor) WindowsDefaultInterfaceIndex(family winipcfg.AddressFamily) (uint32, error) {
var cache *defaultInterfaceCache
wm, _ := m.om.(*winMon)
switch family {
case windows.AF_INET:
if wm != nil {
cache = &wm.defaultInterface4
}
case windows.AF_INET6:
if wm != nil {
cache = &wm.defaultInterface6
}
default:
return 0, errors.New("invalid address family for Windows default interface")
}
lookup := func() (uint32, error) {
iface, err := GetWindowsDefault(family)
if err != nil || iface == nil {
return 0, err
}
return iface.IfIndex, nil
}
if cache == nil {
return lookup()
}
return cache.get(lookup)
}
func (m *winMon) invalidateDefaultInterfaces() {
m.defaultInterface4.invalidate()
m.defaultInterface6.invalidate()
}
// interfaceChanged covers changes to interface metrics, enabled address families,
// and connectivity, which can change the best default without changing a route.
func (m *winMon) interfaceChanged(_ winipcfg.MibNotificationType, _ *winipcfg.MibIPInterfaceRow) {
m.invalidateDefaultInterfaces()
}
func (m *winMon) Close() (ret error) {
m.defaultInterface4.disable()
m.defaultInterface6.disable()
m.cancel()
m.noDeadlockTicker.Stop()
@@ -96,6 +163,15 @@ func (m *winMon) Close() (ret error) {
}
}
if m.interfaceChangeCallback != nil {
if err := m.interfaceChangeCallback.Unregister(); err != nil {
m.logf("interfaceChangeCallback.Unregister error: %v", err)
ret = err
} else {
m.interfaceChangeCallback = nil
}
}
return
}
@@ -131,6 +207,7 @@ func (m *winMon) Receive() (message, error) {
// unicastAddressChanged is the callback we register with Windows to call when unicast address changes.
func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibUnicastIPAddressRow) {
m.invalidateDefaultInterfaces()
if !m.isActive() {
// Avoid starting a goroutine that sends events to messagec,
// or sending messages to messagec directly, if the monitor
@@ -152,6 +229,7 @@ func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, row *wini
// routeChanged is the callback we register with Windows to call when route changes.
func (m *winMon) routeChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibIPforwardRow2) {
m.invalidateDefaultInterfaces()
if !m.isActive() {
// Avoid starting a goroutine that sends events to messagec,
// or sending messages to messagec directly, if the monitor
+82
View File
@@ -0,0 +1,82 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package netmon
import (
"context"
"testing"
"time"
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
)
func TestWindowsDefaultInterfaceIndex(t *testing.T) {
wm := &winMon{}
m := &Monitor{om: wm}
wm.defaultInterface4.get(func() (uint32, error) { return 4, nil })
wm.defaultInterface6.get(func() (uint32, error) { return 6, nil })
for _, tt := range []struct {
family winipcfg.AddressFamily
index uint32
}{
{windows.AF_INET, 4},
{windows.AF_INET6, 6},
} {
if got, err := m.WindowsDefaultInterfaceIndex(tt.family); got != tt.index || err != nil {
t.Errorf("WindowsDefaultInterfaceIndex(%v) = %v, %v; want %v, nil", tt.family, got, err, tt.index)
}
}
if _, err := m.WindowsDefaultInterfaceIndex(windows.AF_UNSPEC); err == nil {
t.Error("WindowsDefaultInterfaceIndex(AF_UNSPEC) succeeded")
}
}
func TestWindowsDefaultInterfaceCacheClose(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
wm := &winMon{
ctx: ctx,
cancel: cancel,
noDeadlockTicker: time.NewTicker(time.Hour),
}
for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} {
cache.get(func() (uint32, error) { return 1, nil })
}
if err := wm.Close(); err != nil {
t.Fatal(err)
}
for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} {
for _, want := range []uint32{2, 3} {
if got, err := cache.get(func() (uint32, error) { return want, nil }); got != want || err != nil {
t.Errorf("get after Close = %v, %v; want %v, nil", got, err, want)
}
}
}
}
func TestWindowsDefaultInterfaceInvalidation(t *testing.T) {
for _, event := range []string{"route", "address", "interface"} {
t.Run(event, func(t *testing.T) {
// Invalidation must work before Start, when the normal event
// delivery path would drop the notification.
wm := &winMon{isActive: func() bool { return false }}
for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} {
cache.get(func() (uint32, error) { return 1, nil })
}
switch event {
case "route":
wm.routeChanged(0, nil)
case "address":
wm.unicastAddressChanged(0, nil)
case "interface":
wm.interfaceChanged(0, nil)
}
for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} {
if got, err := cache.get(func() (uint32, error) { return 2, nil }); got != 2 || err != nil {
t.Errorf("get after %s change = %v, %v; want 2, nil", event, got, err)
}
}
})
}
}
+10 -25
View File
@@ -19,29 +19,9 @@ import (
"tailscale.com/types/logger"
)
func interfaceIndex(iface *winipcfg.IPAdapterAddresses) uint32 {
if iface == nil {
// The zero ifidx means "unspecified". If we end up passing zero
// to bindSocket*(), it unsets the binding and lets the socket
// behave as normal again, which is what we want if there's no
// default route we can use.
return 0
}
return iface.IfIndex
}
func defaultInterfaceIndex(family winipcfg.AddressFamily) (uint32, error) {
iface, err := netmon.GetWindowsDefault(family)
if err != nil {
return 0, err
}
return interfaceIndex(iface), nil
}
func control(logf logger.Logf, _ *netmon.Monitor) func(network, address string, c syscall.RawConn) error {
func control(logf logger.Logf, netMon *netmon.Monitor) func(network, address string, c syscall.RawConn) error {
return func(network, address string, c syscall.RawConn) error {
return controlC(logf, network, address, c)
return controlC(logf, netMon, network, address, c)
}
}
@@ -49,7 +29,7 @@ var bindToInterfaceByRouteEnv = envknob.RegisterBool("TS_BIND_TO_INTERFACE_BY_RO
// controlC binds c to the Windows interface that holds a default
// route, and is not the Tailscale WinTun interface.
func controlC(logf logger.Logf, network, address string, c syscall.RawConn) (err error) {
func controlC(logf logger.Logf, netMon *netmon.Monitor, network, address string, c syscall.RawConn) (err error) {
if isLocalhost(address) {
// Don't bind to an interface for localhost connections,
// otherwise we get:
@@ -68,16 +48,21 @@ func controlC(logf logger.Logf, network, address string, c syscall.RawConn) (err
canV6 = true
}
// defIfaceIdxV4 and defIfaceIdxV6 are the default interface indices for
// each connectable address family. A zero index means there is no
// non-Tailscale default route, and leaves the socket unbound. The monitor
// caches nonzero indices, per family, until a network change
// notification.
var defIfaceIdxV4, defIfaceIdxV6 uint32
if canV4 {
defIfaceIdxV4, err = defaultInterfaceIndex(windows.AF_INET)
defIfaceIdxV4, err = netMon.WindowsDefaultInterfaceIndex(windows.AF_INET)
if err != nil {
return fmt.Errorf("defaultInterfaceIndex(AF_INET): %w", err)
}
}
if canV6 {
defIfaceIdxV6, err = defaultInterfaceIndex(windows.AF_INET6)
defIfaceIdxV6, err = netMon.WindowsDefaultInterfaceIndex(windows.AF_INET6)
if err != nil {
return fmt.Errorf("defaultInterfaceIndex(AF_INET6): %w", err)
}
+4 -2
View File
@@ -9,6 +9,7 @@ import (
"golang.org/x/sys/windows"
"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
"tailscale.com/net/netmon"
"tailscale.com/tsconst"
)
@@ -17,7 +18,8 @@ func TestGetInterfaceIndex(t *testing.T) {
t.Cleanup(func() { bindToInterfaceByRoute.Store(oldVal) })
bindToInterfaceByRoute.Store(true)
defIfaceIdxV4, err := defaultInterfaceIndex(windows.AF_INET)
netMon := netmon.NewStatic()
defIfaceIdxV4, err := netMon.WindowsDefaultInterfaceIndex(windows.AF_INET)
if err != nil {
t.Fatalf("defaultInterfaceIndex(AF_INET) failed: %v", err)
}
@@ -69,7 +71,7 @@ func TestGetInterfaceIndex(t *testing.T) {
t.Skip("no tailscale interface on this machine")
}
defaultIdx, err := defaultInterfaceIndex(windows.AF_INET)
defaultIdx, err := netMon.WindowsDefaultInterfaceIndex(windows.AF_INET)
if err != nil {
t.Fatalf("defaultInterfaceIndex(AF_INET) failed: %v", err)
}