diff --git a/net/netmon/default_interface_cache.go b/net/netmon/default_interface_cache.go new file mode 100644 index 000000000..6ad183788 --- /dev/null +++ b/net/netmon/default_interface_cache.go @@ -0,0 +1,99 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package netmon + +import ( + "sync" +) + +// defaultInterfaceCacheAttempts is the number of lookup and wait attempts a +// single get may make, including waits for another caller's lookup, before +// falling back to one uncached lookup. Bounding them keeps network churn from +// keeping socket setup in this cache forever. +const defaultInterfaceCacheAttempts = 2 + +// defaultInterfaceCache caches a nonzero default interface index until an OS change +// notification invalidates it. Windows uses one cache per address family. +// It is independent of Windows APIs so its concurrency can be tested on any OS. +// The zero value is ready for use. +type defaultInterfaceCache struct { + mu sync.Mutex // guards following + inFlight chan struct{} // closed when the current lookup completes, or nil + generation uint64 + index uint32 + valid bool + disabled bool // notifications are no longer being delivered +} + +func (c *defaultInterfaceCache) invalidate() { + c.mu.Lock() + defer c.mu.Unlock() + c.generation++ + c.valid = false +} + +// disable stops caching when the notification subscription is closed. +func (c *defaultInterfaceCache) disable() { + c.mu.Lock() + defer c.mu.Unlock() + c.disabled = true + c.generation++ + c.valid = false +} + +func (c *defaultInterfaceCache) get(lookup func() (uint32, error)) (uint32, error) { + for attempt := 0; ; attempt++ { + c.mu.Lock() + if c.disabled { + c.mu.Unlock() + return lookup() + } + if c.valid { + index := c.index + c.mu.Unlock() + return index, nil + } + if attempt == defaultInterfaceCacheAttempts { + c.mu.Unlock() + // Churn prevented a stable result. Do one uncached lookup rather + // than fail: a possibly stale interface index beats no socket. + return lookup() + } + if done := c.inFlight; done != nil { + c.mu.Unlock() + <-done + continue + } + // Only one caller performs the expensive lookup. Invalidation must + // not wait for it, since the OS calls can be slow on some systems. + done := make(chan struct{}) + c.inFlight = done + generation := c.generation + c.mu.Unlock() + + index, err := lookup() + c.mu.Lock() + c.inFlight = nil + close(done) + if c.disabled { + c.mu.Unlock() + return index, err + } + if generation != c.generation { + // The network changed during the lookup. Don't publish the old + // result, even if the lookup succeeded. + c.mu.Unlock() + continue + } + if err == nil && index != 0 { + // Never cache "no default interface". Zero leaves a socket + // unconstrained, possibly routing it back into Tailscale. Each + // caller must check the OS again rather than reuse that absence + // while a route-restoration notification is still pending. + c.index, c.valid = index, true + } + c.mu.Unlock() + return index, err + } +} diff --git a/net/netmon/default_interface_cache_test.go b/net/netmon/default_interface_cache_test.go new file mode 100644 index 000000000..b30c3cdfb --- /dev/null +++ b/net/netmon/default_interface_cache_test.go @@ -0,0 +1,275 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package netmon + +import ( + "errors" + "sync/atomic" + "testing" + "testing/synctest" +) + +func BenchmarkDefaultInterfaceCache(b *testing.B) { + var c defaultInterfaceCache + lookup := func() (uint32, error) { return 42, nil } + c.get(lookup) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + c.get(lookup) + } +} + +func TestDefaultInterfaceCache(t *testing.T) { + var c defaultInterfaceCache + var calls int + lookup := func() (uint32, error) { + calls++ + return 42, nil + } + for range 3 { + if got, err := c.get(lookup); got != 42 || err != nil { + t.Fatalf("get = %v, %v; want 42, nil", got, err) + } + } + if calls != 1 { + t.Fatalf("lookups = %v; want 1", calls) + } + c.invalidate() + if got, err := c.get(lookup); got != 42 || err != nil { + t.Fatalf("get after invalidation = %v, %v; want 42, nil", got, err) + } + if calls != 2 { + t.Fatalf("lookups after invalidation = %v; want 2", calls) + } +} + +func TestDefaultInterfaceCacheNoDefault(t *testing.T) { + var c defaultInterfaceCache + var calls int + index := uint32(0) + lookup := func() (uint32, error) { + calls++ + return index, nil + } + for range 2 { + if got, err := c.get(lookup); got != 0 || err != nil { + t.Fatalf("get = %v, %v; want 0, nil", got, err) + } + } + if calls != 2 { + t.Fatalf("lookups with no default = %v; want 2", calls) + } + // The route returns before its notification is delivered. Without an + // explicit invalidation, the very next caller must see the restored route. + index = 42 + for range 2 { + if got, err := c.get(lookup); got != 42 || err != nil { + t.Fatalf("get after route restoration = %v, %v; want 42, nil", got, err) + } + } + if calls != 3 { + t.Fatalf("lookups after route restoration = %v; want 3", calls) + } +} + +func TestDefaultInterfaceCacheChurn(t *testing.T) { + var c defaultInterfaceCache + var calls int + // Every lookup races a network change, so no result is ever stable. + got, err := c.get(func() (uint32, error) { + calls++ + c.invalidate() + return uint32(40 + calls), nil + }) + if got != 43 || err != nil { + t.Fatalf("get during churn = %v, %v; want 43, nil", got, err) + } + if calls != defaultInterfaceCacheAttempts+1 { + t.Fatalf("lookups = %v; want %v", calls, defaultInterfaceCacheAttempts+1) + } + // The unstable result must not be published. + if got, err := c.get(func() (uint32, error) { return 44, nil }); got != 44 || err != nil { + t.Fatalf("get after churn = %v, %v; want 44, nil", got, err) + } +} + +func TestDefaultInterfaceCacheWaitBudget(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var c defaultInterfaceCache + c.inFlight = make(chan struct{}) + var calls atomic.Int32 + done := make(chan struct{}) + go func() { + defer close(done) + // Waiting for other callers' fills consumes the wait budget too. + // After exhausting it, get must do its own uncached lookup rather + // than fail without ever looking anything up. + got, err := c.get(func() (uint32, error) { + return uint32(41 + calls.Add(1)), nil + }) + if got != 42 || err != nil { + t.Errorf("get = %v, %v; want 42, nil", got, err) + } + }() + for attempt := range defaultInterfaceCacheAttempts { + synctest.Wait() + // Simulate other callers taking successive fills. + c.mu.Lock() + previous := c.inFlight + c.inFlight = nil + if attempt+1 < defaultInterfaceCacheAttempts { + c.inFlight = make(chan struct{}) + } + close(previous) + c.mu.Unlock() + } + <-done + if got := calls.Load(); got != 1 { + t.Fatalf("lookups = %v; want 1", got) + } + }) +} + +func TestDefaultInterfaceCacheError(t *testing.T) { + var c defaultInterfaceCache + wantErr := errors.New("lookup failed") + calls := 0 + lookup := func() (uint32, error) { + calls++ + if calls == 1 { + return 0, wantErr + } + return 42, nil + } + if _, err := c.get(lookup); err != wantErr { + t.Fatalf("get error = %v; want %v", err, wantErr) + } + for range 2 { + if got, err := c.get(lookup); got != 42 || err != nil { + t.Fatalf("get = %v, %v; want 42, nil", got, err) + } + } + if calls != 2 { + t.Fatalf("lookups = %v; want 2", calls) + } +} + +func TestDefaultInterfaceCacheConcurrent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var c defaultInterfaceCache + var calls atomic.Int32 + release := make(chan struct{}) + lookup := func() (uint32, error) { + calls.Add(1) + <-release + return 42, nil + } + const n = 20 + done := make(chan struct{}, n) + for range n { + go func() { + if got, err := c.get(lookup); got != 42 || err != nil { + t.Errorf("get = %v, %v; want 42, nil", got, err) + } + done <- struct{}{} + }() + } + synctest.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("concurrent lookups = %v; want 1", got) + } + close(release) + for range n { + <-done + } + if got := calls.Load(); got != 1 { + t.Fatalf("total lookups = %v; want 1", got) + } + }) +} + +func TestDefaultInterfaceCacheInvalidateDuringLookup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var c defaultInterfaceCache + var calls atomic.Int32 + release := make(chan struct{}) + lookup := func() (uint32, error) { + if calls.Add(1) == 1 { + <-release + return 1, nil // obsolete result + } + return 2, nil + } + done := make(chan struct{}) + go func() { + defer close(done) + if got, err := c.get(lookup); got != 2 || err != nil { + t.Errorf("get = %v, %v; want 2, nil", got, err) + } + }() + synctest.Wait() + // Must not block behind the expensive lookup. + c.invalidate() + close(release) + <-done + if got, err := c.get(lookup); got != 2 || err != nil { + t.Fatalf("cached get = %v, %v; want 2, nil", got, err) + } + if got := calls.Load(); got != 2 { + t.Fatalf("lookups = %v; want 2", got) + } + }) +} + +func TestDefaultInterfaceCacheDisableDuringLookup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var c defaultInterfaceCache + var calls atomic.Int32 + release := make(chan struct{}) + lookup := func() (uint32, error) { + n := calls.Add(1) + if n == 1 { + <-release + } + return uint32(n), nil + } + done := make(chan struct{}) + go func() { + defer close(done) + if got, err := c.get(lookup); got != 1 || err != nil { + t.Errorf("get = %v, %v; want 1, nil", got, err) + } + }() + synctest.Wait() + c.disable() + close(release) + <-done + // Even a later notification must not re-enable the cache. + c.invalidate() + for _, want := range []uint32{2, 3} { + if got, err := c.get(lookup); got != want || err != nil { + t.Fatalf("disabled get = %v, %v; want %v, nil", got, err, want) + } + } + }) +} + +func TestDefaultInterfaceCacheDisablePopulated(t *testing.T) { + var c defaultInterfaceCache + var calls uint32 + lookup := func() (uint32, error) { + calls++ + return calls, nil + } + if got, err := c.get(lookup); got != 1 || err != nil { + t.Fatalf("get = %v, %v; want 1, nil", got, err) + } + c.disable() + for _, want := range []uint32{2, 3} { + if got, err := c.get(lookup); got != want || err != nil { + t.Fatalf("disabled get = %v, %v; want %v, nil", got, err, want) + } + } +} diff --git a/net/netmon/netmon_windows.go b/net/netmon/netmon_windows.go index 91c137de0..6a2013905 100644 --- a/net/netmon/netmon_windows.go +++ b/net/netmon/netmon_windows.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" "tailscale.com/net/tsaddr" "tailscale.com/types/logger" @@ -27,13 +28,22 @@ type eventMessage struct { func (eventMessage) ignore() bool { return false } type winMon struct { - logf logger.Logf - ctx context.Context - cancel context.CancelFunc - isActive func() bool - messagec chan eventMessage - addressChangeCallback *winipcfg.UnicastAddressChangeCallback - routeChangeCallback *winipcfg.RouteChangeCallback + logf logger.Logf + ctx context.Context + cancel context.CancelFunc + isActive func() bool + messagec chan eventMessage + addressChangeCallback *winipcfg.UnicastAddressChangeCallback + routeChangeCallback *winipcfg.RouteChangeCallback + interfaceChangeCallback *winipcfg.InterfaceChangeCallback + + // defaultInterface4 and defaultInterface6 cache the default interface + // index for each address family. They are invalidated directly by OS + // change notifications, rather than by the monitor's debounced State, + // which only records the IPv4 default route and no interface or route + // metrics. + defaultInterface4 defaultInterfaceCache + defaultInterface6 defaultInterfaceCache mu sync.Mutex lastLog time.Time // time we last logged about any windows change event @@ -59,22 +69,79 @@ func newOSMon(_ *eventbus.Bus, logf logger.Logf, pm *Monitor) (osMon, error) { m.addressChangeCallback, err = winipcfg.RegisterUnicastAddressChangeCallback(m.unicastAddressChanged) if err != nil { m.logf("winipcfg.RegisterUnicastAddressChangeCallback error: %v", err) - m.cancel() + m.Close() return nil, err } m.routeChangeCallback, err = winipcfg.RegisterRouteChangeCallback(m.routeChanged) if err != nil { - m.addressChangeCallback.Unregister() m.logf("winipcfg.RegisterRouteChangeCallback error: %v", err) - m.cancel() + m.Close() return nil, err } + m.interfaceChangeCallback, err = winipcfg.RegisterInterfaceChangeCallback(m.interfaceChanged) + if err != nil { + m.logf("winipcfg.RegisterInterfaceChangeCallback error: %v; default interface caching disabled", err) + // The cache cannot be invalidated correctly without interface change + // notifications, so disable it. Route and address monitoring is + // unaffected. + m.defaultInterface4.disable() + m.defaultInterface6.disable() + } + return m, nil } +// WindowsDefaultInterfaceIndex returns the index of the non-Tailscale default +// interface for family, which must be AF_INET or AF_INET6. Zero means no default +// interface was found. Nonzero results are cached until an OS network change +// notification; zero results and errors are not cached. Static and closed +// monitors, and monitors without the required notification subscriptions, +// perform uncached lookups, as do lookups that cannot stabilize under repeated +// network changes. +func (m *Monitor) WindowsDefaultInterfaceIndex(family winipcfg.AddressFamily) (uint32, error) { + var cache *defaultInterfaceCache + wm, _ := m.om.(*winMon) + switch family { + case windows.AF_INET: + if wm != nil { + cache = &wm.defaultInterface4 + } + case windows.AF_INET6: + if wm != nil { + cache = &wm.defaultInterface6 + } + default: + return 0, errors.New("invalid address family for Windows default interface") + } + lookup := func() (uint32, error) { + iface, err := GetWindowsDefault(family) + if err != nil || iface == nil { + return 0, err + } + return iface.IfIndex, nil + } + if cache == nil { + return lookup() + } + return cache.get(lookup) +} + +func (m *winMon) invalidateDefaultInterfaces() { + m.defaultInterface4.invalidate() + m.defaultInterface6.invalidate() +} + +// interfaceChanged covers changes to interface metrics, enabled address families, +// and connectivity, which can change the best default without changing a route. +func (m *winMon) interfaceChanged(_ winipcfg.MibNotificationType, _ *winipcfg.MibIPInterfaceRow) { + m.invalidateDefaultInterfaces() +} + func (m *winMon) Close() (ret error) { + m.defaultInterface4.disable() + m.defaultInterface6.disable() m.cancel() m.noDeadlockTicker.Stop() @@ -96,6 +163,15 @@ func (m *winMon) Close() (ret error) { } } + if m.interfaceChangeCallback != nil { + if err := m.interfaceChangeCallback.Unregister(); err != nil { + m.logf("interfaceChangeCallback.Unregister error: %v", err) + ret = err + } else { + m.interfaceChangeCallback = nil + } + } + return } @@ -131,6 +207,7 @@ func (m *winMon) Receive() (message, error) { // unicastAddressChanged is the callback we register with Windows to call when unicast address changes. func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibUnicastIPAddressRow) { + m.invalidateDefaultInterfaces() if !m.isActive() { // Avoid starting a goroutine that sends events to messagec, // or sending messages to messagec directly, if the monitor @@ -152,6 +229,7 @@ func (m *winMon) unicastAddressChanged(_ winipcfg.MibNotificationType, row *wini // routeChanged is the callback we register with Windows to call when route changes. func (m *winMon) routeChanged(_ winipcfg.MibNotificationType, row *winipcfg.MibIPforwardRow2) { + m.invalidateDefaultInterfaces() if !m.isActive() { // Avoid starting a goroutine that sends events to messagec, // or sending messages to messagec directly, if the monitor diff --git a/net/netmon/netmon_windows_test.go b/net/netmon/netmon_windows_test.go new file mode 100644 index 000000000..5f985eafe --- /dev/null +++ b/net/netmon/netmon_windows_test.go @@ -0,0 +1,82 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package netmon + +import ( + "context" + "testing" + "time" + + "golang.org/x/sys/windows" + "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" +) + +func TestWindowsDefaultInterfaceIndex(t *testing.T) { + wm := &winMon{} + m := &Monitor{om: wm} + wm.defaultInterface4.get(func() (uint32, error) { return 4, nil }) + wm.defaultInterface6.get(func() (uint32, error) { return 6, nil }) + for _, tt := range []struct { + family winipcfg.AddressFamily + index uint32 + }{ + {windows.AF_INET, 4}, + {windows.AF_INET6, 6}, + } { + if got, err := m.WindowsDefaultInterfaceIndex(tt.family); got != tt.index || err != nil { + t.Errorf("WindowsDefaultInterfaceIndex(%v) = %v, %v; want %v, nil", tt.family, got, err, tt.index) + } + } + if _, err := m.WindowsDefaultInterfaceIndex(windows.AF_UNSPEC); err == nil { + t.Error("WindowsDefaultInterfaceIndex(AF_UNSPEC) succeeded") + } +} + +func TestWindowsDefaultInterfaceCacheClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + wm := &winMon{ + ctx: ctx, + cancel: cancel, + noDeadlockTicker: time.NewTicker(time.Hour), + } + for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} { + cache.get(func() (uint32, error) { return 1, nil }) + } + if err := wm.Close(); err != nil { + t.Fatal(err) + } + for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} { + for _, want := range []uint32{2, 3} { + if got, err := cache.get(func() (uint32, error) { return want, nil }); got != want || err != nil { + t.Errorf("get after Close = %v, %v; want %v, nil", got, err, want) + } + } + } +} + +func TestWindowsDefaultInterfaceInvalidation(t *testing.T) { + for _, event := range []string{"route", "address", "interface"} { + t.Run(event, func(t *testing.T) { + // Invalidation must work before Start, when the normal event + // delivery path would drop the notification. + wm := &winMon{isActive: func() bool { return false }} + for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} { + cache.get(func() (uint32, error) { return 1, nil }) + } + switch event { + case "route": + wm.routeChanged(0, nil) + case "address": + wm.unicastAddressChanged(0, nil) + case "interface": + wm.interfaceChanged(0, nil) + } + for _, cache := range []*defaultInterfaceCache{&wm.defaultInterface4, &wm.defaultInterface6} { + if got, err := cache.get(func() (uint32, error) { return 2, nil }); got != 2 || err != nil { + t.Errorf("get after %s change = %v, %v; want 2, nil", event, got, err) + } + } + }) + } +} diff --git a/net/netns/netns_windows.go b/net/netns/netns_windows.go index 686c813f6..f13449bd7 100644 --- a/net/netns/netns_windows.go +++ b/net/netns/netns_windows.go @@ -19,29 +19,9 @@ import ( "tailscale.com/types/logger" ) -func interfaceIndex(iface *winipcfg.IPAdapterAddresses) uint32 { - if iface == nil { - // The zero ifidx means "unspecified". If we end up passing zero - // to bindSocket*(), it unsets the binding and lets the socket - // behave as normal again, which is what we want if there's no - // default route we can use. - return 0 - } - return iface.IfIndex -} - -func defaultInterfaceIndex(family winipcfg.AddressFamily) (uint32, error) { - iface, err := netmon.GetWindowsDefault(family) - if err != nil { - return 0, err - } - - return interfaceIndex(iface), nil -} - -func control(logf logger.Logf, _ *netmon.Monitor) func(network, address string, c syscall.RawConn) error { +func control(logf logger.Logf, netMon *netmon.Monitor) func(network, address string, c syscall.RawConn) error { return func(network, address string, c syscall.RawConn) error { - return controlC(logf, network, address, c) + return controlC(logf, netMon, network, address, c) } } @@ -49,7 +29,7 @@ var bindToInterfaceByRouteEnv = envknob.RegisterBool("TS_BIND_TO_INTERFACE_BY_RO // controlC binds c to the Windows interface that holds a default // route, and is not the Tailscale WinTun interface. -func controlC(logf logger.Logf, network, address string, c syscall.RawConn) (err error) { +func controlC(logf logger.Logf, netMon *netmon.Monitor, network, address string, c syscall.RawConn) (err error) { if isLocalhost(address) { // Don't bind to an interface for localhost connections, // otherwise we get: @@ -68,16 +48,21 @@ func controlC(logf logger.Logf, network, address string, c syscall.RawConn) (err canV6 = true } + // defIfaceIdxV4 and defIfaceIdxV6 are the default interface indices for + // each connectable address family. A zero index means there is no + // non-Tailscale default route, and leaves the socket unbound. The monitor + // caches nonzero indices, per family, until a network change + // notification. var defIfaceIdxV4, defIfaceIdxV6 uint32 if canV4 { - defIfaceIdxV4, err = defaultInterfaceIndex(windows.AF_INET) + defIfaceIdxV4, err = netMon.WindowsDefaultInterfaceIndex(windows.AF_INET) if err != nil { return fmt.Errorf("defaultInterfaceIndex(AF_INET): %w", err) } } if canV6 { - defIfaceIdxV6, err = defaultInterfaceIndex(windows.AF_INET6) + defIfaceIdxV6, err = netMon.WindowsDefaultInterfaceIndex(windows.AF_INET6) if err != nil { return fmt.Errorf("defaultInterfaceIndex(AF_INET6): %w", err) } diff --git a/net/netns/netns_windows_test.go b/net/netns/netns_windows_test.go index 67e7b3de8..df393689c 100644 --- a/net/netns/netns_windows_test.go +++ b/net/netns/netns_windows_test.go @@ -9,6 +9,7 @@ import ( "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" + "tailscale.com/net/netmon" "tailscale.com/tsconst" ) @@ -17,7 +18,8 @@ func TestGetInterfaceIndex(t *testing.T) { t.Cleanup(func() { bindToInterfaceByRoute.Store(oldVal) }) bindToInterfaceByRoute.Store(true) - defIfaceIdxV4, err := defaultInterfaceIndex(windows.AF_INET) + netMon := netmon.NewStatic() + defIfaceIdxV4, err := netMon.WindowsDefaultInterfaceIndex(windows.AF_INET) if err != nil { t.Fatalf("defaultInterfaceIndex(AF_INET) failed: %v", err) } @@ -69,7 +71,7 @@ func TestGetInterfaceIndex(t *testing.T) { t.Skip("no tailscale interface on this machine") } - defaultIdx, err := defaultInterfaceIndex(windows.AF_INET) + defaultIdx, err := netMon.WindowsDefaultInterfaceIndex(windows.AF_INET) if err != nil { t.Fatalf("defaultInterfaceIndex(AF_INET) failed: %v", err) }