mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 08:41:19 -08:00
wgengine/magicsock: retry UDP reads on EADDRNOTAVAIL
A customer noticed that UDP reads returned EADDRNOTAVAIL during a link change while the interface had no IPv4 address. magicsock returned the error to wireguard-go, which treated it as permanent and shut down the receive loop. magicsock now retries the read when EADDRNOTAVAIL is returned on Unix. Each error is counted in the magicsock_recv_udp_error_unknown metric. EADDRNOTAVAIL isn't a documented result of a UDP recv. We suspect that certain macOS network extensions return it while the network interface is mid-change and has no address. We expect the error to stop once the interface has an address again. A short delay between retries prevents magicsock from spinning when the address doesn't come back right away. Updates #22097 Change-Id: I70790f78d9488999ce00f9685aec3f0fabc15699 Signed-off-by: Nick Beirne <[email protected]>
This commit is contained in:
1 parent
3ac0005aa2
commit
2c663f0d84
8 files changed
+86
-2
No files matched your search
@@ -17,6 +17,11 @@ func IsUDPReadFeedback(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsUDPReadUnknown always returns false on Plan 9.
|
||||
func IsUDPReadUnknown(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
|
||||
|
||||
@@ -18,6 +18,13 @@ func IsUDPReadFeedback(err error) bool {
|
||||
return errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH)
|
||||
}
|
||||
|
||||
// IsUDPReadUnknown reports whether err is a UDP read error whose meaning is
|
||||
// unclear: it may be temporary or permanent, so callers should not treat it
|
||||
// as fatal on its own.
|
||||
func IsUDPReadUnknown(err error) bool {
|
||||
return errors.Is(err, syscall.EADDRNOTAVAIL)
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// EPIPE/ENOTCONN are common errors when a send fails due to a closed
|
||||
|
||||
@@ -44,3 +44,25 @@ func TestIsUDPReadFeedback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUDPReadErrorUnknown(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{nil, false},
|
||||
{errors.New("unknown error"), false},
|
||||
{syscall.EADDRNOTAVAIL, true},
|
||||
{syscall.EHOSTUNREACH, false},
|
||||
{syscall.ECONNRESET, false},
|
||||
{syscall.EBADF, false},
|
||||
{net.ErrClosed, false},
|
||||
} {
|
||||
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
|
||||
Syscall: "recvfrom", Err: tt.err,
|
||||
}}
|
||||
if got := IsUDPReadUnknown(wrapped); got != tt.want {
|
||||
t.Errorf("IsUDPReadUnknown(wrapped %v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,11 @@ func IsUDPReadFeedback(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsUDPReadUnknown always returns false on this platform.
|
||||
func IsUDPReadUnknown(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
|
||||
|
||||
@@ -18,6 +18,11 @@ func IsUDPReadFeedback(err error) bool {
|
||||
return errors.Is(err, windows.WSAEHOSTUNREACH) || errors.Is(err, windows.WSAENETUNREACH)
|
||||
}
|
||||
|
||||
// IsUDPReadUnknown reports false on windows.
|
||||
func IsUDPReadUnknown(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsClosedPipeError reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
return errors.Is(err, windows.ERROR_NO_DATA) ||
|
||||
|
||||
@@ -70,3 +70,25 @@ func TestIsClosedPipeError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUDPReadErrorUnknown(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{nil, false},
|
||||
{errors.New("unknown error"), false},
|
||||
{windows.WSAEADDRNOTAVAIL, false},
|
||||
{windows.WSAEHOSTUNREACH, false},
|
||||
{windows.WSAECONNRESET, false},
|
||||
{syscall.EADDRNOTAVAIL, false},
|
||||
{net.ErrClosed, false},
|
||||
} {
|
||||
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
|
||||
Syscall: "WSARecvFrom", Err: tt.err,
|
||||
}}
|
||||
if got := IsUDPReadUnknown(wrapped); got != tt.want {
|
||||
t.Errorf("IsUDPReadUnknown(wrapped %v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1753,6 +1753,11 @@ type udpReceiveConfig struct {
|
||||
directBytes, peerRelayBytes *expvar.Int
|
||||
}
|
||||
|
||||
const (
|
||||
// readErrRetryDelay paces a [conn.ReceiveFunc]'s read errors.
|
||||
readErrRetryDelay = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc {
|
||||
ruc := cfg.ruc
|
||||
directPacketMetric, peerRelayPacketMetric := cfg.directPackets, cfg.peerRelayPackets
|
||||
@@ -1793,7 +1798,16 @@ func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc {
|
||||
metricRecvUDPErrorReachability.Add(1)
|
||||
continue
|
||||
}
|
||||
return 0, err
|
||||
if !neterror.IsUDPReadUnknown(err) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
metricRecvUDPErrorUnknown.Add(1)
|
||||
select {
|
||||
case <-c.connCtx.Done():
|
||||
case <-time.After(readErrRetryDelay):
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
reportToCaller := false
|
||||
@@ -4351,6 +4365,8 @@ var (
|
||||
|
||||
// UDP reads matching neterror.IsUDPReadFeedback.
|
||||
metricRecvUDPErrorReachability = clientmetric.NewCounter("magicsock_recv_udp_error_reachability")
|
||||
// UDP reads matching neterror.IsUDPReadUnknown.
|
||||
metricRecvUDPErrorUnknown = clientmetric.NewCounter("magicsock_recv_udp_error_unknown")
|
||||
|
||||
// Sends (data)
|
||||
//
|
||||
|
||||
@@ -42,7 +42,7 @@ func (*scriptedPacketConn) LocalAddr() net.Addr { return &net.UDPAd
|
||||
|
||||
var _ nettype.PacketConn = (*scriptedPacketConn)(nil)
|
||||
|
||||
func TestReceiveFuncReadFeedback(t *testing.T) {
|
||||
func TestReceiveFuncRetriesRecoverableErrors(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
feedback bool
|
||||
@@ -51,6 +51,8 @@ func TestReceiveFuncReadFeedback(t *testing.T) {
|
||||
{syscall.ENETUNREACH, runtime.GOOS != "windows"},
|
||||
{syscall.Errno(10065), runtime.GOOS == "windows"},
|
||||
{syscall.Errno(10051), runtime.GOOS == "windows"},
|
||||
{syscall.EADDRNOTAVAIL, runtime.GOOS != "windows"},
|
||||
{syscall.Errno(10049), false}, // WSAEADDRNOTAVAIL
|
||||
{syscall.EBADF, false},
|
||||
{syscall.ENOTSOCK, false},
|
||||
{syscall.ECONNRESET, false},
|
||||
|
||||
Reference in new issue
Block a user