From 04ed834b368df95fd09f6b8cd9896e6a34528ccf Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Mon, 16 Nov 2026 12:19:54 -0700 Subject: [PATCH] feature/conn25: resolve and count client flows atomically (#22288) The datapath resolved a Magic IP to its Transit IP and then, in a second lock acquisition, counted the new flow against the assignment holding that Transit IP. A concurrent reserveAddresses could expire the assignment in between and hand the Transit IP to an unrelated destination, so the flow was installed DNATing to an address another destination now owned, counted against that stranger's assignment. Merge the two steps into ClientFlowCreatedForMagicIP, which resolves and counts under one c.mu acquisition. Fixes https://github.com/tailscale/corp/issues/44273 Change-Id: Iefae902267fd06a5d2c965a3570b6977921be76f Signed-off-by: Mazdak Nasab --- feature/conn25/client.go | 49 ++++++---- feature/conn25/client_test.go | 168 ++++++++++++++++++++++++++------ feature/conn25/conn25.go | 35 ++++--- feature/conn25/conn25_test.go | 15 ++- feature/conn25/datapath.go | 49 +++++----- feature/conn25/datapath_test.go | 28 +++--- 6 files changed, 241 insertions(+), 103 deletions(-) diff --git a/feature/conn25/client.go b/feature/conn25/client.go index 22826f49d..26e6516c0 100644 --- a/feature/conn25/client.go +++ b/feature/conn25/client.go @@ -42,18 +42,6 @@ type client struct { byConnKey map[key.NodePublic]set.Set[netip.Prefix] } -// transitIPForMagicIP is part of the implementation of the [Conn25Datapath] interface for dataflow lookups. -// See also [Conn25Datapath.ClientTransitIPForMagicIP]. -func (c *client) transitIPForMagicIP(magicIP netip.Addr) (netip.Addr, bool) { - c.mu.Lock() - defer c.mu.Unlock() - v, ok := c.assignments.lookupByMagicIP(magicIP) - if ok { - return v.connectorAddr.transit, true - } - return netip.Addr{}, false -} - // linkLocalAllow returns true if the provided packet with a link-local Dst address has a // Dst that is one of our transit IPs, and false otherwise. // Tailscale's wireguard filters drop link-local unicast packets (see [wgengine/filter/filter.go]) @@ -280,16 +268,43 @@ func (c *client) enqueueAddressAssignment(addrs *addrs) error { } } -func (c *client) flowCreated(transit netip.Addr) func() { +// ClientFlowRef is a client-side flow's reference to the address assignment it was counted against. +type ClientFlowRef struct { + transit netip.Addr + release func() +} + +// TransitIP returns the Transit IP of the assignment the flow was counted +// against, which is the address its packets should be forwarded to. +func (r ClientFlowRef) TransitIP() netip.Addr { + return r.transit +} + +// Release gives up the flow's claim on the address assignment, letting the +// assignment be reclaimed once nothing else holds it. It must be called once +// the flow is gone, or right away if the caller ends up installing no flow at all. +func (r ClientFlowRef) Release() { + if r.release != nil { + r.release() + } +} + +// PrepareNewFlowForMagicIP resolves magicIP to the transit IP currently assigned +// to it and records that a flow is being established against that assignment, +// returning the reference that both names that transit IP and keeps the +// assignment from being reclaimed. It reports whether magicIP had an +// assignment at all. +func (c *client) PrepareNewFlowForMagicIP(magicIP netip.Addr) (ClientFlowRef, bool) { c.mu.Lock() defer c.mu.Unlock() - entry, ok := c.assignments.byTransitIP[transit] + entry, ok := c.assignments.lookupByMagicIP(magicIP) if !ok { - return func() {} + return ClientFlowRef{}, false } + transit := entry.connectorAddr.transit entry.activeFlowCount++ var released bool - return func() { + return ClientFlowRef{transit: transit, release: func() { c.mu.Lock() defer c.mu.Unlock() if released { @@ -304,7 +319,7 @@ func (c *client) flowCreated(transit netip.Addr) func() { if entry.activeFlowCount == 0 { entry.zeroFlowTime = c.assignments.clock.Now() } - } + }}, true } func (c *client) extraWireGuardAllowedIPs(k key.NodePublic) views.Slice[netip.Prefix] { diff --git a/feature/conn25/client_test.go b/feature/conn25/client_test.go index ff24278c6..19c9112f0 100644 --- a/feature/conn25/client_test.go +++ b/feature/conn25/client_test.go @@ -602,7 +602,8 @@ func TestReconfigDoesNotReissueInUseAddresses(t *testing.T) { // TestAddressExpiryDependsOnActiveFlows creates a Conn25 and // // 1. runs a DNS response through it -// 2. uses the ClientFlowCreated API and its cleanup function, and advances the clock +// 2. uses the ClientPrepareNewFlowForMagicIP API and its cleanup function, +// and advances the clock // 3. runs a second DNS response through the Conn25 // 4. asserts things about the expected state of the clients assignments // table based on 1-3 @@ -666,7 +667,7 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // and so we expect the entry for ipOne to be removed, and the entry for // ipTwo to be present. name: "flows-zero", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -680,8 +681,8 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // ipOne. // And so that entry does not get expired. name: "flows-not-zero", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -695,10 +696,10 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // client flow after creating it. // So the expired entry is removed. name: "last-flow-removed-a-while-ago", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - removeFlow := c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + ref, _, _ := c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) - removeFlow() + ref.Release() clock.Advance(3 * time.Minute) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -711,10 +712,10 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // within the cooldown period. // And so the expired entry is not removed. name: "last-flow-recently-removed", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - removeFlow := c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + ref, _, _ := c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) - removeFlow() + ref.Release() clock.Advance(1 * time.Second) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -728,8 +729,8 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // So the entry is not removed. name: "repeated-response-with-expired-and-active-flow", secondDNSResponse: dnsRespIPOne, - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne}), @@ -773,7 +774,7 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { } // there are client flows and time passes - tt.flowsAndTimeFx(c, clock, c.client.assignments.byDomainDst[ipOneDD].connectorAddr.transit) + tt.flowsAndTimeFx(c, clock, c.client.assignments.byDomainDst[ipOneDD].magic) // then a second dns response dnsR2 := tt.secondDNSResponse @@ -845,7 +846,10 @@ func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { // A flow is established against that assignment, and then the profile // changes and the client's state is reset out from under the flow. - removeFlow := c.ClientFlowCreated(transit) + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } c.client.reset() // reset rebuilds the pools, so iteration restarts and the next assignment @@ -863,7 +867,7 @@ func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { // The pre-reset flow is finally torn down. The removal refers to an // assignment that no longer exists, so it must not touch the new one. - removeFlow() + ref.Release() if got := after.activeFlowCount; got != 0 { t.Errorf("activeFlowCount for reissued transit %v = %d, want 0", transit, got) @@ -909,7 +913,10 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { } transit := before.connectorAddr.transit - removeStaleFlow := c.ClientFlowCreated(transit) + staleRef, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } c.client.reset() after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") @@ -925,11 +932,13 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { // The reissued assignment picks up a flow of its own, which is still live // when the pre-reset flow is torn down. - c.ClientFlowCreated(transit) + if _, ok, err := c.ClientPrepareNewFlowForMagicIP(after.magic); err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", after.magic, ok, err) + } if got := after.activeFlowCount; got != 1 { t.Fatalf("activeFlowCount for the reissued assignment = %d, want 1", got) } - removeStaleFlow() + staleRef.Release() if got := after.activeFlowCount; got != 1 { t.Errorf("activeFlowCount after the stale removal = %d, want 1", got) } @@ -941,10 +950,10 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { } } -// TestFlowCleanupRunsOnce checks that a flow's cleanup releases its hold on the -// assignment at most once, so a caller that runs the same cleanup more than -// once cannot undercount the flows still forwarding over that assignment and -// have its addresses reclaimed back to the pools mid-connection. +// TestFlowCleanupRunsOnce checks that a flow's cleanup releases its reference +// to the assignment at most once, so a caller that runs the same cleanup more +// than once cannot undercount the flows still forwarding over that assignment +// and have its addresses reclaimed back to the pools mid-connection. func TestFlowCleanupRunsOnce(t *testing.T) { const appName = "app1" const domain = dnsname.FQDN("example.com.") @@ -967,19 +976,23 @@ func TestFlowCleanupRunsOnce(t *testing.T) { if err != nil { t.Fatal(err) } - transit := as.connectorAddr.transit - // Two flows over the same transit IP. The first is torn down; the second + // Two flows over the same assignment. The first is torn down; the second // stays live. - removeFirstFlow := c.ClientFlowCreated(transit) - c.ClientFlowCreated(transit) + firstRef, ok, err := c.ClientPrepareNewFlowForMagicIP(as.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", as.magic, ok, err) + } + if _, ok, err := c.ClientPrepareNewFlowForMagicIP(as.magic); err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", as.magic, ok, err) + } if got := as.activeFlowCount; got != 2 { t.Fatalf("activeFlowCount = %d, want 2", got) } - removeFirstFlow() - removeFirstFlow() - removeFirstFlow() + firstRef.Release() + firstRef.Release() + firstRef.Release() if got := as.activeFlowCount; got != 1 { t.Errorf("activeFlowCount after repeated cleanup of one flow = %d, want 1", got) } @@ -991,6 +1004,103 @@ func TestFlowCleanupRunsOnce(t *testing.T) { } } +// singleTransitIPPools has exactly one Transit IP per address family, so that +// the next reservation after one is returned to the pool is handed that same address back. +var singleTransitIPPools = appctype.Conn25PoolsAttr{ + V4MagicIPPool: []netipx.IPRange{v4RangeFrom("0", "10")}, + V6MagicIPPool: []netipx.IPRange{v6RangeFrom("0", "10")}, + V4TransitIPPool: []netipx.IPRange{v4RangeFrom("40", "40")}, + V6TransitIPPool: []netipx.IPRange{v6RangeFrom("40", "40")}, +} + +// TestClientPrepareNewFlowForMagicIPPinsAssignment checks that resolving a +// Magic IP for a new flow and counting that flow against its assignment happen +// as one step. The datapath DNATs the flow to the Transit IP this returns, so +// a window between resolving that address and counting the flow would let a +// concurrent reservation expire the assignment and hand the Transit IP to an +// unrelated destination, silently forwarding the flow to the wrong host. +// +// The window is stood in for here by expiring the assignment before resolving +// it: a reservation that runs afterwards sweeps it with popExpired. The test +// first runs that sequence without resolving the Magic IP, to show that the +// sweep really does reissue the address, so that the pinned half is not passing +// vacuously. +func TestClientPrepareNewFlowForMagicIPPinsAssignment(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, singleTransitIPPools, nil) + + // newExpiredAssignment returns a fresh Conn25 holding a single assignment, + // for dstOne, that is past its expiry and so free for the next + // reservation's popExpired sweep to reclaim and return to the pools. + newExpiredAssignment := func(t *testing.T) (*Conn25, *addrs) { + t.Helper() + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + as, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + return c, as + } + + // First, without a flow to pin it: the assignment is swept and its Transit + // IP goes straight back out to a different destination. This is the + // reassignment the pinned half below has to prevent. + c, before := newExpiredAssignment(t) + transit := before.connectorAddr.transit + + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued to %v (got %v); this test no longer covers reassignment", transit, dstTwo, after.connectorAddr.transit) + } + + // Then the same sequence, with the Magic IP resolved through the + // flow-creating API before the second reservation runs. + c, before = newExpiredAssignment(t) + transit = before.connectorAddr.transit + + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } + if got := ref.TransitIP(); got != transit { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) transit = %v, want %v", before.magic, got, transit) + } + if got := before.activeFlowCount; got != 1 { + t.Fatalf("activeFlowCount for the assignment for %v = %d, want 1", dstOne, got) + } + + // The only Transit IP is carrying a flow, so there is nothing left to + // assign. Refusing the reservation is the point: the alternative is handing + // dstTwo an address that dstOne's flow is still being forwarded to. + after, err = c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if !errors.Is(err, errPoolExhausted) { + t.Fatalf("reserveAddresses for %v = (%+v, %v), want errPoolExhausted", dstTwo, after, err) + } + stillAssigned, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if got := stillAssigned.TransitIP(); err != nil || !ok || got != transit { + t.Errorf("after the refused reservation, ClientPrepareNewFlowForMagicIP(%v) = (%v, %v, %v), want (%v, true, nil)", before.magic, got, ok, err, transit) + } + stillAssigned.Release() +} + // TestResendTransitIPMappingConcurrent races resendTransitIPMapping, against a writer of client.assignments. func TestResendTransitIPMappingConcurrent(t *testing.T) { const appName = "a" diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index 0de51b2ff..bb8a2669c 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -303,34 +303,33 @@ func (e *extension) installHooks(dph *datapathHandler) error { return nil } -// ClientTransitIPForMagicIP implements [Conn25Datapath]. -func (c *Conn25) ClientTransitIPForMagicIP(m netip.Addr) (netip.Addr, error) { - if addr, ok := c.client.transitIPForMagicIP(m); ok { - return addr, nil +// ClientPrepareNewFlowForMagicIP implements [Conn25Datapath]. +// The datapath notifies Conn25 that a flow to magic IP is being established, and +// Conn25 resolves m to its Transit IP and counts the flow against that address +// assignment in one step, reporting ok and returning the reference that names +// that Transit IP and keeps it and its associated addresses from being removed +// from Conn25's state and returned to their pools while the flow uses them. +// +// If m has no active address assignment, it reports ok false and returns +// [ErrUnmappedMagicIP] when m falls in a configured Magic IP range, and a nil +// error when it does not, since in that case m is not a Magic IP at all and the +// traffic to it is potentially valid non-app-connector traffic. +func (c *Conn25) ClientPrepareNewFlowForMagicIP(m netip.Addr) (ClientFlowRef, bool, error) { + if ref, ok := c.client.PrepareNewFlowForMagicIP(m); ok { + return ref, true, nil } cfg, ok := c.getConfig() if !ok { - return netip.Addr{}, nil + return ClientFlowRef{}, false, nil } if !cfg.ipSets.v4Magic.Contains(m) && !cfg.ipSets.v6Magic.Contains(m) { - return netip.Addr{}, nil + return ClientFlowRef{}, false, nil } // TODO(tailscale/corp#50405): Just because there is a TransitIP associated with the MagicIP, // that doesn't mean the TransitIP is ready to use. We should also make sure that the TransitIP // is installed in WireGuard for the correct node, which happens after it is registered with the // connector. Perhaps a different error message to represent that. - return netip.Addr{}, ErrUnmappedMagicIP -} - -// ClientFlowCreated implements [Conn25Datapath]. -// The datapath notifies Conn25 that a flow with transitIP has been created so -// that Conn25 can prevent that transit IP and associated addresses from being -// removed from its state and returned to their pools. The returned function -// must be invoked when that flow is removed; it releases the hold on the -// address assignment the flow was counted against. It releases that hold at -// most once: calling it again is a no-op. -func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) func() { - return c.client.flowCreated(transitIP) + return ClientFlowRef{}, false, ErrUnmappedMagicIP } // ConnectorRealIPForTransitIPConnection implements [Conn25Datapath]. diff --git a/feature/conn25/conn25_test.go b/feature/conn25/conn25_test.go index 3ac07c040..da6d16a70 100644 --- a/feature/conn25/conn25_test.go +++ b/feature/conn25/conn25_test.go @@ -3416,7 +3416,7 @@ func TestHandleAddressAssignmentStoresTransitIPs(t *testing.T) { } } -func TestClientTransitIPForMagicIP(t *testing.T) { +func TestClientPrepareNewFlowForMagicIPResolution(t *testing.T) { sn := makeSelfNode(t, []appctype.AppConnectorAttr{{Name: "app1", Mode: appctype.MagicIP}}, appctype.Conn25PoolsAttr{ V4MagicIPPool: []netipx.IPRange{v4RangeFrom("0", "10")}, // 100.64.0.0 - 100.64.0.10 V6MagicIPPool: []netipx.IPRange{v6RangeFrom("0", "10")}, @@ -3439,6 +3439,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name string mip netip.Addr wantTip netip.Addr + wantOk bool wantErr error }{ { @@ -3457,6 +3458,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name: "mapped-magic-ip", mip: mappedMip, wantTip: mappedTip, + wantOk: true, wantErr: nil, }, { @@ -3475,6 +3477,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name: "v6-mapped-magic-ip", mip: v6MappedMip, wantTip: v6MappedTip, + wantOk: true, wantErr: nil, }, } { @@ -3500,10 +3503,16 @@ func TestClientTransitIPForMagicIP(t *testing.T) { }); err != nil { t.Fatal(err) } - tip, err := c.ClientTransitIPForMagicIP(tt.mip) - if tip != tt.wantTip { + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(tt.mip) + if ok { + defer ref.Release() + } + if tip := ref.TransitIP(); tip != tt.wantTip { t.Fatalf("checking transit ip: want %v, got %v", tt.wantTip, tip) } + if ok != tt.wantOk { + t.Fatalf("checking ok: want %v, got %v", tt.wantOk, ok) + } if err != tt.wantErr { t.Fatalf("checking error: want %v, got %v", tt.wantErr, err) } diff --git a/feature/conn25/datapath.go b/feature/conn25/datapath.go index 810774640..acfd8d76d 100644 --- a/feature/conn25/datapath.go +++ b/feature/conn25/datapath.go @@ -31,14 +31,6 @@ var ( // [*Conn25] is the only production implementation; the interface exists to let // datapath tests substitute a lightweight fake. type Conn25Datapath interface { - // ClientTransitIPForMagicIP returns a Transit IP for the given magicIP on a client. - // If the magicIP is within a configured Magic IP range for an app on the client, - // but not mapped to an active Transit IP, implementations should return [ErrUnmappedMagicIP]. - // If magicIP is not within a configured Magic IP range, i.e. it is not actually a Magic IP, - // implementations should return a nil error, and a zero-value [netip.Addr] to indicate - // this potentially valid, non-app-connector traffic. - ClientTransitIPForMagicIP(magicIP netip.Addr) (netip.Addr, error) - // ConnectorRealIPForTransitIPConnection returns a real destination IP for the given // srcIP and transitIP on a connector. If the transitIP is within a configured Transit IP // range for an app on the connector, but not mapped to the client at srcIP, implementations @@ -48,12 +40,18 @@ type Conn25Datapath interface { // non-app-connector traffic. ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) - // ClientFlowCreated is called before a client-side flow for transitIP has - // been installed in the client flow table, and returns a cleanup function - // that the caller must invoke when that flow is removed. The cleanup takes - // effect at most once; implementations must ignore repeat calls rather than - // release the flow's hold on its address assignment more than once. - ClientFlowCreated(transitIP netip.Addr) func() + // ClientPrepareNewFlowForMagicIP is called before a client-side flow to + // magicIP is installed in the client flow table. It reports whether it + // counted the flow against an address assignment, and if so returns the + // reference that names the Transit IP to forward that flow to and keeps + // that address from being reclaimed. + // + // When ok is true, the caller must call Release on the reference exactly + // once: when the flow is removed, or right away if it ends up installing no + // flow at all. Implementations must count nothing and return ok false and + // the zero [ClientFlowRef] otherwise, including whenever they return a + // non-nil error, since the caller does not release on those paths. + ClientPrepareNewFlowForMagicIP(magicIP netip.Addr) (ref ClientFlowRef, ok bool, err error) } // datapathHandler handles packets from the datapath, @@ -234,9 +232,16 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu // tun device, it can only be a new client flow, other (non-app-connector) traffic, // or broken return app-connector traffic on a connector, which needs to be re-established // with a new outbound packet. + // + // Resolve the Magic IP and notify Conn25 that a flow for it is being + // established, before installing that flow in the flow table. Resolving and + // counting together keeps the assignment from expiring out from under the + // flow in between, and guarantees that ClientPrepareNewFlowForMagicIP for + // this flow precedes the cleanup that fires when it is removed. magicIP := p.Dst.Addr() - transitIP, err := dh.conn25.ClientTransitIPForMagicIP(magicIP) + ref, ok, err := dh.conn25.ClientPrepareNewFlowForMagicIP(magicIP) if err != nil { + // No reference to release: an error means nothing was counted. if errors.Is(err, ErrUnmappedMagicIP) { // Couldn't find a mapping. Tell the local application the host is // unreachable so it can recover quickly instead of blackholing. @@ -246,11 +251,12 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu dh.debugLogf("error mapping magic IP, passing packet unmodified: %v", err) return filter.Accept } - - // If this is normal non-app-connector traffic, forward it along unmodified. - if !transitIP.IsValid() { + if !ok { + // This is normal non-app-connector traffic; forward it along + // unmodified. No reference to release: nothing was counted. return filter.Accept } + transitIP := ref.TransitIP() // This is a new outbound client flow. Install a DNAT MagicIP-to-TransitIP action // for the outgoing direction, and an SNAT TransitIP-to-MagicIP action for the @@ -264,15 +270,10 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu Action: dh.snatAction(magicIP), } - // Notify Conn25 that a flow for transitIP is being established before - // installing it in the flow table. This guarantees that ClientFlowCreated - // for this flow precedes the cleanup that fires when it is removed. - flowCleanup := dh.conn25.ClientFlowCreated(transitIP) - dh.clientFlowTable.NewFlow(FlowData{ FromTun: outgoing, FromWG: incoming, - OnRemove: flowCleanup, + OnRemove: ref.Release, }) outgoing.Action(p) return filter.Accept diff --git a/feature/conn25/datapath_test.go b/feature/conn25/datapath_test.go index af43c5a46..4e4d7cdc9 100644 --- a/feature/conn25/datapath_test.go +++ b/feature/conn25/datapath_test.go @@ -25,22 +25,24 @@ import ( ) type testConn25 struct { - clientTransitIPForMagicIPFn func(netip.Addr) (netip.Addr, error) + clientPrepareNewFlowForMagicIPFn func(netip.Addr) (netip.Addr, error) connectorRealIPForTransitIPConnectionFn func(netip.Addr, netip.Addr) (netip.Addr, error) } -func (tc *testConn25) ClientTransitIPForMagicIP(magicIP netip.Addr) (netip.Addr, error) { - return tc.clientTransitIPForMagicIPFn(magicIP) +// ClientPrepareNewFlowForMagicIP counts a flow, and so reports ok, exactly when +// clientPrepareNewFlowForMagicIPFn resolves magicIP to a Transit IP. +func (tc *testConn25) ClientPrepareNewFlowForMagicIP(magicIP netip.Addr) (ClientFlowRef, bool, error) { + transitIP, err := tc.clientPrepareNewFlowForMagicIPFn(magicIP) + if err != nil || !transitIP.IsValid() { + return ClientFlowRef{}, false, err + } + return ClientFlowRef{transit: transitIP}, true, nil } func (tc *testConn25) ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) { return tc.connectorRealIPForTransitIPConnectionFn(srcIP, transitIP) } -func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) func() { - return func() {} -} - // testNet is the cast of addresses the datapath tests run against: a client, // the Magic and Transit IPs it talks through, and the real server behind them, // all in a single address family. It also carries the bits needed to @@ -159,7 +161,7 @@ func (tn testNet) newClientDatapath(t *testing.T, throwMappingErr bool) *datapat t.Helper() mock := &testConn25{} - mock.clientTransitIPForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { + mock.clientPrepareNewFlowForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { if throwMappingErr { return netip.Addr{}, errors.New("synthetic mapping error") } @@ -411,9 +413,9 @@ func TestClientFlowCache(t *testing.T) { getTransitIPCalled := false mock := &testConn25{} - mock.clientTransitIPForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { + mock.clientPrepareNewFlowForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { if getTransitIPCalled { - t.Errorf("ClientGetTransitIPForMagicIP unexpectedly called more than once") + t.Errorf("ClientPrepareNewFlowForMagicIP unexpectedly called more than once") } getTransitIPCalled = true return tn.transitIP, nil @@ -597,14 +599,16 @@ func TestUnmappedTransitIPRejectWithFullInjectionQueue(t *testing.T) { tun := newFakeTUN(t) p := tn.udpPacket( netip.AddrPortFrom(tn.unknownSrcIP, clientPort), - netip.AddrPortFrom(tn.transitIP, serverPort)) + netip.AddrPortFrom(tn.transitIP, serverPort), + ) // Fill the injection queue (depth 1) and leave it unread, // as its reader may be blocked behind wireguard-go's // device.peers lock held by RemovePeer. if err := tun.TryInjectOutbound(tn.udpPacket( netip.AddrPortFrom(tn.clientSrcIP, clientPort), - netip.AddrPortFrom(tn.magicIP, serverPort)).Buffer()); err != nil { + netip.AddrPortFrom(tn.magicIP, serverPort), + ).Buffer()); err != nil { t.Fatal(err) }