From 7ed01385418cfa69df9a1d1cf91fe647ab4f77fd Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Mon, 9 Nov 2026 15:23:33 -0700 Subject: [PATCH] feature/conn25: match client flow removals to the assignment they counted (#22142) flowRemoved identified an address assignment by transit IP alone, but a transit IP does not identify an assignment for the lifetime of a flow. An assignment can be dropped while a flow created against it is still in the client flow table, and the pools then hand the same transit IP back out to an unrelated assignment. The stale removal was charged to that stranger, dropping its activeFlowCount while its own flows were still forwarding, so popExpired reclaimed both of its addresses back to the pools mid-connection. If the stranger had no flows of its own the count went to -1, which can never again satisfy popExpired's zero-flow test, pinning it in the expiry heap and leaking both addresses for the life of the process. flowCreated now returns a function, which the datapath calls on removal to clean up the flow. Fixes https://github.com/tailscale/corp/issues/50760 Change-Id: I3693c487b1031737763b5e49f8f8400c72848e8e Signed-off-by: Mazdak Nasab --- feature/conn25/client.go | 27 +++---- feature/conn25/client_test.go | 138 ++++++++++++++++++++++++++++++-- feature/conn25/conn25.go | 14 ++-- feature/conn25/datapath.go | 23 +++--- feature/conn25/datapath_test.go | 5 +- 5 files changed, 163 insertions(+), 44 deletions(-) diff --git a/feature/conn25/client.go b/feature/conn25/client.go index 6e144ec05..9b1859292 100644 --- a/feature/conn25/client.go +++ b/feature/conn25/client.go @@ -280,26 +280,25 @@ func (c *client) enqueueAddressAssignment(addrs *addrs) error { } } -func (c *client) flowCreated(transit netip.Addr) { +func (c *client) flowCreated(transit netip.Addr) func() { c.mu.Lock() defer c.mu.Unlock() entry, ok := c.assignments.byTransitIP[transit] if !ok { - return + return func() {} } entry.activeFlowCount++ -} - -func (c *client) flowRemoved(transit netip.Addr) { - c.mu.Lock() - defer c.mu.Unlock() - entry, ok := c.assignments.byTransitIP[transit] - if !ok { - return - } - entry.activeFlowCount-- - if entry.activeFlowCount == 0 { - entry.zeroFlowTime = c.assignments.clock.Now() + return func() { + c.mu.Lock() + defer c.mu.Unlock() + if entry.activeFlowCount <= 0 { + c.logf("flow removed for transit %v with no active flows", transit) + return + } + entry.activeFlowCount-- + if entry.activeFlowCount == 0 { + entry.zeroFlowTime = c.assignments.clock.Now() + } } } diff --git a/feature/conn25/client_test.go b/feature/conn25/client_test.go index 73adb0387..33d360a51 100644 --- a/feature/conn25/client_test.go +++ b/feature/conn25/client_test.go @@ -602,7 +602,7 @@ func TestReconfigDoesNotReissueInUseAddresses(t *testing.T) { // TestAddressExpiryDependsOnActiveFlows creates a Conn25 and // // 1. runs a DNS response through it -// 2. uses the ClientFlowCreated/Removed API and advances the clock +// 2. uses the ClientFlowCreated API and its cleanup function, and advances the clock // 3. runs a second DNS response through the Conn25 // 4. asserts things about the expected state of the clients assignments // table based on 1-3 @@ -696,9 +696,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // So the expired entry is removed. name: "last-flow-removed-a-while-ago", flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + removeFlow := c.ClientFlowCreated(transit) clock.Advance(30 * time.Hour) - c.ClientFlowRemoved(transit) + removeFlow() clock.Advance(3 * time.Minute) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -712,9 +712,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // And so the expired entry is not removed. name: "last-flow-recently-removed", flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + removeFlow := c.ClientFlowCreated(transit) clock.Advance(30 * time.Hour) - c.ClientFlowRemoved(transit) + removeFlow() clock.Advance(1 * time.Second) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -813,6 +813,134 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { } } +// TestFlowRemovedAfterTransitIPReassigned checks that a flow removal arriving +// after its assignment was dropped, and the transit IP handed back out, leaves +// the new assignment alone. The datapath creates flows on tun goroutines while +// [client.reset] runs on the profile-change path, so a flow created just before +// a reset can outlive the assignment it was counted against. +func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, arbitraryPools, nil) + + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + transit := before.connectorAddr.transit + + // A flow is established against that assignment, and then the profile + // changes and the client's state is reset out from under the flow. + removeFlow := c.ClientFlowCreated(transit) + c.client.reset() + + // reset rebuilds the pools, so iteration restarts and the next assignment + // is handed the same transit IP, now for a different destination. + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit) + } + if after == before { + t.Fatal("expected a distinct assignment after reset") + } + + // The pre-reset flow is finally torn down. The removal refers to an + // assignment that no longer exists, so it must not touch the new one. + removeFlow() + + if got := after.activeFlowCount; got != 0 { + t.Errorf("activeFlowCount for reissued transit %v = %d, want 0", transit, got) + } + + // A zero flow count must leave the reissued assignment able to expire and + // return its addresses to the pools. A negative count can never satisfy + // popExpired's zero-flow test, which would pin it in the expiry heap. + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + if got := c.client.assignments.popExpired(clock.Now()); got != after { + t.Errorf("popExpired = %+v, want the reissued assignment for %v", got, dstTwo) + } +} + +// TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse is +// [TestFlowRemovedAfterTransitIPReassigned] where the reissued assignment has a +// live flow of its own. A removal misattributed to it would drop its count to +// zero while that flow is still forwarding, reclaiming both of its addresses +// back to the pools mid-connection. Clamping the count at zero does not catch +// this, because the misattributed decrement lands on a positive count. +func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, arbitraryPools, nil) + + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + transit := before.connectorAddr.transit + + removeStaleFlow := c.ClientFlowCreated(transit) + c.client.reset() + + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit) + } + if after == before { + t.Fatal("expected a distinct assignment after reset") + } + + // The reissued assignment picks up a flow of its own, which is still live + // when the pre-reset flow is torn down. + c.ClientFlowCreated(transit) + if got := after.activeFlowCount; got != 1 { + t.Fatalf("activeFlowCount for the reissued assignment = %d, want 1", got) + } + removeStaleFlow() + if got := after.activeFlowCount; got != 1 { + t.Errorf("activeFlowCount after the stale removal = %d, want 1", got) + } + + // With a flow still active, the assignment must hold its addresses. + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + if got := c.client.assignments.popExpired(clock.Now()); got != nil { + t.Errorf("popExpired = %+v, want nil; the assignment for %v is still in use", got, dstTwo) + } +} + // TestResendTransitIPMappingConcurrent races resendTransitIPMapping, against a writer of client.assignments. func TestResendTransitIPMappingConcurrent(t *testing.T) { const appName = "a" diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index 9b78011fb..224370ce2 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -325,15 +325,11 @@ func (c *Conn25) ClientTransitIPForMagicIP(m netip.Addr) (netip.Addr, error) { // ClientFlowCreated implements [Conn25Datapath]. // The datapath notifies Conn25 that a flow with transitIP has been created so // that Conn25 can prevent that transit IP and associated addresses from being -// removed from its state and returned to their pools. -func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) { - c.client.flowCreated(transitIP) -} - -// ClientFlowRemoved implements [Conn25Datapath]. -// See [Conn25.ClientFlowCreated]. -func (c *Conn25) ClientFlowRemoved(transitIP netip.Addr) { - c.client.flowRemoved(transitIP) +// removed from its state and returned to their pools. The returned function +// must be invoked exactly once when that flow is removed; it releases the hold +// on the address assignment the flow was counted against. +func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) func() { + return c.client.flowCreated(transitIP) } // ConnectorRealIPForTransitIPConnection implements [Conn25Datapath]. diff --git a/feature/conn25/datapath.go b/feature/conn25/datapath.go index 3f0bcf2d6..78ed69fed 100644 --- a/feature/conn25/datapath.go +++ b/feature/conn25/datapath.go @@ -48,13 +48,10 @@ type Conn25Datapath interface { // non-app-connector traffic. ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) - // ClientFlowCreated is called after a client-side flow for transitIP has - // been installed in the client flow table. - ClientFlowCreated(transitIP netip.Addr) - // ClientFlowRemoved is called after such a flow is removed. For each - // flow installed in the client flow table, ClientFlowCreated is called - // before any ClientFlowRemoved that fires for it. - ClientFlowRemoved(transitIP netip.Addr) + // ClientFlowCreated is called before a client-side flow for transitIP has + // been installed in the client flow table, and returns a cleanup function that + // the caller must invoke exactly once when that flow is removed. + ClientFlowCreated(transitIP netip.Addr) func() } // datapathHandler handles packets from the datapath, @@ -267,15 +264,13 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu // Notify Conn25 that a flow for transitIP is being established before // installing it in the flow table. This guarantees that ClientFlowCreated - // for this flow precedes any ClientFlowRemoved that fires for it. - dh.conn25.ClientFlowCreated(transitIP) + // for this flow precedes the cleanup that fires when it is removed. + flowCleanup := dh.conn25.ClientFlowCreated(transitIP) dh.clientFlowTable.NewFlow(FlowData{ - FromTun: outgoing, - FromWG: incoming, - OnRemove: func() { - dh.conn25.ClientFlowRemoved(transitIP) - }, + FromTun: outgoing, + FromWG: incoming, + OnRemove: flowCleanup, }) outgoing.Action(p) return filter.Accept diff --git a/feature/conn25/datapath_test.go b/feature/conn25/datapath_test.go index 4290cf75d..af43c5a46 100644 --- a/feature/conn25/datapath_test.go +++ b/feature/conn25/datapath_test.go @@ -37,8 +37,9 @@ func (tc *testConn25) ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, tr return tc.connectorRealIPForTransitIPConnectionFn(srcIP, transitIP) } -func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) {} -func (tc *testConn25) ClientFlowRemoved(transitIP netip.Addr) {} +func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) func() { + return func() {} +} // testNet is the cast of addresses the datapath tests run against: a client, // the Magic and Transit IPs it talks through, and the real server behind them,