diff --git a/feature/conn25/client.go b/feature/conn25/client.go index 6e144ec05..9b1859292 100644 --- a/feature/conn25/client.go +++ b/feature/conn25/client.go @@ -280,26 +280,25 @@ func (c *client) enqueueAddressAssignment(addrs *addrs) error { } } -func (c *client) flowCreated(transit netip.Addr) { +func (c *client) flowCreated(transit netip.Addr) func() { c.mu.Lock() defer c.mu.Unlock() entry, ok := c.assignments.byTransitIP[transit] if !ok { - return + return func() {} } entry.activeFlowCount++ -} - -func (c *client) flowRemoved(transit netip.Addr) { - c.mu.Lock() - defer c.mu.Unlock() - entry, ok := c.assignments.byTransitIP[transit] - if !ok { - return - } - entry.activeFlowCount-- - if entry.activeFlowCount == 0 { - entry.zeroFlowTime = c.assignments.clock.Now() + return func() { + c.mu.Lock() + defer c.mu.Unlock() + if entry.activeFlowCount <= 0 { + c.logf("flow removed for transit %v with no active flows", transit) + return + } + entry.activeFlowCount-- + if entry.activeFlowCount == 0 { + entry.zeroFlowTime = c.assignments.clock.Now() + } } } diff --git a/feature/conn25/client_test.go b/feature/conn25/client_test.go index 73adb0387..33d360a51 100644 --- a/feature/conn25/client_test.go +++ b/feature/conn25/client_test.go @@ -602,7 +602,7 @@ func TestReconfigDoesNotReissueInUseAddresses(t *testing.T) { // TestAddressExpiryDependsOnActiveFlows creates a Conn25 and // // 1. runs a DNS response through it -// 2. uses the ClientFlowCreated/Removed API and advances the clock +// 2. uses the ClientFlowCreated API and its cleanup function, and advances the clock // 3. runs a second DNS response through the Conn25 // 4. asserts things about the expected state of the clients assignments // table based on 1-3 @@ -696,9 +696,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // So the expired entry is removed. name: "last-flow-removed-a-while-ago", flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + removeFlow := c.ClientFlowCreated(transit) clock.Advance(30 * time.Hour) - c.ClientFlowRemoved(transit) + removeFlow() clock.Advance(3 * time.Minute) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -712,9 +712,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // And so the expired entry is not removed. name: "last-flow-recently-removed", flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + removeFlow := c.ClientFlowCreated(transit) clock.Advance(30 * time.Hour) - c.ClientFlowRemoved(transit) + removeFlow() clock.Advance(1 * time.Second) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -813,6 +813,134 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { } } +// TestFlowRemovedAfterTransitIPReassigned checks that a flow removal arriving +// after its assignment was dropped, and the transit IP handed back out, leaves +// the new assignment alone. The datapath creates flows on tun goroutines while +// [client.reset] runs on the profile-change path, so a flow created just before +// a reset can outlive the assignment it was counted against. +func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, arbitraryPools, nil) + + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + transit := before.connectorAddr.transit + + // A flow is established against that assignment, and then the profile + // changes and the client's state is reset out from under the flow. + removeFlow := c.ClientFlowCreated(transit) + c.client.reset() + + // reset rebuilds the pools, so iteration restarts and the next assignment + // is handed the same transit IP, now for a different destination. + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit) + } + if after == before { + t.Fatal("expected a distinct assignment after reset") + } + + // The pre-reset flow is finally torn down. The removal refers to an + // assignment that no longer exists, so it must not touch the new one. + removeFlow() + + if got := after.activeFlowCount; got != 0 { + t.Errorf("activeFlowCount for reissued transit %v = %d, want 0", transit, got) + } + + // A zero flow count must leave the reissued assignment able to expire and + // return its addresses to the pools. A negative count can never satisfy + // popExpired's zero-flow test, which would pin it in the expiry heap. + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + if got := c.client.assignments.popExpired(clock.Now()); got != after { + t.Errorf("popExpired = %+v, want the reissued assignment for %v", got, dstTwo) + } +} + +// TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse is +// [TestFlowRemovedAfterTransitIPReassigned] where the reissued assignment has a +// live flow of its own. A removal misattributed to it would drop its count to +// zero while that flow is still forwarding, reclaiming both of its addresses +// back to the pools mid-connection. Clamping the count at zero does not catch +// this, because the misattributed decrement lands on a positive count. +func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, arbitraryPools, nil) + + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + transit := before.connectorAddr.transit + + removeStaleFlow := c.ClientFlowCreated(transit) + c.client.reset() + + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit) + } + if after == before { + t.Fatal("expected a distinct assignment after reset") + } + + // The reissued assignment picks up a flow of its own, which is still live + // when the pre-reset flow is torn down. + c.ClientFlowCreated(transit) + if got := after.activeFlowCount; got != 1 { + t.Fatalf("activeFlowCount for the reissued assignment = %d, want 1", got) + } + removeStaleFlow() + if got := after.activeFlowCount; got != 1 { + t.Errorf("activeFlowCount after the stale removal = %d, want 1", got) + } + + // With a flow still active, the assignment must hold its addresses. + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + if got := c.client.assignments.popExpired(clock.Now()); got != nil { + t.Errorf("popExpired = %+v, want nil; the assignment for %v is still in use", got, dstTwo) + } +} + // TestResendTransitIPMappingConcurrent races resendTransitIPMapping, against a writer of client.assignments. func TestResendTransitIPMappingConcurrent(t *testing.T) { const appName = "a" diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index 9b78011fb..224370ce2 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -325,15 +325,11 @@ func (c *Conn25) ClientTransitIPForMagicIP(m netip.Addr) (netip.Addr, error) { // ClientFlowCreated implements [Conn25Datapath]. // The datapath notifies Conn25 that a flow with transitIP has been created so // that Conn25 can prevent that transit IP and associated addresses from being -// removed from its state and returned to their pools. -func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) { - c.client.flowCreated(transitIP) -} - -// ClientFlowRemoved implements [Conn25Datapath]. -// See [Conn25.ClientFlowCreated]. -func (c *Conn25) ClientFlowRemoved(transitIP netip.Addr) { - c.client.flowRemoved(transitIP) +// removed from its state and returned to their pools. The returned function +// must be invoked exactly once when that flow is removed; it releases the hold +// on the address assignment the flow was counted against. +func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) func() { + return c.client.flowCreated(transitIP) } // ConnectorRealIPForTransitIPConnection implements [Conn25Datapath]. diff --git a/feature/conn25/datapath.go b/feature/conn25/datapath.go index 3f0bcf2d6..78ed69fed 100644 --- a/feature/conn25/datapath.go +++ b/feature/conn25/datapath.go @@ -48,13 +48,10 @@ type Conn25Datapath interface { // non-app-connector traffic. ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) - // ClientFlowCreated is called after a client-side flow for transitIP has - // been installed in the client flow table. - ClientFlowCreated(transitIP netip.Addr) - // ClientFlowRemoved is called after such a flow is removed. For each - // flow installed in the client flow table, ClientFlowCreated is called - // before any ClientFlowRemoved that fires for it. - ClientFlowRemoved(transitIP netip.Addr) + // ClientFlowCreated is called before a client-side flow for transitIP has + // been installed in the client flow table, and returns a cleanup function that + // the caller must invoke exactly once when that flow is removed. + ClientFlowCreated(transitIP netip.Addr) func() } // datapathHandler handles packets from the datapath, @@ -267,15 +264,13 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu // Notify Conn25 that a flow for transitIP is being established before // installing it in the flow table. This guarantees that ClientFlowCreated - // for this flow precedes any ClientFlowRemoved that fires for it. - dh.conn25.ClientFlowCreated(transitIP) + // for this flow precedes the cleanup that fires when it is removed. + flowCleanup := dh.conn25.ClientFlowCreated(transitIP) dh.clientFlowTable.NewFlow(FlowData{ - FromTun: outgoing, - FromWG: incoming, - OnRemove: func() { - dh.conn25.ClientFlowRemoved(transitIP) - }, + FromTun: outgoing, + FromWG: incoming, + OnRemove: flowCleanup, }) outgoing.Action(p) return filter.Accept diff --git a/feature/conn25/datapath_test.go b/feature/conn25/datapath_test.go index 4290cf75d..af43c5a46 100644 --- a/feature/conn25/datapath_test.go +++ b/feature/conn25/datapath_test.go @@ -37,8 +37,9 @@ func (tc *testConn25) ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, tr return tc.connectorRealIPForTransitIPConnectionFn(srcIP, transitIP) } -func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) {} -func (tc *testConn25) ClientFlowRemoved(transitIP netip.Addr) {} +func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) func() { + return func() {} +} // testNet is the cast of addresses the datapath tests run against: a client, // the Magic and Transit IPs it talks through, and the real server behind them,