feature/conn25: match client flow removals to the assignment they counted (#22142)

flowRemoved identified an address assignment by transit IP alone, but a
transit IP does not identify an assignment for the lifetime of a flow.
An assignment can be dropped while a flow created against it is still in
the client flow table, and the pools then hand the same transit IP back
out to an unrelated assignment. The stale removal was charged to that
stranger, dropping its activeFlowCount while its own flows were still
forwarding, so popExpired reclaimed both of its addresses back to the
pools mid-connection. If the stranger had no flows of its own the count
went to -1, which can never again satisfy popExpired's zero-flow test,
pinning it in the expiry heap and leaking both addresses for the life of
the process.

flowCreated now returns a function, which the datapath calls on removal
to clean up the flow.

Fixes https://github.com/tailscale/corp/issues/50760

Change-Id: I3693c487b1031737763b5e49f8f8400c72848e8e

Signed-off-by: Mazdak Nasab <[email protected]>
This commit is contained in:
Mazdak Nasab authored and GitHub committed 2026-11-09 15:23:33 -07:00
1 parent 6372b9a43a
commit 7ed0138541
5 files changed
+163 -44

No files matched your search

+13 -14
View File
@@ -280,26 +280,25 @@ func (c *client) enqueueAddressAssignment(addrs *addrs) error {
}
}
func (c *client) flowCreated(transit netip.Addr) {
func (c *client) flowCreated(transit netip.Addr) func() {
c.mu.Lock()
defer c.mu.Unlock()
entry, ok := c.assignments.byTransitIP[transit]
if !ok {
return
return func() {}
}
entry.activeFlowCount++
}
func (c *client) flowRemoved(transit netip.Addr) {
c.mu.Lock()
defer c.mu.Unlock()
entry, ok := c.assignments.byTransitIP[transit]
if !ok {
return
}
entry.activeFlowCount--
if entry.activeFlowCount == 0 {
entry.zeroFlowTime = c.assignments.clock.Now()
return func() {
c.mu.Lock()
defer c.mu.Unlock()
if entry.activeFlowCount <= 0 {
c.logf("flow removed for transit %v with no active flows", transit)
return
}
entry.activeFlowCount--
if entry.activeFlowCount == 0 {
entry.zeroFlowTime = c.assignments.clock.Now()
}
}
}
+133 -5
View File
@@ -602,7 +602,7 @@ func TestReconfigDoesNotReissueInUseAddresses(t *testing.T) {
// TestAddressExpiryDependsOnActiveFlows creates a Conn25 and
//
// 1. runs a DNS response through it
// 2. uses the ClientFlowCreated/Removed API and advances the clock
// 2. uses the ClientFlowCreated API and its cleanup function, and advances the clock
// 3. runs a second DNS response through the Conn25
// 4. asserts things about the expected state of the clients assignments
// table based on 1-3
@@ -696,9 +696,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) {
// So the expired entry is removed.
name: "last-flow-removed-a-while-ago",
flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) {
c.ClientFlowCreated(transit)
removeFlow := c.ClientFlowCreated(transit)
clock.Advance(30 * time.Hour)
c.ClientFlowRemoved(transit)
removeFlow()
clock.Advance(3 * time.Minute)
},
wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}),
@@ -712,9 +712,9 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) {
// And so the expired entry is not removed.
name: "last-flow-recently-removed",
flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) {
c.ClientFlowCreated(transit)
removeFlow := c.ClientFlowCreated(transit)
clock.Advance(30 * time.Hour)
c.ClientFlowRemoved(transit)
removeFlow()
clock.Advance(1 * time.Second)
},
wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}),
@@ -813,6 +813,134 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) {
}
}
// TestFlowRemovedAfterTransitIPReassigned checks that a flow removal arriving
// after its assignment was dropped, and the transit IP handed back out, leaves
// the new assignment alone. The datapath creates flows on tun goroutines while
// [client.reset] runs on the profile-change path, so a flow created just before
// a reset can outlive the assignment it was counted against.
func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) {
const appName = "app1"
const domain = dnsname.FQDN("example.com.")
const ttl = 5 * time.Minute
dstOne := netip.MustParseAddr("1.0.0.1")
dstTwo := netip.MustParseAddr("1.0.0.2")
sn := makeSelfNode(t, []appctype.AppConnectorAttr{{
Name: appName,
Connectors: []string{"tag:woo"},
Domains: []string{"example.com"},
Mode: appctype.MagicIP,
}}, arbitraryPools, nil)
c := newConn25(logger.Discard)
clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()})
c.client.assignments.clock = clock
c.reconfig(mustConfig(t, sn))
before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "")
if err != nil {
t.Fatal(err)
}
transit := before.connectorAddr.transit
// A flow is established against that assignment, and then the profile
// changes and the client's state is reset out from under the flow.
removeFlow := c.ClientFlowCreated(transit)
c.client.reset()
// reset rebuilds the pools, so iteration restarts and the next assignment
// is handed the same transit IP, now for a different destination.
after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "")
if err != nil {
t.Fatal(err)
}
if after.connectorAddr.transit != transit {
t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit)
}
if after == before {
t.Fatal("expected a distinct assignment after reset")
}
// The pre-reset flow is finally torn down. The removal refers to an
// assignment that no longer exists, so it must not touch the new one.
removeFlow()
if got := after.activeFlowCount; got != 0 {
t.Errorf("activeFlowCount for reissued transit %v = %d, want 0", transit, got)
}
// A zero flow count must leave the reissued assignment able to expire and
// return its addresses to the pools. A negative count can never satisfy
// popExpired's zero-flow test, which would pin it in the expiry heap.
clock.Advance(ttl + deadFlowWaitTimeout + time.Minute)
if got := c.client.assignments.popExpired(clock.Now()); got != after {
t.Errorf("popExpired = %+v, want the reissued assignment for %v", got, dstTwo)
}
}
// TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse is
// [TestFlowRemovedAfterTransitIPReassigned] where the reissued assignment has a
// live flow of its own. A removal misattributed to it would drop its count to
// zero while that flow is still forwarding, reclaiming both of its addresses
// back to the pools mid-connection. Clamping the count at zero does not catch
// this, because the misattributed decrement lands on a positive count.
func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) {
const appName = "app1"
const domain = dnsname.FQDN("example.com.")
const ttl = 5 * time.Minute
dstOne := netip.MustParseAddr("1.0.0.1")
dstTwo := netip.MustParseAddr("1.0.0.2")
sn := makeSelfNode(t, []appctype.AppConnectorAttr{{
Name: appName,
Connectors: []string{"tag:woo"},
Domains: []string{"example.com"},
Mode: appctype.MagicIP,
}}, arbitraryPools, nil)
c := newConn25(logger.Discard)
clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()})
c.client.assignments.clock = clock
c.reconfig(mustConfig(t, sn))
before, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "")
if err != nil {
t.Fatal(err)
}
transit := before.connectorAddr.transit
removeStaleFlow := c.ClientFlowCreated(transit)
c.client.reset()
after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "")
if err != nil {
t.Fatal(err)
}
if after.connectorAddr.transit != transit {
t.Fatalf("transit %v was not reissued after reset (got %v); this test no longer covers reassignment", transit, after.connectorAddr.transit)
}
if after == before {
t.Fatal("expected a distinct assignment after reset")
}
// The reissued assignment picks up a flow of its own, which is still live
// when the pre-reset flow is torn down.
c.ClientFlowCreated(transit)
if got := after.activeFlowCount; got != 1 {
t.Fatalf("activeFlowCount for the reissued assignment = %d, want 1", got)
}
removeStaleFlow()
if got := after.activeFlowCount; got != 1 {
t.Errorf("activeFlowCount after the stale removal = %d, want 1", got)
}
// With a flow still active, the assignment must hold its addresses.
clock.Advance(ttl + deadFlowWaitTimeout + time.Minute)
if got := c.client.assignments.popExpired(clock.Now()); got != nil {
t.Errorf("popExpired = %+v, want nil; the assignment for %v is still in use", got, dstTwo)
}
}
// TestResendTransitIPMappingConcurrent races resendTransitIPMapping, against a writer of client.assignments.
func TestResendTransitIPMappingConcurrent(t *testing.T) {
const appName = "a"
+5 -9
View File
@@ -325,15 +325,11 @@ func (c *Conn25) ClientTransitIPForMagicIP(m netip.Addr) (netip.Addr, error) {
// ClientFlowCreated implements [Conn25Datapath].
// The datapath notifies Conn25 that a flow with transitIP has been created so
// that Conn25 can prevent that transit IP and associated addresses from being
// removed from its state and returned to their pools.
func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) {
c.client.flowCreated(transitIP)
}
// ClientFlowRemoved implements [Conn25Datapath].
// See [Conn25.ClientFlowCreated].
func (c *Conn25) ClientFlowRemoved(transitIP netip.Addr) {
c.client.flowRemoved(transitIP)
// removed from its state and returned to their pools. The returned function
// must be invoked exactly once when that flow is removed; it releases the hold
// on the address assignment the flow was counted against.
func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) func() {
return c.client.flowCreated(transitIP)
}
// ConnectorRealIPForTransitIPConnection implements [Conn25Datapath].
+9 -14
View File
@@ -48,13 +48,10 @@ type Conn25Datapath interface {
// non-app-connector traffic.
ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error)
// ClientFlowCreated is called after a client-side flow for transitIP has
// been installed in the client flow table.
ClientFlowCreated(transitIP netip.Addr)
// ClientFlowRemoved is called after such a flow is removed. For each
// flow installed in the client flow table, ClientFlowCreated is called
// before any ClientFlowRemoved that fires for it.
ClientFlowRemoved(transitIP netip.Addr)
// ClientFlowCreated is called before a client-side flow for transitIP has
// been installed in the client flow table, and returns a cleanup function that
// the caller must invoke exactly once when that flow is removed.
ClientFlowCreated(transitIP netip.Addr) func()
}
// datapathHandler handles packets from the datapath,
@@ -267,15 +264,13 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu
// Notify Conn25 that a flow for transitIP is being established before
// installing it in the flow table. This guarantees that ClientFlowCreated
// for this flow precedes any ClientFlowRemoved that fires for it.
dh.conn25.ClientFlowCreated(transitIP)
// for this flow precedes the cleanup that fires when it is removed.
flowCleanup := dh.conn25.ClientFlowCreated(transitIP)
dh.clientFlowTable.NewFlow(FlowData{
FromTun: outgoing,
FromWG: incoming,
OnRemove: func() {
dh.conn25.ClientFlowRemoved(transitIP)
},
FromTun: outgoing,
FromWG: incoming,
OnRemove: flowCleanup,
})
outgoing.Action(p)
return filter.Accept
+3 -2
View File
@@ -37,8 +37,9 @@ func (tc *testConn25) ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, tr
return tc.connectorRealIPForTransitIPConnectionFn(srcIP, transitIP)
}
func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) {}
func (tc *testConn25) ClientFlowRemoved(transitIP netip.Addr) {}
func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) func() {
return func() {}
}
// testNet is the cast of addresses the datapath tests run against: a client,
// the Magic and Transit IPs it talks through, and the real server behind them,