diff --git a/feature/conn25/client.go b/feature/conn25/client.go index 22826f49d..26e6516c0 100644 --- a/feature/conn25/client.go +++ b/feature/conn25/client.go @@ -42,18 +42,6 @@ type client struct { byConnKey map[key.NodePublic]set.Set[netip.Prefix] } -// transitIPForMagicIP is part of the implementation of the [Conn25Datapath] interface for dataflow lookups. -// See also [Conn25Datapath.ClientTransitIPForMagicIP]. -func (c *client) transitIPForMagicIP(magicIP netip.Addr) (netip.Addr, bool) { - c.mu.Lock() - defer c.mu.Unlock() - v, ok := c.assignments.lookupByMagicIP(magicIP) - if ok { - return v.connectorAddr.transit, true - } - return netip.Addr{}, false -} - // linkLocalAllow returns true if the provided packet with a link-local Dst address has a // Dst that is one of our transit IPs, and false otherwise. // Tailscale's wireguard filters drop link-local unicast packets (see [wgengine/filter/filter.go]) @@ -280,16 +268,43 @@ func (c *client) enqueueAddressAssignment(addrs *addrs) error { } } -func (c *client) flowCreated(transit netip.Addr) func() { +// ClientFlowRef is a client-side flow's reference to the address assignment it was counted against. +type ClientFlowRef struct { + transit netip.Addr + release func() +} + +// TransitIP returns the Transit IP of the assignment the flow was counted +// against, which is the address its packets should be forwarded to. +func (r ClientFlowRef) TransitIP() netip.Addr { + return r.transit +} + +// Release gives up the flow's claim on the address assignment, letting the +// assignment be reclaimed once nothing else holds it. It must be called once +// the flow is gone, or right away if the caller ends up installing no flow at all. +func (r ClientFlowRef) Release() { + if r.release != nil { + r.release() + } +} + +// PrepareNewFlowForMagicIP resolves magicIP to the transit IP currently assigned +// to it and records that a flow is being established against that assignment, +// returning the reference that both names that transit IP and keeps the +// assignment from being reclaimed. It reports whether magicIP had an +// assignment at all. +func (c *client) PrepareNewFlowForMagicIP(magicIP netip.Addr) (ClientFlowRef, bool) { c.mu.Lock() defer c.mu.Unlock() - entry, ok := c.assignments.byTransitIP[transit] + entry, ok := c.assignments.lookupByMagicIP(magicIP) if !ok { - return func() {} + return ClientFlowRef{}, false } + transit := entry.connectorAddr.transit entry.activeFlowCount++ var released bool - return func() { + return ClientFlowRef{transit: transit, release: func() { c.mu.Lock() defer c.mu.Unlock() if released { @@ -304,7 +319,7 @@ func (c *client) flowCreated(transit netip.Addr) func() { if entry.activeFlowCount == 0 { entry.zeroFlowTime = c.assignments.clock.Now() } - } + }}, true } func (c *client) extraWireGuardAllowedIPs(k key.NodePublic) views.Slice[netip.Prefix] { diff --git a/feature/conn25/client_test.go b/feature/conn25/client_test.go index ff24278c6..19c9112f0 100644 --- a/feature/conn25/client_test.go +++ b/feature/conn25/client_test.go @@ -602,7 +602,8 @@ func TestReconfigDoesNotReissueInUseAddresses(t *testing.T) { // TestAddressExpiryDependsOnActiveFlows creates a Conn25 and // // 1. runs a DNS response through it -// 2. uses the ClientFlowCreated API and its cleanup function, and advances the clock +// 2. uses the ClientPrepareNewFlowForMagicIP API and its cleanup function, +// and advances the clock // 3. runs a second DNS response through the Conn25 // 4. asserts things about the expected state of the clients assignments // table based on 1-3 @@ -666,7 +667,7 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // and so we expect the entry for ipOne to be removed, and the entry for // ipTwo to be present. name: "flows-zero", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -680,8 +681,8 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // ipOne. // And so that entry does not get expired. name: "flows-not-zero", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -695,10 +696,10 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // client flow after creating it. // So the expired entry is removed. name: "last-flow-removed-a-while-ago", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - removeFlow := c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + ref, _, _ := c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) - removeFlow() + ref.Release() clock.Advance(3 * time.Minute) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipTwo}), @@ -711,10 +712,10 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // within the cooldown period. // And so the expired entry is not removed. name: "last-flow-recently-removed", - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - removeFlow := c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + ref, _, _ := c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) - removeFlow() + ref.Release() clock.Advance(1 * time.Second) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne, ipTwo}), @@ -728,8 +729,8 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { // So the entry is not removed. name: "repeated-response-with-expired-and-active-flow", secondDNSResponse: dnsRespIPOne, - flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, transit netip.Addr) { - c.ClientFlowCreated(transit) + flowsAndTimeFx: func(c *Conn25, clock *tstest.Clock, magic netip.Addr) { + c.ClientPrepareNewFlowForMagicIP(magic) clock.Advance(30 * time.Hour) }, wantUnexpiredDstIPs: set.SetOf([]netip.Addr{ipOne}), @@ -773,7 +774,7 @@ func TestAddressExpiryDependsOnActiveFlows(t *testing.T) { } // there are client flows and time passes - tt.flowsAndTimeFx(c, clock, c.client.assignments.byDomainDst[ipOneDD].connectorAddr.transit) + tt.flowsAndTimeFx(c, clock, c.client.assignments.byDomainDst[ipOneDD].magic) // then a second dns response dnsR2 := tt.secondDNSResponse @@ -845,7 +846,10 @@ func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { // A flow is established against that assignment, and then the profile // changes and the client's state is reset out from under the flow. - removeFlow := c.ClientFlowCreated(transit) + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } c.client.reset() // reset rebuilds the pools, so iteration restarts and the next assignment @@ -863,7 +867,7 @@ func TestFlowRemovedAfterTransitIPReassigned(t *testing.T) { // The pre-reset flow is finally torn down. The removal refers to an // assignment that no longer exists, so it must not touch the new one. - removeFlow() + ref.Release() if got := after.activeFlowCount; got != 0 { t.Errorf("activeFlowCount for reissued transit %v = %d, want 0", transit, got) @@ -909,7 +913,10 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { } transit := before.connectorAddr.transit - removeStaleFlow := c.ClientFlowCreated(transit) + staleRef, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } c.client.reset() after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") @@ -925,11 +932,13 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { // The reissued assignment picks up a flow of its own, which is still live // when the pre-reset flow is torn down. - c.ClientFlowCreated(transit) + if _, ok, err := c.ClientPrepareNewFlowForMagicIP(after.magic); err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", after.magic, ok, err) + } if got := after.activeFlowCount; got != 1 { t.Fatalf("activeFlowCount for the reissued assignment = %d, want 1", got) } - removeStaleFlow() + staleRef.Release() if got := after.activeFlowCount; got != 1 { t.Errorf("activeFlowCount after the stale removal = %d, want 1", got) } @@ -941,10 +950,10 @@ func TestFlowRemovedDoesNotReclaimReissuedTransitIPInUse(t *testing.T) { } } -// TestFlowCleanupRunsOnce checks that a flow's cleanup releases its hold on the -// assignment at most once, so a caller that runs the same cleanup more than -// once cannot undercount the flows still forwarding over that assignment and -// have its addresses reclaimed back to the pools mid-connection. +// TestFlowCleanupRunsOnce checks that a flow's cleanup releases its reference +// to the assignment at most once, so a caller that runs the same cleanup more +// than once cannot undercount the flows still forwarding over that assignment +// and have its addresses reclaimed back to the pools mid-connection. func TestFlowCleanupRunsOnce(t *testing.T) { const appName = "app1" const domain = dnsname.FQDN("example.com.") @@ -967,19 +976,23 @@ func TestFlowCleanupRunsOnce(t *testing.T) { if err != nil { t.Fatal(err) } - transit := as.connectorAddr.transit - // Two flows over the same transit IP. The first is torn down; the second + // Two flows over the same assignment. The first is torn down; the second // stays live. - removeFirstFlow := c.ClientFlowCreated(transit) - c.ClientFlowCreated(transit) + firstRef, ok, err := c.ClientPrepareNewFlowForMagicIP(as.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", as.magic, ok, err) + } + if _, ok, err := c.ClientPrepareNewFlowForMagicIP(as.magic); err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", as.magic, ok, err) + } if got := as.activeFlowCount; got != 2 { t.Fatalf("activeFlowCount = %d, want 2", got) } - removeFirstFlow() - removeFirstFlow() - removeFirstFlow() + firstRef.Release() + firstRef.Release() + firstRef.Release() if got := as.activeFlowCount; got != 1 { t.Errorf("activeFlowCount after repeated cleanup of one flow = %d, want 1", got) } @@ -991,6 +1004,103 @@ func TestFlowCleanupRunsOnce(t *testing.T) { } } +// singleTransitIPPools has exactly one Transit IP per address family, so that +// the next reservation after one is returned to the pool is handed that same address back. +var singleTransitIPPools = appctype.Conn25PoolsAttr{ + V4MagicIPPool: []netipx.IPRange{v4RangeFrom("0", "10")}, + V6MagicIPPool: []netipx.IPRange{v6RangeFrom("0", "10")}, + V4TransitIPPool: []netipx.IPRange{v4RangeFrom("40", "40")}, + V6TransitIPPool: []netipx.IPRange{v6RangeFrom("40", "40")}, +} + +// TestClientPrepareNewFlowForMagicIPPinsAssignment checks that resolving a +// Magic IP for a new flow and counting that flow against its assignment happen +// as one step. The datapath DNATs the flow to the Transit IP this returns, so +// a window between resolving that address and counting the flow would let a +// concurrent reservation expire the assignment and hand the Transit IP to an +// unrelated destination, silently forwarding the flow to the wrong host. +// +// The window is stood in for here by expiring the assignment before resolving +// it: a reservation that runs afterwards sweeps it with popExpired. The test +// first runs that sequence without resolving the Magic IP, to show that the +// sweep really does reissue the address, so that the pinned half is not passing +// vacuously. +func TestClientPrepareNewFlowForMagicIPPinsAssignment(t *testing.T) { + const appName = "app1" + const domain = dnsname.FQDN("example.com.") + const ttl = 5 * time.Minute + dstOne := netip.MustParseAddr("1.0.0.1") + dstTwo := netip.MustParseAddr("1.0.0.2") + + sn := makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: appName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, singleTransitIPPools, nil) + + // newExpiredAssignment returns a fresh Conn25 holding a single assignment, + // for dstOne, that is past its expiry and so free for the next + // reservation's popExpired sweep to reclaim and return to the pools. + newExpiredAssignment := func(t *testing.T) (*Conn25, *addrs) { + t.Helper() + c := newConn25(logger.Discard) + clock := tstest.NewClock(tstest.ClockOpts{Start: time.Now()}) + c.client.assignments.clock = clock + c.reconfig(mustConfig(t, sn)) + + as, err := c.client.reserveAddresses(appName, domain, dstOne, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + clock.Advance(ttl + deadFlowWaitTimeout + time.Minute) + return c, as + } + + // First, without a flow to pin it: the assignment is swept and its Transit + // IP goes straight back out to a different destination. This is the + // reassignment the pinned half below has to prevent. + c, before := newExpiredAssignment(t) + transit := before.connectorAddr.transit + + after, err := c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if err != nil { + t.Fatal(err) + } + if after.connectorAddr.transit != transit { + t.Fatalf("transit %v was not reissued to %v (got %v); this test no longer covers reassignment", transit, dstTwo, after.connectorAddr.transit) + } + + // Then the same sequence, with the Magic IP resolved through the + // flow-creating API before the second reservation runs. + c, before = newExpiredAssignment(t) + transit = before.connectorAddr.transit + + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if err != nil || !ok { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) = (_, %v, %v), want (_, true, nil)", before.magic, ok, err) + } + if got := ref.TransitIP(); got != transit { + t.Fatalf("ClientPrepareNewFlowForMagicIP(%v) transit = %v, want %v", before.magic, got, transit) + } + if got := before.activeFlowCount; got != 1 { + t.Fatalf("activeFlowCount for the assignment for %v = %d, want 1", dstOne, got) + } + + // The only Transit IP is carrying a flow, so there is nothing left to + // assign. Refusing the reservation is the point: the alternative is handing + // dstTwo an address that dstOne's flow is still being forwarded to. + after, err = c.client.reserveAddresses(appName, domain, dstTwo, ttl, testConnectorPeer, "") + if !errors.Is(err, errPoolExhausted) { + t.Fatalf("reserveAddresses for %v = (%+v, %v), want errPoolExhausted", dstTwo, after, err) + } + stillAssigned, ok, err := c.ClientPrepareNewFlowForMagicIP(before.magic) + if got := stillAssigned.TransitIP(); err != nil || !ok || got != transit { + t.Errorf("after the refused reservation, ClientPrepareNewFlowForMagicIP(%v) = (%v, %v, %v), want (%v, true, nil)", before.magic, got, ok, err, transit) + } + stillAssigned.Release() +} + // TestResendTransitIPMappingConcurrent races resendTransitIPMapping, against a writer of client.assignments. func TestResendTransitIPMappingConcurrent(t *testing.T) { const appName = "a" diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index 0de51b2ff..bb8a2669c 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -303,34 +303,33 @@ func (e *extension) installHooks(dph *datapathHandler) error { return nil } -// ClientTransitIPForMagicIP implements [Conn25Datapath]. -func (c *Conn25) ClientTransitIPForMagicIP(m netip.Addr) (netip.Addr, error) { - if addr, ok := c.client.transitIPForMagicIP(m); ok { - return addr, nil +// ClientPrepareNewFlowForMagicIP implements [Conn25Datapath]. +// The datapath notifies Conn25 that a flow to magic IP is being established, and +// Conn25 resolves m to its Transit IP and counts the flow against that address +// assignment in one step, reporting ok and returning the reference that names +// that Transit IP and keeps it and its associated addresses from being removed +// from Conn25's state and returned to their pools while the flow uses them. +// +// If m has no active address assignment, it reports ok false and returns +// [ErrUnmappedMagicIP] when m falls in a configured Magic IP range, and a nil +// error when it does not, since in that case m is not a Magic IP at all and the +// traffic to it is potentially valid non-app-connector traffic. +func (c *Conn25) ClientPrepareNewFlowForMagicIP(m netip.Addr) (ClientFlowRef, bool, error) { + if ref, ok := c.client.PrepareNewFlowForMagicIP(m); ok { + return ref, true, nil } cfg, ok := c.getConfig() if !ok { - return netip.Addr{}, nil + return ClientFlowRef{}, false, nil } if !cfg.ipSets.v4Magic.Contains(m) && !cfg.ipSets.v6Magic.Contains(m) { - return netip.Addr{}, nil + return ClientFlowRef{}, false, nil } // TODO(tailscale/corp#50405): Just because there is a TransitIP associated with the MagicIP, // that doesn't mean the TransitIP is ready to use. We should also make sure that the TransitIP // is installed in WireGuard for the correct node, which happens after it is registered with the // connector. Perhaps a different error message to represent that. - return netip.Addr{}, ErrUnmappedMagicIP -} - -// ClientFlowCreated implements [Conn25Datapath]. -// The datapath notifies Conn25 that a flow with transitIP has been created so -// that Conn25 can prevent that transit IP and associated addresses from being -// removed from its state and returned to their pools. The returned function -// must be invoked when that flow is removed; it releases the hold on the -// address assignment the flow was counted against. It releases that hold at -// most once: calling it again is a no-op. -func (c *Conn25) ClientFlowCreated(transitIP netip.Addr) func() { - return c.client.flowCreated(transitIP) + return ClientFlowRef{}, false, ErrUnmappedMagicIP } // ConnectorRealIPForTransitIPConnection implements [Conn25Datapath]. diff --git a/feature/conn25/conn25_test.go b/feature/conn25/conn25_test.go index 3ac07c040..da6d16a70 100644 --- a/feature/conn25/conn25_test.go +++ b/feature/conn25/conn25_test.go @@ -3416,7 +3416,7 @@ func TestHandleAddressAssignmentStoresTransitIPs(t *testing.T) { } } -func TestClientTransitIPForMagicIP(t *testing.T) { +func TestClientPrepareNewFlowForMagicIPResolution(t *testing.T) { sn := makeSelfNode(t, []appctype.AppConnectorAttr{{Name: "app1", Mode: appctype.MagicIP}}, appctype.Conn25PoolsAttr{ V4MagicIPPool: []netipx.IPRange{v4RangeFrom("0", "10")}, // 100.64.0.0 - 100.64.0.10 V6MagicIPPool: []netipx.IPRange{v6RangeFrom("0", "10")}, @@ -3439,6 +3439,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name string mip netip.Addr wantTip netip.Addr + wantOk bool wantErr error }{ { @@ -3457,6 +3458,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name: "mapped-magic-ip", mip: mappedMip, wantTip: mappedTip, + wantOk: true, wantErr: nil, }, { @@ -3475,6 +3477,7 @@ func TestClientTransitIPForMagicIP(t *testing.T) { name: "v6-mapped-magic-ip", mip: v6MappedMip, wantTip: v6MappedTip, + wantOk: true, wantErr: nil, }, } { @@ -3500,10 +3503,16 @@ func TestClientTransitIPForMagicIP(t *testing.T) { }); err != nil { t.Fatal(err) } - tip, err := c.ClientTransitIPForMagicIP(tt.mip) - if tip != tt.wantTip { + ref, ok, err := c.ClientPrepareNewFlowForMagicIP(tt.mip) + if ok { + defer ref.Release() + } + if tip := ref.TransitIP(); tip != tt.wantTip { t.Fatalf("checking transit ip: want %v, got %v", tt.wantTip, tip) } + if ok != tt.wantOk { + t.Fatalf("checking ok: want %v, got %v", tt.wantOk, ok) + } if err != tt.wantErr { t.Fatalf("checking error: want %v, got %v", tt.wantErr, err) } diff --git a/feature/conn25/datapath.go b/feature/conn25/datapath.go index 810774640..acfd8d76d 100644 --- a/feature/conn25/datapath.go +++ b/feature/conn25/datapath.go @@ -31,14 +31,6 @@ var ( // [*Conn25] is the only production implementation; the interface exists to let // datapath tests substitute a lightweight fake. type Conn25Datapath interface { - // ClientTransitIPForMagicIP returns a Transit IP for the given magicIP on a client. - // If the magicIP is within a configured Magic IP range for an app on the client, - // but not mapped to an active Transit IP, implementations should return [ErrUnmappedMagicIP]. - // If magicIP is not within a configured Magic IP range, i.e. it is not actually a Magic IP, - // implementations should return a nil error, and a zero-value [netip.Addr] to indicate - // this potentially valid, non-app-connector traffic. - ClientTransitIPForMagicIP(magicIP netip.Addr) (netip.Addr, error) - // ConnectorRealIPForTransitIPConnection returns a real destination IP for the given // srcIP and transitIP on a connector. If the transitIP is within a configured Transit IP // range for an app on the connector, but not mapped to the client at srcIP, implementations @@ -48,12 +40,18 @@ type Conn25Datapath interface { // non-app-connector traffic. ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) - // ClientFlowCreated is called before a client-side flow for transitIP has - // been installed in the client flow table, and returns a cleanup function - // that the caller must invoke when that flow is removed. The cleanup takes - // effect at most once; implementations must ignore repeat calls rather than - // release the flow's hold on its address assignment more than once. - ClientFlowCreated(transitIP netip.Addr) func() + // ClientPrepareNewFlowForMagicIP is called before a client-side flow to + // magicIP is installed in the client flow table. It reports whether it + // counted the flow against an address assignment, and if so returns the + // reference that names the Transit IP to forward that flow to and keeps + // that address from being reclaimed. + // + // When ok is true, the caller must call Release on the reference exactly + // once: when the flow is removed, or right away if it ends up installing no + // flow at all. Implementations must count nothing and return ok false and + // the zero [ClientFlowRef] otherwise, including whenever they return a + // non-nil error, since the caller does not release on those paths. + ClientPrepareNewFlowForMagicIP(magicIP netip.Addr) (ref ClientFlowRef, ok bool, err error) } // datapathHandler handles packets from the datapath, @@ -234,9 +232,16 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu // tun device, it can only be a new client flow, other (non-app-connector) traffic, // or broken return app-connector traffic on a connector, which needs to be re-established // with a new outbound packet. + // + // Resolve the Magic IP and notify Conn25 that a flow for it is being + // established, before installing that flow in the flow table. Resolving and + // counting together keeps the assignment from expiring out from under the + // flow in between, and guarantees that ClientPrepareNewFlowForMagicIP for + // this flow precedes the cleanup that fires when it is removed. magicIP := p.Dst.Addr() - transitIP, err := dh.conn25.ClientTransitIPForMagicIP(magicIP) + ref, ok, err := dh.conn25.ClientPrepareNewFlowForMagicIP(magicIP) if err != nil { + // No reference to release: an error means nothing was counted. if errors.Is(err, ErrUnmappedMagicIP) { // Couldn't find a mapping. Tell the local application the host is // unreachable so it can recover quickly instead of blackholing. @@ -246,11 +251,12 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu dh.debugLogf("error mapping magic IP, passing packet unmodified: %v", err) return filter.Accept } - - // If this is normal non-app-connector traffic, forward it along unmodified. - if !transitIP.IsValid() { + if !ok { + // This is normal non-app-connector traffic; forward it along + // unmodified. No reference to release: nothing was counted. return filter.Accept } + transitIP := ref.TransitIP() // This is a new outbound client flow. Install a DNAT MagicIP-to-TransitIP action // for the outgoing direction, and an SNAT TransitIP-to-MagicIP action for the @@ -264,15 +270,10 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed, tun *tstu Action: dh.snatAction(magicIP), } - // Notify Conn25 that a flow for transitIP is being established before - // installing it in the flow table. This guarantees that ClientFlowCreated - // for this flow precedes the cleanup that fires when it is removed. - flowCleanup := dh.conn25.ClientFlowCreated(transitIP) - dh.clientFlowTable.NewFlow(FlowData{ FromTun: outgoing, FromWG: incoming, - OnRemove: flowCleanup, + OnRemove: ref.Release, }) outgoing.Action(p) return filter.Accept diff --git a/feature/conn25/datapath_test.go b/feature/conn25/datapath_test.go index af43c5a46..4e4d7cdc9 100644 --- a/feature/conn25/datapath_test.go +++ b/feature/conn25/datapath_test.go @@ -25,22 +25,24 @@ import ( ) type testConn25 struct { - clientTransitIPForMagicIPFn func(netip.Addr) (netip.Addr, error) + clientPrepareNewFlowForMagicIPFn func(netip.Addr) (netip.Addr, error) connectorRealIPForTransitIPConnectionFn func(netip.Addr, netip.Addr) (netip.Addr, error) } -func (tc *testConn25) ClientTransitIPForMagicIP(magicIP netip.Addr) (netip.Addr, error) { - return tc.clientTransitIPForMagicIPFn(magicIP) +// ClientPrepareNewFlowForMagicIP counts a flow, and so reports ok, exactly when +// clientPrepareNewFlowForMagicIPFn resolves magicIP to a Transit IP. +func (tc *testConn25) ClientPrepareNewFlowForMagicIP(magicIP netip.Addr) (ClientFlowRef, bool, error) { + transitIP, err := tc.clientPrepareNewFlowForMagicIPFn(magicIP) + if err != nil || !transitIP.IsValid() { + return ClientFlowRef{}, false, err + } + return ClientFlowRef{transit: transitIP}, true, nil } func (tc *testConn25) ConnectorRealIPForTransitIPConnection(srcIP netip.Addr, transitIP netip.Addr) (netip.Addr, error) { return tc.connectorRealIPForTransitIPConnectionFn(srcIP, transitIP) } -func (tc *testConn25) ClientFlowCreated(transitIP netip.Addr) func() { - return func() {} -} - // testNet is the cast of addresses the datapath tests run against: a client, // the Magic and Transit IPs it talks through, and the real server behind them, // all in a single address family. It also carries the bits needed to @@ -159,7 +161,7 @@ func (tn testNet) newClientDatapath(t *testing.T, throwMappingErr bool) *datapat t.Helper() mock := &testConn25{} - mock.clientTransitIPForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { + mock.clientPrepareNewFlowForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { if throwMappingErr { return netip.Addr{}, errors.New("synthetic mapping error") } @@ -411,9 +413,9 @@ func TestClientFlowCache(t *testing.T) { getTransitIPCalled := false mock := &testConn25{} - mock.clientTransitIPForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { + mock.clientPrepareNewFlowForMagicIPFn = func(mip netip.Addr) (netip.Addr, error) { if getTransitIPCalled { - t.Errorf("ClientGetTransitIPForMagicIP unexpectedly called more than once") + t.Errorf("ClientPrepareNewFlowForMagicIP unexpectedly called more than once") } getTransitIPCalled = true return tn.transitIP, nil @@ -597,14 +599,16 @@ func TestUnmappedTransitIPRejectWithFullInjectionQueue(t *testing.T) { tun := newFakeTUN(t) p := tn.udpPacket( netip.AddrPortFrom(tn.unknownSrcIP, clientPort), - netip.AddrPortFrom(tn.transitIP, serverPort)) + netip.AddrPortFrom(tn.transitIP, serverPort), + ) // Fill the injection queue (depth 1) and leave it unread, // as its reader may be blocked behind wireguard-go's // device.peers lock held by RemovePeer. if err := tun.TryInjectOutbound(tn.udpPacket( netip.AddrPortFrom(tn.clientSrcIP, clientPort), - netip.AddrPortFrom(tn.magicIP, serverPort)).Buffer()); err != nil { + netip.AddrPortFrom(tn.magicIP, serverPort), + ).Buffer()); err != nil { t.Fatal(err) }