mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
derp/xdp: don't write counters back to the per-CPU map on every packet (#22460)
inc_counter looked up the per-CPU counter, incremented it in place, and then called bpf_map_update_elem to store the same value back. For a BPF_MAP_TYPE_PERCPU_HASH the pointer returned by the lookup already is this CPU's slot, so the write-back was redundant, and it is the one step that takes the hash bucket spinlock. The program runs for every received packet, not just STUN, and updates two counters each time, so a busy DERP paid two locked map updates per incoming TCP segment. Keep the update only for the miss path, which runs once per key. While here, make inc_counter and handle_counters return void. Neither caller looked at the result. TestXDP cleared the counters map before every case, so it only ever exercised the key insertion path. Add a subtest that runs the same packet repeatedly without clearing, to cover the in-place increment of an existing key. Regenerate the bpf2go outputs with clang 18.1.8 and the bpf2go from the cilium/ebpf v0.22.0 already in go.mod, which is also why the generated Go files pick up structs.HostLayout and the object name constants. Fixes #22463 Signed-off-by: Mike O'Driscoll <[email protected]> Co-authored-by: Brad Fitzpatrick <[email protected]>
This commit is contained in:
1 parent
7c6e49dfb3
commit
18d7c08613
6 files changed
+165
-26
No files matched your search
+43
-2
@@ -8,11 +8,13 @@ import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"structs"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct {
|
||||
_ structs.HostLayout
|
||||
DstPort uint16
|
||||
DropStun uint16
|
||||
}
|
||||
@@ -54,12 +56,27 @@ const (
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
_ structs.HostLayout
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// Names of all BPF objects in the ELF.
|
||||
//
|
||||
// Used for safe lookups in a Collection or CollectionSpec.
|
||||
const (
|
||||
bpfMapConfigMap = "config_map"
|
||||
bpfMapCountersMap = "counters_map"
|
||||
bpfProgXdpProgFunc = "xdp_prog_func"
|
||||
bpfVarUnusedConfig = "unused_config"
|
||||
bpfVarUnusedCounterKeyAf = "unused_counter_key_af"
|
||||
bpfVarUnusedCounterKeyPacketsBytesAction = "unused_counter_key_packets_bytes_action"
|
||||
bpfVarUnusedCounterKeyProgEnd = "unused_counter_key_prog_end"
|
||||
bpfVarUnusedCountersKey = "unused_counters_key"
|
||||
)
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
@@ -80,7 +97,7 @@ func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
func loadBpfObjects(obj any, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -95,9 +112,10 @@ func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
bpfVariableSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
// bpfProgramSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
@@ -112,12 +130,24 @@ type bpfMapSpecs struct {
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfVariableSpecs contains global variables before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfVariableSpecs struct {
|
||||
UnusedConfig *ebpf.VariableSpec `ebpf:"unused_config"`
|
||||
UnusedCounterKeyAf *ebpf.VariableSpec `ebpf:"unused_counter_key_af"`
|
||||
UnusedCounterKeyPacketsBytesAction *ebpf.VariableSpec `ebpf:"unused_counter_key_packets_bytes_action"`
|
||||
UnusedCounterKeyProgEnd *ebpf.VariableSpec `ebpf:"unused_counter_key_prog_end"`
|
||||
UnusedCountersKey *ebpf.VariableSpec `ebpf:"unused_counters_key"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
bpfVariables
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
@@ -142,6 +172,17 @@ func (m *bpfMaps) Close() error {
|
||||
)
|
||||
}
|
||||
|
||||
// bpfVariables contains all global variables after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfVariables struct {
|
||||
UnusedConfig *ebpf.Variable `ebpf:"unused_config"`
|
||||
UnusedCounterKeyAf *ebpf.Variable `ebpf:"unused_counter_key_af"`
|
||||
UnusedCounterKeyPacketsBytesAction *ebpf.Variable `ebpf:"unused_counter_key_packets_bytes_action"`
|
||||
UnusedCounterKeyProgEnd *ebpf.Variable `ebpf:"unused_counter_key_prog_end"`
|
||||
UnusedCountersKey *ebpf.Variable `ebpf:"unused_counters_key"`
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
|
||||
Binary file not shown.
+44
-3
@@ -1,5 +1,5 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64
|
||||
//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64 || wasm
|
||||
|
||||
package xdp
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
"structs"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type bpfConfig struct {
|
||||
_ structs.HostLayout
|
||||
DstPort uint16
|
||||
DropStun uint16
|
||||
}
|
||||
@@ -54,12 +56,27 @@ const (
|
||||
)
|
||||
|
||||
type bpfCountersKey struct {
|
||||
_ structs.HostLayout
|
||||
Unused uint8
|
||||
Af uint8
|
||||
Pba uint8
|
||||
ProgEnd uint8
|
||||
}
|
||||
|
||||
// Names of all BPF objects in the ELF.
|
||||
//
|
||||
// Used for safe lookups in a Collection or CollectionSpec.
|
||||
const (
|
||||
bpfMapConfigMap = "config_map"
|
||||
bpfMapCountersMap = "counters_map"
|
||||
bpfProgXdpProgFunc = "xdp_prog_func"
|
||||
bpfVarUnusedConfig = "unused_config"
|
||||
bpfVarUnusedCounterKeyAf = "unused_counter_key_af"
|
||||
bpfVarUnusedCounterKeyPacketsBytesAction = "unused_counter_key_packets_bytes_action"
|
||||
bpfVarUnusedCounterKeyProgEnd = "unused_counter_key_prog_end"
|
||||
bpfVarUnusedCountersKey = "unused_counters_key"
|
||||
)
|
||||
|
||||
// loadBpf returns the embedded CollectionSpec for bpf.
|
||||
func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_BpfBytes)
|
||||
@@ -80,7 +97,7 @@ func loadBpf() (*ebpf.CollectionSpec, error) {
|
||||
// *bpfMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
func loadBpfObjects(obj any, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := loadBpf()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -95,9 +112,10 @@ func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
type bpfSpecs struct {
|
||||
bpfProgramSpecs
|
||||
bpfMapSpecs
|
||||
bpfVariableSpecs
|
||||
}
|
||||
|
||||
// bpfSpecs contains programs before they are loaded into the kernel.
|
||||
// bpfProgramSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfProgramSpecs struct {
|
||||
@@ -112,12 +130,24 @@ type bpfMapSpecs struct {
|
||||
CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
|
||||
}
|
||||
|
||||
// bpfVariableSpecs contains global variables before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type bpfVariableSpecs struct {
|
||||
UnusedConfig *ebpf.VariableSpec `ebpf:"unused_config"`
|
||||
UnusedCounterKeyAf *ebpf.VariableSpec `ebpf:"unused_counter_key_af"`
|
||||
UnusedCounterKeyPacketsBytesAction *ebpf.VariableSpec `ebpf:"unused_counter_key_packets_bytes_action"`
|
||||
UnusedCounterKeyProgEnd *ebpf.VariableSpec `ebpf:"unused_counter_key_prog_end"`
|
||||
UnusedCountersKey *ebpf.VariableSpec `ebpf:"unused_counters_key"`
|
||||
}
|
||||
|
||||
// bpfObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfObjects struct {
|
||||
bpfPrograms
|
||||
bpfMaps
|
||||
bpfVariables
|
||||
}
|
||||
|
||||
func (o *bpfObjects) Close() error {
|
||||
@@ -142,6 +172,17 @@ func (m *bpfMaps) Close() error {
|
||||
)
|
||||
}
|
||||
|
||||
// bpfVariables contains all global variables after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type bpfVariables struct {
|
||||
UnusedConfig *ebpf.Variable `ebpf:"unused_config"`
|
||||
UnusedCounterKeyAf *ebpf.Variable `ebpf:"unused_counter_key_af"`
|
||||
UnusedCounterKeyPacketsBytesAction *ebpf.Variable `ebpf:"unused_counter_key_packets_bytes_action"`
|
||||
UnusedCounterKeyProgEnd *ebpf.Variable `ebpf:"unused_counter_key_prog_end"`
|
||||
UnusedCountersKey *ebpf.Variable `ebpf:"unused_counters_key"`
|
||||
}
|
||||
|
||||
// bpfPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
|
||||
Binary file not shown.
+9
-6
@@ -179,16 +179,21 @@ struct packet_context {
|
||||
enum counter_key_prog_end prog_end;
|
||||
};
|
||||
|
||||
static __always_inline int inc_counter(struct counters_key key, __u64 val) {
|
||||
static __always_inline void inc_counter(struct counters_key key, __u64 val) {
|
||||
// inc_counter adds val to this CPU's slot for key in counters_map.
|
||||
// The map is per-CPU, so the pointer from the lookup is already this
|
||||
// CPU's value and can be incremented in place. Only a missing key,
|
||||
// which happens once per key per map lifetime, needs the update call
|
||||
// that takes the hash bucket lock.
|
||||
__u64 *counter = bpf_map_lookup_elem(&counters_map, &key);
|
||||
if (!counter) {
|
||||
return bpf_map_update_elem(&counters_map, &key, &val, BPF_ANY);
|
||||
bpf_map_update_elem(&counters_map, &key, &val, BPF_ANY);
|
||||
return;
|
||||
}
|
||||
*counter += val;
|
||||
return bpf_map_update_elem(&counters_map, &key, counter, BPF_ANY);
|
||||
}
|
||||
|
||||
static __always_inline int handle_counters(struct xdp_md *ctx, int action, struct packet_context *pctx) {
|
||||
static __always_inline void handle_counters(struct xdp_md *ctx, int action, struct packet_context *pctx) {
|
||||
void *data_end = (void *)(long)ctx->data_end;
|
||||
void *data = (void *)(long)ctx->data;
|
||||
|
||||
@@ -229,8 +234,6 @@ static __always_inline int handle_counters(struct xdp_md *ctx, int action, struc
|
||||
|
||||
inc_counter(packets_key, 1);
|
||||
inc_counter(bytes_key, bytes);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define is_ipv6 (pctx->af == COUNTER_KEY_AF_IPV6)
|
||||
|
||||
+69
-15
@@ -990,6 +990,27 @@ func TestXDP(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMetrics reports an error for every counter in server.metrics.last
|
||||
// that differs from want, in either direction.
|
||||
checkMetrics := func(t *testing.T, want map[bpfCountersKey]uint64) {
|
||||
t.Helper()
|
||||
for k, v := range want {
|
||||
gotCounter, ok := server.metrics.last[k]
|
||||
if !ok {
|
||||
t.Errorf("expected counter at key %+v not found", k)
|
||||
}
|
||||
if gotCounter != v {
|
||||
t.Errorf("key: %+v gotCounter: %d != %d", k, gotCounter, v)
|
||||
}
|
||||
}
|
||||
for k := range server.metrics.last {
|
||||
_, ok := want[k]
|
||||
if !ok {
|
||||
t.Errorf("counter at key: %+v incremented unexpectedly", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err = clearCounters()
|
||||
@@ -1019,24 +1040,57 @@ func TestXDP(t *testing.T) {
|
||||
t.Fatalf("error updating metrics: %v", err)
|
||||
}
|
||||
if c.wantMetrics != nil {
|
||||
for k, v := range c.wantMetrics {
|
||||
gotCounter, ok := server.metrics.last[k]
|
||||
if !ok {
|
||||
t.Errorf("expected counter at key %+v not found", k)
|
||||
}
|
||||
if gotCounter != v {
|
||||
t.Errorf("key: %+v gotCounter: %d != %d", k, gotCounter, v)
|
||||
}
|
||||
}
|
||||
for k := range server.metrics.last {
|
||||
_, ok := c.wantMetrics[k]
|
||||
if !ok {
|
||||
t.Errorf("counter at key: %+v incremented unexpectedly", k)
|
||||
}
|
||||
}
|
||||
checkMetrics(t, c.wantMetrics)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The cases above clear the counters map before each run, so they only
|
||||
// ever exercise the key insertion path of inc_counter. Run the same
|
||||
// packet repeatedly without clearing to cover the in-place increment
|
||||
// of an existing key.
|
||||
t.Run("counters-accumulate", func(t *testing.T) {
|
||||
err = clearCounters()
|
||||
if err != nil {
|
||||
t.Fatalf("error clearing counters: %v", err)
|
||||
}
|
||||
err = server.SetDropSTUN(false)
|
||||
if err != nil {
|
||||
t.Fatalf("error setting drop STUN: %v", err)
|
||||
}
|
||||
const runs = 5
|
||||
packetIn := ipv4STUNBindingReqIPProtoPass
|
||||
for i := range runs {
|
||||
opts := ebpf.RunOptions{
|
||||
Data: packetIn,
|
||||
DataOut: make([]byte, 1514),
|
||||
}
|
||||
got, err := server.objs.XdpProgFunc.Run(&opts)
|
||||
if err != nil {
|
||||
t.Fatalf("run %d: error running program: %v", i, err)
|
||||
}
|
||||
if xdpAction(got) != xdpActionPass {
|
||||
t.Fatalf("run %d: got code: %s != %s", i, xdpAction(got), xdpActionPass)
|
||||
}
|
||||
}
|
||||
err = server.updateMetrics()
|
||||
if err != nil {
|
||||
t.Fatalf("error updating metrics: %v", err)
|
||||
}
|
||||
wantMetrics := map[bpfCountersKey]uint64{
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: runs,
|
||||
{
|
||||
Af: uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4),
|
||||
Pba: uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL),
|
||||
ProgEnd: uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED),
|
||||
}: uint64(runs * len(packetIn)),
|
||||
}
|
||||
checkMetrics(t, wantMetrics)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCountersMapKey(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user