util/bufpool: add a general purpose pool of buffers (#20867)

A naive sync.Pool that stores dynamically sized []byte
is incorrect since the Pool assumes that each stored item
carries approximately the same memory cost.

Different cost items should be stored in different pools.
The bufpool tracks buffer pools by powers of two
and provides a Buffer type that mimics the API of bytes.Buffer.

This is a performance optimization. Avoid using this unless
there is a clear performance gain for doing so.
Incorrect use can lead to widespread data corruption
(as is already the case with any incorrect sync.Pool use).

Updates tailscale/corp#21363

Signed-off-by: Joe Tsai <[email protected]>
This commit is contained in:
Joe Tsai authored and GitHub committed 2026-10-14 15:56:12 -07:00
1 parent 5f47f9ce49
commit b83f28ebfb
3 files changed
+552

No files matched your search

+160
View File
@@ -0,0 +1,160 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package bufpool
import (
"io"
"sync"
)
// Buffer is like [bytes.Buffer], but grows using the global buffer pool.
// The Grow method returns an error if the buffer cannot be grown.
// A buffer must be obtained via [Get] or [Pool.Get].
type Buffer struct {
//lint:ignore U1000 intentional unused field
noShallowCopy [0]sync.Mutex
parent *Pool
// buf is the internal buf.
// - buf[:off] is the data that has already been read
// - buf[off:len(buf)] is the data ready to read
// - buf[len(buf):cap(buf)] is the available space to write into
buf []byte
off int
}
// Reset resets the buffer to be empty,
// but it retains the underlying storage for use by future writes.
func (b *Buffer) Reset() {
b.buf = b.buf[:0]
b.off = 0
}
// Len reports the length of the buffer.
func (b *Buffer) Len() int {
return len(b.buf) - b.off
}
// Cap reports the capacity of the buffer.
func (b *Buffer) Cap() int {
return cap(b.buf)
}
// Available report the unused capacity of the buffer.
func (b *Buffer) Available() int {
return cap(b.buf) - len(b.buf)
}
// AvailableBuffer returns an empty buffer with [Buffer.Available] capacity.
// This buffer is intended to be appended to and
// passed to an immediately succeeding [Buffer.Write] call.
// The buffer is only valid until the next write operation on b.
func (b *Buffer) AvailableBuffer() []byte {
return b.buf[len(b.buf):]
}
// Bytes returns the underlying buffer.
//
// The caller must ensure that the underlying buffer is no longer in use
// before putting the Buffer pack in the pool.
func (b *Buffer) Bytes() []byte {
return b.buf[b.off:]
}
// String returns the contents of the unread portion of the buffer as a string.
func (b *Buffer) String() string {
if b == nil {
return "<nil>" // special case for debugging
}
return string(b.Bytes())
}
// Grow expands the capacity of the buffer to accommodate n additional bytes
// without any further regrowth.
func (b *Buffer) Grow(n int) error {
if n < 0 {
panic("negative capacity")
}
if cap(b.buf)-len(b.buf) < n {
b2, err := b.parent.Get(len(b.buf) + n)
if err != nil {
return err
}
b2.buf = append(b2.buf[:0], b.buf[b.off:]...) // does not allocate
b.buf, b2.buf = b2.buf, b.buf
b.off = 0
b.parent.Put(b2)
}
return nil
}
// Next returns a slice containing the next n bytes from the buffer,
// advancing the buffer as if the bytes had been returned by [Buffer.Read].
//
// The caller must ensure that the underlying buffer is no longer in use
// before putting the Buffer pack in the pool.
func (b *Buffer) Next(n int) []byte {
data := b.buf[b.off : b.off+min(n, b.Len())]
b.off += len(data)
return data
}
// Read reads unread bytes from b into p. It implements [io.Reader].
func (b *Buffer) Read(p []byte) (int, error) {
if b.Len() == 0 {
return 0, io.EOF
}
n := copy(p, b.buf[b.off:])
b.off += n
if b.off >= len(b.buf) {
b.Reset()
}
return n, nil
}
// Write writes the provided bytes from p into b. It implements [io.Writer].
func (b *Buffer) Write(p []byte) (int, error) {
if err := b.Grow(len(p)); err != nil {
return 0, err
}
b.buf = b.buf[:len(b.buf)+len(p)]
copy(b.buf[len(b.buf)-len(p):], p) // noop if src and dst are the same
return len(p), nil
}
// ReadFrom reads content from r into b. It implements [io.ReaderFrom].
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
for {
// Always ensure there is at least 1 byte of available capacity.
if err := b.Grow(1); err != nil {
return n, err
}
// Read into the available buffer until io.EOF.
m, err := r.Read(b.buf[len(b.buf):cap(b.buf)])
b.buf = b.buf[:len(b.buf)+m]
n += int64(m)
if err != nil {
if err == io.EOF {
err = nil
}
return n, err
}
}
}
// WriteTo writes content from b into w. It implements [io.WriterTo].
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
if b.Len() > 0 {
var m int
m, err = w.Write(b.buf[b.off:])
n = int64(m)
b.off += m
}
if b.off >= len(b.buf) {
b.Reset()
}
return n, err
}
+167
View File
@@ -0,0 +1,167 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package bufpool
import (
"bytes"
"crypto/rand"
"io"
"slices"
"testing"
"testing/iotest"
qt "github.com/frankban/quicktest"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/sync/errgroup"
"tailscale.com/util/must"
)
// Reduce the range of pool levels for testing purposes.
func init() {
minBits = 8
maxBits = 16
}
func Test(t *testing.T) {
c := qt.New(t)
const maxSize = 1 << 20
p := NewPool()
p.SetMaxSize(maxSize)
c.Run("NoCapacity", func(c *qt.C) {
_, err := p.Get(1 + maxSize)
c.Assert(err, qt.CmpEquals(cmpopts.EquateErrors()), cmpopts.AnyError)
c.Assert(p.counterUsedBytes.Value(), qt.Equals, int64(0))
c.Assert(p.counterUsedBuffers.Value(), qt.Equals, int64(0))
})
c.Run("BackToBack", func(c *qt.C) {
var usedBytes int64
sizes := []int{0, 3, 184, 254, 255, 256, 257, 1381, 1023, 1024, 1025, 30105, 30105, 65534, 65535, 65536, 65537, 184823, 184823}
func() {
for _, n := range sizes {
b := must.Get(p.Get(n))
defer p.Put(b)
if n < 1<<minBits {
c.Assert(b.Cap(), qt.Equals, 1<<minBits)
} else {
c.Assert(n <= b.Cap() && b.Cap() <= 2*n, qt.IsTrue)
}
usedBytes += int64(b.Cap())
}
_, err := p.Get(int(maxSize - usedBytes + 1))
c.Assert(err, qt.CmpEquals(cmpopts.EquateErrors()), cmpopts.AnyError)
c.Assert(p.gaugeUsedBytes.Value(), qt.Equals, usedBytes)
c.Assert(p.gaugeUsedBuffers.Value(), qt.Equals, int64(len(sizes)))
c.Assert(p.counterUsedBytes.Value(), qt.Equals, usedBytes)
c.Assert(p.counterUsedBuffers.Value(), qt.Equals, int64(len(sizes)))
}()
c.Assert(p.gaugeUsedBytes.Value(), qt.Equals, int64(0))
c.Assert(p.gaugeUsedBuffers.Value(), qt.Equals, int64(0))
})
c.Run("Buffers", func(c *qt.C) {
var group errgroup.Group
for range 10 {
group.Go(func() error {
b := must.Get(p.Get(0))
defer p.Put(b)
c.Assert(b.Len(), qt.Equals, 0)
c.Assert(b.Cap(), qt.Equals, 1<<minBits)
c.Assert(b.Bytes(), qt.DeepEquals, []byte{})
n, err := b.Read([]byte{0})
c.Assert(n, qt.Equals, 0)
c.Assert(err, qt.Equals, io.EOF)
want := []byte("hello, world!")
n = must.Get(b.Write(want))
c.Assert(n, qt.Equals, len(want))
c.Assert(b.Len(), qt.Equals, len(want))
c.Assert(b.Cap(), qt.Equals, 1<<minBits)
c.Assert(b.Bytes(), qt.DeepEquals, want)
got := make([]byte, len("hello"))
n = must.Get(b.Read(got))
c.Assert(n, qt.Equals, len(got))
c.Assert(got, qt.DeepEquals, want[:len(got)])
c.Assert(b.Len(), qt.Equals, len(want)-len(got))
c.Assert(b.Cap(), qt.Equals, 1<<minBits)
c.Assert(b.Bytes(), qt.DeepEquals, want[len(got):])
must.Do(b.Grow(1<<minBits - len(want)))
c.Assert(b.Cap(), qt.Equals, 1<<minBits)
must.Do(b.Grow(789))
c.Assert(b.Cap(), qt.Equals, 1<<10)
want2 := make([]byte, cap(b.Bytes())-len(b.Bytes()))
must.Get(rand.Read(want2))
must.Get(b.Write(want2))
c.Assert(b.Len(), qt.Equals, len(want)-len(got)+len(want2))
c.Assert(b.Cap(), qt.Equals, 1<<10)
c.Assert(b.Bytes(), qt.DeepEquals, append(slices.Clone(want[len(got):]), want2...))
want3 := make([]byte, 4000)
must.Get(rand.Read(want3))
must.Get(b.Write(want3))
c.Assert(b.Len(), qt.Equals, len(want)-len(got)+len(want2)+len(want3))
c.Assert(b.Cap(), qt.Equals, 1<<13)
c.Assert(b.Bytes(), qt.DeepEquals, append(append(slices.Clone(want[len(got):]), want2...), want3...))
b.Reset()
must.Get(b.Write(append(b.AvailableBuffer(), "hello"...)))
want4 := must.Get(io.ReadAll(io.LimitReader(rand.Reader, 1<<12)))
must.Get(b.ReadFrom(iotest.HalfReader(bytes.NewReader(want4))))
must.Get(io.ReadFull(b, got))
c.Assert(string(got), qt.Equals, "hello")
got2 := new(bytes.Buffer)
must.Get(b.WriteTo(got2))
c.Assert(got2.Bytes(), qt.DeepEquals, want4)
b.Reset()
b2 := b.AvailableBuffer()
b2 = append(b2, "fizzbuzz"...)
n = must.Get(b.Write(b2))
c.Assert(n, qt.Equals, len(b2))
b2 = b.Next(1000)
c.Assert(b2, qt.DeepEquals, []byte("fizzbuzz"))
b2 = b.Next(1000)
c.Assert(b2, qt.DeepEquals, []byte(""))
return nil
})
}
group.Wait()
c.Assert(p.gaugeUsedBytes.Value(), qt.Equals, int64(0))
c.Assert(p.gaugeUsedBuffers.Value(), qt.Equals, int64(0))
})
}
func BenchmarkAppend(b *testing.B) {
b.ReportAllocs()
pool := NewPool()
pool.SetMaxSize(1 << maxBits)
for range b.N {
buf := must.Get(pool.Get(1 << maxBits))
b := buf.AvailableBuffer() // should have zero length, but large capacity
must.Get(buf.Write(b[:cap(b)])) // write entire capacity of same buffer is noop
pool.Put(buf)
}
}
func BenchmarkGrow(b *testing.B) {
b.ReportAllocs()
pool := NewPool()
pool.SetMaxSize(1 << maxBits)
for range b.N {
buf := must.Get(pool.Get(0))
for range 6 {
buf.Grow(buf.Cap() + 1)
}
pool.Put(buf)
}
}
+225
View File
@@ -0,0 +1,225 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
// Package bufpool provides a pool of variable length buffers.
//
// Buffers retrieved from bufpool can either be used directly
// with an API similar to [bytes.Buffer], or can be used as a raw []byte,
// by calling the [Buffer.Bytes] method. Buffers retrieved from the pool must
// be eventually returned back to the pool. Otherwise, metrics tracking
// the total number of actively used buffers will be incorrect, and
// any limits on the maximum number of active buffers will operate incorrectly.
//
// A limit on the number of active buffers may be specified to refuse future
// attempts to acquire a buffer while running in a high load state.
// This is useful in server applications as a load shedding mechanism where
// it is better to fail certain requests instead of crashing.
//
// Example usage:
//
// b, err := Get(n) // n is minimum capacity
// if err != nil {
// ... // handle err
// }
// defer Put(b)
// ... // use b as if it were a bytes.Buffer
package bufpool
import (
"errors"
"expvar"
"fmt"
"math"
"math/bits"
"strconv"
"sync"
"sync/atomic"
"tailscale.com/metrics"
)
// ErrInsufficientCapacity indicates that [Get] and [Pool.Get] could not acquire
// a buffer because of size limits (see [SetMaxSize] or [Pool.SetMaxSize]).
var ErrInsufficientCapacity = errors.New("insufficient capacity to allocate buffer")
var global = NewPool()
// Get retrieves a [Buffer] with sufficient capacity to hold n bytes.
// If the global pool lacks sufficient capacity, it returns an error.
//
// Every buffer must be returned to back to the pool by calling [Put].
// Leaking a buffer causes metric gauges about actively used buffers
// to incorrectly increase without ever decreasing.
// If [SetMaxSize] is used, then [Get] may perpetually fail.
func Get(n int) (*Buffer, error) { return global.Get(n) }
// Put returns the buffer back to the global pool. The buffer
// (including any []byte obtained through [Buffer.Bytes] or [Buffer.Next])
// must not be used any more.
func Put(b *Buffer) { global.Put(b) }
// ExpVar reports metrics about global pool usage.
// See [Pool.ExpVar].
func ExpVar() expvar.Var { return global.ExpVar() }
// SetMaxSize specifies the maximum number of buffer bytes for active use.
//
// Since this has global effects, it is recommended that this only be set in the
// main package and at a level that's a reasonable fraction of system memory.
//
// See [Pool.SetMaxSize].
func SetMaxSize(n int64) { global.SetMaxSize(n) }
var (
minBits = 10 // 1 KiB
maxBits = 30 // 1 GiB
)
// Pool is a pool of Buffers.
type Pool struct {
//lint:ignore U1000 intentional unused field
noShallowCopy [0]sync.Mutex
maxSize atomic.Int64 // maximum total number of in-use buffer bytes
// pools is a list of sync.Pool,
// where each pool manages buffers of capacity 1<<(minBits+i).
pools []pool
gaugeUsedBytes expvar.Int // current number of buffer bytes in use
gaugeUsedBuffers expvar.Int // current number of buffers in use
counterUsedBytes expvar.Int // total number of buffer bytes ever used
counterUsedBuffers expvar.Int // total number of buffers ever requested
counterRefusedBuffers expvar.Int // total number of buffers refused due to SetMaxSize
gaugeUsedBuffersMap metrics.LabelMap
counterUsedBuffersMap metrics.LabelMap
}
type pool struct {
*sync.Pool
gaugeUsedBuffers *expvar.Int
counterUsedBuffers *expvar.Int
}
// NewPool constructs a new buffer pool.
// By default, there is no limit to the maximum bytes in active use.
// See [Pool.SetMaxSize].
func NewPool() *Pool {
p := &Pool{pools: make([]pool, maxBits-minBits+1)}
p.maxSize.Store(math.MaxInt64)
p.gaugeUsedBuffersMap.Label = "bits"
p.counterUsedBuffersMap.Label = "bits"
for i := range p.pools {
p.pools[i] = pool{
Pool: &sync.Pool{New: func() any {
return &Buffer{parent: p, buf: make([]byte, 0, 1<<(minBits+i))}
}},
gaugeUsedBuffers: p.gaugeUsedBuffersMap.Get(strconv.Itoa(minBits + i)),
counterUsedBuffers: p.counterUsedBuffersMap.Get(strconv.Itoa(minBits + i)),
}
}
return p
}
// ExpVar reports metrics about pool usage:
//
// - gauge_used_bytes: buffer bytes in active use
// - gauge_used_buffers: number of buffers in active use
// - counter_total_used_bytes: total buffer bytes ever acquired
// - counter_total_used_buffers: total number of buffers ever acquired
// - counter_total_refused_buffers: total number of buffers refused
// due to a limit on the total number of active buffer bytes
// - gauge_used_buffers_by_level: a map of the number of buffers
// in active use keyed by the log2 of the buffer size
// - counter_total_used_buffers_by_level: a map of the total number of buffers
// ever acquired keyed by the log2 of the buffer size
func (p *Pool) ExpVar() expvar.Var {
m := new(metrics.Set)
m.Set("gauge_used_bytes", &p.gaugeUsedBytes)
m.Set("gauge_used_buffers", &p.gaugeUsedBuffers)
m.Set("counter_total_used_bytes", &p.counterUsedBytes)
m.Set("counter_total_used_buffers", &p.counterUsedBuffers)
m.Set("counter_total_refused_buffers", &p.counterRefusedBuffers)
m.Set("gauge_used_buffers_by_level", &p.gaugeUsedBuffersMap)
m.Set("counter_total_used_buffers_by_level", &p.counterUsedBuffersMap)
return m
}
// SetMaxSize specifies the maximum number of buffer bytes for active use.
//
// Setting this to a lower limit does not affect active buffers,
// but prevents [Pool.Get] from successfully returning until enough buffers
// have been [Pool.Put] that reduces the total active buffers back to the limit.
//
// A negative value specifies no limit. By default, there is no limit.
func (p *Pool) SetMaxSize(n int64) {
if n < 0 {
n = math.MaxInt64
}
p.maxSize.Store(n)
}
// Get retrieves a [Buffer] with sufficient capacity to hold n bytes.
// If the pool lacks sufficient capacity, it returns an error.
//
// Every buffer must be returned to back to the pool by calling [Pool.Put].
// Leaking a buffer causes metric gauges about actively used buffers
// to incorrectly increase without ever decreasing.
// If [Pool.SetMaxSize] is used, then [Pool.Get] may perpetually fail.
func (p *Pool) Get(n int) (*Buffer, error) {
if n < 0 {
panic("negative capacity")
}
i := bits.UintSize - bits.LeadingZeros(uint(n-1)) // round up to nearest power of two
if n == 0 || i < minBits {
i = minBits
}
c := int64(1 << i)
if c < int64(n) {
panic(fmt.Sprintf("integer overflow computing capacity for %d", n))
}
p.gaugeUsedBytes.Add(+c)
if p.gaugeUsedBytes.Value() > p.maxSize.Load() {
p.gaugeUsedBytes.Add(-c)
p.counterRefusedBuffers.Add(1)
return nil, ErrInsufficientCapacity
}
p.gaugeUsedBuffers.Add(+1)
p.counterUsedBytes.Add(+c)
p.counterUsedBuffers.Add(+1)
if i > maxBits {
// Too large for any of the pre-sized pools.
// Just allocate from the Go heap.
return &Buffer{parent: p, buf: make([]byte, 0, 1<<i)}, nil
}
pp := &p.pools[i-minBits]
pp.gaugeUsedBuffers.Add(+1)
pp.counterUsedBuffers.Add(+1)
b := pp.Get().(*Buffer)
b.Reset()
return b, nil
}
// Put returns the buffer back to the pool. The buffer
// (including any []byte obtained through [Buffer.Bytes] or [Buffer.Next])
// must not be used any more.
func (p *Pool) Put(b *Buffer) {
if p != b.parent {
panic(fmt.Sprintf("mismatching parent pool: %p ≠ %p", p, b.parent))
}
c := int64(cap(b.buf))
i := bits.UintSize - bits.LeadingZeros(uint(c)) - 1 // round down to nearest power of two
p.gaugeUsedBytes.Add(-c)
p.gaugeUsedBuffers.Add(-1)
if !(minBits <= i && i <= maxBits) {
// Too large for any of the pre-sized pools.
// Just ignore the buffer and let the Go GC clean it up later.
return
}
pp := &p.pools[i-minBits]
pp.gaugeUsedBuffers.Add(-1)
pp.Put(b)
}