From 1f36d5a89f8c81bbf9fbe260f768d0ccc2d76ffc Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 18 Nov 2026 14:05:47 +0000 Subject: [PATCH] feature/linuxdnsfight, go.mod: replace gonotify with direct inotify calls The resolv.conf watcher needs one watch on one directory and a blocking read. gonotify v3.0.2 has a bug where it loops forever without advancing when a record's name length runs past the end of a read. It also never closes its fd on cancel and allocates 32 KB per Read. Use x/sys/unix directly, return an error on a malformed record, and stop when the kernel removes the watch. Fixes #16576 Signed-off-by: Brad Fitzpatrick Change-Id: If2bc5e94f8aab2922ee9e07b3c2b75a76f4f9236 --- cmd/tailscaled/depaware.txt | 2 - feature/linuxdnsfight/linuxdnsfight.go | 116 ++++++++--- feature/linuxdnsfight/linuxdnsfight_test.go | 209 ++++++++++++++++++++ flake.nix | 2 +- flakehashes.json | 4 +- go.mod | 1 - go.sum | 2 - net/dns/direct_linux_test.go | 4 +- shell.nix | 2 +- 9 files changed, 303 insertions(+), 39 deletions(-) diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index b74eca585..cf050ebc0 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -146,8 +146,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/huin/goupnp/scpd from github.com/huin/goupnp github.com/huin/goupnp/soap from github.com/huin/goupnp+ github.com/huin/goupnp/ssdp from github.com/huin/goupnp - L 💣 github.com/illarion/gonotify/v3 from tailscale.com/feature/linuxdnsfight - L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/feature/tap L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4 L github.com/insomniacslk/dhcp/interfaces from github.com/insomniacslk/dhcp/dhcpv4 diff --git a/feature/linuxdnsfight/linuxdnsfight.go b/feature/linuxdnsfight/linuxdnsfight.go index 742d23612..2e2cf430c 100644 --- a/feature/linuxdnsfight/linuxdnsfight.go +++ b/feature/linuxdnsfight/linuxdnsfight.go @@ -8,10 +8,15 @@ package linuxdnsfight import ( + "bytes" "context" + "encoding/binary" + "errors" "fmt" + "os" + "path/filepath" - "github.com/illarion/gonotify/v3" + "golang.org/x/sys/unix" "tailscale.com/feature" "tailscale.com/net/dns" ) @@ -23,49 +28,104 @@ func init() { dns.HookWatchFile.Set(watchFile) } +// watchEvents are the inotify events that indicate a file in the watched +// directory may have changed. +const watchEvents = unix.IN_ATTRIB | + unix.IN_CLOSE_WRITE | + unix.IN_CREATE | + unix.IN_DELETE | + unix.IN_MODIFY | + unix.IN_MOVE + // watchFile sets up an inotify watch for a given directory and // calls the callback function every time a particular file is changed. // The filename should be located in the provided directory. +// +// It calls cb once after the watch is in place, so changes made before +// watchFile was called are not missed. func watchFile(ctx context.Context, dir, filename string, cb func()) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - const events = gonotify.IN_ATTRIB | - gonotify.IN_CLOSE_WRITE | - gonotify.IN_CREATE | - gonotify.IN_DELETE | - gonotify.IN_MODIFY | - gonotify.IN_MOVE - - // Watch only dir itself (not recursively) and only for the events - // above. gonotify.NewDirWatcher registers IN_ALL_EVENTS on every - // subdirectory and filters afterwards, so every open or read under - // /etc (including /etc/ld.so.cache on each exec) woke this goroutine. - in, err := gonotify.NewInotify(ctx) + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) if err != nil { - return fmt.Errorf("NewInotify: %w", err) + return fmt.Errorf("inotify_init1: %w", err) } - if _, err := in.AddWatch(dir, events|gonotify.IN_ONLYDIR); err != nil { - return fmt.Errorf("AddWatch: %w", err) + // The fd is non-blocking, so os.NewFile registers it with the runtime + // poller. Reads then block without a thread, and Close from another + // goroutine unblocks them. + f := os.NewFile(uintptr(fd), "inotify") + defer f.Close() + + // Watch only dir itself (not recursively) and only for watchEvents, so + // the kernel drops opens and reads of other files in dir. + if _, err := unix.InotifyAddWatch(fd, dir, watchEvents|unix.IN_ONLYDIR); err != nil { + return fmt.Errorf("inotify_add_watch %q: %w", dir, err) } - // NewDirWatcher used to emit a synthetic event for existing files, - // which ran cb once at startup. Keep that behavior. + stop := context.AfterFunc(ctx, func() { f.Close() }) + defer stop() + cb() + base := filepath.Base(filename) + // The kernel only returns whole records and fails the read with EINVAL + // if the next one doesn't fit, so size buf for one record with the + // longest possible name. Events are rare, so reading one or a few per + // read is fine. + var buf [unix.SizeofInotifyEvent + unix.NAME_MAX + 1]byte for { - evs, err := in.Read() + n, err := f.Read(buf[:]) if err != nil { if ctx.Err() != nil { return ctx.Err() } - return fmt.Errorf("Read: %w", err) + return fmt.Errorf("inotify read: %w", err) } - for _, ev := range evs { - if ev.Name == filename { - cb() - break - } + changed, err := parseEvents(buf[:n], base) + if err != nil { + return err + } + if changed { + cb() } } } + +// errWatchRemoved is returned by [parseEvents] when the kernel removed the +// watch, such as when the watched directory was deleted or unmounted. +var errWatchRemoved = errors.New("inotify watch removed") + +// parseEvents parses the inotify_event records in buf, as returned by one +// read of an inotify fd. It reports whether any of them may indicate a +// change to the file named base, or that events were lost. +// +// It returns an error if buf is malformed. The kernel only returns whole +// records, so a malformed buf means something other than the kernel's +// inotify code produced it. +func parseEvents(buf []byte, base string) (changed bool, err error) { + for len(buf) > 0 { + if len(buf) < unix.SizeofInotifyEvent { + return false, fmt.Errorf("malformed inotify read: %d trailing bytes", len(buf)) + } + mask := binary.NativeEndian.Uint32(buf[4:]) + nameLen := binary.NativeEndian.Uint32(buf[12:]) + if uint64(nameLen) > uint64(len(buf)-unix.SizeofInotifyEvent) { + return false, fmt.Errorf("malformed inotify read: name length %d exceeds remaining %d bytes", nameLen, len(buf)-unix.SizeofInotifyEvent) + } + name := buf[unix.SizeofInotifyEvent:][:nameLen] + buf = buf[unix.SizeofInotifyEvent+int(nameLen):] + + if mask&unix.IN_IGNORED != 0 { + return false, errWatchRemoved + } + if mask&unix.IN_Q_OVERFLOW != 0 { + changed = true + continue + } + if i := bytes.IndexByte(name, 0); i >= 0 { + name = name[:i] + } + if string(name) == base { + changed = true + } + } + return changed, nil +} diff --git a/feature/linuxdnsfight/linuxdnsfight_test.go b/feature/linuxdnsfight/linuxdnsfight_test.go index 52ae06b47..a01930ea0 100644 --- a/feature/linuxdnsfight/linuxdnsfight_test.go +++ b/feature/linuxdnsfight/linuxdnsfight_test.go @@ -6,14 +6,19 @@ package linuxdnsfight import ( + "bytes" "context" + "encoding/binary" "errors" "os" "path/filepath" + "strconv" + "strings" "testing" "time" "golang.org/x/sync/errgroup" + "golang.org/x/sys/unix" ) // startWatch runs watchFile on dir/filename and returns a channel that @@ -78,3 +83,207 @@ func TestWatchFileIgnoresReadsAndOtherFiles(t *testing.T) { case <-time.After(200 * time.Millisecond): } } + +// TestWatchFileLongName checks that the read buffer holds an event for a +// file with the longest name Linux allows, and that watching continues. +func TestWatchFileLongName(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "resolv.conf") + calls := startWatch(t, dir, path) + + long := filepath.Join(dir, strings.Repeat("x", unix.NAME_MAX)) + for range 3 { + if err := os.WriteFile(long, nil, 0644); err != nil { + t.Fatal(err) + } + if err := os.Remove(long); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(path, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + select { + case <-calls: + case <-time.After(5 * time.Second): + t.Fatal("callback was not called after long-name events and a write") + } +} + +// inotifyWatches returns the number of inotify fds held by this process +// and the masks of their watches, read from /proc/self/fdinfo. +func inotifyWatches(t *testing.T) (fds int, masks []uint64) { + t.Helper() + ents, err := os.ReadDir("/proc/self/fd") + if err != nil { + t.Fatal(err) + } + for _, e := range ents { + target, err := os.Readlink("/proc/self/fd/" + e.Name()) + if err != nil || target != "anon_inode:inotify" { + continue + } + fds++ + info, err := os.ReadFile("/proc/self/fdinfo/" + e.Name()) + if err != nil { + t.Fatal(err) + } + for line := range strings.Lines(string(info)) { + if !strings.HasPrefix(line, "inotify ") { + continue + } + for f := range strings.FieldsSeq(line) { + if hex, ok := strings.CutPrefix(f, "mask:"); ok { + m, err := strconv.ParseUint(hex, 16, 32) + if err != nil { + t.Fatalf("bad fdinfo line %q: %v", line, err) + } + masks = append(masks, m) + } + } + } + } + return fds, masks +} + +func TestWatchFileKernelMask(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "sub"), 0755); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + started := make(chan struct{}, 1) + errc := make(chan error, 1) + go func() { + errc <- watchFile(ctx, dir, filepath.Join(dir, "resolv.conf"), func() { + select { + case started <- struct{}{}: + default: + } + }) + }() + <-started + + fds, masks := inotifyWatches(t) + if fds != 1 || len(masks) != 1 { + t.Fatalf("got %d inotify fds with %d watches; want 1 fd with 1 watch", fds, len(masks)) + } + const readEvents = unix.IN_OPEN | unix.IN_ACCESS | unix.IN_CLOSE_NOWRITE + if masks[0]&readEvents != 0 { + t.Errorf("watch mask %#x includes read events", masks[0]) + } + + cancel() + if err := <-errc; !errors.Is(err, context.Canceled) { + t.Errorf("watchFile = %v; want context.Canceled", err) + } + if fds, _ := inotifyWatches(t); fds != 0 { + t.Errorf("%d inotify fds still open after watchFile returned", fds) + } +} + +func TestWatchFileDirRemoved(t *testing.T) { + dir := filepath.Join(t.TempDir(), "etc") + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatal(err) + } + started := make(chan struct{}) + errc := make(chan error, 1) + go func() { + errc <- watchFile(t.Context(), dir, filepath.Join(dir, "resolv.conf"), func() { + close(started) + }) + }() + <-started + if err := os.Remove(dir); err != nil { + t.Fatal(err) + } + select { + case err := <-errc: + if !errors.Is(err, errWatchRemoved) { + t.Errorf("watchFile = %v; want errWatchRemoved", err) + } + case <-time.After(5 * time.Second): + t.Fatal("watchFile did not return after its directory was removed") + } +} + +// event encodes one inotify_event record, padding the name the way the +// kernel does. +func event(wd int32, mask uint32, name string) []byte { + nameLen := 0 + if name != "" { + nameLen = (len(name) + unix.SizeofInotifyEvent) &^ (unix.SizeofInotifyEvent - 1) + } + b := make([]byte, unix.SizeofInotifyEvent+nameLen) + binary.NativeEndian.PutUint32(b[0:], uint32(wd)) + binary.NativeEndian.PutUint32(b[4:], mask) + binary.NativeEndian.PutUint32(b[12:], uint32(nameLen)) + copy(b[unix.SizeofInotifyEvent:], name) + return b +} + +// badLen returns an inotify_event header whose name length is nameLen, +// with no name bytes after it. +func badLen(nameLen uint32) []byte { + return binary.NativeEndian.AppendUint32(make([]byte, 12), nameLen) +} + +func TestParseEvents(t *testing.T) { + cat := func(bs ...[]byte) []byte { return bytes.Join(bs, nil) } + tests := []struct { + name string + buf []byte + wantChanged bool + wantErr string // substring; empty means no error + }{ + {name: "empty"}, + {name: "match", buf: event(1, unix.IN_CLOSE_WRITE, "resolv.conf"), wantChanged: true}, + {name: "other-file", buf: event(1, unix.IN_CLOSE_WRITE, "hostname")}, + {name: "longer-name", buf: event(1, unix.IN_CREATE, "resolv.conf.tmp")}, + {name: "16-byte-name", buf: event(1, unix.IN_CREATE, "resolv.conf.abcd")}, + { + name: "match-after-others", + buf: cat( + event(1, unix.IN_CREATE, "resolv.conf.tmp"), + event(1, unix.IN_MOVED_FROM, "resolv.conf.tmp"), + event(1, unix.IN_MOVED_TO, "resolv.conf"), + ), + wantChanged: true, + }, + {name: "overflow", buf: event(-1, unix.IN_Q_OVERFLOW, ""), wantChanged: true}, + {name: "watch-removed", buf: event(1, unix.IN_IGNORED, ""), wantErr: "watch removed"}, + {name: "short-header", buf: event(1, unix.IN_CREATE, "resolv.conf")[:10], wantErr: "10 trailing bytes"}, + {name: "trailing-bytes", buf: cat(event(1, unix.IN_CREATE, "a"), []byte{1, 2, 3}), wantErr: "3 trailing bytes"}, + { + // gonotify v3.0.2 looped forever on a record like this, + // whose name runs past the end of the read (#16576). + // 761623152 is "pre-", as seen in #14699. + name: "name-past-end", + buf: badLen(761623152), + wantErr: "name length 761623152", + }, + { + name: "max-name-len-after-valid", + buf: cat(event(1, unix.IN_CREATE, "resolv.conf"), badLen(0xffffffff)), + wantErr: "name length 4294967295", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + changed, err := parseEvents(tt.buf, "resolv.conf") + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v; want containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if changed != tt.wantChanged { + t.Errorf("changed = %v; want %v", changed, tt.wantChanged) + } + }) + } +} diff --git a/flake.nix b/flake.nix index 61ebb1aeb..d68f5a45b 100644 --- a/flake.nix +++ b/flake.nix @@ -172,4 +172,4 @@ }); }; } -# nix-direnv cache busting line: sha256-2FUoQbRKmhAbpWH356v/TpgjT9KrcZekMO5E1HbLtMA= +# nix-direnv cache busting line: sha256-ZcegeCjeDDLaX7adwdoxndOjukb4H+JNKHx0xGCWXiQ= diff --git a/flakehashes.json b/flakehashes.json index b0242b71a..64818eeba 100644 --- a/flakehashes.json +++ b/flakehashes.json @@ -4,7 +4,7 @@ "sri": "sha256-Up4oMG6TVLQkV9inAYF3uYbxu+bMHeU7q9BLcXIllow=" }, "vendor": { - "goModSum": "sha256-uGmj1yPXUtncPrsDPi9pVW4y+ZJuWPCUWXpgIiGJfIo=", - "sri": "sha256-2FUoQbRKmhAbpWH356v/TpgjT9KrcZekMO5E1HbLtMA=" + "goModSum": "sha256-tp8fiukPKErrTuxmaT8TqSiJqSR6BjntVSnWnBWW7RQ=", + "sri": "sha256-ZcegeCjeDDLaX7adwdoxndOjukb4H+JNKHx0xGCWXiQ=" } } diff --git a/go.mod b/go.mod index 5318b37c8..0e06e8426 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,6 @@ require ( github.com/hashicorp/raft-boltdb/v2 v2.3.1 github.com/hdevalence/ed25519consensus v0.2.0 github.com/huin/goupnp v1.3.0 - github.com/illarion/gonotify/v3 v3.0.2 github.com/inetaf/tcpproxy v0.0.0-20260515195445-c159a6051109 github.com/insomniacslk/dhcp v0.0.0-20260901064844-234b97448fae github.com/jellydator/ttlcache/v3 v3.4.1 diff --git a/go.sum b/go.sum index b0968b636..065777532 100644 --- a/go.sum +++ b/go.sum @@ -710,8 +710,6 @@ github.com/hugelgupf/vmtest v0.0.0-20240307030256-5d9f3d34a58d h1:nP8SfQJqruIVSW github.com/hugelgupf/vmtest v0.0.0-20240307030256-5d9f3d34a58d/go.mod h1:B63hDJMhTupLWCHwopAyEo7wRFowx9kOc8m8j1sfOqE= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/illarion/gonotify/v3 v3.0.2 h1:O7S6vcopHexutmpObkeWsnzMJt/r1hONIEogeVNmJMk= -github.com/illarion/gonotify/v3 v3.0.2/go.mod h1:HWGPdPe817GfvY3w7cx6zkbzNZfi3QjcBm/wgVvEL1U= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inetaf/tcpproxy v0.0.0-20260515195445-c159a6051109 h1:H3ztMPCBcyG60Ks0plTncU3Vs6jIrxhn3nEThpMEpo4= diff --git a/net/dns/direct_linux_test.go b/net/dns/direct_linux_test.go index c053db178..9428a5e8d 100644 --- a/net/dns/direct_linux_test.go +++ b/net/dns/direct_linux_test.go @@ -76,8 +76,8 @@ search ts.net ts-dns.test } // watchFile is a test implementation of the file watcher that uses a timer -// instead of inotify. Real inotify (gonotify.NewDirWatcher) creates goroutines -// that block on real syscalls, which don't work inside synctest's fake-time +// instead of inotify. The real watcher blocks reading an inotify fd, which +// doesn't work inside synctest's fake-time // bubble. Instead, we use a one-shot timer that synctest.Wait() will advance, // triggering a callback to check for file trampling. func watchFile(ctx context.Context, dir, filename string, cb func()) error { diff --git a/shell.nix b/shell.nix index 9747f4fbf..6ee20e616 100644 --- a/shell.nix +++ b/shell.nix @@ -16,4 +16,4 @@ ) { src = ./.; }).shellNix -# nix-direnv cache busting line: sha256-2FUoQbRKmhAbpWH356v/TpgjT9KrcZekMO5E1HbLtMA= +# nix-direnv cache busting line: sha256-ZcegeCjeDDLaX7adwdoxndOjukb4H+JNKHx0xGCWXiQ=