mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
logtail: add support for authenticated log uploads (#20482)
Control via nodecap.LogUploadAuth may notify the node that there is an authorization token to use when uploading logs. If specified, then the logtail.Logger will use that token when uploading. Unfortunately, there is a time-of-use and time-of-check discrepancy that complicates logging. When logs are written under the context of being for a particular node, we cannot store the token, as it may be invalid by the time that the logs are uploaded. Instead, we add Logger.SetAuthID to annotate subsequent log entries with an identifier for the token to later use. Only when we upload do we strip out the auth ID, lookup the auth token to use, and then perform the upload. We assume that control will notify the client of new tokens well before they expire. Fetching a refreshed token is not the job of the client. Updates tailscale/corp#39092 Signed-off-by: Joe Tsai <[email protected]>
This commit is contained in:
1 parent
027c2a8253
commit
09d54b6ce2
13 files changed
+855
-51
No files matched your search
@@ -777,6 +777,7 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
|
||||
lb.SetVarRoot(opts.VarRoot)
|
||||
if logPol != nil {
|
||||
lb.SetLogFlusher(logPol.Logtail.StartFlush)
|
||||
lb.SetLogUploader(logPol.Logtail)
|
||||
}
|
||||
if root := lb.TailscaleVarRoot(); root != "" {
|
||||
dnsfallback.SetCachePath(filepath.Join(root, "derpmap.cached.json"), logf)
|
||||
|
||||
+37
-6
@@ -274,10 +274,15 @@ type LocalBackend struct {
|
||||
pushDeviceToken syncs.AtomicValue[string]
|
||||
backendLogID logid.PublicID // or zero value if logging not in use
|
||||
unregisterSysPolicyWatch func()
|
||||
varRoot string // or empty if SetVarRoot never called
|
||||
logFlushFunc func() // or nil if SetLogFlusher wasn't called
|
||||
em *expiryManager // non-nil; TODO(nickkhyl): move to nodeBackend
|
||||
sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend
|
||||
varRoot string // or empty if SetVarRoot never called
|
||||
logFlushFunc func() // or nil if SetLogFlusher wasn't called
|
||||
// logUploader is the process-wide logtail logger for tailscaled's main
|
||||
// log stream. Set via [LocalBackend.SetLogUploader]. May be nil.
|
||||
logUploader *logtail.Logger
|
||||
// logAuth caches tokens for authenticated log uploads (for node logs and netlog).
|
||||
logAuth logAuthCache
|
||||
em *expiryManager // non-nil; TODO(nickkhyl): move to nodeBackend
|
||||
sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend
|
||||
// webClientAtomicBool controls whether the web client is running. This should
|
||||
// be true unless the disable-web-client node attribute has been set.
|
||||
webClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend
|
||||
@@ -638,6 +643,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
|
||||
em: newExpiryManager(logf, sys.Bus.Get()),
|
||||
loginFlags: loginFlags,
|
||||
clock: clock,
|
||||
logAuth: logAuthCache{logf: logf},
|
||||
}
|
||||
|
||||
sys.NoiseRoundTripper.Set(noiseRoundTripper{b})
|
||||
@@ -6619,11 +6625,15 @@ func (b *LocalBackend) SetLogFlusher(flushFunc func()) {
|
||||
}
|
||||
|
||||
// SetLogUploader sets the process-wide logtail logger used for tailscaled's
|
||||
// main log stream.
|
||||
// main log stream. It enables authenticated log uploads when the node is in a
|
||||
// tailnet (see [LocalBackend.updateLogUploadAuth]).
|
||||
//
|
||||
// It should only be called before the LocalBackend is used.
|
||||
func (b *LocalBackend) SetLogUploader(lg *logtail.Logger) {
|
||||
// TODO: Implemented later for authenticated log uploads.
|
||||
if !buildfeatures.HasLogTail {
|
||||
return
|
||||
}
|
||||
b.logUploader = lg
|
||||
}
|
||||
|
||||
// TryFlushLogs calls the log flush function. It returns false if a log flush
|
||||
@@ -7695,6 +7705,15 @@ func (b *LocalBackend) resolveExitNodeInPrefsLocked(prefs *ipn.Prefs) (changed b
|
||||
// received nm. If nm is nil, it resets all configuration as though
|
||||
// Tailscale is turned off.
|
||||
func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
|
||||
// Cache any log-upload bearer token, then remove it from nm before any
|
||||
// subsystem observes the netmap. The token must not be installed into
|
||||
// the published map (status, WhoIs, SelfChange, the disk cache, or
|
||||
// debug dumps).
|
||||
if buildfeatures.HasLogTail {
|
||||
b.updateLogUploadAuth(nm)
|
||||
}
|
||||
stripLogUploadAuth(nm)
|
||||
|
||||
if buildfeatures.HasCacheNetMap {
|
||||
// As a defensive measure, if something triggers a panic when we are
|
||||
// installing a network map, make an effort to discard any cached netmaps.
|
||||
@@ -8535,6 +8554,18 @@ func (s netLogNodeSource) NetLogIDs() (nodeID, domainID logid.PrivateID, logExit
|
||||
return nodeID, domainID, nm.SelfNode.HasCap(nodecap.LogExitFlows), true
|
||||
}
|
||||
|
||||
// LogUploadAuth implements [wgengine.NetLogSource].
|
||||
func (s netLogNodeSource) LogUploadAuth() (authID string, authToken func() string) {
|
||||
nm := s.b.NetMap()
|
||||
if nm == nil || !nm.SelfNode.Valid() {
|
||||
return "", nil
|
||||
}
|
||||
id := formatAuthID(nm.SelfNode.ID())
|
||||
return string(id), func() string {
|
||||
return string(s.b.logAuth.loadToken(id))
|
||||
}
|
||||
}
|
||||
|
||||
// Compile-time assertion that netLogNodeSource implements
|
||||
// [wgengine.NetLogSource].
|
||||
var _ wgengine.NetLogSource = netLogNodeSource{}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package ipnlocal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
"tailscale.com/feature/buildfeatures"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/tailcfg/nodecap"
|
||||
"tailscale.com/types/logger"
|
||||
"tailscale.com/types/netmap"
|
||||
"tailscale.com/util/mak"
|
||||
)
|
||||
|
||||
type logAuthID string
|
||||
type logAuthToken string
|
||||
|
||||
// logAuthCache caches log upload auth tokens keyed by auth ID ("nodeid:<nodeID>").
|
||||
// Tokens arrive proactively from control via [nodecap.LogUploadAuth]
|
||||
// on netmap updates; the client does not refresh them itself.
|
||||
//
|
||||
// All methods are safe for concurrent use.
|
||||
type logAuthCache struct {
|
||||
logf logger.Logf
|
||||
|
||||
mu sync.Mutex
|
||||
tokens map[logAuthID]logAuthToken
|
||||
}
|
||||
|
||||
// storeToken stores token for authID.
|
||||
func (c *logAuthCache) storeToken(id logAuthID, tok logAuthToken) {
|
||||
if id == "" || tok == "" {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
mak.Set(&c.tokens, id, tok)
|
||||
}
|
||||
|
||||
// loadToken returns the cached bearer token for authID, or "" if none.
|
||||
// An empty result means the uploader proceeds unauthenticated (matching
|
||||
// historical logtail behavior when no auth is available).
|
||||
func (c *logAuthCache) loadToken(id logAuthID) logAuthToken {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.tokens[id]
|
||||
}
|
||||
|
||||
// updateLogUploadAuth applies netmap-driven log upload authentication
|
||||
// to the main tailscaled [logtail.Logger] and updates the token cache.
|
||||
//
|
||||
// When nm is nil or lacks a self node, new log entries are stamped with an
|
||||
// empty auth ID, but previously cached tokens are retained so buffered logs
|
||||
// from the prior node can still upload with Authorization.
|
||||
//
|
||||
// When CapMap carries [nodecap.LogUploadAuth], the token is cached.
|
||||
// Control is expected to push a refreshed token via netmap
|
||||
// before the previous one expires.
|
||||
//
|
||||
// Even when a new auth token arrives as a concise netmap delta
|
||||
// with just the self CapMap for [nodecap.LogUploadAuth],
|
||||
// the LocalBackend still triggers a full netmap install.
|
||||
// Note that peer deltas cannot carry this attribute.
|
||||
//
|
||||
// [LocalBackend.setNetMapLocked] caches the token via this method, then
|
||||
// [stripLogUploadAuth] removes the capability before the map is installed.
|
||||
// A token the log service rejects is unstuck by the service, not by falling
|
||||
// back to an unauthenticated upload. See [tailscale.com/logtail.Logger.upload].
|
||||
func (b *LocalBackend) updateLogUploadAuth(nm *netmap.NetworkMap) {
|
||||
if !buildfeatures.HasLogTail {
|
||||
return
|
||||
}
|
||||
|
||||
lg := b.logUploader
|
||||
if lg == nil || nm == nil || !nm.SelfNode.Valid() {
|
||||
if lg != nil {
|
||||
lg.SetAuthID("")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Cache any token provided in the netmap CapMap.
|
||||
authID := formatAuthID(nm.SelfNode.ID())
|
||||
tokens, err := tailcfg.UnmarshalNodeCapViewJSON[string](nm.SelfNode.CapMap(), nodecap.LogUploadAuth)
|
||||
if err != nil {
|
||||
b.logf("logauth: CapMap %s: %v", nodecap.LogUploadAuth, err)
|
||||
return
|
||||
}
|
||||
for _, tok := range tokens {
|
||||
if tok != "" {
|
||||
b.logAuth.storeToken(authID, logAuthToken(tok))
|
||||
}
|
||||
}
|
||||
|
||||
// Configure the logger to log under the context of the current node.
|
||||
lg.SetAuthID(string(authID))
|
||||
lg.RegisterAuthToken(string(authID), func() string {
|
||||
return string(b.logAuth.loadToken(authID))
|
||||
})
|
||||
}
|
||||
|
||||
// formatAuthID returns the canonical auth ID for authenticated log uploads:
|
||||
// "nodeid:<nodeID>". A node ID change implies a tailnet change, so the auth ID
|
||||
// does not include a separate tailnet component.
|
||||
func formatAuthID(nodeID tailcfg.NodeID) logAuthID {
|
||||
return logAuthID(fmt.Sprintf("nodeid:%d", nodeID))
|
||||
}
|
||||
|
||||
// stripLogUploadAuth removes [nodecap.LogUploadAuth] from nm.
|
||||
//
|
||||
// The capability value is a bearer token. Leaving it on the installed netmap
|
||||
// publishes it through status JSON, WhoIs, SelfChange notifications, the
|
||||
// netmap disk cache, and debug netmap dumps. Callers must cache the token
|
||||
// (see [LocalBackend.updateLogUploadAuth]) before calling this.
|
||||
//
|
||||
// nm is updated in place. The self node and AllCaps set are cloned first so a
|
||||
// control-client copy that aliases them keeps the original token.
|
||||
func stripLogUploadAuth(nm *netmap.NetworkMap) {
|
||||
if nm == nil || !nm.SelfNode.Valid() {
|
||||
return
|
||||
}
|
||||
if !nm.SelfNode.CapMap().Contains(nodecap.LogUploadAuth) && !nm.AllCaps.Contains(nodecap.LogUploadAuth) {
|
||||
return
|
||||
}
|
||||
self := nm.SelfNode.AsStruct()
|
||||
delete(self.CapMap, nodecap.LogUploadAuth)
|
||||
self.Capabilities = slices.DeleteFunc(self.Capabilities, func(c nodecap.Cap) bool {
|
||||
return c == nodecap.LogUploadAuth
|
||||
})
|
||||
nm.SelfNode = self.View()
|
||||
if nm.AllCaps.Contains(nodecap.LogUploadAuth) {
|
||||
// AllCaps may alias the control client's cap set.
|
||||
nm.AllCaps = nm.AllCaps.Clone()
|
||||
nm.AllCaps.Delete(nodecap.LogUploadAuth)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package ipnlocal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"tailscale.com/feature/buildfeatures"
|
||||
"tailscale.com/logtail"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/tailcfg/nodecap"
|
||||
"tailscale.com/types/netmap"
|
||||
"tailscale.com/util/set"
|
||||
)
|
||||
|
||||
func TestLogAuthCachePutAndToken(t *testing.T) {
|
||||
c := &logAuthCache{logf: t.Logf}
|
||||
authID := formatAuthID(2)
|
||||
const tok = "test-jwt-token"
|
||||
c.storeToken(authID, tok)
|
||||
if got := c.loadToken(authID); got != tok {
|
||||
t.Errorf("token = %q; want %q", got, tok)
|
||||
}
|
||||
if got := c.loadToken("missing"); got != "" {
|
||||
t.Errorf("missing authID token = %q; want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateLogUploadAuth(t *testing.T) {
|
||||
lg := logtail.NewLogger(logtail.Config{
|
||||
BaseURL: "http://127.0.0.1:1", // never dialed; we only check SetAuthID
|
||||
HTTPC: nil,
|
||||
}, t.Logf)
|
||||
// Immediately shut down the upload goroutine to avoid noise.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_ = lg.Shutdown(ctx)
|
||||
|
||||
b := &LocalBackend{
|
||||
logf: t.Logf,
|
||||
logUploader: lg,
|
||||
logAuth: logAuthCache{logf: t.Logf},
|
||||
}
|
||||
|
||||
const tok = "eyJ.test.jwt"
|
||||
node := &tailcfg.Node{
|
||||
ID: 42,
|
||||
CapMap: tailcfg.NodeCapMap{
|
||||
nodecap.LogUploadAuth: {tailcfg.RawMessage(`"` + tok + `"`)},
|
||||
},
|
||||
}
|
||||
nm := &netmap.NetworkMap{
|
||||
SelfNode: node.View(),
|
||||
}
|
||||
b.updateLogUploadAuth(nm)
|
||||
wantID := formatAuthID(42)
|
||||
if got := b.logAuth.loadToken(wantID); got != tok {
|
||||
t.Errorf("cached token = %q; want %q", got, tok)
|
||||
}
|
||||
|
||||
// A later netmap can push a refreshed token for the same auth ID.
|
||||
const tok2 = "eyJ.refreshed.jwt"
|
||||
node2 := &tailcfg.Node{
|
||||
ID: 42,
|
||||
CapMap: tailcfg.NodeCapMap{
|
||||
nodecap.LogUploadAuth: {tailcfg.RawMessage(`"` + tok2 + `"`)},
|
||||
},
|
||||
}
|
||||
nm2 := &netmap.NetworkMap{
|
||||
SelfNode: node2.View(),
|
||||
}
|
||||
b.updateLogUploadAuth(nm2)
|
||||
if got := b.logAuth.loadToken(wantID); got != tok2 {
|
||||
t.Errorf("after refresh push, token = %q; want %q", got, tok2)
|
||||
}
|
||||
|
||||
// Logout / nil netmap: clear SetAuthID but keep cached tokens for buffered logs.
|
||||
b.updateLogUploadAuth(nil)
|
||||
if got := b.logAuth.loadToken(wantID); got != tok2 {
|
||||
t.Errorf("after logout, cached token = %q; want retained %q", got, tok2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatAuthID(t *testing.T) {
|
||||
const want = "nodeid:456"
|
||||
got := formatAuthID(456)
|
||||
if got != want {
|
||||
t.Errorf("formatAuthID = %q; want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogUploadAuthNotPublished(t *testing.T) {
|
||||
b := newTestLocalBackend(t)
|
||||
if buildfeatures.HasLogTail {
|
||||
lg := logtail.NewLogger(logtail.Config{
|
||||
BaseURL: "http://127.0.0.1:1", // never dialed
|
||||
}, t.Logf)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_ = lg.Shutdown(ctx)
|
||||
b.SetLogUploader(lg)
|
||||
}
|
||||
|
||||
const tok = "super-secret-log-token"
|
||||
const other = "keep-me"
|
||||
node := &tailcfg.Node{
|
||||
ID: 7,
|
||||
User: 1,
|
||||
Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.7/32")},
|
||||
Capabilities: []nodecap.Cap{nodecap.LogUploadAuth, nodecap.Debug},
|
||||
CapMap: tailcfg.NodeCapMap{
|
||||
nodecap.LogUploadAuth: {tailcfg.RawMessage(`"` + tok + `"`)},
|
||||
nodecap.Debug: {tailcfg.RawMessage(`"` + other + `"`)},
|
||||
},
|
||||
}
|
||||
caps := set.Of(nodecap.LogUploadAuth, nodecap.Debug)
|
||||
nm := &netmap.NetworkMap{
|
||||
SelfNode: node.View(),
|
||||
AllCaps: caps,
|
||||
UserProfiles: map[tailcfg.UserID]tailcfg.UserProfileView{
|
||||
1: (&tailcfg.UserProfile{ID: 1, LoginName: "[email protected]"}).View(),
|
||||
},
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.setNetMapLocked(nm)
|
||||
published := b.currentNode().NetMap()
|
||||
b.mu.Unlock()
|
||||
|
||||
if published.SelfNode.CapMap().Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("installed CapMap still contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
if published.AllCaps.Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("installed AllCaps still contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
if !published.SelfNode.CapMap().Contains(nodecap.Debug) {
|
||||
t.Errorf("installed CapMap dropped %s", nodecap.Debug)
|
||||
}
|
||||
if slices.Contains(published.SelfNode.Capabilities().AsSlice(), nodecap.LogUploadAuth) {
|
||||
t.Errorf("installed Capabilities still contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
if !slices.Contains(published.SelfNode.Capabilities().AsSlice(), nodecap.Debug) {
|
||||
t.Errorf("installed Capabilities dropped %s", nodecap.Debug)
|
||||
}
|
||||
if !published.AllCaps.Contains(nodecap.Debug) {
|
||||
t.Errorf("installed AllCaps dropped %s", nodecap.Debug)
|
||||
}
|
||||
// The control-client node and cap set must be left intact.
|
||||
if !node.CapMap.Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("source node CapMap was mutated")
|
||||
}
|
||||
if !caps.Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("source AllCaps was mutated")
|
||||
}
|
||||
if buildfeatures.HasLogTail {
|
||||
if got := b.logAuth.loadToken(formatAuthID(node.ID)); got != tok {
|
||||
t.Errorf("cached token = %q; want %q", got, tok)
|
||||
}
|
||||
}
|
||||
|
||||
// Same netmap pointer the caller uses for SelfChange notifications.
|
||||
if nm.SelfNode.CapMap().Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("SelfChange netmap still contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
|
||||
st := b.Status()
|
||||
if st.Self == nil {
|
||||
t.Fatal("status self node is nil")
|
||||
}
|
||||
if st.Self.CapMap.Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("status CapMap contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
if !st.Self.CapMap.Contains(nodecap.Debug) {
|
||||
t.Errorf("status CapMap dropped %s", nodecap.Debug)
|
||||
}
|
||||
|
||||
who, _, ok := b.WhoIs("", netip.AddrPortFrom(netip.MustParseAddr("100.64.0.7"), 0))
|
||||
if !ok {
|
||||
t.Fatal("WhoIs(self) failed")
|
||||
}
|
||||
if who.CapMap().Contains(nodecap.LogUploadAuth) {
|
||||
t.Errorf("WhoIs CapMap contains %s", nodecap.LogUploadAuth)
|
||||
}
|
||||
|
||||
// Status JSON, WhoIs, and the netmap used by the c2n debug dump.
|
||||
publishedJSON, err := json.Marshal(published)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal published netmap: %v", err)
|
||||
}
|
||||
statusJSON, err := json.Marshal(st)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal status: %v", err)
|
||||
}
|
||||
whoJSON, err := json.Marshal(who)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal WhoIs node: %v", err)
|
||||
}
|
||||
dump := b.currentNode().netMapWithPeers()
|
||||
dumpJSON, err := json.Marshal(dump)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal netMapWithPeers: %v", err)
|
||||
}
|
||||
for _, blob := range []struct {
|
||||
name string
|
||||
js []byte
|
||||
}{
|
||||
{"published netmap", publishedJSON},
|
||||
{"status", statusJSON},
|
||||
{"WhoIs", whoJSON},
|
||||
{"c2n netmap", dumpJSON},
|
||||
} {
|
||||
if bytes.Contains(blob.js, []byte(tok)) {
|
||||
t.Errorf("%s JSON contains the log upload token", blob.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
+252
-33
@@ -38,6 +38,7 @@ import (
|
||||
tslogger "tailscale.com/types/logger"
|
||||
"tailscale.com/types/logid"
|
||||
"tailscale.com/util/eventbus"
|
||||
"tailscale.com/util/mak"
|
||||
"tailscale.com/util/set"
|
||||
"tailscale.com/util/truncate"
|
||||
"tailscale.com/util/zstdframe"
|
||||
@@ -154,6 +155,11 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
|
||||
// Zero-valued fields are omitted when an entry is uploaded. [UploadLogs] fills
|
||||
// in any fields the caller leaves zero from its [Config] (e.g. ClientTime and
|
||||
// ProcID), so most callers can leave this empty.
|
||||
//
|
||||
// AuthID is used only while an entry is buffered.
|
||||
// [Logger] writes it into each entry from [Logger.SetAuthID] and
|
||||
// strips it before the HTTP POST so it never appears
|
||||
// in the payload the log server receives.
|
||||
type Logtail struct {
|
||||
// ClientTime is the time the entry was generated. If zero, [UploadLogs]
|
||||
// fills it with the current time unless Config.SkipClientTime is set.
|
||||
@@ -163,6 +169,9 @@ type Logtail struct {
|
||||
// ProcSeq is an ephemeral per-process sequence number; see
|
||||
// Config.IncludeProcSequence.
|
||||
ProcSeq uint64 `json:"proc_seq,omitzero"`
|
||||
// AuthID is an opaque authentication ID (see [Logger.SetAuthID]).
|
||||
// It is stripped before upload and is not part of the log payload.
|
||||
AuthID string `json:"auth_id,omitzero"`
|
||||
}
|
||||
|
||||
// LogEntry is a single log entry to be uploaded via [UploadLogs].
|
||||
@@ -238,7 +247,9 @@ func UploadLogs[T any](ctx context.Context, conf Config, entries iter.Seq[LogEnt
|
||||
}
|
||||
// upload is synchronous, so it is safe to reuse body's backing array
|
||||
// for the next batch once upload returns.
|
||||
if _, err := lg.upload(ctx, out, origlen); err != nil {
|
||||
// UploadLogs does not use auth IDs; callers that need authenticated
|
||||
// uploads should use [Logger] with [Logger.SetAuthID].
|
||||
if _, err := lg.upload(ctx, out, origlen, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
body = body[:len("[")]
|
||||
@@ -319,6 +330,28 @@ type Logger struct {
|
||||
// Disable kill switch, which takes precedence. Toggled by SetEnabled.
|
||||
disabled atomic.Bool
|
||||
|
||||
// authID is the current authentication ID to stamp onto new log entries
|
||||
// (see [Logger.SetAuthID]). Empty string means unauthenticated.
|
||||
authID atomic.Value // string
|
||||
|
||||
// authTokens maps authID to a function that returns the Authorization
|
||||
// token to use when uploading a batch for that authID.
|
||||
// Guarded by authTokensMu.
|
||||
authTokensMu sync.Mutex
|
||||
authTokens map[string]func() string
|
||||
|
||||
// linePending is a line from Buffer.TryReadLine that was pulled into a
|
||||
// drain but belongs to the next upload batch (different auth_id).
|
||||
//
|
||||
// It aliases the buffer's returned slice. Buffer documents that the
|
||||
// slice is only valid until the next TryReadLine; that is safe here
|
||||
// because nextDrainLine is the sole TryReadLine call site, and it
|
||||
// always returns a non-nil linePending (clearing the field) before
|
||||
// calling TryReadLine again. Owned by the uploading goroutine.
|
||||
linePending []byte // owned by [Logger.uploading] goroutine for use by [Logger.drainPending]
|
||||
lineBuf bytes.Buffer // owned by [Logger.uploading] goroutine for use by [Logger.appendEntry]
|
||||
lineDec jsontext.Decoder // owned by [Logger.uploading] goroutine for use by [Logger.appendEntry]
|
||||
|
||||
writeLock sync.Mutex // guards procSequence, flushTimer, buffer.Write calls
|
||||
procSequence uint64
|
||||
flushTimer tstime.TimerController // used when flushDelay is >0
|
||||
@@ -363,6 +396,56 @@ func (lg *Logger) SetSockstatsLabel(label sockstats.Label) {
|
||||
lg.sockstatsLabel.Store(label)
|
||||
}
|
||||
|
||||
// SetAuthID specifies an arbitrary authentication ID
|
||||
// (e.g., "nodeid:456") to annotate every subsequent log entry.
|
||||
// This is deliberately an opaque ID to allow the application-layer
|
||||
// decide how to associate particular authentication tokens
|
||||
// to authentication IDs (see [Logger.RegisterAuthToken]).
|
||||
//
|
||||
// This ID is not uploaded as part of the log entry payload,
|
||||
// but used at the moment of uploading to determine
|
||||
// the authentication token to use when uploading.
|
||||
//
|
||||
// It is safe to concurrently call this method with other Logger methods.
|
||||
func (lg *Logger) SetAuthID(id string) {
|
||||
lg.authID.Store(id)
|
||||
}
|
||||
|
||||
// getAuthID returns the auth ID set by [Logger.SetAuthID].
|
||||
func (lg *Logger) getAuthID() string {
|
||||
v, _ := lg.authID.Load().(string)
|
||||
return v
|
||||
}
|
||||
|
||||
// RegisterAuthToken registers a dynamically-derived authToken to be called
|
||||
// for uploading logs for a particular authID (see [Logger.SetAuthID]),
|
||||
// replacing any previously registered authToken for the given authID.
|
||||
// If authToken is nil or returns an empty string, then no authorization
|
||||
// token is specified when uploading logs. It is permitted to register
|
||||
// an authToken for an empty authID.
|
||||
//
|
||||
// It is safe to concurrently call this method with other Logger methods.
|
||||
func (lg *Logger) RegisterAuthToken(authID string, authToken func() string) {
|
||||
lg.authTokensMu.Lock()
|
||||
defer lg.authTokensMu.Unlock()
|
||||
if authToken != nil {
|
||||
mak.Set(&lg.authTokens, authID, authToken)
|
||||
} else {
|
||||
delete(lg.authTokens, authID)
|
||||
}
|
||||
}
|
||||
|
||||
// lookupAuthToken returns the Authorization token for authID, or "" if none.
|
||||
func (lg *Logger) lookupAuthToken(authID string) string {
|
||||
lg.authTokensMu.Lock()
|
||||
fn := lg.authTokens[authID]
|
||||
lg.authTokensMu.Unlock()
|
||||
if fn == nil {
|
||||
return ""
|
||||
}
|
||||
return fn()
|
||||
}
|
||||
|
||||
// PrivateID returns the logger's private log ID.
|
||||
//
|
||||
// It exists for internal use only.
|
||||
@@ -432,8 +515,14 @@ func (lg *Logger) drainBlock() (shuttingDown bool) {
|
||||
|
||||
// drainPending drains and encodes a batch of logs from the buffer for upload.
|
||||
// If no logs are available, drainPending blocks until logs are available.
|
||||
//
|
||||
// Every entry in the returned batch shares the same authID (the value of
|
||||
// /logtail/auth_id when the entry was written). Entries with a different
|
||||
// auth_id are held for a subsequent batch. Auth IDs are stripped from the
|
||||
// body so they are never uploaded.
|
||||
//
|
||||
// The returned buffer is only valid until the next call to drainPending.
|
||||
func (lg *Logger) drainPending() (b []byte) {
|
||||
func (lg *Logger) drainPending() (b []byte, authID string) {
|
||||
b = lg.drainBuf[:0]
|
||||
b = append(b, '[')
|
||||
defer func() {
|
||||
@@ -446,21 +535,25 @@ func (lg *Logger) drainPending() (b []byte) {
|
||||
}()
|
||||
|
||||
maxLen := cmp.Or(lg.maxUploadSize, maxSize)
|
||||
var batchAuthID string
|
||||
var batchStarted bool
|
||||
for len(b) < maxLen {
|
||||
line, err := lg.buffer.TryReadLine()
|
||||
line, err := lg.nextDrainLine()
|
||||
switch {
|
||||
case err == io.EOF:
|
||||
return b
|
||||
return b, batchAuthID
|
||||
case err != nil:
|
||||
// Synthesized error entries inherit the batch authID if any so far,
|
||||
// otherwise an empty authID (unauthenticated).
|
||||
b = append(b, '{')
|
||||
b = lg.appendMetadata(b, false, true, 0, 0, "reading ringbuffer: "+err.Error(), nil, 0)
|
||||
b = lg.appendMetadata(b, false, true, 0, 0, "", "reading ringbuffer: "+err.Error(), nil, 0)
|
||||
b = bytes.TrimRight(b, ",")
|
||||
b = append(b, '}')
|
||||
return b
|
||||
return b, batchAuthID
|
||||
case line == nil:
|
||||
// If we read at least some log entries, return immediately.
|
||||
if len(b) > len("[") {
|
||||
return b
|
||||
return b, batchAuthID
|
||||
}
|
||||
|
||||
// We're about to block. If we're holding on to too much memory
|
||||
@@ -471,22 +564,37 @@ func (lg *Logger) drainPending() (b []byte) {
|
||||
}
|
||||
|
||||
if shuttingDown := lg.drainBlock(); shuttingDown {
|
||||
return b
|
||||
return b, batchAuthID
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case len(line) == 0:
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
case line[0] == '{' && jsontext.Value(line).IsValid():
|
||||
// This is already a valid JSON object, so just append it.
|
||||
// This may exceed maxLen, but should be no larger than maxSize
|
||||
// so long as logic writing into the buffer enforces the limit.
|
||||
b = append(b, line...)
|
||||
default:
|
||||
// This is probably a log added to stderr by filch
|
||||
// outside of the logtail logger. Encode it.
|
||||
}
|
||||
|
||||
// This may exceed maxLen, but should be no larger than maxSize
|
||||
// so long as logic writing into the buffer enforces the limit.
|
||||
next, entryAuthID, valid := lg.appendEntry(b, line)
|
||||
if !valid {
|
||||
// Non-JSON (or otherwise unparseable) line.
|
||||
// Typically raw stderr captured by filch outside logtail.
|
||||
// There is no right auth ID to use, so use the current
|
||||
// SetAuthID as a chronologically likely guess.
|
||||
entryAuthID = lg.getAuthID()
|
||||
}
|
||||
if batchStarted && entryAuthID != batchAuthID {
|
||||
// Different auth ID: hold this line for the next batch.
|
||||
// Leave b unchanged (do not assign next).
|
||||
// See [Logger.linePending] for why it is safe not to copy the line.
|
||||
lg.linePending = line
|
||||
return b, batchAuthID
|
||||
}
|
||||
|
||||
b = next
|
||||
if !valid {
|
||||
// Commit the RAW line only after the auth batch check so we
|
||||
// do not print/encode twice when the line is held for later.
|
||||
if !lg.explainedRaw {
|
||||
fmt.Fprintf(lg.stderr, "RAW-STDERR: ***\n")
|
||||
fmt.Fprintf(lg.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
|
||||
@@ -494,15 +602,102 @@ func (lg *Logger) drainPending() (b []byte) {
|
||||
fmt.Fprintf(lg.stderr, "RAW-STDERR:\n")
|
||||
lg.explainedRaw = true
|
||||
}
|
||||
fmt.Fprintf(lg.stderr, "RAW-STDERR: %s", b)
|
||||
fmt.Fprintf(lg.stderr, "RAW-STDERR: %s", line)
|
||||
// Do not add a client time, as it could be really old.
|
||||
// Do not include instance key or ID either,
|
||||
// since this came from a different instance.
|
||||
b = lg.appendText(b, line, true, 0, 0, 0)
|
||||
// Do not stamp auth_id into the body since the batch's
|
||||
// Authorization header carries entryAuthID from getAuthID above.
|
||||
b = lg.appendText(b, line, true, 0, 0, "", 0)
|
||||
}
|
||||
if !batchStarted {
|
||||
batchAuthID = entryAuthID // may be empty
|
||||
batchStarted = true
|
||||
}
|
||||
b = append(b, ',')
|
||||
}
|
||||
return b
|
||||
return b, batchAuthID
|
||||
}
|
||||
|
||||
// nextDrainLine returns the next line to drain: either a previously held
|
||||
// pending line, or a fresh line from the buffer.
|
||||
//
|
||||
// See [Logger.linePending] for the lifetime invariant with TryReadLine.
|
||||
func (lg *Logger) nextDrainLine() ([]byte, error) {
|
||||
if line := lg.linePending; line != nil {
|
||||
lg.linePending = nil
|
||||
return line, nil
|
||||
}
|
||||
return lg.buffer.TryReadLine()
|
||||
}
|
||||
|
||||
// appendEntry appends a JSON log entry in src to dst if it is valid.
|
||||
// If /logtail/auth_id is populated, it extracts it out of the log entry.
|
||||
// If valid is false, then out is identical to dst.
|
||||
func (lg *Logger) appendEntry(dst, src []byte) (out []byte, authID string, valid bool) {
|
||||
lg.lineBuf = *bytes.NewBuffer(src)
|
||||
defer func() { lg.lineBuf = bytes.Buffer{} }() // avoid pinning src
|
||||
dec := &lg.lineDec
|
||||
dec.Reset(&lg.lineBuf)
|
||||
|
||||
// Scan through the JSON entry, identifying the offsets
|
||||
// for the /logtail/auth_id field, so that we can strip it out later.
|
||||
var startAuthIDOffset, endAuthIDOffset int64
|
||||
if tok, err := dec.ReadToken(); err != nil || tok.Kind() != '{' {
|
||||
return dst, "", false
|
||||
}
|
||||
for dec.PeekKind() != '}' {
|
||||
switch tok, err := dec.ReadToken(); {
|
||||
case err != nil:
|
||||
return dst, "", false
|
||||
case tok.String() == "logtail":
|
||||
if tok, err := dec.ReadToken(); err != nil || tok.Kind() != '{' {
|
||||
return dst, "", false
|
||||
}
|
||||
for dec.PeekKind() != '}' {
|
||||
startOffset := dec.InputOffset()
|
||||
switch tok, err := dec.ReadToken(); {
|
||||
case err != nil:
|
||||
return dst, "", false
|
||||
case tok.String() == "auth_id":
|
||||
tok, err := dec.ReadToken()
|
||||
if err != nil || tok.Kind() != '"' {
|
||||
return dst, "", false
|
||||
}
|
||||
authID = tok.String()
|
||||
startAuthIDOffset = startOffset
|
||||
endAuthIDOffset = dec.InputOffset()
|
||||
|
||||
// If this is the first member and there is a following member,
|
||||
// then we need to strip the subsequent comma.
|
||||
// We check n == 2 since we already read one member.
|
||||
if _, n := dec.StackIndex(dec.StackDepth()); n == 2 && dec.PeekKind() != '}' {
|
||||
endAuthIDOffset += int64(len(dec.UnreadBuffer()) - len(bytes.TrimLeft(dec.UnreadBuffer(), " \n\r\t")) + len(","))
|
||||
}
|
||||
default:
|
||||
if err := dec.SkipValue(); err != nil {
|
||||
return dst, "", false
|
||||
}
|
||||
}
|
||||
}
|
||||
if tok, err := dec.ReadToken(); err != nil || tok.Kind() != '}' {
|
||||
return dst, "", false
|
||||
}
|
||||
default:
|
||||
if err := dec.SkipValue(); err != nil {
|
||||
return dst, "", false
|
||||
}
|
||||
}
|
||||
}
|
||||
if tok, err := dec.ReadToken(); err != nil || tok.Kind() != '}' {
|
||||
return dst, "", false
|
||||
}
|
||||
if _, err := dec.ReadValue(); err != io.EOF {
|
||||
return dst, "", false // implies trailing data after JSON object
|
||||
}
|
||||
|
||||
// Append the log entry, stripping out /logtail/auth_id field if present.
|
||||
return append(append(dst, src[:startAuthIDOffset]...), src[endAuthIDOffset:]...), authID, true
|
||||
}
|
||||
|
||||
// This is the goroutine that repeatedly uploads logs in the background.
|
||||
@@ -510,7 +705,7 @@ func (lg *Logger) uploading(ctx context.Context) {
|
||||
defer close(lg.shutdownDone)
|
||||
|
||||
for {
|
||||
body := lg.drainPending()
|
||||
body, authID := lg.drainPending()
|
||||
origlen := -1 // sentinel value: uncompressed
|
||||
// Don't attempt to compress tiny bodies; not worth the CPU cycles.
|
||||
if lg.compressLogs && len(body) > 256 {
|
||||
@@ -533,8 +728,10 @@ func (lg *Logger) uploading(ctx context.Context) {
|
||||
if logtailDisabled.Load() || lg.disabled.Load() {
|
||||
break
|
||||
}
|
||||
retryAfter, err := lg.upload(ctx, body, origlen)
|
||||
retryAfter, err := lg.upload(ctx, body, origlen, authID)
|
||||
if err != nil {
|
||||
// Retries call upload again, which re-reads the auth token.
|
||||
// The log service unsticks a rejected token; see [Logger.upload].
|
||||
numFailures++
|
||||
firstFailure = lg.clock.Now()
|
||||
|
||||
@@ -631,7 +828,18 @@ func (lg *Logger) awaitInternetUp(ctx context.Context) {
|
||||
// upload uploads body to the log server.
|
||||
// origlen indicates the pre-compression body length.
|
||||
// origlen of -1 indicates that the body is not compressed.
|
||||
func (lg *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAfter time.Duration, err error) {
|
||||
// authID is the authentication ID shared by every entry in body
|
||||
// (already stripped from the JSON); when a non-empty token is registered,
|
||||
// the request includes an Authorization: Bearer header.
|
||||
//
|
||||
// The log service unsticks a client that presents a bad auth token. It may
|
||||
// deliberately tell the client the token is bad (any non-200) and have the
|
||||
// client retry. Each attempt reads the token again, so a later try succeeds
|
||||
// after control pushes a new log token. Or the service may accept the upload
|
||||
// (and drop the logs or record them as unauthenticated).
|
||||
// The client does not fall back to uploading the batch without Authorization.
|
||||
// Obtaining a replacement token is control's job, via the netmap.
|
||||
func (lg *Logger) upload(ctx context.Context, body []byte, origlen int, authID string) (retryAfter time.Duration, err error) {
|
||||
lg.uploadCalls.Add(1)
|
||||
startUpload := time.Now()
|
||||
|
||||
@@ -651,6 +859,13 @@ func (lg *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAf
|
||||
req.Header.Add("Content-Encoding", "zstd")
|
||||
req.Header.Add("Orig-Content-Length", strconv.Itoa(origlen))
|
||||
}
|
||||
if token := lg.lookupAuthToken(authID); token != "" {
|
||||
// TODO: If authID is non-empty, but the token is empty,
|
||||
// then it implies that we might have just logged in as a node,
|
||||
// but not yet received a full netmap with the token.
|
||||
// Perhaps wait a bit before checking again?
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if runtime.GOOS == "js" {
|
||||
// We once advertised we'd accept optional client certs (for internal use)
|
||||
// on log.tailscale.com but then Tailscale SSH js/wasm clients prompted
|
||||
@@ -806,9 +1021,8 @@ func (lg *Logger) sendLocked(jsonBlob []byte) (int, error) {
|
||||
// appendMetadata appends optional "logtail", "metrics", and "v" JSON members.
|
||||
// This assumes dst is already within a JSON object.
|
||||
// Each member is comma-terminated.
|
||||
func (lg *Logger) appendMetadata(dst []byte, skipClientTime, skipMetrics bool, procID uint32, procSequence uint64, errDetail string, errData jsontext.Value, level int) []byte {
|
||||
// Append optional logtail metadata.
|
||||
if !skipClientTime || procID != 0 || procSequence != 0 || errDetail != "" || errData != nil {
|
||||
func (lg *Logger) appendMetadata(dst []byte, skipClientTime, skipMetrics bool, procID uint32, procSequence uint64, authID, errDetail string, errData jsontext.Value, level int) []byte {
|
||||
if !skipClientTime || procID != 0 || procSequence != 0 || errDetail != "" || errData != nil || authID != "" {
|
||||
dst = append(dst, `"logtail":{`...)
|
||||
if !skipClientTime {
|
||||
dst = append(dst, `"client_time":"`...)
|
||||
@@ -825,6 +1039,11 @@ func (lg *Logger) appendMetadata(dst []byte, skipClientTime, skipMetrics bool, p
|
||||
dst = strconv.AppendUint(dst, procSequence, 10)
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
if authID != "" {
|
||||
dst = append(dst, `"auth_id":`...)
|
||||
dst, _ = jsontext.AppendQuote(dst, authID)
|
||||
dst = append(dst, ',')
|
||||
}
|
||||
if errDetail != "" || errData != nil {
|
||||
dst = append(dst, `"error":{`...)
|
||||
if errDetail != "" {
|
||||
@@ -866,10 +1085,10 @@ func (lg *Logger) appendMetadata(dst []byte, skipClientTime, skipMetrics bool, p
|
||||
}
|
||||
|
||||
// appendText appends a raw text message in the Tailscale JSON log entry format.
|
||||
func (lg *Logger) appendText(dst, src []byte, skipClientTime bool, procID uint32, procSequence uint64, level int) []byte {
|
||||
func (lg *Logger) appendText(dst, src []byte, skipClientTime bool, procID uint32, procSequence uint64, authID string, level int) []byte {
|
||||
dst = slices.Grow(dst, len(src))
|
||||
dst = append(dst, '{')
|
||||
dst = lg.appendMetadata(dst, skipClientTime, false, procID, procSequence, "", nil, level)
|
||||
dst = lg.appendMetadata(dst, skipClientTime, false, procID, procSequence, authID, "", nil, level)
|
||||
if len(src) == 0 {
|
||||
dst = bytes.TrimRight(dst, ",")
|
||||
return append(dst, "}\n"...)
|
||||
@@ -904,7 +1123,7 @@ func (lg *Logger) appendTextOrJSONLocked(dst, src []byte, level int) []byte {
|
||||
lg.procSequence++
|
||||
}
|
||||
if len(src) == 0 || src[0] != '{' {
|
||||
return lg.appendText(dst, src, lg.skipClientTime, lg.procID, lg.procSequence, level)
|
||||
return lg.appendText(dst, src, lg.skipClientTime, lg.procID, lg.procSequence, lg.getAuthID(), level)
|
||||
}
|
||||
|
||||
// Check whether the input is a valid JSON object and
|
||||
@@ -952,7 +1171,7 @@ func (lg *Logger) appendTextOrJSONLocked(dst, src []byte, level int) []byte {
|
||||
|
||||
// Treat invalid JSON as a raw text message.
|
||||
if !validJSON {
|
||||
return lg.appendText(dst, src, lg.skipClientTime, lg.procID, lg.procSequence, level)
|
||||
return lg.appendText(dst, src, lg.skipClientTime, lg.procID, lg.procSequence, lg.getAuthID(), level)
|
||||
}
|
||||
|
||||
// Check whether the JSON payload is too large.
|
||||
@@ -966,7 +1185,7 @@ func (lg *Logger) appendTextOrJSONLocked(dst, src []byte, level int) []byte {
|
||||
errData := appendTruncatedString(nil, src, maxLen/len(`\uffff`)) // escaping could increase size
|
||||
|
||||
dst = append(dst, '{')
|
||||
dst = lg.appendMetadata(dst, lg.skipClientTime, true, lg.procID, lg.procSequence, errDetail, errData, level)
|
||||
dst = lg.appendMetadata(dst, lg.skipClientTime, true, lg.procID, lg.procSequence, lg.getAuthID(), errDetail, errData, level)
|
||||
dst = bytes.TrimRight(dst, ",")
|
||||
return append(dst, "}\n"...)
|
||||
}
|
||||
@@ -983,7 +1202,7 @@ func (lg *Logger) appendTextOrJSONLocked(dst, src []byte, level int) []byte {
|
||||
}
|
||||
dst = slices.Grow(dst, len(src))
|
||||
dst = append(dst, '{')
|
||||
dst = lg.appendMetadata(dst, lg.skipClientTime, true, lg.procID, lg.procSequence, errDetail, errData, level)
|
||||
dst = lg.appendMetadata(dst, lg.skipClientTime, true, lg.procID, lg.procSequence, lg.getAuthID(), errDetail, errData, level)
|
||||
if logtailValLength > 0 {
|
||||
// Exclude original logtail member from the message.
|
||||
dst = appendWithoutNewline(dst, src[len("{"):logtailKeyOffset])
|
||||
|
||||
@@ -24,6 +24,7 @@ type Logtail struct {
|
||||
ClientTime time.Time `json:"client_time,omitzero"`
|
||||
ProcID uint32 `json:"proc_id,omitzero"`
|
||||
ProcSeq uint64 `json:"proc_seq,omitzero"`
|
||||
AuthID string `json:"auth_id,omitzero"`
|
||||
}
|
||||
|
||||
type LogEntry[T any] struct {
|
||||
@@ -56,6 +57,9 @@ func (l *Logger) SetSockstatsLabel(label any) {}
|
||||
func (l *Logger) PrivateID() logid.PrivateID { return logid.PrivateID{} }
|
||||
func (l *Logger) StartFlush() {}
|
||||
|
||||
func (*Logger) SetAuthID(id string) {}
|
||||
func (*Logger) RegisterAuthToken(authID string, authToken func() string) {}
|
||||
|
||||
func RegisterLogTap(dst chan<- string) (unregister func()) {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
+161
-11
@@ -628,7 +628,7 @@ func TestUploadMetrics(t *testing.T) {
|
||||
}
|
||||
return &http.Response{StatusCode: tt.status, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
|
||||
})}
|
||||
_, err := lg.upload(t.Context(), []byte(tt.body), tt.origlen)
|
||||
_, err := lg.upload(t.Context(), []byte(tt.body), tt.origlen, "")
|
||||
wantErr := tt.transportErr || tt.status != http.StatusOK
|
||||
if (err != nil) != wantErr {
|
||||
t.Fatalf("upload error = %v; wantErr %v", err, wantErr)
|
||||
@@ -662,6 +662,146 @@ func TestUploadMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendEntry(t *testing.T) {
|
||||
var lg Logger
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
wantAuthID string
|
||||
wantValid bool
|
||||
}{
|
||||
{``, ``, ``, false},
|
||||
{`fizz`, ``, ``, false},
|
||||
{` {} `, ` {} `, ``, true},
|
||||
{` { "logtail": {} } `, ` { "logtail": {} } `, ``, true},
|
||||
{` { "logtail": { "auth_id" : "foo" } } `, ` { "logtail": { } } `, `foo`, true},
|
||||
{` { "logtail": { "before" : "value" , "auth_id" : "foo" } } `, ` { "logtail": { "before" : "value" } } `, `foo`, true},
|
||||
{` { "logtail": { "auth_id" : "foo" , "after" : "value" } } `, ` { "logtail": { "after" : "value" } } `, `foo`, true},
|
||||
{`{"logtail":{"auth_id":"foo","after":"value"}}`, `{"logtail":{"after":"value"}}`, `foo`, true},
|
||||
{` { "logtail": { "before" : "value" , "auth_id" : "foo" , "after" : "value" } , "fizz":"buzz"} `, ` { "logtail": { "before" : "value" , "after" : "value" } , "fizz":"buzz"} `, `foo`, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, gotAuthID, gotValid := lg.appendEntry(nil, []byte(tt.in))
|
||||
if string(got) != tt.want || gotAuthID != tt.wantAuthID || gotValid != tt.wantValid {
|
||||
t.Errorf("appendEntry(%q) = (%q, %q, %v), want (%q, %q, %v)", tt.in, got, gotAuthID, gotValid, tt.want, tt.wantAuthID, tt.wantValid)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestDrainPendingBatchesByAuthID(t *testing.T) {
|
||||
buf := NewMemoryBuffer(100)
|
||||
lg := &Logger{
|
||||
clock: tstest.NewClock(tstest.ClockOpts{Start: time.Unix(123, 0)}),
|
||||
buffer: buf,
|
||||
}
|
||||
lg.SetAuthID("auth-a")
|
||||
must.Get(lg.Write([]byte("a1")))
|
||||
must.Get(lg.Write([]byte("a2")))
|
||||
lg.SetAuthID("auth-b")
|
||||
must.Get(lg.Write([]byte("b1")))
|
||||
|
||||
body1, id1 := lg.drainPending()
|
||||
if id1 != "auth-a" {
|
||||
t.Errorf("first batch authID = %q; want auth-a", id1)
|
||||
}
|
||||
if strings.Contains(string(body1), "auth_id") {
|
||||
t.Errorf("first batch still has auth_id: %s", body1)
|
||||
}
|
||||
if !strings.Contains(string(body1), "a1") || !strings.Contains(string(body1), "a2") {
|
||||
t.Errorf("first batch missing a-entries: %s", body1)
|
||||
}
|
||||
if strings.Contains(string(body1), "b1") {
|
||||
t.Errorf("first batch leaked b-entry: %s", body1)
|
||||
}
|
||||
|
||||
body2, id2 := lg.drainPending()
|
||||
if id2 != "auth-b" {
|
||||
t.Errorf("second batch authID = %q; want auth-b", id2)
|
||||
}
|
||||
if !strings.Contains(string(body2), "b1") {
|
||||
t.Errorf("second batch missing b1: %s", body2)
|
||||
}
|
||||
if strings.Contains(string(body2), "auth_id") {
|
||||
t.Errorf("second batch still has auth_id: %s", body2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAuthorizationHeader(t *testing.T) { synctest.Test(t, synctestUploadAuthorizationHeader) }
|
||||
|
||||
func synctestUploadAuthorizationHeader(t *testing.T) {
|
||||
type capture struct {
|
||||
auth string
|
||||
body []byte
|
||||
}
|
||||
uploaded := make(chan capture, 8)
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Errorf("read body: %v", err)
|
||||
}
|
||||
uploaded <- capture{auth: r.Header.Get("Authorization"), body: body}
|
||||
})
|
||||
ln := memnet.Listen("logtail-test:0")
|
||||
httpsrv := &http.Server{Handler: handler}
|
||||
go httpsrv.Serve(ln)
|
||||
t.Cleanup(func() {
|
||||
httpsrv.Close()
|
||||
ln.Close()
|
||||
})
|
||||
|
||||
// Disable startup banner so it doesn't consume a capture.
|
||||
envknob.Setenv("TS_DEBUG_LOGTAIL", "")
|
||||
logger := NewLogger(Config{
|
||||
BaseURL: "http://" + ln.Addr().String(),
|
||||
HTTPC: &http.Client{Transport: &http.Transport{DialContext: ln.Dial}},
|
||||
Bus: eventbustest.NewBus(t),
|
||||
}, t.Logf)
|
||||
|
||||
const authID = "nodeid:8"
|
||||
const token = "test-jwt-token"
|
||||
logger.SetAuthID(authID)
|
||||
logger.RegisterAuthToken(authID, func() string { return token })
|
||||
io.WriteString(logger, "authed log")
|
||||
|
||||
got := <-uploaded
|
||||
if want := "Bearer " + token; got.auth != want {
|
||||
t.Errorf("Authorization = %q; want %q", got.auth, want)
|
||||
}
|
||||
if strings.Contains(string(got.body), "auth_id") {
|
||||
t.Errorf("uploaded body still has auth_id: %s", got.body)
|
||||
}
|
||||
if !strings.Contains(string(got.body), "authed log") {
|
||||
t.Errorf("uploaded body missing text: %s", got.body)
|
||||
}
|
||||
|
||||
// Empty token: no Authorization header.
|
||||
logger.SetAuthID("other")
|
||||
logger.RegisterAuthToken("other", func() string { return "" })
|
||||
io.WriteString(logger, "no auth")
|
||||
got = <-uploaded
|
||||
if got.auth != "" {
|
||||
t.Errorf("expected no Authorization, got %q", got.auth)
|
||||
}
|
||||
|
||||
if err := logger.Shutdown(context.Background()); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterAuthTokenNilClears(t *testing.T) {
|
||||
lg := &Logger{}
|
||||
lg.RegisterAuthToken("id", func() string { return "tok" })
|
||||
if got := lg.lookupAuthToken("id"); got != "tok" {
|
||||
t.Errorf("got %q; want tok", got)
|
||||
}
|
||||
lg.RegisterAuthToken("id", nil)
|
||||
if got := lg.lookupAuthToken("id"); got != "" {
|
||||
t.Errorf("after nil register got %q; want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLoggerDisabled(t *testing.T) { synctest.Test(t, synctestNewLoggerDisabled) }
|
||||
|
||||
func synctestNewLoggerDisabled(t *testing.T) {
|
||||
@@ -800,6 +940,7 @@ func TestAppendMetadata(t *testing.T) {
|
||||
skipMetrics bool
|
||||
procID uint32
|
||||
procSeq uint64
|
||||
authID string
|
||||
errDetail string
|
||||
errData jsontext.Value
|
||||
level int
|
||||
@@ -813,15 +954,17 @@ func TestAppendMetadata(t *testing.T) {
|
||||
{skipClientTime: true, skipMetrics: true, procSeq: 2, want: `"logtail":{"proc_seq":2},`},
|
||||
{skipClientTime: true, skipMetrics: true, procID: 1, procSeq: 2, want: `"logtail":{"proc_id":1,"proc_seq":2},`},
|
||||
{skipMetrics: true, procID: 1, procSeq: 2, want: `"logtail":{"client_time":"2000-01-01T00:00:00Z","proc_id":1,"proc_seq":2},`},
|
||||
{skipClientTime: true, skipMetrics: true, authID: "nodeid:abc", want: `"logtail":{"auth_id":"nodeid:abc"},`},
|
||||
{skipClientTime: true, skipMetrics: true, authID: `quote"and\slash`, want: `"logtail":{"auth_id":"quote\"and\\slash"},`},
|
||||
{skipClientTime: true, skipMetrics: true, errDetail: "error", want: `"logtail":{"error":{"detail":"error"}},`},
|
||||
{skipClientTime: true, skipMetrics: true, errData: jsontext.Value("null"), want: `"logtail":{"error":{"bad_data":null}},`},
|
||||
{skipClientTime: true, skipMetrics: true, level: 5, want: `"v":5,`},
|
||||
{procID: 1, procSeq: 2, errDetail: "error", errData: jsontext.Value(`["something","bad","happened"]`), level: 2,
|
||||
want: `"logtail":{"client_time":"2000-01-01T00:00:00Z","proc_id":1,"proc_seq":2,"error":{"detail":"error","bad_data":["something","bad","happened"]}},"metrics":"metrics","v":2,`},
|
||||
{procID: 1, procSeq: 2, authID: "nodeid:xyz", errDetail: "error", errData: jsontext.Value(`["something","bad","happened"]`), level: 2,
|
||||
want: `"logtail":{"client_time":"2000-01-01T00:00:00Z","proc_id":1,"proc_seq":2,"auth_id":"nodeid:xyz","error":{"detail":"error","bad_data":["something","bad","happened"]}},"metrics":"metrics","v":2,`},
|
||||
} {
|
||||
got := string(lg.appendMetadata(nil, tt.skipClientTime, tt.skipMetrics, tt.procID, tt.procSeq, tt.errDetail, tt.errData, tt.level))
|
||||
got := string(lg.appendMetadata(nil, tt.skipClientTime, tt.skipMetrics, tt.procID, tt.procSeq, tt.authID, tt.errDetail, tt.errData, tt.level))
|
||||
if got != tt.want {
|
||||
t.Errorf("appendMetadata(%v, %v, %v, %v, %v, %v, %v):\n\tgot %s\n\twant %s", tt.skipClientTime, tt.skipMetrics, tt.procID, tt.procSeq, tt.errDetail, tt.errData, tt.level, got, tt.want)
|
||||
t.Errorf("appendMetadata(%v, %v, %v, %v, %q, %v, %v, %v):\n\tgot %s\n\twant %s", tt.skipClientTime, tt.skipMetrics, tt.procID, tt.procSeq, tt.authID, tt.errDetail, tt.errData, tt.level, got, tt.want)
|
||||
}
|
||||
gotObj := "{" + strings.TrimSuffix(got, ",") + "}"
|
||||
if !jsontext.Value(gotObj).IsValid() {
|
||||
@@ -840,23 +983,26 @@ func TestAppendText(t *testing.T) {
|
||||
skipClientTime bool
|
||||
procID uint32
|
||||
procSeq uint64
|
||||
authID string
|
||||
level int
|
||||
want string
|
||||
}{
|
||||
{want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics"}`},
|
||||
{skipClientTime: true, want: `{"metrics":"metrics"}`},
|
||||
{skipClientTime: true, procID: 1, procSeq: 2, want: `{"logtail":{"proc_id":1,"proc_seq":2},"metrics":"metrics"}`},
|
||||
{skipClientTime: true, authID: "nodeid:abc", want: `{"logtail":{"auth_id":"nodeid:abc"},"metrics":"metrics"}`},
|
||||
{text: "fizz buzz", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","text":"fizz buzz"}`},
|
||||
{text: "fizz buzz", authID: "nodeid:xyz", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","auth_id":"nodeid:xyz"},"metrics":"metrics","text":"fizz buzz"}`},
|
||||
{text: "\b\f\n\r\t\"\\", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","text":"\b\f\n\r\t\"\\"}`},
|
||||
{text: "x" + strings.Repeat("😐", maxSize), want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","text":"x` + strings.Repeat("😐", 4095) + `…+1032196"}`},
|
||||
} {
|
||||
got := string(lg.appendText(nil, []byte(tt.text), tt.skipClientTime, tt.procID, tt.procSeq, tt.level))
|
||||
got := string(lg.appendText(nil, []byte(tt.text), tt.skipClientTime, tt.procID, tt.procSeq, tt.authID, tt.level))
|
||||
if !strings.HasSuffix(got, "\n") {
|
||||
t.Errorf("`%s` does not end with a newline", got)
|
||||
}
|
||||
got = got[:len(got)-1]
|
||||
if got != tt.want {
|
||||
t.Errorf("appendText(%v, %v, %v, %v, %v):\n\tgot %s\n\twant %s", tt.text[:min(len(tt.text), 256)], tt.skipClientTime, tt.procID, tt.procSeq, tt.level, got, tt.want)
|
||||
t.Errorf("appendText(%v, %v, %v, %v, %q, %v):\n\tgot %s\n\twant %s", tt.text[:min(len(tt.text), 256)], tt.skipClientTime, tt.procID, tt.procSeq, tt.authID, tt.level, got, tt.want)
|
||||
}
|
||||
if !jsontext.Value(got).IsValid() {
|
||||
t.Errorf("`%s`.IsValid() = false, want true", got)
|
||||
@@ -870,14 +1016,17 @@ func TestAppendTextOrJSON(t *testing.T) {
|
||||
lg.metricsDelta = func() string { return "metrics" }
|
||||
|
||||
for _, tt := range []struct {
|
||||
in string
|
||||
level int
|
||||
want string
|
||||
in string
|
||||
authID string
|
||||
level int
|
||||
want string
|
||||
}{
|
||||
{want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics"}`},
|
||||
{in: "[]", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","text":"[]"}`},
|
||||
{level: 1, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","v":1}`},
|
||||
{in: `{}`, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"}}`},
|
||||
{in: `{}`, authID: "nodeid:abc", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","auth_id":"nodeid:abc"}}`},
|
||||
{in: "hello", authID: "nodeid:abc", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","auth_id":"nodeid:abc"},"metrics":"metrics","text":"hello"}`},
|
||||
{in: `{}{}`, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"metrics":"metrics","text":"{}{}"}`},
|
||||
{in: "{\n\"fizz\"\n:\n\"buzz\"\n}", want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z"},"fizz":"buzz"}`},
|
||||
{in: `{ "logtail" : "duplicate" }`, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","error":{"detail":"duplicate logtail member","bad_data":"duplicate"}}}`},
|
||||
@@ -886,6 +1035,7 @@ func TestAppendTextOrJSON(t *testing.T) {
|
||||
{in: `{ "fizz" : "buzz" , "logtail" : "duplicate" , "wizz" : "wuzz" }`, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","error":{"detail":"duplicate logtail member","bad_data":"duplicate"}}, "fizz" : "buzz" , "wizz" : "wuzz"}`},
|
||||
{in: `{"long":"` + strings.Repeat("a", maxSize) + `"}`, want: `{"logtail":{"client_time":"2000-01-01T00:00:00Z","error":{"detail":"entry too large: 262155 bytes","bad_data":"{\"long\":\"` + strings.Repeat("a", 43681) + `…+218465"}}}`},
|
||||
} {
|
||||
lg.SetAuthID(tt.authID)
|
||||
got := string(lg.appendTextOrJSONLocked(nil, []byte(tt.in), tt.level))
|
||||
if !strings.HasSuffix(got, "\n") {
|
||||
t.Errorf("`%s` does not end with a newline", got)
|
||||
@@ -908,7 +1058,7 @@ func TestAppendTextAllocs(t *testing.T) {
|
||||
procID := uint32(0x24d32ee9)
|
||||
procSequence := uint64(0x12346)
|
||||
must.Do(tstest.MinAllocsPerRun(t, 0, func() {
|
||||
sink = lg.appendText(sink[:0], inBuf, false, procID, procSequence, 0)
|
||||
sink = lg.appendText(sink[:0], inBuf, false, procID, procSequence, "", 0)
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -199,6 +199,17 @@ const (
|
||||
// LogExitFlows enables exit node destinations in network flow logs.
|
||||
LogExitFlows Cap = "log-exit-flows"
|
||||
|
||||
// LogUploadAuth specifies the HTTP Authorization token
|
||||
// to use when uploading logs. The CapMap value is a JSON string containing
|
||||
// the token. Control pushes a refreshed token via netmap CapMap updates before
|
||||
// the previous token expires. The client caches the latest value and
|
||||
// does not fetch refreshes itself.
|
||||
//
|
||||
// The value is a secret. The client removes this capability from the netmap
|
||||
// before publishing it, so the token does not appear in local status, WhoIs,
|
||||
// netmap notifications, the on-disk netmap cache, or debug netmap dumps.
|
||||
LogUploadAuth Cap = "log-upload-auth"
|
||||
|
||||
// AutoExitNode permits the automatic exit nodes feature.
|
||||
AutoExitNode Cap = "auto-exit-node"
|
||||
|
||||
|
||||
+2
-1
@@ -199,7 +199,8 @@ type CapabilityVersion int
|
||||
// - 148: 2026-09-15: Client understands [Node.StableTailnetID]
|
||||
// - 149: 2026-10-07: Sentinel version to request the server send back [NodeAttrCacheNetworkMaps] set to enabled
|
||||
// - 150: 2026-10-13: Client handles app connectors with "mode": "magicip" as conn25.
|
||||
const CurrentCapabilityVersion CapabilityVersion = 150
|
||||
// - 151: 2026-11-13: Client understands [nodecap.LogUploadAuth] for authenticated log uploads
|
||||
const CurrentCapabilityVersion CapabilityVersion = 151
|
||||
|
||||
// ID is an integer ID for a user, node, or login allocated by the
|
||||
// control plane.
|
||||
|
||||
@@ -1014,6 +1014,10 @@ func (s *Server) start() (reterr error) {
|
||||
}
|
||||
lb.SetTCPHandlerForFunnelFlow(s.getTCPHandlerForFunnelFlow)
|
||||
lb.SetVarRoot(s.rootPath)
|
||||
if s.logtail != nil {
|
||||
lb.SetLogUploader(s.logtail)
|
||||
lb.SetLogFlusher(s.logtail.StartFlush)
|
||||
}
|
||||
s.logf("tsnet starting with hostname %q, varRoot %q", s.hostname, s.rootPath)
|
||||
s.lb = lb
|
||||
if err := ns.Start(lb); err != nil {
|
||||
|
||||
@@ -62,6 +62,11 @@ type NodeSource interface {
|
||||
// its owning user profile.
|
||||
// ok is false if no node is known to own addr.
|
||||
NodeByAddr(addr netip.Addr) (node tailcfg.NodeView, user tailcfg.UserProfileView, ok bool)
|
||||
|
||||
// LogUploadAuth returns the opaque auth ID and a token function for
|
||||
// authenticated log uploads. authID is empty when not in a tailnet
|
||||
// context; authToken may be nil.
|
||||
LogUploadAuth() (authID string, authToken func() string)
|
||||
}
|
||||
|
||||
// Logger logs statistics about every connection.
|
||||
@@ -165,6 +170,14 @@ func (nl *Logger) Startup(logf logger.Logf, source NodeSource, nodeLogID, domain
|
||||
}, logf)
|
||||
logger.SetSockstatsLabel(sockstats.LabelNetlogLogger)
|
||||
|
||||
// Network flow logging is always tied to a single tailnet+node for the
|
||||
// lifetime of this logger instance. Register auth once at construction;
|
||||
// there is no need to clear it (the logger is shut down on identity change).
|
||||
if authID, authToken := source.LogUploadAuth(); authID != "" {
|
||||
logger.SetAuthID(authID)
|
||||
logger.RegisterAuthToken(authID, authToken)
|
||||
}
|
||||
|
||||
// Register the connection tracker into the TUN device.
|
||||
tun = cmp.Or[Device](tun, noopDevice{})
|
||||
tun.SetConnectionCounter(nl.updateVirtConn)
|
||||
|
||||
@@ -38,6 +38,8 @@ type fakeNodeSource struct {
|
||||
byAddr map[netip.Addr]nodeAndUser
|
||||
}
|
||||
|
||||
func (s *fakeNodeSource) LogUploadAuth() (string, func() string) { return "", nil }
|
||||
|
||||
func (s *fakeNodeSource) SelfNode() (tailcfg.NodeView, tailcfg.UserProfileView) {
|
||||
return s.self, s.selfUser
|
||||
}
|
||||
|
||||
@@ -35,6 +35,12 @@ type NetLogSource interface {
|
||||
// along with whether exit node flows should be logged.
|
||||
// ok is false if network flow logging should not run.
|
||||
NetLogIDs() (nodeID, domainID logid.PrivateID, logExitFlows bool, ok bool)
|
||||
|
||||
// LogUploadAuth returns the opaque auth ID and a token function for
|
||||
// authenticated log uploads (see logtail.Logger.SetAuthID and
|
||||
// RegisterAuthToken). authID is empty when there is no self node;
|
||||
// authToken may be nil.
|
||||
LogUploadAuth() (authID string, authToken func() string)
|
||||
}
|
||||
|
||||
// NetLogger is the engine's handle on the network flow logger
|
||||
|
||||
Reference in new issue
Block a user