mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-21 09:11:18 -08:00
feature/acme: only allow one async renewal at a time
Currently we serialise ACME account setup and ACME issuances for domains which don't yet have a valid certificate, but not for async renewals of still-valid certificates. This patch adds a check that we only have one async renewal for a domain in-flight at a time, and a test that ensures we de-dupe these renewals. Updates tailscale/corp#46420 Change-Id: Ibbaa537cd28b8adf238b5c9396403e845f95b7e7 Signed-off-by: Alex Chan <[email protected]>
This commit is contained in:
1 parent
ffaa5a361f
commit
2d98e75249
3 files changed
+158
-8
No files matched your search
@@ -83,11 +83,15 @@ type extension struct {
|
||||
// accounts.
|
||||
accountMu syncs.Mutex
|
||||
|
||||
// renewMu guards renewCertAt.
|
||||
// renewMu guards renewCertAt and renewingCertDomains.
|
||||
// Lock order: per-domain lock before renewMu.
|
||||
renewMu syncs.Mutex
|
||||
renewCertAt map[string]time.Time // lazily initialized under renewMu
|
||||
|
||||
// renewingCertDomains tracks domains for which an async renewal is
|
||||
// currently in flight.
|
||||
renewingCertDomains set.Set[string]
|
||||
|
||||
// pendingACMETLSALPNCerts maps SNI names to short-lived ACME
|
||||
// tls-alpn-01 challenge certificates while an ACME order is
|
||||
// waiting for validation. Entries are deleted by the cleanup
|
||||
|
||||
+38
-7
@@ -136,13 +136,18 @@ func (e *extension) getCertPEMWithValidity(ctx context.Context, b *ipnlocal.Loca
|
||||
return pair, nil
|
||||
}
|
||||
if minValidity == 0 {
|
||||
logf("starting async renewal")
|
||||
// Start renewal in the background, return current valid cert.
|
||||
e.Go(func() {
|
||||
if _, err := getCertPEM(context.Background(), e, b, cs, logf, traceACME, certDomain, now, minValidity); err != nil {
|
||||
logf("async renewal failed: getCertPem: %v", err)
|
||||
}
|
||||
})
|
||||
// Start renewal in the background if there isn't one already in progress,
|
||||
// return current valid cert.
|
||||
if e.beginAsyncRenewal(certDomain) {
|
||||
logf("starting async renewal")
|
||||
|
||||
e.Go(func() {
|
||||
defer e.endAsyncRenewal(certDomain)
|
||||
if _, err := getCertPEM(context.Background(), e, b, cs, logf, traceACME, certDomain, now, minValidity); err != nil {
|
||||
logf("async renewal failed: getCertPem: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
return pair, nil
|
||||
}
|
||||
// If the caller requested a specific validity duration, fall through
|
||||
@@ -199,6 +204,32 @@ func (e *extension) domainRenewed(domain string) {
|
||||
delete(e.renewCertAt, domain)
|
||||
}
|
||||
|
||||
// beginAsyncRenewal marks a domain as having an async renewal in flight,
|
||||
// and reports whether this caller started it.
|
||||
//
|
||||
// The caller must arrange for endAsyncRenewal(domain) when the renewal
|
||||
// finishes, regardless of result.
|
||||
func (e *extension) beginAsyncRenewal(domain string) bool {
|
||||
e.renewMu.Lock()
|
||||
defer e.renewMu.Unlock()
|
||||
if e.renewingCertDomains.Contains(domain) {
|
||||
return false
|
||||
}
|
||||
e.renewingCertDomains.Make()
|
||||
e.renewingCertDomains.Add(domain)
|
||||
return true
|
||||
}
|
||||
|
||||
// endAsyncRenewal clears the in-flight marker for domain, allowing
|
||||
// a subsequent renewal to begin.
|
||||
//
|
||||
// It should only be called by the caller that began the renewal.
|
||||
func (e *extension) endAsyncRenewal(domain string) {
|
||||
e.renewMu.Lock()
|
||||
defer e.renewMu.Unlock()
|
||||
e.renewingCertDomains.Delete(domain)
|
||||
}
|
||||
|
||||
func domainRenewalTimeByExpiry(pair *ipnlocal.TLSCertKeyPair) (time.Time, error) {
|
||||
cert, err := parseCertificate(pair)
|
||||
if err != nil {
|
||||
|
||||
@@ -882,6 +882,121 @@ func TestGetCertPEMWithValidityTrimsTrailingDot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAsyncRenewalDedup checks that only one async renewal for a domain
|
||||
// is allowed at a time; multiple calls to renew the same domain should
|
||||
// be de-duplicated.
|
||||
func TestAsyncRenewalDedup(t *testing.T) {
|
||||
tstest.AssertNotParallel(t)
|
||||
|
||||
const domain = "example.com"
|
||||
b := ipnlocaltest.NewBackend(t)
|
||||
b.SetVarRoot(t.TempDir())
|
||||
e := extOf(t, b)
|
||||
|
||||
b.ForTest().SetNetMap(&netmap.NetworkMap{
|
||||
SelfNode: (&tailcfg.Node{}).View(),
|
||||
DNS: tailcfg.DNSConfig{
|
||||
CertDomains: []string{domain},
|
||||
},
|
||||
})
|
||||
|
||||
certDirPath, err := certDir(b)
|
||||
if err != nil {
|
||||
t.Fatalf("certDir error: %v", err)
|
||||
}
|
||||
if _, err := e.getCertStore(b); err != nil {
|
||||
t.Fatalf("getCertStore error: %v", err)
|
||||
}
|
||||
testRoot, err := certTestFS.ReadFile("testdata/rootCA.pem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(testRoot) {
|
||||
t.Fatal("Unable to add test CA to the cert pool")
|
||||
}
|
||||
testX509Roots = roots
|
||||
defer func() { testX509Roots = nil }()
|
||||
|
||||
// Store a valid, cached cert under a time at which it is due for
|
||||
// renewal (the same epoch as TestGetCertPEMWithValidity's
|
||||
// "renewal_needed" case).
|
||||
os.MkdirAll(certDirPath, 0755)
|
||||
if err := os.WriteFile(filepath.Join(certDirPath, "example.com.crt"),
|
||||
must.Get(certTestFS.ReadFile("testdata/example.com.pem")), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(certDirPath, "example.com.key"),
|
||||
must.Get(certTestFS.ReadFile("testdata/example.com-key.pem")), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.ForTest().SetClock(tstest.NewClock(tstest.ClockOpts{
|
||||
Start: time.Date(2025, time.May, 1, 0, 0, 0, 0, time.UTC),
|
||||
}))
|
||||
|
||||
// Keep the first async renewal in flight so we can observe a second
|
||||
// trigger that should be deduplicated.
|
||||
renewalStarted := make(chan struct{}, 1)
|
||||
renewalComplete := make(chan struct{})
|
||||
var completeRenewalOnce sync.Once
|
||||
completeFirstRenewal := func() { completeRenewalOnce.Do(func() { close(renewalComplete) }) }
|
||||
defer completeFirstRenewal()
|
||||
orig := getCertPEM
|
||||
getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*ipnlocal.TLSCertKeyPair, error) {
|
||||
t.Logf("calling getCertPEM")
|
||||
renewalStarted <- struct{}{}
|
||||
<-renewalComplete
|
||||
return nil, nil
|
||||
}
|
||||
t.Cleanup(func() { getCertPEM = orig })
|
||||
|
||||
// First trigger starts the async renewal.
|
||||
prevGo := e.goroutinesStarted.Load()
|
||||
pair, err := b.GetCertPEMWithValidity(context.Background(), domain, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("first GetCertPEMWithValidity error: %v", err)
|
||||
}
|
||||
if pair == nil || !pair.Cached {
|
||||
t.Fatalf("first GetCertPEMWithValidity = %+v, want cached cert", pair)
|
||||
}
|
||||
if got := e.goroutinesStarted.Load() - prevGo; got != 1 {
|
||||
t.Fatalf("first call spawned %d goroutine(s), want 1 (async renewal started)", got)
|
||||
}
|
||||
select {
|
||||
case <-renewalStarted:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for async renewal to enter getCertPEM")
|
||||
}
|
||||
|
||||
// A second trigger while the renewal is still in flight must be
|
||||
// deduplicated: no new goroutine.
|
||||
prevGo = e.goroutinesStarted.Load()
|
||||
pair2, err := b.GetCertPEMWithValidity(context.Background(), domain, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("second GetCertPEMWithValidity error: %v", err)
|
||||
}
|
||||
if pair2 == nil || !pair2.Cached {
|
||||
t.Fatalf("second GetCertPEMWithValidity = %+v, want cached cert", pair2)
|
||||
}
|
||||
if got := e.goroutinesStarted.Load() - prevGo; got != 0 {
|
||||
t.Fatalf("second call spawned %d goroutine(s), want 0 (deduplicated while renewal in flight)", got)
|
||||
}
|
||||
|
||||
// Let the in-flight renewal finish and verify the marker is cleared so
|
||||
// a later renewal can be attempted again.
|
||||
completeFirstRenewal()
|
||||
done := make(chan struct{})
|
||||
go func() { e.wg.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for in-flight async renewal goroutine to finish")
|
||||
}
|
||||
if e.renewingCertDomains.Contains(domain) {
|
||||
t.Fatal("renewingCertDomains still contains domain after renewal finished")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCertPendingWarnable(t *testing.T) {
|
||||
b := ipnlocaltest.NewBackend(t)
|
||||
e := extOf(t, b)
|
||||
|
||||
Reference in new issue
Block a user