feature/acme: only allow one async renewal at a time

Currently we serialise ACME account setup and ACME issuances for domains
which don't yet have a valid certificate, but not for async renewals of
still-valid certificates.

This patch adds a check that we only have one async renewal for a domain
in-flight at a time, and a test that ensures we de-dupe these renewals.

Updates tailscale/corp#46420

Change-Id: Ibbaa537cd28b8adf238b5c9396403e845f95b7e7
Signed-off-by: Alex Chan <[email protected]>
This commit is contained in:
Alex Chan authored and Alex Chan committed 2026-08-14 20:15:30 +01:00
1 parent ffaa5a361f
commit 2d98e75249
3 files changed
+158 -8

No files matched your search

+5 -1
View File
@@ -83,11 +83,15 @@ type extension struct {
// accounts.
accountMu syncs.Mutex
// renewMu guards renewCertAt.
// renewMu guards renewCertAt and renewingCertDomains.
// Lock order: per-domain lock before renewMu.
renewMu syncs.Mutex
renewCertAt map[string]time.Time // lazily initialized under renewMu
// renewingCertDomains tracks domains for which an async renewal is
// currently in flight.
renewingCertDomains set.Set[string]
// pendingACMETLSALPNCerts maps SNI names to short-lived ACME
// tls-alpn-01 challenge certificates while an ACME order is
// waiting for validation. Entries are deleted by the cleanup
+38 -7
View File
@@ -136,13 +136,18 @@ func (e *extension) getCertPEMWithValidity(ctx context.Context, b *ipnlocal.Loca
return pair, nil
}
if minValidity == 0 {
logf("starting async renewal")
// Start renewal in the background, return current valid cert.
e.Go(func() {
if _, err := getCertPEM(context.Background(), e, b, cs, logf, traceACME, certDomain, now, minValidity); err != nil {
logf("async renewal failed: getCertPem: %v", err)
}
})
// Start renewal in the background if there isn't one already in progress,
// return current valid cert.
if e.beginAsyncRenewal(certDomain) {
logf("starting async renewal")
e.Go(func() {
defer e.endAsyncRenewal(certDomain)
if _, err := getCertPEM(context.Background(), e, b, cs, logf, traceACME, certDomain, now, minValidity); err != nil {
logf("async renewal failed: getCertPem: %v", err)
}
})
}
return pair, nil
}
// If the caller requested a specific validity duration, fall through
@@ -199,6 +204,32 @@ func (e *extension) domainRenewed(domain string) {
delete(e.renewCertAt, domain)
}
// beginAsyncRenewal marks a domain as having an async renewal in flight,
// and reports whether this caller started it.
//
// The caller must arrange for endAsyncRenewal(domain) when the renewal
// finishes, regardless of result.
func (e *extension) beginAsyncRenewal(domain string) bool {
e.renewMu.Lock()
defer e.renewMu.Unlock()
if e.renewingCertDomains.Contains(domain) {
return false
}
e.renewingCertDomains.Make()
e.renewingCertDomains.Add(domain)
return true
}
// endAsyncRenewal clears the in-flight marker for domain, allowing
// a subsequent renewal to begin.
//
// It should only be called by the caller that began the renewal.
func (e *extension) endAsyncRenewal(domain string) {
e.renewMu.Lock()
defer e.renewMu.Unlock()
e.renewingCertDomains.Delete(domain)
}
func domainRenewalTimeByExpiry(pair *ipnlocal.TLSCertKeyPair) (time.Time, error) {
cert, err := parseCertificate(pair)
if err != nil {
+115
View File
@@ -882,6 +882,121 @@ func TestGetCertPEMWithValidityTrimsTrailingDot(t *testing.T) {
}
}
// TestAsyncRenewalDedup checks that only one async renewal for a domain
// is allowed at a time; multiple calls to renew the same domain should
// be de-duplicated.
func TestAsyncRenewalDedup(t *testing.T) {
tstest.AssertNotParallel(t)
const domain = "example.com"
b := ipnlocaltest.NewBackend(t)
b.SetVarRoot(t.TempDir())
e := extOf(t, b)
b.ForTest().SetNetMap(&netmap.NetworkMap{
SelfNode: (&tailcfg.Node{}).View(),
DNS: tailcfg.DNSConfig{
CertDomains: []string{domain},
},
})
certDirPath, err := certDir(b)
if err != nil {
t.Fatalf("certDir error: %v", err)
}
if _, err := e.getCertStore(b); err != nil {
t.Fatalf("getCertStore error: %v", err)
}
testRoot, err := certTestFS.ReadFile("testdata/rootCA.pem")
if err != nil {
t.Fatal(err)
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(testRoot) {
t.Fatal("Unable to add test CA to the cert pool")
}
testX509Roots = roots
defer func() { testX509Roots = nil }()
// Store a valid, cached cert under a time at which it is due for
// renewal (the same epoch as TestGetCertPEMWithValidity's
// "renewal_needed" case).
os.MkdirAll(certDirPath, 0755)
if err := os.WriteFile(filepath.Join(certDirPath, "example.com.crt"),
must.Get(certTestFS.ReadFile("testdata/example.com.pem")), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(certDirPath, "example.com.key"),
must.Get(certTestFS.ReadFile("testdata/example.com-key.pem")), 0644); err != nil {
t.Fatal(err)
}
b.ForTest().SetClock(tstest.NewClock(tstest.ClockOpts{
Start: time.Date(2025, time.May, 1, 0, 0, 0, 0, time.UTC),
}))
// Keep the first async renewal in flight so we can observe a second
// trigger that should be deduplicated.
renewalStarted := make(chan struct{}, 1)
renewalComplete := make(chan struct{})
var completeRenewalOnce sync.Once
completeFirstRenewal := func() { completeRenewalOnce.Do(func() { close(renewalComplete) }) }
defer completeFirstRenewal()
orig := getCertPEM
getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*ipnlocal.TLSCertKeyPair, error) {
t.Logf("calling getCertPEM")
renewalStarted <- struct{}{}
<-renewalComplete
return nil, nil
}
t.Cleanup(func() { getCertPEM = orig })
// First trigger starts the async renewal.
prevGo := e.goroutinesStarted.Load()
pair, err := b.GetCertPEMWithValidity(context.Background(), domain, 0)
if err != nil {
t.Fatalf("first GetCertPEMWithValidity error: %v", err)
}
if pair == nil || !pair.Cached {
t.Fatalf("first GetCertPEMWithValidity = %+v, want cached cert", pair)
}
if got := e.goroutinesStarted.Load() - prevGo; got != 1 {
t.Fatalf("first call spawned %d goroutine(s), want 1 (async renewal started)", got)
}
select {
case <-renewalStarted:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for async renewal to enter getCertPEM")
}
// A second trigger while the renewal is still in flight must be
// deduplicated: no new goroutine.
prevGo = e.goroutinesStarted.Load()
pair2, err := b.GetCertPEMWithValidity(context.Background(), domain, 0)
if err != nil {
t.Fatalf("second GetCertPEMWithValidity error: %v", err)
}
if pair2 == nil || !pair2.Cached {
t.Fatalf("second GetCertPEMWithValidity = %+v, want cached cert", pair2)
}
if got := e.goroutinesStarted.Load() - prevGo; got != 0 {
t.Fatalf("second call spawned %d goroutine(s), want 0 (deduplicated while renewal in flight)", got)
}
// Let the in-flight renewal finish and verify the marker is cleared so
// a later renewal can be attempted again.
completeFirstRenewal()
done := make(chan struct{})
go func() { e.wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for in-flight async renewal goroutine to finish")
}
if e.renewingCertDomains.Contains(domain) {
t.Fatal("renewingCertDomains still contains domain after renewal finished")
}
}
func TestCertPendingWarnable(t *testing.T) {
b := ipnlocaltest.NewBackend(t)
e := extOf(t, b)