From 2993b5e6de3b8b164bbf30463516631fc3e551be Mon Sep 17 00:00:00 2001 From: David Bond Date: Thu, 5 Nov 2026 08:35:13 +0000 Subject: [PATCH] cmd/k8s-operator: move Recorder reconciler into its own package (#20891) Move the Recorder reconciler and its object specs into k8s-operator/reconciler/recorder, following the earlier ProxyClass, DNSConfig and nameserver moves, and switch it to the shared finalizer, resource tracker and CreateOrUpdate helpers. Hoist two duplicated helpers into tailscaled.PrefsFromStateSecret and reconciler.IsOptimisticLockError, and add k8s-operator/reconciler/reconcilertest for the test helpers each package had copied, including one fake Tailscale API client in place of four. No behaviour change; the Recorder's watches and labels are untouched. Fixes tailscale/corp#37086 Signed-off-by: David Bond --- cmd/k8s-operator/api-server-proxy-pg.go | 6 +- cmd/k8s-operator/api-server-proxy-pg_test.go | 3 +- cmd/k8s-operator/connector.go | 4 +- cmd/k8s-operator/depaware.txt | 1 + cmd/k8s-operator/egress-services.go | 3 +- cmd/k8s-operator/ingress-for-pg.go | 3 +- cmd/k8s-operator/ingress-for-pg_test.go | 13 +- cmd/k8s-operator/ingress.go | 3 +- cmd/k8s-operator/operator.go | 31 +- cmd/k8s-operator/proxygroup.go | 12 +- cmd/k8s-operator/proxygroup_test.go | 8 +- cmd/k8s-operator/sts.go | 6 + cmd/k8s-operator/svc-for-pg.go | 6 +- cmd/k8s-operator/svc.go | 3 +- cmd/k8s-operator/testutils_test.go | 3 + cmd/k8s-operator/tsrecorder_specs_test.go | 156 ---- cmd/k8s-operator/tsrecorder_test.go | 321 ------- .../reconciler/dnsrecords/dnsrecords.go | 5 +- .../reconciler/dnsrecords/dnsrecords_test.go | 131 +-- .../reconciler/nameserver/nameserver.go | 5 +- .../reconciler/nameserver/nameserver_test.go | 85 +- .../reconciler/peerrelay/peerrelay_test.go | 190 +---- .../reconciler/proxyclass/proxyclass_test.go | 30 +- .../proxygrouppolicy/proxygrouppolicy_test.go | 4 +- k8s-operator/reconciler/reconciler.go | 13 + k8s-operator/reconciler/reconciler_test.go | 33 + .../reconciler/reconcilertest/conditions.go | 59 ++ .../reconcilertest/reconcilertest.go | 202 +++++ .../reconciler/reconcilertest/tsclient.go | 174 ++++ k8s-operator/reconciler/recorder/authkey.go | 91 ++ .../reconciler/recorder/recorder.go | 370 ++++---- .../reconciler/recorder/recorder_test.go | 803 ++++++++++++++++++ .../reconciler/recorder/specs.go | 23 +- .../reconciler/tailnet/tailnet_test.go | 4 +- k8s-operator/reconciler/tailscaled/prefs.go | 52 ++ .../reconciler/tailscaled/prefs_test.go | 104 +++ .../reconciler/tailscaled/reissuer_test.go | 52 +- 37 files changed, 1868 insertions(+), 1144 deletions(-) delete mode 100644 cmd/k8s-operator/tsrecorder_specs_test.go delete mode 100644 cmd/k8s-operator/tsrecorder_test.go create mode 100644 k8s-operator/reconciler/reconcilertest/conditions.go create mode 100644 k8s-operator/reconciler/reconcilertest/reconcilertest.go create mode 100644 k8s-operator/reconciler/reconcilertest/tsclient.go create mode 100644 k8s-operator/reconciler/recorder/authkey.go rename cmd/k8s-operator/tsrecorder.go => k8s-operator/reconciler/recorder/recorder.go (61%) create mode 100644 k8s-operator/reconciler/recorder/recorder_test.go rename cmd/k8s-operator/tsrecorder_specs.go => k8s-operator/reconciler/recorder/specs.go (91%) create mode 100644 k8s-operator/reconciler/tailscaled/prefs.go create mode 100644 k8s-operator/reconciler/tailscaled/prefs_test.go diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 08bb405b9..94ecac7be 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -12,7 +12,6 @@ import ( "fmt" "maps" "slices" - "strings" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -27,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" @@ -85,7 +85,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") - if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) { + if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return res, nil } @@ -95,7 +95,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient) if err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go index b7c35f437..a7ace3da7 100644 --- a/cmd/k8s-operator/api-server-proxy-pg_test.go +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -20,6 +20,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" @@ -199,7 +200,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { // Simulate Pod prefs updated with advertised services; should see Configured condition updated to true. mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) } diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index bfe751fbb..41e21dad0 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -11,7 +11,6 @@ import ( "fmt" "net/netip" "slices" - "strings" "sync" "time" @@ -28,6 +27,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" @@ -146,7 +146,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil { reason := reasonConnectorCreationFailed message := fmt.Sprintf(messageConnectorCreationFailed, err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { reason = reasonConnectorCreating message = fmt.Sprintf("optimistic lock error, retrying: %s", err) err = nil diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 554dc4ec8..3a3c960d3 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -774,6 +774,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/recorder from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/staticendpoints from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailscaled from tailscale.com/k8s-operator/reconciler/peerrelay+ diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 6f0480a94..7ec758fd7 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -34,6 +34,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" @@ -161,7 +162,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re } if err := esr.maybeProvision(ctx, svc, lg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { lg.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 9b8c70380..545f90d88 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -34,6 +34,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -851,7 +852,7 @@ func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, p var count int for _, secret := range secrets.Items { - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return 0, fmt.Errorf("error getting node metadata: %w", err) } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 8312dc5f7..38c2e7f93 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -30,6 +30,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -563,7 +564,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -689,7 +690,7 @@ func TestIngressPGReconciler_HTTPRedirect(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -824,7 +825,7 @@ func TestIngressPGReconciler_HTTPEndpointAndRedirectConflict(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -1151,7 +1152,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { } mustCreate(t, fc, pgCfgSecret) - pr := prefs{} + pr := tailscaled.Prefs{} pr.Config.UserProfile.LoginName = "test.ts.net" pr.Config.NodeID = "test" @@ -1168,8 +1169,8 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), }, Data: map[string][]byte{ - currentProfileKey: []byte("test"), - "test": p, + string(ipn.CurrentProfileStateKey): []byte("test"), + "test": p, }, } mustCreate(t, fc, pgStateSecret) diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index cf2f37b06..a09524d59 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/ipn" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" "tailscale.com/util/clientmetric" @@ -82,7 +83,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request } if err := a.maybeProvision(ctx, logger, ing); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 430ac3015..10420331f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -58,6 +58,7 @@ import ( "tailscale.com/k8s-operator/reconciler/peerrelay" "tailscale.com/k8s-operator/reconciler/proxyclass" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" + "tailscale.com/k8s-operator/reconciler/recorder" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" @@ -674,26 +675,16 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create DNS records reconciler: %v", err) } - // Recorder reconciler. - recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{}) - err = builder.ControllerManagedBy(mgr). - For(&tsapi.Recorder{}). - Named("recorder-reconciler"). - Watches(&appsv1.StatefulSet{}, recorderFilter). - Watches(&corev1.ServiceAccount{}, recorderFilter). - Watches(&corev1.Secret{}, recorderFilter). - Watches(&rbacv1.Role{}, recorderFilter). - Watches(&rbacv1.RoleBinding{}, recorderFilter). - Complete(&RecorderReconciler{ - recorder: eventRecorder, - tsNamespace: opts.tailscaleNamespace, - Client: mgr.GetClient(), - log: opts.log.Named("recorder-reconciler"), - clock: tstime.DefaultClock{}, - clients: clients, - reissuer: tailscaled.NewReissuer(), - }) - if err != nil { + recorderOptions := recorder.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Clients: clients, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + } + + if err = recorder.NewReconciler(recorderOptions).Register(mgr); err != nil { startlog.Fatalf("could not create Recorder reconciler: %v", err) } diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index b8455e4b2..cf1c25e9f 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -34,6 +34,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/staticendpoints" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" @@ -57,9 +58,6 @@ const ( reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable" reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion" - // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" - // The minimum tailcfg.CapabilityVersion that deployed clients are expected // to support to be compatible with the current ProxyGroup controller. // If the controller needs to depend on newer client behaviour, it should @@ -150,7 +148,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } @@ -475,7 +473,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za defer func() { if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(updateErr) { logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) updateErr = nil } @@ -992,7 +990,7 @@ func getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup, cl client.Client stateSecret: &secret, } - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return nil, err } @@ -1101,7 +1099,7 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool { func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) logger.Info(msg) return notReady(reasonProxyGroupCreating, msg) diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 986b6ad5c..21a50c36b 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -2296,10 +2296,10 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG } mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) { s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, - kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), - kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i), + string(ipn.CurrentProfileStateKey): []byte(key), + key: bytes, + kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), + kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i), // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. // Consolidate on this one. kubetypes.KeyDeviceID: fmt.Appendf(nil, "nodeid-%d", i), diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 68414d1d8..2e5680cdc 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -1524,6 +1524,12 @@ func nameForService(svc *corev1.Service) string { return svc.Namespace + "-" + svc.Name } +// markedForDeletion reports whether obj has a deletion timestamp, i.e. the API server is waiting on finalizers +// before it can garbage collect it. +func markedForDeletion(obj metav1.Object) bool { + return !obj.GetDeletionTimestamp().IsZero() +} + // proxyClassForObject returns the proxy class for the given object. If the // object does not have a proxy class label, it returns the default proxy class func proxyClassForObject(o client.Object, proxyDefaultClass string) string { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index daa983b78..61ff57cf8 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -31,6 +31,8 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" @@ -188,7 +190,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque needsRequeue := false needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient) if err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err @@ -762,7 +764,7 @@ func (r *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName var count int for _, secret := range secrets.Items { - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return 0, fmt.Errorf("error getting node metadata: %w", err) } diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index eb39d0029..daa5edf9c 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -26,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/net/dns/resolvconffile" "tailscale.com/tstime" @@ -123,7 +124,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request } if err := a.maybeProvision(ctx, logger, svc); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index c4c322904..11c4565a2 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -45,6 +45,9 @@ import ( const ( vipTestIP = "5.6.7.8" + + // tsNamespace is the namespace the operator is installed in for tests that don't override it. + tsNamespace = "tailscale" ) // confgOpts contains configuration options for creating cluster resources for diff --git a/cmd/k8s-operator/tsrecorder_specs_test.go b/cmd/k8s-operator/tsrecorder_specs_test.go deleted file mode 100644 index 13da8a3c8..000000000 --- a/cmd/k8s-operator/tsrecorder_specs_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package main - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" -) - -func TestRecorderSpecs(t *testing.T) { - t.Run("spec-fields-passthrough", func(t *testing.T) { - tsr := &tsapi.Recorder{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: tsapi.RecorderSpec{ - Replicas: new(int32(3)), - StatefulSet: tsapi.RecorderStatefulSet{ - Labels: map[string]string{ - "ss-label-key": "ss-label-value", - }, - Annotations: map[string]string{ - "ss-annotation-key": "ss-annotation-value", - }, - Pod: tsapi.RecorderPod{ - Labels: map[string]string{ - "pod-label-key": "pod-label-value", - }, - Annotations: map[string]string{ - "pod-annotation-key": "pod-annotation-value", - }, - Affinity: &corev1.Affinity{ - PodAffinity: &corev1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "match-label": "match-value", - }, - }}, - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: new(int64(1000)), - }, - ImagePullSecrets: []corev1.LocalObjectReference{{ - Name: "img-pull", - }}, - NodeSelector: map[string]string{ - "some-node": "selector", - }, - Tolerations: []corev1.Toleration{{ - Key: "key", - Value: "value", - TolerationSeconds: new(int64(60)), - }}, - Container: tsapi.RecorderContainer{ - Env: []tsapi.Env{{ - Name: "some_env", - Value: "env_value", - }}, - Image: "custom-image", - ImagePullPolicy: corev1.PullAlways, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - }, - }, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - }, - }, - }, - }, - }, - }, - } - - ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) - - // StatefulSet-level. - if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - // Pod-level. - if diff := cmp.Diff(ss.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - // Container-level. - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, tsrEnv(tsr, tsLoginServer)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - if *ss.Spec.Replicas != *tsr.Spec.Replicas { - t.Errorf("expected %d replicas, got %d", *tsr.Spec.Replicas, *ss.Spec.Replicas) - } - - if len(ss.Spec.Template.Spec.Volumes) != int(*tsr.Spec.Replicas)+1 { - t.Errorf("expected %d volumes, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Volumes)) - } - - if len(ss.Spec.Template.Spec.Containers[0].VolumeMounts) != int(*tsr.Spec.Replicas)+1 { - t.Errorf("expected %d volume mounts, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Containers[0].VolumeMounts)) - } - }) -} diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go deleted file mode 100644 index 44e199bc7..000000000 --- a/cmd/k8s-operator/tsrecorder_test.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "go.uber.org/zap" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "tailscale.com/client/tailscale/v2" - - tsoperator "tailscale.com/k8s-operator" - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/k8s-operator/reconciler/tailscaled" - "tailscale.com/k8s-operator/tsclient" - "tailscale.com/tstest" -) - -const ( - tsNamespace = "tailscale" - tsLoginServer = "example.tailscale.com" -) - -func TestRecorder(t *testing.T) { - tsr := &tsapi.Recorder{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Finalizers: []string{"tailscale.com/finalizer"}, - }, - Spec: tsapi.RecorderSpec{ - Replicas: new(int32(3)), - }, - } - - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). - WithObjects(tsr). - WithStatusSubresource(tsr). - Build() - tsClient := &fakeTSClient{loginURL: tsLoginServer} - zl, _ := zap.NewDevelopment() - fr := record.NewFakeRecorder(2) - cl := tstest.NewClock(tstest.ClockOpts{}) - reconciler := &RecorderReconciler{ - tsNamespace: tsNamespace, - Client: fc, - clients: tsclient.NewProvider(tsClient), - recorder: fr, - log: zl.Sugar(), - clock: cl, - reissuer: tailscaled.NewReissuer(), - } - - t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { - expectReconciled(t, reconciler, "", tsr.Name) - - msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible" - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, false) - - expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible" - expectEvents(t, fr, []string{expectedEvent}) - - tsr.Spec.EnableUI = true - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ - "invalid space characters": "test", - } - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - expectReconciled(t, reconciler, "", tsr.Name) - - expectedEvent = "Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible" - expectEvents(t, fr, []string{expectedEvent}) - - tsr.Spec.Storage.S3 = &tsapi.S3{} - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - expectReconciled(t, reconciler, "", tsr.Name) - - // Only check part of this error message, because it's defined in an - // external package and may change. - if err := fc.Get(context.Background(), client.ObjectKey{ - Name: tsr.Name, - }, tsr); err != nil { - t.Fatal(err) - } - if len(tsr.Status.Conditions) != 1 { - t.Fatalf("expected 1 condition, got %d", len(tsr.Status.Conditions)) - } - cond := tsr.Status.Conditions[0] - if cond.Type != string(tsapi.RecorderReady) || cond.Status != metav1.ConditionFalse || cond.Reason != reasonRecorderInvalid { - t.Fatalf("expected condition RecorderReady false due to RecorderInvalid, got %v", cond) - } - for _, msg := range []string{cond.Message, <-fr.Events} { - if !strings.Contains(msg, `"invalid space characters"`) { - t.Fatalf("expected invalid annotation key in error message, got %q", cond.Message) - } - } - }) - - t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) { - mustCreate(t, fc, &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pre-existing-sa", - Namespace: tsNamespace, - }, - }) - - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa" - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace` - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - - expectedEvent := "Warning RecorderInvalid " + msg - expectEvents(t, fr, []string{expectedEvent}) - }) - - t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) { - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "" - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 1; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, true) - }) - - t.Run("valid_service_account_config", func(t *testing.T) { - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa" - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ - "test": "test", - } - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - expectEqual(t, fc, tsr) - if expected := 1; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, true) - - // Get the service account and check the annotations. - sa := &corev1.ServiceAccount{} - if err := fc.Get(context.Background(), client.ObjectKey{ - Name: tsr.Spec.StatefulSet.Pod.ServiceAccount.Name, - Namespace: tsNamespace, - }, sa); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" { - t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff) - } - if sa.Name != tsr.Spec.StatefulSet.Pod.ServiceAccount.Name { - t.Fatalf("unexpected service account name: got %q, want %q", sa.Name, tsr.Spec.StatefulSet.Pod.ServiceAccount.Name) - } - - expectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name) - }) - - t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) { - const key = "profile-abc" - - for replica := range *tsr.Spec.Replicas { - bytes, err := json.Marshal(map[string]any{ - "Config": map[string]any{ - "NodeID": fmt.Sprintf("node-%d", replica), - "UserProfile": map[string]any{ - "LoginName": fmt.Sprintf("test-%d.example.ts.net", replica), - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - name := fmt.Sprintf("%s-%d", "test", replica) - mustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) { - s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, - } - }) - } - - tsClient.devices = []tailscale.Device{ - { - ID: "node-0", - Hostname: "hostname-node-0", - Addresses: []string{"1.2.3.4", "::1"}, - }, - { - ID: "node-1", - Hostname: "hostname-node-1", - Addresses: []string{"1.2.3.4", "::1"}, - }, - { - ID: "node-2", - Hostname: "hostname-node-2", - Addresses: []string{"1.2.3.4", "::1"}, - }, - } - - expectReconciled(t, reconciler, "", tsr.Name) - tsr.Status.Devices = []tsapi.RecorderTailnetDevice{ - { - Hostname: "hostname-node-0", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-0.example.ts.net", - }, - { - Hostname: "hostname-node-1", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-1.example.ts.net", - }, - { - Hostname: "hostname-node-2", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-2.example.ts.net", - }, - } - expectEqual(t, fc, tsr) - }) - - t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) { - // Replica 0 comes first, so an unreported device there must not end cleanup early. - setUnreportedDevice(t, fc, tsClient, "test-0", "unreported-0") - if err := fc.Delete(context.Background(), tsr); err != nil { - t.Fatal(err) - } - - expectReconciled(t, reconciler, "", tsr.Name) - - expectMissing[tsapi.Recorder](t, fc, "", tsr.Name) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - if diff := cmp.Diff(tsClient.deleted, []string{"unreported-0", "node-1", "node-2"}); diff != "" { - t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) - } - // The fake client does not clean up objects whose owner has been - // deleted, so we can't test for the owned resources getting deleted. - }) -} - -func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) { - t.Helper() - - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - role := tsrRole(tsr, tsNamespace) - roleBinding := tsrRoleBinding(tsr, tsNamespace) - serviceAccount := tsrServiceAccount(tsr, tsNamespace) - statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) - - if shouldExist { - expectEqual(t, fc, role) - expectEqual(t, fc, roleBinding) - expectEqual(t, fc, serviceAccount) - expectEqual(t, fc, statefulSet, removeResourceReqs) - } else { - expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) - expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) - expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) - expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) - } - - for replica := range replicas { - auth := tsrAuthSecret(tsr, tsNamespace, "new-authkey", replica) - state := tsrStateSecret(tsr, tsNamespace, replica) - - if shouldExist { - expectEqual(t, fc, auth) - expectEqual(t, fc, state) - } else { - expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name) - expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name) - } - } -} diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index 2e0d0e1d4..30daf87b9 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -46,8 +46,7 @@ const ( serviceTypeIngress = "ingress" serviceTypeSvc = "svc" - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" - shortRequeue = 5 * time.Second + shortRequeue = 5 * time.Second // AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN. AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" @@ -157,7 +156,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res } if err := r.maybeProvision(ctx, proxySvc, logger); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go index a5d83b19c..43f7f0256 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go @@ -21,7 +21,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -30,6 +29,7 @@ import ( "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/dnsrecords" "tailscale.com/k8s-operator/reconciler/nameserver" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" ) @@ -70,8 +70,7 @@ func TestDNSRecordsReconciler(t *testing.T) { }, } cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}} - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(cm). WithObjects(dnsConfig). WithObjects(ing). @@ -83,7 +82,7 @@ func TestDNSRecordsReconciler(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) // Set the ready condition of the DNSConfig - mustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { + reconcilertest.MustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar()) }) dnsRR := dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{ @@ -109,11 +108,11 @@ func TestDNSRecordsReconciler(t *testing.T) { ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4) epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6) - mustCreate(t, fc, egressSvcFQDN) - mustCreate(t, fc, headlessForEgressSvcFQDN) - mustCreate(t, fc, ep) - mustCreate(t, fc, epv6) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.MustCreate(t, fc, egressSvcFQDN) + reconcilertest.MustCreate(t, fc, headlessForEgressSvcFQDN) + reconcilertest.MustCreate(t, fc, ep) + reconcilertest.MustCreate(t, fc, epv6) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service // ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7 wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} wantHostsIPv6 := map[string][]string{"foo.bar.ts.net": {"2600:1900:4011:161:0:d:0:d"}} @@ -121,57 +120,57 @@ func TestDNSRecordsReconciler(t *testing.T) { // 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's // value changes - mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) { svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net" }) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}} expectHostsRecords(t, fc, wantHosts) // 3. DNS record is updated if the IP address of the proxy Pod changes. ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4) - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Addresses = []string{"10.6.5.4"} }) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}} expectHostsRecords(t, fc, wantHosts) // 4. DNS record is created for an ingress proxy configured via Ingress headlessForIngress := headlessSvcForParent(ing, "ingress") ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4) - mustCreate(t, fc, headlessForIngress) - mustCreate(t, fc, ep) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service + reconcilertest.MustCreate(t, fc, headlessForIngress) + reconcilertest.MustCreate(t, fc, ep) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"} expectHostsRecords(t, fc, wantHosts) // 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0]) t.Log("test case 5") - mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) { + reconcilertest.MustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) { ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net" }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service delete(wantHosts, "cluster.ingress.ts.net") wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"} expectHostsRecords(t, fc, wantHosts) // 6. DNS records are updated if Ingress proxy's Pod IP changes - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Addresses = []string{"7.8.9.10"} }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"} expectHostsRecords(t, fc, wantHosts) // 7. A not-ready Endpoint is removed from DNS config. - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Conditions.Ready = new(false) ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"1.2.3.4"}, }) }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"} expectHostsRecords(t, fc, wantHosts) @@ -192,7 +191,7 @@ func TestDNSRecordsReconciler(t *testing.T) { ExternalName: "unused", }, } - mustCreate(t, fc, parentEgressSvc) + reconcilertest.MustCreate(t, fc, parentEgressSvc) proxyGroupEgressSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -246,9 +245,9 @@ func TestDNSRecordsReconciler(t *testing.T) { }}, } - mustCreate(t, fc, proxyGroupEgressSvc) - mustCreate(t, fc, proxyGroupEps) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.MustCreate(t, fc, proxyGroupEgressSvc) + reconcilertest.MustCreate(t, fc, proxyGroupEps) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") // Verify DNS record uses ClusterIP Service IP, not Pod IPs wantHosts["external-service.example.ts.net"] = []string{"10.0.100.50"} @@ -256,17 +255,17 @@ func TestDNSRecordsReconciler(t *testing.T) { // 9. ProxyGroup egress DNS record updates when ClusterIP changes t.Log("test case 9: ProxyGroup egress ClusterIP change") - mustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) { svc.Spec.ClusterIP = "10.0.100.51" }) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") wantHosts["external-service.example.ts.net"] = []string{"10.0.100.51"} expectHostsRecords(t, fc, wantHosts) // 10. Test ProxyGroup service deletion and DNS cleanup t.Log("test case 10: ProxyGroup egress service deletion") - mustDeleteAll(t, fc, proxyGroupEgressSvc) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.MustDeleteAll(t, fc, proxyGroupEgressSvc) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") delete(wantHosts, "external-service.example.ts.net") expectHostsRecords(t, fc, wantHosts) } @@ -322,9 +321,8 @@ func TestDNSRecordsReconcilerOptimisticLockError(t *testing.T) { }, } - f := fake.NewClientBuilder(). + f := reconcilertest.NewClientBuilder(). WithInterceptorFuncs(funcs). - WithScheme(tsapi.GlobalScheme). WithObjects(dnsCfg, proxyGroupEgressSvc, egressSvc). WithStatusSubresource(dnsCfg). Build() @@ -410,8 +408,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(dnsCfg, ing, headlessSvc, epv4, epv6, cm). WithStatusSubresource(dnsCfg). Build() @@ -423,7 +420,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }) // Test dual-stack service records - expectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") + reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") wantIPv4 := map[string][]string{"dual-stack.example.ts.net": {"10.1.2.3"}} wantIPv6 := map[string][]string{"dual-stack.example.ts.net": {"2001:db8::1"}} @@ -468,9 +465,9 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }, } - mustCreate(t, fc, parentEgressSvc) - mustCreate(t, fc, proxyGroupSvc) - expectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack") + reconcilertest.MustCreate(t, fc, parentEgressSvc) + reconcilertest.MustCreate(t, fc, proxyGroupSvc) + reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack") wantIPv4["pg-service.example.ts.net"] = []string{"10.96.0.100"} wantIPv6["pg-service.example.ts.net"] = []string{"2001:db8::100"} @@ -519,7 +516,7 @@ func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.Add func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) { t.Helper() cm := new(corev1.ConfigMap) - if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { + if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { t.Fatalf("getting dnsconfig ConfigMap: %v", err) } if cm.Data == nil { @@ -541,7 +538,7 @@ func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][] func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, wantsHostsIPv6 map[string][]string) { t.Helper() cm := new(corev1.ConfigMap) - if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { + if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { t.Fatalf("getting dnsconfig ConfigMap: %v", err) } if cm.Data == nil { @@ -562,57 +559,3 @@ func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, t.Fatalf("unexpected IPv6 dns config (-got +want):\n%s", diff) } } - -func expectReconciled(t *testing.T, r *dnsrecords.Reconciler, ns, name string) { - t.Helper() - req := reconcile.Request{ - NamespacedName: types.NamespacedName{Namespace: ns, Name: name}, - } - res, err := r.Reconcile(context.Background(), req) - if err != nil { - t.Fatalf("Reconcile: unexpected error: %v", err) - } - if res.Requeue { - t.Fatalf("unexpected immediate requeue") - } -} - -func mustCreate(t *testing.T, c client.Client, obj client.Object) { - t.Helper() - if err := c.Create(context.Background(), obj); err != nil { - t.Fatalf("creating %q: %v", obj.GetName(), err) - } -} - -func mustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) { - t.Helper() - for _, obj := range objs { - if err := c.Delete(context.Background(), obj); err != nil { - t.Fatalf("deleting %q: %v", obj.GetName(), err) - } - } -} - -func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { - t.Fatalf("getting %q: %v", name, err) - } - update(obj) - if err := c.Update(context.Background(), obj); err != nil { - t.Fatalf("updating %q: %v", name, err) - } -} - -func mustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { - t.Fatalf("getting %q: %v", name, err) - } - update(obj) - if err := c.Status().Update(context.Background(), obj); err != nil { - t.Fatalf("updating status %q: %v", name, err) - } -} diff --git a/k8s-operator/reconciler/nameserver/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 9add71705..705b7e177 100644 --- a/k8s-operator/reconciler/nameserver/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -12,7 +12,6 @@ import ( _ "embed" "errors" "fmt" - "strings" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" @@ -50,8 +49,6 @@ const ( defaultNameserverImageRepo = "tailscale/k8s-nameserver" defaultNameserverImageTag = "stable" - - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) @@ -161,7 +158,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg) } if err = r.maybeProvision(ctx, &dnsCfg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } else { diff --git a/k8s-operator/reconciler/nameserver/nameserver_test.go b/k8s-operator/reconciler/nameserver/nameserver_test.go index 594917154..7cd0761e1 100644 --- a/k8s-operator/reconciler/nameserver/nameserver_test.go +++ b/k8s-operator/reconciler/nameserver/nameserver_test.go @@ -6,30 +6,23 @@ package nameserver_test import ( - "context" _ "embed" "encoding/json" - "reflect" "testing" "time" - "github.com/google/go-cmp/cmp" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/nameserver" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -96,8 +89,7 @@ func TestNameserverReconciler(t *testing.T) { }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(dnsConfig). WithStatusSubresource(dnsConfig). Build() @@ -115,7 +107,7 @@ func TestNameserverReconciler(t *testing.T) { Logger: logger.Sugar(), Clock: clock, }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) nameserverLabels := map[string]string{ @@ -169,7 +161,7 @@ func TestNameserverReconciler(t *testing.T) { {Name: "some-secret"}, } - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}} @@ -181,16 +173,16 @@ func TestNameserverReconciler(t *testing.T) { wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference} wantsSvc.Namespace = tsNamespace wantsSvc.ObjectMeta.Labels = nameserverLabels - expectEqual(t, fc, wantsSvc) + reconcilertest.ExpectEqual(t, fc, wantsSvc) }) t.Run("dns-config-status-is-set", func(t *testing.T) { // Verify that DNSConfig advertises the nameserver's Service IP address, // has the ready status condition and tailscale finalizer. - mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { svc.Spec.ClusterIP = "1.2.3.4" }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") dnsConfig.Finalizers = []string{reconciler.Finalizer} dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ @@ -204,17 +196,17 @@ func TestNameserverReconciler(t *testing.T) { LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, }) - expectEqual(t, fc, dnsConfig) + reconcilertest.ExpectEqual(t, fc, dnsConfig) }) t.Run("nameserver-image-updated", func(t *testing.T) { // Verify that nameserver image gets updated to match DNSConfig spec. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) t.Run("reconciler-preserves-custom-config", func(t *testing.T) { @@ -226,11 +218,11 @@ func TestNameserverReconciler(t *testing.T) { t.Fatalf("error marshalling ConfigMap contents: %v", err) } - mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { + reconcilertest.MustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { mak.Set(&cm.Data, "records.json", string(bs)) }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantCm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -243,60 +235,17 @@ func TestNameserverReconciler(t *testing.T) { Data: map[string]string{"records.json": string(bs)}, } - expectEqual(t, fc, wantCm) + reconcilertest.ExpectEqual(t, fc, wantCm) }) t.Run("uses-default-nameserver-image", func(t *testing.T) { // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, // the nameserver image defaults to tailscale/k8s-nameserver:stable. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image = nil }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:stable" - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) } - -func mustReconcile(t *testing.T, r *nameserver.Reconciler, req reconcile.Request) { - t.Helper() - if _, err := r.Reconcile(context.Background(), req); err != nil { - t.Fatalf("unexpected reconcile error: %v", err) - } -} - -func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { - t.Fatalf("getting object: %v", err) - } - update(obj) - if err := c.Update(context.Background(), obj); err != nil { - t.Fatalf("updating object: %v", err) - } -} - -func expectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O) { - t.Helper() - got := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{ - Name: want.GetName(), - Namespace: want.GetNamespace(), - }, got); err != nil { - t.Fatalf("getting %q: %v", want.GetName(), err) - } - // The resource version changes eagerly whenever the operator does even a - // no-op update. Asserting a specific value leads to overly brittle tests, - // so just remove it from both got and want. - got.SetResourceVersion("") - want.SetResourceVersion("") - // controller-runtime v0.20+ populates TypeMeta on objects returned by the - // fake client. Strip it so tests can continue to build expected objects - // without setting Kind/APIVersion explicitly. - got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) - want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) - if diff := cmp.Diff(got, want); diff != "" { - t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff) - } -} diff --git a/k8s-operator/reconciler/peerrelay/peerrelay_test.go b/k8s-operator/reconciler/peerrelay/peerrelay_test.go index 885a3427e..6254542da 100644 --- a/k8s-operator/reconciler/peerrelay/peerrelay_test.go +++ b/k8s-operator/reconciler/peerrelay/peerrelay_test.go @@ -14,7 +14,6 @@ import ( "net/netip" "slices" "strings" - "sync" "testing" "go.uber.org/zap" @@ -25,8 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" tailscaleclient "tailscale.com/client/tailscale/v2" @@ -34,7 +31,7 @@ import ( "tailscale.com/ipn" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/peerrelay" - "tailscale.com/k8s-operator/tsclient" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/types/key" ) @@ -1064,10 +1061,9 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - builder := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + builder := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). - WithInterceptorFuncs(applyPatchInterceptor()) + WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()) if tc.PeerRelay != nil { builder = builder.WithObjects(tc.PeerRelay) } @@ -1079,7 +1075,7 @@ func TestReconciler_Reconcile(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1149,7 +1145,7 @@ func TestReconciler_Reconcile(t *testing.T) { } if tc.ExpectedReadyStatus != "" || tc.ExpectedReadyReason != "" { - cond := readyCondition(&pr) + cond := reconcilertest.Condition(t, pr.Status.Conditions, tsapi.PeerRelayReady) if tc.ExpectedReadyStatus != "" && cond.Status != tc.ExpectedReadyStatus { t.Errorf("expected PeerRelayReady status %s, got %q", tc.ExpectedReadyStatus, cond.Status) } @@ -1294,16 +1290,6 @@ func assertSecretsForType(t *testing.T, fc client.Client, prName, secretType, la } } -func readyCondition(pr *tsapi.PeerRelay) metav1.Condition { - for _, cond := range pr.Status.Conditions { - if cond.Type == string(tsapi.PeerRelayReady) { - return cond - } - } - - return metav1.Condition{} -} - func assertService(t *testing.T, want expectedService, got *corev1.Service) { t.Helper() @@ -1455,8 +1441,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) { Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects( pr, @@ -1471,7 +1456,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{loginURL: testLoginURL}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(testLoginURL))), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1535,104 +1520,6 @@ func readTailscaledConfig(t *testing.T, fc client.Client, secretName string) ipn return conf } -func applyPatchInterceptor() interceptor.Funcs { - return interceptor.Funcs{ - Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if patch.Type() != types.ApplyPatchType { - return cl.Patch(ctx, obj, patch, opts...) - } - - key := client.ObjectKeyFromObject(obj) - existing := obj.DeepCopyObject().(client.Object) - if err := cl.Get(ctx, key, existing); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - - return cl.Create(ctx, obj) - } - - obj.SetResourceVersion(existing.GetResourceVersion()) - return cl.Update(ctx, obj) - }, - } -} - -type fakeClientProvider struct { - client tsclient.Client - err error -} - -func (p *fakeClientProvider) For(_ string) (tsclient.Client, error) { return p.client, p.err } - -type fakeTSClient struct { - tsclient.Client - - loginURL string - - mu sync.Mutex - keyCalls []tailscaleclient.CreateKeyRequest - deviceDeletes []string - devices []tailscaleclient.Device - nextKey []string -} - -func (c *fakeTSClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) } -func (c *fakeTSClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) } -func (c *fakeTSClient) LoginURL() string { return c.loginURL } - -func (c *fakeTSClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest { - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.keyCalls) -} - -func (c *fakeTSClient) DeviceDeletes() []string { - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.deviceDeletes) -} - -type fakeKeys fakeTSClient - -func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) { - c := (*fakeTSClient)(k) - c.mu.Lock() - defer c.mu.Unlock() - c.keyCalls = append(c.keyCalls, req) - - var key string - if len(c.nextKey) > 0 { - key, c.nextKey = c.nextKey[0], c.nextKey[1:] - } else { - key = fmt.Sprintf("auth-key-%d", len(c.keyCalls)) - } - return &tailscaleclient.Key{Key: key}, nil -} - -func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { return nil, nil } - -type fakeDevices fakeTSClient - -func (d *fakeDevices) Delete(_ context.Context, id string) error { - c := (*fakeTSClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - c.deviceDeletes = append(c.deviceDeletes, id) - return nil -} - -func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { - c := (*fakeTSClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.devices), nil -} - -func (d *fakeDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) { - return nil, nil -} - func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Parallel() @@ -1643,19 +1530,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Run("mints-key-on-first-reconcile", func(t *testing.T) { pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{nextKey: []string{"tskey-abc"}} + tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-abc")) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1681,19 +1567,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Run("reuses-existing-key-across-reconciles", func(t *testing.T) { pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{nextKey: []string{"tskey-first", "tskey-second"}} + tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-first", "tskey-second")) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1721,20 +1606,19 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { Spec: tsapi.PeerRelaySpec{Tags: tsapi.Tags{"tag:custom"}}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{} + tsc := reconcilertest.NewFakeClient() r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1801,8 +1685,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { unreported := stateSecret("test", "peerrelay-test-2", 2, "") unreported.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects( pr, @@ -1813,16 +1696,17 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { ). Build() - tsc := &fakeTSClient{devices: []tailscaleclient.Device{ - {NodeID: "device-bbb", MachineKey: mk.Public().String()}, - {NodeID: "device-other", MachineKey: key.NewMachine().Public().String()}, - }} + tsc := reconcilertest.NewFakeClient() + tsc.SetDevices( + tailscaleclient.Device{NodeID: "device-bbb", MachineKey: mk.Public().String()}, + tailscaleclient.Device{NodeID: "device-other", MachineKey: key.NewMachine().Public().String()}, + ) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1856,8 +1740,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects( pr, @@ -1867,13 +1750,13 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { ). Build() - tsc := &fakeTSClient{} + tsc := reconcilertest.NewFakeClient() r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1916,8 +1799,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { Spec: tsapi.PeerRelaySpec{Tailnet: "missing"}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() @@ -1927,7 +1809,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{err: errors.New("tailnet missing: not ready")}, + Clients: reconcilertest.NewFailingClientProvider(errors.New("tailnet missing: not ready")), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1941,13 +1823,10 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { t.Fatal(err) } - cond := readyCondition(&got) - if cond.Status != metav1.ConditionFalse { - t.Errorf("expected PeerRelayReady=False, got %q", cond.Status) - } - if cond.Reason != peerrelay.ReasonTailnetUnavailable { - t.Errorf("expected reason=%s, got %q", peerrelay.ReasonTailnetUnavailable, cond.Reason) - } + reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.PeerRelayReady, + metav1.ConditionFalse, peerrelay.ReasonTailnetUnavailable) + + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.PeerRelayReady) if !strings.Contains(cond.Message, "not ready") { t.Errorf("expected condition message to include resolver error, got %q", cond.Message) } @@ -1997,8 +1876,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { }, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects(pr, pc). Build() @@ -2008,7 +1886,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()), Resolver: testResolver, Logger: logger.Sugar(), }) diff --git a/k8s-operator/reconciler/proxyclass/proxyclass_test.go b/k8s-operator/reconciler/proxyclass/proxyclass_test.go index 54363ab61..cdd0315ad 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass_test.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass_test.go @@ -16,11 +16,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/proxyclass" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/tstest" ) @@ -51,8 +51,7 @@ func TestProxyClass(t *testing.T) { }, }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(pc). WithStatusSubresource(pc). Build() @@ -118,16 +117,7 @@ func TestProxyClass(t *testing.T) { if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil { t.Fatalf("Get: %v", err) } - var cond *metav1.Condition - for i := range got.Status.Conditions { - if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) { - cond = &got.Status.Conditions[i] - break - } - } - if cond == nil { - t.Fatalf("ProxyClassReady condition not set") - } + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady) if cond.Status != wantStatus { t.Errorf("condition Status: got %q, want %q", cond.Status, wantStatus) } @@ -337,8 +327,7 @@ func TestProxyClassValidation(t *testing.T) { objs = append(objs, existingPC) } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(objs...). WithStatusSubresource(pc, existingPC). Build() @@ -365,16 +354,7 @@ func TestProxyClassValidation(t *testing.T) { t.Fatalf("Get: %v", err) } - var ready *metav1.Condition - for i := range got.Status.Conditions { - if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) { - ready = &got.Status.Conditions[i] - break - } - } - if ready == nil { - t.Fatal("Ready condition not set") - } + ready := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady) wantStatus := metav1.ConditionTrue if !tc.valid { diff --git a/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go b/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go index d5c0b6d35..6e4b8cb51 100644 --- a/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go +++ b/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go @@ -12,11 +12,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" + "tailscale.com/k8s-operator/reconciler/reconcilertest" ) func TestReconciler_Reconcile(t *testing.T) { @@ -102,7 +102,7 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - bldr := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) + bldr := reconcilertest.NewClientBuilder() bldr = bldr.WithObjects(tc.ExistingResources...) fc := bldr.Build() diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go index 5825f73ae..66339fc96 100644 --- a/k8s-operator/reconciler/reconciler.go +++ b/k8s-operator/reconciler/reconciler.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "slices" + "strings" "sync" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -40,8 +41,20 @@ const ( // LabelParentNamespace identifies the namespace of the owning Tailscale CRD. It is only stamped when the parent // CRD is namespaced; cluster-scoped parents omit it. LabelParentNamespace = "tailscale.com/parent-resource-ns" + + // optimisticLockErrorMsg is the message the API server returns when an Update is rejected because the object + // has been modified since it was read. The API server does not give us a typed error for the conflict in all + // cases, so callers match on the message via IsOptimisticLockError. + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) +// IsOptimisticLockError reports whether err was caused by an optimistic locking conflict, i.e. the object being +// updated was modified after the reconciler read it. Such an error is transient: the reconciler should requeue and +// retry against a freshly read object rather than surface the error. +func IsOptimisticLockError(err error) bool { + return err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) +} + // SetFinalizer adds name to obj's finalizers if not already present. Most CRD reconcilers pass Finalizer; per- // reconciler names are used when a single resource may carry finalizers from multiple reconcilers that each need to run // their own cleanup. diff --git a/k8s-operator/reconciler/reconciler_test.go b/k8s-operator/reconciler/reconciler_test.go index 7ae76df42..1e6f54bdf 100644 --- a/k8s-operator/reconciler/reconciler_test.go +++ b/k8s-operator/reconciler/reconciler_test.go @@ -6,6 +6,8 @@ package reconciler_test import ( + "errors" + "fmt" "maps" "slices" "testing" @@ -144,6 +146,37 @@ func TestEnqueueForChild(t *testing.T) { } } +func TestIsOptimisticLockError(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + err error + want bool + }{ + "nil": {}, + "conflict": { + err: errors.New("Operation cannot be fulfilled on secrets \"test\": the object has been modified; please apply your changes to the latest version and try again"), + want: true, + }, + "unrelated": { + err: errors.New("secrets \"test\" not found"), + }, + "wrapped-conflict": { + err: fmt.Errorf("failed to update Secret: %w", errors.New("the object has been modified; please apply your changes to the latest version and try again")), + want: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + if got := reconciler.IsOptimisticLockError(tc.err); got != tc.want { + t.Errorf("expected %v, got %v", tc.want, got) + } + }) + } +} + func TestResourceTracker(t *testing.T) { t.Parallel() diff --git a/k8s-operator/reconciler/reconcilertest/conditions.go b/k8s-operator/reconciler/reconcilertest/conditions.go new file mode 100644 index 000000000..0b88b9ceb --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/conditions.go @@ -0,0 +1,59 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package reconcilertest + +import ( + "testing" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +// The condition helpers below take a []metav1.Condition rather than the owning object. Every Tailscale CRD stores its +// conditions in Status.Conditions, but the CRD types live in tsapi and so can't grow a shared accessor method from +// here; taking the slice keeps these usable from every reconciler's tests (and for corev1.Service conditions) without +// per-CRD plumbing. Pair them with MustGet: +// +// tsr := &tsapi.Recorder{} +// reconcilertest.MustGet(t, fc, "", "test", tsr) +// reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, metav1.ConditionTrue, reason, message) + +// Condition returns the condition of the given type, failing the test if it isn't present. Use it when a test needs to +// inspect a condition's fields directly, e.g. to match part of a message generated outside this repo; prefer +// ExpectCondition for whole-condition assertions. +func Condition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType) metav1.Condition { + t.Helper() + cond := apimeta.FindStatusCondition(conds, string(conditionType)) + if cond == nil { + t.Fatalf("no %s condition found, got %+v", conditionType, conds) + } + return *cond +} + +// ExpectCondition asserts that a condition of the given type is present with the given status, reason and message. +// LastTransitionTime and ObservedGeneration are not compared, so tests don't need a fixed clock or to track +// generations in order to assert on the parts that describe the outcome. +func ExpectCondition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) { + t.Helper() + got := Condition(t, conds, conditionType) + if got.Status != status || got.Reason != reason || got.Message != message { + t.Fatalf("unexpected %s condition: got (%s, %s, %q), want (%s, %s, %q)", + conditionType, got.Status, got.Reason, got.Message, status, reason, message) + } +} + +// ExpectConditionStatus asserts a condition's status and reason, ignoring its message. Use it when the message is +// long, generated outside this repo, or otherwise not the point of the test. +func ExpectConditionStatus(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason string) { + t.Helper() + got := Condition(t, conds, conditionType) + if got.Status != status || got.Reason != reason { + t.Fatalf("unexpected %s condition: got (%s, %s), want (%s, %s)", + conditionType, got.Status, got.Reason, status, reason) + } +} diff --git a/k8s-operator/reconciler/reconcilertest/reconcilertest.go b/k8s-operator/reconciler/reconcilertest/reconcilertest.go new file mode 100644 index 000000000..c2529915c --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/reconcilertest.go @@ -0,0 +1,202 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package reconcilertest provides helpers for testing the CRD reconcilers under k8s-operator/reconciler. It covers +// the parts of a reconciler test that are the same whichever CRD is under test: building a fake client with the +// operator's scheme, creating and mutating objects in it, asserting on what a reconcile left behind, and faking the +// Tailscale API. +// +// Assertions come in two flavours. Must* helpers set up test preconditions and call t.Fatalf if the cluster doesn't +// behave, because a failed precondition makes the rest of the test meaningless. Expect* helpers assert on the state +// a reconcile produced. +// +// Per-CRD assertions belong in the reconciler's own test package; only helpers that are genuinely reconciler- +// agnostic belong here. +package reconcilertest + +import ( + "context" + "reflect" + "slices" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" +) + +// NewClientBuilder returns a fake client builder preconfigured with the operator's scheme. Callers add their own +// objects, status subresources and interceptors before calling Build. +func NewClientBuilder() *fake.ClientBuilder { + return fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) +} + +// ApplyPatchInterceptor returns interceptor funcs that emulate server-side apply, which the fake client does not +// implement. Apply patches become a Create if the object is absent and an Update if it is present; every other patch +// type is passed through untouched. Reconcilers that apply their children (rather than using +// reconciler.CreateOrUpdate) need this to be testable against the fake client. +func ApplyPatchInterceptor() interceptor.Funcs { + return interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if patch.Type() != types.ApplyPatchType { + return cl.Patch(ctx, obj, patch, opts...) + } + + key := client.ObjectKeyFromObject(obj) + existing := obj.DeepCopyObject().(client.Object) + if err := cl.Get(ctx, key, existing); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + + return cl.Create(ctx, obj) + } + + obj.SetResourceVersion(existing.GetResourceVersion()) + return cl.Update(ctx, obj) + }, + } +} + +// MustCreate creates obj, failing the test if the cluster rejects it. +func MustCreate(t *testing.T, c client.Client, obj client.Object) { + t.Helper() + if err := c.Create(t.Context(), obj); err != nil { + t.Fatalf("creating %q: %v", obj.GetName(), err) + } +} + +// MustGet reads ns/name into obj, failing the test if it is absent. Use ExpectMissing to assert the opposite. +func MustGet(t *testing.T, c client.Client, ns, name string, obj client.Object) { + t.Helper() + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } +} + +// MustDeleteAll deletes each of objs in order. +func MustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + if err := c.Delete(t.Context(), obj); err != nil { + t.Fatalf("deleting %q: %v", obj.GetName(), err) + } + } +} + +// MustUpdate reads ns/name, applies update to it and writes it back. It reads the object fresh rather than taking one +// from the caller so the update is applied to the current ResourceVersion. +func MustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Update(t.Context(), obj); err != nil { + t.Fatalf("updating %q: %v", name, err) + } +} + +// MustUpdateStatus is MustUpdate for the status subresource. The object's type must have been registered with +// WithStatusSubresource on the fake client builder. +func MustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Status().Update(t.Context(), obj); err != nil { + t.Fatalf("updating status %q: %v", name, err) + } +} + +// ExpectEqual asserts that the object named by want exists and matches it. ResourceVersion and TypeMeta are cleared +// on both sides first: the former changes on every no-op write and the latter is populated by the fake client but not +// by the callers building expected objects, so asserting on either makes tests brittle. Pass modifiers to blank out +// any further fields that shouldn't be compared; each one runs against both want and got. +func ExpectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O, modifiers ...func(O)) { + t.Helper() + got := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{ + Name: want.GetName(), + Namespace: want.GetNamespace(), + }, got); err != nil { + t.Fatalf("getting %q: %v", want.GetName(), err) + } + got.SetResourceVersion("") + want.SetResourceVersion("") + got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + for _, modifier := range modifiers { + modifier(want) + modifier(got) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff) + } +} + +// ExpectMissing asserts that no object of type T exists at ns/name. +func ExpectMissing[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string) { + t.Helper() + obj := O(new(T)) + err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj) + if !apierrors.IsNotFound(err) { + t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name) + } +} + +// ExpectReconciled runs a single reconcile for ns/name and asserts it succeeded without asking to be requeued, i.e. +// the reconciler considers the resource fully settled. +func ExpectReconciled(t *testing.T, r reconcile.Reconciler, ns, name string) { + t.Helper() + res, err := r.Reconcile(t.Context(), request(ns, name)) + if err != nil { + t.Fatalf("Reconcile: unexpected error: %v", err) + } + if res.Requeue { + t.Fatalf("unexpected immediate requeue") + } + if res.RequeueAfter != 0 { + t.Fatalf("unexpected timed requeue (%v)", res.RequeueAfter) + } +} + +func request(ns, name string) reconcile.Request { + return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: name}} +} + +// ExpectEvents asserts that len(want) events are emitted on rec, each of which is in want, waiting up to five seconds +// for each. Events are not required to arrive in order, because a reconcile may emit them from more than one code path. +func ExpectEvents(t *testing.T, rec *record.FakeRecorder, want []string) { + t.Helper() + seen := make([]string, 0, len(want)) + for range want { + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case got := <-rec.Events: + if !slices.Contains(want, got) { + t.Errorf("got unexpected event %q, expected events: %+#v", got, want) + continue + } + seen = append(seen, got) + case <-timer.C: + t.Errorf("timeout waiting for an event, wants events %+#v, got events %+#v", want, seen) + } + } +} diff --git a/k8s-operator/reconciler/reconcilertest/tsclient.go b/k8s-operator/reconciler/reconcilertest/tsclient.go new file mode 100644 index 000000000..a90aea029 --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/tsclient.go @@ -0,0 +1,174 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package reconcilertest + +import ( + "context" + "fmt" + "net/http" + "slices" + "sync" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/k8s-operator/tsclient" +) + +// FakeClient is a tsclient.Client that records the calls a reconciler makes to the Tailscale API and serves canned +// responses. The zero value is usable: keys are minted with generated values, no devices exist, and the login URL is +// blank. FakeClient is safe for concurrent use. +// +// Only the endpoints the reconcilers actually call are implemented. Anything else panics via the embedded nil +// interface rather than silently returning a zero value, so a reconciler that starts calling a new endpoint fails +// loudly instead of being tested against a fake that quietly does nothing. +type FakeClient struct { + tsclient.Client + + mu sync.Mutex + + loginURL string + + // Devices are returned by Devices().Get and List. Get matches on Device.ID and returns a 404 API error when no + // device matches, mirroring control's behaviour for an unknown or already-deleted device. + devices []tailscaleclient.Device + + // nextKeys are handed out by CreateAuthKey in order. Once exhausted, generated values are returned instead, so + // tests only script the keys they assert on. + nextKeys []string + + keyCalls []tailscaleclient.CreateKeyRequest + deviceDeletes []string +} + +// FakeClientOption configures a FakeClient. +type FakeClientOption func(*FakeClient) + +// WithLoginURL sets the URL returned by LoginURL, which reconcilers embed in the workloads they create. +func WithLoginURL(url string) FakeClientOption { + return func(c *FakeClient) { c.loginURL = url } +} + +// WithAuthKeys sets the auth keys CreateAuthKey returns, in order. Use it when a test needs to tell successive keys +// apart, e.g. to assert that a replica picked up a reissued key. +func WithAuthKeys(keys ...string) FakeClientOption { + return func(c *FakeClient) { c.nextKeys = keys } +} + +// NewFakeClient returns a FakeClient configured by opts. +func NewFakeClient(opts ...FakeClientOption) *FakeClient { + c := &FakeClient{} + for _, opt := range opts { + opt(c) + } + return c +} + +// LoginURL implements tsclient.Client. +func (c *FakeClient) LoginURL() string { return c.loginURL } + +// Devices implements tsclient.Client. +func (c *FakeClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) } + +// Keys implements tsclient.Client. +func (c *FakeClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) } + +// CreateAuthKeyCalls returns every auth key request the reconciler has made, in order. +func (c *FakeClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.keyCalls) +} + +// DeviceDeletes returns the IDs of every device the reconciler has deleted, in order. +func (c *FakeClient) DeviceDeletes() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.deviceDeletes) +} + +// SetDevices replaces the devices served by the fake. Use it to simulate control catching up partway through a test, +// e.g. once a replica has authenticated. +func (c *FakeClient) SetDevices(devices ...tailscaleclient.Device) { + c.mu.Lock() + defer c.mu.Unlock() + c.devices = devices +} + +type fakeDevices FakeClient + +func (d *fakeDevices) Delete(_ context.Context, id string) error { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + c.deviceDeletes = append(c.deviceDeletes, id) + return nil +} + +func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.devices), nil +} + +func (d *fakeDevices) Get(_ context.Context, id string) (*tailscaleclient.Device, error) { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + for _, dev := range c.devices { + if dev.ID == id { + return &dev, nil + } + } + return nil, tailscaleclient.APIError{Status: http.StatusNotFound} +} + +type fakeKeys FakeClient + +func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) { + c := (*FakeClient)(k) + c.mu.Lock() + defer c.mu.Unlock() + c.keyCalls = append(c.keyCalls, req) + + var key string + if len(c.nextKeys) > 0 { + key, c.nextKeys = c.nextKeys[0], c.nextKeys[1:] + } else { + key = fmt.Sprintf("auth-key-%d", len(c.keyCalls)) + } + return &tailscaleclient.Key{Key: key}, nil +} + +func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { + return nil, nil +} + +// FakeClientProvider is a tailscaled.ClientProvider that hands out a single client for every tailnet, or fails with a +// fixed error. Reconcilers resolve their API client through a provider, so tests that exercise the +// tailnet-unavailable path need one that can fail. +type FakeClientProvider struct { + client tsclient.Client + err error +} + +// NewFakeClientProvider returns a provider that resolves every tailnet to client. +func NewFakeClientProvider(client tsclient.Client) *FakeClientProvider { + return &FakeClientProvider{client: client} +} + +// NewFailingClientProvider returns a provider that fails to resolve any tailnet with err. +func NewFailingClientProvider(err error) *FakeClientProvider { + return &FakeClientProvider{err: err} +} + +// For implements tailscaled.ClientProvider. +func (p *FakeClientProvider) For(_ string) (tsclient.Client, error) { + if p.err != nil { + return nil, p.err + } + return p.client, nil +} diff --git a/k8s-operator/reconciler/recorder/authkey.go b/k8s-operator/reconciler/recorder/authkey.go new file mode 100644 index 000000000..ef8fba988 --- /dev/null +++ b/k8s-operator/reconciler/recorder/authkey.go @@ -0,0 +1,91 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package recorder + +import ( + "context" + "fmt" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" + "tailscale.com/k8s-operator/tsclient" +) + +// authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret. +const authKeySecretKey = "authkey" + +// recorderTags returns the tags to apply to auth keys minted for tsr's replicas. +func recorderTags(tsr *tsapi.Recorder) []string { + if len(tsr.Spec.Tags) == 0 { + return tsapi.Tags{"tag:k8s"}.Stringify() + } + + return tsr.Spec.Tags.Stringify() +} + +// ensureAuthSecretsCreated makes sure every replica has an auth Secret holding a usable auth key. A key is minted +// when the Secret doesn't exist yet, or when the replica has requested a reissue via its state Secret because the +// existing key can no longer authenticate it. +func (r *Reconciler) ensureAuthSecretsCreated(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + tags := recorderTags(tsr) + + for replica := range replicas(tsr) { + key := types.NamespacedName{ + Namespace: r.tsNamespace, + Name: authSecretName(tsr.Name, replica), + } + + existingSecret := &corev1.Secret{} + err := r.Get(ctx, key, existingSecret) + switch { + case err == nil: + stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) + if err != nil { + return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err) + } + cfgAuthKey := string(existingSecret.Data[authKeySecretKey]) + reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, logger, tailscaled.ReissueInput{ + ParentName: tsr.Name, + ReplicaName: stateSecretName(tsr.Name, replica), + Kind: tailscaled.KindRecorder, + StateSecret: stateSecret, + CfgAuthKey: &cfgAuthKey, + }) + if err != nil { + return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) + } + if !reissue { + logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) + continue + } + authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags) + if err != nil { + return err + } + existingSecret.Data[authKeySecretKey] = []byte(authKey) + if err = r.Update(ctx, existingSecret); err != nil { + return err + } + case apierrors.IsNotFound(err): + authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags) + if err != nil { + return err + } + if err = r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { + return err + } + default: + return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) + } + } + + return nil +} diff --git a/cmd/k8s-operator/tsrecorder.go b/k8s-operator/reconciler/recorder/recorder.go similarity index 61% rename from cmd/k8s-operator/tsrecorder.go rename to k8s-operator/reconciler/recorder/recorder.go index 74ee7051c..60497b578 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/k8s-operator/reconciler/recorder/recorder.go @@ -3,20 +3,21 @@ //go:build !plan9 -package main +// Package recorder provides reconciliation logic for the Recorder custom resource definition. It is responsible for +// managing the lifecycle of tsrecorder devices, including the StatefulSet, ServiceAccount and RBAC resources used to +// run them. +package recorder import ( "context" - "encoding/json" "errors" "fmt" "slices" "strconv" "strings" - "sync" + "time" "go.uber.org/zap" - xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -24,68 +25,132 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apivalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/client/tailscale/v2" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" - "tailscale.com/tailcfg" "tailscale.com/tstime" "tailscale.com/util/clientmetric" - "tailscale.com/util/set" ) const ( - reasonRecorderCreationFailed = "RecorderCreationFailed" - reasonRecorderCreating = "RecorderCreating" - reasonRecorderCreated = "RecorderCreated" - reasonRecorderInvalid = "RecorderInvalid" - reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" + reconcilerName = "recorder-reconciler" - currentProfileKey = "_current-profile" - - // authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret. - authKeySecretKey = "authkey" + // shortRequeue is how long to wait before retrying a reconcile that made progress but hasn't finished, e.g. + // cleanup that is waiting on control to catch up. + shortRequeue = 5 * time.Second ) +// Condition reasons set on Recorder resources. +const ( + reasonRecorderCreated = "RecorderCreated" + reasonRecorderCreationFailed = "RecorderCreationFailed" + reasonRecorderCreating = "RecorderCreating" + reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" +) + +// gaugeRecorderResources tracks the overall number of Recorder resources currently managed by this operator instance. var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount) -// RecorderReconciler syncs Recorder statefulsets with their definition in -// Recorder CRs. -type RecorderReconciler struct { - client.Client - log *zap.SugaredLogger - recorder record.EventRecorder - clock tstime.Clock - clients ClientProvider - tsNamespace string - reissuer *tailscaled.Reissuer - mu sync.Mutex // protects following - recorders set.Slice[types.UID] // for recorders gauge +type ( + // Reconciler is a reconcile.Reconciler implementation that syncs Recorder StatefulSets with their definition in + // Recorder custom resources. + Reconciler struct { + client.Client + + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock + clients tailscaled.ClientProvider + tsNamespace string + reissuer *tailscaled.Reissuer + + tracker *reconciler.ResourceTracker + } + + // ReconcilerOptions contains configuration values for the Reconciler. + ReconcilerOptions struct { + // Client is used to interact with the Kubernetes API. + Client client.Client + // Recorder is used to emit Kubernetes events. + Recorder record.EventRecorder + // TailscaleNamespace is the namespace the operator is installed in. Recorder-managed resources + // (StatefulSets, Secrets, etc.) are created within this namespace. + TailscaleNamespace string + // Clients resolves the Tailscale API client for a given tailnet name. Used to mint auth keys for each + // replica and to delete their devices on cleanup. Blank tailnet returns the operator's default client. + Clients tailscaled.ClientProvider + // Logger is the logger to use for this Reconciler. + Logger *zap.SugaredLogger + // Clock is used to stamp condition transitions. Defaults to a real clock when unset. + Clock tstime.Clock + } +) + +// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Recorder +// custom resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + clock := options.Clock + if clock == nil { + clock = tstime.DefaultClock{} + } + + return &Reconciler{ + Client: options.Client, + recorder: options.Recorder, + logger: options.Logger.Named(reconcilerName), + clock: clock, + clients: options.Clients, + tsNamespace: options.TailscaleNamespace, + reissuer: tailscaled.NewReissuer(), + tracker: reconciler.NewResourceTracker(gaugeRecorderResources), + } } -func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger { - return r.log.With("Recorder", name) +// Register the Reconciler onto the given manager.Manager implementation. It watches Recorder resources directly, as +// well as the child resources it manages, so that drift is corrected by a reconcile of the owning Recorder. All +// children carry a controller owner reference to their Recorder, so they are matched by owner rather than by label. +func (r *Reconciler) Register(mgr manager.Manager) error { + enqueue := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{}) + return builder. + ControllerManagedBy(mgr). + For(&tsapi.Recorder{}). + Named(reconcilerName). + Watches(&appsv1.StatefulSet{}, enqueue). + Watches(&corev1.ServiceAccount{}, enqueue). + Watches(&corev1.Secret{}, enqueue). + Watches(&rbacv1.Role{}, enqueue). + Watches(&rbacv1.RoleBinding{}, enqueue). + Complete(r) } -func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - logger := r.logger(req.Name) +// Reconcile is invoked when a change occurs to Recorder resources within the cluster. On create/update it ensures a +// StatefulSet and the Secrets and RBAC resources each replica needs exist. On delete it removes each replica's +// device from the tailnet before releasing the finalizer; the in-cluster resources are garbage collected via their +// owner references. +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + logger := r.logger.With("Recorder", req.Name) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") tsr := new(tsapi.Recorder) err := r.Get(ctx, req.NamespacedName, tsr) - if apierrors.IsNotFound(err) { + switch { + case apierrors.IsNotFound(err): logger.Debugf("Recorder not found, assuming it was deleted") return reconcile.Result{}, nil - } else if err != nil { + case err != nil: return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err) } @@ -107,36 +172,33 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error()) } - if markedForDeletion(tsr) { + if !tsr.DeletionTimestamp.IsZero() { logger.Debugf("Recorder is being deleted, cleaning up resources") - ix := xslices.Index(tsr.Finalizers, FinalizerName) - if ix < 0 { + if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) { logger.Debugf("no finalizer, nothing to do") return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, tsr, tsClient); err != nil { + if done, err := r.maybeCleanup(ctx, logger, tsr, tsClient); err != nil { return reconcile.Result{}, err } else if !done { logger.Debugf("Recorder resource cleanup not yet finished, will retry...") return reconcile.Result{RequeueAfter: shortRequeue}, nil } - tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1) - if err = r.Update(ctx, tsr); err != nil { + if err = reconciler.ClearFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil { return reconcile.Result{}, err } return reconcile.Result{}, nil } - if !slices.Contains(tsr.Finalizers, FinalizerName) { + if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, // this is a nice place to log that the high level, multi-reconcile // operation is underway. logger.Infof("ensuring Recorder is set up") - tsr.Finalizers = append(tsr.Finalizers, FinalizerName) - if err = r.Update(ctx, tsr); err != nil { + if err = reconciler.EnsureFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil { return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed) } } @@ -147,10 +209,10 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message) } - if err = r.maybeProvision(ctx, tsClient, tsr); err != nil { + if err = r.maybeProvision(ctx, logger, tsClient, tsr); err != nil { reason := reasonRecorderCreationFailed message := fmt.Sprintf("failed creating Recorder: %s", err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { reason = reasonRecorderCreating message = fmt.Sprintf("optimistic lock error, retrying: %s", err) err = nil @@ -165,28 +227,20 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated) } -func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { - logger := r.logger(tsr.Name) +func (r *Reconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + replicas := replicas(tsr) - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - r.mu.Lock() - r.recorders.Add(tsr.UID) - gaugeRecorderResources.Set(int64(r.recorders.Len())) - r.mu.Unlock() + r.tracker.Add(tsr.UID) r.reissuer.EnsureState(tsr.Name, int(replicas)) - if err := r.ensureAuthSecretsCreated(ctx, tsClient, tsr); err != nil { + if err := r.ensureAuthSecretsCreated(ctx, logger, tsClient, tsr); err != nil { return fmt.Errorf("error creating secrets: %w", err) } // State Secrets are pre-created so we can use the Recorder CR as its owner ref. for replica := range replicas { sec := tsrStateSecret(tsr, r.tsNamespace, replica) - _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { + _, err := reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { s.ObjectMeta.Labels = sec.ObjectMeta.Labels s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations }) @@ -196,7 +250,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } sa := tsrServiceAccount(tsr, r.tsNamespace) - _, err := createOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { + _, err := reconciler.CreateOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { // Perform this check within the update function to make sure we don't // have a race condition between the previous check and the update. if err := saOwnedByRecorder(s, tsr); err != nil { @@ -213,7 +267,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } role := tsrRole(tsr, r.tsNamespace) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels r.ObjectMeta.Annotations = role.ObjectMeta.Annotations r.Rules = role.Rules @@ -223,7 +277,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } roleBinding := tsrRoleBinding(tsr, r.tsNamespace) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations r.RoleRef = roleBinding.RoleRef @@ -234,7 +288,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } ss := tsrStatefulSet(tsr, r.tsNamespace, tsClient.LoginURL()) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.Spec = ss.Spec @@ -246,13 +300,13 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie // ServiceAccount name may have changed, in which case we need to clean up // the previous ServiceAccount. RoleBinding will already be updated to point // to the new ServiceAccount. - if err = r.maybeCleanupServiceAccounts(ctx, tsr, sa.Name); err != nil { + if err = r.maybeCleanupServiceAccounts(ctx, logger, tsr, sa.Name); err != nil { return fmt.Errorf("error cleaning up ServiceAccounts: %w", err) } // If we have scaled the recorder down, we will have dangling state secrets // that we need to clean up. - if err = r.maybeCleanupSecrets(ctx, tsClient, tsr); err != nil { + if err = r.maybeCleanupSecrets(ctx, logger, tsClient, tsr); err != nil { return fmt.Errorf("error cleaning up Secrets: %w", err) } @@ -292,9 +346,7 @@ func saOwnedByRecorder(sa *corev1.ServiceAccount, tsr *tsapi.Recorder) error { // accumulating a large amount of garbage. // // This is a no-op if the ServiceAccount name has not changed. -func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, tsr *tsapi.Recorder, currentName string) error { - logger := r.logger(tsr.Name) - +func (r *Reconciler) maybeCleanupServiceAccounts(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, currentName string) error { options := []client.ListOption{ client.InNamespace(r.tsNamespace), client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), @@ -323,7 +375,9 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts return nil } -func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { +// maybeCleanupSecrets deletes the state and auth Secrets of any replica beyond the Recorder's current replica count, +// along with each one's tailnet device. Their ordinals are read back from the Secret names. +func (r *Reconciler) maybeCleanupSecrets(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { options := []client.ListOption{ client.InNamespace(r.tsNamespace), client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), @@ -336,10 +390,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t // Get the largest ordinal suffix that we expect. Then we'll go through the list of secrets owned by this // recorder and remove them. - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } + replicas := replicas(tsr) for _, secret := range secrets.Items { parts := strings.Split(secret.Name, "-") @@ -356,22 +407,16 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t continue } - devicePrefs, ok, err := getDevicePrefs(&secret) + devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return err } if ok { - r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID) - err = tsClient.Devices().Delete(ctx, string(devicePrefs.Config.NodeID)) - switch { - case tailscale.IsNotFound(err): - // This device has possibly already been deleted in the admin console. So we can ignore this - // and move on to removing the secret. - case err != nil: + if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil { return err } - } else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, r.log, &secret); err != nil { + } else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, logger, &secret); err != nil { return err } @@ -386,25 +431,18 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a Recorder will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) { - logger := r.logger(tsr.Name) - - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - for replica := range replicas { +func (r *Reconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) { + for replica := range replicas(tsr) { secret, err := r.getStateSecret(ctx, tsr.Name, replica) if err != nil { return false, err } if secret == nil { - logger.Debugf("state Secret %s-%d not found, continuing cleanup", tsr.Name, replica) + logger.Debugf("state Secret %s not found, continuing cleanup", stateSecretName(tsr.Name, replica)) continue } - devicePrefs, ok, err := getDevicePrefs(secret) + devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(secret) if err != nil { return false, err } @@ -415,18 +453,9 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record continue } - nodeID := string(devicePrefs.Config.NodeID) - logger.Debugf("deleting device %s from control", nodeID) - err = tsClient.Devices().Delete(ctx, nodeID) - switch { - case tailscale.IsNotFound(err): - logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID) - continue - case err != nil: - return false, fmt.Errorf("error deleting device: %w", err) + if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil { + return false, err } - - logger.Debugf("device %s deleted from control", nodeID) } // Unlike most log entries in the reconcile loop, this will get printed @@ -434,83 +463,13 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record // cleanup removes the tailscale finalizer, which will make all future // reconciles exit early. logger.Infof("cleaned up Recorder resources") - r.mu.Lock() - r.recorders.Remove(tsr.UID) - gaugeRecorderResources.Set(int64(r.recorders.Len())) - r.mu.Unlock() + r.tracker.Remove(tsr.UID) r.reissuer.RemoveState(tsr.Name) return true, nil } -func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - tags := tsr.Spec.Tags - if len(tags) == 0 { - tags = tsapi.Tags{"tag:k8s"} - } - - logger := r.logger(tsr.Name) - - for replica := range replicas { - key := types.NamespacedName{ - Namespace: r.tsNamespace, - Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), - } - - existingSecret := &corev1.Secret{} - err := r.Get(ctx, key, existingSecret) - switch { - case err == nil: - stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) - if err != nil { - return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err) - } - cfgAuthKey := string(existingSecret.Data[authKeySecretKey]) - reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, r.log, tailscaled.ReissueInput{ - ParentName: tsr.Name, - ReplicaName: fmt.Sprintf("%s-%d", tsr.Name, replica), - Kind: tailscaled.KindRecorder, - StateSecret: stateSecret, - CfgAuthKey: &cfgAuthKey, - }) - if err != nil { - return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) - } - if !reissue { - logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) - continue - } - authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) - if err != nil { - return err - } - existingSecret.Data[authKeySecretKey] = []byte(authKey) - if err = r.Update(ctx, existingSecret); err != nil { - return err - } - continue - case apierrors.IsNotFound(err): - authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) - if err != nil { - return err - } - if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { - return err - } - default: - return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) - } - } - - return nil -} - -func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *Reconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error { if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil { return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible") } @@ -535,7 +494,7 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) case apierrors.IsNotFound(err): // ServiceAccount doesn't exist, so no conflict. case err != nil: - return fmt.Errorf("error getting ServiceAccount %q for validation: %w", tsr.Spec.StatefulSet.Pod.ServiceAccount.Name, err) + return fmt.Errorf("error getting ServiceAccount %q for validation: %w", specSA.Name, err) default: // ServiceAccount exists, check if it's owned by the Recorder. if err := saOwnedByRecorder(sa, tsr); err != nil { @@ -552,11 +511,13 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) return nil } -func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) { +// getStateSecret returns the tailscaled state Secret for the given Recorder replica, or nil if it does not exist +// yet. +func (r *Reconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: r.tsNamespace, - Name: fmt.Sprintf("%s-%d", tsrName, replica), + Name: stateSecretName(tsrName, replica), }, } if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { @@ -570,33 +531,17 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, return secret, nil } -// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName -// is expected to always be non-empty if the node ID is, but not required. -func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) { - // TODO(tomhjp): Should maybe use ipn to parse the following info instead. - currentProfile, ok := secret.Data[currentProfileKey] - if !ok { - return prefs, false, nil - } - profileBytes, ok := secret.Data[string(currentProfile)] - if !ok { - return prefs, false, nil - } - if err := json.Unmarshal(profileBytes, &prefs); err != nil { - return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err) - } - - ok = prefs.Config.NodeID != "" - return prefs, ok, nil -} - -func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { +func (r *Reconciler) getDevicePrefs(ctx context.Context, tsrName string, replica int32) (prefs tailscaled.Prefs, ok bool, err error) { secret, err := r.getStateSecret(ctx, tsrName, replica) if err != nil || secret == nil { - return tsapi.RecorderTailnetDevice{}, false, err + return prefs, false, err } - prefs, ok, err := getDevicePrefs(secret) + return tailscaled.PrefsFromStateSecret(secret) +} + +func (r *Reconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { + prefs, ok, err := r.getDevicePrefs(ctx, tsrName, replica) if !ok || err != nil { return tsapi.RecorderTailnetDevice{}, false, err } @@ -620,20 +565,11 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclien return d, true, nil } -// [prefs] is a subset of the ipn.Prefs struct used for extracting information -// from the state Secret of Tailscale devices. -type prefs struct { - Config struct { - NodeID tailcfg.StableNodeID `json:"NodeID"` - UserProfile struct { - // LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net. - LoginName string `json:"LoginName"` - } `json:"UserProfile"` - } `json:"Config"` +// replicas returns the number of replicas the Recorder asks for, defaulting to one. +func replicas(tsr *tsapi.Recorder) int32 { + if tsr.Spec.Replicas != nil { + return *tsr.Spec.Replicas + } - AdvertiseServices []string `json:"AdvertiseServices"` -} - -func markedForDeletion(obj metav1.Object) bool { - return !obj.GetDeletionTimestamp().IsZero() + return 1 } diff --git a/k8s-operator/reconciler/recorder/recorder_test.go b/k8s-operator/reconciler/recorder/recorder_test.go new file mode 100644 index 000000000..15b5ede59 --- /dev/null +++ b/k8s-operator/reconciler/recorder/recorder_test.go @@ -0,0 +1,803 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package recorder_test + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/reconcilertest" + "tailscale.com/k8s-operator/reconciler/recorder" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" + "tailscale.com/types/key" + "tailscale.com/util/clientmetric" + "tailscale.com/util/mak" + "tailscale.com/version" +) + +const ( + tsNamespace = "tailscale" + tsLoginServer = "example.tailscale.com" +) + +func TestRecorder(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer)) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + fr := record.NewFakeRecorder(2) + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: fr, + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(tsClient), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid", + "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible") + expectManagedCount(t, 0) + expectRecorderResources(t, fc, tsr, false) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"}) + + tsr.Spec.EnableUI = true + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ + "invalid space characters": "test", + } + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible"}) + + tsr.Spec.Storage.S3 = &tsapi.S3{} + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Only check part of this error message, because it's defined in an + // external package and may change. + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.RecorderReady, metav1.ConditionFalse, "RecorderInvalid") + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.RecorderReady) + for _, msg := range []string{cond.Message, <-fr.Events} { + if !strings.Contains(msg, `"invalid space characters"`) { + t.Fatalf("expected invalid annotation key in error message, got %q", msg) + } + } + }) + + t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) { + reconcilertest.MustCreate(t, fc, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pre-existing-sa", + Namespace: tsNamespace, + }, + }) + + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa" + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace` + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid", msg) + expectManagedCount(t, 0) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid " + msg}) + }) + + t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) { + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "" + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // The reason is part of the CRD's observable status, so assert it as a literal: a rename of the constant + // should fail here. + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionTrue, "RecorderCreated", "RecorderCreated") + expectManagedCount(t, 1) + expectRecorderResources(t, fc, tsr, true) + + // Every replica's auth key was minted with the default tag, and its + // StatefulSet is pointed at the login server of its tailnet. + if got := len(tsClient.CreateAuthKeyCalls()); got != int(*tsr.Spec.Replicas) { + t.Fatalf("expected %d auth keys to be created, got %d", *tsr.Spec.Replicas, got) + } + for _, req := range tsClient.CreateAuthKeyCalls() { + if diff := cmp.Diff(req.Capabilities.Devices.Create.Tags, []string{"tag:k8s"}); diff != "" { + t.Errorf("unexpected auth key tags (-got +want):\n%s", diff) + } + } + + ss := &appsv1.StatefulSet{} + reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, ss) + if got := envValue(ss, "TSRECORDER_LOGIN_SERVER"); got != tsLoginServer { + t.Errorf("unexpected login server env value: got %q, want %q", got, tsLoginServer) + } + }) + + t.Run("valid_service_account_config", func(t *testing.T) { + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa" + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ + "test": "test", + } + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + expectManagedCount(t, 1) + expectRecorderResources(t, fc, tsr, true) + + // Get the service account and check the annotations. + sa := &corev1.ServiceAccount{} + reconcilertest.MustGet(t, fc, tsNamespace, "test-sa", sa) + if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" { + t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff) + } + + // The ServiceAccount named after the Recorder is no longer referenced, + // so it should have been cleaned up. + reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name) + }) + + t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) { + for replica := range *tsr.Spec.Replicas { + seedStateSecret(t, fc, fmt.Sprintf("test-%d", replica), fmt.Sprintf("node-%d", replica), fmt.Sprintf("test-%d.example.ts.net", replica)) + } + + tsClient.SetDevices( + tailscaleclient.Device{ + ID: "node-0", + Hostname: "hostname-node-0", + Addresses: []string{"1.2.3.4", "::1"}, + }, + tailscaleclient.Device{ + ID: "node-1", + Hostname: "hostname-node-1", + Addresses: []string{"1.2.3.4", "::1"}, + }, + tailscaleclient.Device{ + ID: "node-2", + Hostname: "hostname-node-2", + Addresses: []string{"1.2.3.4", "::1"}, + }, + ) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + want := []tsapi.RecorderTailnetDevice{ + { + Hostname: "hostname-node-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-0.example.ts.net", + }, + { + Hostname: "hostname-node-1", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-1.example.ts.net", + }, + { + Hostname: "hostname-node-2", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-2.example.ts.net", + }, + } + + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + if diff := cmp.Diff(got.Status.Devices, want); diff != "" { + t.Fatalf("unexpected devices in status (-got +want):\n%s", diff) + } + }) + + t.Run("scale_down_deletes_dangling_replica_secrets_and_devices", func(t *testing.T) { + tsr.Spec.Replicas = new(int32(1)) + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Replicas 1 and 2 are gone from the cluster and from the tailnet; + // replica 0 is untouched. + for _, name := range []string{"test-1", "test-auth-1", "test-2", "test-auth-2"} { + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, name) + } + reconcilertest.MustGet(t, fc, tsNamespace, "test-0", &corev1.Secret{}) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } + + // The Role should no longer grant access to the scaled-down replicas' + // Secrets. + role := &rbacv1.Role{} + reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, role) + if diff := cmp.Diff(role.Rules[0].ResourceNames, []string{"test-0", "test-auth-0"}); diff != "" { + t.Fatalf("unexpected Role resource names (-got +want):\n%s", diff) + } + }) + + t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) { + reconcilertest.MustDeleteAll(t, fc, tsr) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name) + expectManagedCount(t, 0) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2", "node-0"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } + // The fake client does not clean up objects whose owner has been + // deleted, so we can't test for the owned resources getting deleted. + }) +} + +func TestRecorderTailnetUnavailable(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.RecorderSpec{EnableUI: true, Tailnet: "missing"}, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFailingClientProvider(errors.New(`tailnet "missing" not found`)), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + reconcilertest.ExpectCondition(t, got.Status.Conditions, tsapi.RecorderReady, + metav1.ConditionFalse, "RecorderTailnetUnavailable", `tailnet "missing" not found`) + + // No resources should have been created for a Recorder we can't mint keys + // for, and no finalizer should have been added. + reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, tsNamespace, tsr.Name) + if len(got.Finalizers) > 0 { + t.Fatalf("expected no finalizers, got %v", got.Finalizers) + } +} + +func TestRecorderCleanupContinuesPastUnreportedDevice(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + DeletionTimestamp: new(metav1.Now()), + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + Storage: tsapi.Storage{S3: &tsapi.S3{}}, + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects( + tsr, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-0", Namespace: tsNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: tsNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Namespace: tsNamespace}}, + ). + WithStatusSubresource(tsr). + Build() + tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer)) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(tsClient), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + // Replica 0 registered with control but never reported its device ID. It comes first, so it must not end cleanup + // early. + mk := key.NewMachine() + mkText, err := mk.MarshalText() + if err != nil { + t.Fatal(err) + } + reconcilertest.MustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) { + s.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText} + }) + seedStateSecret(t, fc, "test-1", "node-1", "test-1.example.ts.net") + seedStateSecret(t, fc, "test-2", "node-2", "test-2.example.ts.net") + tsClient.SetDevices( + tailscaleclient.Device{NodeID: "unreported-0", MachineKey: mk.Public().String()}, + tailscaleclient.Device{NodeID: "other", MachineKey: key.NewMachine().Public().String()}, + ) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"unreported-0", "node-1", "node-2"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } +} + +func TestRecorderStatefulSetSpec(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + EnableUI: true, + Storage: tsapi.Storage{ + S3: &tsapi.S3{ + Endpoint: "s3.example.com", + Bucket: "recordings", + Credentials: tsapi.S3Credentials{ + Secret: tsapi.S3Secret{Name: "s3-creds"}, + }, + }, + }, + StatefulSet: tsapi.RecorderStatefulSet{ + Labels: map[string]string{ + "ss-label-key": "ss-label-value", + }, + Annotations: map[string]string{ + "ss-annotation-key": "ss-annotation-value", + }, + Pod: tsapi.RecorderPod{ + Labels: map[string]string{ + "pod-label-key": "pod-label-value", + }, + Annotations: map[string]string{ + "pod-annotation-key": "pod-annotation-value", + }, + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "match-label": "match-value", + }, + }}, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: new(int64(1000)), + }, + ImagePullSecrets: []corev1.LocalObjectReference{{ + Name: "img-pull", + }}, + NodeSelector: map[string]string{ + "some-node": "selector", + }, + Tolerations: []corev1.Toleration{{ + Key: "key", + Value: "value", + TolerationSeconds: new(int64(60)), + }}, + ServiceAccount: tsapi.RecorderServiceAccount{ + Name: "custom-sa", + }, + Container: tsapi.RecorderContainer{ + Env: []tsapi.Env{{ + Name: "some_env", + Value: "env_value", + }}, + Image: "custom-image", + ImagePullPolicy: corev1.PullAlways, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + }, + }, + }, + }, + }, + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer))), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Every field that the spec passes through ends up on the Recorder's children. + expectRecorderResources(t, fc, tsr, true) +} + +// expectRecorderResources asserts that the full set of resources for each of tsr's replicas either exists or is +// absent. The test builds each expected child itself, without the reconciler's spec builders, so a change to what the +// reconciler writes fails here. +func expectRecorderResources(t *testing.T, fc client.Client, tsr *tsapi.Recorder, shouldExist bool) { + t.Helper() + + // The owner references carry the UID of the stored Recorder. + stored := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, stored) + stored.Spec = tsr.Spec + + role := wantRole(stored) + roleBinding := wantRoleBinding(stored) + serviceAccount := wantServiceAccount(stored) + statefulSet := wantStatefulSet(stored) + + if !shouldExist { + reconcilertest.ExpectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) + reconcilertest.ExpectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) + reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) + reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) + for replica := range replicaCount(stored) { + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, authSecretName(stored, replica)) + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, stateSecretName(stored, replica)) + } + return + } + + reconcilertest.ExpectEqual(t, fc, role) + reconcilertest.ExpectEqual(t, fc, roleBinding) + reconcilertest.ExpectEqual(t, fc, serviceAccount) + reconcilertest.ExpectEqual(t, fc, statefulSet) + + for replica := range replicaCount(stored) { + auth := &corev1.Secret{} + reconcilertest.MustGet(t, fc, tsNamespace, authSecretName(stored, replica), auth) + if auth.StringData["authkey"] == "" { + t.Errorf("auth Secret %q has no auth key", auth.Name) + } + + // The fake Tailscale client generates each auth key, so leave the key out of the comparison. + reconcilertest.ExpectEqual(t, fc, wantSecret(stored, authSecretName(stored, replica)), func(s *corev1.Secret) { + s.StringData = nil + }) + reconcilertest.ExpectEqual(t, fc, wantSecret(stored, stateSecretName(stored, replica))) + } +} + +func replicaCount(tsr *tsapi.Recorder) int32 { + if tsr.Spec.Replicas == nil { + return 1 + } + return *tsr.Spec.Replicas +} + +func authSecretName(tsr *tsapi.Recorder, replica int32) string { + return fmt.Sprintf("%s-auth-%d", tsr.Name, replica) +} + +func stateSecretName(tsr *tsapi.Recorder, replica int32) string { + return fmt.Sprintf("%s-%d", tsr.Name, replica) +} + +func wantServiceAccountName(tsr *tsapi.Recorder) string { + if name := tsr.Spec.StatefulSet.Pod.ServiceAccount.Name; name != "" { + return name + } + return tsr.Name +} + +func wantLabels(tsr *tsapi.Recorder, custom map[string]string) map[string]string { + labels := maps.Clone(custom) + mak.Set(&labels, "app.kubernetes.io/name", "recorder") + mak.Set(&labels, "app.kubernetes.io/instance", tsr.Name) + mak.Set(&labels, "app.kubernetes.io/managed-by", "tailscale-operator") + return labels +} + +func wantObjectMeta(tsr *tsapi.Recorder, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, nil), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "tailscale.com/v1alpha1", + Kind: "Recorder", + Name: tsr.Name, + UID: tsr.UID, + Controller: new(true), + BlockOwnerDeletion: new(true), + }}, + } +} + +func wantSecret(tsr *tsapi.Recorder, name string) *corev1.Secret { + return &corev1.Secret{ObjectMeta: wantObjectMeta(tsr, name)} +} + +func wantServiceAccount(tsr *tsapi.Recorder) *corev1.ServiceAccount { + sa := &corev1.ServiceAccount{ObjectMeta: wantObjectMeta(tsr, wantServiceAccountName(tsr))} + sa.Annotations = tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations + return sa +} + +func wantRole(tsr *tsapi.Recorder) *rbacv1.Role { + var secrets []string + for replica := range replicaCount(tsr) { + secrets = append(secrets, stateSecretName(tsr, replica), authSecretName(tsr, replica)) + } + + return &rbacv1.Role{ + ObjectMeta: wantObjectMeta(tsr, tsr.Name), + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: secrets, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"get", "create", "patch"}, + }, + }, + } +} + +func wantRoleBinding(tsr *tsapi.Recorder) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: wantObjectMeta(tsr, tsr.Name), + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: wantServiceAccountName(tsr), + Namespace: tsNamespace, + }}, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: tsr.Name, + }, + } +} + +func wantStatefulSet(tsr *tsapi.Recorder) *appsv1.StatefulSet { + pod := tsr.Spec.StatefulSet.Pod + + image := pod.Container.Image + if image == "" { + meta := version.GetMeta() + prefix := "" + if meta.UnstableBranch { + prefix = "unstable-" + } + image = fmt.Sprintf("tailscale/tsrecorder:%sv%s", prefix, meta.MajorMinorPatch) + } + + env := []corev1.EnvVar{ + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, + {Name: "TS_AUTHKEY_FILE", Value: "/etc/tailscaled/$(POD_NAME)/authkey"}, + {Name: "TS_STATE", Value: "kube:$(POD_NAME)"}, + {Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)"}, + {Name: "TSRECORDER_LOGIN_SERVER", Value: tsLoginServer}, + } + for _, e := range pod.Container.Env { + env = append(env, corev1.EnvVar{Name: string(e.Name), Value: e.Value}) + } + + var envFrom []corev1.EnvFromSource + if s3 := tsr.Spec.Storage.S3; s3 != nil { + env = append(env, + corev1.EnvVar{Name: "TSRECORDER_DST", Value: "s3://" + s3.Endpoint}, + corev1.EnvVar{Name: "TSRECORDER_BUCKET", Value: s3.Bucket}, + ) + if s3.Credentials.Secret.Name != "" { + envFrom = []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: s3.Credentials.Secret.Name}, + }, + }} + } + } else { + env = append(env, corev1.EnvVar{Name: "TSRECORDER_DST", Value: "/data/recordings"}) + } + if tsr.Spec.EnableUI { + env = append(env, corev1.EnvVar{Name: "TSRECORDER_UI", Value: "true"}) + } + + mounts := []corev1.VolumeMount{{Name: "data", MountPath: "/data"}} + volumes := []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + for replica := range replicaCount(tsr) { + name := fmt.Sprintf("authkey-%d", replica) + mounts = append(mounts, corev1.VolumeMount{ + Name: name, + ReadOnly: true, + MountPath: "/etc/tailscaled/" + stateSecretName(tsr, replica), + }) + volumes = append(volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: authSecretName(tsr, replica), + Items: []corev1.KeyToPath{{Key: "authkey", Path: "authkey"}}, + }, + }, + }) + } + + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: tsr.Name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, tsr.Spec.StatefulSet.Labels), + Annotations: tsr.Spec.StatefulSet.Annotations, + OwnerReferences: wantObjectMeta(tsr, tsr.Name).OwnerReferences, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: new(replicaCount(tsr)), + Selector: &metav1.LabelSelector{MatchLabels: wantLabels(tsr, pod.Labels)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: tsr.Name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, pod.Labels), + Annotations: pod.Annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: wantServiceAccountName(tsr), + Affinity: pod.Affinity, + SecurityContext: pod.SecurityContext, + ImagePullSecrets: pod.ImagePullSecrets, + NodeSelector: pod.NodeSelector, + Tolerations: pod.Tolerations, + Containers: []corev1.Container{{ + Name: "recorder", + Image: image, + ImagePullPolicy: pod.Container.ImagePullPolicy, + Resources: pod.Container.Resources, + SecurityContext: pod.Container.SecurityContext, + Env: env, + EnvFrom: envFrom, + Command: []string{"/tsrecorder"}, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } +} + +// expectReadyCondition asserts the Recorder's RecorderReady condition. +func expectReadyCondition(t *testing.T, fc client.Client, name string, status metav1.ConditionStatus, reason, message string) { + t.Helper() + tsr := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", name, tsr) + reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, status, reason, message) +} + +// expectManagedCount asserts the number of Recorders the reconciler reports as managed via its clientmetric gauge. +func expectManagedCount(t *testing.T, want int64) { + t.Helper() + for _, m := range clientmetric.Metrics() { + if m.Name() == kubetypes.MetricRecorderCount { + if got := m.Value(); got != want { + t.Fatalf("expected %d recorders, got %d", want, got) + } + return + } + } + t.Fatalf("metric %q not found", kubetypes.MetricRecorderCount) +} + +// seedStateSecret writes the profile data that tailscaled would have written to a replica's state Secret once it has +// authenticated. +func seedStateSecret(t *testing.T, fc client.Client, name, nodeID, loginName string) { + t.Helper() + const key = "profile-abc" + + body, err := json.Marshal(map[string]any{ + "Config": map[string]any{ + "NodeID": nodeID, + "UserProfile": map[string]any{ + "LoginName": loginName, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + reconcilertest.MustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + string(ipn.CurrentProfileStateKey): []byte(key), + key: body, + } + }) +} + +func envValue(ss *appsv1.StatefulSet, name string) string { + for _, env := range ss.Spec.Template.Spec.Containers[0].Env { + if env.Name == name { + return env.Value + } + } + return "" +} diff --git a/cmd/k8s-operator/tsrecorder_specs.go b/k8s-operator/reconciler/recorder/specs.go similarity index 91% rename from cmd/k8s-operator/tsrecorder_specs.go rename to k8s-operator/reconciler/recorder/specs.go index 73ad5f431..d92e7e4e0 100644 --- a/cmd/k8s-operator/tsrecorder_specs.go +++ b/k8s-operator/reconciler/recorder/specs.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package recorder import ( "fmt" @@ -115,7 +115,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) * Name: volumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + SecretName: authSecretName(tsr.Name, replica), Items: []corev1.KeyToPath{{Key: authKeySecretKey, Path: authKeySecretKey}}, }, }, @@ -156,8 +156,8 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role { resourceNames := make([]string, 0) for replica := range replicas { resourceNames = append(resourceNames, - fmt.Sprintf("%s-%d", tsr.Name, replica), // State secret. - fmt.Sprintf("%s-auth-%d", tsr.Name, replica), // Auth key secret. + stateSecretName(tsr.Name, replica), // State secret. + authSecretName(tsr.Name, replica), // Auth key secret. ) } @@ -218,7 +218,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + Name: authSecretName(tsr.Name, replica), Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, @@ -231,7 +231,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", tsr.Name, replica), + Name: stateSecretName(tsr.Name, replica), Namespace: namespace, Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), @@ -239,6 +239,17 @@ func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev } } +// stateSecretName returns the name of the tailscaled state Secret for the given Recorder replica. It matches the +// replica's Pod name, which is what tsrecorder reads from TS_STATE. +func stateSecretName(tsrName string, replica int32) string { + return fmt.Sprintf("%s-%d", tsrName, replica) +} + +// authSecretName returns the name of the Secret holding the auth key for the given Recorder replica. +func authSecretName(tsrName string, replica int32) string { + return fmt.Sprintf("%s-auth-%d", tsrName, replica) +} + func tsrEnv(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar { envs := []corev1.EnvVar{ { diff --git a/k8s-operator/reconciler/tailnet/tailnet_test.go b/k8s-operator/reconciler/tailnet/tailnet_test.go index 36a7ea774..23e5445ef 100644 --- a/k8s-operator/reconciler/tailnet/tailnet_test.go +++ b/k8s-operator/reconciler/tailnet/tailnet_test.go @@ -13,10 +13,10 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/tsclient" "tailscale.com/tstest" @@ -374,7 +374,7 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) + builder := reconcilertest.NewClientBuilder() if tc.Tailnet != nil { builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet) } diff --git a/k8s-operator/reconciler/tailscaled/prefs.go b/k8s-operator/reconciler/tailscaled/prefs.go new file mode 100644 index 000000000..0ed72327f --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/prefs.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + + "tailscale.com/ipn" + "tailscale.com/tailcfg" +) + +// Prefs is the subset of ipn.Prefs that operator reconcilers need to read back out of a tailscaled state Secret. +// tailscaled writes the full prefs for the current profile; only the fields declared here are unmarshalled. +type Prefs struct { + Config struct { + NodeID tailcfg.StableNodeID `json:"NodeID"` + UserProfile struct { + // LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net. + LoginName string `json:"LoginName"` + } `json:"UserProfile"` + } `json:"Config"` + + AdvertiseServices []string `json:"AdvertiseServices"` +} + +// PrefsFromStateSecret returns the prefs tailscaled recorded for its current profile in secret. ok reports whether +// a node ID was found; it is false when the device hasn't finished authenticating yet, in which case prefs is not +// meaningful. LoginName is expected to be set whenever the node ID is, but that is not required. +// +// TODO(tomhjp): Should maybe use ipn to parse the following info instead. +func PrefsFromStateSecret(secret *corev1.Secret) (prefs Prefs, ok bool, err error) { + currentProfile, ok := secret.Data[string(ipn.CurrentProfileStateKey)] + if !ok { + return prefs, false, nil + } + profileBytes, ok := secret.Data[string(currentProfile)] + if !ok { + return prefs, false, nil + } + if err := json.Unmarshal(profileBytes, &prefs); err != nil { + return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err) + } + + ok = prefs.Config.NodeID != "" + return prefs, ok, nil +} diff --git a/k8s-operator/reconciler/tailscaled/prefs_test.go b/k8s-operator/reconciler/tailscaled/prefs_test.go new file mode 100644 index 000000000..bd713acd5 --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/prefs_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "tailscale.com/k8s-operator/reconciler/tailscaled" +) + +func TestPrefsFromStateSecret(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + data map[string][]byte + wantOK bool + wantErr bool + wantNodeID string + wantLogin string + wantSvcs []string + }{ + "populated": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`{"Config":{"NodeID":"node-foo","UserProfile":{"LoginName":"foo.example.ts.net"}},"AdvertiseServices":["svc:bar"]}`), + }, + wantOK: true, + wantNodeID: "node-foo", + wantLogin: "foo.example.ts.net", + wantSvcs: []string{"svc:bar"}, + }, + "no-current-profile": { + data: map[string][]byte{ + "profile-abc": []byte(`{"Config":{"NodeID":"node-foo"}}`), + }, + }, + "current-profile-points-at-missing-profile": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + }, + }, + // The device hasn't finished authenticating, so there's no node ID yet. + "no-node-id": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`{"Config":{"UserProfile":{"LoginName":"foo.example.ts.net"}}}`), + }, + wantLogin: "foo.example.ts.net", + }, + "empty": {}, + "malformed-profile": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`not json`), + }, + wantErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "tailscale"}, + Data: tc.data, + } + + prefs, ok, err := tailscaled.PrefsFromStateSecret(secret) + switch { + case tc.wantErr && err == nil: + t.Fatal("expected an error, got nil") + case !tc.wantErr && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.wantErr: + return + } + + if ok != tc.wantOK { + t.Errorf("ok = %v, want %v", ok, tc.wantOK) + } + if got := string(prefs.Config.NodeID); got != tc.wantNodeID { + t.Errorf("NodeID = %q, want %q", got, tc.wantNodeID) + } + if got := prefs.Config.UserProfile.LoginName; got != tc.wantLogin { + t.Errorf("LoginName = %q, want %q", got, tc.wantLogin) + } + if got, want := len(prefs.AdvertiseServices), len(tc.wantSvcs); got != want { + t.Fatalf("len(AdvertiseServices) = %d, want %d", got, want) + } + for i, svc := range tc.wantSvcs { + if prefs.AdvertiseServices[i] != svc { + t.Errorf("AdvertiseServices[%d] = %q, want %q", i, prefs.AdvertiseServices[i], svc) + } + } + }) + } +} diff --git a/k8s-operator/reconciler/tailscaled/reissuer_test.go b/k8s-operator/reconciler/tailscaled/reissuer_test.go index aebd34d38..ead8fe796 100644 --- a/k8s-operator/reconciler/tailscaled/reissuer_test.go +++ b/k8s-operator/reconciler/tailscaled/reissuer_test.go @@ -6,18 +6,15 @@ package tailscaled_test import ( - "context" "slices" "strings" - "sync" "testing" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" - tailscaleclient "tailscale.com/client/tailscale/v2" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/k8s-operator/reconciler/tailscaled" - "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" ) @@ -72,7 +69,7 @@ func TestReissuer_ShouldReissue(t *testing.T) { stateSecret = &corev1.Secret{Data: tc.stateData} } - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -89,8 +86,8 @@ func TestReissuer_ShouldReissue(t *testing.T) { if got != tc.wantReissue { t.Errorf("shouldReissue = %v, want %v", got, tc.wantReissue) } - if !slices.Equal(tsc.deleted, tc.wantDeleted) { - t.Errorf("deleted devices = %v, want %v", tsc.deleted, tc.wantDeleted) + if !slices.Equal(tsc.DeviceDeletes(), tc.wantDeleted) { + t.Errorf("deleted devices = %v, want %v", tsc.DeviceDeletes(), tc.wantDeleted) } }) } @@ -101,7 +98,7 @@ func TestReissuer_ShouldReissue(t *testing.T) { // state Secret. func TestReissuer_InFlight(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -137,7 +134,7 @@ func TestReissuer_InFlight(t *testing.T) { // across replicas, blocks re-issuance once its burst is exhausted. func TestReissuer_RateLimit(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() // Burst of 1: the first replica's reissue consumes the only token. r.EnsureState("pg", 1) @@ -169,7 +166,7 @@ func TestReissuer_RateLimit(t *testing.T) { // EnsureState reissues instead of panicking on a nil limiter. func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() broken := &corev1.Secret{Data: map[string][]byte{ @@ -183,8 +180,8 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { if err != nil || !got { t.Fatalf("got %v, err %v; want true, nil", got, err) } - if !slices.Equal(tsc.deleted, []string{"nodeid-0"}) { - t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.deleted) + if !slices.Equal(tsc.DeviceDeletes(), []string{"nodeid-0"}) { + t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.DeviceDeletes()) } } @@ -192,7 +189,7 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { // is cleared, so a later reissue request for the same replica triggers afresh. func TestReissuer_RemoveState(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -226,7 +223,7 @@ func TestReissuer_RemoveState(t *testing.T) { // parent scaled down then deleted doesn't strand in-flight entries. func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 2) @@ -260,30 +257,3 @@ func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) { t.Fatalf("pg-1 after RemoveState: got %v, err %v; want true, nil", got, err) } } - -type fakeReissueClient struct { - tsclient.Client - - mu sync.Mutex - deleted []string -} - -func (c *fakeReissueClient) Devices() tsclient.DeviceResource { return (*fakeReissueDevices)(c) } - -type fakeReissueDevices fakeReissueClient - -func (d *fakeReissueDevices) Delete(_ context.Context, id string) error { - c := (*fakeReissueClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - c.deleted = append(c.deleted, id) - return nil -} - -func (d *fakeReissueDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { - return nil, nil -} - -func (d *fakeReissueDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) { - return nil, nil -}