diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 08bb405b9..94ecac7be 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -12,7 +12,6 @@ import ( "fmt" "maps" "slices" - "strings" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -27,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" @@ -85,7 +85,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") - if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) { + if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return res, nil } @@ -95,7 +95,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient) if err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go index b7c35f437..a7ace3da7 100644 --- a/cmd/k8s-operator/api-server-proxy-pg_test.go +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -20,6 +20,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" @@ -199,7 +200,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { // Simulate Pod prefs updated with advertised services; should see Configured condition updated to true. mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) } diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index bfe751fbb..41e21dad0 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -11,7 +11,6 @@ import ( "fmt" "net/netip" "slices" - "strings" "sync" "time" @@ -28,6 +27,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" @@ -146,7 +146,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil { reason := reasonConnectorCreationFailed message := fmt.Sprintf(messageConnectorCreationFailed, err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { reason = reasonConnectorCreating message = fmt.Sprintf("optimistic lock error, retrying: %s", err) err = nil diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 554dc4ec8..3a3c960d3 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -774,6 +774,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/recorder from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/staticendpoints from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailscaled from tailscale.com/k8s-operator/reconciler/peerrelay+ diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 6f0480a94..7ec758fd7 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -34,6 +34,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" @@ -161,7 +162,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re } if err := esr.maybeProvision(ctx, svc, lg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { lg.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 9b8c70380..545f90d88 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -34,6 +34,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -851,7 +852,7 @@ func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, p var count int for _, secret := range secrets.Items { - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return 0, fmt.Errorf("error getting node metadata: %w", err) } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 8312dc5f7..38c2e7f93 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -30,6 +30,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -563,7 +564,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -689,7 +690,7 @@ func TestIngressPGReconciler_HTTPRedirect(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -824,7 +825,7 @@ func TestIngressPGReconciler_HTTPEndpointAndRedirectConflict(t *testing.T) { // Add the Tailscale Service to prefs to have the Ingress recognised as ready. mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { - var p prefs + var p tailscaled.Prefs var err error if err = json.Unmarshal(o.Data["test"], &p); err != nil { t.Errorf("failed to unmarshal preferences: %v", err) @@ -1151,7 +1152,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { } mustCreate(t, fc, pgCfgSecret) - pr := prefs{} + pr := tailscaled.Prefs{} pr.Config.UserProfile.LoginName = "test.ts.net" pr.Config.NodeID = "test" @@ -1168,8 +1169,8 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), }, Data: map[string][]byte{ - currentProfileKey: []byte("test"), - "test": p, + string(ipn.CurrentProfileStateKey): []byte("test"), + "test": p, }, } mustCreate(t, fc, pgStateSecret) diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index cf2f37b06..a09524d59 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/ipn" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" "tailscale.com/util/clientmetric" @@ -82,7 +83,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request } if err := a.maybeProvision(ctx, logger, ing); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 430ac3015..10420331f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -58,6 +58,7 @@ import ( "tailscale.com/k8s-operator/reconciler/peerrelay" "tailscale.com/k8s-operator/reconciler/proxyclass" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" + "tailscale.com/k8s-operator/reconciler/recorder" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" @@ -674,26 +675,16 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create DNS records reconciler: %v", err) } - // Recorder reconciler. - recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{}) - err = builder.ControllerManagedBy(mgr). - For(&tsapi.Recorder{}). - Named("recorder-reconciler"). - Watches(&appsv1.StatefulSet{}, recorderFilter). - Watches(&corev1.ServiceAccount{}, recorderFilter). - Watches(&corev1.Secret{}, recorderFilter). - Watches(&rbacv1.Role{}, recorderFilter). - Watches(&rbacv1.RoleBinding{}, recorderFilter). - Complete(&RecorderReconciler{ - recorder: eventRecorder, - tsNamespace: opts.tailscaleNamespace, - Client: mgr.GetClient(), - log: opts.log.Named("recorder-reconciler"), - clock: tstime.DefaultClock{}, - clients: clients, - reissuer: tailscaled.NewReissuer(), - }) - if err != nil { + recorderOptions := recorder.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Clients: clients, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + } + + if err = recorder.NewReconciler(recorderOptions).Register(mgr); err != nil { startlog.Fatalf("could not create Recorder reconciler: %v", err) } diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index b8455e4b2..cf1c25e9f 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -34,6 +34,7 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/staticendpoints" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" @@ -57,9 +58,6 @@ const ( reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable" reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion" - // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" - // The minimum tailcfg.CapabilityVersion that deployed clients are expected // to support to be compatible with the current ProxyGroup controller. // If the controller needs to depend on newer client behaviour, it should @@ -150,7 +148,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } @@ -475,7 +473,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za defer func() { if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(updateErr) { logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) updateErr = nil } @@ -992,7 +990,7 @@ func getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup, cl client.Client stateSecret: &secret, } - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return nil, err } @@ -1101,7 +1099,7 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool { func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) logger.Info(msg) return notReady(reasonProxyGroupCreating, msg) diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 986b6ad5c..21a50c36b 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -2296,10 +2296,10 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG } mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) { s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, - kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), - kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i), + string(ipn.CurrentProfileStateKey): []byte(key), + key: bytes, + kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), + kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i), // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. // Consolidate on this one. kubetypes.KeyDeviceID: fmt.Appendf(nil, "nodeid-%d", i), diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 68414d1d8..2e5680cdc 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -1524,6 +1524,12 @@ func nameForService(svc *corev1.Service) string { return svc.Namespace + "-" + svc.Name } +// markedForDeletion reports whether obj has a deletion timestamp, i.e. the API server is waiting on finalizers +// before it can garbage collect it. +func markedForDeletion(obj metav1.Object) bool { + return !obj.GetDeletionTimestamp().IsZero() +} + // proxyClassForObject returns the proxy class for the given object. If the // object does not have a proxy class label, it returns the default proxy class func proxyClassForObject(o client.Object, proxyDefaultClass string) string { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index daa983b78..61ff57cf8 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -31,6 +31,8 @@ import ( "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" @@ -188,7 +190,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque needsRequeue := false needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient) if err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err @@ -762,7 +764,7 @@ func (r *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName var count int for _, secret := range secrets.Items { - prefs, ok, err := getDevicePrefs(&secret) + prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return 0, fmt.Errorf("error getting node metadata: %w", err) } diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index eb39d0029..daa5edf9c 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -26,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/net/dns/resolvconffile" "tailscale.com/tstime" @@ -123,7 +124,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request } if err := a.maybeProvision(ctx, logger, svc); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index c4c322904..11c4565a2 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -45,6 +45,9 @@ import ( const ( vipTestIP = "5.6.7.8" + + // tsNamespace is the namespace the operator is installed in for tests that don't override it. + tsNamespace = "tailscale" ) // confgOpts contains configuration options for creating cluster resources for diff --git a/cmd/k8s-operator/tsrecorder_specs_test.go b/cmd/k8s-operator/tsrecorder_specs_test.go deleted file mode 100644 index 13da8a3c8..000000000 --- a/cmd/k8s-operator/tsrecorder_specs_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package main - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" -) - -func TestRecorderSpecs(t *testing.T) { - t.Run("spec-fields-passthrough", func(t *testing.T) { - tsr := &tsapi.Recorder{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: tsapi.RecorderSpec{ - Replicas: new(int32(3)), - StatefulSet: tsapi.RecorderStatefulSet{ - Labels: map[string]string{ - "ss-label-key": "ss-label-value", - }, - Annotations: map[string]string{ - "ss-annotation-key": "ss-annotation-value", - }, - Pod: tsapi.RecorderPod{ - Labels: map[string]string{ - "pod-label-key": "pod-label-value", - }, - Annotations: map[string]string{ - "pod-annotation-key": "pod-annotation-value", - }, - Affinity: &corev1.Affinity{ - PodAffinity: &corev1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "match-label": "match-value", - }, - }}, - }, - }, - }, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: new(int64(1000)), - }, - ImagePullSecrets: []corev1.LocalObjectReference{{ - Name: "img-pull", - }}, - NodeSelector: map[string]string{ - "some-node": "selector", - }, - Tolerations: []corev1.Toleration{{ - Key: "key", - Value: "value", - TolerationSeconds: new(int64(60)), - }}, - Container: tsapi.RecorderContainer{ - Env: []tsapi.Env{{ - Name: "some_env", - Value: "env_value", - }}, - Image: "custom-image", - ImagePullPolicy: corev1.PullAlways, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - }, - }, - }, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - }, - }, - }, - }, - }, - }, - } - - ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) - - // StatefulSet-level. - if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - // Pod-level. - if diff := cmp.Diff(ss.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - // Container-level. - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, tsrEnv(tsr, tsLoginServer)); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" { - t.Errorf("(-got +want):\n%s", diff) - } - - if *ss.Spec.Replicas != *tsr.Spec.Replicas { - t.Errorf("expected %d replicas, got %d", *tsr.Spec.Replicas, *ss.Spec.Replicas) - } - - if len(ss.Spec.Template.Spec.Volumes) != int(*tsr.Spec.Replicas)+1 { - t.Errorf("expected %d volumes, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Volumes)) - } - - if len(ss.Spec.Template.Spec.Containers[0].VolumeMounts) != int(*tsr.Spec.Replicas)+1 { - t.Errorf("expected %d volume mounts, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Containers[0].VolumeMounts)) - } - }) -} diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go deleted file mode 100644 index 44e199bc7..000000000 --- a/cmd/k8s-operator/tsrecorder_test.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "go.uber.org/zap" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "tailscale.com/client/tailscale/v2" - - tsoperator "tailscale.com/k8s-operator" - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/k8s-operator/reconciler/tailscaled" - "tailscale.com/k8s-operator/tsclient" - "tailscale.com/tstest" -) - -const ( - tsNamespace = "tailscale" - tsLoginServer = "example.tailscale.com" -) - -func TestRecorder(t *testing.T) { - tsr := &tsapi.Recorder{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Finalizers: []string{"tailscale.com/finalizer"}, - }, - Spec: tsapi.RecorderSpec{ - Replicas: new(int32(3)), - }, - } - - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). - WithObjects(tsr). - WithStatusSubresource(tsr). - Build() - tsClient := &fakeTSClient{loginURL: tsLoginServer} - zl, _ := zap.NewDevelopment() - fr := record.NewFakeRecorder(2) - cl := tstest.NewClock(tstest.ClockOpts{}) - reconciler := &RecorderReconciler{ - tsNamespace: tsNamespace, - Client: fc, - clients: tsclient.NewProvider(tsClient), - recorder: fr, - log: zl.Sugar(), - clock: cl, - reissuer: tailscaled.NewReissuer(), - } - - t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { - expectReconciled(t, reconciler, "", tsr.Name) - - msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible" - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, false) - - expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible" - expectEvents(t, fr, []string{expectedEvent}) - - tsr.Spec.EnableUI = true - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ - "invalid space characters": "test", - } - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - expectReconciled(t, reconciler, "", tsr.Name) - - expectedEvent = "Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible" - expectEvents(t, fr, []string{expectedEvent}) - - tsr.Spec.Storage.S3 = &tsapi.S3{} - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - expectReconciled(t, reconciler, "", tsr.Name) - - // Only check part of this error message, because it's defined in an - // external package and may change. - if err := fc.Get(context.Background(), client.ObjectKey{ - Name: tsr.Name, - }, tsr); err != nil { - t.Fatal(err) - } - if len(tsr.Status.Conditions) != 1 { - t.Fatalf("expected 1 condition, got %d", len(tsr.Status.Conditions)) - } - cond := tsr.Status.Conditions[0] - if cond.Type != string(tsapi.RecorderReady) || cond.Status != metav1.ConditionFalse || cond.Reason != reasonRecorderInvalid { - t.Fatalf("expected condition RecorderReady false due to RecorderInvalid, got %v", cond) - } - for _, msg := range []string{cond.Message, <-fr.Events} { - if !strings.Contains(msg, `"invalid space characters"`) { - t.Fatalf("expected invalid annotation key in error message, got %q", cond.Message) - } - } - }) - - t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) { - mustCreate(t, fc, &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pre-existing-sa", - Namespace: tsNamespace, - }, - }) - - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa" - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace` - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - - expectedEvent := "Warning RecorderInvalid " + msg - expectEvents(t, fr, []string{expectedEvent}) - }) - - t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) { - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "" - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar()) - expectEqual(t, fc, tsr) - if expected := 1; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, true) - }) - - t.Run("valid_service_account_config", func(t *testing.T) { - tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa" - tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ - "test": "test", - } - mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { - t.Spec = tsr.Spec - }) - - expectReconciled(t, reconciler, "", tsr.Name) - - expectEqual(t, fc, tsr) - if expected := 1; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - expectRecorderResources(t, fc, tsr, true) - - // Get the service account and check the annotations. - sa := &corev1.ServiceAccount{} - if err := fc.Get(context.Background(), client.ObjectKey{ - Name: tsr.Spec.StatefulSet.Pod.ServiceAccount.Name, - Namespace: tsNamespace, - }, sa); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" { - t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff) - } - if sa.Name != tsr.Spec.StatefulSet.Pod.ServiceAccount.Name { - t.Fatalf("unexpected service account name: got %q, want %q", sa.Name, tsr.Spec.StatefulSet.Pod.ServiceAccount.Name) - } - - expectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name) - }) - - t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) { - const key = "profile-abc" - - for replica := range *tsr.Spec.Replicas { - bytes, err := json.Marshal(map[string]any{ - "Config": map[string]any{ - "NodeID": fmt.Sprintf("node-%d", replica), - "UserProfile": map[string]any{ - "LoginName": fmt.Sprintf("test-%d.example.ts.net", replica), - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - name := fmt.Sprintf("%s-%d", "test", replica) - mustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) { - s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, - } - }) - } - - tsClient.devices = []tailscale.Device{ - { - ID: "node-0", - Hostname: "hostname-node-0", - Addresses: []string{"1.2.3.4", "::1"}, - }, - { - ID: "node-1", - Hostname: "hostname-node-1", - Addresses: []string{"1.2.3.4", "::1"}, - }, - { - ID: "node-2", - Hostname: "hostname-node-2", - Addresses: []string{"1.2.3.4", "::1"}, - }, - } - - expectReconciled(t, reconciler, "", tsr.Name) - tsr.Status.Devices = []tsapi.RecorderTailnetDevice{ - { - Hostname: "hostname-node-0", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-0.example.ts.net", - }, - { - Hostname: "hostname-node-1", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-1.example.ts.net", - }, - { - Hostname: "hostname-node-2", - TailnetIPs: []string{"1.2.3.4", "::1"}, - URL: "https://test-2.example.ts.net", - }, - } - expectEqual(t, fc, tsr) - }) - - t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) { - // Replica 0 comes first, so an unreported device there must not end cleanup early. - setUnreportedDevice(t, fc, tsClient, "test-0", "unreported-0") - if err := fc.Delete(context.Background(), tsr); err != nil { - t.Fatal(err) - } - - expectReconciled(t, reconciler, "", tsr.Name) - - expectMissing[tsapi.Recorder](t, fc, "", tsr.Name) - if expected := 0; reconciler.recorders.Len() != expected { - t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) - } - if diff := cmp.Diff(tsClient.deleted, []string{"unreported-0", "node-1", "node-2"}); diff != "" { - t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) - } - // The fake client does not clean up objects whose owner has been - // deleted, so we can't test for the owned resources getting deleted. - }) -} - -func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) { - t.Helper() - - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - role := tsrRole(tsr, tsNamespace) - roleBinding := tsrRoleBinding(tsr, tsNamespace) - serviceAccount := tsrServiceAccount(tsr, tsNamespace) - statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) - - if shouldExist { - expectEqual(t, fc, role) - expectEqual(t, fc, roleBinding) - expectEqual(t, fc, serviceAccount) - expectEqual(t, fc, statefulSet, removeResourceReqs) - } else { - expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) - expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) - expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) - expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) - } - - for replica := range replicas { - auth := tsrAuthSecret(tsr, tsNamespace, "new-authkey", replica) - state := tsrStateSecret(tsr, tsNamespace, replica) - - if shouldExist { - expectEqual(t, fc, auth) - expectEqual(t, fc, state) - } else { - expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name) - expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name) - } - } -} diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index 2e0d0e1d4..30daf87b9 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -46,8 +46,7 @@ const ( serviceTypeIngress = "ingress" serviceTypeSvc = "svc" - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" - shortRequeue = 5 * time.Second + shortRequeue = 5 * time.Second // AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN. AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" @@ -157,7 +156,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res } if err := r.maybeProvision(ctx, proxySvc, logger); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go index a5d83b19c..43f7f0256 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go @@ -21,7 +21,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -30,6 +29,7 @@ import ( "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/dnsrecords" "tailscale.com/k8s-operator/reconciler/nameserver" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" ) @@ -70,8 +70,7 @@ func TestDNSRecordsReconciler(t *testing.T) { }, } cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}} - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(cm). WithObjects(dnsConfig). WithObjects(ing). @@ -83,7 +82,7 @@ func TestDNSRecordsReconciler(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) // Set the ready condition of the DNSConfig - mustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { + reconcilertest.MustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar()) }) dnsRR := dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{ @@ -109,11 +108,11 @@ func TestDNSRecordsReconciler(t *testing.T) { ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4) epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6) - mustCreate(t, fc, egressSvcFQDN) - mustCreate(t, fc, headlessForEgressSvcFQDN) - mustCreate(t, fc, ep) - mustCreate(t, fc, epv6) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.MustCreate(t, fc, egressSvcFQDN) + reconcilertest.MustCreate(t, fc, headlessForEgressSvcFQDN) + reconcilertest.MustCreate(t, fc, ep) + reconcilertest.MustCreate(t, fc, epv6) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service // ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7 wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} wantHostsIPv6 := map[string][]string{"foo.bar.ts.net": {"2600:1900:4011:161:0:d:0:d"}} @@ -121,57 +120,57 @@ func TestDNSRecordsReconciler(t *testing.T) { // 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's // value changes - mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) { svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net" }) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}} expectHostsRecords(t, fc, wantHosts) // 3. DNS record is updated if the IP address of the proxy Pod changes. ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4) - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Addresses = []string{"10.6.5.4"} }) - expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}} expectHostsRecords(t, fc, wantHosts) // 4. DNS record is created for an ingress proxy configured via Ingress headlessForIngress := headlessSvcForParent(ing, "ingress") ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4) - mustCreate(t, fc, headlessForIngress) - mustCreate(t, fc, ep) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service + reconcilertest.MustCreate(t, fc, headlessForIngress) + reconcilertest.MustCreate(t, fc, ep) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"} expectHostsRecords(t, fc, wantHosts) // 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0]) t.Log("test case 5") - mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) { + reconcilertest.MustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) { ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net" }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service delete(wantHosts, "cluster.ingress.ts.net") wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"} expectHostsRecords(t, fc, wantHosts) // 6. DNS records are updated if Ingress proxy's Pod IP changes - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Addresses = []string{"7.8.9.10"} }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"} expectHostsRecords(t, fc, wantHosts) // 7. A not-ready Endpoint is removed from DNS config. - mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { ep.Endpoints[0].Conditions.Ready = new(false) ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"1.2.3.4"}, }) }) - expectReconciled(t, dnsRR, "tailscale", "ts-ingress") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"} expectHostsRecords(t, fc, wantHosts) @@ -192,7 +191,7 @@ func TestDNSRecordsReconciler(t *testing.T) { ExternalName: "unused", }, } - mustCreate(t, fc, parentEgressSvc) + reconcilertest.MustCreate(t, fc, parentEgressSvc) proxyGroupEgressSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -246,9 +245,9 @@ func TestDNSRecordsReconciler(t *testing.T) { }}, } - mustCreate(t, fc, proxyGroupEgressSvc) - mustCreate(t, fc, proxyGroupEps) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.MustCreate(t, fc, proxyGroupEgressSvc) + reconcilertest.MustCreate(t, fc, proxyGroupEps) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") // Verify DNS record uses ClusterIP Service IP, not Pod IPs wantHosts["external-service.example.ts.net"] = []string{"10.0.100.50"} @@ -256,17 +255,17 @@ func TestDNSRecordsReconciler(t *testing.T) { // 9. ProxyGroup egress DNS record updates when ClusterIP changes t.Log("test case 9: ProxyGroup egress ClusterIP change") - mustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) { svc.Spec.ClusterIP = "10.0.100.51" }) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") wantHosts["external-service.example.ts.net"] = []string{"10.0.100.51"} expectHostsRecords(t, fc, wantHosts) // 10. Test ProxyGroup service deletion and DNS cleanup t.Log("test case 10: ProxyGroup egress service deletion") - mustDeleteAll(t, fc, proxyGroupEgressSvc) - expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + reconcilertest.MustDeleteAll(t, fc, proxyGroupEgressSvc) + reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") delete(wantHosts, "external-service.example.ts.net") expectHostsRecords(t, fc, wantHosts) } @@ -322,9 +321,8 @@ func TestDNSRecordsReconcilerOptimisticLockError(t *testing.T) { }, } - f := fake.NewClientBuilder(). + f := reconcilertest.NewClientBuilder(). WithInterceptorFuncs(funcs). - WithScheme(tsapi.GlobalScheme). WithObjects(dnsCfg, proxyGroupEgressSvc, egressSvc). WithStatusSubresource(dnsCfg). Build() @@ -410,8 +408,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(dnsCfg, ing, headlessSvc, epv4, epv6, cm). WithStatusSubresource(dnsCfg). Build() @@ -423,7 +420,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }) // Test dual-stack service records - expectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") + reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") wantIPv4 := map[string][]string{"dual-stack.example.ts.net": {"10.1.2.3"}} wantIPv6 := map[string][]string{"dual-stack.example.ts.net": {"2001:db8::1"}} @@ -468,9 +465,9 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { }, } - mustCreate(t, fc, parentEgressSvc) - mustCreate(t, fc, proxyGroupSvc) - expectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack") + reconcilertest.MustCreate(t, fc, parentEgressSvc) + reconcilertest.MustCreate(t, fc, proxyGroupSvc) + reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack") wantIPv4["pg-service.example.ts.net"] = []string{"10.96.0.100"} wantIPv6["pg-service.example.ts.net"] = []string{"2001:db8::100"} @@ -519,7 +516,7 @@ func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.Add func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) { t.Helper() cm := new(corev1.ConfigMap) - if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { + if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { t.Fatalf("getting dnsconfig ConfigMap: %v", err) } if cm.Data == nil { @@ -541,7 +538,7 @@ func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][] func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, wantsHostsIPv6 map[string][]string) { t.Helper() cm := new(corev1.ConfigMap) - if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { + if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { t.Fatalf("getting dnsconfig ConfigMap: %v", err) } if cm.Data == nil { @@ -562,57 +559,3 @@ func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, t.Fatalf("unexpected IPv6 dns config (-got +want):\n%s", diff) } } - -func expectReconciled(t *testing.T, r *dnsrecords.Reconciler, ns, name string) { - t.Helper() - req := reconcile.Request{ - NamespacedName: types.NamespacedName{Namespace: ns, Name: name}, - } - res, err := r.Reconcile(context.Background(), req) - if err != nil { - t.Fatalf("Reconcile: unexpected error: %v", err) - } - if res.Requeue { - t.Fatalf("unexpected immediate requeue") - } -} - -func mustCreate(t *testing.T, c client.Client, obj client.Object) { - t.Helper() - if err := c.Create(context.Background(), obj); err != nil { - t.Fatalf("creating %q: %v", obj.GetName(), err) - } -} - -func mustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) { - t.Helper() - for _, obj := range objs { - if err := c.Delete(context.Background(), obj); err != nil { - t.Fatalf("deleting %q: %v", obj.GetName(), err) - } - } -} - -func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { - t.Fatalf("getting %q: %v", name, err) - } - update(obj) - if err := c.Update(context.Background(), obj); err != nil { - t.Fatalf("updating %q: %v", name, err) - } -} - -func mustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil { - t.Fatalf("getting %q: %v", name, err) - } - update(obj) - if err := c.Status().Update(context.Background(), obj); err != nil { - t.Fatalf("updating status %q: %v", name, err) - } -} diff --git a/k8s-operator/reconciler/nameserver/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 9add71705..705b7e177 100644 --- a/k8s-operator/reconciler/nameserver/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -12,7 +12,6 @@ import ( _ "embed" "errors" "fmt" - "strings" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" @@ -50,8 +49,6 @@ const ( defaultNameserverImageRepo = "tailscale/k8s-nameserver" defaultNameserverImageTag = "stable" - - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) @@ -161,7 +158,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg) } if err = r.maybeProvision(ctx, &dnsCfg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil } else { diff --git a/k8s-operator/reconciler/nameserver/nameserver_test.go b/k8s-operator/reconciler/nameserver/nameserver_test.go index 594917154..7cd0761e1 100644 --- a/k8s-operator/reconciler/nameserver/nameserver_test.go +++ b/k8s-operator/reconciler/nameserver/nameserver_test.go @@ -6,30 +6,23 @@ package nameserver_test import ( - "context" _ "embed" "encoding/json" - "reflect" "testing" "time" - "github.com/google/go-cmp/cmp" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/nameserver" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -96,8 +89,7 @@ func TestNameserverReconciler(t *testing.T) { }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(dnsConfig). WithStatusSubresource(dnsConfig). Build() @@ -115,7 +107,7 @@ func TestNameserverReconciler(t *testing.T) { Logger: logger.Sugar(), Clock: clock, }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) nameserverLabels := map[string]string{ @@ -169,7 +161,7 @@ func TestNameserverReconciler(t *testing.T) { {Name: "some-secret"}, } - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}} @@ -181,16 +173,16 @@ func TestNameserverReconciler(t *testing.T) { wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference} wantsSvc.Namespace = tsNamespace wantsSvc.ObjectMeta.Labels = nameserverLabels - expectEqual(t, fc, wantsSvc) + reconcilertest.ExpectEqual(t, fc, wantsSvc) }) t.Run("dns-config-status-is-set", func(t *testing.T) { // Verify that DNSConfig advertises the nameserver's Service IP address, // has the ready status condition and tailscale finalizer. - mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { svc.Spec.ClusterIP = "1.2.3.4" }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") dnsConfig.Finalizers = []string{reconciler.Finalizer} dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ @@ -204,17 +196,17 @@ func TestNameserverReconciler(t *testing.T) { LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, }) - expectEqual(t, fc, dnsConfig) + reconcilertest.ExpectEqual(t, fc, dnsConfig) }) t.Run("nameserver-image-updated", func(t *testing.T) { // Verify that nameserver image gets updated to match DNSConfig spec. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) t.Run("reconciler-preserves-custom-config", func(t *testing.T) { @@ -226,11 +218,11 @@ func TestNameserverReconciler(t *testing.T) { t.Fatalf("error marshalling ConfigMap contents: %v", err) } - mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { + reconcilertest.MustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { mak.Set(&cm.Data, "records.json", string(bs)) }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantCm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -243,60 +235,17 @@ func TestNameserverReconciler(t *testing.T) { Data: map[string]string{"records.json": string(bs)}, } - expectEqual(t, fc, wantCm) + reconcilertest.ExpectEqual(t, fc, wantCm) }) t.Run("uses-default-nameserver-image", func(t *testing.T) { // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, // the nameserver image defaults to tailscale/k8s-nameserver:stable. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { dnsCfg.Spec.Nameserver.Image = nil }) - mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}) + reconcilertest.ExpectReconciled(t, r, "", "test") wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:stable" - expectEqual(t, fc, wantsDeploy) + reconcilertest.ExpectEqual(t, fc, wantsDeploy) }) } - -func mustReconcile(t *testing.T, r *nameserver.Reconciler, req reconcile.Request) { - t.Helper() - if _, err := r.Reconcile(context.Background(), req); err != nil { - t.Fatalf("unexpected reconcile error: %v", err) - } -} - -func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { - t.Helper() - obj := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { - t.Fatalf("getting object: %v", err) - } - update(obj) - if err := c.Update(context.Background(), obj); err != nil { - t.Fatalf("updating object: %v", err) - } -} - -func expectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O) { - t.Helper() - got := O(new(T)) - if err := c.Get(context.Background(), types.NamespacedName{ - Name: want.GetName(), - Namespace: want.GetNamespace(), - }, got); err != nil { - t.Fatalf("getting %q: %v", want.GetName(), err) - } - // The resource version changes eagerly whenever the operator does even a - // no-op update. Asserting a specific value leads to overly brittle tests, - // so just remove it from both got and want. - got.SetResourceVersion("") - want.SetResourceVersion("") - // controller-runtime v0.20+ populates TypeMeta on objects returned by the - // fake client. Strip it so tests can continue to build expected objects - // without setting Kind/APIVersion explicitly. - got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) - want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) - if diff := cmp.Diff(got, want); diff != "" { - t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff) - } -} diff --git a/k8s-operator/reconciler/peerrelay/peerrelay_test.go b/k8s-operator/reconciler/peerrelay/peerrelay_test.go index 885a3427e..6254542da 100644 --- a/k8s-operator/reconciler/peerrelay/peerrelay_test.go +++ b/k8s-operator/reconciler/peerrelay/peerrelay_test.go @@ -14,7 +14,6 @@ import ( "net/netip" "slices" "strings" - "sync" "testing" "go.uber.org/zap" @@ -25,8 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" tailscaleclient "tailscale.com/client/tailscale/v2" @@ -34,7 +31,7 @@ import ( "tailscale.com/ipn" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/peerrelay" - "tailscale.com/k8s-operator/tsclient" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/types/key" ) @@ -1064,10 +1061,9 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - builder := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + builder := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). - WithInterceptorFuncs(applyPatchInterceptor()) + WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()) if tc.PeerRelay != nil { builder = builder.WithObjects(tc.PeerRelay) } @@ -1079,7 +1075,7 @@ func TestReconciler_Reconcile(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1149,7 +1145,7 @@ func TestReconciler_Reconcile(t *testing.T) { } if tc.ExpectedReadyStatus != "" || tc.ExpectedReadyReason != "" { - cond := readyCondition(&pr) + cond := reconcilertest.Condition(t, pr.Status.Conditions, tsapi.PeerRelayReady) if tc.ExpectedReadyStatus != "" && cond.Status != tc.ExpectedReadyStatus { t.Errorf("expected PeerRelayReady status %s, got %q", tc.ExpectedReadyStatus, cond.Status) } @@ -1294,16 +1290,6 @@ func assertSecretsForType(t *testing.T, fc client.Client, prName, secretType, la } } -func readyCondition(pr *tsapi.PeerRelay) metav1.Condition { - for _, cond := range pr.Status.Conditions { - if cond.Type == string(tsapi.PeerRelayReady) { - return cond - } - } - - return metav1.Condition{} -} - func assertService(t *testing.T, want expectedService, got *corev1.Service) { t.Helper() @@ -1455,8 +1441,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) { Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects( pr, @@ -1471,7 +1456,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{loginURL: testLoginURL}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(testLoginURL))), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1535,104 +1520,6 @@ func readTailscaledConfig(t *testing.T, fc client.Client, secretName string) ipn return conf } -func applyPatchInterceptor() interceptor.Funcs { - return interceptor.Funcs{ - Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if patch.Type() != types.ApplyPatchType { - return cl.Patch(ctx, obj, patch, opts...) - } - - key := client.ObjectKeyFromObject(obj) - existing := obj.DeepCopyObject().(client.Object) - if err := cl.Get(ctx, key, existing); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - - return cl.Create(ctx, obj) - } - - obj.SetResourceVersion(existing.GetResourceVersion()) - return cl.Update(ctx, obj) - }, - } -} - -type fakeClientProvider struct { - client tsclient.Client - err error -} - -func (p *fakeClientProvider) For(_ string) (tsclient.Client, error) { return p.client, p.err } - -type fakeTSClient struct { - tsclient.Client - - loginURL string - - mu sync.Mutex - keyCalls []tailscaleclient.CreateKeyRequest - deviceDeletes []string - devices []tailscaleclient.Device - nextKey []string -} - -func (c *fakeTSClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) } -func (c *fakeTSClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) } -func (c *fakeTSClient) LoginURL() string { return c.loginURL } - -func (c *fakeTSClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest { - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.keyCalls) -} - -func (c *fakeTSClient) DeviceDeletes() []string { - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.deviceDeletes) -} - -type fakeKeys fakeTSClient - -func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) { - c := (*fakeTSClient)(k) - c.mu.Lock() - defer c.mu.Unlock() - c.keyCalls = append(c.keyCalls, req) - - var key string - if len(c.nextKey) > 0 { - key, c.nextKey = c.nextKey[0], c.nextKey[1:] - } else { - key = fmt.Sprintf("auth-key-%d", len(c.keyCalls)) - } - return &tailscaleclient.Key{Key: key}, nil -} - -func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { return nil, nil } - -type fakeDevices fakeTSClient - -func (d *fakeDevices) Delete(_ context.Context, id string) error { - c := (*fakeTSClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - c.deviceDeletes = append(c.deviceDeletes, id) - return nil -} - -func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { - c := (*fakeTSClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - return slices.Clone(c.devices), nil -} - -func (d *fakeDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) { - return nil, nil -} - func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Parallel() @@ -1643,19 +1530,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Run("mints-key-on-first-reconcile", func(t *testing.T) { pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{nextKey: []string{"tskey-abc"}} + tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-abc")) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1681,19 +1567,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { t.Run("reuses-existing-key-across-reconciles", func(t *testing.T) { pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{nextKey: []string{"tskey-first", "tskey-second"}} + tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-first", "tskey-second")) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1721,20 +1606,19 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) { Spec: tsapi.PeerRelaySpec{Tags: tsapi.Tags{"tag:custom"}}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() - tsc := &fakeTSClient{} + tsc := reconcilertest.NewFakeClient() r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:k8s-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1801,8 +1685,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { unreported := stateSecret("test", "peerrelay-test-2", 2, "") unreported.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText} - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects( pr, @@ -1813,16 +1696,17 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { ). Build() - tsc := &fakeTSClient{devices: []tailscaleclient.Device{ - {NodeID: "device-bbb", MachineKey: mk.Public().String()}, - {NodeID: "device-other", MachineKey: key.NewMachine().Public().String()}, - }} + tsc := reconcilertest.NewFakeClient() + tsc.SetDevices( + tailscaleclient.Device{NodeID: "device-bbb", MachineKey: mk.Public().String()}, + tailscaleclient.Device{NodeID: "device-other", MachineKey: key.NewMachine().Public().String()}, + ) r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1856,8 +1740,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects( pr, @@ -1867,13 +1750,13 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) { ). Build() - tsc := &fakeTSClient{} + tsc := reconcilertest.NewFakeClient() r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ Client: fc, TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: tsc}, + Clients: reconcilertest.NewFakeClientProvider(tsc), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1916,8 +1799,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { Spec: tsapi.PeerRelaySpec{Tailnet: "missing"}, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}). WithObjects(pr). Build() @@ -1927,7 +1809,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{err: errors.New("tailnet missing: not ready")}, + Clients: reconcilertest.NewFailingClientProvider(errors.New("tailnet missing: not ready")), Resolver: testResolver, Logger: logger.Sugar(), }) @@ -1941,13 +1823,10 @@ func TestReconciler_TailnetUnavailable(t *testing.T) { t.Fatal(err) } - cond := readyCondition(&got) - if cond.Status != metav1.ConditionFalse { - t.Errorf("expected PeerRelayReady=False, got %q", cond.Status) - } - if cond.Reason != peerrelay.ReasonTailnetUnavailable { - t.Errorf("expected reason=%s, got %q", peerrelay.ReasonTailnetUnavailable, cond.Reason) - } + reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.PeerRelayReady, + metav1.ConditionFalse, peerrelay.ReasonTailnetUnavailable) + + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.PeerRelayReady) if !strings.Contains(cond.Message, "not ready") { t.Errorf("expected condition message to include resolver error, got %q", cond.Message) } @@ -1997,8 +1876,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { }, } - fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). WithObjects(pr, pc). Build() @@ -2008,7 +1886,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { TailscaleNamespace: tailscaleNamespace, ProxyImage: testProxyImage, DefaultTags: []string{"tag:test-peer-relay"}, - Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()), Resolver: testResolver, Logger: logger.Sugar(), }) diff --git a/k8s-operator/reconciler/proxyclass/proxyclass_test.go b/k8s-operator/reconciler/proxyclass/proxyclass_test.go index 54363ab61..cdd0315ad 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass_test.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass_test.go @@ -16,11 +16,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/proxyclass" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/tstest" ) @@ -51,8 +51,7 @@ func TestProxyClass(t *testing.T) { }, }, } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(pc). WithStatusSubresource(pc). Build() @@ -118,16 +117,7 @@ func TestProxyClass(t *testing.T) { if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil { t.Fatalf("Get: %v", err) } - var cond *metav1.Condition - for i := range got.Status.Conditions { - if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) { - cond = &got.Status.Conditions[i] - break - } - } - if cond == nil { - t.Fatalf("ProxyClassReady condition not set") - } + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady) if cond.Status != wantStatus { t.Errorf("condition Status: got %q, want %q", cond.Status, wantStatus) } @@ -337,8 +327,7 @@ func TestProxyClassValidation(t *testing.T) { objs = append(objs, existingPC) } - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). + fc := reconcilertest.NewClientBuilder(). WithObjects(objs...). WithStatusSubresource(pc, existingPC). Build() @@ -365,16 +354,7 @@ func TestProxyClassValidation(t *testing.T) { t.Fatalf("Get: %v", err) } - var ready *metav1.Condition - for i := range got.Status.Conditions { - if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) { - ready = &got.Status.Conditions[i] - break - } - } - if ready == nil { - t.Fatal("Ready condition not set") - } + ready := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady) wantStatus := metav1.ConditionTrue if !tc.valid { diff --git a/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go b/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go index d5c0b6d35..6e4b8cb51 100644 --- a/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go +++ b/k8s-operator/reconciler/proxygrouppolicy/proxygrouppolicy_test.go @@ -12,11 +12,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" + "tailscale.com/k8s-operator/reconciler/reconcilertest" ) func TestReconciler_Reconcile(t *testing.T) { @@ -102,7 +102,7 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - bldr := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) + bldr := reconcilertest.NewClientBuilder() bldr = bldr.WithObjects(tc.ExistingResources...) fc := bldr.Build() diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go index 5825f73ae..66339fc96 100644 --- a/k8s-operator/reconciler/reconciler.go +++ b/k8s-operator/reconciler/reconciler.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "slices" + "strings" "sync" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -40,8 +41,20 @@ const ( // LabelParentNamespace identifies the namespace of the owning Tailscale CRD. It is only stamped when the parent // CRD is namespaced; cluster-scoped parents omit it. LabelParentNamespace = "tailscale.com/parent-resource-ns" + + // optimisticLockErrorMsg is the message the API server returns when an Update is rejected because the object + // has been modified since it was read. The API server does not give us a typed error for the conflict in all + // cases, so callers match on the message via IsOptimisticLockError. + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) +// IsOptimisticLockError reports whether err was caused by an optimistic locking conflict, i.e. the object being +// updated was modified after the reconciler read it. Such an error is transient: the reconciler should requeue and +// retry against a freshly read object rather than surface the error. +func IsOptimisticLockError(err error) bool { + return err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) +} + // SetFinalizer adds name to obj's finalizers if not already present. Most CRD reconcilers pass Finalizer; per- // reconciler names are used when a single resource may carry finalizers from multiple reconcilers that each need to run // their own cleanup. diff --git a/k8s-operator/reconciler/reconciler_test.go b/k8s-operator/reconciler/reconciler_test.go index 7ae76df42..1e6f54bdf 100644 --- a/k8s-operator/reconciler/reconciler_test.go +++ b/k8s-operator/reconciler/reconciler_test.go @@ -6,6 +6,8 @@ package reconciler_test import ( + "errors" + "fmt" "maps" "slices" "testing" @@ -144,6 +146,37 @@ func TestEnqueueForChild(t *testing.T) { } } +func TestIsOptimisticLockError(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + err error + want bool + }{ + "nil": {}, + "conflict": { + err: errors.New("Operation cannot be fulfilled on secrets \"test\": the object has been modified; please apply your changes to the latest version and try again"), + want: true, + }, + "unrelated": { + err: errors.New("secrets \"test\" not found"), + }, + "wrapped-conflict": { + err: fmt.Errorf("failed to update Secret: %w", errors.New("the object has been modified; please apply your changes to the latest version and try again")), + want: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + if got := reconciler.IsOptimisticLockError(tc.err); got != tc.want { + t.Errorf("expected %v, got %v", tc.want, got) + } + }) + } +} + func TestResourceTracker(t *testing.T) { t.Parallel() diff --git a/k8s-operator/reconciler/reconcilertest/conditions.go b/k8s-operator/reconciler/reconcilertest/conditions.go new file mode 100644 index 000000000..0b88b9ceb --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/conditions.go @@ -0,0 +1,59 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package reconcilertest + +import ( + "testing" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +// The condition helpers below take a []metav1.Condition rather than the owning object. Every Tailscale CRD stores its +// conditions in Status.Conditions, but the CRD types live in tsapi and so can't grow a shared accessor method from +// here; taking the slice keeps these usable from every reconciler's tests (and for corev1.Service conditions) without +// per-CRD plumbing. Pair them with MustGet: +// +// tsr := &tsapi.Recorder{} +// reconcilertest.MustGet(t, fc, "", "test", tsr) +// reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, metav1.ConditionTrue, reason, message) + +// Condition returns the condition of the given type, failing the test if it isn't present. Use it when a test needs to +// inspect a condition's fields directly, e.g. to match part of a message generated outside this repo; prefer +// ExpectCondition for whole-condition assertions. +func Condition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType) metav1.Condition { + t.Helper() + cond := apimeta.FindStatusCondition(conds, string(conditionType)) + if cond == nil { + t.Fatalf("no %s condition found, got %+v", conditionType, conds) + } + return *cond +} + +// ExpectCondition asserts that a condition of the given type is present with the given status, reason and message. +// LastTransitionTime and ObservedGeneration are not compared, so tests don't need a fixed clock or to track +// generations in order to assert on the parts that describe the outcome. +func ExpectCondition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) { + t.Helper() + got := Condition(t, conds, conditionType) + if got.Status != status || got.Reason != reason || got.Message != message { + t.Fatalf("unexpected %s condition: got (%s, %s, %q), want (%s, %s, %q)", + conditionType, got.Status, got.Reason, got.Message, status, reason, message) + } +} + +// ExpectConditionStatus asserts a condition's status and reason, ignoring its message. Use it when the message is +// long, generated outside this repo, or otherwise not the point of the test. +func ExpectConditionStatus(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason string) { + t.Helper() + got := Condition(t, conds, conditionType) + if got.Status != status || got.Reason != reason { + t.Fatalf("unexpected %s condition: got (%s, %s), want (%s, %s)", + conditionType, got.Status, got.Reason, status, reason) + } +} diff --git a/k8s-operator/reconciler/reconcilertest/reconcilertest.go b/k8s-operator/reconciler/reconcilertest/reconcilertest.go new file mode 100644 index 000000000..c2529915c --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/reconcilertest.go @@ -0,0 +1,202 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package reconcilertest provides helpers for testing the CRD reconcilers under k8s-operator/reconciler. It covers +// the parts of a reconciler test that are the same whichever CRD is under test: building a fake client with the +// operator's scheme, creating and mutating objects in it, asserting on what a reconcile left behind, and faking the +// Tailscale API. +// +// Assertions come in two flavours. Must* helpers set up test preconditions and call t.Fatalf if the cluster doesn't +// behave, because a failed precondition makes the rest of the test meaningless. Expect* helpers assert on the state +// a reconcile produced. +// +// Per-CRD assertions belong in the reconciler's own test package; only helpers that are genuinely reconciler- +// agnostic belong here. +package reconcilertest + +import ( + "context" + "reflect" + "slices" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" +) + +// NewClientBuilder returns a fake client builder preconfigured with the operator's scheme. Callers add their own +// objects, status subresources and interceptors before calling Build. +func NewClientBuilder() *fake.ClientBuilder { + return fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) +} + +// ApplyPatchInterceptor returns interceptor funcs that emulate server-side apply, which the fake client does not +// implement. Apply patches become a Create if the object is absent and an Update if it is present; every other patch +// type is passed through untouched. Reconcilers that apply their children (rather than using +// reconciler.CreateOrUpdate) need this to be testable against the fake client. +func ApplyPatchInterceptor() interceptor.Funcs { + return interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if patch.Type() != types.ApplyPatchType { + return cl.Patch(ctx, obj, patch, opts...) + } + + key := client.ObjectKeyFromObject(obj) + existing := obj.DeepCopyObject().(client.Object) + if err := cl.Get(ctx, key, existing); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + + return cl.Create(ctx, obj) + } + + obj.SetResourceVersion(existing.GetResourceVersion()) + return cl.Update(ctx, obj) + }, + } +} + +// MustCreate creates obj, failing the test if the cluster rejects it. +func MustCreate(t *testing.T, c client.Client, obj client.Object) { + t.Helper() + if err := c.Create(t.Context(), obj); err != nil { + t.Fatalf("creating %q: %v", obj.GetName(), err) + } +} + +// MustGet reads ns/name into obj, failing the test if it is absent. Use ExpectMissing to assert the opposite. +func MustGet(t *testing.T, c client.Client, ns, name string, obj client.Object) { + t.Helper() + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } +} + +// MustDeleteAll deletes each of objs in order. +func MustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + if err := c.Delete(t.Context(), obj); err != nil { + t.Fatalf("deleting %q: %v", obj.GetName(), err) + } + } +} + +// MustUpdate reads ns/name, applies update to it and writes it back. It reads the object fresh rather than taking one +// from the caller so the update is applied to the current ResourceVersion. +func MustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Update(t.Context(), obj); err != nil { + t.Fatalf("updating %q: %v", name, err) + } +} + +// MustUpdateStatus is MustUpdate for the status subresource. The object's type must have been registered with +// WithStatusSubresource on the fake client builder. +func MustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) { + t.Helper() + obj := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil { + t.Fatalf("getting %q: %v", name, err) + } + update(obj) + if err := c.Status().Update(t.Context(), obj); err != nil { + t.Fatalf("updating status %q: %v", name, err) + } +} + +// ExpectEqual asserts that the object named by want exists and matches it. ResourceVersion and TypeMeta are cleared +// on both sides first: the former changes on every no-op write and the latter is populated by the fake client but not +// by the callers building expected objects, so asserting on either makes tests brittle. Pass modifiers to blank out +// any further fields that shouldn't be compared; each one runs against both want and got. +func ExpectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O, modifiers ...func(O)) { + t.Helper() + got := O(new(T)) + if err := c.Get(t.Context(), types.NamespacedName{ + Name: want.GetName(), + Namespace: want.GetNamespace(), + }, got); err != nil { + t.Fatalf("getting %q: %v", want.GetName(), err) + } + got.SetResourceVersion("") + want.SetResourceVersion("") + got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + for _, modifier := range modifiers { + modifier(want) + modifier(got) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff) + } +} + +// ExpectMissing asserts that no object of type T exists at ns/name. +func ExpectMissing[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string) { + t.Helper() + obj := O(new(T)) + err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj) + if !apierrors.IsNotFound(err) { + t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name) + } +} + +// ExpectReconciled runs a single reconcile for ns/name and asserts it succeeded without asking to be requeued, i.e. +// the reconciler considers the resource fully settled. +func ExpectReconciled(t *testing.T, r reconcile.Reconciler, ns, name string) { + t.Helper() + res, err := r.Reconcile(t.Context(), request(ns, name)) + if err != nil { + t.Fatalf("Reconcile: unexpected error: %v", err) + } + if res.Requeue { + t.Fatalf("unexpected immediate requeue") + } + if res.RequeueAfter != 0 { + t.Fatalf("unexpected timed requeue (%v)", res.RequeueAfter) + } +} + +func request(ns, name string) reconcile.Request { + return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: name}} +} + +// ExpectEvents asserts that len(want) events are emitted on rec, each of which is in want, waiting up to five seconds +// for each. Events are not required to arrive in order, because a reconcile may emit them from more than one code path. +func ExpectEvents(t *testing.T, rec *record.FakeRecorder, want []string) { + t.Helper() + seen := make([]string, 0, len(want)) + for range want { + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case got := <-rec.Events: + if !slices.Contains(want, got) { + t.Errorf("got unexpected event %q, expected events: %+#v", got, want) + continue + } + seen = append(seen, got) + case <-timer.C: + t.Errorf("timeout waiting for an event, wants events %+#v, got events %+#v", want, seen) + } + } +} diff --git a/k8s-operator/reconciler/reconcilertest/tsclient.go b/k8s-operator/reconciler/reconcilertest/tsclient.go new file mode 100644 index 000000000..a90aea029 --- /dev/null +++ b/k8s-operator/reconciler/reconcilertest/tsclient.go @@ -0,0 +1,174 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package reconcilertest + +import ( + "context" + "fmt" + "net/http" + "slices" + "sync" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/k8s-operator/tsclient" +) + +// FakeClient is a tsclient.Client that records the calls a reconciler makes to the Tailscale API and serves canned +// responses. The zero value is usable: keys are minted with generated values, no devices exist, and the login URL is +// blank. FakeClient is safe for concurrent use. +// +// Only the endpoints the reconcilers actually call are implemented. Anything else panics via the embedded nil +// interface rather than silently returning a zero value, so a reconciler that starts calling a new endpoint fails +// loudly instead of being tested against a fake that quietly does nothing. +type FakeClient struct { + tsclient.Client + + mu sync.Mutex + + loginURL string + + // Devices are returned by Devices().Get and List. Get matches on Device.ID and returns a 404 API error when no + // device matches, mirroring control's behaviour for an unknown or already-deleted device. + devices []tailscaleclient.Device + + // nextKeys are handed out by CreateAuthKey in order. Once exhausted, generated values are returned instead, so + // tests only script the keys they assert on. + nextKeys []string + + keyCalls []tailscaleclient.CreateKeyRequest + deviceDeletes []string +} + +// FakeClientOption configures a FakeClient. +type FakeClientOption func(*FakeClient) + +// WithLoginURL sets the URL returned by LoginURL, which reconcilers embed in the workloads they create. +func WithLoginURL(url string) FakeClientOption { + return func(c *FakeClient) { c.loginURL = url } +} + +// WithAuthKeys sets the auth keys CreateAuthKey returns, in order. Use it when a test needs to tell successive keys +// apart, e.g. to assert that a replica picked up a reissued key. +func WithAuthKeys(keys ...string) FakeClientOption { + return func(c *FakeClient) { c.nextKeys = keys } +} + +// NewFakeClient returns a FakeClient configured by opts. +func NewFakeClient(opts ...FakeClientOption) *FakeClient { + c := &FakeClient{} + for _, opt := range opts { + opt(c) + } + return c +} + +// LoginURL implements tsclient.Client. +func (c *FakeClient) LoginURL() string { return c.loginURL } + +// Devices implements tsclient.Client. +func (c *FakeClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) } + +// Keys implements tsclient.Client. +func (c *FakeClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) } + +// CreateAuthKeyCalls returns every auth key request the reconciler has made, in order. +func (c *FakeClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.keyCalls) +} + +// DeviceDeletes returns the IDs of every device the reconciler has deleted, in order. +func (c *FakeClient) DeviceDeletes() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.deviceDeletes) +} + +// SetDevices replaces the devices served by the fake. Use it to simulate control catching up partway through a test, +// e.g. once a replica has authenticated. +func (c *FakeClient) SetDevices(devices ...tailscaleclient.Device) { + c.mu.Lock() + defer c.mu.Unlock() + c.devices = devices +} + +type fakeDevices FakeClient + +func (d *fakeDevices) Delete(_ context.Context, id string) error { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + c.deviceDeletes = append(c.deviceDeletes, id) + return nil +} + +func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.devices), nil +} + +func (d *fakeDevices) Get(_ context.Context, id string) (*tailscaleclient.Device, error) { + c := (*FakeClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + for _, dev := range c.devices { + if dev.ID == id { + return &dev, nil + } + } + return nil, tailscaleclient.APIError{Status: http.StatusNotFound} +} + +type fakeKeys FakeClient + +func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) { + c := (*FakeClient)(k) + c.mu.Lock() + defer c.mu.Unlock() + c.keyCalls = append(c.keyCalls, req) + + var key string + if len(c.nextKeys) > 0 { + key, c.nextKeys = c.nextKeys[0], c.nextKeys[1:] + } else { + key = fmt.Sprintf("auth-key-%d", len(c.keyCalls)) + } + return &tailscaleclient.Key{Key: key}, nil +} + +func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { + return nil, nil +} + +// FakeClientProvider is a tailscaled.ClientProvider that hands out a single client for every tailnet, or fails with a +// fixed error. Reconcilers resolve their API client through a provider, so tests that exercise the +// tailnet-unavailable path need one that can fail. +type FakeClientProvider struct { + client tsclient.Client + err error +} + +// NewFakeClientProvider returns a provider that resolves every tailnet to client. +func NewFakeClientProvider(client tsclient.Client) *FakeClientProvider { + return &FakeClientProvider{client: client} +} + +// NewFailingClientProvider returns a provider that fails to resolve any tailnet with err. +func NewFailingClientProvider(err error) *FakeClientProvider { + return &FakeClientProvider{err: err} +} + +// For implements tailscaled.ClientProvider. +func (p *FakeClientProvider) For(_ string) (tsclient.Client, error) { + if p.err != nil { + return nil, p.err + } + return p.client, nil +} diff --git a/k8s-operator/reconciler/recorder/authkey.go b/k8s-operator/reconciler/recorder/authkey.go new file mode 100644 index 000000000..ef8fba988 --- /dev/null +++ b/k8s-operator/reconciler/recorder/authkey.go @@ -0,0 +1,91 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package recorder + +import ( + "context" + "fmt" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" + "tailscale.com/k8s-operator/tsclient" +) + +// authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret. +const authKeySecretKey = "authkey" + +// recorderTags returns the tags to apply to auth keys minted for tsr's replicas. +func recorderTags(tsr *tsapi.Recorder) []string { + if len(tsr.Spec.Tags) == 0 { + return tsapi.Tags{"tag:k8s"}.Stringify() + } + + return tsr.Spec.Tags.Stringify() +} + +// ensureAuthSecretsCreated makes sure every replica has an auth Secret holding a usable auth key. A key is minted +// when the Secret doesn't exist yet, or when the replica has requested a reissue via its state Secret because the +// existing key can no longer authenticate it. +func (r *Reconciler) ensureAuthSecretsCreated(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + tags := recorderTags(tsr) + + for replica := range replicas(tsr) { + key := types.NamespacedName{ + Namespace: r.tsNamespace, + Name: authSecretName(tsr.Name, replica), + } + + existingSecret := &corev1.Secret{} + err := r.Get(ctx, key, existingSecret) + switch { + case err == nil: + stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) + if err != nil { + return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err) + } + cfgAuthKey := string(existingSecret.Data[authKeySecretKey]) + reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, logger, tailscaled.ReissueInput{ + ParentName: tsr.Name, + ReplicaName: stateSecretName(tsr.Name, replica), + Kind: tailscaled.KindRecorder, + StateSecret: stateSecret, + CfgAuthKey: &cfgAuthKey, + }) + if err != nil { + return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) + } + if !reissue { + logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) + continue + } + authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags) + if err != nil { + return err + } + existingSecret.Data[authKeySecretKey] = []byte(authKey) + if err = r.Update(ctx, existingSecret); err != nil { + return err + } + case apierrors.IsNotFound(err): + authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags) + if err != nil { + return err + } + if err = r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { + return err + } + default: + return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) + } + } + + return nil +} diff --git a/cmd/k8s-operator/tsrecorder.go b/k8s-operator/reconciler/recorder/recorder.go similarity index 61% rename from cmd/k8s-operator/tsrecorder.go rename to k8s-operator/reconciler/recorder/recorder.go index 74ee7051c..60497b578 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/k8s-operator/reconciler/recorder/recorder.go @@ -3,20 +3,21 @@ //go:build !plan9 -package main +// Package recorder provides reconciliation logic for the Recorder custom resource definition. It is responsible for +// managing the lifecycle of tsrecorder devices, including the StatefulSet, ServiceAccount and RBAC resources used to +// run them. +package recorder import ( "context" - "encoding/json" "errors" "fmt" "slices" "strconv" "strings" - "sync" + "time" "go.uber.org/zap" - xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -24,68 +25,132 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apivalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/client/tailscale/v2" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" - "tailscale.com/tailcfg" "tailscale.com/tstime" "tailscale.com/util/clientmetric" - "tailscale.com/util/set" ) const ( - reasonRecorderCreationFailed = "RecorderCreationFailed" - reasonRecorderCreating = "RecorderCreating" - reasonRecorderCreated = "RecorderCreated" - reasonRecorderInvalid = "RecorderInvalid" - reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" + reconcilerName = "recorder-reconciler" - currentProfileKey = "_current-profile" - - // authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret. - authKeySecretKey = "authkey" + // shortRequeue is how long to wait before retrying a reconcile that made progress but hasn't finished, e.g. + // cleanup that is waiting on control to catch up. + shortRequeue = 5 * time.Second ) +// Condition reasons set on Recorder resources. +const ( + reasonRecorderCreated = "RecorderCreated" + reasonRecorderCreationFailed = "RecorderCreationFailed" + reasonRecorderCreating = "RecorderCreating" + reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" +) + +// gaugeRecorderResources tracks the overall number of Recorder resources currently managed by this operator instance. var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount) -// RecorderReconciler syncs Recorder statefulsets with their definition in -// Recorder CRs. -type RecorderReconciler struct { - client.Client - log *zap.SugaredLogger - recorder record.EventRecorder - clock tstime.Clock - clients ClientProvider - tsNamespace string - reissuer *tailscaled.Reissuer - mu sync.Mutex // protects following - recorders set.Slice[types.UID] // for recorders gauge +type ( + // Reconciler is a reconcile.Reconciler implementation that syncs Recorder StatefulSets with their definition in + // Recorder custom resources. + Reconciler struct { + client.Client + + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock + clients tailscaled.ClientProvider + tsNamespace string + reissuer *tailscaled.Reissuer + + tracker *reconciler.ResourceTracker + } + + // ReconcilerOptions contains configuration values for the Reconciler. + ReconcilerOptions struct { + // Client is used to interact with the Kubernetes API. + Client client.Client + // Recorder is used to emit Kubernetes events. + Recorder record.EventRecorder + // TailscaleNamespace is the namespace the operator is installed in. Recorder-managed resources + // (StatefulSets, Secrets, etc.) are created within this namespace. + TailscaleNamespace string + // Clients resolves the Tailscale API client for a given tailnet name. Used to mint auth keys for each + // replica and to delete their devices on cleanup. Blank tailnet returns the operator's default client. + Clients tailscaled.ClientProvider + // Logger is the logger to use for this Reconciler. + Logger *zap.SugaredLogger + // Clock is used to stamp condition transitions. Defaults to a real clock when unset. + Clock tstime.Clock + } +) + +// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Recorder +// custom resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + clock := options.Clock + if clock == nil { + clock = tstime.DefaultClock{} + } + + return &Reconciler{ + Client: options.Client, + recorder: options.Recorder, + logger: options.Logger.Named(reconcilerName), + clock: clock, + clients: options.Clients, + tsNamespace: options.TailscaleNamespace, + reissuer: tailscaled.NewReissuer(), + tracker: reconciler.NewResourceTracker(gaugeRecorderResources), + } } -func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger { - return r.log.With("Recorder", name) +// Register the Reconciler onto the given manager.Manager implementation. It watches Recorder resources directly, as +// well as the child resources it manages, so that drift is corrected by a reconcile of the owning Recorder. All +// children carry a controller owner reference to their Recorder, so they are matched by owner rather than by label. +func (r *Reconciler) Register(mgr manager.Manager) error { + enqueue := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{}) + return builder. + ControllerManagedBy(mgr). + For(&tsapi.Recorder{}). + Named(reconcilerName). + Watches(&appsv1.StatefulSet{}, enqueue). + Watches(&corev1.ServiceAccount{}, enqueue). + Watches(&corev1.Secret{}, enqueue). + Watches(&rbacv1.Role{}, enqueue). + Watches(&rbacv1.RoleBinding{}, enqueue). + Complete(r) } -func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - logger := r.logger(req.Name) +// Reconcile is invoked when a change occurs to Recorder resources within the cluster. On create/update it ensures a +// StatefulSet and the Secrets and RBAC resources each replica needs exist. On delete it removes each replica's +// device from the tailnet before releasing the finalizer; the in-cluster resources are garbage collected via their +// owner references. +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + logger := r.logger.With("Recorder", req.Name) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") tsr := new(tsapi.Recorder) err := r.Get(ctx, req.NamespacedName, tsr) - if apierrors.IsNotFound(err) { + switch { + case apierrors.IsNotFound(err): logger.Debugf("Recorder not found, assuming it was deleted") return reconcile.Result{}, nil - } else if err != nil { + case err != nil: return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err) } @@ -107,36 +172,33 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error()) } - if markedForDeletion(tsr) { + if !tsr.DeletionTimestamp.IsZero() { logger.Debugf("Recorder is being deleted, cleaning up resources") - ix := xslices.Index(tsr.Finalizers, FinalizerName) - if ix < 0 { + if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) { logger.Debugf("no finalizer, nothing to do") return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, tsr, tsClient); err != nil { + if done, err := r.maybeCleanup(ctx, logger, tsr, tsClient); err != nil { return reconcile.Result{}, err } else if !done { logger.Debugf("Recorder resource cleanup not yet finished, will retry...") return reconcile.Result{RequeueAfter: shortRequeue}, nil } - tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1) - if err = r.Update(ctx, tsr); err != nil { + if err = reconciler.ClearFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil { return reconcile.Result{}, err } return reconcile.Result{}, nil } - if !slices.Contains(tsr.Finalizers, FinalizerName) { + if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, // this is a nice place to log that the high level, multi-reconcile // operation is underway. logger.Infof("ensuring Recorder is set up") - tsr.Finalizers = append(tsr.Finalizers, FinalizerName) - if err = r.Update(ctx, tsr); err != nil { + if err = reconciler.EnsureFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil { return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed) } } @@ -147,10 +209,10 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message) } - if err = r.maybeProvision(ctx, tsClient, tsr); err != nil { + if err = r.maybeProvision(ctx, logger, tsClient, tsr); err != nil { reason := reasonRecorderCreationFailed message := fmt.Sprintf("failed creating Recorder: %s", err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { reason = reasonRecorderCreating message = fmt.Sprintf("optimistic lock error, retrying: %s", err) err = nil @@ -165,28 +227,20 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated) } -func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { - logger := r.logger(tsr.Name) +func (r *Reconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + replicas := replicas(tsr) - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - r.mu.Lock() - r.recorders.Add(tsr.UID) - gaugeRecorderResources.Set(int64(r.recorders.Len())) - r.mu.Unlock() + r.tracker.Add(tsr.UID) r.reissuer.EnsureState(tsr.Name, int(replicas)) - if err := r.ensureAuthSecretsCreated(ctx, tsClient, tsr); err != nil { + if err := r.ensureAuthSecretsCreated(ctx, logger, tsClient, tsr); err != nil { return fmt.Errorf("error creating secrets: %w", err) } // State Secrets are pre-created so we can use the Recorder CR as its owner ref. for replica := range replicas { sec := tsrStateSecret(tsr, r.tsNamespace, replica) - _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { + _, err := reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { s.ObjectMeta.Labels = sec.ObjectMeta.Labels s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations }) @@ -196,7 +250,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } sa := tsrServiceAccount(tsr, r.tsNamespace) - _, err := createOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { + _, err := reconciler.CreateOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { // Perform this check within the update function to make sure we don't // have a race condition between the previous check and the update. if err := saOwnedByRecorder(s, tsr); err != nil { @@ -213,7 +267,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } role := tsrRole(tsr, r.tsNamespace) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels r.ObjectMeta.Annotations = role.ObjectMeta.Annotations r.Rules = role.Rules @@ -223,7 +277,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } roleBinding := tsrRoleBinding(tsr, r.tsNamespace) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations r.RoleRef = roleBinding.RoleRef @@ -234,7 +288,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie } ss := tsrStatefulSet(tsr, r.tsNamespace, tsClient.LoginURL()) - _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + _, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.Spec = ss.Spec @@ -246,13 +300,13 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie // ServiceAccount name may have changed, in which case we need to clean up // the previous ServiceAccount. RoleBinding will already be updated to point // to the new ServiceAccount. - if err = r.maybeCleanupServiceAccounts(ctx, tsr, sa.Name); err != nil { + if err = r.maybeCleanupServiceAccounts(ctx, logger, tsr, sa.Name); err != nil { return fmt.Errorf("error cleaning up ServiceAccounts: %w", err) } // If we have scaled the recorder down, we will have dangling state secrets // that we need to clean up. - if err = r.maybeCleanupSecrets(ctx, tsClient, tsr); err != nil { + if err = r.maybeCleanupSecrets(ctx, logger, tsClient, tsr); err != nil { return fmt.Errorf("error cleaning up Secrets: %w", err) } @@ -292,9 +346,7 @@ func saOwnedByRecorder(sa *corev1.ServiceAccount, tsr *tsapi.Recorder) error { // accumulating a large amount of garbage. // // This is a no-op if the ServiceAccount name has not changed. -func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, tsr *tsapi.Recorder, currentName string) error { - logger := r.logger(tsr.Name) - +func (r *Reconciler) maybeCleanupServiceAccounts(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, currentName string) error { options := []client.ListOption{ client.InNamespace(r.tsNamespace), client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), @@ -323,7 +375,9 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts return nil } -func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { +// maybeCleanupSecrets deletes the state and auth Secrets of any replica beyond the Recorder's current replica count, +// along with each one's tailnet device. Their ordinals are read back from the Secret names. +func (r *Reconciler) maybeCleanupSecrets(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error { options := []client.ListOption{ client.InNamespace(r.tsNamespace), client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), @@ -336,10 +390,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t // Get the largest ordinal suffix that we expect. Then we'll go through the list of secrets owned by this // recorder and remove them. - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } + replicas := replicas(tsr) for _, secret := range secrets.Items { parts := strings.Split(secret.Name, "-") @@ -356,22 +407,16 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t continue } - devicePrefs, ok, err := getDevicePrefs(&secret) + devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(&secret) if err != nil { return err } if ok { - r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID) - err = tsClient.Devices().Delete(ctx, string(devicePrefs.Config.NodeID)) - switch { - case tailscale.IsNotFound(err): - // This device has possibly already been deleted in the admin console. So we can ignore this - // and move on to removing the secret. - case err != nil: + if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil { return err } - } else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, r.log, &secret); err != nil { + } else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, logger, &secret); err != nil { return err } @@ -386,25 +431,18 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a Recorder will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) { - logger := r.logger(tsr.Name) - - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - for replica := range replicas { +func (r *Reconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) { + for replica := range replicas(tsr) { secret, err := r.getStateSecret(ctx, tsr.Name, replica) if err != nil { return false, err } if secret == nil { - logger.Debugf("state Secret %s-%d not found, continuing cleanup", tsr.Name, replica) + logger.Debugf("state Secret %s not found, continuing cleanup", stateSecretName(tsr.Name, replica)) continue } - devicePrefs, ok, err := getDevicePrefs(secret) + devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(secret) if err != nil { return false, err } @@ -415,18 +453,9 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record continue } - nodeID := string(devicePrefs.Config.NodeID) - logger.Debugf("deleting device %s from control", nodeID) - err = tsClient.Devices().Delete(ctx, nodeID) - switch { - case tailscale.IsNotFound(err): - logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID) - continue - case err != nil: - return false, fmt.Errorf("error deleting device: %w", err) + if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil { + return false, err } - - logger.Debugf("device %s deleted from control", nodeID) } // Unlike most log entries in the reconcile loop, this will get printed @@ -434,83 +463,13 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record // cleanup removes the tailscale finalizer, which will make all future // reconciles exit early. logger.Infof("cleaned up Recorder resources") - r.mu.Lock() - r.recorders.Remove(tsr.UID) - gaugeRecorderResources.Set(int64(r.recorders.Len())) - r.mu.Unlock() + r.tracker.Remove(tsr.UID) r.reissuer.RemoveState(tsr.Name) return true, nil } -func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { - var replicas int32 = 1 - if tsr.Spec.Replicas != nil { - replicas = *tsr.Spec.Replicas - } - - tags := tsr.Spec.Tags - if len(tags) == 0 { - tags = tsapi.Tags{"tag:k8s"} - } - - logger := r.logger(tsr.Name) - - for replica := range replicas { - key := types.NamespacedName{ - Namespace: r.tsNamespace, - Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), - } - - existingSecret := &corev1.Secret{} - err := r.Get(ctx, key, existingSecret) - switch { - case err == nil: - stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) - if err != nil { - return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err) - } - cfgAuthKey := string(existingSecret.Data[authKeySecretKey]) - reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, r.log, tailscaled.ReissueInput{ - ParentName: tsr.Name, - ReplicaName: fmt.Sprintf("%s-%d", tsr.Name, replica), - Kind: tailscaled.KindRecorder, - StateSecret: stateSecret, - CfgAuthKey: &cfgAuthKey, - }) - if err != nil { - return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) - } - if !reissue { - logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) - continue - } - authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) - if err != nil { - return err - } - existingSecret.Data[authKeySecretKey] = []byte(authKey) - if err = r.Update(ctx, existingSecret); err != nil { - return err - } - continue - case apierrors.IsNotFound(err): - authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) - if err != nil { - return err - } - if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { - return err - } - default: - return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) - } - } - - return nil -} - -func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *Reconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error { if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil { return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible") } @@ -535,7 +494,7 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) case apierrors.IsNotFound(err): // ServiceAccount doesn't exist, so no conflict. case err != nil: - return fmt.Errorf("error getting ServiceAccount %q for validation: %w", tsr.Spec.StatefulSet.Pod.ServiceAccount.Name, err) + return fmt.Errorf("error getting ServiceAccount %q for validation: %w", specSA.Name, err) default: // ServiceAccount exists, check if it's owned by the Recorder. if err := saOwnedByRecorder(sa, tsr); err != nil { @@ -552,11 +511,13 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) return nil } -func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) { +// getStateSecret returns the tailscaled state Secret for the given Recorder replica, or nil if it does not exist +// yet. +func (r *Reconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: r.tsNamespace, - Name: fmt.Sprintf("%s-%d", tsrName, replica), + Name: stateSecretName(tsrName, replica), }, } if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { @@ -570,33 +531,17 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, return secret, nil } -// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName -// is expected to always be non-empty if the node ID is, but not required. -func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) { - // TODO(tomhjp): Should maybe use ipn to parse the following info instead. - currentProfile, ok := secret.Data[currentProfileKey] - if !ok { - return prefs, false, nil - } - profileBytes, ok := secret.Data[string(currentProfile)] - if !ok { - return prefs, false, nil - } - if err := json.Unmarshal(profileBytes, &prefs); err != nil { - return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err) - } - - ok = prefs.Config.NodeID != "" - return prefs, ok, nil -} - -func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { +func (r *Reconciler) getDevicePrefs(ctx context.Context, tsrName string, replica int32) (prefs tailscaled.Prefs, ok bool, err error) { secret, err := r.getStateSecret(ctx, tsrName, replica) if err != nil || secret == nil { - return tsapi.RecorderTailnetDevice{}, false, err + return prefs, false, err } - prefs, ok, err := getDevicePrefs(secret) + return tailscaled.PrefsFromStateSecret(secret) +} + +func (r *Reconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { + prefs, ok, err := r.getDevicePrefs(ctx, tsrName, replica) if !ok || err != nil { return tsapi.RecorderTailnetDevice{}, false, err } @@ -620,20 +565,11 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclien return d, true, nil } -// [prefs] is a subset of the ipn.Prefs struct used for extracting information -// from the state Secret of Tailscale devices. -type prefs struct { - Config struct { - NodeID tailcfg.StableNodeID `json:"NodeID"` - UserProfile struct { - // LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net. - LoginName string `json:"LoginName"` - } `json:"UserProfile"` - } `json:"Config"` +// replicas returns the number of replicas the Recorder asks for, defaulting to one. +func replicas(tsr *tsapi.Recorder) int32 { + if tsr.Spec.Replicas != nil { + return *tsr.Spec.Replicas + } - AdvertiseServices []string `json:"AdvertiseServices"` -} - -func markedForDeletion(obj metav1.Object) bool { - return !obj.GetDeletionTimestamp().IsZero() + return 1 } diff --git a/k8s-operator/reconciler/recorder/recorder_test.go b/k8s-operator/reconciler/recorder/recorder_test.go new file mode 100644 index 000000000..15b5ede59 --- /dev/null +++ b/k8s-operator/reconciler/recorder/recorder_test.go @@ -0,0 +1,803 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package recorder_test + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/reconcilertest" + "tailscale.com/k8s-operator/reconciler/recorder" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" + "tailscale.com/types/key" + "tailscale.com/util/clientmetric" + "tailscale.com/util/mak" + "tailscale.com/version" +) + +const ( + tsNamespace = "tailscale" + tsLoginServer = "example.tailscale.com" +) + +func TestRecorder(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer)) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + fr := record.NewFakeRecorder(2) + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: fr, + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(tsClient), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid", + "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible") + expectManagedCount(t, 0) + expectRecorderResources(t, fc, tsr, false) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"}) + + tsr.Spec.EnableUI = true + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ + "invalid space characters": "test", + } + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible"}) + + tsr.Spec.Storage.S3 = &tsapi.S3{} + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Only check part of this error message, because it's defined in an + // external package and may change. + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.RecorderReady, metav1.ConditionFalse, "RecorderInvalid") + cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.RecorderReady) + for _, msg := range []string{cond.Message, <-fr.Events} { + if !strings.Contains(msg, `"invalid space characters"`) { + t.Fatalf("expected invalid annotation key in error message, got %q", msg) + } + } + }) + + t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) { + reconcilertest.MustCreate(t, fc, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pre-existing-sa", + Namespace: tsNamespace, + }, + }) + + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa" + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace` + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid", msg) + expectManagedCount(t, 0) + + reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid " + msg}) + }) + + t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) { + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "" + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // The reason is part of the CRD's observable status, so assert it as a literal: a rename of the constant + // should fail here. + expectReadyCondition(t, fc, tsr.Name, metav1.ConditionTrue, "RecorderCreated", "RecorderCreated") + expectManagedCount(t, 1) + expectRecorderResources(t, fc, tsr, true) + + // Every replica's auth key was minted with the default tag, and its + // StatefulSet is pointed at the login server of its tailnet. + if got := len(tsClient.CreateAuthKeyCalls()); got != int(*tsr.Spec.Replicas) { + t.Fatalf("expected %d auth keys to be created, got %d", *tsr.Spec.Replicas, got) + } + for _, req := range tsClient.CreateAuthKeyCalls() { + if diff := cmp.Diff(req.Capabilities.Devices.Create.Tags, []string{"tag:k8s"}); diff != "" { + t.Errorf("unexpected auth key tags (-got +want):\n%s", diff) + } + } + + ss := &appsv1.StatefulSet{} + reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, ss) + if got := envValue(ss, "TSRECORDER_LOGIN_SERVER"); got != tsLoginServer { + t.Errorf("unexpected login server env value: got %q, want %q", got, tsLoginServer) + } + }) + + t.Run("valid_service_account_config", func(t *testing.T) { + tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa" + tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{ + "test": "test", + } + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + expectManagedCount(t, 1) + expectRecorderResources(t, fc, tsr, true) + + // Get the service account and check the annotations. + sa := &corev1.ServiceAccount{} + reconcilertest.MustGet(t, fc, tsNamespace, "test-sa", sa) + if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" { + t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff) + } + + // The ServiceAccount named after the Recorder is no longer referenced, + // so it should have been cleaned up. + reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name) + }) + + t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) { + for replica := range *tsr.Spec.Replicas { + seedStateSecret(t, fc, fmt.Sprintf("test-%d", replica), fmt.Sprintf("node-%d", replica), fmt.Sprintf("test-%d.example.ts.net", replica)) + } + + tsClient.SetDevices( + tailscaleclient.Device{ + ID: "node-0", + Hostname: "hostname-node-0", + Addresses: []string{"1.2.3.4", "::1"}, + }, + tailscaleclient.Device{ + ID: "node-1", + Hostname: "hostname-node-1", + Addresses: []string{"1.2.3.4", "::1"}, + }, + tailscaleclient.Device{ + ID: "node-2", + Hostname: "hostname-node-2", + Addresses: []string{"1.2.3.4", "::1"}, + }, + ) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + want := []tsapi.RecorderTailnetDevice{ + { + Hostname: "hostname-node-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-0.example.ts.net", + }, + { + Hostname: "hostname-node-1", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-1.example.ts.net", + }, + { + Hostname: "hostname-node-2", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-2.example.ts.net", + }, + } + + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + if diff := cmp.Diff(got.Status.Devices, want); diff != "" { + t.Fatalf("unexpected devices in status (-got +want):\n%s", diff) + } + }) + + t.Run("scale_down_deletes_dangling_replica_secrets_and_devices", func(t *testing.T) { + tsr.Spec.Replicas = new(int32(1)) + reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) { + o.Spec = tsr.Spec + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Replicas 1 and 2 are gone from the cluster and from the tailnet; + // replica 0 is untouched. + for _, name := range []string{"test-1", "test-auth-1", "test-2", "test-auth-2"} { + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, name) + } + reconcilertest.MustGet(t, fc, tsNamespace, "test-0", &corev1.Secret{}) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } + + // The Role should no longer grant access to the scaled-down replicas' + // Secrets. + role := &rbacv1.Role{} + reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, role) + if diff := cmp.Diff(role.Rules[0].ResourceNames, []string{"test-0", "test-auth-0"}); diff != "" { + t.Fatalf("unexpected Role resource names (-got +want):\n%s", diff) + } + }) + + t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) { + reconcilertest.MustDeleteAll(t, fc, tsr) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name) + expectManagedCount(t, 0) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2", "node-0"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } + // The fake client does not clean up objects whose owner has been + // deleted, so we can't test for the owned resources getting deleted. + }) +} + +func TestRecorderTailnetUnavailable(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.RecorderSpec{EnableUI: true, Tailnet: "missing"}, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFailingClientProvider(errors.New(`tailnet "missing" not found`)), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + got := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, got) + reconcilertest.ExpectCondition(t, got.Status.Conditions, tsapi.RecorderReady, + metav1.ConditionFalse, "RecorderTailnetUnavailable", `tailnet "missing" not found`) + + // No resources should have been created for a Recorder we can't mint keys + // for, and no finalizer should have been added. + reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, tsNamespace, tsr.Name) + if len(got.Finalizers) > 0 { + t.Fatalf("expected no finalizers, got %v", got.Finalizers) + } +} + +func TestRecorderCleanupContinuesPastUnreportedDevice(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + DeletionTimestamp: new(metav1.Now()), + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + Storage: tsapi.Storage{S3: &tsapi.S3{}}, + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects( + tsr, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-0", Namespace: tsNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: tsNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Namespace: tsNamespace}}, + ). + WithStatusSubresource(tsr). + Build() + tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer)) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(tsClient), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + // Replica 0 registered with control but never reported its device ID. It comes first, so it must not end cleanup + // early. + mk := key.NewMachine() + mkText, err := mk.MarshalText() + if err != nil { + t.Fatal(err) + } + reconcilertest.MustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) { + s.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText} + }) + seedStateSecret(t, fc, "test-1", "node-1", "test-1.example.ts.net") + seedStateSecret(t, fc, "test-2", "node-2", "test-2.example.ts.net") + tsClient.SetDevices( + tailscaleclient.Device{NodeID: "unreported-0", MachineKey: mk.Public().String()}, + tailscaleclient.Device{NodeID: "other", MachineKey: key.NewMachine().Public().String()}, + ) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name) + if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"unreported-0", "node-1", "node-2"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } +} + +func TestRecorderStatefulSetSpec(t *testing.T) { + tsr := &tsapi.Recorder{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + EnableUI: true, + Storage: tsapi.Storage{ + S3: &tsapi.S3{ + Endpoint: "s3.example.com", + Bucket: "recordings", + Credentials: tsapi.S3Credentials{ + Secret: tsapi.S3Secret{Name: "s3-creds"}, + }, + }, + }, + StatefulSet: tsapi.RecorderStatefulSet{ + Labels: map[string]string{ + "ss-label-key": "ss-label-value", + }, + Annotations: map[string]string{ + "ss-annotation-key": "ss-annotation-value", + }, + Pod: tsapi.RecorderPod{ + Labels: map[string]string{ + "pod-label-key": "pod-label-value", + }, + Annotations: map[string]string{ + "pod-annotation-key": "pod-annotation-value", + }, + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "match-label": "match-value", + }, + }}, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: new(int64(1000)), + }, + ImagePullSecrets: []corev1.LocalObjectReference{{ + Name: "img-pull", + }}, + NodeSelector: map[string]string{ + "some-node": "selector", + }, + Tolerations: []corev1.Toleration{{ + Key: "key", + Value: "value", + TolerationSeconds: new(int64(60)), + }}, + ServiceAccount: tsapi.RecorderServiceAccount{ + Name: "custom-sa", + }, + Container: tsapi.RecorderContainer{ + Env: []tsapi.Env{{ + Name: "some_env", + Value: "env_value", + }}, + Image: "custom-image", + ImagePullPolicy: corev1.PullAlways, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + }, + }, + }, + }, + }, + }, + } + + fc := reconcilertest.NewClientBuilder(). + WithObjects(tsr). + WithStatusSubresource(tsr). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + r := recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: fc, + Recorder: record.NewFakeRecorder(1), + TailscaleNamespace: tsNamespace, + Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer))), + Logger: zl.Sugar(), + Clock: tstest.NewClock(tstest.ClockOpts{}), + }) + + reconcilertest.ExpectReconciled(t, r, "", tsr.Name) + + // Every field that the spec passes through ends up on the Recorder's children. + expectRecorderResources(t, fc, tsr, true) +} + +// expectRecorderResources asserts that the full set of resources for each of tsr's replicas either exists or is +// absent. The test builds each expected child itself, without the reconciler's spec builders, so a change to what the +// reconciler writes fails here. +func expectRecorderResources(t *testing.T, fc client.Client, tsr *tsapi.Recorder, shouldExist bool) { + t.Helper() + + // The owner references carry the UID of the stored Recorder. + stored := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", tsr.Name, stored) + stored.Spec = tsr.Spec + + role := wantRole(stored) + roleBinding := wantRoleBinding(stored) + serviceAccount := wantServiceAccount(stored) + statefulSet := wantStatefulSet(stored) + + if !shouldExist { + reconcilertest.ExpectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) + reconcilertest.ExpectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) + reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) + reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) + for replica := range replicaCount(stored) { + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, authSecretName(stored, replica)) + reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, stateSecretName(stored, replica)) + } + return + } + + reconcilertest.ExpectEqual(t, fc, role) + reconcilertest.ExpectEqual(t, fc, roleBinding) + reconcilertest.ExpectEqual(t, fc, serviceAccount) + reconcilertest.ExpectEqual(t, fc, statefulSet) + + for replica := range replicaCount(stored) { + auth := &corev1.Secret{} + reconcilertest.MustGet(t, fc, tsNamespace, authSecretName(stored, replica), auth) + if auth.StringData["authkey"] == "" { + t.Errorf("auth Secret %q has no auth key", auth.Name) + } + + // The fake Tailscale client generates each auth key, so leave the key out of the comparison. + reconcilertest.ExpectEqual(t, fc, wantSecret(stored, authSecretName(stored, replica)), func(s *corev1.Secret) { + s.StringData = nil + }) + reconcilertest.ExpectEqual(t, fc, wantSecret(stored, stateSecretName(stored, replica))) + } +} + +func replicaCount(tsr *tsapi.Recorder) int32 { + if tsr.Spec.Replicas == nil { + return 1 + } + return *tsr.Spec.Replicas +} + +func authSecretName(tsr *tsapi.Recorder, replica int32) string { + return fmt.Sprintf("%s-auth-%d", tsr.Name, replica) +} + +func stateSecretName(tsr *tsapi.Recorder, replica int32) string { + return fmt.Sprintf("%s-%d", tsr.Name, replica) +} + +func wantServiceAccountName(tsr *tsapi.Recorder) string { + if name := tsr.Spec.StatefulSet.Pod.ServiceAccount.Name; name != "" { + return name + } + return tsr.Name +} + +func wantLabels(tsr *tsapi.Recorder, custom map[string]string) map[string]string { + labels := maps.Clone(custom) + mak.Set(&labels, "app.kubernetes.io/name", "recorder") + mak.Set(&labels, "app.kubernetes.io/instance", tsr.Name) + mak.Set(&labels, "app.kubernetes.io/managed-by", "tailscale-operator") + return labels +} + +func wantObjectMeta(tsr *tsapi.Recorder, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, nil), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "tailscale.com/v1alpha1", + Kind: "Recorder", + Name: tsr.Name, + UID: tsr.UID, + Controller: new(true), + BlockOwnerDeletion: new(true), + }}, + } +} + +func wantSecret(tsr *tsapi.Recorder, name string) *corev1.Secret { + return &corev1.Secret{ObjectMeta: wantObjectMeta(tsr, name)} +} + +func wantServiceAccount(tsr *tsapi.Recorder) *corev1.ServiceAccount { + sa := &corev1.ServiceAccount{ObjectMeta: wantObjectMeta(tsr, wantServiceAccountName(tsr))} + sa.Annotations = tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations + return sa +} + +func wantRole(tsr *tsapi.Recorder) *rbacv1.Role { + var secrets []string + for replica := range replicaCount(tsr) { + secrets = append(secrets, stateSecretName(tsr, replica), authSecretName(tsr, replica)) + } + + return &rbacv1.Role{ + ObjectMeta: wantObjectMeta(tsr, tsr.Name), + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: secrets, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"get", "create", "patch"}, + }, + }, + } +} + +func wantRoleBinding(tsr *tsapi.Recorder) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: wantObjectMeta(tsr, tsr.Name), + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: wantServiceAccountName(tsr), + Namespace: tsNamespace, + }}, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: tsr.Name, + }, + } +} + +func wantStatefulSet(tsr *tsapi.Recorder) *appsv1.StatefulSet { + pod := tsr.Spec.StatefulSet.Pod + + image := pod.Container.Image + if image == "" { + meta := version.GetMeta() + prefix := "" + if meta.UnstableBranch { + prefix = "unstable-" + } + image = fmt.Sprintf("tailscale/tsrecorder:%sv%s", prefix, meta.MajorMinorPatch) + } + + env := []corev1.EnvVar{ + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, + {Name: "TS_AUTHKEY_FILE", Value: "/etc/tailscaled/$(POD_NAME)/authkey"}, + {Name: "TS_STATE", Value: "kube:$(POD_NAME)"}, + {Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)"}, + {Name: "TSRECORDER_LOGIN_SERVER", Value: tsLoginServer}, + } + for _, e := range pod.Container.Env { + env = append(env, corev1.EnvVar{Name: string(e.Name), Value: e.Value}) + } + + var envFrom []corev1.EnvFromSource + if s3 := tsr.Spec.Storage.S3; s3 != nil { + env = append(env, + corev1.EnvVar{Name: "TSRECORDER_DST", Value: "s3://" + s3.Endpoint}, + corev1.EnvVar{Name: "TSRECORDER_BUCKET", Value: s3.Bucket}, + ) + if s3.Credentials.Secret.Name != "" { + envFrom = []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: s3.Credentials.Secret.Name}, + }, + }} + } + } else { + env = append(env, corev1.EnvVar{Name: "TSRECORDER_DST", Value: "/data/recordings"}) + } + if tsr.Spec.EnableUI { + env = append(env, corev1.EnvVar{Name: "TSRECORDER_UI", Value: "true"}) + } + + mounts := []corev1.VolumeMount{{Name: "data", MountPath: "/data"}} + volumes := []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + for replica := range replicaCount(tsr) { + name := fmt.Sprintf("authkey-%d", replica) + mounts = append(mounts, corev1.VolumeMount{ + Name: name, + ReadOnly: true, + MountPath: "/etc/tailscaled/" + stateSecretName(tsr, replica), + }) + volumes = append(volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: authSecretName(tsr, replica), + Items: []corev1.KeyToPath{{Key: "authkey", Path: "authkey"}}, + }, + }, + }) + } + + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: tsr.Name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, tsr.Spec.StatefulSet.Labels), + Annotations: tsr.Spec.StatefulSet.Annotations, + OwnerReferences: wantObjectMeta(tsr, tsr.Name).OwnerReferences, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: new(replicaCount(tsr)), + Selector: &metav1.LabelSelector{MatchLabels: wantLabels(tsr, pod.Labels)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: tsr.Name, + Namespace: tsNamespace, + Labels: wantLabels(tsr, pod.Labels), + Annotations: pod.Annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: wantServiceAccountName(tsr), + Affinity: pod.Affinity, + SecurityContext: pod.SecurityContext, + ImagePullSecrets: pod.ImagePullSecrets, + NodeSelector: pod.NodeSelector, + Tolerations: pod.Tolerations, + Containers: []corev1.Container{{ + Name: "recorder", + Image: image, + ImagePullPolicy: pod.Container.ImagePullPolicy, + Resources: pod.Container.Resources, + SecurityContext: pod.Container.SecurityContext, + Env: env, + EnvFrom: envFrom, + Command: []string{"/tsrecorder"}, + VolumeMounts: mounts, + }}, + Volumes: volumes, + }, + }, + }, + } +} + +// expectReadyCondition asserts the Recorder's RecorderReady condition. +func expectReadyCondition(t *testing.T, fc client.Client, name string, status metav1.ConditionStatus, reason, message string) { + t.Helper() + tsr := &tsapi.Recorder{} + reconcilertest.MustGet(t, fc, "", name, tsr) + reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, status, reason, message) +} + +// expectManagedCount asserts the number of Recorders the reconciler reports as managed via its clientmetric gauge. +func expectManagedCount(t *testing.T, want int64) { + t.Helper() + for _, m := range clientmetric.Metrics() { + if m.Name() == kubetypes.MetricRecorderCount { + if got := m.Value(); got != want { + t.Fatalf("expected %d recorders, got %d", want, got) + } + return + } + } + t.Fatalf("metric %q not found", kubetypes.MetricRecorderCount) +} + +// seedStateSecret writes the profile data that tailscaled would have written to a replica's state Secret once it has +// authenticated. +func seedStateSecret(t *testing.T, fc client.Client, name, nodeID, loginName string) { + t.Helper() + const key = "profile-abc" + + body, err := json.Marshal(map[string]any{ + "Config": map[string]any{ + "NodeID": nodeID, + "UserProfile": map[string]any{ + "LoginName": loginName, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + reconcilertest.MustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + string(ipn.CurrentProfileStateKey): []byte(key), + key: body, + } + }) +} + +func envValue(ss *appsv1.StatefulSet, name string) string { + for _, env := range ss.Spec.Template.Spec.Containers[0].Env { + if env.Name == name { + return env.Value + } + } + return "" +} diff --git a/cmd/k8s-operator/tsrecorder_specs.go b/k8s-operator/reconciler/recorder/specs.go similarity index 91% rename from cmd/k8s-operator/tsrecorder_specs.go rename to k8s-operator/reconciler/recorder/specs.go index 73ad5f431..d92e7e4e0 100644 --- a/cmd/k8s-operator/tsrecorder_specs.go +++ b/k8s-operator/reconciler/recorder/specs.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package recorder import ( "fmt" @@ -115,7 +115,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) * Name: volumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + SecretName: authSecretName(tsr.Name, replica), Items: []corev1.KeyToPath{{Key: authKeySecretKey, Path: authKeySecretKey}}, }, }, @@ -156,8 +156,8 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role { resourceNames := make([]string, 0) for replica := range replicas { resourceNames = append(resourceNames, - fmt.Sprintf("%s-%d", tsr.Name, replica), // State secret. - fmt.Sprintf("%s-auth-%d", tsr.Name, replica), // Auth key secret. + stateSecretName(tsr.Name, replica), // State secret. + authSecretName(tsr.Name, replica), // Auth key secret. ) } @@ -218,7 +218,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + Name: authSecretName(tsr.Name, replica), Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, @@ -231,7 +231,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", tsr.Name, replica), + Name: stateSecretName(tsr.Name, replica), Namespace: namespace, Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), @@ -239,6 +239,17 @@ func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev } } +// stateSecretName returns the name of the tailscaled state Secret for the given Recorder replica. It matches the +// replica's Pod name, which is what tsrecorder reads from TS_STATE. +func stateSecretName(tsrName string, replica int32) string { + return fmt.Sprintf("%s-%d", tsrName, replica) +} + +// authSecretName returns the name of the Secret holding the auth key for the given Recorder replica. +func authSecretName(tsrName string, replica int32) string { + return fmt.Sprintf("%s-auth-%d", tsrName, replica) +} + func tsrEnv(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar { envs := []corev1.EnvVar{ { diff --git a/k8s-operator/reconciler/tailnet/tailnet_test.go b/k8s-operator/reconciler/tailnet/tailnet_test.go index 36a7ea774..23e5445ef 100644 --- a/k8s-operator/reconciler/tailnet/tailnet_test.go +++ b/k8s-operator/reconciler/tailnet/tailnet_test.go @@ -13,10 +13,10 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/tsclient" "tailscale.com/tstest" @@ -374,7 +374,7 @@ func TestReconciler_Reconcile(t *testing.T) { for _, tc := range tt { t.Run(tc.Name, func(t *testing.T) { - builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) + builder := reconcilertest.NewClientBuilder() if tc.Tailnet != nil { builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet) } diff --git a/k8s-operator/reconciler/tailscaled/prefs.go b/k8s-operator/reconciler/tailscaled/prefs.go new file mode 100644 index 000000000..0ed72327f --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/prefs.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + + "tailscale.com/ipn" + "tailscale.com/tailcfg" +) + +// Prefs is the subset of ipn.Prefs that operator reconcilers need to read back out of a tailscaled state Secret. +// tailscaled writes the full prefs for the current profile; only the fields declared here are unmarshalled. +type Prefs struct { + Config struct { + NodeID tailcfg.StableNodeID `json:"NodeID"` + UserProfile struct { + // LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net. + LoginName string `json:"LoginName"` + } `json:"UserProfile"` + } `json:"Config"` + + AdvertiseServices []string `json:"AdvertiseServices"` +} + +// PrefsFromStateSecret returns the prefs tailscaled recorded for its current profile in secret. ok reports whether +// a node ID was found; it is false when the device hasn't finished authenticating yet, in which case prefs is not +// meaningful. LoginName is expected to be set whenever the node ID is, but that is not required. +// +// TODO(tomhjp): Should maybe use ipn to parse the following info instead. +func PrefsFromStateSecret(secret *corev1.Secret) (prefs Prefs, ok bool, err error) { + currentProfile, ok := secret.Data[string(ipn.CurrentProfileStateKey)] + if !ok { + return prefs, false, nil + } + profileBytes, ok := secret.Data[string(currentProfile)] + if !ok { + return prefs, false, nil + } + if err := json.Unmarshal(profileBytes, &prefs); err != nil { + return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err) + } + + ok = prefs.Config.NodeID != "" + return prefs, ok, nil +} diff --git a/k8s-operator/reconciler/tailscaled/prefs_test.go b/k8s-operator/reconciler/tailscaled/prefs_test.go new file mode 100644 index 000000000..bd713acd5 --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/prefs_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "tailscale.com/k8s-operator/reconciler/tailscaled" +) + +func TestPrefsFromStateSecret(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + data map[string][]byte + wantOK bool + wantErr bool + wantNodeID string + wantLogin string + wantSvcs []string + }{ + "populated": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`{"Config":{"NodeID":"node-foo","UserProfile":{"LoginName":"foo.example.ts.net"}},"AdvertiseServices":["svc:bar"]}`), + }, + wantOK: true, + wantNodeID: "node-foo", + wantLogin: "foo.example.ts.net", + wantSvcs: []string{"svc:bar"}, + }, + "no-current-profile": { + data: map[string][]byte{ + "profile-abc": []byte(`{"Config":{"NodeID":"node-foo"}}`), + }, + }, + "current-profile-points-at-missing-profile": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + }, + }, + // The device hasn't finished authenticating, so there's no node ID yet. + "no-node-id": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`{"Config":{"UserProfile":{"LoginName":"foo.example.ts.net"}}}`), + }, + wantLogin: "foo.example.ts.net", + }, + "empty": {}, + "malformed-profile": { + data: map[string][]byte{ + "_current-profile": []byte("profile-abc"), + "profile-abc": []byte(`not json`), + }, + wantErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "tailscale"}, + Data: tc.data, + } + + prefs, ok, err := tailscaled.PrefsFromStateSecret(secret) + switch { + case tc.wantErr && err == nil: + t.Fatal("expected an error, got nil") + case !tc.wantErr && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.wantErr: + return + } + + if ok != tc.wantOK { + t.Errorf("ok = %v, want %v", ok, tc.wantOK) + } + if got := string(prefs.Config.NodeID); got != tc.wantNodeID { + t.Errorf("NodeID = %q, want %q", got, tc.wantNodeID) + } + if got := prefs.Config.UserProfile.LoginName; got != tc.wantLogin { + t.Errorf("LoginName = %q, want %q", got, tc.wantLogin) + } + if got, want := len(prefs.AdvertiseServices), len(tc.wantSvcs); got != want { + t.Fatalf("len(AdvertiseServices) = %d, want %d", got, want) + } + for i, svc := range tc.wantSvcs { + if prefs.AdvertiseServices[i] != svc { + t.Errorf("AdvertiseServices[%d] = %q, want %q", i, prefs.AdvertiseServices[i], svc) + } + } + }) + } +} diff --git a/k8s-operator/reconciler/tailscaled/reissuer_test.go b/k8s-operator/reconciler/tailscaled/reissuer_test.go index aebd34d38..ead8fe796 100644 --- a/k8s-operator/reconciler/tailscaled/reissuer_test.go +++ b/k8s-operator/reconciler/tailscaled/reissuer_test.go @@ -6,18 +6,15 @@ package tailscaled_test import ( - "context" "slices" "strings" - "sync" "testing" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" - tailscaleclient "tailscale.com/client/tailscale/v2" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/k8s-operator/reconciler/tailscaled" - "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" ) @@ -72,7 +69,7 @@ func TestReissuer_ShouldReissue(t *testing.T) { stateSecret = &corev1.Secret{Data: tc.stateData} } - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -89,8 +86,8 @@ func TestReissuer_ShouldReissue(t *testing.T) { if got != tc.wantReissue { t.Errorf("shouldReissue = %v, want %v", got, tc.wantReissue) } - if !slices.Equal(tsc.deleted, tc.wantDeleted) { - t.Errorf("deleted devices = %v, want %v", tsc.deleted, tc.wantDeleted) + if !slices.Equal(tsc.DeviceDeletes(), tc.wantDeleted) { + t.Errorf("deleted devices = %v, want %v", tsc.DeviceDeletes(), tc.wantDeleted) } }) } @@ -101,7 +98,7 @@ func TestReissuer_ShouldReissue(t *testing.T) { // state Secret. func TestReissuer_InFlight(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -137,7 +134,7 @@ func TestReissuer_InFlight(t *testing.T) { // across replicas, blocks re-issuance once its burst is exhausted. func TestReissuer_RateLimit(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() // Burst of 1: the first replica's reissue consumes the only token. r.EnsureState("pg", 1) @@ -169,7 +166,7 @@ func TestReissuer_RateLimit(t *testing.T) { // EnsureState reissues instead of panicking on a nil limiter. func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() broken := &corev1.Secret{Data: map[string][]byte{ @@ -183,8 +180,8 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { if err != nil || !got { t.Fatalf("got %v, err %v; want true, nil", got, err) } - if !slices.Equal(tsc.deleted, []string{"nodeid-0"}) { - t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.deleted) + if !slices.Equal(tsc.DeviceDeletes(), []string{"nodeid-0"}) { + t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.DeviceDeletes()) } } @@ -192,7 +189,7 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) { // is cleared, so a later reissue request for the same replica triggers afresh. func TestReissuer_RemoveState(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 1) @@ -226,7 +223,7 @@ func TestReissuer_RemoveState(t *testing.T) { // parent scaled down then deleted doesn't strand in-flight entries. func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) { key := "broken-key" - tsc := &fakeReissueClient{} + tsc := reconcilertest.NewFakeClient() r := tailscaled.NewReissuer() r.EnsureState("pg", 2) @@ -260,30 +257,3 @@ func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) { t.Fatalf("pg-1 after RemoveState: got %v, err %v; want true, nil", got, err) } } - -type fakeReissueClient struct { - tsclient.Client - - mu sync.Mutex - deleted []string -} - -func (c *fakeReissueClient) Devices() tsclient.DeviceResource { return (*fakeReissueDevices)(c) } - -type fakeReissueDevices fakeReissueClient - -func (d *fakeReissueDevices) Delete(_ context.Context, id string) error { - c := (*fakeReissueClient)(d) - c.mu.Lock() - defer c.mu.Unlock() - c.deleted = append(c.deleted, id) - return nil -} - -func (d *fakeReissueDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { - return nil, nil -} - -func (d *fakeReissueDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) { - return nil, nil -}