cmd/k8s-operator: move Recorder reconciler into its own package (#20891)

Move the Recorder reconciler and its object specs into
k8s-operator/reconciler/recorder, following the earlier ProxyClass,
DNSConfig and nameserver moves, and switch it to the shared finalizer,
resource tracker and CreateOrUpdate helpers.

Hoist two duplicated helpers into tailscaled.PrefsFromStateSecret and
reconciler.IsOptimisticLockError, and add
k8s-operator/reconciler/reconcilertest for the test helpers each package
had copied, including one fake Tailscale API client in place of four.

No behaviour change; the Recorder's watches and labels are untouched.

Fixes tailscale/corp#37086

Signed-off-by: David Bond <[email protected]>
This commit is contained in:
David Bond authored and GitHub committed 2026-11-05 08:35:13 +00:00
1 parent dc69869769
commit 2993b5e6de
37 files changed
+1868 -1144

No files matched your search

+3 -3
View File
@@ -12,7 +12,6 @@ import (
"fmt"
"maps"
"slices"
"strings"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
@@ -27,6 +26,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
@@ -85,7 +85,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
if markedForDeletion(pg) {
logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up")
if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) {
if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return res, nil
}
@@ -95,7 +95,7 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
+2 -1
View File
@@ -20,6 +20,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
@@ -199,7 +200,7 @@ func TestAPIServerProxyReconciler(t *testing.T) {
// Simulate Pod prefs updated with advertised services; should see Configured condition updated to true.
mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) {
var p prefs
var p tailscaled.Prefs
if err = json.Unmarshal(o.Data["test"], &p); err != nil {
t.Errorf("failed to unmarshal preferences: %v", err)
}
+2 -2
View File
@@ -11,7 +11,6 @@ import (
"fmt"
"net/netip"
"slices"
"strings"
"sync"
"time"
@@ -28,6 +27,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/netutil"
"tailscale.com/net/tsaddr"
@@ -146,7 +146,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil {
reason := reasonConnectorCreationFailed
message := fmt.Sprintf(messageConnectorCreationFailed, err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
reason = reasonConnectorCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
err = nil
+1
View File
@@ -774,6 +774,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/recorder from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/staticendpoints from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/tailscaled from tailscale.com/k8s-operator/reconciler/peerrelay+
+2 -1
View File
@@ -34,6 +34,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstime"
@@ -161,7 +162,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
}
if err := esr.maybeProvision(ctx, svc, lg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
lg.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
+2 -1
View File
@@ -34,6 +34,7 @@ import (
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
@@ -851,7 +852,7 @@ func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, p
var count int
for _, secret := range secrets.Items {
prefs, ok, err := getDevicePrefs(&secret)
prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret)
if err != nil {
return 0, fmt.Errorf("error getting node metadata: %w", err)
}
+7 -6
View File
@@ -30,6 +30,7 @@ import (
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
@@ -563,7 +564,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
// Add the Tailscale Service to prefs to have the Ingress recognised as ready.
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) {
var p prefs
var p tailscaled.Prefs
var err error
if err = json.Unmarshal(o.Data["test"], &p); err != nil {
t.Errorf("failed to unmarshal preferences: %v", err)
@@ -689,7 +690,7 @@ func TestIngressPGReconciler_HTTPRedirect(t *testing.T) {
// Add the Tailscale Service to prefs to have the Ingress recognised as ready.
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) {
var p prefs
var p tailscaled.Prefs
var err error
if err = json.Unmarshal(o.Data["test"], &p); err != nil {
t.Errorf("failed to unmarshal preferences: %v", err)
@@ -824,7 +825,7 @@ func TestIngressPGReconciler_HTTPEndpointAndRedirectConflict(t *testing.T) {
// Add the Tailscale Service to prefs to have the Ingress recognised as ready.
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) {
var p prefs
var p tailscaled.Prefs
var err error
if err = json.Unmarshal(o.Data["test"], &p); err != nil {
t.Errorf("failed to unmarshal preferences: %v", err)
@@ -1151,7 +1152,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) {
}
mustCreate(t, fc, pgCfgSecret)
pr := prefs{}
pr := tailscaled.Prefs{}
pr.Config.UserProfile.LoginName = "test.ts.net"
pr.Config.NodeID = "test"
@@ -1168,8 +1169,8 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) {
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
},
Data: map[string][]byte{
currentProfileKey: []byte("test"),
"test": p,
string(ipn.CurrentProfileStateKey): []byte("test"),
"test": p,
},
}
mustCreate(t, fc, pgStateSecret)
+2 -1
View File
@@ -25,6 +25,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/ipn"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/opt"
"tailscale.com/util/clientmetric"
@@ -82,7 +83,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
if err := a.maybeProvision(ctx, logger, ing); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
+11 -20
View File
@@ -58,6 +58,7 @@ import (
"tailscale.com/k8s-operator/reconciler/peerrelay"
"tailscale.com/k8s-operator/reconciler/proxyclass"
"tailscale.com/k8s-operator/reconciler/proxygrouppolicy"
"tailscale.com/k8s-operator/reconciler/recorder"
"tailscale.com/k8s-operator/reconciler/tailnet"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
@@ -674,26 +675,16 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create DNS records reconciler: %v", err)
}
// Recorder reconciler.
recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
err = builder.ControllerManagedBy(mgr).
For(&tsapi.Recorder{}).
Named("recorder-reconciler").
Watches(&appsv1.StatefulSet{}, recorderFilter).
Watches(&corev1.ServiceAccount{}, recorderFilter).
Watches(&corev1.Secret{}, recorderFilter).
Watches(&rbacv1.Role{}, recorderFilter).
Watches(&rbacv1.RoleBinding{}, recorderFilter).
Complete(&RecorderReconciler{
recorder: eventRecorder,
tsNamespace: opts.tailscaleNamespace,
Client: mgr.GetClient(),
log: opts.log.Named("recorder-reconciler"),
clock: tstime.DefaultClock{},
clients: clients,
reissuer: tailscaled.NewReissuer(),
})
if err != nil {
recorderOptions := recorder.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
Clients: clients,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}
if err = recorder.NewReconciler(recorderOptions).Register(mgr); err != nil {
startlog.Fatalf("could not create Recorder reconciler: %v", err)
}
+5 -7
View File
@@ -34,6 +34,7 @@ import (
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/staticendpoints"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
@@ -57,9 +58,6 @@ const (
reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable"
reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion"
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
// The minimum tailcfg.CapabilityVersion that deployed clients are expected
// to support to be compatible with the current ProxyGroup controller.
// If the controller needs to depend on newer client behaviour, it should
@@ -150,7 +148,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
@@ -475,7 +473,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
defer func() {
if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) {
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(updateErr) {
logger.Infof("optimistic lock error updating status, retrying: %s", updateErr)
updateErr = nil
}
@@ -992,7 +990,7 @@ func getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup, cl client.Client
stateSecret: &secret,
}
prefs, ok, err := getDevicePrefs(&secret)
prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret)
if err != nil {
return nil, err
}
@@ -1101,7 +1099,7 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool {
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
err := fmt.Errorf(format, a...)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error())
logger.Info(msg)
return notReady(reasonProxyGroupCreating, msg)
+4 -4
View File
@@ -2296,10 +2296,10 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
}
mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) {
s.Data = map[string][]byte{
currentProfileKey: []byte(key),
key: bytes,
kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`),
kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i),
string(ipn.CurrentProfileStateKey): []byte(key),
key: bytes,
kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`),
kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i),
// TODO(tomhjp): We have two different mechanisms to retrieve device IDs.
// Consolidate on this one.
kubetypes.KeyDeviceID: fmt.Appendf(nil, "nodeid-%d", i),
+6
View File
@@ -1524,6 +1524,12 @@ func nameForService(svc *corev1.Service) string {
return svc.Namespace + "-" + svc.Name
}
// markedForDeletion reports whether obj has a deletion timestamp, i.e. the API server is waiting on finalizers
// before it can garbage collect it.
func markedForDeletion(obj metav1.Object) bool {
return !obj.GetDeletionTimestamp().IsZero()
}
// proxyClassForObject returns the proxy class for the given object. If the
// object does not have a proxy class label, it returns the default proxy class
func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+4 -2
View File
@@ -31,6 +31,8 @@ import (
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubetypes"
@@ -188,7 +190,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque
needsRequeue := false
needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
@@ -762,7 +764,7 @@ func (r *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName
var count int
for _, secret := range secrets.Items {
prefs, ok, err := getDevicePrefs(&secret)
prefs, ok, err := tailscaled.PrefsFromStateSecret(&secret)
if err != nil {
return 0, fmt.Errorf("error getting node metadata: %w", err)
}
+2 -1
View File
@@ -26,6 +26,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/dns/resolvconffile"
"tailscale.com/tstime"
@@ -123,7 +124,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
if err := a.maybeProvision(ctx, logger, svc); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
+3
View File
@@ -45,6 +45,9 @@ import (
const (
vipTestIP = "5.6.7.8"
// tsNamespace is the namespace the operator is installed in for tests that don't override it.
tsNamespace = "tailscale"
)
// confgOpts contains configuration options for creating cluster resources for
-156
View File
@@ -1,156 +0,0 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"testing"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
)
func TestRecorderSpecs(t *testing.T) {
t.Run("spec-fields-passthrough", func(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.RecorderSpec{
Replicas: new(int32(3)),
StatefulSet: tsapi.RecorderStatefulSet{
Labels: map[string]string{
"ss-label-key": "ss-label-value",
},
Annotations: map[string]string{
"ss-annotation-key": "ss-annotation-value",
},
Pod: tsapi.RecorderPod{
Labels: map[string]string{
"pod-label-key": "pod-label-value",
},
Annotations: map[string]string{
"pod-annotation-key": "pod-annotation-value",
},
Affinity: &corev1.Affinity{
PodAffinity: &corev1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"match-label": "match-value",
},
}},
},
},
},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: new(int64(1000)),
},
ImagePullSecrets: []corev1.LocalObjectReference{{
Name: "img-pull",
}},
NodeSelector: map[string]string{
"some-node": "selector",
},
Tolerations: []corev1.Toleration{{
Key: "key",
Value: "value",
TolerationSeconds: new(int64(60)),
}},
Container: tsapi.RecorderContainer{
Env: []tsapi.Env{{
Name: "some_env",
Value: "env_value",
}},
Image: "custom-image",
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
},
},
},
},
},
},
}
ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer)
// StatefulSet-level.
if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Annotations, tsr.Spec.StatefulSet.Pod.Annotations); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
// Pod-level.
if diff := cmp.Diff(ss.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.SecurityContext, tsr.Spec.StatefulSet.Pod.SecurityContext); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.ImagePullSecrets, tsr.Spec.StatefulSet.Pod.ImagePullSecrets); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.NodeSelector, tsr.Spec.StatefulSet.Pod.NodeSelector); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Tolerations, tsr.Spec.StatefulSet.Pod.Tolerations); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
// Container-level.
if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, tsrEnv(tsr, tsLoginServer)); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].ImagePullPolicy, tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].SecurityContext, tsr.Spec.StatefulSet.Pod.Container.SecurityContext); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" {
t.Errorf("(-got +want):\n%s", diff)
}
if *ss.Spec.Replicas != *tsr.Spec.Replicas {
t.Errorf("expected %d replicas, got %d", *tsr.Spec.Replicas, *ss.Spec.Replicas)
}
if len(ss.Spec.Template.Spec.Volumes) != int(*tsr.Spec.Replicas)+1 {
t.Errorf("expected %d volumes, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Volumes))
}
if len(ss.Spec.Template.Spec.Containers[0].VolumeMounts) != int(*tsr.Spec.Replicas)+1 {
t.Errorf("expected %d volume mounts, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Containers[0].VolumeMounts))
}
})
}
-321
View File
@@ -1,321 +0,0 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/client/tailscale/v2"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/tstest"
)
const (
tsNamespace = "tailscale"
tsLoginServer = "example.tailscale.com"
)
func TestRecorder(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
},
Spec: tsapi.RecorderSpec{
Replicas: new(int32(3)),
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(tsr).
WithStatusSubresource(tsr).
Build()
tsClient := &fakeTSClient{loginURL: tsLoginServer}
zl, _ := zap.NewDevelopment()
fr := record.NewFakeRecorder(2)
cl := tstest.NewClock(tstest.ClockOpts{})
reconciler := &RecorderReconciler{
tsNamespace: tsNamespace,
Client: fc,
clients: tsclient.NewProvider(tsClient),
recorder: fr,
log: zl.Sugar(),
clock: cl,
reissuer: tailscaled.NewReissuer(),
}
t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) {
expectReconciled(t, reconciler, "", tsr.Name)
msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, tsr)
if expected := 0; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
expectRecorderResources(t, fc, tsr, false)
expectedEvent := "Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
expectEvents(t, fr, []string{expectedEvent})
tsr.Spec.EnableUI = true
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{
"invalid space characters": "test",
}
mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
t.Spec = tsr.Spec
})
expectReconciled(t, reconciler, "", tsr.Name)
expectedEvent = "Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible"
expectEvents(t, fr, []string{expectedEvent})
tsr.Spec.Storage.S3 = &tsapi.S3{}
mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
t.Spec = tsr.Spec
})
expectReconciled(t, reconciler, "", tsr.Name)
// Only check part of this error message, because it's defined in an
// external package and may change.
if err := fc.Get(context.Background(), client.ObjectKey{
Name: tsr.Name,
}, tsr); err != nil {
t.Fatal(err)
}
if len(tsr.Status.Conditions) != 1 {
t.Fatalf("expected 1 condition, got %d", len(tsr.Status.Conditions))
}
cond := tsr.Status.Conditions[0]
if cond.Type != string(tsapi.RecorderReady) || cond.Status != metav1.ConditionFalse || cond.Reason != reasonRecorderInvalid {
t.Fatalf("expected condition RecorderReady false due to RecorderInvalid, got %v", cond)
}
for _, msg := range []string{cond.Message, <-fr.Events} {
if !strings.Contains(msg, `"invalid space characters"`) {
t.Fatalf("expected invalid annotation key in error message, got %q", cond.Message)
}
}
})
t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) {
mustCreate(t, fc, &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "pre-existing-sa",
Namespace: tsNamespace,
},
})
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa"
mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
t.Spec = tsr.Spec
})
expectReconciled(t, reconciler, "", tsr.Name)
msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace`
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, tsr)
if expected := 0; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
expectedEvent := "Warning RecorderInvalid " + msg
expectEvents(t, fr, []string{expectedEvent})
})
t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) {
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = ""
mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
t.Spec = tsr.Spec
})
expectReconciled(t, reconciler, "", tsr.Name)
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar())
expectEqual(t, fc, tsr)
if expected := 1; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
expectRecorderResources(t, fc, tsr, true)
})
t.Run("valid_service_account_config", func(t *testing.T) {
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa"
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{
"test": "test",
}
mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) {
t.Spec = tsr.Spec
})
expectReconciled(t, reconciler, "", tsr.Name)
expectEqual(t, fc, tsr)
if expected := 1; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
expectRecorderResources(t, fc, tsr, true)
// Get the service account and check the annotations.
sa := &corev1.ServiceAccount{}
if err := fc.Get(context.Background(), client.ObjectKey{
Name: tsr.Spec.StatefulSet.Pod.ServiceAccount.Name,
Namespace: tsNamespace,
}, sa); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" {
t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff)
}
if sa.Name != tsr.Spec.StatefulSet.Pod.ServiceAccount.Name {
t.Fatalf("unexpected service account name: got %q, want %q", sa.Name, tsr.Spec.StatefulSet.Pod.ServiceAccount.Name)
}
expectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name)
})
t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) {
const key = "profile-abc"
for replica := range *tsr.Spec.Replicas {
bytes, err := json.Marshal(map[string]any{
"Config": map[string]any{
"NodeID": fmt.Sprintf("node-%d", replica),
"UserProfile": map[string]any{
"LoginName": fmt.Sprintf("test-%d.example.ts.net", replica),
},
},
})
if err != nil {
t.Fatal(err)
}
name := fmt.Sprintf("%s-%d", "test", replica)
mustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) {
s.Data = map[string][]byte{
currentProfileKey: []byte(key),
key: bytes,
}
})
}
tsClient.devices = []tailscale.Device{
{
ID: "node-0",
Hostname: "hostname-node-0",
Addresses: []string{"1.2.3.4", "::1"},
},
{
ID: "node-1",
Hostname: "hostname-node-1",
Addresses: []string{"1.2.3.4", "::1"},
},
{
ID: "node-2",
Hostname: "hostname-node-2",
Addresses: []string{"1.2.3.4", "::1"},
},
}
expectReconciled(t, reconciler, "", tsr.Name)
tsr.Status.Devices = []tsapi.RecorderTailnetDevice{
{
Hostname: "hostname-node-0",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-0.example.ts.net",
},
{
Hostname: "hostname-node-1",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-1.example.ts.net",
},
{
Hostname: "hostname-node-2",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-2.example.ts.net",
},
}
expectEqual(t, fc, tsr)
})
t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) {
// Replica 0 comes first, so an unreported device there must not end cleanup early.
setUnreportedDevice(t, fc, tsClient, "test-0", "unreported-0")
if err := fc.Delete(context.Background(), tsr); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", tsr.Name)
expectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
if expected := 0; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
if diff := cmp.Diff(tsClient.deleted, []string{"unreported-0", "node-1", "node-2"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
// The fake client does not clean up objects whose owner has been
// deleted, so we can't test for the owned resources getting deleted.
})
}
func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) {
t.Helper()
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
}
role := tsrRole(tsr, tsNamespace)
roleBinding := tsrRoleBinding(tsr, tsNamespace)
serviceAccount := tsrServiceAccount(tsr, tsNamespace)
statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer)
if shouldExist {
expectEqual(t, fc, role)
expectEqual(t, fc, roleBinding)
expectEqual(t, fc, serviceAccount)
expectEqual(t, fc, statefulSet, removeResourceReqs)
} else {
expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
}
for replica := range replicas {
auth := tsrAuthSecret(tsr, tsNamespace, "new-authkey", replica)
state := tsrStateSecret(tsr, tsNamespace, replica)
if shouldExist {
expectEqual(t, fc, auth)
expectEqual(t, fc, state)
} else {
expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name)
expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name)
}
}
}
@@ -46,8 +46,7 @@ const (
serviceTypeIngress = "ingress"
serviceTypeSvc = "svc"
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
shortRequeue = 5 * time.Second
shortRequeue = 5 * time.Second
// AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN.
AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
@@ -157,7 +156,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
}
if err := r.maybeProvision(ctx, proxySvc, logger); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
} else {
@@ -21,7 +21,6 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -30,6 +29,7 @@ import (
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/dnsrecords"
"tailscale.com/k8s-operator/reconciler/nameserver"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
)
@@ -70,8 +70,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
},
}
cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().
WithObjects(cm).
WithObjects(dnsConfig).
WithObjects(ing).
@@ -83,7 +82,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
}
cl := tstest.NewClock(tstest.ClockOpts{})
// Set the ready condition of the DNSConfig
mustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) {
reconcilertest.MustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) {
operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar())
})
dnsRR := dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{
@@ -109,11 +108,11 @@ func TestDNSRecordsReconciler(t *testing.T) {
ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7", discoveryv1.AddressTypeIPv4)
epv6 := endpointSliceForService(headlessForEgressSvcFQDN, "2600:1900:4011:161:0:d:0:d", discoveryv1.AddressTypeIPv6)
mustCreate(t, fc, egressSvcFQDN)
mustCreate(t, fc, headlessForEgressSvcFQDN)
mustCreate(t, fc, ep)
mustCreate(t, fc, epv6)
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
reconcilertest.MustCreate(t, fc, egressSvcFQDN)
reconcilertest.MustCreate(t, fc, headlessForEgressSvcFQDN)
reconcilertest.MustCreate(t, fc, ep)
reconcilertest.MustCreate(t, fc, epv6)
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
wantHostsIPv6 := map[string][]string{"foo.bar.ts.net": {"2600:1900:4011:161:0:d:0:d"}}
@@ -121,57 +120,57 @@ func TestDNSRecordsReconciler(t *testing.T) {
// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
// value changes
mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) {
svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net"
})
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}}
expectHostsRecords(t, fc, wantHosts)
// 3. DNS record is updated if the IP address of the proxy Pod changes.
ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4", discoveryv1.AddressTypeIPv4)
mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
})
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}}
expectHostsRecords(t, fc, wantHosts)
// 4. DNS record is created for an ingress proxy configured via Ingress
headlessForIngress := headlessSvcForParent(ing, "ingress")
ep = endpointSliceForService(headlessForIngress, "10.9.8.7", discoveryv1.AddressTypeIPv4)
mustCreate(t, fc, headlessForIngress)
mustCreate(t, fc, ep)
expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
reconcilertest.MustCreate(t, fc, headlessForIngress)
reconcilertest.MustCreate(t, fc, ep)
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"}
expectHostsRecords(t, fc, wantHosts)
// 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0])
t.Log("test case 5")
mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) {
reconcilertest.MustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) {
ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net"
})
expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
delete(wantHosts, "cluster.ingress.ts.net")
wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"}
expectHostsRecords(t, fc, wantHosts)
// 6. DNS records are updated if Ingress proxy's Pod IP changes
mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
ep.Endpoints[0].Addresses = []string{"7.8.9.10"}
})
expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress")
wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
expectHostsRecords(t, fc, wantHosts)
// 7. A not-ready Endpoint is removed from DNS config.
mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
ep.Endpoints[0].Conditions.Ready = new(false)
ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{
Addresses: []string{"1.2.3.4"},
})
})
expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-ingress")
wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"}
expectHostsRecords(t, fc, wantHosts)
@@ -192,7 +191,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
ExternalName: "unused",
},
}
mustCreate(t, fc, parentEgressSvc)
reconcilertest.MustCreate(t, fc, parentEgressSvc)
proxyGroupEgressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -246,9 +245,9 @@ func TestDNSRecordsReconciler(t *testing.T) {
}},
}
mustCreate(t, fc, proxyGroupEgressSvc)
mustCreate(t, fc, proxyGroupEps)
expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
reconcilertest.MustCreate(t, fc, proxyGroupEgressSvc)
reconcilertest.MustCreate(t, fc, proxyGroupEps)
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
// Verify DNS record uses ClusterIP Service IP, not Pod IPs
wantHosts["external-service.example.ts.net"] = []string{"10.0.100.50"}
@@ -256,17 +255,17 @@ func TestDNSRecordsReconciler(t *testing.T) {
// 9. ProxyGroup egress DNS record updates when ClusterIP changes
t.Log("test case 9: ProxyGroup egress ClusterIP change")
mustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) {
svc.Spec.ClusterIP = "10.0.100.51"
})
expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
wantHosts["external-service.example.ts.net"] = []string{"10.0.100.51"}
expectHostsRecords(t, fc, wantHosts)
// 10. Test ProxyGroup service deletion and DNS cleanup
t.Log("test case 10: ProxyGroup egress service deletion")
mustDeleteAll(t, fc, proxyGroupEgressSvc)
expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
reconcilertest.MustDeleteAll(t, fc, proxyGroupEgressSvc)
reconcilertest.ExpectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1")
delete(wantHosts, "external-service.example.ts.net")
expectHostsRecords(t, fc, wantHosts)
}
@@ -322,9 +321,8 @@ func TestDNSRecordsReconcilerOptimisticLockError(t *testing.T) {
},
}
f := fake.NewClientBuilder().
f := reconcilertest.NewClientBuilder().
WithInterceptorFuncs(funcs).
WithScheme(tsapi.GlobalScheme).
WithObjects(dnsCfg, proxyGroupEgressSvc, egressSvc).
WithStatusSubresource(dnsCfg).
Build()
@@ -410,8 +408,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) {
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().
WithObjects(dnsCfg, ing, headlessSvc, epv4, epv6, cm).
WithStatusSubresource(dnsCfg).
Build()
@@ -423,7 +420,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) {
})
// Test dual-stack service records
expectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress")
reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress")
wantIPv4 := map[string][]string{"dual-stack.example.ts.net": {"10.1.2.3"}}
wantIPv6 := map[string][]string{"dual-stack.example.ts.net": {"2001:db8::1"}}
@@ -468,9 +465,9 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) {
},
}
mustCreate(t, fc, parentEgressSvc)
mustCreate(t, fc, proxyGroupSvc)
expectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack")
reconcilertest.MustCreate(t, fc, parentEgressSvc)
reconcilertest.MustCreate(t, fc, proxyGroupSvc)
reconcilertest.ExpectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack")
wantIPv4["pg-service.example.ts.net"] = []string{"10.96.0.100"}
wantIPv6["pg-service.example.ts.net"] = []string{"2001:db8::100"}
@@ -519,7 +516,7 @@ func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.Add
func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) {
t.Helper()
cm := new(corev1.ConfigMap)
if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
t.Fatalf("getting dnsconfig ConfigMap: %v", err)
}
if cm.Data == nil {
@@ -541,7 +538,7 @@ func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]
func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, wantsHostsIPv6 map[string][]string) {
t.Helper()
cm := new(corev1.ConfigMap)
if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
if err := cl.Get(t.Context(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
t.Fatalf("getting dnsconfig ConfigMap: %v", err)
}
if cm.Data == nil {
@@ -562,57 +559,3 @@ func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4,
t.Fatalf("unexpected IPv6 dns config (-got +want):\n%s", diff)
}
}
func expectReconciled(t *testing.T, r *dnsrecords.Reconciler, ns, name string) {
t.Helper()
req := reconcile.Request{
NamespacedName: types.NamespacedName{Namespace: ns, Name: name},
}
res, err := r.Reconcile(context.Background(), req)
if err != nil {
t.Fatalf("Reconcile: unexpected error: %v", err)
}
if res.Requeue {
t.Fatalf("unexpected immediate requeue")
}
}
func mustCreate(t *testing.T, c client.Client, obj client.Object) {
t.Helper()
if err := c.Create(context.Background(), obj); err != nil {
t.Fatalf("creating %q: %v", obj.GetName(), err)
}
}
func mustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) {
t.Helper()
for _, obj := range objs {
if err := c.Delete(context.Background(), obj); err != nil {
t.Fatalf("deleting %q: %v", obj.GetName(), err)
}
}
}
func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) {
t.Helper()
obj := O(new(T))
if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil {
t.Fatalf("getting %q: %v", name, err)
}
update(obj)
if err := c.Update(context.Background(), obj); err != nil {
t.Fatalf("updating %q: %v", name, err)
}
}
func mustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) {
t.Helper()
obj := O(new(T))
if err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, obj); err != nil {
t.Fatalf("getting %q: %v", name, err)
}
update(obj)
if err := c.Status().Update(context.Background(), obj); err != nil {
t.Fatalf("updating status %q: %v", name, err)
}
}
@@ -12,7 +12,6 @@ import (
_ "embed"
"errors"
"fmt"
"strings"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
@@ -50,8 +49,6 @@ const (
defaultNameserverImageRepo = "tailscale/k8s-nameserver"
defaultNameserverImageTag = "stable"
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)
var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
@@ -161,7 +158,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
}
if err = r.maybeProvision(ctx, &dnsCfg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
} else {
@@ -6,30 +6,23 @@
package nameserver_test
import (
"context"
_ "embed"
"encoding/json"
"reflect"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/yaml"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/nameserver"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/tstest"
"tailscale.com/util/mak"
)
@@ -96,8 +89,7 @@ func TestNameserverReconciler(t *testing.T) {
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().
WithObjects(dnsConfig).
WithStatusSubresource(dnsConfig).
Build()
@@ -115,7 +107,7 @@ func TestNameserverReconciler(t *testing.T) {
Logger: logger.Sugar(),
Clock: clock,
})
mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}})
reconcilertest.ExpectReconciled(t, r, "", "test")
ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
nameserverLabels := map[string]string{
@@ -169,7 +161,7 @@ func TestNameserverReconciler(t *testing.T) {
{Name: "some-secret"},
}
expectEqual(t, fc, wantsDeploy)
reconcilertest.ExpectEqual(t, fc, wantsDeploy)
})
wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}}
@@ -181,16 +173,16 @@ func TestNameserverReconciler(t *testing.T) {
wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference}
wantsSvc.Namespace = tsNamespace
wantsSvc.ObjectMeta.Labels = nameserverLabels
expectEqual(t, fc, wantsSvc)
reconcilertest.ExpectEqual(t, fc, wantsSvc)
})
t.Run("dns-config-status-is-set", func(t *testing.T) {
// Verify that DNSConfig advertises the nameserver's Service IP address,
// has the ready status condition and tailscale finalizer.
mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
svc.Spec.ClusterIP = "1.2.3.4"
})
mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}})
reconcilertest.ExpectReconciled(t, r, "", "test")
dnsConfig.Finalizers = []string{reconciler.Finalizer}
dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{
@@ -204,17 +196,17 @@ func TestNameserverReconciler(t *testing.T) {
LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)},
})
expectEqual(t, fc, dnsConfig)
reconcilertest.ExpectEqual(t, fc, dnsConfig)
})
t.Run("nameserver-image-updated", func(t *testing.T) {
// Verify that nameserver image gets updated to match DNSConfig spec.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
})
mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}})
reconcilertest.ExpectReconciled(t, r, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
expectEqual(t, fc, wantsDeploy)
reconcilertest.ExpectEqual(t, fc, wantsDeploy)
})
t.Run("reconciler-preserves-custom-config", func(t *testing.T) {
@@ -226,11 +218,11 @@ func TestNameserverReconciler(t *testing.T) {
t.Fatalf("error marshalling ConfigMap contents: %v", err)
}
mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
reconcilertest.MustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
mak.Set(&cm.Data, "records.json", string(bs))
})
mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}})
reconcilertest.ExpectReconciled(t, r, "", "test")
wantCm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
@@ -243,60 +235,17 @@ func TestNameserverReconciler(t *testing.T) {
Data: map[string]string{"records.json": string(bs)},
}
expectEqual(t, fc, wantCm)
reconcilertest.ExpectEqual(t, fc, wantCm)
})
t.Run("uses-default-nameserver-image", func(t *testing.T) {
// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
// the nameserver image defaults to tailscale/k8s-nameserver:stable.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
reconcilertest.MustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image = nil
})
mustReconcile(t, r, reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}})
reconcilertest.ExpectReconciled(t, r, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:stable"
expectEqual(t, fc, wantsDeploy)
reconcilertest.ExpectEqual(t, fc, wantsDeploy)
})
}
func mustReconcile(t *testing.T, r *nameserver.Reconciler, req reconcile.Request) {
t.Helper()
if _, err := r.Reconcile(context.Background(), req); err != nil {
t.Fatalf("unexpected reconcile error: %v", err)
}
}
func mustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) {
t.Helper()
obj := O(new(T))
if err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil {
t.Fatalf("getting object: %v", err)
}
update(obj)
if err := c.Update(context.Background(), obj); err != nil {
t.Fatalf("updating object: %v", err)
}
}
func expectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O) {
t.Helper()
got := O(new(T))
if err := c.Get(context.Background(), types.NamespacedName{
Name: want.GetName(),
Namespace: want.GetNamespace(),
}, got); err != nil {
t.Fatalf("getting %q: %v", want.GetName(), err)
}
// The resource version changes eagerly whenever the operator does even a
// no-op update. Asserting a specific value leads to overly brittle tests,
// so just remove it from both got and want.
got.SetResourceVersion("")
want.SetResourceVersion("")
// controller-runtime v0.20+ populates TypeMeta on objects returned by the
// fake client. Strip it so tests can continue to build expected objects
// without setting Kind/APIVersion explicitly.
got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{})
want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{})
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
}
}
@@ -14,7 +14,6 @@ import (
"net/netip"
"slices"
"strings"
"sync"
"testing"
"go.uber.org/zap"
@@ -25,8 +24,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tailscaleclient "tailscale.com/client/tailscale/v2"
@@ -34,7 +31,7 @@ import (
"tailscale.com/ipn"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/peerrelay"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/types/key"
)
@@ -1064,10 +1061,9 @@ func TestReconciler_Reconcile(t *testing.T) {
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
builder := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
builder := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}).
WithInterceptorFuncs(applyPatchInterceptor())
WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor())
if tc.PeerRelay != nil {
builder = builder.WithObjects(tc.PeerRelay)
}
@@ -1079,7 +1075,7 @@ func TestReconciler_Reconcile(t *testing.T) {
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{client: &fakeTSClient{}},
Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1149,7 +1145,7 @@ func TestReconciler_Reconcile(t *testing.T) {
}
if tc.ExpectedReadyStatus != "" || tc.ExpectedReadyReason != "" {
cond := readyCondition(&pr)
cond := reconcilertest.Condition(t, pr.Status.Conditions, tsapi.PeerRelayReady)
if tc.ExpectedReadyStatus != "" && cond.Status != tc.ExpectedReadyStatus {
t.Errorf("expected PeerRelayReady status %s, got %q", tc.ExpectedReadyStatus, cond.Status)
}
@@ -1294,16 +1290,6 @@ func assertSecretsForType(t *testing.T, fc client.Client, prName, secretType, la
}
}
func readyCondition(pr *tsapi.PeerRelay) metav1.Condition {
for _, cond := range pr.Status.Conditions {
if cond.Type == string(tsapi.PeerRelayReady) {
return cond
}
}
return metav1.Condition{}
}
func assertService(t *testing.T, want expectedService, got *corev1.Service) {
t.Helper()
@@ -1455,8 +1441,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) {
Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))},
}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}).
WithObjects(
pr,
@@ -1471,7 +1456,7 @@ func TestReconciler_TailscaledConfig(t *testing.T) {
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{client: &fakeTSClient{loginURL: testLoginURL}},
Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(testLoginURL))),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1535,104 +1520,6 @@ func readTailscaledConfig(t *testing.T, fc client.Client, secretName string) ipn
return conf
}
func applyPatchInterceptor() interceptor.Funcs {
return interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if patch.Type() != types.ApplyPatchType {
return cl.Patch(ctx, obj, patch, opts...)
}
key := client.ObjectKeyFromObject(obj)
existing := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, key, existing); err != nil {
if !apierrors.IsNotFound(err) {
return err
}
return cl.Create(ctx, obj)
}
obj.SetResourceVersion(existing.GetResourceVersion())
return cl.Update(ctx, obj)
},
}
}
type fakeClientProvider struct {
client tsclient.Client
err error
}
func (p *fakeClientProvider) For(_ string) (tsclient.Client, error) { return p.client, p.err }
type fakeTSClient struct {
tsclient.Client
loginURL string
mu sync.Mutex
keyCalls []tailscaleclient.CreateKeyRequest
deviceDeletes []string
devices []tailscaleclient.Device
nextKey []string
}
func (c *fakeTSClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) }
func (c *fakeTSClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) }
func (c *fakeTSClient) LoginURL() string { return c.loginURL }
func (c *fakeTSClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.keyCalls)
}
func (c *fakeTSClient) DeviceDeletes() []string {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.deviceDeletes)
}
type fakeKeys fakeTSClient
func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) {
c := (*fakeTSClient)(k)
c.mu.Lock()
defer c.mu.Unlock()
c.keyCalls = append(c.keyCalls, req)
var key string
if len(c.nextKey) > 0 {
key, c.nextKey = c.nextKey[0], c.nextKey[1:]
} else {
key = fmt.Sprintf("auth-key-%d", len(c.keyCalls))
}
return &tailscaleclient.Key{Key: key}, nil
}
func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { return nil, nil }
type fakeDevices fakeTSClient
func (d *fakeDevices) Delete(_ context.Context, id string) error {
c := (*fakeTSClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
c.deviceDeletes = append(c.deviceDeletes, id)
return nil
}
func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) {
c := (*fakeTSClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.devices), nil
}
func (d *fakeDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) {
return nil, nil
}
func TestReconciler_AuthKey_Lifecycle(t *testing.T) {
t.Parallel()
@@ -1643,19 +1530,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) {
t.Run("mints-key-on-first-reconcile", func(t *testing.T) {
pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}).
WithObjects(pr).
Build()
tsc := &fakeTSClient{nextKey: []string{"tskey-abc"}}
tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-abc"))
r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: fc,
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:k8s-peer-relay"},
Clients: &fakeClientProvider{client: tsc},
Clients: reconcilertest.NewFakeClientProvider(tsc),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1681,19 +1567,18 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) {
t.Run("reuses-existing-key-across-reconciles", func(t *testing.T) {
pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}).
WithObjects(pr).
Build()
tsc := &fakeTSClient{nextKey: []string{"tskey-first", "tskey-second"}}
tsc := reconcilertest.NewFakeClient(reconcilertest.WithAuthKeys("tskey-first", "tskey-second"))
r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: fc,
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:k8s-peer-relay"},
Clients: &fakeClientProvider{client: tsc},
Clients: reconcilertest.NewFakeClientProvider(tsc),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1721,20 +1606,19 @@ func TestReconciler_AuthKey_Lifecycle(t *testing.T) {
Spec: tsapi.PeerRelaySpec{Tags: tsapi.Tags{"tag:custom"}},
}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}).
WithObjects(pr).
Build()
tsc := &fakeTSClient{}
tsc := reconcilertest.NewFakeClient()
r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: fc,
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:k8s-peer-relay"},
Clients: &fakeClientProvider{client: tsc},
Clients: reconcilertest.NewFakeClientProvider(tsc),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1801,8 +1685,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) {
unreported := stateSecret("test", "peerrelay-test-2", 2, "")
unreported.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}).
WithObjects(
pr,
@@ -1813,16 +1696,17 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) {
).
Build()
tsc := &fakeTSClient{devices: []tailscaleclient.Device{
{NodeID: "device-bbb", MachineKey: mk.Public().String()},
{NodeID: "device-other", MachineKey: key.NewMachine().Public().String()},
}}
tsc := reconcilertest.NewFakeClient()
tsc.SetDevices(
tailscaleclient.Device{NodeID: "device-bbb", MachineKey: mk.Public().String()},
tailscaleclient.Device{NodeID: "device-other", MachineKey: key.NewMachine().Public().String()},
)
r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: fc,
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{client: tsc},
Clients: reconcilertest.NewFakeClientProvider(tsc),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1856,8 +1740,7 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) {
Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))},
}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}).
WithObjects(
pr,
@@ -1867,13 +1750,13 @@ func TestReconciler_DeletesTailnetDevices(t *testing.T) {
).
Build()
tsc := &fakeTSClient{}
tsc := reconcilertest.NewFakeClient()
r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: fc,
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{client: tsc},
Clients: reconcilertest.NewFakeClientProvider(tsc),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1916,8 +1799,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) {
Spec: tsapi.PeerRelaySpec{Tailnet: "missing"},
}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}).
WithObjects(pr).
Build()
@@ -1927,7 +1809,7 @@ func TestReconciler_TailnetUnavailable(t *testing.T) {
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{err: errors.New("tailnet missing: not ready")},
Clients: reconcilertest.NewFailingClientProvider(errors.New("tailnet missing: not ready")),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -1941,13 +1823,10 @@ func TestReconciler_TailnetUnavailable(t *testing.T) {
t.Fatal(err)
}
cond := readyCondition(&got)
if cond.Status != metav1.ConditionFalse {
t.Errorf("expected PeerRelayReady=False, got %q", cond.Status)
}
if cond.Reason != peerrelay.ReasonTailnetUnavailable {
t.Errorf("expected reason=%s, got %q", peerrelay.ReasonTailnetUnavailable, cond.Reason)
}
reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.PeerRelayReady,
metav1.ConditionFalse, peerrelay.ReasonTailnetUnavailable)
cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.PeerRelayReady)
if !strings.Contains(cond.Message, "not ready") {
t.Errorf("expected condition message to include resolver error, got %q", cond.Message)
}
@@ -1997,8 +1876,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) {
},
}
fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()).
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()).
WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}).
WithObjects(pr, pc).
Build()
@@ -2008,7 +1886,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) {
TailscaleNamespace: tailscaleNamespace,
ProxyImage: testProxyImage,
DefaultTags: []string{"tag:test-peer-relay"},
Clients: &fakeClientProvider{client: &fakeTSClient{}},
Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient()),
Resolver: testResolver,
Logger: logger.Sugar(),
})
@@ -16,11 +16,11 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/proxyclass"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/tstest"
)
@@ -51,8 +51,7 @@ func TestProxyClass(t *testing.T) {
},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().
WithObjects(pc).
WithStatusSubresource(pc).
Build()
@@ -118,16 +117,7 @@ func TestProxyClass(t *testing.T) {
if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil {
t.Fatalf("Get: %v", err)
}
var cond *metav1.Condition
for i := range got.Status.Conditions {
if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) {
cond = &got.Status.Conditions[i]
break
}
}
if cond == nil {
t.Fatalf("ProxyClassReady condition not set")
}
cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady)
if cond.Status != wantStatus {
t.Errorf("condition Status: got %q, want %q", cond.Status, wantStatus)
}
@@ -337,8 +327,7 @@ func TestProxyClassValidation(t *testing.T) {
objs = append(objs, existingPC)
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
fc := reconcilertest.NewClientBuilder().
WithObjects(objs...).
WithStatusSubresource(pc, existingPC).
Build()
@@ -365,16 +354,7 @@ func TestProxyClassValidation(t *testing.T) {
t.Fatalf("Get: %v", err)
}
var ready *metav1.Condition
for i := range got.Status.Conditions {
if got.Status.Conditions[i].Type == string(tsapi.ProxyClassReady) {
ready = &got.Status.Conditions[i]
break
}
}
if ready == nil {
t.Fatal("Ready condition not set")
}
ready := reconcilertest.Condition(t, got.Status.Conditions, tsapi.ProxyClassReady)
wantStatus := metav1.ConditionTrue
if !tc.valid {
@@ -12,11 +12,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/proxygrouppolicy"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
)
func TestReconciler_Reconcile(t *testing.T) {
@@ -102,7 +102,7 @@ func TestReconciler_Reconcile(t *testing.T) {
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
bldr := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme)
bldr := reconcilertest.NewClientBuilder()
bldr = bldr.WithObjects(tc.ExistingResources...)
fc := bldr.Build()
+13
View File
@@ -11,6 +11,7 @@ import (
"context"
"fmt"
"slices"
"strings"
"sync"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -40,8 +41,20 @@ const (
// LabelParentNamespace identifies the namespace of the owning Tailscale CRD. It is only stamped when the parent
// CRD is namespaced; cluster-scoped parents omit it.
LabelParentNamespace = "tailscale.com/parent-resource-ns"
// optimisticLockErrorMsg is the message the API server returns when an Update is rejected because the object
// has been modified since it was read. The API server does not give us a typed error for the conflict in all
// cases, so callers match on the message via IsOptimisticLockError.
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)
// IsOptimisticLockError reports whether err was caused by an optimistic locking conflict, i.e. the object being
// updated was modified after the reconciler read it. Such an error is transient: the reconciler should requeue and
// retry against a freshly read object rather than surface the error.
func IsOptimisticLockError(err error) bool {
return err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg)
}
// SetFinalizer adds name to obj's finalizers if not already present. Most CRD reconcilers pass Finalizer; per-
// reconciler names are used when a single resource may carry finalizers from multiple reconcilers that each need to run
// their own cleanup.
@@ -6,6 +6,8 @@
package reconciler_test
import (
"errors"
"fmt"
"maps"
"slices"
"testing"
@@ -144,6 +146,37 @@ func TestEnqueueForChild(t *testing.T) {
}
}
func TestIsOptimisticLockError(t *testing.T) {
t.Parallel()
tests := map[string]struct {
err error
want bool
}{
"nil": {},
"conflict": {
err: errors.New("Operation cannot be fulfilled on secrets \"test\": the object has been modified; please apply your changes to the latest version and try again"),
want: true,
},
"unrelated": {
err: errors.New("secrets \"test\" not found"),
},
"wrapped-conflict": {
err: fmt.Errorf("failed to update Secret: %w", errors.New("the object has been modified; please apply your changes to the latest version and try again")),
want: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
if got := reconciler.IsOptimisticLockError(tc.err); got != tc.want {
t.Errorf("expected %v, got %v", tc.want, got)
}
})
}
}
func TestResourceTracker(t *testing.T) {
t.Parallel()
@@ -0,0 +1,59 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package reconcilertest
import (
"testing"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
)
// The condition helpers below take a []metav1.Condition rather than the owning object. Every Tailscale CRD stores its
// conditions in Status.Conditions, but the CRD types live in tsapi and so can't grow a shared accessor method from
// here; taking the slice keeps these usable from every reconciler's tests (and for corev1.Service conditions) without
// per-CRD plumbing. Pair them with MustGet:
//
// tsr := &tsapi.Recorder{}
// reconcilertest.MustGet(t, fc, "", "test", tsr)
// reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, metav1.ConditionTrue, reason, message)
// Condition returns the condition of the given type, failing the test if it isn't present. Use it when a test needs to
// inspect a condition's fields directly, e.g. to match part of a message generated outside this repo; prefer
// ExpectCondition for whole-condition assertions.
func Condition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType) metav1.Condition {
t.Helper()
cond := apimeta.FindStatusCondition(conds, string(conditionType))
if cond == nil {
t.Fatalf("no %s condition found, got %+v", conditionType, conds)
}
return *cond
}
// ExpectCondition asserts that a condition of the given type is present with the given status, reason and message.
// LastTransitionTime and ObservedGeneration are not compared, so tests don't need a fixed clock or to track
// generations in order to assert on the parts that describe the outcome.
func ExpectCondition(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) {
t.Helper()
got := Condition(t, conds, conditionType)
if got.Status != status || got.Reason != reason || got.Message != message {
t.Fatalf("unexpected %s condition: got (%s, %s, %q), want (%s, %s, %q)",
conditionType, got.Status, got.Reason, got.Message, status, reason, message)
}
}
// ExpectConditionStatus asserts a condition's status and reason, ignoring its message. Use it when the message is
// long, generated outside this repo, or otherwise not the point of the test.
func ExpectConditionStatus(t *testing.T, conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason string) {
t.Helper()
got := Condition(t, conds, conditionType)
if got.Status != status || got.Reason != reason {
t.Fatalf("unexpected %s condition: got (%s, %s), want (%s, %s)",
conditionType, got.Status, got.Reason, status, reason)
}
}
@@ -0,0 +1,202 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package reconcilertest provides helpers for testing the CRD reconcilers under k8s-operator/reconciler. It covers
// the parts of a reconciler test that are the same whichever CRD is under test: building a fake client with the
// operator's scheme, creating and mutating objects in it, asserting on what a reconcile left behind, and faking the
// Tailscale API.
//
// Assertions come in two flavours. Must* helpers set up test preconditions and call t.Fatalf if the cluster doesn't
// behave, because a failed precondition makes the rest of the test meaningless. Expect* helpers assert on the state
// a reconcile produced.
//
// Per-CRD assertions belong in the reconciler's own test package; only helpers that are genuinely reconciler-
// agnostic belong here.
package reconcilertest
import (
"context"
"reflect"
"slices"
"testing"
"time"
"github.com/google/go-cmp/cmp"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
)
// NewClientBuilder returns a fake client builder preconfigured with the operator's scheme. Callers add their own
// objects, status subresources and interceptors before calling Build.
func NewClientBuilder() *fake.ClientBuilder {
return fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme)
}
// ApplyPatchInterceptor returns interceptor funcs that emulate server-side apply, which the fake client does not
// implement. Apply patches become a Create if the object is absent and an Update if it is present; every other patch
// type is passed through untouched. Reconcilers that apply their children (rather than using
// reconciler.CreateOrUpdate) need this to be testable against the fake client.
func ApplyPatchInterceptor() interceptor.Funcs {
return interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if patch.Type() != types.ApplyPatchType {
return cl.Patch(ctx, obj, patch, opts...)
}
key := client.ObjectKeyFromObject(obj)
existing := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, key, existing); err != nil {
if !apierrors.IsNotFound(err) {
return err
}
return cl.Create(ctx, obj)
}
obj.SetResourceVersion(existing.GetResourceVersion())
return cl.Update(ctx, obj)
},
}
}
// MustCreate creates obj, failing the test if the cluster rejects it.
func MustCreate(t *testing.T, c client.Client, obj client.Object) {
t.Helper()
if err := c.Create(t.Context(), obj); err != nil {
t.Fatalf("creating %q: %v", obj.GetName(), err)
}
}
// MustGet reads ns/name into obj, failing the test if it is absent. Use ExpectMissing to assert the opposite.
func MustGet(t *testing.T, c client.Client, ns, name string, obj client.Object) {
t.Helper()
if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil {
t.Fatalf("getting %q: %v", name, err)
}
}
// MustDeleteAll deletes each of objs in order.
func MustDeleteAll(t *testing.T, c client.Client, objs ...client.Object) {
t.Helper()
for _, obj := range objs {
if err := c.Delete(t.Context(), obj); err != nil {
t.Fatalf("deleting %q: %v", obj.GetName(), err)
}
}
}
// MustUpdate reads ns/name, applies update to it and writes it back. It reads the object fresh rather than taking one
// from the caller so the update is applied to the current ResourceVersion.
func MustUpdate[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) {
t.Helper()
obj := O(new(T))
if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil {
t.Fatalf("getting %q: %v", name, err)
}
update(obj)
if err := c.Update(t.Context(), obj); err != nil {
t.Fatalf("updating %q: %v", name, err)
}
}
// MustUpdateStatus is MustUpdate for the status subresource. The object's type must have been registered with
// WithStatusSubresource on the fake client builder.
func MustUpdateStatus[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string, update func(O)) {
t.Helper()
obj := O(new(T))
if err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj); err != nil {
t.Fatalf("getting %q: %v", name, err)
}
update(obj)
if err := c.Status().Update(t.Context(), obj); err != nil {
t.Fatalf("updating status %q: %v", name, err)
}
}
// ExpectEqual asserts that the object named by want exists and matches it. ResourceVersion and TypeMeta are cleared
// on both sides first: the former changes on every no-op write and the latter is populated by the fake client but not
// by the callers building expected objects, so asserting on either makes tests brittle. Pass modifiers to blank out
// any further fields that shouldn't be compared; each one runs against both want and got.
func ExpectEqual[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, want O, modifiers ...func(O)) {
t.Helper()
got := O(new(T))
if err := c.Get(t.Context(), types.NamespacedName{
Name: want.GetName(),
Namespace: want.GetNamespace(),
}, got); err != nil {
t.Fatalf("getting %q: %v", want.GetName(), err)
}
got.SetResourceVersion("")
want.SetResourceVersion("")
got.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{})
want.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{})
for _, modifier := range modifiers {
modifier(want)
modifier(got)
}
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected %s (-got +want):\n%s", reflect.TypeOf(want).Elem().Name(), diff)
}
}
// ExpectMissing asserts that no object of type T exists at ns/name.
func ExpectMissing[T any, O reconciler.PtrObject[T]](t *testing.T, c client.Client, ns, name string) {
t.Helper()
obj := O(new(T))
err := c.Get(t.Context(), types.NamespacedName{Namespace: ns, Name: name}, obj)
if !apierrors.IsNotFound(err) {
t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
}
}
// ExpectReconciled runs a single reconcile for ns/name and asserts it succeeded without asking to be requeued, i.e.
// the reconciler considers the resource fully settled.
func ExpectReconciled(t *testing.T, r reconcile.Reconciler, ns, name string) {
t.Helper()
res, err := r.Reconcile(t.Context(), request(ns, name))
if err != nil {
t.Fatalf("Reconcile: unexpected error: %v", err)
}
if res.Requeue {
t.Fatalf("unexpected immediate requeue")
}
if res.RequeueAfter != 0 {
t.Fatalf("unexpected timed requeue (%v)", res.RequeueAfter)
}
}
func request(ns, name string) reconcile.Request {
return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: name}}
}
// ExpectEvents asserts that len(want) events are emitted on rec, each of which is in want, waiting up to five seconds
// for each. Events are not required to arrive in order, because a reconcile may emit them from more than one code path.
func ExpectEvents(t *testing.T, rec *record.FakeRecorder, want []string) {
t.Helper()
seen := make([]string, 0, len(want))
for range want {
timer := time.NewTimer(5 * time.Second)
defer timer.Stop()
select {
case got := <-rec.Events:
if !slices.Contains(want, got) {
t.Errorf("got unexpected event %q, expected events: %+#v", got, want)
continue
}
seen = append(seen, got)
case <-timer.C:
t.Errorf("timeout waiting for an event, wants events %+#v, got events %+#v", want, seen)
}
}
}
@@ -0,0 +1,174 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package reconcilertest
import (
"context"
"fmt"
"net/http"
"slices"
"sync"
tailscaleclient "tailscale.com/client/tailscale/v2"
"tailscale.com/k8s-operator/tsclient"
)
// FakeClient is a tsclient.Client that records the calls a reconciler makes to the Tailscale API and serves canned
// responses. The zero value is usable: keys are minted with generated values, no devices exist, and the login URL is
// blank. FakeClient is safe for concurrent use.
//
// Only the endpoints the reconcilers actually call are implemented. Anything else panics via the embedded nil
// interface rather than silently returning a zero value, so a reconciler that starts calling a new endpoint fails
// loudly instead of being tested against a fake that quietly does nothing.
type FakeClient struct {
tsclient.Client
mu sync.Mutex
loginURL string
// Devices are returned by Devices().Get and List. Get matches on Device.ID and returns a 404 API error when no
// device matches, mirroring control's behaviour for an unknown or already-deleted device.
devices []tailscaleclient.Device
// nextKeys are handed out by CreateAuthKey in order. Once exhausted, generated values are returned instead, so
// tests only script the keys they assert on.
nextKeys []string
keyCalls []tailscaleclient.CreateKeyRequest
deviceDeletes []string
}
// FakeClientOption configures a FakeClient.
type FakeClientOption func(*FakeClient)
// WithLoginURL sets the URL returned by LoginURL, which reconcilers embed in the workloads they create.
func WithLoginURL(url string) FakeClientOption {
return func(c *FakeClient) { c.loginURL = url }
}
// WithAuthKeys sets the auth keys CreateAuthKey returns, in order. Use it when a test needs to tell successive keys
// apart, e.g. to assert that a replica picked up a reissued key.
func WithAuthKeys(keys ...string) FakeClientOption {
return func(c *FakeClient) { c.nextKeys = keys }
}
// NewFakeClient returns a FakeClient configured by opts.
func NewFakeClient(opts ...FakeClientOption) *FakeClient {
c := &FakeClient{}
for _, opt := range opts {
opt(c)
}
return c
}
// LoginURL implements tsclient.Client.
func (c *FakeClient) LoginURL() string { return c.loginURL }
// Devices implements tsclient.Client.
func (c *FakeClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) }
// Keys implements tsclient.Client.
func (c *FakeClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) }
// CreateAuthKeyCalls returns every auth key request the reconciler has made, in order.
func (c *FakeClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.keyCalls)
}
// DeviceDeletes returns the IDs of every device the reconciler has deleted, in order.
func (c *FakeClient) DeviceDeletes() []string {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.deviceDeletes)
}
// SetDevices replaces the devices served by the fake. Use it to simulate control catching up partway through a test,
// e.g. once a replica has authenticated.
func (c *FakeClient) SetDevices(devices ...tailscaleclient.Device) {
c.mu.Lock()
defer c.mu.Unlock()
c.devices = devices
}
type fakeDevices FakeClient
func (d *fakeDevices) Delete(_ context.Context, id string) error {
c := (*FakeClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
c.deviceDeletes = append(c.deviceDeletes, id)
return nil
}
func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) {
c := (*FakeClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.devices), nil
}
func (d *fakeDevices) Get(_ context.Context, id string) (*tailscaleclient.Device, error) {
c := (*FakeClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
for _, dev := range c.devices {
if dev.ID == id {
return &dev, nil
}
}
return nil, tailscaleclient.APIError{Status: http.StatusNotFound}
}
type fakeKeys FakeClient
func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) {
c := (*FakeClient)(k)
c.mu.Lock()
defer c.mu.Unlock()
c.keyCalls = append(c.keyCalls, req)
var key string
if len(c.nextKeys) > 0 {
key, c.nextKeys = c.nextKeys[0], c.nextKeys[1:]
} else {
key = fmt.Sprintf("auth-key-%d", len(c.keyCalls))
}
return &tailscaleclient.Key{Key: key}, nil
}
func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) {
return nil, nil
}
// FakeClientProvider is a tailscaled.ClientProvider that hands out a single client for every tailnet, or fails with a
// fixed error. Reconcilers resolve their API client through a provider, so tests that exercise the
// tailnet-unavailable path need one that can fail.
type FakeClientProvider struct {
client tsclient.Client
err error
}
// NewFakeClientProvider returns a provider that resolves every tailnet to client.
func NewFakeClientProvider(client tsclient.Client) *FakeClientProvider {
return &FakeClientProvider{client: client}
}
// NewFailingClientProvider returns a provider that fails to resolve any tailnet with err.
func NewFailingClientProvider(err error) *FakeClientProvider {
return &FakeClientProvider{err: err}
}
// For implements tailscaled.ClientProvider.
func (p *FakeClientProvider) For(_ string) (tsclient.Client, error) {
if p.err != nil {
return nil, p.err
}
return p.client, nil
}
@@ -0,0 +1,91 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package recorder
import (
"context"
"fmt"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
)
// authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret.
const authKeySecretKey = "authkey"
// recorderTags returns the tags to apply to auth keys minted for tsr's replicas.
func recorderTags(tsr *tsapi.Recorder) []string {
if len(tsr.Spec.Tags) == 0 {
return tsapi.Tags{"tag:k8s"}.Stringify()
}
return tsr.Spec.Tags.Stringify()
}
// ensureAuthSecretsCreated makes sure every replica has an auth Secret holding a usable auth key. A key is minted
// when the Secret doesn't exist yet, or when the replica has requested a reissue via its state Secret because the
// existing key can no longer authenticate it.
func (r *Reconciler) ensureAuthSecretsCreated(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
tags := recorderTags(tsr)
for replica := range replicas(tsr) {
key := types.NamespacedName{
Namespace: r.tsNamespace,
Name: authSecretName(tsr.Name, replica),
}
existingSecret := &corev1.Secret{}
err := r.Get(ctx, key, existingSecret)
switch {
case err == nil:
stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica)
if err != nil {
return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err)
}
cfgAuthKey := string(existingSecret.Data[authKeySecretKey])
reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, logger, tailscaled.ReissueInput{
ParentName: tsr.Name,
ReplicaName: stateSecretName(tsr.Name, replica),
Kind: tailscaled.KindRecorder,
StateSecret: stateSecret,
CfgAuthKey: &cfgAuthKey,
})
if err != nil {
return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err)
}
if !reissue {
logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name)
continue
}
authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags)
if err != nil {
return err
}
existingSecret.Data[authKeySecretKey] = []byte(authKey)
if err = r.Update(ctx, existingSecret); err != nil {
return err
}
case apierrors.IsNotFound(err):
authKey, err := tailscaled.NewAuthKey(ctx, tsClient, tags)
if err != nil {
return err
}
if err = r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil {
return err
}
default:
return fmt.Errorf("failed to get Secret %q: %w", key.Name, err)
}
}
return nil
}
@@ -3,20 +3,21 @@
//go:build !plan9
package main
// Package recorder provides reconciliation logic for the Recorder custom resource definition. It is responsible for
// managing the lifecycle of tsrecorder devices, including the StatefulSet, ServiceAccount and RBAC resources used to
// run them.
package recorder
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"sync"
"time"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -24,68 +25,132 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
apivalidation "k8s.io/apimachinery/pkg/api/validation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/client/tailscale/v2"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/set"
)
const (
reasonRecorderCreationFailed = "RecorderCreationFailed"
reasonRecorderCreating = "RecorderCreating"
reasonRecorderCreated = "RecorderCreated"
reasonRecorderInvalid = "RecorderInvalid"
reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable"
reconcilerName = "recorder-reconciler"
currentProfileKey = "_current-profile"
// authKeySecretKey is the data key under which a Recorder replica's auth key is stored in its auth Secret.
authKeySecretKey = "authkey"
// shortRequeue is how long to wait before retrying a reconcile that made progress but hasn't finished, e.g.
// cleanup that is waiting on control to catch up.
shortRequeue = 5 * time.Second
)
// Condition reasons set on Recorder resources.
const (
reasonRecorderCreated = "RecorderCreated"
reasonRecorderCreationFailed = "RecorderCreationFailed"
reasonRecorderCreating = "RecorderCreating"
reasonRecorderInvalid = "RecorderInvalid"
reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable"
)
// gaugeRecorderResources tracks the overall number of Recorder resources currently managed by this operator instance.
var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount)
// RecorderReconciler syncs Recorder statefulsets with their definition in
// Recorder CRs.
type RecorderReconciler struct {
client.Client
log *zap.SugaredLogger
recorder record.EventRecorder
clock tstime.Clock
clients ClientProvider
tsNamespace string
reissuer *tailscaled.Reissuer
mu sync.Mutex // protects following
recorders set.Slice[types.UID] // for recorders gauge
type (
// Reconciler is a reconcile.Reconciler implementation that syncs Recorder StatefulSets with their definition in
// Recorder custom resources.
Reconciler struct {
client.Client
recorder record.EventRecorder
logger *zap.SugaredLogger
clock tstime.Clock
clients tailscaled.ClientProvider
tsNamespace string
reissuer *tailscaled.Reissuer
tracker *reconciler.ResourceTracker
}
// ReconcilerOptions contains configuration values for the Reconciler.
ReconcilerOptions struct {
// Client is used to interact with the Kubernetes API.
Client client.Client
// Recorder is used to emit Kubernetes events.
Recorder record.EventRecorder
// TailscaleNamespace is the namespace the operator is installed in. Recorder-managed resources
// (StatefulSets, Secrets, etc.) are created within this namespace.
TailscaleNamespace string
// Clients resolves the Tailscale API client for a given tailnet name. Used to mint auth keys for each
// replica and to delete their devices on cleanup. Blank tailnet returns the operator's default client.
Clients tailscaled.ClientProvider
// Logger is the logger to use for this Reconciler.
Logger *zap.SugaredLogger
// Clock is used to stamp condition transitions. Defaults to a real clock when unset.
Clock tstime.Clock
}
)
// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Recorder
// custom resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler.
func NewReconciler(options ReconcilerOptions) *Reconciler {
clock := options.Clock
if clock == nil {
clock = tstime.DefaultClock{}
}
return &Reconciler{
Client: options.Client,
recorder: options.Recorder,
logger: options.Logger.Named(reconcilerName),
clock: clock,
clients: options.Clients,
tsNamespace: options.TailscaleNamespace,
reissuer: tailscaled.NewReissuer(),
tracker: reconciler.NewResourceTracker(gaugeRecorderResources),
}
}
func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger {
return r.log.With("Recorder", name)
// Register the Reconciler onto the given manager.Manager implementation. It watches Recorder resources directly, as
// well as the child resources it manages, so that drift is corrected by a reconcile of the owning Recorder. All
// children carry a controller owner reference to their Recorder, so they are matched by owner rather than by label.
func (r *Reconciler) Register(mgr manager.Manager) error {
enqueue := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
return builder.
ControllerManagedBy(mgr).
For(&tsapi.Recorder{}).
Named(reconcilerName).
Watches(&appsv1.StatefulSet{}, enqueue).
Watches(&corev1.ServiceAccount{}, enqueue).
Watches(&corev1.Secret{}, enqueue).
Watches(&rbacv1.Role{}, enqueue).
Watches(&rbacv1.RoleBinding{}, enqueue).
Complete(r)
}
func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
logger := r.logger(req.Name)
// Reconcile is invoked when a change occurs to Recorder resources within the cluster. On create/update it ensures a
// StatefulSet and the Secrets and RBAC resources each replica needs exist. On delete it removes each replica's
// device from the tailnet before releasing the finalizer; the in-cluster resources are garbage collected via their
// owner references.
func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
logger := r.logger.With("Recorder", req.Name)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
tsr := new(tsapi.Recorder)
err := r.Get(ctx, req.NamespacedName, tsr)
if apierrors.IsNotFound(err) {
switch {
case apierrors.IsNotFound(err):
logger.Debugf("Recorder not found, assuming it was deleted")
return reconcile.Result{}, nil
} else if err != nil {
case err != nil:
return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err)
}
@@ -107,36 +172,33 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error())
}
if markedForDeletion(tsr) {
if !tsr.DeletionTimestamp.IsZero() {
logger.Debugf("Recorder is being deleted, cleaning up resources")
ix := xslices.Index(tsr.Finalizers, FinalizerName)
if ix < 0 {
if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) {
logger.Debugf("no finalizer, nothing to do")
return reconcile.Result{}, nil
}
if done, err := r.maybeCleanup(ctx, tsr, tsClient); err != nil {
if done, err := r.maybeCleanup(ctx, logger, tsr, tsClient); err != nil {
return reconcile.Result{}, err
} else if !done {
logger.Debugf("Recorder resource cleanup not yet finished, will retry...")
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1)
if err = r.Update(ctx, tsr); err != nil {
if err = reconciler.ClearFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
if !slices.Contains(tsr.Finalizers, FinalizerName) {
if !slices.Contains(tsr.Finalizers, reconciler.Finalizer) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to log that the high level, multi-reconcile
// operation is underway.
logger.Infof("ensuring Recorder is set up")
tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
if err = r.Update(ctx, tsr); err != nil {
if err = reconciler.EnsureFinalizer(ctx, r.Client, tsr, reconciler.Finalizer); err != nil {
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
}
}
@@ -147,10 +209,10 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
}
if err = r.maybeProvision(ctx, tsClient, tsr); err != nil {
if err = r.maybeProvision(ctx, logger, tsClient, tsr); err != nil {
reason := reasonRecorderCreationFailed
message := fmt.Sprintf("failed creating Recorder: %s", err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
reason = reasonRecorderCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
err = nil
@@ -165,28 +227,20 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated)
}
func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
logger := r.logger(tsr.Name)
func (r *Reconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
replicas := replicas(tsr)
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
}
r.mu.Lock()
r.recorders.Add(tsr.UID)
gaugeRecorderResources.Set(int64(r.recorders.Len()))
r.mu.Unlock()
r.tracker.Add(tsr.UID)
r.reissuer.EnsureState(tsr.Name, int(replicas))
if err := r.ensureAuthSecretsCreated(ctx, tsClient, tsr); err != nil {
if err := r.ensureAuthSecretsCreated(ctx, logger, tsClient, tsr); err != nil {
return fmt.Errorf("error creating secrets: %w", err)
}
// State Secrets are pre-created so we can use the Recorder CR as its owner ref.
for replica := range replicas {
sec := tsrStateSecret(tsr, r.tsNamespace, replica)
_, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
_, err := reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) {
s.ObjectMeta.Labels = sec.ObjectMeta.Labels
s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
})
@@ -196,7 +250,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie
}
sa := tsrServiceAccount(tsr, r.tsNamespace)
_, err := createOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error {
_, err := reconciler.CreateOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error {
// Perform this check within the update function to make sure we don't
// have a race condition between the previous check and the update.
if err := saOwnedByRecorder(s, tsr); err != nil {
@@ -213,7 +267,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie
}
role := tsrRole(tsr, r.tsNamespace)
_, err = createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
_, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
r.ObjectMeta.Labels = role.ObjectMeta.Labels
r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
r.Rules = role.Rules
@@ -223,7 +277,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie
}
roleBinding := tsrRoleBinding(tsr, r.tsNamespace)
_, err = createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
_, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations
r.RoleRef = roleBinding.RoleRef
@@ -234,7 +288,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie
}
ss := tsrStatefulSet(tsr, r.tsNamespace, tsClient.LoginURL())
_, err = createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
_, err = reconciler.CreateOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
s.ObjectMeta.Labels = ss.ObjectMeta.Labels
s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
s.Spec = ss.Spec
@@ -246,13 +300,13 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclie
// ServiceAccount name may have changed, in which case we need to clean up
// the previous ServiceAccount. RoleBinding will already be updated to point
// to the new ServiceAccount.
if err = r.maybeCleanupServiceAccounts(ctx, tsr, sa.Name); err != nil {
if err = r.maybeCleanupServiceAccounts(ctx, logger, tsr, sa.Name); err != nil {
return fmt.Errorf("error cleaning up ServiceAccounts: %w", err)
}
// If we have scaled the recorder down, we will have dangling state secrets
// that we need to clean up.
if err = r.maybeCleanupSecrets(ctx, tsClient, tsr); err != nil {
if err = r.maybeCleanupSecrets(ctx, logger, tsClient, tsr); err != nil {
return fmt.Errorf("error cleaning up Secrets: %w", err)
}
@@ -292,9 +346,7 @@ func saOwnedByRecorder(sa *corev1.ServiceAccount, tsr *tsapi.Recorder) error {
// accumulating a large amount of garbage.
//
// This is a no-op if the ServiceAccount name has not changed.
func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, tsr *tsapi.Recorder, currentName string) error {
logger := r.logger(tsr.Name)
func (r *Reconciler) maybeCleanupServiceAccounts(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, currentName string) error {
options := []client.ListOption{
client.InNamespace(r.tsNamespace),
client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)),
@@ -323,7 +375,9 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts
return nil
}
func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
// maybeCleanupSecrets deletes the state and auth Secrets of any replica beyond the Recorder's current replica count,
// along with each one's tailnet device. Their ordinals are read back from the Secret names.
func (r *Reconciler) maybeCleanupSecrets(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
options := []client.ListOption{
client.InNamespace(r.tsNamespace),
client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)),
@@ -336,10 +390,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t
// Get the largest ordinal suffix that we expect. Then we'll go through the list of secrets owned by this
// recorder and remove them.
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
}
replicas := replicas(tsr)
for _, secret := range secrets.Items {
parts := strings.Split(secret.Name, "-")
@@ -356,22 +407,16 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t
continue
}
devicePrefs, ok, err := getDevicePrefs(&secret)
devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(&secret)
if err != nil {
return err
}
if ok {
r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID)
err = tsClient.Devices().Delete(ctx, string(devicePrefs.Config.NodeID))
switch {
case tailscale.IsNotFound(err):
// This device has possibly already been deleted in the admin console. So we can ignore this
// and move on to removing the secret.
case err != nil:
if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil {
return err
}
} else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, r.log, &secret); err != nil {
} else if err = tailscaled.EnsureUnreportedDeviceDeleted(ctx, tsClient, logger, &secret); err != nil {
return err
}
@@ -386,25 +431,18 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient t
// maybeCleanup just deletes the device from the tailnet. All the kubernetes
// resources linked to a Recorder will get cleaned up via owner references
// (which we can use because they are all in the same namespace).
func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) {
logger := r.logger(tsr.Name)
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
}
for replica := range replicas {
func (r *Reconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) {
for replica := range replicas(tsr) {
secret, err := r.getStateSecret(ctx, tsr.Name, replica)
if err != nil {
return false, err
}
if secret == nil {
logger.Debugf("state Secret %s-%d not found, continuing cleanup", tsr.Name, replica)
logger.Debugf("state Secret %s not found, continuing cleanup", stateSecretName(tsr.Name, replica))
continue
}
devicePrefs, ok, err := getDevicePrefs(secret)
devicePrefs, ok, err := tailscaled.PrefsFromStateSecret(secret)
if err != nil {
return false, err
}
@@ -415,18 +453,9 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
continue
}
nodeID := string(devicePrefs.Config.NodeID)
logger.Debugf("deleting device %s from control", nodeID)
err = tsClient.Devices().Delete(ctx, nodeID)
switch {
case tailscale.IsNotFound(err):
logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID)
continue
case err != nil:
return false, fmt.Errorf("error deleting device: %w", err)
if err = tailscaled.EnsureDeviceDeleted(ctx, tsClient, logger, devicePrefs.Config.NodeID); err != nil {
return false, err
}
logger.Debugf("device %s deleted from control", nodeID)
}
// Unlike most log entries in the reconcile loop, this will get printed
@@ -434,83 +463,13 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record
// cleanup removes the tailscale finalizer, which will make all future
// reconciles exit early.
logger.Infof("cleaned up Recorder resources")
r.mu.Lock()
r.recorders.Remove(tsr.UID)
gaugeRecorderResources.Set(int64(r.recorders.Len()))
r.mu.Unlock()
r.tracker.Remove(tsr.UID)
r.reissuer.RemoveState(tsr.Name)
return true, nil
}
func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error {
var replicas int32 = 1
if tsr.Spec.Replicas != nil {
replicas = *tsr.Spec.Replicas
}
tags := tsr.Spec.Tags
if len(tags) == 0 {
tags = tsapi.Tags{"tag:k8s"}
}
logger := r.logger(tsr.Name)
for replica := range replicas {
key := types.NamespacedName{
Namespace: r.tsNamespace,
Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica),
}
existingSecret := &corev1.Secret{}
err := r.Get(ctx, key, existingSecret)
switch {
case err == nil:
stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica)
if err != nil {
return fmt.Errorf("error getting state Secret for replica %d: %w", replica, err)
}
cfgAuthKey := string(existingSecret.Data[authKeySecretKey])
reissue, err := r.reissuer.ShouldReissue(ctx, tsClient, r.log, tailscaled.ReissueInput{
ParentName: tsr.Name,
ReplicaName: fmt.Sprintf("%s-%d", tsr.Name, replica),
Kind: tailscaled.KindRecorder,
StateSecret: stateSecret,
CfgAuthKey: &cfgAuthKey,
})
if err != nil {
return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err)
}
if !reissue {
logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name)
continue
}
authKey, err := newAuthKey(ctx, tsClient, tags.Stringify())
if err != nil {
return err
}
existingSecret.Data[authKeySecretKey] = []byte(authKey)
if err = r.Update(ctx, existingSecret); err != nil {
return err
}
continue
case apierrors.IsNotFound(err):
authKey, err := newAuthKey(ctx, tsClient, tags.Stringify())
if err != nil {
return err
}
if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil {
return err
}
default:
return fmt.Errorf("failed to get Secret %q: %w", key.Name, err)
}
}
return nil
}
func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error {
func (r *Reconciler) validate(ctx context.Context, tsr *tsapi.Recorder) error {
if !tsr.Spec.EnableUI && tsr.Spec.Storage.S3 == nil {
return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible")
}
@@ -535,7 +494,7 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder)
case apierrors.IsNotFound(err):
// ServiceAccount doesn't exist, so no conflict.
case err != nil:
return fmt.Errorf("error getting ServiceAccount %q for validation: %w", tsr.Spec.StatefulSet.Pod.ServiceAccount.Name, err)
return fmt.Errorf("error getting ServiceAccount %q for validation: %w", specSA.Name, err)
default:
// ServiceAccount exists, check if it's owned by the Recorder.
if err := saOwnedByRecorder(sa, tsr); err != nil {
@@ -552,11 +511,13 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder)
return nil
}
func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) {
// getStateSecret returns the tailscaled state Secret for the given Recorder replica, or nil if it does not exist
// yet.
func (r *Reconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: r.tsNamespace,
Name: fmt.Sprintf("%s-%d", tsrName, replica),
Name: stateSecretName(tsrName, replica),
},
}
if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
@@ -570,33 +531,17 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string,
return secret, nil
}
// getDevicePrefs returns 'ok == true' iff the node ID is found. The dnsName
// is expected to always be non-empty if the node ID is, but not required.
func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) {
// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
currentProfile, ok := secret.Data[currentProfileKey]
if !ok {
return prefs, false, nil
}
profileBytes, ok := secret.Data[string(currentProfile)]
if !ok {
return prefs, false, nil
}
if err := json.Unmarshal(profileBytes, &prefs); err != nil {
return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
}
ok = prefs.Config.NodeID != ""
return prefs, ok, nil
}
func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
func (r *Reconciler) getDevicePrefs(ctx context.Context, tsrName string, replica int32) (prefs tailscaled.Prefs, ok bool, err error) {
secret, err := r.getStateSecret(ctx, tsrName, replica)
if err != nil || secret == nil {
return tsapi.RecorderTailnetDevice{}, false, err
return prefs, false, err
}
prefs, ok, err := getDevicePrefs(secret)
return tailscaled.PrefsFromStateSecret(secret)
}
func (r *Reconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) {
prefs, ok, err := r.getDevicePrefs(ctx, tsrName, replica)
if !ok || err != nil {
return tsapi.RecorderTailnetDevice{}, false, err
}
@@ -620,20 +565,11 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclien
return d, true, nil
}
// [prefs] is a subset of the ipn.Prefs struct used for extracting information
// from the state Secret of Tailscale devices.
type prefs struct {
Config struct {
NodeID tailcfg.StableNodeID `json:"NodeID"`
UserProfile struct {
// LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net.
LoginName string `json:"LoginName"`
} `json:"UserProfile"`
} `json:"Config"`
// replicas returns the number of replicas the Recorder asks for, defaulting to one.
func replicas(tsr *tsapi.Recorder) int32 {
if tsr.Spec.Replicas != nil {
return *tsr.Spec.Replicas
}
AdvertiseServices []string `json:"AdvertiseServices"`
}
func markedForDeletion(obj metav1.Object) bool {
return !obj.GetDeletionTimestamp().IsZero()
return 1
}
@@ -0,0 +1,803 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package recorder_test
import (
"encoding/json"
"errors"
"fmt"
"maps"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
tailscaleclient "tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/k8s-operator/reconciler/recorder"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/key"
"tailscale.com/util/clientmetric"
"tailscale.com/util/mak"
"tailscale.com/version"
)
const (
tsNamespace = "tailscale"
tsLoginServer = "example.tailscale.com"
)
func TestRecorder(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
},
Spec: tsapi.RecorderSpec{
Replicas: new(int32(3)),
},
}
fc := reconcilertest.NewClientBuilder().
WithObjects(tsr).
WithStatusSubresource(tsr).
Build()
tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer))
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
fr := record.NewFakeRecorder(2)
r := recorder.NewReconciler(recorder.ReconcilerOptions{
Client: fc,
Recorder: fr,
TailscaleNamespace: tsNamespace,
Clients: reconcilertest.NewFakeClientProvider(tsClient),
Logger: zl.Sugar(),
Clock: tstest.NewClock(tstest.ClockOpts{}),
})
t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) {
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid",
"Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible")
expectManagedCount(t, 0)
expectRecorderResources(t, fc, tsr, false)
reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"})
tsr.Spec.EnableUI = true
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{
"invalid space characters": "test",
}
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible"})
tsr.Spec.Storage.S3 = &tsapi.S3{}
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
// Only check part of this error message, because it's defined in an
// external package and may change.
got := &tsapi.Recorder{}
reconcilertest.MustGet(t, fc, "", tsr.Name, got)
reconcilertest.ExpectConditionStatus(t, got.Status.Conditions, tsapi.RecorderReady, metav1.ConditionFalse, "RecorderInvalid")
cond := reconcilertest.Condition(t, got.Status.Conditions, tsapi.RecorderReady)
for _, msg := range []string{cond.Message, <-fr.Events} {
if !strings.Contains(msg, `"invalid space characters"`) {
t.Fatalf("expected invalid annotation key in error message, got %q", msg)
}
}
})
t.Run("conflicting_service_account_config_marked_as_invalid", func(t *testing.T) {
reconcilertest.MustCreate(t, fc, &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "pre-existing-sa",
Namespace: tsNamespace,
},
})
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = nil
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "pre-existing-sa"
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
msg := `Recorder is invalid: custom ServiceAccount name "pre-existing-sa" specified but conflicts with a pre-existing ServiceAccount in the tailscale namespace`
expectReadyCondition(t, fc, tsr.Name, metav1.ConditionFalse, "RecorderInvalid", msg)
expectManagedCount(t, 0)
reconcilertest.ExpectEvents(t, fr, []string{"Warning RecorderInvalid " + msg})
})
t.Run("observe_Ready_true_status_condition_for_a_valid_spec", func(t *testing.T) {
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = ""
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
// The reason is part of the CRD's observable status, so assert it as a literal: a rename of the constant
// should fail here.
expectReadyCondition(t, fc, tsr.Name, metav1.ConditionTrue, "RecorderCreated", "RecorderCreated")
expectManagedCount(t, 1)
expectRecorderResources(t, fc, tsr, true)
// Every replica's auth key was minted with the default tag, and its
// StatefulSet is pointed at the login server of its tailnet.
if got := len(tsClient.CreateAuthKeyCalls()); got != int(*tsr.Spec.Replicas) {
t.Fatalf("expected %d auth keys to be created, got %d", *tsr.Spec.Replicas, got)
}
for _, req := range tsClient.CreateAuthKeyCalls() {
if diff := cmp.Diff(req.Capabilities.Devices.Create.Tags, []string{"tag:k8s"}); diff != "" {
t.Errorf("unexpected auth key tags (-got +want):\n%s", diff)
}
}
ss := &appsv1.StatefulSet{}
reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, ss)
if got := envValue(ss, "TSRECORDER_LOGIN_SERVER"); got != tsLoginServer {
t.Errorf("unexpected login server env value: got %q, want %q", got, tsLoginServer)
}
})
t.Run("valid_service_account_config", func(t *testing.T) {
tsr.Spec.StatefulSet.Pod.ServiceAccount.Name = "test-sa"
tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations = map[string]string{
"test": "test",
}
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
expectManagedCount(t, 1)
expectRecorderResources(t, fc, tsr, true)
// Get the service account and check the annotations.
sa := &corev1.ServiceAccount{}
reconcilertest.MustGet(t, fc, tsNamespace, "test-sa", sa)
if diff := cmp.Diff(sa.Annotations, tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations); diff != "" {
t.Fatalf("unexpected service account annotations (-got +want):\n%s", diff)
}
// The ServiceAccount named after the Recorder is no longer referenced,
// so it should have been cleaned up.
reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, tsNamespace, tsr.Name)
})
t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) {
for replica := range *tsr.Spec.Replicas {
seedStateSecret(t, fc, fmt.Sprintf("test-%d", replica), fmt.Sprintf("node-%d", replica), fmt.Sprintf("test-%d.example.ts.net", replica))
}
tsClient.SetDevices(
tailscaleclient.Device{
ID: "node-0",
Hostname: "hostname-node-0",
Addresses: []string{"1.2.3.4", "::1"},
},
tailscaleclient.Device{
ID: "node-1",
Hostname: "hostname-node-1",
Addresses: []string{"1.2.3.4", "::1"},
},
tailscaleclient.Device{
ID: "node-2",
Hostname: "hostname-node-2",
Addresses: []string{"1.2.3.4", "::1"},
},
)
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
want := []tsapi.RecorderTailnetDevice{
{
Hostname: "hostname-node-0",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-0.example.ts.net",
},
{
Hostname: "hostname-node-1",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-1.example.ts.net",
},
{
Hostname: "hostname-node-2",
TailnetIPs: []string{"1.2.3.4", "::1"},
URL: "https://test-2.example.ts.net",
},
}
got := &tsapi.Recorder{}
reconcilertest.MustGet(t, fc, "", tsr.Name, got)
if diff := cmp.Diff(got.Status.Devices, want); diff != "" {
t.Fatalf("unexpected devices in status (-got +want):\n%s", diff)
}
})
t.Run("scale_down_deletes_dangling_replica_secrets_and_devices", func(t *testing.T) {
tsr.Spec.Replicas = new(int32(1))
reconcilertest.MustUpdate(t, fc, "", "test", func(o *tsapi.Recorder) {
o.Spec = tsr.Spec
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
// Replicas 1 and 2 are gone from the cluster and from the tailnet;
// replica 0 is untouched.
for _, name := range []string{"test-1", "test-auth-1", "test-2", "test-auth-2"} {
reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, name)
}
reconcilertest.MustGet(t, fc, tsNamespace, "test-0", &corev1.Secret{})
if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
// The Role should no longer grant access to the scaled-down replicas'
// Secrets.
role := &rbacv1.Role{}
reconcilertest.MustGet(t, fc, tsNamespace, tsr.Name, role)
if diff := cmp.Diff(role.Rules[0].ResourceNames, []string{"test-0", "test-auth-0"}); diff != "" {
t.Fatalf("unexpected Role resource names (-got +want):\n%s", diff)
}
})
t.Run("delete_the_Recorder_and_observe_cleanup", func(t *testing.T) {
reconcilertest.MustDeleteAll(t, fc, tsr)
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
expectManagedCount(t, 0)
if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"node-1", "node-2", "node-0"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
// The fake client does not clean up objects whose owner has been
// deleted, so we can't test for the owned resources getting deleted.
})
}
func TestRecorderTailnetUnavailable(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
Spec: tsapi.RecorderSpec{EnableUI: true, Tailnet: "missing"},
}
fc := reconcilertest.NewClientBuilder().
WithObjects(tsr).
WithStatusSubresource(tsr).
Build()
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
r := recorder.NewReconciler(recorder.ReconcilerOptions{
Client: fc,
Recorder: record.NewFakeRecorder(1),
TailscaleNamespace: tsNamespace,
Clients: reconcilertest.NewFailingClientProvider(errors.New(`tailnet "missing" not found`)),
Logger: zl.Sugar(),
Clock: tstest.NewClock(tstest.ClockOpts{}),
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
got := &tsapi.Recorder{}
reconcilertest.MustGet(t, fc, "", tsr.Name, got)
reconcilertest.ExpectCondition(t, got.Status.Conditions, tsapi.RecorderReady,
metav1.ConditionFalse, "RecorderTailnetUnavailable", `tailnet "missing" not found`)
// No resources should have been created for a Recorder we can't mint keys
// for, and no finalizer should have been added.
reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, tsNamespace, tsr.Name)
if len(got.Finalizers) > 0 {
t.Fatalf("expected no finalizers, got %v", got.Finalizers)
}
}
func TestRecorderCleanupContinuesPastUnreportedDevice(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
DeletionTimestamp: new(metav1.Now()),
},
Spec: tsapi.RecorderSpec{
Replicas: new(int32(3)),
Storage: tsapi.Storage{S3: &tsapi.S3{}},
},
}
fc := reconcilertest.NewClientBuilder().
WithObjects(
tsr,
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-0", Namespace: tsNamespace}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Namespace: tsNamespace}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Namespace: tsNamespace}},
).
WithStatusSubresource(tsr).
Build()
tsClient := reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer))
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
r := recorder.NewReconciler(recorder.ReconcilerOptions{
Client: fc,
Recorder: record.NewFakeRecorder(1),
TailscaleNamespace: tsNamespace,
Clients: reconcilertest.NewFakeClientProvider(tsClient),
Logger: zl.Sugar(),
Clock: tstest.NewClock(tstest.ClockOpts{}),
})
// Replica 0 registered with control but never reported its device ID. It comes first, so it must not end cleanup
// early.
mk := key.NewMachine()
mkText, err := mk.MarshalText()
if err != nil {
t.Fatal(err)
}
reconcilertest.MustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) {
s.Data = map[string][]byte{string(ipn.MachineKeyStateKey): mkText}
})
seedStateSecret(t, fc, "test-1", "node-1", "test-1.example.ts.net")
seedStateSecret(t, fc, "test-2", "node-2", "test-2.example.ts.net")
tsClient.SetDevices(
tailscaleclient.Device{NodeID: "unreported-0", MachineKey: mk.Public().String()},
tailscaleclient.Device{NodeID: "other", MachineKey: key.NewMachine().Public().String()},
)
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
reconcilertest.ExpectMissing[tsapi.Recorder](t, fc, "", tsr.Name)
if diff := cmp.Diff(tsClient.DeviceDeletes(), []string{"unreported-0", "node-1", "node-2"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
}
func TestRecorderStatefulSetSpec(t *testing.T) {
tsr := &tsapi.Recorder{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
Spec: tsapi.RecorderSpec{
Replicas: new(int32(3)),
EnableUI: true,
Storage: tsapi.Storage{
S3: &tsapi.S3{
Endpoint: "s3.example.com",
Bucket: "recordings",
Credentials: tsapi.S3Credentials{
Secret: tsapi.S3Secret{Name: "s3-creds"},
},
},
},
StatefulSet: tsapi.RecorderStatefulSet{
Labels: map[string]string{
"ss-label-key": "ss-label-value",
},
Annotations: map[string]string{
"ss-annotation-key": "ss-annotation-value",
},
Pod: tsapi.RecorderPod{
Labels: map[string]string{
"pod-label-key": "pod-label-value",
},
Annotations: map[string]string{
"pod-annotation-key": "pod-annotation-value",
},
Affinity: &corev1.Affinity{
PodAffinity: &corev1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"match-label": "match-value",
},
}},
},
},
},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: new(int64(1000)),
},
ImagePullSecrets: []corev1.LocalObjectReference{{
Name: "img-pull",
}},
NodeSelector: map[string]string{
"some-node": "selector",
},
Tolerations: []corev1.Toleration{{
Key: "key",
Value: "value",
TolerationSeconds: new(int64(60)),
}},
ServiceAccount: tsapi.RecorderServiceAccount{
Name: "custom-sa",
},
Container: tsapi.RecorderContainer{
Env: []tsapi.Env{{
Name: "some_env",
Value: "env_value",
}},
Image: "custom-image",
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
},
},
},
},
},
},
}
fc := reconcilertest.NewClientBuilder().
WithObjects(tsr).
WithStatusSubresource(tsr).
Build()
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
r := recorder.NewReconciler(recorder.ReconcilerOptions{
Client: fc,
Recorder: record.NewFakeRecorder(1),
TailscaleNamespace: tsNamespace,
Clients: reconcilertest.NewFakeClientProvider(reconcilertest.NewFakeClient(reconcilertest.WithLoginURL(tsLoginServer))),
Logger: zl.Sugar(),
Clock: tstest.NewClock(tstest.ClockOpts{}),
})
reconcilertest.ExpectReconciled(t, r, "", tsr.Name)
// Every field that the spec passes through ends up on the Recorder's children.
expectRecorderResources(t, fc, tsr, true)
}
// expectRecorderResources asserts that the full set of resources for each of tsr's replicas either exists or is
// absent. The test builds each expected child itself, without the reconciler's spec builders, so a change to what the
// reconciler writes fails here.
func expectRecorderResources(t *testing.T, fc client.Client, tsr *tsapi.Recorder, shouldExist bool) {
t.Helper()
// The owner references carry the UID of the stored Recorder.
stored := &tsapi.Recorder{}
reconcilertest.MustGet(t, fc, "", tsr.Name, stored)
stored.Spec = tsr.Spec
role := wantRole(stored)
roleBinding := wantRoleBinding(stored)
serviceAccount := wantServiceAccount(stored)
statefulSet := wantStatefulSet(stored)
if !shouldExist {
reconcilertest.ExpectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
reconcilertest.ExpectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)
reconcilertest.ExpectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name)
reconcilertest.ExpectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name)
for replica := range replicaCount(stored) {
reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, authSecretName(stored, replica))
reconcilertest.ExpectMissing[corev1.Secret](t, fc, tsNamespace, stateSecretName(stored, replica))
}
return
}
reconcilertest.ExpectEqual(t, fc, role)
reconcilertest.ExpectEqual(t, fc, roleBinding)
reconcilertest.ExpectEqual(t, fc, serviceAccount)
reconcilertest.ExpectEqual(t, fc, statefulSet)
for replica := range replicaCount(stored) {
auth := &corev1.Secret{}
reconcilertest.MustGet(t, fc, tsNamespace, authSecretName(stored, replica), auth)
if auth.StringData["authkey"] == "" {
t.Errorf("auth Secret %q has no auth key", auth.Name)
}
// The fake Tailscale client generates each auth key, so leave the key out of the comparison.
reconcilertest.ExpectEqual(t, fc, wantSecret(stored, authSecretName(stored, replica)), func(s *corev1.Secret) {
s.StringData = nil
})
reconcilertest.ExpectEqual(t, fc, wantSecret(stored, stateSecretName(stored, replica)))
}
}
func replicaCount(tsr *tsapi.Recorder) int32 {
if tsr.Spec.Replicas == nil {
return 1
}
return *tsr.Spec.Replicas
}
func authSecretName(tsr *tsapi.Recorder, replica int32) string {
return fmt.Sprintf("%s-auth-%d", tsr.Name, replica)
}
func stateSecretName(tsr *tsapi.Recorder, replica int32) string {
return fmt.Sprintf("%s-%d", tsr.Name, replica)
}
func wantServiceAccountName(tsr *tsapi.Recorder) string {
if name := tsr.Spec.StatefulSet.Pod.ServiceAccount.Name; name != "" {
return name
}
return tsr.Name
}
func wantLabels(tsr *tsapi.Recorder, custom map[string]string) map[string]string {
labels := maps.Clone(custom)
mak.Set(&labels, "app.kubernetes.io/name", "recorder")
mak.Set(&labels, "app.kubernetes.io/instance", tsr.Name)
mak.Set(&labels, "app.kubernetes.io/managed-by", "tailscale-operator")
return labels
}
func wantObjectMeta(tsr *tsapi.Recorder, name string) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: name,
Namespace: tsNamespace,
Labels: wantLabels(tsr, nil),
OwnerReferences: []metav1.OwnerReference{{
APIVersion: "tailscale.com/v1alpha1",
Kind: "Recorder",
Name: tsr.Name,
UID: tsr.UID,
Controller: new(true),
BlockOwnerDeletion: new(true),
}},
}
}
func wantSecret(tsr *tsapi.Recorder, name string) *corev1.Secret {
return &corev1.Secret{ObjectMeta: wantObjectMeta(tsr, name)}
}
func wantServiceAccount(tsr *tsapi.Recorder) *corev1.ServiceAccount {
sa := &corev1.ServiceAccount{ObjectMeta: wantObjectMeta(tsr, wantServiceAccountName(tsr))}
sa.Annotations = tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations
return sa
}
func wantRole(tsr *tsapi.Recorder) *rbacv1.Role {
var secrets []string
for replica := range replicaCount(tsr) {
secrets = append(secrets, stateSecretName(tsr, replica), authSecretName(tsr, replica))
}
return &rbacv1.Role{
ObjectMeta: wantObjectMeta(tsr, tsr.Name),
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"secrets"},
Verbs: []string{"get", "patch", "update"},
ResourceNames: secrets,
},
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{"get", "create", "patch"},
},
},
}
}
func wantRoleBinding(tsr *tsapi.Recorder) *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
ObjectMeta: wantObjectMeta(tsr, tsr.Name),
Subjects: []rbacv1.Subject{{
Kind: "ServiceAccount",
Name: wantServiceAccountName(tsr),
Namespace: tsNamespace,
}},
RoleRef: rbacv1.RoleRef{
Kind: "Role",
Name: tsr.Name,
},
}
}
func wantStatefulSet(tsr *tsapi.Recorder) *appsv1.StatefulSet {
pod := tsr.Spec.StatefulSet.Pod
image := pod.Container.Image
if image == "" {
meta := version.GetMeta()
prefix := ""
if meta.UnstableBranch {
prefix = "unstable-"
}
image = fmt.Sprintf("tailscale/tsrecorder:%sv%s", prefix, meta.MajorMinorPatch)
}
env := []corev1.EnvVar{
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}},
{Name: "TS_AUTHKEY_FILE", Value: "/etc/tailscaled/$(POD_NAME)/authkey"},
{Name: "TS_STATE", Value: "kube:$(POD_NAME)"},
{Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)"},
{Name: "TSRECORDER_LOGIN_SERVER", Value: tsLoginServer},
}
for _, e := range pod.Container.Env {
env = append(env, corev1.EnvVar{Name: string(e.Name), Value: e.Value})
}
var envFrom []corev1.EnvFromSource
if s3 := tsr.Spec.Storage.S3; s3 != nil {
env = append(env,
corev1.EnvVar{Name: "TSRECORDER_DST", Value: "s3://" + s3.Endpoint},
corev1.EnvVar{Name: "TSRECORDER_BUCKET", Value: s3.Bucket},
)
if s3.Credentials.Secret.Name != "" {
envFrom = []corev1.EnvFromSource{{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{Name: s3.Credentials.Secret.Name},
},
}}
}
} else {
env = append(env, corev1.EnvVar{Name: "TSRECORDER_DST", Value: "/data/recordings"})
}
if tsr.Spec.EnableUI {
env = append(env, corev1.EnvVar{Name: "TSRECORDER_UI", Value: "true"})
}
mounts := []corev1.VolumeMount{{Name: "data", MountPath: "/data"}}
volumes := []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}
for replica := range replicaCount(tsr) {
name := fmt.Sprintf("authkey-%d", replica)
mounts = append(mounts, corev1.VolumeMount{
Name: name,
ReadOnly: true,
MountPath: "/etc/tailscaled/" + stateSecretName(tsr, replica),
})
volumes = append(volumes, corev1.Volume{
Name: name,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: authSecretName(tsr, replica),
Items: []corev1.KeyToPath{{Key: "authkey", Path: "authkey"}},
},
},
})
}
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: tsr.Name,
Namespace: tsNamespace,
Labels: wantLabels(tsr, tsr.Spec.StatefulSet.Labels),
Annotations: tsr.Spec.StatefulSet.Annotations,
OwnerReferences: wantObjectMeta(tsr, tsr.Name).OwnerReferences,
},
Spec: appsv1.StatefulSetSpec{
Replicas: new(replicaCount(tsr)),
Selector: &metav1.LabelSelector{MatchLabels: wantLabels(tsr, pod.Labels)},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: tsr.Name,
Namespace: tsNamespace,
Labels: wantLabels(tsr, pod.Labels),
Annotations: pod.Annotations,
},
Spec: corev1.PodSpec{
ServiceAccountName: wantServiceAccountName(tsr),
Affinity: pod.Affinity,
SecurityContext: pod.SecurityContext,
ImagePullSecrets: pod.ImagePullSecrets,
NodeSelector: pod.NodeSelector,
Tolerations: pod.Tolerations,
Containers: []corev1.Container{{
Name: "recorder",
Image: image,
ImagePullPolicy: pod.Container.ImagePullPolicy,
Resources: pod.Container.Resources,
SecurityContext: pod.Container.SecurityContext,
Env: env,
EnvFrom: envFrom,
Command: []string{"/tsrecorder"},
VolumeMounts: mounts,
}},
Volumes: volumes,
},
},
},
}
}
// expectReadyCondition asserts the Recorder's RecorderReady condition.
func expectReadyCondition(t *testing.T, fc client.Client, name string, status metav1.ConditionStatus, reason, message string) {
t.Helper()
tsr := &tsapi.Recorder{}
reconcilertest.MustGet(t, fc, "", name, tsr)
reconcilertest.ExpectCondition(t, tsr.Status.Conditions, tsapi.RecorderReady, status, reason, message)
}
// expectManagedCount asserts the number of Recorders the reconciler reports as managed via its clientmetric gauge.
func expectManagedCount(t *testing.T, want int64) {
t.Helper()
for _, m := range clientmetric.Metrics() {
if m.Name() == kubetypes.MetricRecorderCount {
if got := m.Value(); got != want {
t.Fatalf("expected %d recorders, got %d", want, got)
}
return
}
}
t.Fatalf("metric %q not found", kubetypes.MetricRecorderCount)
}
// seedStateSecret writes the profile data that tailscaled would have written to a replica's state Secret once it has
// authenticated.
func seedStateSecret(t *testing.T, fc client.Client, name, nodeID, loginName string) {
t.Helper()
const key = "profile-abc"
body, err := json.Marshal(map[string]any{
"Config": map[string]any{
"NodeID": nodeID,
"UserProfile": map[string]any{
"LoginName": loginName,
},
},
})
if err != nil {
t.Fatal(err)
}
reconcilertest.MustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) {
s.Data = map[string][]byte{
string(ipn.CurrentProfileStateKey): []byte(key),
key: body,
}
})
}
func envValue(ss *appsv1.StatefulSet, name string) string {
for _, env := range ss.Spec.Template.Spec.Containers[0].Env {
if env.Name == name {
return env.Value
}
}
return ""
}
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package recorder
import (
"fmt"
@@ -115,7 +115,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) *
Name: volumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: fmt.Sprintf("%s-auth-%d", tsr.Name, replica),
SecretName: authSecretName(tsr.Name, replica),
Items: []corev1.KeyToPath{{Key: authKeySecretKey, Path: authKeySecretKey}},
},
},
@@ -156,8 +156,8 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
resourceNames := make([]string, 0)
for replica := range replicas {
resourceNames = append(resourceNames,
fmt.Sprintf("%s-%d", tsr.Name, replica), // State secret.
fmt.Sprintf("%s-auth-%d", tsr.Name, replica), // Auth key secret.
stateSecretName(tsr.Name, replica), // State secret.
authSecretName(tsr.Name, replica), // Auth key secret.
)
}
@@ -218,7 +218,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica),
Name: authSecretName(tsr.Name, replica),
Labels: tsrLabels("recorder", tsr.Name, nil),
OwnerReferences: tsrOwnerReference(tsr),
},
@@ -231,7 +231,7 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replic
func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", tsr.Name, replica),
Name: stateSecretName(tsr.Name, replica),
Namespace: namespace,
Labels: tsrLabels("recorder", tsr.Name, nil),
OwnerReferences: tsrOwnerReference(tsr),
@@ -239,6 +239,17 @@ func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev
}
}
// stateSecretName returns the name of the tailscaled state Secret for the given Recorder replica. It matches the
// replica's Pod name, which is what tsrecorder reads from TS_STATE.
func stateSecretName(tsrName string, replica int32) string {
return fmt.Sprintf("%s-%d", tsrName, replica)
}
// authSecretName returns the name of the Secret holding the auth key for the given Recorder replica.
func authSecretName(tsrName string, replica int32) string {
return fmt.Sprintf("%s-auth-%d", tsrName, replica)
}
func tsrEnv(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar {
envs := []corev1.EnvVar{
{
@@ -13,10 +13,10 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/k8s-operator/reconciler/tailnet"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/tstest"
@@ -374,7 +374,7 @@ func TestReconciler_Reconcile(t *testing.T) {
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme)
builder := reconcilertest.NewClientBuilder()
if tc.Tailnet != nil {
builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet)
}
@@ -0,0 +1,52 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package tailscaled
import (
"encoding/json"
"fmt"
corev1 "k8s.io/api/core/v1"
"tailscale.com/ipn"
"tailscale.com/tailcfg"
)
// Prefs is the subset of ipn.Prefs that operator reconcilers need to read back out of a tailscaled state Secret.
// tailscaled writes the full prefs for the current profile; only the fields declared here are unmarshalled.
type Prefs struct {
Config struct {
NodeID tailcfg.StableNodeID `json:"NodeID"`
UserProfile struct {
// LoginName is the MagicDNS name of the device, e.g. foo.tail-scale.ts.net.
LoginName string `json:"LoginName"`
} `json:"UserProfile"`
} `json:"Config"`
AdvertiseServices []string `json:"AdvertiseServices"`
}
// PrefsFromStateSecret returns the prefs tailscaled recorded for its current profile in secret. ok reports whether
// a node ID was found; it is false when the device hasn't finished authenticating yet, in which case prefs is not
// meaningful. LoginName is expected to be set whenever the node ID is, but that is not required.
//
// TODO(tomhjp): Should maybe use ipn to parse the following info instead.
func PrefsFromStateSecret(secret *corev1.Secret) (prefs Prefs, ok bool, err error) {
currentProfile, ok := secret.Data[string(ipn.CurrentProfileStateKey)]
if !ok {
return prefs, false, nil
}
profileBytes, ok := secret.Data[string(currentProfile)]
if !ok {
return prefs, false, nil
}
if err := json.Unmarshal(profileBytes, &prefs); err != nil {
return prefs, false, fmt.Errorf("failed to extract node profile info from state Secret %s: %w", secret.Name, err)
}
ok = prefs.Config.NodeID != ""
return prefs, ok, nil
}
@@ -0,0 +1,104 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package tailscaled_test
import (
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"tailscale.com/k8s-operator/reconciler/tailscaled"
)
func TestPrefsFromStateSecret(t *testing.T) {
t.Parallel()
tests := map[string]struct {
data map[string][]byte
wantOK bool
wantErr bool
wantNodeID string
wantLogin string
wantSvcs []string
}{
"populated": {
data: map[string][]byte{
"_current-profile": []byte("profile-abc"),
"profile-abc": []byte(`{"Config":{"NodeID":"node-foo","UserProfile":{"LoginName":"foo.example.ts.net"}},"AdvertiseServices":["svc:bar"]}`),
},
wantOK: true,
wantNodeID: "node-foo",
wantLogin: "foo.example.ts.net",
wantSvcs: []string{"svc:bar"},
},
"no-current-profile": {
data: map[string][]byte{
"profile-abc": []byte(`{"Config":{"NodeID":"node-foo"}}`),
},
},
"current-profile-points-at-missing-profile": {
data: map[string][]byte{
"_current-profile": []byte("profile-abc"),
},
},
// The device hasn't finished authenticating, so there's no node ID yet.
"no-node-id": {
data: map[string][]byte{
"_current-profile": []byte("profile-abc"),
"profile-abc": []byte(`{"Config":{"UserProfile":{"LoginName":"foo.example.ts.net"}}}`),
},
wantLogin: "foo.example.ts.net",
},
"empty": {},
"malformed-profile": {
data: map[string][]byte{
"_current-profile": []byte("profile-abc"),
"profile-abc": []byte(`not json`),
},
wantErr: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "tailscale"},
Data: tc.data,
}
prefs, ok, err := tailscaled.PrefsFromStateSecret(secret)
switch {
case tc.wantErr && err == nil:
t.Fatal("expected an error, got nil")
case !tc.wantErr && err != nil:
t.Fatalf("unexpected error: %v", err)
case tc.wantErr:
return
}
if ok != tc.wantOK {
t.Errorf("ok = %v, want %v", ok, tc.wantOK)
}
if got := string(prefs.Config.NodeID); got != tc.wantNodeID {
t.Errorf("NodeID = %q, want %q", got, tc.wantNodeID)
}
if got := prefs.Config.UserProfile.LoginName; got != tc.wantLogin {
t.Errorf("LoginName = %q, want %q", got, tc.wantLogin)
}
if got, want := len(prefs.AdvertiseServices), len(tc.wantSvcs); got != want {
t.Fatalf("len(AdvertiseServices) = %d, want %d", got, want)
}
for i, svc := range tc.wantSvcs {
if prefs.AdvertiseServices[i] != svc {
t.Errorf("AdvertiseServices[%d] = %q, want %q", i, prefs.AdvertiseServices[i], svc)
}
}
})
}
}
@@ -6,18 +6,15 @@
package tailscaled_test
import (
"context"
"slices"
"strings"
"sync"
"testing"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
tailscaleclient "tailscale.com/client/tailscale/v2"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
)
@@ -72,7 +69,7 @@ func TestReissuer_ShouldReissue(t *testing.T) {
stateSecret = &corev1.Secret{Data: tc.stateData}
}
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
r.EnsureState("pg", 1)
@@ -89,8 +86,8 @@ func TestReissuer_ShouldReissue(t *testing.T) {
if got != tc.wantReissue {
t.Errorf("shouldReissue = %v, want %v", got, tc.wantReissue)
}
if !slices.Equal(tsc.deleted, tc.wantDeleted) {
t.Errorf("deleted devices = %v, want %v", tsc.deleted, tc.wantDeleted)
if !slices.Equal(tsc.DeviceDeletes(), tc.wantDeleted) {
t.Errorf("deleted devices = %v, want %v", tsc.DeviceDeletes(), tc.wantDeleted)
}
})
}
@@ -101,7 +98,7 @@ func TestReissuer_ShouldReissue(t *testing.T) {
// state Secret.
func TestReissuer_InFlight(t *testing.T) {
key := "broken-key"
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
r.EnsureState("pg", 1)
@@ -137,7 +134,7 @@ func TestReissuer_InFlight(t *testing.T) {
// across replicas, blocks re-issuance once its burst is exhausted.
func TestReissuer_RateLimit(t *testing.T) {
key := "broken-key"
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
// Burst of 1: the first replica's reissue consumes the only token.
r.EnsureState("pg", 1)
@@ -169,7 +166,7 @@ func TestReissuer_RateLimit(t *testing.T) {
// EnsureState reissues instead of panicking on a nil limiter.
func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) {
key := "broken-key"
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
broken := &corev1.Secret{Data: map[string][]byte{
@@ -183,8 +180,8 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) {
if err != nil || !got {
t.Fatalf("got %v, err %v; want true, nil", got, err)
}
if !slices.Equal(tsc.deleted, []string{"nodeid-0"}) {
t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.deleted)
if !slices.Equal(tsc.DeviceDeletes(), []string{"nodeid-0"}) {
t.Errorf("deleted devices = %v, want [nodeid-0]", tsc.DeviceDeletes())
}
}
@@ -192,7 +189,7 @@ func TestReissuer_ShouldReissue_NoEnsureState(t *testing.T) {
// is cleared, so a later reissue request for the same replica triggers afresh.
func TestReissuer_RemoveState(t *testing.T) {
key := "broken-key"
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
r.EnsureState("pg", 1)
@@ -226,7 +223,7 @@ func TestReissuer_RemoveState(t *testing.T) {
// parent scaled down then deleted doesn't strand in-flight entries.
func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) {
key := "broken-key"
tsc := &fakeReissueClient{}
tsc := reconcilertest.NewFakeClient()
r := tailscaled.NewReissuer()
r.EnsureState("pg", 2)
@@ -260,30 +257,3 @@ func TestReissuer_RemoveState_AfterScaleDown(t *testing.T) {
t.Fatalf("pg-1 after RemoveState: got %v, err %v; want true, nil", got, err)
}
}
type fakeReissueClient struct {
tsclient.Client
mu sync.Mutex
deleted []string
}
func (c *fakeReissueClient) Devices() tsclient.DeviceResource { return (*fakeReissueDevices)(c) }
type fakeReissueDevices fakeReissueClient
func (d *fakeReissueDevices) Delete(_ context.Context, id string) error {
c := (*fakeReissueClient)(d)
c.mu.Lock()
defer c.mu.Unlock()
c.deleted = append(c.deleted, id)
return nil
}
func (d *fakeReissueDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) {
return nil, nil
}
func (d *fakeReissueDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) {
return nil, nil
}