mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 08:41:19 -08:00
Move the Recorder reconciler and its object specs into k8s-operator/reconciler/recorder, following the earlier ProxyClass, DNSConfig and nameserver moves, and switch it to the shared finalizer, resource tracker and CreateOrUpdate helpers. Hoist two duplicated helpers into tailscaled.PrefsFromStateSecret and reconciler.IsOptimisticLockError, and add k8s-operator/reconciler/reconcilertest for the test helpers each package had copied, including one fake Tailscale API client in place of four. No behaviour change; the Recorder's watches and labels are untouched. Fixes tailscale/corp#37086 Signed-off-by: David Bond <[email protected]>
394 lines
14 KiB
Go
394 lines
14 KiB
Go
// Copyright (c) Tailscale Inc & contributors
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"go.uber.org/zap"
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"tailscale.com/client/tailscale/v2"
|
|
|
|
tsoperator "tailscale.com/k8s-operator"
|
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
|
"tailscale.com/k8s-operator/reconciler/tailscaled"
|
|
"tailscale.com/k8s-operator/tsclient"
|
|
"tailscale.com/kube/k8s-proxy/conf"
|
|
"tailscale.com/kube/kubetypes"
|
|
"tailscale.com/tailcfg"
|
|
"tailscale.com/tstest"
|
|
"tailscale.com/types/opt"
|
|
)
|
|
|
|
func TestAPIServerProxyReconciler(t *testing.T) {
|
|
const (
|
|
pgName = "test-pg"
|
|
pgUID = "test-pg-uid"
|
|
ns = "operator-ns"
|
|
defaultDomain = "test-pg.ts.net"
|
|
)
|
|
pg := &tsapi.ProxyGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: pgName,
|
|
Generation: 1,
|
|
UID: pgUID,
|
|
},
|
|
Spec: tsapi.ProxyGroupSpec{
|
|
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
|
|
},
|
|
Status: tsapi.ProxyGroupStatus{
|
|
Conditions: []metav1.Condition{
|
|
{
|
|
Type: string(tsapi.ProxyGroupAvailable),
|
|
Status: metav1.ConditionTrue,
|
|
ObservedGeneration: 1,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
initialCfg := &conf.VersionedConfig{
|
|
Version: "v1alpha1",
|
|
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
|
|
AuthKey: new("test-key"),
|
|
APIServerProxy: &conf.APIServerProxyConfig{
|
|
Enabled: opt.NewBool(true),
|
|
},
|
|
},
|
|
}
|
|
expectedCfg := *initialCfg
|
|
initialCfgB, err := json.Marshal(initialCfg)
|
|
if err != nil {
|
|
t.Fatalf("marshaling initial config: %v", err)
|
|
}
|
|
pgCfgSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: pgConfigSecretName(pgName, 0),
|
|
Namespace: ns,
|
|
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
|
|
},
|
|
Data: map[string][]byte{
|
|
// Existing config should be preserved.
|
|
kubetypes.KubeAPIServerConfigFile: initialCfgB,
|
|
},
|
|
}
|
|
fc := fake.NewClientBuilder().
|
|
WithScheme(tsapi.GlobalScheme).
|
|
WithObjects(pg, pgCfgSecret).
|
|
WithStatusSubresource(pg).
|
|
Build()
|
|
expectCfg := func(c *conf.VersionedConfig) {
|
|
t.Helper()
|
|
cBytes, err := json.Marshal(c)
|
|
if err != nil {
|
|
t.Fatalf("marshaling expected config: %v", err)
|
|
}
|
|
pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes
|
|
expectEqual(t, fc, pgCfgSecret)
|
|
}
|
|
|
|
ft := &fakeTSClient{
|
|
vipServices: make(map[string]tailscale.VIPService),
|
|
}
|
|
ingressTSSvc := tailscale.VIPService{
|
|
Name: "svc:some-ingress-hostname",
|
|
Comment: managedTSServiceComment,
|
|
Annotations: map[string]string{
|
|
// No resource field.
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`,
|
|
},
|
|
Ports: []string{"tcp:443"},
|
|
Tags: []string{"tag:k8s"},
|
|
Addrs: []string{"5.6.7.8"},
|
|
}
|
|
ft.VIPServices().CreateOrUpdate(t.Context(), ingressTSSvc)
|
|
|
|
r := &KubeAPIServerTSServiceReconciler{
|
|
Client: fc,
|
|
clients: tsclient.NewProvider(ft),
|
|
defaultTags: []string{"tag:k8s"},
|
|
tsNamespace: ns,
|
|
logger: zap.Must(zap.NewDevelopment()).Sugar(),
|
|
recorder: record.NewFakeRecorder(10),
|
|
clock: tstest.NewClock(tstest.ClockOpts{}),
|
|
operatorID: "self-id",
|
|
}
|
|
|
|
// Create a Tailscale Service that will conflict with the initial config.
|
|
if err := ft.VIPServices().CreateOrUpdate(t.Context(), tailscale.VIPService{
|
|
Name: "svc:" + pgName,
|
|
}); err != nil {
|
|
t.Fatalf("creating initial Tailscale Service: %v", err)
|
|
}
|
|
expectReconciled(t, r, "", pgName)
|
|
pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName}
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger)
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
|
|
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
|
|
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
|
|
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
|
|
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
|
|
expectEqual(t, fc, pgCfgSecret) // Unchanged.
|
|
|
|
// Delete Tailscale Service; should see Service created and valid condition updated to true.
|
|
if err := ft.VIPServices().Delete(t.Context(), "svc:"+pgName); err != nil {
|
|
t.Fatalf("deleting initial Tailscale Service: %v", err)
|
|
}
|
|
|
|
// Create the state secret for the ProxyGroup without services being advertised.
|
|
mustCreate(t, fc, &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pg-0",
|
|
Namespace: ns,
|
|
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
|
|
},
|
|
Data: map[string][]byte{
|
|
"_current-profile": []byte("test"),
|
|
"test": []byte(`{"Config":{"NodeID":"node-foo", "UserProfile": {"LoginName": "test-pg.ts.net" }}}`),
|
|
},
|
|
})
|
|
|
|
expectReconciled(t, r, "", pgName)
|
|
|
|
tsSvc, err := ft.VIPServices().Get(t.Context(), "svc:"+pgName)
|
|
if err != nil {
|
|
t.Fatalf("getting Tailscale Service: %v", err)
|
|
}
|
|
if tsSvc == nil {
|
|
t.Fatalf("expected Tailscale Service to be created, but got nil")
|
|
}
|
|
expectedTSSvc := &tailscale.VIPService{
|
|
Name: "svc:" + pgName,
|
|
Comment: managedTSServiceComment,
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`,
|
|
},
|
|
Ports: []string{"tcp:443"},
|
|
Tags: []string{"tag:k8s"},
|
|
Addrs: []string{"5.6.7.8"},
|
|
}
|
|
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
|
|
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
|
|
}
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger)
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
|
|
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
|
|
|
|
expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName))
|
|
expectCfg(&expectedCfg)
|
|
|
|
expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg))
|
|
expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain))
|
|
expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain))
|
|
|
|
// Simulate certs being issued; should observe AdvertiseServices config change.
|
|
populateTLSSecret(t, fc, pgName, defaultDomain)
|
|
expectReconciled(t, r, "", pgName)
|
|
|
|
expectedCfg.AdvertiseServices = []string{"svc:" + pgName}
|
|
expectCfg(&expectedCfg)
|
|
|
|
expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status.
|
|
|
|
// Simulate Pod prefs updated with advertised services; should see Configured condition updated to true.
|
|
mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) {
|
|
var p tailscaled.Prefs
|
|
if err = json.Unmarshal(o.Data["test"], &p); err != nil {
|
|
t.Errorf("failed to unmarshal preferences: %v", err)
|
|
}
|
|
|
|
p.AdvertiseServices = []string{"svc:test-pg"}
|
|
o.Data["test"], err = json.Marshal(p)
|
|
if err != nil {
|
|
t.Errorf("failed to marshal preferences: %v", err)
|
|
}
|
|
})
|
|
|
|
expectReconciled(t, r, "", pgName)
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
|
|
pg.Status.URL = "https://" + defaultDomain
|
|
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
|
|
|
|
// Rename the Tailscale Service - old one + cert resources should be cleaned up.
|
|
updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed")
|
|
updatedDomain := "test-pg-renamed.ts.net"
|
|
pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{
|
|
Hostname: updatedServiceName.WithoutPrefix(),
|
|
}
|
|
mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) {
|
|
p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer
|
|
})
|
|
expectReconciled(t, r, "", pgName)
|
|
_, err = ft.VIPServices().Get(t.Context(), "svc:"+pgName)
|
|
if !tailscale.IsNotFound(err) {
|
|
t.Fatalf("Expected 404, got: %v", err)
|
|
}
|
|
tsSvc, err = ft.VIPServices().Get(t.Context(), updatedServiceName.String())
|
|
if err != nil {
|
|
t.Fatalf("Expected renamed svc, got error: %v", err)
|
|
}
|
|
expectedTSSvc.Name = updatedServiceName.String()
|
|
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
|
|
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
|
|
}
|
|
// Check cfg and status reset until TLS certs are available again.
|
|
expectedCfg.APIServerProxy.ServiceName = new(updatedServiceName)
|
|
expectedCfg.AdvertiseServices = nil
|
|
expectCfg(&expectedCfg)
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
|
|
pg.Status.URL = ""
|
|
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
|
|
|
|
expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg))
|
|
expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain))
|
|
expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain))
|
|
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
|
|
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
|
|
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
|
|
|
|
// Check we get the new hostname in the status once ready.
|
|
populateTLSSecret(t, fc, pgName, updatedDomain)
|
|
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) {
|
|
s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`)
|
|
})
|
|
expectReconciled(t, r, "", pgName)
|
|
expectedCfg.AdvertiseServices = []string{updatedServiceName.String()}
|
|
expectCfg(&expectedCfg)
|
|
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
|
|
pg.Status.URL = "https://" + updatedDomain
|
|
|
|
// Delete the ProxyGroup and verify Tailscale Service and cert resources are
|
|
// cleaned up. Delete the state Secret first, as a Foreground delete does
|
|
// on older versions of the operator, so the cleanup cannot read the DNS
|
|
// name from it.
|
|
mustDeleteAll(t, fc, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "test-pg-0", Namespace: ns}})
|
|
if err := fc.Delete(t.Context(), pg); err != nil {
|
|
t.Fatalf("deleting ProxyGroup: %v", err)
|
|
}
|
|
expectReconciled(t, r, "", pgName)
|
|
expectMissing[corev1.Secret](t, fc, ns, updatedDomain)
|
|
expectMissing[rbacv1.Role](t, fc, ns, updatedDomain)
|
|
expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain)
|
|
_, err = ft.VIPServices().Get(t.Context(), updatedServiceName.String())
|
|
if !tailscale.IsNotFound(err) {
|
|
t.Fatalf("Expected 404, got: %v", err)
|
|
}
|
|
|
|
// Ingress Tailscale Service should not be affected.
|
|
svc, err := ft.VIPServices().Get(t.Context(), ingressTSSvc.Name)
|
|
if err != nil {
|
|
t.Fatalf("getting ingress Tailscale Service: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(svc, &ingressTSSvc) {
|
|
t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc)
|
|
}
|
|
}
|
|
|
|
func TestExclusiveOwnerAnnotations(t *testing.T) {
|
|
pg := &tsapi.ProxyGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "pg1",
|
|
UID: "pg1-uid",
|
|
},
|
|
}
|
|
const (
|
|
pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}`
|
|
)
|
|
|
|
for name, tc := range map[string]struct {
|
|
svc *tailscale.VIPService
|
|
wantErr string
|
|
}{
|
|
"no_svc": {
|
|
svc: nil,
|
|
},
|
|
"empty_svc": {
|
|
svc: &tailscale.VIPService{},
|
|
wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator",
|
|
},
|
|
"already_owner": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: pg1Owner,
|
|
},
|
|
},
|
|
},
|
|
"already_owner_name_updated": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`,
|
|
},
|
|
},
|
|
},
|
|
"preserves_existing_annotations": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
"existing": "annotation",
|
|
ownerAnnotation: pg1Owner,
|
|
},
|
|
},
|
|
},
|
|
"owned_by_another_operator": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`,
|
|
},
|
|
},
|
|
wantErr: "already owned by other operator(s)",
|
|
},
|
|
"owned_by_an_ingress": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet).
|
|
},
|
|
},
|
|
wantErr: "does not reference an owning resource",
|
|
},
|
|
"owned_by_another_pg": {
|
|
svc: &tailscale.VIPService{
|
|
Annotations: map[string]string{
|
|
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`,
|
|
},
|
|
},
|
|
wantErr: "already owned by another resource",
|
|
},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc)
|
|
if tc.wantErr != "" {
|
|
if !strings.Contains(err.Error(), tc.wantErr) {
|
|
t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr)
|
|
}
|
|
} else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" {
|
|
t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff)
|
|
}
|
|
if tc.svc == nil {
|
|
return // Don't check annotations being preserved.
|
|
}
|
|
for k, v := range tc.svc.Annotations {
|
|
if k == ownerAnnotation {
|
|
continue
|
|
}
|
|
if got[k] != v {
|
|
t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) {
|
|
for i := range p.Status.Conditions {
|
|
// Don't bother validating the message.
|
|
p.Status.Conditions[i].Message = ""
|
|
}
|
|
}
|