Files
tailscale/cmd/k8s-operator/operator_test.go
David Bond 689a0c155f cmd/k8s-operator,k8s-operator/reconciler/staticendpoints: support static endpoints for Connector resources (#21172)
Previously ProxyClass spec.staticEndpoints was only implemented for
ProxyGroups and silently ignored when the ProxyClass was referenced by
a Connector. Extract the NodePort Service provisioning, port
allocation, and node ExternalIP discovery logic shared with ProxyGroup
into a new k8s-operator/reconciler/staticendpoints package, and wire
it into the StatefulSet reconciler so that each Connector replica gets
a per-pod NodePort Service, the discovered endpoints are written to
its tailscaled config as static endpoints, and tailscaled listens on
the Services' target port via the PORT env var. Also reconcile
Connectors on Node changes, clean up NodePort Services on scale down,
Connector deletion, and when static endpoints are removed, and
surface the endpoints in the Connector's device status.

Add e2e tests for static endpoints on both Connectors and ProxyGroups.
They verify the per-replica NodePort Services, the PORT env var, the
endpoints reported in status and advertised to control, and the
cleanup on scale down and on removal of the static endpoints
configuration. The tests skip on clusters whose Nodes have no
ExternalIP addresses, such as kind.

Fixes #18819

Change-Id: Ia134916062471e6c0088692834257aafabd0e713

Signed-off-by: David Bond <[email protected]>
2026-10-01 12:07:16 +01:00

2246 lines
70 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/k8s-operator/apis/v1alpha1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/dns/resolvconffile"
"tailscale.com/tstest"
"tailscale.com/tstime"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
)
func TestLoadBalancerClass(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
recorder: record.NewFakeRecorder(100),
}
// Create a service that we should manage, but start with a miconfiguration
// in the annotations.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: "invalid.example.com",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
// The expected value of .status.conditions[0].LastTransitionTime until the
// proxy becomes ready.
t0 := conditionTime(clock)
// Should have an error about invalid config.
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: "invalid.example.com",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
Status: corev1.ServiceStatus{
Conditions: []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionFalse,
LastTransitionTime: t0,
Reason: reasonProxyInvalid,
Message: `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-fqdn: "invalid.example.com" does not appear to be a valid MagicDNS name`,
}},
},
}
expectEqual(t, fc, want)
// Delete the misconfiguration so the proxy starts getting created on the
// next reconcile.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = nil
})
clock.Advance(time.Second)
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
want.Annotations = nil
want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
want.Status = corev1.ServiceStatus{
Conditions: []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionFalse,
LastTransitionTime: t0, // Status is still false, no update to transition time
Reason: reasonProxyPending,
Message: "no Tailscale hostname known yet, waiting for proxy pod to finish auth",
}},
}
expectEqual(t, fc, want)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
// that we get to the end.
mustUpdate(t, fc, "operator-ns", fullName, func(s *corev1.Secret) {
if s.Data == nil {
s.Data = map[string][]byte{}
}
s.Data["device_id"] = []byte("ts-id-1234")
s.Data["device_fqdn"] = []byte("tailscale.device.name.")
s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
})
clock.Advance(time.Second)
expectReconciled(t, sr, "default", "test")
want.Status.Conditions = proxyCreatedCondition(clock)
want.Status.LoadBalancer = corev1.LoadBalancerStatus{
Ingress: []corev1.LoadBalancerIngress{
{
Hostname: "tailscale.device.name",
},
{
IP: "100.99.98.97",
},
},
}
// Perform an additional reconciliation loop here to ensure resources don't change through side effects. Mainly
// to prevent infinite reconciliation
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, want)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Type = corev1.ServiceTypeClusterIP
s.Spec.LoadBalancerClass = nil
})
mustUpdateStatus(t, fc, "default", "test", func(s *corev1.Service) {
// Fake client doesn't automatically delete the LoadBalancer status when
// changing away from the LoadBalancer type, we have to do
// controller-manager's work by hand.
s.Status = corev1.ServiceStatus{}
})
// synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// didn't create any child resources since this is all faked, so the
// deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// The deletion triggers another reconcile, to finish the cleanup.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
// Note that the Tailscale-specific condition status should be gone now.
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
}
func TestTailnetTargetFQDNAnnotation(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
tailnetTargetFQDN := "foo.bar.ts.net."
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
"foo": "bar",
},
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
tailnetTargetFQDN: tailnetTargetFQDN,
hostname: "default-test",
app: kubetypes.AppEgressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
},
},
Spec: corev1.ServiceSpec{
ExternalName: fmt.Sprintf("%s.operator-ns.svc.cluster.local", shortName),
Type: corev1.ServiceTypeExternalName,
Selector: nil,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
// Change the tailscale-target-fqdn annotation which should update the
// StatefulSet
tailnetTargetFQDN = "bar.baz.ts.net"
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{
AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
}
})
// Remove the tailscale-target-fqdn annotation which should make the
// operator clean up
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{}
})
expectReconciled(t, sr, "default", "test")
// // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// // didn't create any child resources since this is all faked, so the
// // deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// // The deletion triggers another reconcile, to finish the cleanup.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
}
func TestTailnetTargetIPAnnotation(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
tailnetTargetIP := "100.66.66.66"
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
"foo": "bar",
},
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
tailnetTargetIP: tailnetTargetIP,
hostname: "default-test",
app: kubetypes.AppEgressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ExternalName: fmt.Sprintf("%s.operator-ns.svc.cluster.local", shortName),
Type: corev1.ServiceTypeExternalName,
Selector: nil,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
tailnetTargetIP = "100.77.77.77"
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
}
})
// Remove the tailscale-target-ip annotation which should make the
// operator clean up
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{}
})
expectReconciled(t, sr, "default", "test")
// synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// didn't create any child resources since this is all faked, so the
// deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// The deletion triggers another reconcile, to finish the cleanup.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
}
func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
recorder: record.NewFakeRecorder(100),
}
tailnetTargetIP := "invalid-ip"
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
t0 := conditionTime(clock)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
Status: corev1.ServiceStatus{
Conditions: []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionFalse,
LastTransitionTime: t0,
Reason: reasonProxyInvalid,
Message: `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "invalid-ip" could not be parsed as a valid IP Address, error: ParseAddr("invalid-ip"): unable to parse IP`,
}},
},
}
expectEqual(t, fc, want)
}
func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
recorder: record.NewFakeRecorder(100),
}
tailnetTargetIP := "999.999.999.999"
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
t0 := conditionTime(clock)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetIP: tailnetTargetIP,
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
Status: corev1.ServiceStatus{
Conditions: []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionFalse,
LastTransitionTime: t0,
Reason: reasonProxyInvalid,
Message: `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-ip: "999.999.999.999" could not be parsed as a valid IP Address, error: ParseAddr("999.999.999.999"): IPv4 field has value >255`,
}},
},
}
expectEqual(t, fc, want)
}
func TestAnnotations(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
delete(s.ObjectMeta.Annotations, "tailscale.com/expose")
})
// synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// didn't create any child resources since this is all faked, so the
// deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// Second time around, the rest of cleanup happens.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
}
func TestAnnotationIntoLB(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, since it would have normally happened at
// this point and the LoadBalancer is going to expect this.
mustUpdate(t, fc, "operator-ns", fullName, func(s *corev1.Secret) {
if s.Data == nil {
s.Data = map[string][]byte{}
}
s.Data["device_id"] = []byte("ts-id-1234")
s.Data["device_fqdn"] = []byte("tailscale.device.name.")
s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
})
expectReconciled(t, sr, "default", "test")
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
// Remove Tailscale's annotation, and at the same time convert the service
// into a tailscale LoadBalancer.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
delete(s.ObjectMeta.Annotations, "tailscale.com/expose")
s.Spec.Type = corev1.ServiceTypeLoadBalancer
s.Spec.LoadBalancerClass = new("tailscale")
})
expectReconciled(t, sr, "default", "test")
// None of the proxy machinery should have changed...
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
// ... but the service should have a LoadBalancer status.
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
Status: corev1.ServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{
Ingress: []corev1.LoadBalancerIngress{
{
Hostname: "tailscale.device.name",
},
{
IP: "100.99.98.97",
},
},
},
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
}
func TestLBIntoAnnotation(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
// that we get to the end.
mustUpdate(t, fc, "operator-ns", fullName, func(s *corev1.Secret) {
if s.Data == nil {
s.Data = map[string][]byte{}
}
s.Data["device_id"] = []byte("ts-id-1234")
s.Data["device_fqdn"] = []byte("tailscale.device.name.")
s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
})
expectReconciled(t, sr, "default", "test")
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
Status: corev1.ServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{
Ingress: []corev1.LoadBalancerIngress{
{
Hostname: "tailscale.device.name",
},
{
IP: "100.99.98.97",
},
},
},
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
// Turn the service back into a ClusterIP service, but also add the
// tailscale annotation.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.ObjectMeta.Annotations = map[string]string{
"tailscale.com/expose": "true",
}
s.Spec.Type = corev1.ServiceTypeClusterIP
s.Spec.LoadBalancerClass = nil
})
mustUpdateStatus(t, fc, "default", "test", func(s *corev1.Service) {
// Fake client doesn't automatically delete the LoadBalancer status when
// changing away from the LoadBalancer type, we have to do
// controller-manager's work by hand.
s.Status = corev1.ServiceStatus{}
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
}
func TestCustomHostname(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/hostname": "reindeer-flotilla",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "reindeer-flotilla",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
Finalizers: []string{"tailscale.com/finalizer"},
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/hostname": "reindeer-flotilla",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
Status: corev1.ServiceStatus{
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
delete(s.ObjectMeta.Annotations, "tailscale.com/expose")
})
// synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
// didn't create any child resources since this is all faked, so the
// deletion goes through immediately.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
// Second time around, the rest of cleanup happens.
expectReconciled(t, sr, "default", "test")
expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/hostname": "reindeer-flotilla",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
}
func TestCustomPriorityClassName(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
proxyPriorityClassName: "custom-priority-class-name",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/expose": "true",
"tailscale.com/hostname": "tailscale-critical",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "tailscale-critical",
priorityClassName: "custom-priority-class-name",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
}
func TestServiceProxyClassAnnotation(t *testing.T) {
cl := tstest.NewClock(tstest.ClockOpts{})
zl := zap.Must(zap.NewDevelopment())
pcIfNotPresent := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{
Name: "if-not-present",
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleContainer: &v1alpha1.Container{
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
},
},
}
pcAlways := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{
Name: "always",
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleContainer: &v1alpha1.Container{
ImagePullPolicy: corev1.PullAlways,
},
},
},
},
}
builder := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme)
builder = builder.WithObjects(pcIfNotPresent, pcAlways).
WithStatusSubresource(pcIfNotPresent, pcAlways)
fc := builder.Build()
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
},
}
mustCreate(t, fc, svc)
testCases := []struct {
name string
proxyClassAnnotation string
proxyClassLabel string
proxyClassDefault string
expectedProxyClass string
expectEvents []string
}{
{
name: "via_label",
proxyClassLabel: pcIfNotPresent.Name,
expectedProxyClass: pcIfNotPresent.Name,
},
{
name: "via_annotation",
proxyClassAnnotation: pcIfNotPresent.Name,
expectedProxyClass: pcIfNotPresent.Name,
},
{
name: "via_default",
proxyClassDefault: pcIfNotPresent.Name,
expectedProxyClass: pcIfNotPresent.Name,
},
{
name: "via_label_override_annotation",
proxyClassLabel: pcIfNotPresent.Name,
proxyClassAnnotation: pcAlways.Name,
expectedProxyClass: pcIfNotPresent.Name,
},
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
ft := &fakeTSClient{}
if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" {
name := tt.proxyClassDefault
if name == "" {
name = tt.proxyClassLabel
if name == "" {
name = tt.proxyClassAnnotation
}
}
setProxyClassReady(t, fc, cl, name)
}
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
defaultProxyClass: tt.proxyClassDefault,
logger: zl.Sugar(),
clock: cl,
isDefaultLoadBalancer: true,
}
if tt.proxyClassLabel != "" {
svc.Labels = map[string]string{
LabelAnnotationProxyClass: tt.proxyClassLabel,
}
}
if tt.proxyClassAnnotation != "" {
svc.Annotations = map[string]string{
LabelAnnotationProxyClass: tt.proxyClassAnnotation,
}
}
mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) {
s.Labels = svc.Labels
s.Annotations = svc.Annotations
})
expectReconciled(t, sr, "default", "test")
list := &corev1.ServiceList{}
fc.List(context.Background(), list, client.InNamespace("default"))
for _, i := range list.Items {
t.Logf("found service %s", i.Name)
}
slist := &corev1.SecretList{}
fc.List(context.Background(), slist, client.InNamespace("operator-ns"))
for _, i := range slist.Items {
labels, _ := json.Marshal(i.Labels)
t.Logf("found secret %q with labels %q ", i.Name, string(labels))
}
_, shortName := findGenName(t, fc, "default", "test", "svc")
sts := &appsv1.StatefulSet{}
if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
t.Fatalf("failed to get StatefulSet: %v", err)
}
switch tt.expectedProxyClass {
case pcIfNotPresent.Name:
for _, cont := range sts.Spec.Template.Spec.Containers {
if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullIfNotPresent {
t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcIfNotPresent.Name, pcIfNotPresent.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy)
}
}
case pcAlways.Name:
for _, cont := range sts.Spec.Template.Spec.Containers {
if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullAlways {
t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcAlways.Name, pcAlways.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy)
}
}
default:
t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass)
}
})
}
}
func TestProxyClassForService(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{
TailscaleConfig: &tsapi.TailscaleConfig{
AcceptRoutes: true,
},
StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}},
},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// 1. A new tailscale LoadBalancer Service is created without any
// ProxyClass. Resources get created for it as usual.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
// 2. The Service gets updated with tailscale.com/proxy-class label
// pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not
// yet ready, so no changes are actually applied to the proxy resources.
mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
mak.Set(&svc.Labels, LabelAnnotationProxyClass, "custom-metadata")
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts))
// 3. ProxyClass is set to Ready, the Service gets reconciled by the
// services-reconciler and the customization from the ProxyClass is
// applied to the proxy resources.
mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
pc.Status = tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
Type: string(tsapi.ProxyClassReady),
ObservedGeneration: pc.Generation,
}},
}
})
opts.proxyClass = pc.Name
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
// 4. tailscale.com/proxy-class label is removed from the Service, the
// configuration from the ProxyClass is removed from the cluster
// resources.
mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
delete(svc.Labels, LabelAnnotationProxyClass)
})
opts.proxyClass = ""
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
}
func TestDefaultLoadBalancer(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
isDefaultLoadBalancer: true,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
}
func TestProxyFirewallMode(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
tsFirewallMode: "nftables",
},
logger: zl.Sugar(),
clock: clock,
isDefaultLoadBalancer: true,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
firewallMode: "nftables",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
}
func Test_isMagicDNSName(t *testing.T) {
tests := []struct {
name string
in string
want bool
}{
{
name: "foo-tail4567-ts-net",
in: "foo.tail4567.ts.net",
want: true,
},
{
name: "foo-tail4567-ts-net-trailing-dot",
in: "foo.tail4567.ts.net.",
want: true,
},
{
name: "foo-tail4567",
in: "foo.tail4567",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isMagicDNSName(tt.in); got != tt.want {
t.Errorf("isMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want)
}
})
}
}
func Test_HeadlessService(t *testing.T) {
fc := fake.NewFakeClient()
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
},
logger: zl.Sugar(),
clock: clock,
recorder: record.NewFakeRecorder(100),
}
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
Annotations: map[string]string{
AnnotationExpose: "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "None",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
t0 := conditionTime(clock)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationExpose: "true",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "None",
Type: corev1.ServiceTypeClusterIP,
},
Status: corev1.ServiceStatus{
Conditions: []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionFalse,
LastTransitionTime: t0,
Reason: reasonProxyInvalid,
Message: `unable to provision proxy resources: invalid Service: headless Services are not supported.`,
}},
},
}
expectEqual(t, fc, want)
}
func Test_serviceHandlerForIngress(t *testing.T) {
const tailscaleIngressClassName = "tailscale"
fc := fake.NewFakeClient()
zl := zap.Must(zap.NewDevelopment())
// 1. An event on a headless Service for a tailscale Ingress results in
// the Ingress being reconciled.
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing-1",
Namespace: "ns-1",
},
Spec: networkingv1.IngressSpec{IngressClassName: new(tailscaleIngressClassName)},
})
svc1 := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "headless-1",
Namespace: "tailscale",
Labels: map[string]string{
kubetypes.LabelManaged: "true",
LabelParentName: "ing-1",
LabelParentNamespace: "ns-1",
LabelParentType: "ingress",
},
},
}
mustCreate(t, fc, svc1)
wantReqs := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-1", Name: "ing-1"}}}
gotReqs := serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), svc1)
if diff := cmp.Diff(gotReqs, wantReqs); diff != "" {
t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff)
}
// 2. An event on a Service that is the default backend for a tailscale
// Ingress results in the Ingress being reconciled.
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing-2",
Namespace: "ns-2",
},
Spec: networkingv1.IngressSpec{
DefaultBackend: &networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{Name: "def-backend"},
},
IngressClassName: new(tailscaleIngressClassName),
},
})
backendSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "def-backend",
Namespace: "ns-2",
},
}
mustCreate(t, fc, backendSvc)
wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-2", Name: "ing-2"}}}
gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc)
if diff := cmp.Diff(gotReqs, wantReqs); diff != "" {
t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff)
}
// 3. An event on a Service that is one of the non-default backends for
// a tailscale Ingress results in the Ingress being reconciled.
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing-3",
Namespace: "ns-3",
},
Spec: networkingv1.IngressSpec{
IngressClassName: new(tailscaleIngressClassName),
Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{
Paths: []networkingv1.HTTPIngressPath{
{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}},
},
}}}},
},
})
backendSvc2 := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "backend",
Namespace: "ns-3",
},
}
mustCreate(t, fc, backendSvc2)
wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-3", Name: "ing-3"}}}
gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc2)
if diff := cmp.Diff(gotReqs, wantReqs); diff != "" {
t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff)
}
// 4. An event on a Service that is a backend for an Ingress that is not
// tailscale Ingress does not result in an Ingress reconcile.
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing-4",
Namespace: "ns-4",
},
Spec: networkingv1.IngressSpec{
Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{
Paths: []networkingv1.HTTPIngressPath{
{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}},
},
}}}},
},
})
nonTSBackend := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "non-ts-backend",
Namespace: "ns-4",
},
}
mustCreate(t, fc, nonTSBackend)
gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), nonTSBackend)
if len(gotReqs) > 0 {
t.Errorf("unexpected reconcile request for a Service that does not belong to a Tailscale Ingress: %#+v\n", gotReqs)
}
// 5. An event on a Service not related to any Ingress does not result
// in an Ingress reconcile.
someSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "some-svc",
Namespace: "ns-4",
},
}
mustCreate(t, fc, someSvc)
gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), someSvc)
if len(gotReqs) > 0 {
t.Errorf("unexpected reconcile request for a Service that does not belong to any Ingress: %#+v\n", gotReqs)
}
}
func Test_serviceHandlerForIngress_multipleIngressClasses(t *testing.T) {
fc := fake.NewFakeClient()
zl := zap.Must(zap.NewDevelopment())
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: "default"},
}
mustCreate(t, fc, svc)
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{Name: "nginx-ing", Namespace: "default"},
Spec: networkingv1.IngressSpec{
IngressClassName: new("nginx"),
DefaultBackend: &networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}},
},
})
mustCreate(t, fc, &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{Name: "ts-ing", Namespace: "default"},
Spec: networkingv1.IngressSpec{
IngressClassName: new("tailscale"),
DefaultBackend: &networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}},
},
})
got := serviceHandlerForIngress(fc, zl.Sugar(), "tailscale")(context.Background(), svc)
want := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "default", Name: "ts-ing"}}}
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff)
}
}
func Test_clusterDomainFromResolverConf(t *testing.T) {
zl := zap.Must(zap.NewDevelopment())
tests := []struct {
name string
conf *resolvconffile.Config
namespace string
want string
}{
{
name: "success-custom-domain",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
},
namespace: "foo",
want: "department.org.io",
},
{
name: "success-default-domain",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.cluster.local."), toFQDN(t, "svc.cluster.local."), toFQDN(t, "cluster.local.")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "only-two-search-domains",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "first-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.bar.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "second-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "foo.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "third-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := clusterDomainFromResolverConf(tt.conf, tt.namespace, zl.Sugar()); got != tt.want {
t.Errorf("clusterDomainFromResolverConf() = %v, want %v", got, tt.want)
}
})
}
}
func Test_authKeyRemoval(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
// 1. A new Service that should be exposed via Tailscale gets created, a Secret with a config that contains auth
// key is generated.
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: "1234-UID",
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
replicas: new(int32(1)),
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
// 2. Apply update to the Secret that imitates the proxy setting device_id.
s := expectedSecret(t, fc, opts)
mustUpdate(t, fc, s.Namespace, s.Name, func(s *corev1.Secret) {
mak.Set(&s.Data, "device_id", []byte("dkkdi4CNTRL"))
})
// 3. Config should no longer contain auth key
expectReconciled(t, sr, "default", "test")
opts.shouldRemoveAuthKey = true
opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
expectEqual(t, fc, expectedSecret(t, fc, opts))
}
func Test_externalNameService(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
// 1. A External name Service that should be exposed via Tailscale gets
// created.
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// 1. Create an ExternalName Service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
AnnotationExpose: "true",
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeExternalName,
ExternalName: "foo.com",
},
})
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: new(int32(1)),
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
hostname: "default-test",
clusterTargetDNS: "foo.com",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
// 2. Change the ExternalName and verify that changes get propagated.
mustUpdate(t, sr, "default", "test", func(s *corev1.Service) {
s.Spec.ExternalName = "bar.com"
})
expectReconciled(t, sr, "default", "test")
opts.clusterTargetDNS = "bar.com"
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
}
func Test_metricsResourceCreation(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
Spec: tsapi.ProxyClassSpec{},
Status: tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
Type: string(tsapi.ProxyClassReady),
ObservedGeneration: 1,
}},
},
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Labels: map[string]string{LabelAnnotationProxyClass: "metrics"},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, svc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
operatorNamespace: "operator-ns",
},
logger: zl.Sugar(),
clock: clock,
}
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
tailscaleNamespace: "operator-ns",
hostname: "default-test",
namespaced: true,
proxyType: proxyTypeIngressService,
app: kubetypes.AppIngressProxy,
resourceVersion: "1",
}
// 1. Enable metrics- expect metrics Service to be created
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}}
})
expectReconciled(t, sr, "default", "test")
opts.enableMetrics = true
expectEqual(t, fc, expectedMetricsService(opts))
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
})
expectReconciled(t, sr, "default", "test")
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
mustCreate(t, fc, crd)
expectReconciled(t, sr, "default", "test")
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"}
})
expectReconciled(t, sr, "default", "test")
opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
opts.resourceVersion = "2"
expectEqual(t, fc, expectedMetricsService(opts))
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 5. Disable metrics- expect metrics Service to be deleted
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics = nil
})
expectReconciled(t, sr, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName))
// ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service
// object). We cannot test this using the fake client.
}
func TestIgnorePGService(t *testing.T) {
// NOTE: creating proxygroup stuff just to be sure that it's all ignored
_, _, fc, _, _ := setupServiceTest(t)
ft := &fakeTSClient{}
zl := zap.Must(zap.NewDevelopment())
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(ft),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
clock: clock,
}
// Create a service that we should manage, and check that the initial round
// of objects looks right.
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: "1234-UID",
Annotations: map[string]string{
"tailscale.com/proxygroup": "test-pg",
},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeClusterIP,
},
})
expectReconciled(t, sr, "default", "test")
findNoGenName(t, fc, "default", "test", "svc")
}
// Regression test: the service-pg-reconciler used to watch ProxyGroups with
// ingressesFromIngressProxyGroup, which lists Ingresses and returns Ingress
// keys. Fed to the Service reconciler those keys never match a Service, so a
// ProxyGroup becoming Available did not re-enqueue the HA Services waiting on
// it and provisioning only happened on the next operator restart (full
// re-list).
func Test_servicesFromIngressProxyGroup(t *testing.T) {
ingressPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{Name: "ingress-pg"},
Spec: tsapi.ProxyGroupSpec{Type: tsapi.ProxyGroupTypeIngress},
}
egressPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{Name: "egress-pg"},
Spec: tsapi.ProxyGroupSpec{Type: tsapi.ProxyGroupTypeEgress},
}
svcForIngressPG := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-1",
Namespace: "ns-1",
Annotations: map[string]string{AnnotationProxyGroup: "ingress-pg"},
},
}
svcForOtherPG := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "svc-2",
Namespace: "ns-2",
Annotations: map[string]string{AnnotationProxyGroup: "other-pg"},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(svcForIngressPG, svcForOtherPG).
WithIndex(new(corev1.Service), indexIngressProxyGroup, indexPGIngresses).
Build()
zl := zap.Must(zap.NewDevelopment())
h := servicesFromIngressProxyGroup(fc, zl.Sugar())
// An event on an ingress ProxyGroup enqueues only the Services that are
// annotated for it.
gotReqs := h(t.Context(), ingressPG)
wantReqs := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-1", Name: "svc-1"}}}
if diff := cmp.Diff(gotReqs, wantReqs); diff != "" {
t.Errorf("servicesFromIngressProxyGroup(ingress-pg) mismatch (-got +want):\n%s", diff)
}
// An egress ProxyGroup is handled by a different reconciler and must not
// enqueue anything here.
if gotReqs := h(t.Context(), egressPG); len(gotReqs) != 0 {
t.Errorf("servicesFromIngressProxyGroup(egress-pg) = %v, want none", gotReqs)
}
}
func toFQDN(t *testing.T, s string) dnsname.FQDN {
t.Helper()
fqdn, err := dnsname.ToFQDN(s)
if err != nil {
t.Fatalf("error coverting %q to dnsname.FQDN: %v", s, err)
}
return fqdn
}
func proxyCreatedCondition(clock tstime.Clock) []metav1.Condition {
return []metav1.Condition{{
Type: string(tsapi.ProxyReady),
Status: metav1.ConditionTrue,
ObservedGeneration: 0,
LastTransitionTime: conditionTime(clock),
Reason: reasonProxyCreated,
Message: reasonProxyCreated,
}}
}
func conditionTime(clock tstime.Clock) metav1.Time {
return metav1.NewTime(clock.Now().Truncate(time.Second))
}
func TestNodeHandlerForConnector(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "static-endpoints"},
Spec: tsapi.ProxyClassSpec{
StaticEndpoints: &tsapi.StaticEndpointsConfig{
NodePort: &tsapi.NodePortConfig{
Ports: []tsapi.PortRange{{Port: 30001, EndPort: 30003}},
Selector: map[string]string{"zone": "eu"},
},
},
},
}
noStaticPC := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "no-static-endpoints"},
}
connectors := []*tsapi.Connector{
// Matches the node via the ProxyClass selector.
{
ObjectMeta: metav1.ObjectMeta{Name: "matching"},
Spec: tsapi.ConnectorSpec{ProxyClass: pc.Name},
},
// References a ProxyClass that does not exist. This must not
// prevent other Connectors from being reconciled.
{
ObjectMeta: metav1.ObjectMeta{Name: "missing-proxyclass"},
Spec: tsapi.ConnectorSpec{ProxyClass: "does-not-exist"},
},
// References a ProxyClass without static endpoints.
{
ObjectMeta: metav1.ObjectMeta{Name: "no-static-endpoints"},
Spec: tsapi.ConnectorSpec{ProxyClass: noStaticPC.Name},
},
// No ProxyClass at all.
{
ObjectMeta: metav1.ObjectMeta{Name: "no-proxyclass"},
},
// Also matches the node, listed after the Connector with the
// missing ProxyClass.
{
ObjectMeta: metav1.ObjectMeta{Name: "matching-after-missing"},
Spec: tsapi.ConnectorSpec{ProxyClass: pc.Name},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, noStaticPC).
Build()
for _, cn := range connectors {
mustCreate(t, fc, cn)
}
handler := nodeHandlerForConnector(fc, zap.Must(zap.NewDevelopment()).Sugar())
t.Run("matching_node", func(t *testing.T) {
node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", Labels: map[string]string{"zone": "eu"}}}
got := handler(t.Context(), node)
want := []reconcile.Request{
{NamespacedName: types.NamespacedName{Name: "matching"}},
{NamespacedName: types.NamespacedName{Name: "matching-after-missing"}},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected reconcile requests (-want +got):\n%s", diff)
}
})
t.Run("non_matching_node", func(t *testing.T) {
node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-b", Labels: map[string]string{"zone": "us"}}}
got := handler(t.Context(), node)
if len(got) != 0 {
t.Fatalf("expected no reconcile requests, got %v", got)
}
})
}