Files
ess-helm/tests/integration/test_networking.py
T

267 lines
13 KiB
Python

# Copyright 2024-2025 New Vector Ltd
# Copyright 2025-2026 Element Creations Ltd
#
# SPDX-License-Identifier: AGPL-3.0-only
import asyncio
import os
import pytest
from lightkube import AsyncClient
from lightkube import operators as op
from lightkube.resources.apps_v1 import ReplicaSet
from lightkube.resources.core_v1 import Pod, Service
from prometheus_client.parser import text_string_to_metric_families
from .fixtures.data import ESSData
from .lib.helpers import run_pod_with_args, wait_for_all_deployments_rolled_out, wait_for_endpoint_ready
from .lib.utils import read_service_monitor_kind
@pytest.mark.asyncio_cooperative
async def test_services_have_matching_labels(kube_client: AsyncClient, generated_data: ESSData, matrix_stack):
ignored_labels = [
"app.kubernetes.io/managed-by",
"helm.sh/chart",
"k8s.element.io/service-type",
"k8s.element.io/synapse-instance",
"replica",
]
async for service in kube_client.list(
Service, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
):
assert service.spec, f"Encountered a service without spec : {service}"
assert service.spec.selector, f"Encountered a service missing a selector : {service}"
assert service.metadata, f"Encountered a service without metadata : {service}"
assert service.metadata.labels, f"Encountered a service without labels : {service}"
label_selectors = {label: value for label, value in service.spec.selector.items()}
async for pod in kube_client.list(Pod, namespace=generated_data.ess_namespace, labels=label_selectors):
if pod.status and pod.status.phase in ("Terminating", "Succeeded"):
continue # Skip terminating pods
# For Pods part of a replicaset we must ignore pods which template-hash do not match
# the latest replicaset `pod-template-hash`
if pod.metadata and pod.metadata.labels and pod.metadata.labels.get("pod-template-hash"):
async for rs in kube_client.list(
ReplicaSet,
namespace=generated_data.ess_namespace,
labels={"pod-template-hash": op.equal(pod.metadata.labels["pod-template-hash"])},
):
# we check if the rs desires replicas
if rs.spec and rs.spec.replicas:
break
else:
# Skip pods which do not have a ReplicaSet desiring replicas
continue
assert service.metadata, f"Encountered a service without metadata : {service}"
assert pod.metadata, f"Encountered a pod without metadata : {pod}"
assert pod.metadata.labels, f"Encountered a pod without labels : {pod}"
has_target_label = any(label.startswith("k8s.element.io/target-") for label in service.metadata.labels)
for label, value in service.metadata.labels.items():
if label in ["k8s.element.io/owner-name", "k8s.element.io/owner-group-kind"]:
assert label not in pod.metadata.labels
continue
elif label in ignored_labels or (
has_target_label and label in ["app.kubernetes.io/name", "app.kubernetes.io/instance"]
):
continue
assert label.replace("k8s.element.io/target-", "app.kubernetes.io/") in pod.metadata.labels
assert value.startswith(
pod.metadata.labels[label.replace("k8s.element.io/target-", "app.kubernetes.io/")]
), (
f"Service/{service.metadata.name} has {label=}={value=} "
f"which does not start with the value from the Pod/{pod.metadata.name} "
f"{label.replace('k8s.element.io/target-', 'app.kubernetes.io/')}="
f"{pod.metadata.labels[label.replace('k8s.element.io/target-', 'app.kubernetes.io/')]} "
f"(pod status phase : {pod.status.phase if pod.status else 'N/A'}"
)
@pytest.mark.asyncio_cooperative
async def test_services_have_endpoints(cluster, kube_client: AsyncClient, generated_data: ESSData, matrix_stack):
# Helm will stop waiting when 1 of replicas are ready
# We need to wait for all replicas to be ready to check that services all have endpoints
await wait_for_all_deployments_rolled_out(kube_client, generated_data.ess_namespace)
endpoints_to_wait = []
services = {}
async for service in kube_client.list(
Service, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
):
assert service.metadata, f"Encountered a service without metadata : {service}"
assert service.spec, f"Encountered a service without spec : {service}"
endpoints_to_wait.append(
wait_for_endpoint_ready(service.metadata.name, generated_data.ess_namespace, cluster, kube_client)
)
services[service.metadata.name] = service
for endpoint in await asyncio.gather(*endpoints_to_wait):
assert endpoint.metadata is not None, f"Encountered an endpoint without metadata : {endpoint}"
assert endpoint.subsets, f"Endpoint {endpoint.metadata.name} has no subsets"
ports = []
for subset in endpoint.subsets:
assert subset.addresses, f"Endpoint {endpoint.metadata.name} has no addresses"
assert not subset.notReadyAddresses, f"Endpoint {endpoint.metadata.name} has notReadyAddresses"
assert subset.ports, f"Endpoint {endpoint.metadata.name} has no ports"
ports += subset.ports
port_names = [port.name for port in ports if port.name]
port_numbers = [port.port for port in ports]
service_from_endpoint = services[endpoint.metadata.name]
assert service_from_endpoint.spec, f"Service {endpoint.metadata.name} has no spec"
assert service_from_endpoint.spec.ports, f"Service {endpoint.metadata.name} has no port"
for port in service_from_endpoint.spec.ports:
if port.name:
assert port.name in port_names
else:
assert port.port in port_numbers
@pytest.mark.skipif(
os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "true", reason="ServiceMonitors not deployed"
)
@pytest.mark.asyncio_cooperative
@pytest.mark.usefixtures("matrix_stack")
async def test_pods_monitored(
kube_client: AsyncClient,
generated_data: ESSData,
):
# Helm will stop waiting when 1 of replicas are ready
# We need to wait for all replicas to be ready to be able to compute monitorable and monitored pods
await wait_for_all_deployments_rolled_out(kube_client, generated_data.ess_namespace)
all_running_pods = list[Pod]()
all_monitorable_pod_names = set[str]()
async for pod in kube_client.list(
Pod, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
):
if pod.status and pod.status.phase in ("Terminating", "Succeeded"):
continue # Skip terminating pods
elif pod.metadata:
assert pod.metadata.name, "Encountered a pod without a name"
all_running_pods.append(pod)
if pod.metadata.annotations and "has-no-service-monitor" in pod.metadata.annotations:
continue
all_monitorable_pod_names.add(pod.metadata.name)
else:
raise RuntimeError(f"Pod {pod} has no metadata")
monitored_pod_names = set[str]()
async for service_monitor in kube_client.list(
await read_service_monitor_kind(kube_client),
namespace=generated_data.ess_namespace,
labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])},
):
service_monitor_is_useful = False
async for service in kube_client.list(
Service, namespace=generated_data.ess_namespace, labels=service_monitor["spec"]["selector"]["matchLabels"]
):
assert service.metadata, f"Encountered a service without metadata : {service}"
assert service.spec, f"Encountered a service without spec : {service}"
assert service.spec.ports, f"Ecountered a service without port : {service}"
assert service.spec.selector, f"Ecountered a service without selectors : {service}"
for endpoint in service_monitor["spec"]["endpoints"]:
service_port_names = [port.name for port in service.spec.ports if port.name]
if endpoint["port"] in service_port_names:
break
def matches_labels(pod, selector):
pod_labels = pod.metadata.labels
for label, value in selector.items():
if label not in pod_labels:
return False
if pod_labels[label] != value:
return False
return True
for covered_pod in [pod for pod in all_running_pods if matches_labels(pod, service.spec.selector)]:
assert covered_pod.metadata
if not covered_pod.metadata:
raise RuntimeError(f"Pod {covered_pod} has no metadata")
# Something monitored by multiple ServiceMonitors smells like a bug
assert covered_pod.metadata.name not in monitored_pod_names, (
f"Pod {covered_pod.metadata.name} is monitored multiple times"
)
assert covered_pod.metadata.name
monitored_pod_names.add(covered_pod.metadata.name)
service_monitor_is_useful = True
assert service_monitor_is_useful, f"ServiceMonitor {service_monitor['metadata']['name']} does not cover any pod"
assert all_monitorable_pod_names == monitored_pod_names, (
f"Some pods are not monitored : {', '.join(list(set(all_monitorable_pod_names) ^ set(monitored_pod_names)))}"
)
@pytest.mark.skipif(
os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "true", reason="ServiceMonitors not deployed"
)
@pytest.mark.skipif(os.environ.get("MATRIX_TEST_FROM_REF", "") == "25.6.2", reason="This fails against 25.6.2.")
@pytest.mark.asyncio_cooperative
@pytest.mark.usefixtures("matrix_stack")
async def test_service_monitors_point_to_metrics(
kube_client: AsyncClient,
generated_data: ESSData,
):
async for service_monitor in kube_client.list(
await read_service_monitor_kind(kube_client),
namespace=generated_data.ess_namespace,
labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])},
):
found_metrics = False
async for service in kube_client.list(
Service, namespace=generated_data.ess_namespace, labels=service_monitor["spec"]["selector"]["matchLabels"]
):
assert service.metadata, f"Encountered a service without metadata : {service}"
assert service.spec, f"Encountered a service without spec : {service}"
assert service.spec.ports, f"Ecountered a service without port : {service}"
assert service.spec.selector, f"Ecountered a service without selectors : {service}"
for endpoint in service_monitor["spec"]["endpoints"]:
service_port_names = [port.name for port in service.spec.ports if port.name]
if endpoint["port"] in service_port_names:
assert await has_actual_metrics_on_endpoint(
kube_client, generated_data, service, service_monitor["spec"]["endpoints"]
)
found_metrics = True
assert found_metrics, (
f"ServiceMonitor {service_monitor['metadata']['name']} does not point to any /metrics endpoint"
)
async def has_actual_metrics_on_endpoint(
kube_client: AsyncClient, generated_data: ESSData, service: Service, endpoints
):
assert service.metadata
assert service.spec
assert service.spec.ports
found_metrics = False
for endpoint in endpoints:
for port_spec in service.spec.ports:
if port_spec.name == endpoint["port"]:
metrics_data = await run_pod_with_args(
kube_client,
generated_data,
"curlimages/curl:latest",
"curl",
[
"-s",
"--connect-timeout",
"2",
"--max-time",
"5",
f"http://{service.metadata.name}.{generated_data.ess_namespace}.svc.cluster.local:{port_spec.port}/metrics",
],
restart_policy="OnFailure",
)
for metric_family in text_string_to_metric_families(metrics_data):
assert metric_family.name, "Metric family has no name"
found_metrics = True
return found_metrics