mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-21 01:21:33 -08:00
270 lines
13 KiB
Python
270 lines
13 KiB
Python
# Copyright 2024-2025 New Vector Ltd
|
|
# Copyright 2025-2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
import asyncio
|
|
import os
|
|
|
|
import pytest
|
|
from lightkube import AsyncClient
|
|
from lightkube import operators as op
|
|
from lightkube.resources.apps_v1 import ReplicaSet
|
|
from lightkube.resources.core_v1 import Pod, Service
|
|
from prometheus_client.parser import text_string_to_metric_families
|
|
|
|
from .fixtures.data import ESSData
|
|
from .lib.helpers import run_pod_with_args, wait_for_all_deployments_rolled_out, wait_for_endpoint_ready
|
|
from .lib.utils import read_service_monitor_kind
|
|
|
|
|
|
@pytest.mark.asyncio_cooperative
|
|
async def test_services_have_matching_labels(kube_client: AsyncClient, generated_data: ESSData, matrix_stack):
|
|
ignored_labels = [
|
|
"app.kubernetes.io/managed-by",
|
|
"helm.sh/chart",
|
|
"k8s.element.io/service-type",
|
|
"k8s.element.io/synapse-instance",
|
|
"replica",
|
|
]
|
|
|
|
async for service in kube_client.list(
|
|
Service, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
|
|
):
|
|
assert service.spec, f"Encountered a service without spec : {service}"
|
|
assert service.spec.selector, f"Encountered a service missing a selector : {service}"
|
|
assert service.metadata, f"Encountered a service without metadata : {service}"
|
|
assert service.metadata.labels, f"Encountered a service without labels : {service}"
|
|
label_selectors = {label: value for label, value in service.spec.selector.items()}
|
|
|
|
async for pod in kube_client.list(Pod, namespace=generated_data.ess_namespace, labels=label_selectors):
|
|
if pod.status and pod.status.phase in ("Terminating", "Succeeded"):
|
|
continue # Skip terminating pods
|
|
# For Pods part of a replicaset we must ignore pods which template-hash do not match
|
|
# the latest replicaset `pod-template-hash`
|
|
if pod.metadata and pod.metadata.labels and pod.metadata.labels.get("pod-template-hash"):
|
|
async for rs in kube_client.list(
|
|
ReplicaSet,
|
|
namespace=generated_data.ess_namespace,
|
|
labels={"pod-template-hash": op.equal(pod.metadata.labels["pod-template-hash"])},
|
|
):
|
|
# we check if the rs desires replicas
|
|
if rs.spec and rs.spec.replicas:
|
|
break
|
|
else:
|
|
# Skip pods which do not have a ReplicaSet desiring replicas
|
|
continue
|
|
assert service.metadata, f"Encountered a service without metadata : {service}"
|
|
assert pod.metadata, f"Encountered a pod without metadata : {pod}"
|
|
assert pod.metadata.labels, f"Encountered a pod without labels : {pod}"
|
|
has_target_label = any(label.startswith("k8s.element.io/target-") for label in service.metadata.labels)
|
|
for label, value in service.metadata.labels.items():
|
|
if label in ["k8s.element.io/owner-name", "k8s.element.io/owner-group-kind"]:
|
|
assert label not in pod.metadata.labels
|
|
continue
|
|
elif label in ignored_labels or (
|
|
has_target_label and label in ["app.kubernetes.io/name", "app.kubernetes.io/instance"]
|
|
):
|
|
continue
|
|
assert label.replace("k8s.element.io/target-", "app.kubernetes.io/") in pod.metadata.labels
|
|
assert value.startswith(
|
|
pod.metadata.labels[label.replace("k8s.element.io/target-", "app.kubernetes.io/")]
|
|
), (
|
|
f"Service/{service.metadata.name} has {label=}={value=} "
|
|
f"which does not start with the value from the Pod/{pod.metadata.name} "
|
|
f"{label.replace('k8s.element.io/target-', 'app.kubernetes.io/')}="
|
|
f"{pod.metadata.labels[label.replace('k8s.element.io/target-', 'app.kubernetes.io/')]} "
|
|
f"(pod status phase : {pod.status.phase if pod.status else 'N/A'}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio_cooperative
|
|
async def test_services_have_endpoints(kube_client: AsyncClient, generated_data: ESSData, matrix_stack):
|
|
# Helm will stop waiting when 1 of replicas are ready
|
|
# We need to wait for all replicas to be ready to check that services all have endpoints
|
|
await wait_for_all_deployments_rolled_out(kube_client, generated_data.ess_namespace)
|
|
endpoints_to_wait = []
|
|
services = {}
|
|
async for service in kube_client.list(
|
|
Service, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
|
|
):
|
|
assert service.metadata, f"Encountered a service without metadata : {service}"
|
|
assert service.spec, f"Encountered a service without spec : {service}"
|
|
endpoints_to_wait.append(
|
|
wait_for_endpoint_ready(service.metadata.name, generated_data.ess_namespace, kube_client)
|
|
)
|
|
services[service.metadata.name] = service
|
|
|
|
for endpoint in await asyncio.gather(*endpoints_to_wait):
|
|
assert endpoint.metadata is not None, f"Encountered an endpoint without metadata : {endpoint}"
|
|
assert endpoint.subsets, f"Endpoint {endpoint.metadata.name} has no subsets"
|
|
|
|
ports = []
|
|
for subset in endpoint.subsets:
|
|
assert subset.addresses, f"Endpoint {endpoint.metadata.name} has no addresses"
|
|
assert not subset.notReadyAddresses, f"Endpoint {endpoint.metadata.name} has notReadyAddresses"
|
|
assert subset.ports, f"Endpoint {endpoint.metadata.name} has no ports"
|
|
ports += subset.ports
|
|
|
|
port_names = [port.name for port in ports if port.name]
|
|
port_numbers = [port.port for port in ports]
|
|
service_from_endpoint = services[endpoint.metadata.name]
|
|
assert service_from_endpoint.spec, f"Service {endpoint.metadata.name} has no spec"
|
|
assert service_from_endpoint.spec.ports, f"Service {endpoint.metadata.name} has no port"
|
|
for port in service_from_endpoint.spec.ports:
|
|
if port.name:
|
|
assert port.name in port_names
|
|
else:
|
|
assert port.port in port_numbers
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "true", reason="ServiceMonitors not deployed"
|
|
)
|
|
@pytest.mark.asyncio_cooperative
|
|
@pytest.mark.usefixtures("matrix_stack")
|
|
async def test_pods_monitored(
|
|
kube_client: AsyncClient,
|
|
generated_data: ESSData,
|
|
):
|
|
# Helm will stop waiting when 1 of replicas are ready
|
|
# We need to wait for all replicas to be ready to be able to compute monitorable and monitored pods
|
|
await wait_for_all_deployments_rolled_out(kube_client, generated_data.ess_namespace)
|
|
all_running_pods = list[Pod]()
|
|
all_monitorable_pod_names = set[str]()
|
|
async for pod in kube_client.list(
|
|
Pod, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}
|
|
):
|
|
if pod.status and pod.status.phase in ("Terminating", "Succeeded"):
|
|
continue # Skip terminating pods
|
|
|
|
elif pod.metadata:
|
|
assert pod.metadata.name, "Encountered a pod without a name"
|
|
all_running_pods.append(pod)
|
|
|
|
if pod.metadata.annotations and "has-no-service-monitor" in pod.metadata.annotations:
|
|
continue
|
|
|
|
all_monitorable_pod_names.add(pod.metadata.name)
|
|
else:
|
|
raise RuntimeError(f"Pod {pod} has no metadata")
|
|
|
|
monitored_pod_names = set[str]()
|
|
async for service_monitor in kube_client.list(
|
|
await read_service_monitor_kind(kube_client),
|
|
namespace=generated_data.ess_namespace,
|
|
labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])},
|
|
):
|
|
service_monitor_is_useful = False
|
|
async for service in kube_client.list(
|
|
Service, namespace=generated_data.ess_namespace, labels=service_monitor["spec"]["selector"]["matchLabels"]
|
|
):
|
|
assert service.metadata, f"Encountered a service without metadata : {service}"
|
|
assert service.spec, f"Encountered a service without spec : {service}"
|
|
assert service.spec.ports, f"Ecountered a service without port : {service}"
|
|
assert service.spec.selector, f"Ecountered a service without selectors : {service}"
|
|
|
|
for endpoint in service_monitor["spec"]["endpoints"]:
|
|
service_port_names = [port.name for port in service.spec.ports if port.name]
|
|
if endpoint["port"] in service_port_names:
|
|
break
|
|
|
|
def matches_labels(pod, selector):
|
|
pod_labels = pod.metadata.labels
|
|
for label, value in selector.items():
|
|
if label not in pod_labels:
|
|
return False
|
|
|
|
if pod_labels[label] != value:
|
|
return False
|
|
return True
|
|
|
|
for covered_pod in [pod for pod in all_running_pods if matches_labels(pod, service.spec.selector)]:
|
|
assert covered_pod.metadata
|
|
if not covered_pod.metadata:
|
|
raise RuntimeError(f"Pod {covered_pod} has no metadata")
|
|
|
|
# Something monitored by multiple ServiceMonitors smells like a bug
|
|
assert covered_pod.metadata.name not in monitored_pod_names, (
|
|
f"Pod {covered_pod.metadata.name} is monitored multiple times"
|
|
)
|
|
assert covered_pod.metadata.name
|
|
monitored_pod_names.add(covered_pod.metadata.name)
|
|
service_monitor_is_useful = True
|
|
|
|
assert service_monitor_is_useful, f"ServiceMonitor {service_monitor['metadata']['name']} does not cover any pod"
|
|
|
|
assert all_monitorable_pod_names == monitored_pod_names, (
|
|
f"Some pods are not monitored : {', '.join(list(set(all_monitorable_pod_names) ^ set(monitored_pod_names)))}"
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "true", reason="ServiceMonitors not deployed"
|
|
)
|
|
@pytest.mark.asyncio_cooperative
|
|
@pytest.mark.usefixtures("matrix_stack")
|
|
async def test_service_monitors_point_to_metrics(
|
|
kube_client: AsyncClient,
|
|
generated_data: ESSData,
|
|
):
|
|
async for service_monitor in kube_client.list(
|
|
await read_service_monitor_kind(kube_client),
|
|
namespace=generated_data.ess_namespace,
|
|
labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])},
|
|
):
|
|
found_metrics = False
|
|
async for service in kube_client.list(
|
|
Service, namespace=generated_data.ess_namespace, labels=service_monitor["spec"]["selector"]["matchLabels"]
|
|
):
|
|
assert service.metadata, f"Encountered a service without metadata : {service}"
|
|
assert service.spec, f"Encountered a service without spec : {service}"
|
|
assert service.spec.ports, f"Ecountered a service without port : {service}"
|
|
assert service.spec.selector, f"Ecountered a service without selectors : {service}"
|
|
|
|
for endpoint in service_monitor["spec"]["endpoints"]:
|
|
service_port_names = [port.name for port in service.spec.ports if port.name]
|
|
if endpoint["port"] in service_port_names:
|
|
assert await has_actual_metrics_on_endpoint(
|
|
kube_client, generated_data, service, service_monitor["spec"]["endpoints"]
|
|
)
|
|
found_metrics = True
|
|
assert found_metrics, (
|
|
f"ServiceMonitor {service_monitor['metadata']['name']} does not point to any /metrics endpoint"
|
|
)
|
|
|
|
|
|
async def has_actual_metrics_on_endpoint(
|
|
kube_client: AsyncClient, generated_data: ESSData, service: Service, endpoints
|
|
):
|
|
assert service.metadata
|
|
assert service.spec
|
|
assert service.spec.ports
|
|
found_metrics = False
|
|
for endpoint in endpoints:
|
|
for port_spec in service.spec.ports:
|
|
if port_spec.name == endpoint["port"]:
|
|
metrics_data = await run_pod_with_args(
|
|
kube_client,
|
|
generated_data,
|
|
"curlimages/curl:latest",
|
|
"curl",
|
|
[
|
|
"-s",
|
|
"--connect-timeout",
|
|
"2",
|
|
"--max-time",
|
|
"5",
|
|
"--retry",
|
|
"6",
|
|
"--retry-all-errors",
|
|
"--retry-delay",
|
|
"2",
|
|
f"http://{service.metadata.name}.{generated_data.ess_namespace}.svc.cluster.local:{port_spec.port}/metrics",
|
|
],
|
|
)
|
|
for metric_family in text_string_to_metric_families(metrics_data):
|
|
assert metric_family.name, "Metric family has no name"
|
|
found_metrics = True
|
|
return found_metrics
|