mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-21 17:41:44 -08:00
292 lines
12 KiB
Python
292 lines
12 KiB
Python
# Copyright 2024-2025 New Vector Ltd
|
|
# Copyright 2025-2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
import asyncio
|
|
import base64
|
|
import contextlib
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import httpx_retries
|
|
import pyhelm3
|
|
import pytest
|
|
import yaml
|
|
from lightkube import ApiError, AsyncClient, KubeConfig
|
|
from lightkube.config.client_adapter import ConnectionParams, verify_cluster
|
|
from lightkube.config.kubeconfig import SingleConfig
|
|
from lightkube.core.generic_client import GenericAsyncClient
|
|
from lightkube.generic_resource import create_global_resource, create_namespaced_resource
|
|
from lightkube.models.meta_v1 import ObjectMeta
|
|
from lightkube.resources.core_v1 import Namespace, Secret, Service
|
|
from pytest_kubernetes.options import ClusterOptions
|
|
from pytest_kubernetes.providers import K3dManagerBase
|
|
|
|
from ..lib.utils import async_retry_with_timeout, b64encode, chart_from_ci_cache
|
|
from .data import ESSData
|
|
|
|
ClusterIssuer = create_global_resource(
|
|
group="cert-manager.io",
|
|
version="v1",
|
|
plural="clusterissuers",
|
|
kind="ClusterIssuer",
|
|
)
|
|
|
|
Certificate = create_namespaced_resource(
|
|
group="cert-manager.io",
|
|
version="v1",
|
|
plural="certificates",
|
|
kind="Certificate",
|
|
)
|
|
|
|
|
|
def FixedAsyncClient(config: SingleConfig, conn_parameters: ConnectionParams) -> httpx.AsyncClient:
|
|
# A transport with a SSLContext can't be pickled. This happens with the deepcopy that `asdict` does
|
|
# inside conn_parameters.httpx_params(). Save it off, null it out for the `asdict` and pass it explicitly
|
|
transport: httpx.AsyncBaseTransport | None = conn_parameters.transport # type: ignore[assignment]
|
|
conn_parameters.transport = None
|
|
args = conn_parameters.httpx_params(config)
|
|
args["transport"] = transport
|
|
return httpx.AsyncClient(**args)
|
|
|
|
|
|
GenericAsyncClient.AdapterClient = staticmethod(FixedAsyncClient)
|
|
|
|
|
|
class PotentiallyExistingK3dCluster(K3dManagerBase):
|
|
def __init__(self, cluster_name, provider_config=None):
|
|
super().__init__(cluster_name, provider_config)
|
|
|
|
clusters = self._exec(["cluster", "list"])
|
|
if any([line.startswith(cluster_name + " ") for line in clusters.stdout.decode("utf-8").split("\n")]):
|
|
self.existing_cluster = True
|
|
else:
|
|
self.existing_cluster = False
|
|
|
|
def _on_create(self, cluster_options, **kwargs):
|
|
if self.existing_cluster:
|
|
self._exec(
|
|
[
|
|
"kubeconfig",
|
|
"print",
|
|
self.cluster_name,
|
|
">",
|
|
str(cluster_options.kubeconfig_path),
|
|
]
|
|
)
|
|
else:
|
|
# The cluster requires extraMounts. These are relative paths from the cluster config file
|
|
# as they'll be different for everyone + CI.
|
|
# We save off the current working directory incase it is important, change to the folder
|
|
# with the cluster config file and then change back afterwards
|
|
cwd = os.getcwd()
|
|
try:
|
|
fixtures_folder = Path(__file__).parent.resolve()
|
|
os.chdir(fixtures_folder / Path("files/clusters"))
|
|
super()._on_create(cluster_options, **kwargs)
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
def _on_delete(self):
|
|
# We always keep around an existing cluster, it can always be deleted with scripts/destroy-test-cluster.sh
|
|
if not self.existing_cluster and os.environ.get("PYTEST_KEEP_CLUSTER", "") != "1":
|
|
return super()._on_delete()
|
|
|
|
|
|
@pytest.fixture(autouse=True, scope="session")
|
|
async def cluster():
|
|
# Both these names must match what `setup_test_cluster.sh` would create
|
|
this_cluster = PotentiallyExistingK3dCluster("ess-helm")
|
|
this_cluster.create(
|
|
ClusterOptions(cluster_name="ess-helm", provider_config=Path(__file__).parent / Path("files/clusters/k3d.yml")),
|
|
options=os.environ.get("K3D_EXTRA_OPTIONS", "").split(" "),
|
|
)
|
|
|
|
yield this_cluster
|
|
|
|
this_cluster.delete()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def helm_client(cluster):
|
|
return pyhelm3.Client(kubeconfig=cluster.kubeconfig, kubecontext=cluster.context)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def kube_client(cluster):
|
|
kube_config = KubeConfig.from_file(cluster.kubeconfig)
|
|
config = kube_config.get()
|
|
|
|
# We've seen 429 errors with storage is (re)initializing. Let's retry those
|
|
ssl_context = verify_cluster(config.cluster, config.user, config.abs_file)
|
|
wrapped_transport = httpx.AsyncHTTPTransport(verify=ssl_context)
|
|
transport = httpx_retries.RetryTransport(
|
|
transport=wrapped_transport, retry=httpx_retries.Retry(status_forcelist=[429])
|
|
)
|
|
return AsyncClient(config=kube_config, transport=transport)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def ingress(cluster, kube_client):
|
|
attempt = 0
|
|
while attempt < 180:
|
|
try:
|
|
# We can't just kubectl wait as that doesn't work with non-existent objects
|
|
# This can be setup before the LB port is accessible externally, so we do it afterwards
|
|
service = await kube_client.get(Service, name="traefik", namespace="kube-system")
|
|
await asyncio.to_thread(
|
|
cluster.wait,
|
|
name="service/traefik",
|
|
waitfor="jsonpath='{.status.loadBalancer.ingress[0].ip}'",
|
|
namespace="kube-system",
|
|
)
|
|
return service.spec.clusterIP
|
|
except ApiError:
|
|
await asyncio.sleep(1)
|
|
attempt += 1
|
|
raise Exception("Couldn't fetch Trafeik Service IP after 180s")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def cert_manager(helm_client, kube_client):
|
|
if os.environ.get("SKIP_CERT_MANAGER", "false") != "false":
|
|
return
|
|
|
|
chart = await chart_from_ci_cache(helm_client, "oci://quay.io/jetstack/charts/cert-manager")
|
|
await helm_client.install_or_upgrade_release(
|
|
"cert-manager",
|
|
chart,
|
|
yaml.safe_load((Path(__file__).parent / "files/charts/cert-manager.yml").open()),
|
|
namespace="cert-manager",
|
|
create_namespace=True,
|
|
atomic="CI" not in os.environ,
|
|
wait=True,
|
|
)
|
|
|
|
ca_folder = Path(__file__).parent.parent.parent.parent / ".ca"
|
|
if not ca_folder.exists():
|
|
ca_folder.mkdir()
|
|
ca_crt_path = Path(ca_folder) / "ca.crt"
|
|
ca_pem_path = Path(ca_folder) / "ca.pem"
|
|
if not (ca_crt_path.exists() and ca_pem_path.exists()):
|
|
await kube_client.create(ClusterIssuer(metadata={"name": "ess-ca"}, spec={"selfSigned": {}}))
|
|
await kube_client.create(
|
|
Certificate(
|
|
metadata={"name": "ess-ca", "namespace": "cert-manager"},
|
|
spec={
|
|
"isCA": True,
|
|
"commonName": "ess-ca",
|
|
"secretName": "ess-ca",
|
|
"duration": "87660h0m0s",
|
|
"privateKey": {"algorithm": "RSA"},
|
|
"issuerRef": {"name": "ess-ca", "kind": "ClusterIssuer", "group": "cert-manager.io"},
|
|
},
|
|
)
|
|
)
|
|
|
|
# Wait for certificate to be ready
|
|
cert_ready = False
|
|
while not cert_ready:
|
|
cert = await kube_client.get(Certificate, name="ess-ca", namespace="cert-manager")
|
|
if cert.status and cert.status.get("conditions"):
|
|
for condition in cert.status["conditions"]:
|
|
if condition["type"] == "Ready" and condition["status"] == "True":
|
|
cert_ready = True
|
|
break
|
|
await asyncio.sleep(1)
|
|
else:
|
|
# Delete existing resources
|
|
with contextlib.suppress(Exception):
|
|
await kube_client.delete(ClusterIssuer, name="ess-ca")
|
|
with contextlib.suppress(Exception):
|
|
await kube_client.delete(Certificate, name="ess-ca", namespace="cert-manager")
|
|
with contextlib.suppress(Exception):
|
|
await kube_client.delete(Secret, name="ess-ca", namespace="cert-manager")
|
|
|
|
await kube_client.create(
|
|
Secret(
|
|
metadata=ObjectMeta(name="ess-ca", namespace="cert-manager"),
|
|
data={
|
|
"tls.crt": b64encode(ca_crt_path.read_text()),
|
|
"tls.key": b64encode(ca_pem_path.read_text()),
|
|
"ca.crt": b64encode(ca_crt_path.read_text()),
|
|
},
|
|
)
|
|
)
|
|
await kube_client.apply(
|
|
ClusterIssuer(metadata={"name": "ess-selfsigned"}, spec={"ca": {"secretName": "ess-ca"}}),
|
|
field_manager="pytest",
|
|
)
|
|
ess_ca_secret = await kube_client.get(Secret, name="ess-ca", namespace="cert-manager")
|
|
|
|
if not ca_crt_path.exists() or not ca_pem_path.exists():
|
|
with open(ca_crt_path, "w") as crt_file, open(ca_pem_path, "w") as pem_file:
|
|
crt_file.write(base64.standard_b64decode(ess_ca_secret.data["ca.crt"]).decode("utf-8"))
|
|
pem_file.write(base64.standard_b64decode(ess_ca_secret.data["tls.key"]).decode("utf-8"))
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def prometheus_operator_crds(helm_client: pyhelm3.Client):
|
|
if os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "false":
|
|
chart = await chart_from_ci_cache(
|
|
helm_client,
|
|
"oci://ghcr.io/prometheus-community/charts/prometheus-operator-crds",
|
|
)
|
|
|
|
async def _setup_prometheus_operator_crds():
|
|
# Install or upgrade a release
|
|
await helm_client.install_or_upgrade_release(
|
|
"prometheus-operator-crds",
|
|
chart,
|
|
{},
|
|
namespace="prometheus-operator",
|
|
create_namespace=True,
|
|
atomic="CI" not in os.environ,
|
|
timeout="1m",
|
|
wait=True,
|
|
)
|
|
|
|
# We retry 5 times, maximum for 5m
|
|
# Each helm install timeouts after 1m so that we early fail if the CRDs fail to setup
|
|
# This should be revised when https://github.com/helm/helm/issues/31824 gets resolved
|
|
await async_retry_with_timeout(
|
|
_setup_prometheus_operator_crds,
|
|
should_retry=lambda e: type(e) is pyhelm3.errors.Error,
|
|
max_retries=5,
|
|
timeout_seconds=300,
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def ess_namespace(cluster: PotentiallyExistingK3dCluster, kube_client: AsyncClient, generated_data: ESSData):
|
|
(major_version, minor_version) = cluster.version()
|
|
try:
|
|
await kube_client.get(Namespace, name=generated_data.ess_namespace)
|
|
except ApiError:
|
|
await kube_client.create(
|
|
Namespace(
|
|
metadata=ObjectMeta(
|
|
name=generated_data.ess_namespace,
|
|
labels={
|
|
"app.kubernetes.io/managed-by": "pytest",
|
|
# We do turn on enforce here to cause test failures.
|
|
# If we actually need restricted functionality then the tests can drop this
|
|
# and parse the audit logs
|
|
"pod-security.kubernetes.io/enforce": "restricted",
|
|
"pod-security.kubernetes.io/enforce-version": f"v{major_version}.{minor_version}",
|
|
"pod-security.kubernetes.io/audit": "restricted",
|
|
"pod-security.kubernetes.io/audit-version": f"v{major_version}.{minor_version}",
|
|
"pod-security.kubernetes.io/warn": "restricted",
|
|
"pod-security.kubernetes.io/warn-version": f"v{major_version}.{minor_version}",
|
|
},
|
|
)
|
|
)
|
|
)
|
|
|
|
yield
|
|
|
|
if os.environ.get("PYTEST_KEEP_CLUSTER", "") != "1":
|
|
await kube_client.delete(Namespace, name=generated_data.ess_namespace)
|