mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-20 00:51:27 -08:00
334 lines
12 KiB
Python
334 lines
12 KiB
Python
# Copyright 2024-2025 New Vector Ltd
|
|
# Copyright 2025-2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import os
|
|
import shutil
|
|
from collections.abc import AsyncGenerator, Awaitable, Callable
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from ssl import SSLContext
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import aiohttp
|
|
import pyhelm3
|
|
import yaml
|
|
from aiohttp_retry import JitterRetry, RetryClient
|
|
from lightkube import AsyncClient
|
|
from lightkube.generic_resource import (
|
|
async_load_in_cluster_generic_resources,
|
|
get_generic_resource,
|
|
)
|
|
from lightkube.resources.apiextensions_v1 import CustomResourceDefinition
|
|
from lightkube.resources.core_v1 import Pod
|
|
from pytest_kubernetes.providers import AClusterManager
|
|
|
|
retry_options = JitterRetry(
|
|
attempts=12,
|
|
statuses={
|
|
500,
|
|
503,
|
|
}
|
|
| set(int(code) for code in os.environ.get("PYTEST_EXPECTED_HTTP_STATUS_CODES", "").split(",") if code),
|
|
retry_all_server_errors=False,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class DockerAuth:
|
|
registry: str
|
|
username: str
|
|
password: str
|
|
|
|
|
|
@dataclass
|
|
class KubeCtl:
|
|
"""A class to execute kubectl against a pod asynchronously"""
|
|
|
|
cluster: AClusterManager
|
|
|
|
async def exec(self, pod, namespace, cmd):
|
|
return await asyncio.to_thread(
|
|
self.cluster.kubectl, as_dict=False, args=["exec", "-t", pod, "-n", namespace, "--", *cmd]
|
|
)
|
|
|
|
|
|
def docker_config_json(auths: list[DockerAuth]) -> str:
|
|
docker_config_auths = {}
|
|
for auth in auths:
|
|
docker_config_auths[auth.registry] = {
|
|
"username": auth.username,
|
|
"password": auth.password,
|
|
"auth": b64encode(f"{auth.username}:{auth.password}"),
|
|
}
|
|
|
|
return json.dumps({"auths": docker_config_auths})
|
|
|
|
|
|
def b64encode(value: str):
|
|
return base64.b64encode(value.encode("utf-8")).decode("utf-8")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def aiohttp_client(ssl_context: SSLContext) -> AsyncGenerator[RetryClient]:
|
|
async with (
|
|
aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session,
|
|
RetryClient(session, retry_options=retry_options, raise_for_status=True) as client,
|
|
):
|
|
yield client
|
|
|
|
|
|
async def aiohttp_get_json(url: str, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP GET against a url, retry exponentially on 429s. It expects a JSON response.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
headers (dict): Any headers to add
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError(f"{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.get(
|
|
url.replace(host, "127.0.0.1"),
|
|
headers=headers | {"Host": host},
|
|
server_hostname=host,
|
|
) as response,
|
|
):
|
|
return await response.json()
|
|
|
|
|
|
async def aiohttp_post_json(url: str, data: dict, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP POST against a url, retry exponentially on 429s. IT expects a JSON resposne.
|
|
|
|
Due to synapse bootstrap, when helm has finished deploying, HAProxy can still return
|
|
429s because it did not detect the backend servers ready yet.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
data (dict): The data to post
|
|
headers (dict): Headers to use
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError("f{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.post(
|
|
url.replace(host, "127.0.0.1"), headers=headers | {"Host": host}, server_hostname=host, json=data
|
|
) as response,
|
|
):
|
|
# If we can 204: NO CONTENT, we dont want to try to parse json
|
|
if response.status != 204:
|
|
return await response.json()
|
|
else:
|
|
return {}
|
|
|
|
|
|
async def aiohttp_put_json(url: str, data: dict, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP PUT against a url, retry exponentially on 429s. IT expects a JSON resposne.
|
|
|
|
Due to synapse bootstrap, when helm has finished deploying, HAProxy can still return
|
|
429s because it did not detect the backend servers ready yet.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
data (dict): The data to post
|
|
headers (dict): Headers to use
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError("f{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.put(
|
|
url.replace(host, "127.0.0.1"), headers=headers | {"Host": host}, server_hostname=host, json=data
|
|
) as response,
|
|
):
|
|
# If we can 204: NO CONTENT, we dont want to try to parse json
|
|
if response.status != 204:
|
|
return await response.json()
|
|
else:
|
|
return {}
|
|
|
|
|
|
def merge(a: dict, b: dict, path=None):
|
|
if not path:
|
|
path = []
|
|
for key in b:
|
|
if key in a:
|
|
if isinstance(a[key], dict) and isinstance(b[key], dict):
|
|
merge(a[key], b[key], path + [str(key)])
|
|
elif type(a[key]) is not type(b[key]):
|
|
raise Exception("Conflict at " + ".".join(path + [str(key)]))
|
|
else:
|
|
a[key] = b[key]
|
|
else:
|
|
a[key] = b[key]
|
|
return a
|
|
|
|
|
|
def value_file_has(property_path, expected=None):
|
|
"""
|
|
Check if a nested property (given as a dot-separated string) is would be true if the chart was installed/templated.
|
|
"""
|
|
# If we do not have TEST_VALUES_FILE defined, we return False by default
|
|
if not os.environ.get("TEST_VALUES_FILE"):
|
|
return False
|
|
|
|
with (
|
|
open(Path().resolve() / "charts" / "matrix-stack" / "values.yaml") as base_value_file,
|
|
open(os.environ["TEST_VALUES_FILE"]) as test_value_file,
|
|
):
|
|
data = merge(yaml.safe_load(base_value_file), yaml.safe_load(test_value_file))
|
|
|
|
keys = property_path.split(".")
|
|
for key in keys:
|
|
if isinstance(data, dict) and key in data:
|
|
data = data[key]
|
|
else:
|
|
return False
|
|
return data == expected if expected is not None else True
|
|
|
|
|
|
async def read_service_monitor_kind(kube_client):
|
|
async for r in kube_client.list(CustomResourceDefinition):
|
|
if r.spec.group == "monitoring.coreos.com" and r.spec.names.kind == "ServiceMonitor":
|
|
generic_resource = get_generic_resource(f"{r.spec.group}/{r.spec.versions[0].name}", r.spec.names.kind)
|
|
if generic_resource is None:
|
|
await async_load_in_cluster_generic_resources(kube_client)
|
|
generic_resource = get_generic_resource(f"{r.spec.group}/{r.spec.versions[0].name}", r.spec.names.kind)
|
|
return generic_resource
|
|
raise Exception("Could not find ServiceMonitor CRD")
|
|
|
|
|
|
async def get_pods_matching_labels(kube_client: AsyncClient, namespace: str, labels: dict):
|
|
active_pods = set[str]()
|
|
async for event, pod in kube_client.watch(Pod, namespace=namespace, labels=labels):
|
|
assert pod
|
|
assert pod.metadata
|
|
assert pod.metadata.name
|
|
assert pod.status
|
|
assert pod.status.phase
|
|
pod_name = pod.metadata.name
|
|
if event in ["ADDED", "MODIFIED"] and pod_name not in active_pods and pod.status.phase == "Running":
|
|
active_pods.add(pod_name)
|
|
yield pod_name
|
|
|
|
if event in ["MODIFIED", "DELETED"] and pod_name in active_pods and pod.status.phase != "Running":
|
|
active_pods.remove(pod_name)
|
|
|
|
|
|
async def stream_logs_from_pod(kube_client: AsyncClient, namespace: str, pod_name: str, log_queue: asyncio.Queue):
|
|
async for line in kube_client.log(pod_name, namespace=namespace, follow=True, newlines=False):
|
|
log_queue.put_nowait(line)
|
|
|
|
|
|
async def stream_logs_from_pods_matching_labels(
|
|
kube_client: AsyncClient, namespace: str, labels: dict[str, str], log_queue: asyncio.Queue
|
|
):
|
|
async with asyncio.TaskGroup() as task_group:
|
|
async for pod_name in get_pods_matching_labels(kube_client, namespace, labels):
|
|
task_group.create_task(stream_logs_from_pod(kube_client, namespace, pod_name, log_queue))
|
|
|
|
|
|
async def forward_matching_logs(input_logs_queue: asyncio.Queue, output_matchers_to_logs_queues: list):
|
|
while True:
|
|
log_line = await input_logs_queue.get()
|
|
for matcher, output_logs_queue in output_matchers_to_logs_queues:
|
|
if matcher(log_line):
|
|
output_logs_queue.put_nowait(log_line)
|
|
|
|
|
|
async def chart_from_ci_cache(helm_client: pyhelm3.Client, chart_ref: str) -> pyhelm3.Chart:
|
|
project_root = Path(__file__).parent.parent.parent.parent
|
|
if os.environ.get("CI"):
|
|
helm_cache = project_root / ".pyhelm_cache"
|
|
helm_cache.mkdir(parents=True, exist_ok=True)
|
|
chart_name = chart_ref.split("/")[-1]
|
|
cached_ref = helm_cache / chart_name
|
|
try:
|
|
# Always try to fetch the remote chart first
|
|
async with helm_client.pull_chart(chart_ref) as chart:
|
|
if cached_ref.exists():
|
|
shutil.rmtree(cached_ref)
|
|
# pull_chart returns a Chart object, not a Path, type annotation is wrong
|
|
chart.ref.copy(cached_ref) # type: ignore
|
|
# pull_chart removes the pulled chart, we return the cached one
|
|
return await helm_client.get_chart(cached_ref) # type: ignore
|
|
except Exception as e:
|
|
# Fall back to cache if remote fetch fails
|
|
if cached_ref.exists():
|
|
return await helm_client.get_chart(cached_ref)
|
|
else:
|
|
raise FileNotFoundError(
|
|
f"Could not find chart {chart_ref} in cache at {cached_ref}. "
|
|
f"Remote fetch also failed with error: {e}"
|
|
) from e
|
|
else:
|
|
return await helm_client.get_chart(chart_ref)
|
|
|
|
|
|
async def async_retry_with_timeout(
|
|
retriable_callable: Callable[[], Awaitable[Any]],
|
|
max_retries=15,
|
|
timeout_seconds=60,
|
|
delay=1,
|
|
should_retry: Callable[[BaseException], bool] | None = None,
|
|
):
|
|
begin_time = datetime.now()
|
|
last_exception = None
|
|
attempts_ts: list[datetime] = []
|
|
|
|
for attempt in range(max_retries):
|
|
# Check if timeout has been exceeded
|
|
if datetime.now() > begin_time + timedelta(seconds=timeout_seconds):
|
|
raise TimeoutError(
|
|
f"Timeout of {timeout_seconds} seconds exceeded. "
|
|
f"Last exception: {last_exception if last_exception else 'None'}"
|
|
f"Tried attempts: {[a.strftime('%H:%M:%S') for a in attempts_ts]}"
|
|
)
|
|
|
|
try:
|
|
attempts_ts.append(datetime.now())
|
|
return await retriable_callable()
|
|
except BaseException as e:
|
|
last_exception = e
|
|
# If no should_retry function is provided, retry on all exceptions
|
|
if should_retry is None or should_retry(e):
|
|
print(f"Attempt {attempt}/{max_retries} failed: {e}")
|
|
if attempt < max_retries:
|
|
await asyncio.sleep(delay)
|
|
continue # Explicitly continue to the next attempt
|
|
raise # Re-raise if should_retry returns False
|
|
|
|
if last_exception:
|
|
raise RuntimeError(
|
|
f"All {max_retries} attempts failed"
|
|
f"Last exception: {last_exception if last_exception else 'None'}"
|
|
f"Tried attempts: {[a.strftime('%H:%M:%S') for a in attempts_ts]}"
|
|
) # Re-raise the last exception if all retries fail
|