mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-20 00:51:27 -08:00
456 lines
17 KiB
Python
456 lines
17 KiB
Python
# Copyright 2024-2025 New Vector Ltd
|
|
# Copyright 2025-2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import random
|
|
import shutil
|
|
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from ssl import SSLContext
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import aiohttp
|
|
import pyhelm3
|
|
import yaml
|
|
from aiohttp_retry import JitterRetry, RetryClient
|
|
from lightkube import AsyncClient
|
|
from lightkube.generic_resource import (
|
|
async_load_in_cluster_generic_resources,
|
|
get_generic_resource,
|
|
)
|
|
from lightkube.resources.apiextensions_v1 import CustomResourceDefinition
|
|
from lightkube.resources.apps_v1 import Deployment, StatefulSet
|
|
from lightkube.resources.core_v1 import Pod
|
|
|
|
retry_options = JitterRetry(
|
|
attempts=12,
|
|
statuses={
|
|
429,
|
|
500,
|
|
503,
|
|
}
|
|
| set(int(code) for code in os.environ.get("PYTEST_EXPECTED_HTTP_STATUS_CODES", "").split(",") if code),
|
|
retry_all_server_errors=False,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class DockerAuth:
|
|
registry: str
|
|
username: str
|
|
password: str
|
|
|
|
|
|
def docker_config_json(auths: list[DockerAuth]) -> str:
|
|
docker_config_auths = {}
|
|
for auth in auths:
|
|
docker_config_auths[auth.registry] = {
|
|
"username": auth.username,
|
|
"password": auth.password,
|
|
"auth": b64encode(f"{auth.username}:{auth.password}"),
|
|
}
|
|
|
|
return json.dumps({"auths": docker_config_auths})
|
|
|
|
|
|
def b64encode(value: str):
|
|
return base64.b64encode(value.encode("utf-8")).decode("utf-8")
|
|
|
|
|
|
@asynccontextmanager
|
|
async def aiohttp_client(ssl_context: SSLContext) -> AsyncGenerator[RetryClient]:
|
|
async with (
|
|
aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session,
|
|
RetryClient(session, retry_options=retry_options, raise_for_status=True) as client,
|
|
):
|
|
yield client
|
|
|
|
|
|
async def aiohttp_get_json(url: str, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP GET against a url, retry exponentially on 429s. It expects a JSON response.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
headers (dict): Any headers to add
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError(f"{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.get(
|
|
url.replace(host, "127.0.0.1"),
|
|
headers=headers | {"Host": host},
|
|
server_hostname=host,
|
|
) as response,
|
|
):
|
|
return await response.json()
|
|
|
|
|
|
async def aiohttp_post_json(url: str, data: dict, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP POST against a url, retry exponentially on 429s. IT expects a JSON resposne.
|
|
|
|
Due to synapse bootstrap, when helm has finished deploying, HAProxy can still return
|
|
429s because it did not detect the backend servers ready yet.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
data (dict): The data to post
|
|
headers (dict): Headers to use
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError("f{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.post(
|
|
url.replace(host, "127.0.0.1"), headers=headers | {"Host": host}, server_hostname=host, json=data
|
|
) as response,
|
|
):
|
|
# If we can 204: NO CONTENT, we dont want to try to parse json
|
|
if response.status != 204:
|
|
return await response.json()
|
|
else:
|
|
return {}
|
|
|
|
|
|
async def aiohttp_put_json(url: str, data: dict, headers: dict, ssl_context: SSLContext) -> Any:
|
|
"""Do an async HTTP PUT against a url, retry exponentially on 429s. IT expects a JSON resposne.
|
|
|
|
Due to synapse bootstrap, when helm has finished deploying, HAProxy can still return
|
|
429s because it did not detect the backend servers ready yet.
|
|
|
|
Args:
|
|
url (str): The URL to hit
|
|
data (dict): The data to post
|
|
headers (dict): Headers to use
|
|
ssl_context (SSLContext): The SSL Context with test CA loaded
|
|
|
|
Returns:
|
|
Any: the Json dict response
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError("f{url} does not have a hostname")
|
|
|
|
async with (
|
|
aiohttp_client(ssl_context) as client,
|
|
client.put(
|
|
url.replace(host, "127.0.0.1"), headers=headers | {"Host": host}, server_hostname=host, json=data
|
|
) as response,
|
|
):
|
|
# If we can 204: NO CONTENT, we dont want to try to parse json
|
|
if response.status != 204:
|
|
return await response.json()
|
|
else:
|
|
return {}
|
|
|
|
|
|
def merge(a: dict, b: dict, path=None):
|
|
if not path:
|
|
path = []
|
|
for key in b:
|
|
if key in a:
|
|
if isinstance(a[key], dict) and isinstance(b[key], dict):
|
|
merge(a[key], b[key], path + [str(key)])
|
|
elif type(a[key]) is not type(b[key]):
|
|
raise Exception("Conflict at " + ".".join(path + [str(key)]))
|
|
else:
|
|
a[key] = b[key]
|
|
else:
|
|
a[key] = b[key]
|
|
return a
|
|
|
|
|
|
def value_file_has(property_path, expected=None):
|
|
"""
|
|
Check if a nested property (given as a dot-separated string) is would be true if the chart was installed/templated.
|
|
"""
|
|
# If we do not have TEST_VALUES_FILE defined, we return False by default
|
|
if not os.environ.get("TEST_VALUES_FILE"):
|
|
return False
|
|
|
|
with (
|
|
open(Path().resolve() / "charts" / "matrix-stack" / "values.yaml") as base_value_file,
|
|
open(os.environ["TEST_VALUES_FILE"]) as test_value_file,
|
|
):
|
|
data = merge(yaml.safe_load(base_value_file), yaml.safe_load(test_value_file))
|
|
|
|
if os.environ.get("ADDITIONAL_TEST_VALUES_FILE"):
|
|
with open(os.environ["ADDITIONAL_TEST_VALUES_FILE"]) as additional_test_value_file:
|
|
data = merge(data, yaml.safe_load(additional_test_value_file))
|
|
|
|
keys = property_path.split(".")
|
|
for key in keys:
|
|
if isinstance(data, dict) and key in data:
|
|
data = data[key]
|
|
else:
|
|
return False
|
|
return data == expected if expected is not None else True
|
|
|
|
|
|
async def read_service_monitor_kind(kube_client):
|
|
async for r in kube_client.list(CustomResourceDefinition):
|
|
if r.spec.group == "monitoring.coreos.com" and r.spec.names.kind == "ServiceMonitor":
|
|
generic_resource = get_generic_resource(f"{r.spec.group}/{r.spec.versions[0].name}", r.spec.names.kind)
|
|
if generic_resource is None:
|
|
await async_load_in_cluster_generic_resources(kube_client)
|
|
generic_resource = get_generic_resource(f"{r.spec.group}/{r.spec.versions[0].name}", r.spec.names.kind)
|
|
return generic_resource
|
|
raise Exception("Could not find ServiceMonitor CRD")
|
|
|
|
|
|
async def get_pods_matching_labels(kube_client: AsyncClient, namespace: str, labels: dict):
|
|
active_pods = set[str]()
|
|
async for event, pod in kube_client.watch(Pod, namespace=namespace, labels=labels):
|
|
assert pod
|
|
assert pod.metadata
|
|
assert pod.metadata.name
|
|
assert pod.status
|
|
assert pod.status.phase
|
|
pod_name = pod.metadata.name
|
|
if event in ["ADDED", "MODIFIED"] and pod_name not in active_pods and pod.status.phase == "Running":
|
|
active_pods.add(pod_name)
|
|
yield pod_name
|
|
|
|
if event in ["MODIFIED", "DELETED"] and pod_name in active_pods and pod.status.phase != "Running":
|
|
active_pods.remove(pod_name)
|
|
|
|
|
|
async def stream_logs_from_pod(kube_client: AsyncClient, namespace: str, pod_name: str, log_queue: asyncio.Queue):
|
|
async for line in kube_client.log(pod_name, namespace=namespace, follow=True, newlines=False):
|
|
log_queue.put_nowait(line)
|
|
|
|
|
|
async def stream_logs_from_pods_matching_labels(
|
|
kube_client: AsyncClient, namespace: str, labels: dict[str, str], log_queue: asyncio.Queue
|
|
):
|
|
async with asyncio.TaskGroup() as task_group:
|
|
async for pod_name in get_pods_matching_labels(kube_client, namespace, labels):
|
|
task_group.create_task(stream_logs_from_pod(kube_client, namespace, pod_name, log_queue))
|
|
|
|
|
|
async def forward_matching_logs(input_logs_queue: asyncio.Queue, output_matchers_to_logs_queues: list):
|
|
while True:
|
|
log_line = await input_logs_queue.get()
|
|
for matcher, output_logs_queue in output_matchers_to_logs_queues:
|
|
if matcher(log_line):
|
|
output_logs_queue.put_nowait(log_line)
|
|
|
|
|
|
async def chart_from_ci_cache(helm_client: pyhelm3.Client, chart_ref: str) -> pyhelm3.Chart:
|
|
if os.environ.get("GITHUB_WORKSPACE"):
|
|
helm_cache = Path(os.environ["GITHUB_WORKSPACE"]) / ".pyhelm_cache"
|
|
helm_cache.mkdir(parents=True, exist_ok=True)
|
|
chart_name = chart_ref.split("/")[-1]
|
|
cached_ref = helm_cache / chart_name
|
|
try:
|
|
# Always try to fetch the remote chart first
|
|
async with helm_client.pull_chart(chart_ref) as chart:
|
|
if cached_ref.exists():
|
|
shutil.rmtree(cached_ref)
|
|
# pull_chart returns a Chart object, not a Path, type annotation is wrong
|
|
shutil.copytree(chart.ref, cached_ref) # type: ignore
|
|
# pull_chart removes the pulled chart, we return the cached one
|
|
return await helm_client.get_chart(cached_ref) # type: ignore
|
|
except Exception as e:
|
|
# Fall back to cache if remote fetch fails
|
|
if cached_ref.exists():
|
|
return await helm_client.get_chart(cached_ref)
|
|
else:
|
|
raise FileNotFoundError(
|
|
f"Could not find chart {chart_ref} in cache at {cached_ref}. "
|
|
f"The cache {helm_cache} contains : {helm_cache.glob('*')}"
|
|
f"Remote fetch also failed with error: {e}"
|
|
) from e
|
|
else:
|
|
return await helm_client.get_chart(chart_ref)
|
|
|
|
|
|
async def install_release_and_wait_for_resources(
|
|
helm_client: pyhelm3.Client,
|
|
kube_client: AsyncClient,
|
|
release_name: str,
|
|
chart: pyhelm3.Chart,
|
|
values: dict,
|
|
namespace: str,
|
|
create_namespace: bool,
|
|
):
|
|
"""Install or upgrade a Helm release without --wait, then poll its resources until ready.
|
|
|
|
We don't use helm's --wait: in flaky runs helm 4's status watcher never
|
|
sees the resources as ready, even though they are, and hangs until the timeout.
|
|
See https://github.com/helm/helm/issues/32615
|
|
We also don't use --atomic/--rollback-on-failure as it implies --wait.
|
|
|
|
The Deployments, StatefulSets and CustomResourceDefinitions of the release
|
|
are read from its manifest, and their readiness is polled instead.
|
|
Note that CRDs placed in the chart's `crds/` folder are not part of the
|
|
release manifest, so they are not waited on.
|
|
"""
|
|
revision = await helm_client.install_or_upgrade_release(
|
|
release_name,
|
|
chart,
|
|
values,
|
|
namespace=namespace,
|
|
create_namespace=create_namespace,
|
|
)
|
|
|
|
# The manifest does not always carry the namespace. Helm then installs
|
|
# the resource in the release namespace.
|
|
deployments = []
|
|
statefulsets = []
|
|
crd_names = []
|
|
for resource in await revision.resources():
|
|
# The manifest can hold empty documents between its --- separators.
|
|
# They are parsed as None, skip them.
|
|
if not resource:
|
|
continue
|
|
kind = resource.get("kind")
|
|
metadata = resource.get("metadata") or {}
|
|
name = metadata.get("name")
|
|
if not name:
|
|
continue
|
|
if kind == "Deployment":
|
|
deployments.append((name, metadata.get("namespace") or namespace))
|
|
elif kind == "StatefulSet":
|
|
statefulsets.append((name, metadata.get("namespace") or namespace))
|
|
elif kind == "CustomResourceDefinition":
|
|
crd_names.append(name)
|
|
|
|
async def _deployments_are_ready():
|
|
for deployment_name, deployment_namespace in deployments:
|
|
deployment = await kube_client.get(Deployment, name=deployment_name, namespace=deployment_namespace)
|
|
conditions = deployment.status.conditions if deployment.status else []
|
|
if not any(condition.type == "Available" and condition.status == "True" for condition in conditions):
|
|
await asyncio.sleep(random.uniform(0, 1))
|
|
raise AssertionError(f"Deployment {deployment_name} not ready, conditions: {conditions}")
|
|
|
|
async def _statefulsets_are_ready():
|
|
for statefulset_name, statefulset_namespace in statefulsets:
|
|
statefulset = await kube_client.get(StatefulSet, name=statefulset_name, namespace=statefulset_namespace)
|
|
replicas = statefulset.spec.replicas if statefulset.spec else 0
|
|
ready_replicas = statefulset.status.readyReplicas if statefulset.status else 0
|
|
# StatefulSets only expose conditions on newer Kubernetes versions,
|
|
# so compare the ready replicas against the desired ones instead
|
|
if ready_replicas != replicas:
|
|
await asyncio.sleep(random.uniform(0, 1))
|
|
raise AssertionError(
|
|
f"StatefulSet {statefulset_name} not ready, readyReplicas: {ready_replicas}/{replicas}"
|
|
)
|
|
|
|
async def _crds_are_established():
|
|
for crd_name in crd_names:
|
|
crd = await kube_client.get(CustomResourceDefinition, name=crd_name)
|
|
conditions = crd.status.conditions if crd.status else []
|
|
if not any(condition.type == "Established" and condition.status == "True" for condition in conditions):
|
|
await asyncio.sleep(random.uniform(0, 1))
|
|
raise AssertionError(f"{crd_name} CRD not established, conditions: {conditions}")
|
|
|
|
# The three checks run concurrently on each attempt. return_exceptions=True
|
|
# lets them all run to completion, so that the reported failure isn't
|
|
# whichever kind happened to fail first
|
|
async def _resources_are_ready():
|
|
results = await asyncio.gather(
|
|
_deployments_are_ready(),
|
|
_statefulsets_are_ready(),
|
|
_crds_are_established(),
|
|
return_exceptions=True,
|
|
)
|
|
for result in results:
|
|
if isinstance(result, BaseException):
|
|
raise result
|
|
|
|
# Pulling images on a cold runner can take a while, and building the API
|
|
# handlers for a batch of CRDs is CPU intensive for the kube-apiserver,
|
|
# so we allow up to 5 minutes for all of the release's resources.
|
|
await async_retry_with_timeout(
|
|
_resources_are_ready,
|
|
max_retries=150,
|
|
delay=2,
|
|
timeout_seconds=300,
|
|
)
|
|
|
|
|
|
async def async_iterable_payload(value: AsyncIterator[bytes], file_size: int) -> aiohttp.payload.AsyncIterablePayload:
|
|
# Workaround to the fact that aiohttp hardcodes sending a chuncked upload
|
|
# when it receives an async iterator
|
|
# See https://github.com/fsspec/filesystem_spec/issues/1390#issuecomment-1765886388
|
|
payload = aiohttp.payload.AsyncIterablePayload(value)
|
|
payload._size = file_size
|
|
return payload
|
|
|
|
|
|
async def random_bytes(
|
|
size_in_bytes: int,
|
|
sha256_hash: hashlib._Hash,
|
|
chunk_size: int = 4096,
|
|
) -> AsyncIterator[bytes]:
|
|
remaining = size_in_bytes
|
|
while remaining > 0:
|
|
chunk = os.urandom(min(chunk_size, remaining))
|
|
sha256_hash.update(chunk)
|
|
yield chunk
|
|
remaining -= len(chunk)
|
|
|
|
|
|
async def async_retry_with_timeout(
|
|
retriable_callable: Callable[[], Awaitable[Any]],
|
|
max_retries=15,
|
|
timeout_seconds=60,
|
|
delay=1,
|
|
should_retry: Callable[[BaseException], bool] | None = None,
|
|
):
|
|
begin_time = datetime.now()
|
|
last_exception = None
|
|
attempts_ts: list[datetime] = []
|
|
|
|
for attempt in range(max_retries):
|
|
# Check if timeout has been exceeded
|
|
if datetime.now() > begin_time + timedelta(seconds=timeout_seconds):
|
|
raise TimeoutError(
|
|
f"Timeout of {timeout_seconds} seconds exceeded. "
|
|
f"Last exception: {last_exception if last_exception else 'None'}"
|
|
f"Tried attempts: {[a.strftime('%H:%M:%S') for a in attempts_ts]}"
|
|
)
|
|
|
|
try:
|
|
attempts_ts.append(datetime.now())
|
|
return await retriable_callable()
|
|
except BaseException as e:
|
|
last_exception = e
|
|
# If no should_retry function is provided, retry on all exceptions
|
|
if should_retry is None or should_retry(e):
|
|
print(f"Attempt {attempt}/{max_retries} failed: {e}")
|
|
if attempt < max_retries:
|
|
await asyncio.sleep(delay)
|
|
continue # Explicitly continue to the next attempt
|
|
raise # Re-raise if should_retry returns False
|
|
|
|
if last_exception:
|
|
raise RuntimeError(
|
|
f"All {max_retries} attempts failed"
|
|
f"Last exception: {last_exception if last_exception else 'None'}"
|
|
f"Tried attempts: {[a.strftime('%H:%M:%S') for a in attempts_ts]}"
|
|
) # Re-raise the last exception if all retries fail
|