mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-21 17:41:44 -08:00
Add a script to collect logs
This commit is contained in:
4 files changed
+113
-24
No files matched your search
@@ -160,30 +160,7 @@ jobs:
|
||||
if: ${{ failure() }}
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir ess-helm-logs
|
||||
k3d kubeconfig merge ess-helm -ds
|
||||
for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do
|
||||
mkdir -p "./ess-helm-logs/$ns"
|
||||
for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers); do
|
||||
kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors --previous "$pod" > "./ess-helm-logs/$ns/$pod.previous"
|
||||
kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors "$pod" > "./ess-helm-logs/$ns/$pod.logs"
|
||||
done
|
||||
resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints")
|
||||
for i in "${resources[@]}"; do
|
||||
kubectl --context k3d-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$ns/$i.txt"
|
||||
echo "----" >> "./ess-helm-logs/$ns/$i.txt"
|
||||
kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$ns/$i.txt"
|
||||
done
|
||||
kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > "./ess-helm-logs/$ns/events.txt"
|
||||
kubectl --context k3d-ess-helm get secrets -n "$ns" > "./ess-helm-logs/$ns/secrets.txt"
|
||||
echo "----" >> "./ess-helm-logs/$ns/secrets.txt"
|
||||
kubectl --context k3d-ess-helm get secrets -o yaml -n "$ns" | yq eval '.items[].data |= with_entries(.value = "censored") | .items[].stringData |= with_entries(.value = "censored")' > "./ess-helm-logs/$ns/secrets.txt"
|
||||
done
|
||||
kubectl --context k3d-ess-helm get crds > "./ess-helm-logs/crds.txt"
|
||||
echo "----" >> "./ess-helm-logs/$ns/crds.txt"
|
||||
kubectl --context k3d-ess-helm get crds -o yaml | yq 'del .items[].spec' > "./ess-helm-logs/crds.txt"
|
||||
docker logs k3d-ess-helm-server-0 > "./ess-helm-logs/k3d-ess-helm-server-0.logs" 2>&1
|
||||
k3d cluster delete ess-helm
|
||||
collect-ess-logs --system-logs
|
||||
|
||||
- name: Upload logs
|
||||
if: ${{ failure() }}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
CI: Use a python script to generate logs.
|
||||
@@ -2,6 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Literal
|
||||
@@ -13,6 +14,115 @@ HERE = Path(__file__).resolve().parent
|
||||
existing_test_suites = [file.stem for file in (HERE / "env").glob("*.rc")]
|
||||
|
||||
|
||||
def run_command(cmd):
|
||||
result = subprocess.run(cmd, shell=True, check=True, text=True, capture_output=True)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def run_command_to_file(cmd, output_file, check=True, text=True):
|
||||
"""Run a command and write its output directly to a file."""
|
||||
with open(output_file, "w") as f:
|
||||
f.write("----\n")
|
||||
subprocess.run(cmd, shell=True, check=check, stdout=f, stderr=subprocess.PIPE, text=text)
|
||||
|
||||
|
||||
def censor_secrets_yaml(yaml_content):
|
||||
"""Censor all data and stringData fields in a Kubernetes secrets YAML."""
|
||||
secrets = yaml.safe_load(yaml_content)
|
||||
if secrets and "items" in secrets:
|
||||
for item in secrets["items"]:
|
||||
if "data" in item:
|
||||
for key in item["data"]:
|
||||
item["data"][key] = "censored"
|
||||
if "stringData" in item:
|
||||
for key in item["stringData"]:
|
||||
item["stringData"][key] = "censored"
|
||||
return yaml.dump(secrets, default_flow_style=False)
|
||||
|
||||
|
||||
def collect_ess_logs():
|
||||
def _collect_logs(destination: Path = Path("ess-helm-logs"), system_logs: bool = False):
|
||||
Path(destination).mkdir(exist_ok=True)
|
||||
|
||||
# Merge kubeconfig
|
||||
subprocess.run("k3d kubeconfig merge ess-helm -ds", shell=True, check=True)
|
||||
|
||||
# Get all namespaces
|
||||
namespaces = run_command(
|
||||
"kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers"
|
||||
).splitlines()
|
||||
|
||||
resources = [
|
||||
"pods",
|
||||
"deployments",
|
||||
"statefulsets",
|
||||
"services",
|
||||
"configmaps",
|
||||
"ingresses",
|
||||
"persistentvolumes",
|
||||
"persistentvolumeclaims",
|
||||
"endpoints",
|
||||
]
|
||||
|
||||
for ns in namespaces:
|
||||
if not system_logs and "ess" not in ns:
|
||||
continue
|
||||
(destination / ns).mkdir(exist_ok=True)
|
||||
# Get all pods in the namespace
|
||||
pods = run_command(
|
||||
f"kubectl --context=k3d-ess-helm get pods -n {ns} -o custom-columns=NAME:.metadata.name --no-headers"
|
||||
).splitlines()
|
||||
|
||||
for pod in pods:
|
||||
# Get previous logs
|
||||
run_command_to_file(
|
||||
f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps "
|
||||
f"--ignore-errors --previous {pod}",
|
||||
f"{destination}/{ns}/{pod}.previous",
|
||||
check=False,
|
||||
)
|
||||
# Get current logs
|
||||
run_command_to_file(
|
||||
f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps "
|
||||
f"--ignore-errors {pod}",
|
||||
f"{destination}/{ns}/{pod}.logs",
|
||||
)
|
||||
|
||||
# Get resources
|
||||
for resource in resources:
|
||||
output_file = f"{destination}/{ns}/{resource}.txt"
|
||||
run_command_to_file(f"kubectl --context=k3d-ess-helm get {resource} -n {ns}", output_file)
|
||||
run_command_to_file(
|
||||
f"kubectl --context=k3d-ess-helm get {resource} -o yaml -n {ns}", f"{output_file}.yaml"
|
||||
)
|
||||
|
||||
# Get events
|
||||
run_command_to_file(
|
||||
f"kubectl --context=k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n {ns}",
|
||||
f"{destination}/{ns}/events.txt",
|
||||
)
|
||||
|
||||
# Get secrets (censored)
|
||||
secrets_yaml = run_command(f"kubectl --context=k3d-ess-helm get secrets -o yaml -n {ns}")
|
||||
censored_yaml = censor_secrets_yaml(secrets_yaml)
|
||||
with open(f"{destination}/{ns}/secrets.txt", "w") as f:
|
||||
f.write("----\n")
|
||||
f.write(censored_yaml)
|
||||
|
||||
if system_logs:
|
||||
# Get CRDs
|
||||
run_command_to_file(
|
||||
"kubectl --context=k3d-ess-helm get crds",
|
||||
f"{destination}/crds.txt",
|
||||
)
|
||||
|
||||
# Get k3d server logs
|
||||
run_command_to_file("docker logs k3d-ess-helm-server-0", f"{destination}/k3d-ess-helm-server-0.logs")
|
||||
typer.echo(f"Logs and resources collected in {destination}")
|
||||
|
||||
typer.run(_collect_logs)
|
||||
|
||||
|
||||
def run_tests():
|
||||
def _run_tests(
|
||||
test_suite: Annotated[Literal[*existing_test_suites], typer.Option()],
|
||||
|
||||
@@ -41,6 +41,7 @@ dependencies = [
|
||||
[project.scripts]
|
||||
setup-ess-cluster = "integration:scripts.setup_cluster"
|
||||
pytest-ess = "integration:scripts.run_tests"
|
||||
collect-ess-logs = "integration:scripts.collect_ess_logs"
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.10.9,<0.11.0"]
|
||||
|
||||
Reference in new issue
Block a user