Add a script to collect logs

This commit is contained in:
Gaël Goinvic committed 2026-03-26 17:45:04 +01:00
1 parent db7f7ef6e8
commit b73a29d40e
4 files changed
+113 -24

No files matched your search

+1 -24
View File
@@ -160,30 +160,7 @@ jobs:
if: ${{ failure() }}
shell: bash
run: |
mkdir ess-helm-logs
k3d kubeconfig merge ess-helm -ds
for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do
mkdir -p "./ess-helm-logs/$ns"
for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers); do
kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors --previous "$pod" > "./ess-helm-logs/$ns/$pod.previous"
kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors "$pod" > "./ess-helm-logs/$ns/$pod.logs"
done
resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints")
for i in "${resources[@]}"; do
kubectl --context k3d-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$ns/$i.txt"
echo "----" >> "./ess-helm-logs/$ns/$i.txt"
kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$ns/$i.txt"
done
kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > "./ess-helm-logs/$ns/events.txt"
kubectl --context k3d-ess-helm get secrets -n "$ns" > "./ess-helm-logs/$ns/secrets.txt"
echo "----" >> "./ess-helm-logs/$ns/secrets.txt"
kubectl --context k3d-ess-helm get secrets -o yaml -n "$ns" | yq eval '.items[].data |= with_entries(.value = "censored") | .items[].stringData |= with_entries(.value = "censored")' > "./ess-helm-logs/$ns/secrets.txt"
done
kubectl --context k3d-ess-helm get crds > "./ess-helm-logs/crds.txt"
echo "----" >> "./ess-helm-logs/$ns/crds.txt"
kubectl --context k3d-ess-helm get crds -o yaml | yq 'del .items[].spec' > "./ess-helm-logs/crds.txt"
docker logs k3d-ess-helm-server-0 > "./ess-helm-logs/k3d-ess-helm-server-0.logs" 2>&1
k3d cluster delete ess-helm
collect-ess-logs --system-logs
- name: Upload logs
if: ${{ failure() }}
+1
View File
@@ -0,0 +1 @@
CI: Use a python script to generate logs.
+110
View File
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0-only
import subprocess
import sys
from pathlib import Path
from typing import Annotated, Literal
@@ -13,6 +14,115 @@ HERE = Path(__file__).resolve().parent
existing_test_suites = [file.stem for file in (HERE / "env").glob("*.rc")]
def run_command(cmd):
result = subprocess.run(cmd, shell=True, check=True, text=True, capture_output=True)
return result.stdout
def run_command_to_file(cmd, output_file, check=True, text=True):
"""Run a command and write its output directly to a file."""
with open(output_file, "w") as f:
f.write("----\n")
subprocess.run(cmd, shell=True, check=check, stdout=f, stderr=subprocess.PIPE, text=text)
def censor_secrets_yaml(yaml_content):
"""Censor all data and stringData fields in a Kubernetes secrets YAML."""
secrets = yaml.safe_load(yaml_content)
if secrets and "items" in secrets:
for item in secrets["items"]:
if "data" in item:
for key in item["data"]:
item["data"][key] = "censored"
if "stringData" in item:
for key in item["stringData"]:
item["stringData"][key] = "censored"
return yaml.dump(secrets, default_flow_style=False)
def collect_ess_logs():
def _collect_logs(destination: Path = Path("ess-helm-logs"), system_logs: bool = False):
Path(destination).mkdir(exist_ok=True)
# Merge kubeconfig
subprocess.run("k3d kubeconfig merge ess-helm -ds", shell=True, check=True)
# Get all namespaces
namespaces = run_command(
"kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers"
).splitlines()
resources = [
"pods",
"deployments",
"statefulsets",
"services",
"configmaps",
"ingresses",
"persistentvolumes",
"persistentvolumeclaims",
"endpoints",
]
for ns in namespaces:
if not system_logs and "ess" not in ns:
continue
(destination / ns).mkdir(exist_ok=True)
# Get all pods in the namespace
pods = run_command(
f"kubectl --context=k3d-ess-helm get pods -n {ns} -o custom-columns=NAME:.metadata.name --no-headers"
).splitlines()
for pod in pods:
# Get previous logs
run_command_to_file(
f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps "
f"--ignore-errors --previous {pod}",
f"{destination}/{ns}/{pod}.previous",
check=False,
)
# Get current logs
run_command_to_file(
f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps "
f"--ignore-errors {pod}",
f"{destination}/{ns}/{pod}.logs",
)
# Get resources
for resource in resources:
output_file = f"{destination}/{ns}/{resource}.txt"
run_command_to_file(f"kubectl --context=k3d-ess-helm get {resource} -n {ns}", output_file)
run_command_to_file(
f"kubectl --context=k3d-ess-helm get {resource} -o yaml -n {ns}", f"{output_file}.yaml"
)
# Get events
run_command_to_file(
f"kubectl --context=k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n {ns}",
f"{destination}/{ns}/events.txt",
)
# Get secrets (censored)
secrets_yaml = run_command(f"kubectl --context=k3d-ess-helm get secrets -o yaml -n {ns}")
censored_yaml = censor_secrets_yaml(secrets_yaml)
with open(f"{destination}/{ns}/secrets.txt", "w") as f:
f.write("----\n")
f.write(censored_yaml)
if system_logs:
# Get CRDs
run_command_to_file(
"kubectl --context=k3d-ess-helm get crds",
f"{destination}/crds.txt",
)
# Get k3d server logs
run_command_to_file("docker logs k3d-ess-helm-server-0", f"{destination}/k3d-ess-helm-server-0.logs")
typer.echo(f"Logs and resources collected in {destination}")
typer.run(_collect_logs)
def run_tests():
def _run_tests(
test_suite: Annotated[Literal[*existing_test_suites], typer.Option()],
+1
View File
@@ -41,6 +41,7 @@ dependencies = [
[project.scripts]
setup-ess-cluster = "integration:scripts.setup_cluster"
pytest-ess = "integration:scripts.run_tests"
collect-ess-logs = "integration:scripts.collect_ess_logs"
[build-system]
requires = ["uv_build>=0.10.9,<0.11.0"]