diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b7d73f9a..349d2f13 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -160,30 +160,7 @@ jobs: if: ${{ failure() }} shell: bash run: | - mkdir ess-helm-logs - k3d kubeconfig merge ess-helm -ds - for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do - mkdir -p "./ess-helm-logs/$ns" - for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers); do - kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors --previous "$pod" > "./ess-helm-logs/$ns/$pod.previous" - kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors "$pod" > "./ess-helm-logs/$ns/$pod.logs" - done - resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints") - for i in "${resources[@]}"; do - kubectl --context k3d-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$ns/$i.txt" - echo "----" >> "./ess-helm-logs/$ns/$i.txt" - kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$ns/$i.txt" - done - kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > "./ess-helm-logs/$ns/events.txt" - kubectl --context k3d-ess-helm get secrets -n "$ns" > "./ess-helm-logs/$ns/secrets.txt" - echo "----" >> "./ess-helm-logs/$ns/secrets.txt" - kubectl --context k3d-ess-helm get secrets -o yaml -n "$ns" | yq eval '.items[].data |= with_entries(.value = "censored") | .items[].stringData |= with_entries(.value = "censored")' > "./ess-helm-logs/$ns/secrets.txt" - done - kubectl --context k3d-ess-helm get crds > "./ess-helm-logs/crds.txt" - echo "----" >> "./ess-helm-logs/$ns/crds.txt" - kubectl --context k3d-ess-helm get crds -o yaml | yq 'del .items[].spec' > "./ess-helm-logs/crds.txt" - docker logs k3d-ess-helm-server-0 > "./ess-helm-logs/k3d-ess-helm-server-0.logs" 2>&1 - k3d cluster delete ess-helm + collect-ess-logs --system-logs - name: Upload logs if: ${{ failure() }} diff --git a/newsfragments/1192.internal.md b/newsfragments/1192.internal.md new file mode 100644 index 00000000..b51f5d0c --- /dev/null +++ b/newsfragments/1192.internal.md @@ -0,0 +1 @@ +CI: Use a python script to generate logs. diff --git a/tests/integration/scripts.py b/tests/integration/scripts.py index 579f998a..c1f526d0 100644 --- a/tests/integration/scripts.py +++ b/tests/integration/scripts.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only +import subprocess import sys from pathlib import Path from typing import Annotated, Literal @@ -13,6 +14,115 @@ HERE = Path(__file__).resolve().parent existing_test_suites = [file.stem for file in (HERE / "env").glob("*.rc")] +def run_command(cmd): + result = subprocess.run(cmd, shell=True, check=True, text=True, capture_output=True) + return result.stdout + + +def run_command_to_file(cmd, output_file, check=True, text=True): + """Run a command and write its output directly to a file.""" + with open(output_file, "w") as f: + f.write("----\n") + subprocess.run(cmd, shell=True, check=check, stdout=f, stderr=subprocess.PIPE, text=text) + + +def censor_secrets_yaml(yaml_content): + """Censor all data and stringData fields in a Kubernetes secrets YAML.""" + secrets = yaml.safe_load(yaml_content) + if secrets and "items" in secrets: + for item in secrets["items"]: + if "data" in item: + for key in item["data"]: + item["data"][key] = "censored" + if "stringData" in item: + for key in item["stringData"]: + item["stringData"][key] = "censored" + return yaml.dump(secrets, default_flow_style=False) + + +def collect_ess_logs(): + def _collect_logs(destination: Path = Path("ess-helm-logs"), system_logs: bool = False): + Path(destination).mkdir(exist_ok=True) + + # Merge kubeconfig + subprocess.run("k3d kubeconfig merge ess-helm -ds", shell=True, check=True) + + # Get all namespaces + namespaces = run_command( + "kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers" + ).splitlines() + + resources = [ + "pods", + "deployments", + "statefulsets", + "services", + "configmaps", + "ingresses", + "persistentvolumes", + "persistentvolumeclaims", + "endpoints", + ] + + for ns in namespaces: + if not system_logs and "ess" not in ns: + continue + (destination / ns).mkdir(exist_ok=True) + # Get all pods in the namespace + pods = run_command( + f"kubectl --context=k3d-ess-helm get pods -n {ns} -o custom-columns=NAME:.metadata.name --no-headers" + ).splitlines() + + for pod in pods: + # Get previous logs + run_command_to_file( + f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps " + f"--ignore-errors --previous {pod}", + f"{destination}/{ns}/{pod}.previous", + check=False, + ) + # Get current logs + run_command_to_file( + f"kubectl --context=k3d-ess-helm -n {ns} logs --all-containers --prefix --timestamps " + f"--ignore-errors {pod}", + f"{destination}/{ns}/{pod}.logs", + ) + + # Get resources + for resource in resources: + output_file = f"{destination}/{ns}/{resource}.txt" + run_command_to_file(f"kubectl --context=k3d-ess-helm get {resource} -n {ns}", output_file) + run_command_to_file( + f"kubectl --context=k3d-ess-helm get {resource} -o yaml -n {ns}", f"{output_file}.yaml" + ) + + # Get events + run_command_to_file( + f"kubectl --context=k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n {ns}", + f"{destination}/{ns}/events.txt", + ) + + # Get secrets (censored) + secrets_yaml = run_command(f"kubectl --context=k3d-ess-helm get secrets -o yaml -n {ns}") + censored_yaml = censor_secrets_yaml(secrets_yaml) + with open(f"{destination}/{ns}/secrets.txt", "w") as f: + f.write("----\n") + f.write(censored_yaml) + + if system_logs: + # Get CRDs + run_command_to_file( + "kubectl --context=k3d-ess-helm get crds", + f"{destination}/crds.txt", + ) + + # Get k3d server logs + run_command_to_file("docker logs k3d-ess-helm-server-0", f"{destination}/k3d-ess-helm-server-0.logs") + typer.echo(f"Logs and resources collected in {destination}") + + typer.run(_collect_logs) + + def run_tests(): def _run_tests( test_suite: Annotated[Literal[*existing_test_suites], typer.Option()], diff --git a/tests/pyproject.toml b/tests/pyproject.toml index 97198d5b..19be1279 100644 --- a/tests/pyproject.toml +++ b/tests/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ [project.scripts] setup-ess-cluster = "integration:scripts.setup_cluster" pytest-ess = "integration:scripts.run_tests" +collect-ess-logs = "integration:scripts.collect_ess_logs" [build-system] requires = ["uv_build>=0.10.9,<0.11.0"]