# Copyright 2024-2025 New Vector Ltd # Copyright 2025-2026 Element Creations Ltd # # SPDX-License-Identifier: AGPL-3.0-only name: Run PyTest tests - Integration and Manifests on: pull_request: push: branches: - main - 'maintenance/*' workflow_dispatch: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: pytest-setup: runs-on: ubuntu-latest outputs: envFiles: ${{ steps.data.outputs.envFiles }} manifestTests: ${{ steps.data.outputs.manifestTests }} upgradeFrom: ${{ steps.data.outputs.upgradeFrom }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false fetch-depth: 0 fetch-tags: true - id: data run: | echo "envFiles=$(find tests/integration/env -name '*.rc' | sed 's|tests/integration/env/||' | sed 's/.rc$//' | jq -ncR '{envFiles: [inputs]}')" | tee -a "$GITHUB_OUTPUT" echo "manifestTests=$(find tests/manifests -name 'test_*.py' | sed 's|tests/manifests/||' | sed 's/.py$//' | jq -ncR '{manifestTests: [inputs]}')" | tee -a "$GITHUB_OUTPUT" LATEST_TAG=$(git describe --abbrev=0 --tags --match '[0-9]*.[0-9]*.[0-9]*') echo "upgradeFrom=$LATEST_TAG" | tee -a "$GITHUB_OUTPUT" pytest-integration: runs-on: ubuntu-latest needs: pytest-setup strategy: fail-fast: false matrix: test-component: ${{ fromJSON(needs.pytest-setup.outputs.envFiles).envFiles }} test-from-ref: - "${{ github.event.pull_request.head.sha }}" - "${{ needs.pytest-setup.outputs.upgradeFrom }}" env: MATRIX_TEST_COMPONENT: ${{ matrix.test-component }} MATRIX_TEST_FROM_REF: ${{ matrix.test-from-ref }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false fetch-depth: 0 fetch-tags: true - name: Install uv uses: astral-sh/setup-uv@6ee6290f1cbc4156c0bdd66691b2c144ef8df19a # v7 with: enable-cache: true activate-environment: true - name: Set up UV environment run: uv sync - name: Get current week number and home dir id: cache-input run: | echo "week=$(date +%Y%U)" | tee -a "$GITHUB_OUTPUT" echo "runner-home=$HOME" | tee -a "$GITHUB_OUTPUT" - name: Restore pyhelm cache id: cache-pyhelm uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: .pyhelm_cache key: pyhelm-integration-cache-${{ hashFiles('tests/integration/fixtures/cluster.py') }}-week-${{ steps.cache-input.outputs.week }} restore-keys: | pyhelm-integration-cache- - name: List pyhelm cache contents run: | echo "Listing contents of .pyhelm_cache:" ls -la .pyhelm_cache || echo "Cache directory does not exist or is empty" - name: Login to Dockerhub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 if: ${{ github.event.pull_request.head.repo.full_name == 'element-hq/ess-helm' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} with: registry: docker.io username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: ./.github/actions/asdf-cache with: tool-versions: | k3d 5.8.3 - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 - name: Checkout values files from previous ref run: | git checkout --no-overlay -f "${MATRIX_TEST_FROM_REF}" -- charts/matrix-stack tests/integration/env/ if: ${{ matrix.test-from-ref != github.event.pull_request.head.sha }} - name: Deploy with pytest once env: PYTEST_CI_FIRST_STEP: "1" # Drop me after releasing next 26.3.x # We are changing the k3d ports range, so we cannot setup matrix-rtc with the old values if: ${{ env.MATRIX_TEST_FROM_REF == '26.2.3' && !contains(env.MATRIX_TEST_COMPONENT, 'matrix-rtc') || env.MATRIX_TEST_FROM_REF != '26.2.3' }} run: | if [ -f "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc" ]; then # shellcheck source=/dev/null . "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc" PYTEST_KEEP_CLUSTER=1 pytest -vv tests/integration else echo "The integration test does not exist in previous ref, skipping..." fi - name: Checkout values files back run: | git checkout --no-overlay -f HEAD -- charts/matrix-stack tests/integration/env/ - name: On upgrade, Synapse can restart, expect 429 to occur run: | echo "PYTEST_EXPECTED_HTTP_STATUS_CODES=429,502,504" >> "$GITHUB_ENV" if: ${{ matrix.test-from-ref != github.event.pull_request.head.sha }} - name: Test with pytest (upgrade or idempotent setup) env: PYTEST_CI_SECOND_STEP: "1" run: | # shellcheck source=/dev/null . "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc" PYTEST_KEEP_CLUSTER=1 pytest -vv tests/integration # syn2mas is a special case that cannot run twice by design if: ${{ matrix.test-component != 'matrix-authentication-service-syn2mas' }} - name: Save pyhelm cache if: always() && steps.cache-pyhelm.outputs.cache-hit != 'true' uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: key: pyhelm-integration-cache-${{ hashFiles('tests/integration/fixtures/cluster.py') }}-week-${{ steps.cache-input.outputs.week }} path: .pyhelm_cache - name: Export logs if: ${{ failure() }} shell: bash run: | mkdir ess-helm-logs k3d kubeconfig merge ess-helm -ds for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do mkdir -p "./ess-helm-logs/$ns" for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers); do kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors --previous "$pod" > "./ess-helm-logs/$ns/$pod.previous" kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps --ignore-errors "$pod" > "./ess-helm-logs/$ns/$pod.logs" done resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints") for i in "${resources[@]}"; do kubectl --context k3d-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$ns/$i.txt" echo "----" >> "./ess-helm-logs/$ns/$i.txt" kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$ns/$i.txt" done kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > "./ess-helm-logs/$ns/events.txt" kubectl --context k3d-ess-helm get secrets -n "$ns" > "./ess-helm-logs/$ns/secrets.txt" echo "----" >> "./ess-helm-logs/$ns/secrets.txt" kubectl --context k3d-ess-helm get secrets -o yaml -n "$ns" | yq eval '.items[].data |= with_entries(.value = "censored") | .items[].stringData |= with_entries(.value = "censored")' > "./ess-helm-logs/$ns/secrets.txt" done kubectl --context k3d-ess-helm get crds > "./ess-helm-logs/crds.txt" echo "----" >> "./ess-helm-logs/$ns/crds.txt" kubectl --context k3d-ess-helm get crds -o yaml | yq 'del .items[].spec' > "./ess-helm-logs/crds.txt" docker logs k3d-ess-helm-server-0 > "./ess-helm-logs/k3d-ess-helm-server-0.logs" 2>&1 k3d cluster delete ess-helm - name: Upload logs if: ${{ failure() }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: ess-helm-logs-${{ matrix.test-component }}-${{ matrix.test-from-ref }} path: ess-helm-logs retention-days: 1 pytest-manifests: runs-on: ubuntu-latest needs: pytest-setup strategy: fail-fast: false matrix: manifest-test: ${{ fromJSON(needs.pytest-setup.outputs.manifestTests).manifestTests }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@6ee6290f1cbc4156c0bdd66691b2c144ef8df19a # v7 with: enable-cache: true activate-environment: true - name: Set up UV environment run: uv sync # Helm >= 3.19.5 (including Helm v4) have https://github.com/helm/helm/issues/31643 # which breaks one of the Ingress tests - name: Pin Helm with asdf uses: ./.github/actions/asdf-cache with: tool-versions: | helm 3.19.4 - name: Restore pytest cache uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: .pytest_cache key: pytest-manifests-cache-${{ matrix.manifest-test }}-${{ hashFiles('charts/matrix-stack/values.yaml') }} restore-keys: | pytest-manifests-cache- - name: Run our manifest test env: MANIFEST_TEST: ${{ matrix.manifest-test }} run: pytest -vv "tests/manifests/${MANIFEST_TEST}.py"