Files
tailscale/.github/workflows/natlab-matrix.yml
Claus Lensbøl eec8725650 tstest/natlab/vmtest{,/cmd},.github/workflows: run generated tests in shards
Instead of having one runner do all the work for a test, split it up
into multiple shards so as the number of tests grows, we can add tests
without running out of time on the runners.

The test runner is also moved out of the generator to make it easier to
work on it without having to generate and without having to write the
code in the template.

Updates #21817

Signed-off-by: Claus Lensbøl <[email protected]>
2026-11-18 10:00:39 -05:00

152 lines
6.2 KiB
YAML

# Run TestMatrix from tstest/natlab/vmtest/matrix_generated_test.go on weekends
# for flake detection.
#
# This is intentionally separate from natlab-test.yml, which runs the full
# vmtest suite. That workflow excludes TestMatrix from its discovery; this one
# runs nothing else.
#
# A lightweight prepare job counts the entries in matrix_generated_test.go and
# fans out into N parallel test jobs of up to 'shard_size' subtests each. The
# shard count is computed automatically, so new tests added to the matrix
# require no changes to this file.
#
# All cache handling and pre-test preparation follows natlab-test.yml.
name: "natlab-matrix"
on:
workflow_dispatch:
schedule:
- cron: "3 1-23/2 * * 6" # Saturday from 01:03 UTC (Friday evening Pacific), every 2h
- cron: "3 */2 * * 0" # all day Sunday, every 2h
- cron: "3 1-5/2 * * 1" # Monday until 05:03 UTC (early morning London), every 2h
jobs:
prepare:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
matrix-shards: ${{ steps.shards.outputs.matrix-shards }}
steps:
- name: Check out code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Restore tsgo toolchain
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/.cache/tsgo
key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}
- name: Restore Go modules
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/go/pkg/mod
key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}
- name: Compute TestMatrix shards
id: shards
run: |
set -euo pipefail
shard_size=25
total=$(./tool/go test -run '^TestMatrix$' -count=1 ./tstest/natlab/vmtest/ -args -matrix-total | head -1)
shards=$(( (total + shard_size - 1) / shard_size ))
matrix_shards=$(seq 1 "$shards" | jq -R . | jq -sc --arg m "$shards" '[ .[] | . + "/" + $m ]')
echo "matrix-shards=${matrix_shards}" >> "$GITHUB_OUTPUT"
echo "TestMatrix: ${total} tests -> ${shards} shard(s) of up to ${shard_size}"
test:
needs: prepare
runs-on: ubuntu-latest
timeout-minutes: 100
name: "TestMatrix ${{ matrix.shard }}"
strategy:
fail-fast: false
matrix:
shard: ${{ fromJson(needs.prepare.outputs.matrix-shards) }}
steps:
- name: Check out code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
# ./tool/go downloads and extracts the tsgo toolchain on a cold runner.
# go.toolchain.rev names an immutable release, so an exact hit is enough.
- name: Restore tsgo toolchain
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/.cache/tsgo
key: natlab-tsgo-${{ runner.os }}-${{ hashFiles('go.toolchain.rev') }}
- name: Restore Go modules
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/go/pkg/mod
key: natlab-gomod-${{ runner.os }}-${{ hashFiles('go.mod', 'go.sum') }}
# Saved at the end of the job. The run_id suffix makes each save unique,
# and restore-keys finds an earlier one.
- name: Restore Go build cache
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/.cache/go-build
key: natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-${{ github.run_id }}
restore-keys: |
natlab-gobuild-${{ runner.os }}-${{ hashFiles('go.sum') }}-
natlab-gobuild-${{ runner.os }}-
# The cloud VM image cache is keyed only on images.go (image URLs and
# SHAs), so it survives across workflow runs and is invalidated only
# when a new image source is added.
- name: Cache cloud VM images
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: ~/.cache/tailscale/vmtest/images
key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }}
# The gokrazy VM image is keyed by github.sha. That means we rebuild
# it once per commit but matrix test jobs in the same run all share
# the result. Per-PR re-runs of the same sha (e.g. a rerun-failed)
# also get the cache.
- name: Cache gokrazy VM image
id: gokrazy-cache
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
path: gokrazy/natlabapp.qcow2
key: natlab-gokrazy-${{ github.sha }}
# qemu-utils provides qemu-img, which the gokrazy Makefile uses to
# convert natlabapp.img to qcow2.
- name: Install qemu
run: |
sudo rm -f /var/lib/man-db/auto-update
sudo apt-get -y update
sudo apt-get -y remove man-db
sudo apt-get install -y qemu-system-x86 qemu-utils
- name: Download cloud VM images
# natlabprep is idempotent: it checks the cache before downloading.
run: |
./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep
# This also builds the minimal-feature gokrazy variants, which warms
# the Go build cache for their tagged tailscale and tailscaled.
- name: Build gokrazy VM image
if: steps.gokrazy-cache.outputs.cache-hit != 'true'
run: |
./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep --gokrazy
# No Go source imports this module, so nothing above fetches it. Pulling
# it here puts it in the module cache the matrix jobs restore.
- name: Download kernel.amd64 module
run: |
./tool/go mod download github.com/gokrazy/kernel.amd64
- name: Enable KVM
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- name: Run TestMatrix shard ${{ matrix.shard }}
run: |
./tool/go test -v -timeout=90m -count=1 ./tstest/natlab/vmtest/ \
-run '^TestMatrix$' --run-vm-tests -args -matrix-shard=${{ matrix.shard }}