Files
tailscale/kernel/linux/scripts/ec2.sh
T
Brad Fitzpatrick 25d5ffacd7 kernel/linux/scripts: fix debug kernel caching, and sort EC2 hosts numerically
gokrazy-debug-kernel.sh hashed sha256sum's output for its cache key,
which includes the config's random mktemp path, so every run built
another 3.5GB debug kernel; seven of them filled the disk. Hash the
files' contents instead.

ec2.sh sorted hosts by name, which put bradfitz-tswg-10 before
bradfitz-tswg-2; sort by number.

Updates #426

Signed-off-by: Brad Fitzpatrick <[email protected]>
Change-Id: Ia758dae0077c54a2ef8f6112ae3c43d6604f66e1
2026-11-07 18:51:40 +00:00

287 lines
10 KiB
Bash
Executable File

#!/bin/sh
# SPDX-License-Identifier: GPL-2.0
#
# ec2.sh creates and destroys EC2 instances for benchmarking tswg on real
# cloud VMs (and bare metal) rather than nested QEMU VMs, with tailscale.com's
# lean harness in remote mode (TSWG_REMOTE, see tstest/tswgbench).
#
# Everything it creates is named bradfitz-tswg... and tagged
# owner=claude-tswg: a key pair (from ~/.ssh/bradfitz-tswg, generated if
# missing), a security group allowing SSH and the harness's agent from this
# machine's public addresses and all traffic between its members, a cluster
# placement group, and the instances, which run Ubuntu 24.04, require IMDSv2
# (the sandbox org's SCP insists), and get an IPv6 address too.
#
# "up" writes the instances' addresses to $TSWG_EC2_HOSTS (default
# ~/.cache/tslinux/ec2/hosts.json), the file TSWG_REMOTE takes.
#
# ec2-idle-stop.sh (run from a systemd user timer) stops the instances once
# nothing has touched ~/.cache/tslinux/ec2/last-used for a while; every
# command here touches it, as does the lean harness while it runs. "start"
# brings stopped instances back (with new public addresses).
#
# Environment:
# AWS_PROFILE default tswg-sandbox
# AWS_REGION default us-west-2
# TSWG_EC2_SUBNET default subnet-c9385194 (the default VPC's IPv6 subnet in us-west-2c)
#
# usage:
# ec2.sh up [COUNT [TYPE]] launch COUNT (default 2) instances of TYPE (default c7i.xlarge)
# ec2.sh start start stopped instances, and rewrite the hosts file
# ec2.sh stop stop the instances (as ec2-idle-stop.sh does)
# ec2.sh hosts rewrite the hosts file from the running instances
# ec2.sh ssh N [CMD...] ssh to instance N (0-based)
# ec2.sh down terminate the instances and delete everything else
# ec2.sh status list our resources
set -eu
export AWS_PROFILE=${AWS_PROFILE:-tswg-sandbox}
export AWS_REGION=${AWS_REGION:-us-west-2}
subnet=${TSWG_EC2_SUBNET:-subnet-c9385194}
name=bradfitz-tswg
key=$HOME/.ssh/$name
hosts=${TSWG_EC2_HOSTS:-$HOME/.cache/tslinux/ec2/hosts.json}
owner='Key=owner,Value=claude-tswg'
marker=$(dirname "$hosts")/last-used
# Keep ec2-idle-stop.sh from stopping the instances while we use them.
mkdir -p "$(dirname "$marker")"
touch "$marker"
ec2() { aws ec2 "$@"; }
# tags NAME RESOURCE_TYPE prints a --tag-specifications value.
tags() { echo "ResourceType=$2,Tags=[{$owner},{Key=Name,Value=$1}]"; }
vpc_of_subnet() {
ec2 describe-subnets --subnet-ids "$subnet" --query 'Subnets[0].VpcId' --output text
}
sg_id() {
ec2 describe-security-groups --filters "Name=group-name,Values=$name" \
"Name=tag:owner,Values=claude-tswg" --query 'SecurityGroups[0].GroupId' --output text
}
instance_ids() {
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text
}
ensure_key() {
if [ ! -f "$key" ]; then
ssh-keygen -q -t ed25519 -N '' -C "$name" -f "$key"
fi
if ! ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 import-key-pair --key-name "$name" --public-key-material "fileb://$key.pub" \
--tag-specifications "$(tags "$name" key-pair)" > /dev/null
echo "imported key pair $name" >&2
fi
}
ensure_sg() {
sg=$(sg_id)
if [ "$sg" = "None" ] || [ -z "$sg" ]; then
sg=$(ec2 create-security-group --group-name "$name" --vpc-id "$(vpc_of_subnet)" \
--description "tswg benchmarks (claude-tswg)" \
--tag-specifications "$(tags "$name" security-group)" \
--query GroupId --output text)
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=-1,UserIdGroupPairs=[{GroupId=$sg}]" > /dev/null
echo "created security group $sg" >&2
fi
# Allow SSH and the agent from this machine, whatever its address is now.
my4=$(curl -s -m 5 https://checkip.amazonaws.com || true)
my6=$(curl -s -m 5 -6 https://api6.ipify.org || true)
for port in 22 8080; do
if [ -n "$my4" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,IpRanges=[{CidrIp=$my4/32}]" \
> /dev/null 2>&1 || true
fi
if [ -n "$my6" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,Ipv6Ranges=[{CidrIpv6=$my6/128}]" \
> /dev/null 2>&1 || true
fi
done
echo "$sg"
}
ensure_pg() {
if ! ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 create-placement-group --group-name "$name" --strategy cluster \
--tag-specifications "$(tags "$name" placement-group)" > /dev/null
echo "created placement group $name" >&2
fi
}
# wait_ready waits for SSH to every host in the hosts file, and for
# user-data to have installed the build tools.
wait_ready() {
for ip in $(python3 -c 'import json,sys; print(" ".join(h["public"] for h in json.load(open(sys.argv[1]))))' "$hosts"); do
n=0
until ssh -i "$key" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
-o ConnectTimeout=5 "ubuntu@$ip" test -f /var/lib/tswg-ready 2> /dev/null; do
n=$((n + 1))
if [ $n -gt 120 ]; then
echo "$ip: not ready after 10 minutes" >&2
exit 1
fi
sleep 5
done
echo "$ip: ready ($(ssh -i "$key" -o BatchMode=yes "ubuntu@$ip" uname -r))" >&2
done
}
# The instances install what's needed to build tswg.ko against their own
# kernel, with the compiler the kernel was built with.
userdata() {
cat <<'EOF'
#!/bin/sh
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update -q
apt-get install -qy build-essential gcc-13 gcc-14 nftables "linux-headers-$(uname -r)"
touch /var/lib/tswg-ready
EOF
}
write_hosts() {
mkdir -p "$(dirname "$hosts")"
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=running" \
--query 'Reservations[].Instances[].{name:Tags[?Key==`Name`]|[0].Value,id:InstanceId,type:InstanceType,public:PublicIpAddress,private4:PrivateIpAddress,ip6:Ipv6Address}' \
--output json | python3 -c '
import json, sys
hosts = sorted(json.load(sys.stdin), key=lambda h: int(h["name"].rsplit("-", 1)[1]))
for h in hosts:
h["user"] = "ubuntu"
h["key"] = sys.argv[1]
json.dump(hosts, sys.stdout, indent=2)
print()
' "$key" > "$hosts"
echo "wrote $hosts" >&2
cat "$hosts"
}
cmd=${1:-}
[ $# -gt 0 ] && shift
case $cmd in
up)
count=${1:-2}
type=${2:-c7i.xlarge}
ensure_key
sg=$(ensure_sg)
ensure_pg
ami=$(aws ssm get-parameter \
--name /aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id \
--query Parameter.Value --output text)
stopped=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=stopping,stopped" \
--query 'length(Reservations[].Instances[])' --output text)
if [ "$stopped" != 0 ]; then
echo "$stopped instances are stopped; use \"$0 start\" (or \"$0 down\" first)" >&2
exit 1
fi
existing=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running" \
--query 'length(Reservations[].Instances[])' --output text)
ids=""
i=$existing
while [ "$i" -lt $((existing + count)) ]; do
id=$(ec2 run-instances --image-id "$ami" --instance-type "$type" \
--key-name "$name" --subnet-id "$subnet" --security-group-ids "$sg" \
--placement "GroupName=$name" --ipv6-address-count 1 \
--associate-public-ip-address \
--metadata-options HttpTokens=required \
--block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=30,VolumeType=gp3,DeleteOnTermination=true}' \
--user-data "$(userdata)" \
--tag-specifications "$(tags "$name-$i" instance)" "$(tags "$name-$i" volume)" \
--query 'Instances[0].InstanceId' --output text)
echo "launched $name-$i ($type): $id" >&2
# Let it forward packets that aren't its own, as a subnet router does.
ec2 wait instance-exists --instance-ids "$id"
ec2 modify-instance-attribute --instance-id "$id" --no-source-dest-check
ids="$ids $id"
i=$((i + 1))
done
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
start)
ids=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 wait instance-stopped --instance-ids $ids
# shellcheck disable=SC2086
ec2 start-instances --instance-ids $ids > /dev/null
echo "starting $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
fi
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
stop)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 stop-instances --instance-ids $ids > /dev/null
echo "stopping $ids" >&2
fi
;;
hosts)
write_hosts
;;
ssh)
n=$1
shift
ip=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[int(sys.argv[2])]["public"])' "$hosts" "$n")
exec ssh -i "$key" -o StrictHostKeyChecking=accept-new "ubuntu@$ip" "$@"
;;
down)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 terminate-instances --instance-ids $ids > /dev/null
echo "terminating $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-terminated --instance-ids $ids
fi
sg=$(sg_id)
if [ "$sg" != "None" ] && [ -n "$sg" ]; then
ec2 delete-security-group --group-id "$sg"
echo "deleted security group $sg" >&2
fi
if ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 delete-placement-group --group-name "$name"
echo "deleted placement group $name" >&2
fi
if ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 delete-key-pair --key-name "$name"
echo "deleted key pair $name" >&2
fi
rm -f "$hosts"
;;
status)
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
--query 'Reservations[].Instances[].[Tags[?Key==`Name`]|[0].Value,InstanceId,InstanceType,State.Name,PublicIpAddress]' \
--output text
echo "security group: $(sg_id)"
ec2 describe-placement-groups --group-names "$name" --query 'PlacementGroups[0].GroupName' --output text 2> /dev/null || echo "no placement group"
ec2 describe-key-pairs --key-names "$name" --query 'KeyPairs[0].KeyName' --output text 2> /dev/null || echo "no key pair"
;;
*)
sed -n 's/^# \{0,1\}//; /^usage:/,/^$/p' "$0" >&2
exit 2
;;
esac