Files
tailscale/kernel/linux/scripts/ec2.sh
T
Brad Fitzpatrick 95c00517eb kernel/linux/{scripts,DESIGN.md}: resize EC2 instances, and record 50 Gbps results
ec2.sh gets "resize TYPE N..." (for stopped instances) and "start N..."
(some instances), and numbers new instances after all existing ones,
stopped ones included.

With a c7i.48xlarge router (50 Gbps) and 8 clients, tswg forwards 31.5
Gbps at MTU 1280 (wireguard-go 7.2, kernel WireGuard 12.3 using 137 of
192 CPUs), limited by EC2's packet rate per NIC receive queue (about
715k pps, 7.3 Gbps of tunnel packets) with only 8 tunnel flows to spread
over 16 queues; and at line rate with a jumbo tunnel MTU, a data point
only, since MTU 1280 is the target.

Updates #426

Signed-off-by: Brad Fitzpatrick <[email protected]>
Change-Id: I70744651f3385343bf8890607911d9b4e0f70620
2026-11-07 19:16:21 +00:00

310 lines
11 KiB
Bash
Executable File

#!/bin/sh
# SPDX-License-Identifier: GPL-2.0
#
# ec2.sh creates and destroys EC2 instances for benchmarking tswg on real
# cloud VMs (and bare metal) rather than nested QEMU VMs, with tailscale.com's
# lean harness in remote mode (TSWG_REMOTE, see tstest/tswgbench).
#
# Everything it creates is named bradfitz-tswg... and tagged
# owner=claude-tswg: a key pair (from ~/.ssh/bradfitz-tswg, generated if
# missing), a security group allowing SSH and the harness's agent from this
# machine's public addresses and all traffic between its members, a cluster
# placement group, and the instances, which run Ubuntu 24.04, require IMDSv2
# (the sandbox org's SCP insists), and get an IPv6 address too.
#
# "up" writes the instances' addresses to $TSWG_EC2_HOSTS (default
# ~/.cache/tslinux/ec2/hosts.json), the file TSWG_REMOTE takes.
#
# ec2-idle-stop.sh (run from a systemd user timer) stops the instances once
# nothing has touched ~/.cache/tslinux/ec2/last-used for a while; every
# command here touches it, as does the lean harness while it runs. "start"
# brings stopped instances back (with new public addresses).
#
# Environment:
# AWS_PROFILE default tswg-sandbox
# AWS_REGION default us-west-2
# TSWG_EC2_SUBNET default subnet-c9385194 (the default VPC's IPv6 subnet in us-west-2c)
#
# usage:
# ec2.sh up [COUNT [TYPE]] launch COUNT (default 2) instances of TYPE (default c7i.xlarge)
# ec2.sh start [N...] start stopped instances (all, or those numbered N),
# and rewrite the hosts file
# ec2.sh resize TYPE N... change stopped instances N... to TYPE
# ec2.sh stop stop the instances (as ec2-idle-stop.sh does)
# ec2.sh hosts rewrite the hosts file from the running instances
# ec2.sh ssh N [CMD...] ssh to instance N (0-based)
# ec2.sh down terminate the instances and delete everything else
# ec2.sh status list our resources
set -eu
export AWS_PROFILE=${AWS_PROFILE:-tswg-sandbox}
export AWS_REGION=${AWS_REGION:-us-west-2}
subnet=${TSWG_EC2_SUBNET:-subnet-c9385194}
name=bradfitz-tswg
key=$HOME/.ssh/$name
hosts=${TSWG_EC2_HOSTS:-$HOME/.cache/tslinux/ec2/hosts.json}
owner='Key=owner,Value=claude-tswg'
marker=$(dirname "$hosts")/last-used
# Keep ec2-idle-stop.sh from stopping the instances while we use them.
mkdir -p "$(dirname "$marker")"
touch "$marker"
ec2() { aws ec2 "$@"; }
# tags NAME RESOURCE_TYPE prints a --tag-specifications value.
tags() { echo "ResourceType=$2,Tags=[{$owner},{Key=Name,Value=$1}]"; }
vpc_of_subnet() {
ec2 describe-subnets --subnet-ids "$subnet" --query 'Subnets[0].VpcId' --output text
}
sg_id() {
ec2 describe-security-groups --filters "Name=group-name,Values=$name" \
"Name=tag:owner,Values=claude-tswg" --query 'SecurityGroups[0].GroupId' --output text
}
instance_ids() {
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text
}
# ids_of N... prints the IDs of our (not terminated) instances numbered N.
ids_of() {
for n in "$@"; do
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=tag:Name,Values=$name-$n" \
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text
done | tr '\n' ' '
}
ensure_key() {
if [ ! -f "$key" ]; then
ssh-keygen -q -t ed25519 -N '' -C "$name" -f "$key"
fi
if ! ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 import-key-pair --key-name "$name" --public-key-material "fileb://$key.pub" \
--tag-specifications "$(tags "$name" key-pair)" > /dev/null
echo "imported key pair $name" >&2
fi
}
ensure_sg() {
sg=$(sg_id)
if [ "$sg" = "None" ] || [ -z "$sg" ]; then
sg=$(ec2 create-security-group --group-name "$name" --vpc-id "$(vpc_of_subnet)" \
--description "tswg benchmarks (claude-tswg)" \
--tag-specifications "$(tags "$name" security-group)" \
--query GroupId --output text)
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=-1,UserIdGroupPairs=[{GroupId=$sg}]" > /dev/null
echo "created security group $sg" >&2
fi
# Allow SSH and the agent from this machine, whatever its address is now.
my4=$(curl -s -m 5 https://checkip.amazonaws.com || true)
my6=$(curl -s -m 5 -6 https://api6.ipify.org || true)
for port in 22 8080; do
if [ -n "$my4" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,IpRanges=[{CidrIp=$my4/32}]" \
> /dev/null 2>&1 || true
fi
if [ -n "$my6" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,Ipv6Ranges=[{CidrIpv6=$my6/128}]" \
> /dev/null 2>&1 || true
fi
done
echo "$sg"
}
ensure_pg() {
if ! ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 create-placement-group --group-name "$name" --strategy cluster \
--tag-specifications "$(tags "$name" placement-group)" > /dev/null
echo "created placement group $name" >&2
fi
}
# wait_ready waits for SSH to every host in the hosts file, and for
# user-data to have installed the build tools.
wait_ready() {
for ip in $(python3 -c 'import json,sys; print(" ".join(h["public"] for h in json.load(open(sys.argv[1]))))' "$hosts"); do
n=0
until ssh -i "$key" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
-o ConnectTimeout=5 "ubuntu@$ip" test -f /var/lib/tswg-ready 2> /dev/null; do
n=$((n + 1))
if [ $n -gt 120 ]; then
echo "$ip: not ready after 10 minutes" >&2
exit 1
fi
sleep 5
done
echo "$ip: ready ($(ssh -i "$key" -o BatchMode=yes "ubuntu@$ip" uname -r))" >&2
done
}
# The instances install what's needed to build tswg.ko against their own
# kernel, with the compiler the kernel was built with.
userdata() {
cat <<'EOF'
#!/bin/sh
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update -q
apt-get install -qy build-essential gcc-13 gcc-14 nftables "linux-headers-$(uname -r)"
touch /var/lib/tswg-ready
EOF
}
write_hosts() {
mkdir -p "$(dirname "$hosts")"
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=running" \
--query 'Reservations[].Instances[].{name:Tags[?Key==`Name`]|[0].Value,id:InstanceId,type:InstanceType,public:PublicIpAddress,private4:PrivateIpAddress,ip6:Ipv6Address}' \
--output json | python3 -c '
import json, sys
hosts = sorted(json.load(sys.stdin), key=lambda h: int(h["name"].rsplit("-", 1)[1]))
for h in hosts:
h["user"] = "ubuntu"
h["key"] = sys.argv[1]
json.dump(hosts, sys.stdout, indent=2)
print()
' "$key" > "$hosts"
echo "wrote $hosts" >&2
cat "$hosts"
}
cmd=${1:-}
[ $# -gt 0 ] && shift
case $cmd in
up)
count=${1:-2}
type=${2:-c7i.xlarge}
ensure_key
sg=$(ensure_sg)
ensure_pg
ami=$(aws ssm get-parameter \
--name /aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id \
--query Parameter.Value --output text)
# Number new instances after all existing ones, stopped ones included.
existing=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
--query 'Reservations[].Instances[].Tags[?Key==`Name`].Value[]' --output text |
tr '\t' '\n' | sed -n "s/^$name-//p" | sort -n | tail -1)
existing=$((${existing:--1} + 1))
ids=""
i=$existing
while [ "$i" -lt $((existing + count)) ]; do
id=$(ec2 run-instances --image-id "$ami" --instance-type "$type" \
--key-name "$name" --subnet-id "$subnet" --security-group-ids "$sg" \
--placement "GroupName=$name" --ipv6-address-count 1 \
--associate-public-ip-address \
--metadata-options HttpTokens=required \
--block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=30,VolumeType=gp3,DeleteOnTermination=true}' \
--user-data "$(userdata)" \
--tag-specifications "$(tags "$name-$i" instance)" "$(tags "$name-$i" volume)" \
--query 'Instances[0].InstanceId' --output text)
echo "launched $name-$i ($type): $id" >&2
# Let it forward packets that aren't its own, as a subnet router does.
ec2 wait instance-exists --instance-ids "$id"
ec2 modify-instance-attribute --instance-id "$id" --no-source-dest-check
ids="$ids $id"
i=$((i + 1))
done
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
start)
if [ $# -gt 0 ]; then
ids=$(ids_of "$@")
else
ids=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text)
fi
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 wait instance-stopped --instance-ids $ids
# shellcheck disable=SC2086
ec2 start-instances --instance-ids $ids > /dev/null
echo "starting $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
fi
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
resize)
type=$1
shift
ids=$(ids_of "$@")
# shellcheck disable=SC2086
ec2 wait instance-stopped --instance-ids $ids
for id in $ids; do
ec2 modify-instance-attribute --instance-id "$id" --instance-type "Value=$type"
echo "$id is now $type" >&2
done
;;
stop)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 stop-instances --instance-ids $ids > /dev/null
echo "stopping $ids" >&2
fi
;;
hosts)
write_hosts
;;
ssh)
n=$1
shift
ip=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[int(sys.argv[2])]["public"])' "$hosts" "$n")
exec ssh -i "$key" -o StrictHostKeyChecking=accept-new "ubuntu@$ip" "$@"
;;
down)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 terminate-instances --instance-ids $ids > /dev/null
echo "terminating $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-terminated --instance-ids $ids
fi
sg=$(sg_id)
if [ "$sg" != "None" ] && [ -n "$sg" ]; then
ec2 delete-security-group --group-id "$sg"
echo "deleted security group $sg" >&2
fi
if ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 delete-placement-group --group-name "$name"
echo "deleted placement group $name" >&2
fi
if ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 delete-key-pair --key-name "$name"
echo "deleted key pair $name" >&2
fi
rm -f "$hosts"
;;
status)
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
--query 'Reservations[].Instances[].[Tags[?Key==`Name`]|[0].Value,InstanceId,InstanceType,State.Name,PublicIpAddress]' \
--output text
echo "security group: $(sg_id)"
ec2 describe-placement-groups --group-names "$name" --query 'PlacementGroups[0].GroupName' --output text 2> /dev/null || echo "no placement group"
ec2 describe-key-pairs --key-names "$name" --query 'KeyPairs[0].KeyName' --output text 2> /dev/null || echo "no key pair"
;;
*)
sed -n 's/^# \{0,1\}//; /^usage:/,/^$/p' "$0" >&2
exit 2
;;
esac