Files
tailscale/kernel/linux/scripts/ec2.sh
T
Brad Fitzpatrick df191a1199 kernel/linux/{scripts,DESIGN.md}: add the subnet router benchmark and its results
subnetbench.sh runs tailscale.com's TestLeanSubnetRouter on the EC2
instances for each router CPU count given. ec2.sh now turns off source
and destination checks on its instances, so that one can forward as a
subnet router does, and installs nftables (for the router's masquerade).

DESIGN.md records the results: with 4 clients, tswg as the subnet
router forwards 2.1x (4 CPUs) to 3.2x (8 CPUs) what wireguard-go does,
which uses about 3 CPUs however many it has, at the least CPU per byte
of any tunnel, and reaches the 25 Gbps link at 16 CPUs. Zero-copy
receive was tried there and was 7% worse, so isn't in.

Updates #426

Signed-off-by: Brad Fitzpatrick <[email protected]>
Change-Id: I6b1da8bb8eaa17f9b20cbe36896426411b8b905f
2026-11-07 18:02:37 +00:00

287 lines
10 KiB
Bash
Executable File

#!/bin/sh
# SPDX-License-Identifier: GPL-2.0
#
# ec2.sh creates and destroys EC2 instances for benchmarking tswg on real
# cloud VMs (and bare metal) rather than nested QEMU VMs, with tailscale.com's
# lean harness in remote mode (TSWG_REMOTE, see tstest/tswgbench).
#
# Everything it creates is named bradfitz-tswg... and tagged
# owner=claude-tswg: a key pair (from ~/.ssh/bradfitz-tswg, generated if
# missing), a security group allowing SSH and the harness's agent from this
# machine's public addresses and all traffic between its members, a cluster
# placement group, and the instances, which run Ubuntu 24.04, require IMDSv2
# (the sandbox org's SCP insists), and get an IPv6 address too.
#
# "up" writes the instances' addresses to $TSWG_EC2_HOSTS (default
# ~/.cache/tslinux/ec2/hosts.json), the file TSWG_REMOTE takes.
#
# ec2-idle-stop.sh (run from a systemd user timer) stops the instances once
# nothing has touched ~/.cache/tslinux/ec2/last-used for a while; every
# command here touches it, as does the lean harness while it runs. "start"
# brings stopped instances back (with new public addresses).
#
# Environment:
# AWS_PROFILE default tswg-sandbox
# AWS_REGION default us-west-2
# TSWG_EC2_SUBNET default subnet-c9385194 (the default VPC's IPv6 subnet in us-west-2c)
#
# usage:
# ec2.sh up [COUNT [TYPE]] launch COUNT (default 2) instances of TYPE (default c7i.xlarge)
# ec2.sh start start stopped instances, and rewrite the hosts file
# ec2.sh stop stop the instances (as ec2-idle-stop.sh does)
# ec2.sh hosts rewrite the hosts file from the running instances
# ec2.sh ssh N [CMD...] ssh to instance N (0-based)
# ec2.sh down terminate the instances and delete everything else
# ec2.sh status list our resources
set -eu
export AWS_PROFILE=${AWS_PROFILE:-tswg-sandbox}
export AWS_REGION=${AWS_REGION:-us-west-2}
subnet=${TSWG_EC2_SUBNET:-subnet-c9385194}
name=bradfitz-tswg
key=$HOME/.ssh/$name
hosts=${TSWG_EC2_HOSTS:-$HOME/.cache/tslinux/ec2/hosts.json}
owner='Key=owner,Value=claude-tswg'
marker=$(dirname "$hosts")/last-used
# Keep ec2-idle-stop.sh from stopping the instances while we use them.
mkdir -p "$(dirname "$marker")"
touch "$marker"
ec2() { aws ec2 "$@"; }
# tags NAME RESOURCE_TYPE prints a --tag-specifications value.
tags() { echo "ResourceType=$2,Tags=[{$owner},{Key=Name,Value=$1}]"; }
vpc_of_subnet() {
ec2 describe-subnets --subnet-ids "$subnet" --query 'Subnets[0].VpcId' --output text
}
sg_id() {
ec2 describe-security-groups --filters "Name=group-name,Values=$name" \
"Name=tag:owner,Values=claude-tswg" --query 'SecurityGroups[0].GroupId' --output text
}
instance_ids() {
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running,stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text
}
ensure_key() {
if [ ! -f "$key" ]; then
ssh-keygen -q -t ed25519 -N '' -C "$name" -f "$key"
fi
if ! ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 import-key-pair --key-name "$name" --public-key-material "fileb://$key.pub" \
--tag-specifications "$(tags "$name" key-pair)" > /dev/null
echo "imported key pair $name" >&2
fi
}
ensure_sg() {
sg=$(sg_id)
if [ "$sg" = "None" ] || [ -z "$sg" ]; then
sg=$(ec2 create-security-group --group-name "$name" --vpc-id "$(vpc_of_subnet)" \
--description "tswg benchmarks (claude-tswg)" \
--tag-specifications "$(tags "$name" security-group)" \
--query GroupId --output text)
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=-1,UserIdGroupPairs=[{GroupId=$sg}]" > /dev/null
echo "created security group $sg" >&2
fi
# Allow SSH and the agent from this machine, whatever its address is now.
my4=$(curl -s -m 5 https://checkip.amazonaws.com || true)
my6=$(curl -s -m 5 -6 https://api6.ipify.org || true)
for port in 22 8080; do
if [ -n "$my4" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,IpRanges=[{CidrIp=$my4/32}]" \
> /dev/null 2>&1 || true
fi
if [ -n "$my6" ]; then
ec2 authorize-security-group-ingress --group-id "$sg" \
--ip-permissions "IpProtocol=tcp,FromPort=$port,ToPort=$port,Ipv6Ranges=[{CidrIpv6=$my6/128}]" \
> /dev/null 2>&1 || true
fi
done
echo "$sg"
}
ensure_pg() {
if ! ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 create-placement-group --group-name "$name" --strategy cluster \
--tag-specifications "$(tags "$name" placement-group)" > /dev/null
echo "created placement group $name" >&2
fi
}
# wait_ready waits for SSH to every host in the hosts file, and for
# user-data to have installed the build tools.
wait_ready() {
for ip in $(python3 -c 'import json,sys; print(" ".join(h["public"] for h in json.load(open(sys.argv[1]))))' "$hosts"); do
n=0
until ssh -i "$key" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
-o ConnectTimeout=5 "ubuntu@$ip" test -f /var/lib/tswg-ready 2> /dev/null; do
n=$((n + 1))
if [ $n -gt 120 ]; then
echo "$ip: not ready after 10 minutes" >&2
exit 1
fi
sleep 5
done
echo "$ip: ready ($(ssh -i "$key" -o BatchMode=yes "ubuntu@$ip" uname -r))" >&2
done
}
# The instances install what's needed to build tswg.ko against their own
# kernel, with the compiler the kernel was built with.
userdata() {
cat <<'EOF'
#!/bin/sh
set -e
export DEBIAN_FRONTEND=noninteractive
apt-get update -q
apt-get install -qy build-essential gcc-13 gcc-14 nftables "linux-headers-$(uname -r)"
touch /var/lib/tswg-ready
EOF
}
write_hosts() {
mkdir -p "$(dirname "$hosts")"
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=running" \
--query 'Reservations[].Instances[].{name:Tags[?Key==`Name`]|[0].Value,id:InstanceId,type:InstanceType,public:PublicIpAddress,private4:PrivateIpAddress,ip6:Ipv6Address}' \
--output json | python3 -c '
import json, sys
hosts = sorted(json.load(sys.stdin), key=lambda h: h["name"])
for h in hosts:
h["user"] = "ubuntu"
h["key"] = sys.argv[1]
json.dump(hosts, sys.stdout, indent=2)
print()
' "$key" > "$hosts"
echo "wrote $hosts" >&2
cat "$hosts"
}
cmd=${1:-}
[ $# -gt 0 ] && shift
case $cmd in
up)
count=${1:-2}
type=${2:-c7i.xlarge}
ensure_key
sg=$(ensure_sg)
ensure_pg
ami=$(aws ssm get-parameter \
--name /aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id \
--query Parameter.Value --output text)
stopped=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=stopping,stopped" \
--query 'length(Reservations[].Instances[])' --output text)
if [ "$stopped" != 0 ]; then
echo "$stopped instances are stopped; use \"$0 start\" (or \"$0 down\" first)" >&2
exit 1
fi
existing=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=pending,running" \
--query 'length(Reservations[].Instances[])' --output text)
ids=""
i=$existing
while [ "$i" -lt $((existing + count)) ]; do
id=$(ec2 run-instances --image-id "$ami" --instance-type "$type" \
--key-name "$name" --subnet-id "$subnet" --security-group-ids "$sg" \
--placement "GroupName=$name" --ipv6-address-count 1 \
--associate-public-ip-address \
--metadata-options HttpTokens=required \
--block-device-mappings 'DeviceName=/dev/sda1,Ebs={VolumeSize=30,VolumeType=gp3,DeleteOnTermination=true}' \
--user-data "$(userdata)" \
--tag-specifications "$(tags "$name-$i" instance)" "$(tags "$name-$i" volume)" \
--query 'Instances[0].InstanceId' --output text)
echo "launched $name-$i ($type): $id" >&2
# Let it forward packets that aren't its own, as a subnet router does.
ec2 wait instance-exists --instance-ids "$id"
ec2 modify-instance-attribute --instance-id "$id" --no-source-dest-check
ids="$ids $id"
i=$((i + 1))
done
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
start)
ids=$(ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
"Name=instance-state-name,Values=stopping,stopped" \
--query 'Reservations[].Instances[].InstanceId' --output text)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 wait instance-stopped --instance-ids $ids
# shellcheck disable=SC2086
ec2 start-instances --instance-ids $ids > /dev/null
echo "starting $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-running --instance-ids $ids
fi
write_hosts > /dev/null
wait_ready
cat "$hosts"
;;
stop)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 stop-instances --instance-ids $ids > /dev/null
echo "stopping $ids" >&2
fi
;;
hosts)
write_hosts
;;
ssh)
n=$1
shift
ip=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[int(sys.argv[2])]["public"])' "$hosts" "$n")
exec ssh -i "$key" -o StrictHostKeyChecking=accept-new "ubuntu@$ip" "$@"
;;
down)
ids=$(instance_ids)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
ec2 terminate-instances --instance-ids $ids > /dev/null
echo "terminating $ids" >&2
# shellcheck disable=SC2086
ec2 wait instance-terminated --instance-ids $ids
fi
sg=$(sg_id)
if [ "$sg" != "None" ] && [ -n "$sg" ]; then
ec2 delete-security-group --group-id "$sg"
echo "deleted security group $sg" >&2
fi
if ec2 describe-placement-groups --group-names "$name" > /dev/null 2>&1; then
ec2 delete-placement-group --group-name "$name"
echo "deleted placement group $name" >&2
fi
if ec2 describe-key-pairs --key-names "$name" > /dev/null 2>&1; then
ec2 delete-key-pair --key-name "$name"
echo "deleted key pair $name" >&2
fi
rm -f "$hosts"
;;
status)
ec2 describe-instances --filters "Name=tag:owner,Values=claude-tswg" \
--query 'Reservations[].Instances[].[Tags[?Key==`Name`]|[0].Value,InstanceId,InstanceType,State.Name,PublicIpAddress]' \
--output text
echo "security group: $(sg_id)"
ec2 describe-placement-groups --group-names "$name" --query 'PlacementGroups[0].GroupName' --output text 2> /dev/null || echo "no placement group"
ec2 describe-key-pairs --key-names "$name" --query 'KeyPairs[0].KeyName' --output text 2> /dev/null || echo "no key pair"
;;
*)
sed -n 's/^# \{0,1\}//; /^usage:/,/^$/p' "$0" >&2
exit 2
;;
esac