mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-20 00:31:17 -08:00
tstest/natlab/vmtest, gokrazy, cmd/tailscale/cli: add minimal-build vmtests
Add vmtests that boot pairs of gokrazy VMs whose tailscale and tailscaled are minimal builds and check that IP traffic flows between them over WireGuard, so we notice if a minimal build stops doing its one job: build_dist.sh --extra-small (direct path through NATs), that minus NAT traversal (DERP across NATs, direct on a shared LAN), and DERP-only. They found that "tailscale up" built with ts_omit_ipnbus returned before calling Start or EditPrefs, making it a no-op. Fix that, and have it wait for the Running state by polling tailscaled's status instead of watching the IPN bus, so scripts can rely on it as with normal builds. The extra-small feature list moves to featuretags.ExtraSmall so build_dist.sh (via a new "featuretags --extra-small") and vmtest share it. The gokrazy builder gains --output and --go-build-tags flags to build the variant images, and natlabprep --gokrazy prebuilds them all for CI. Updates #13038 Updates #12614 Signed-off-by: Brad Fitzpatrick <[email protected]> Change-Id: I0caa2c7b3adf2399da09b6d07a942f1dc34e8486
This commit is contained in:
1 parent
3e2dca47c5
commit
654c105f8b
13 files changed
+463
-79
No files matched your search
@@ -110,10 +110,12 @@ jobs:
|
||||
run: |
|
||||
./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep
|
||||
|
||||
- name: Build gokrazy VM image
|
||||
# This also builds the minimal-feature gokrazy variants, which warms
|
||||
# the Go build cache for their tagged tailscale and tailscaled.
|
||||
- name: Build gokrazy VM images
|
||||
if: steps.gokrazy-cache.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
make -C gokrazy natlab
|
||||
./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep --gokrazy
|
||||
|
||||
# No Go source imports this module, so nothing above fetches it. Pulling
|
||||
# it here puts it in the module cache the matrix jobs restore.
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ while [ "$#" -gt 1 ]; do
|
||||
fi
|
||||
shift
|
||||
ldflags="$ldflags -w -s"
|
||||
tags="${tags:+$tags,},$(GOOS= GOARCH= $go run ./cmd/featuretags --min --add=osrouter,nattraversal)"
|
||||
tags="${tags:+$tags,},$(GOOS= GOARCH= $go run ./cmd/featuretags --extra-small)"
|
||||
;;
|
||||
--min)
|
||||
# --min is like --extra-small but even smaller, removing all features,
|
||||
|
||||
@@ -22,11 +22,29 @@ var (
|
||||
remove = flag.String("remove", "", "a comma-separated list of features to remove from the build. (without the 'ts_omit_' prefix)")
|
||||
add = flag.String("add", "", "a comma-separated list of features or tags to add, if --min is used.")
|
||||
list = flag.Bool("list", false, "if true, list all known features and what they do")
|
||||
|
||||
extraSmall = flag.Bool("extra-small", false, "shorthand for build_dist.sh --extra-small's tags: --min --add="+joinFeatures(featuretags.ExtraSmall))
|
||||
)
|
||||
|
||||
func joinFeatures(fts []featuretags.FeatureTag) string {
|
||||
var ss []string
|
||||
for _, ft := range fts {
|
||||
ss = append(ss, string(ft))
|
||||
}
|
||||
return strings.Join(ss, ",")
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if *extraSmall {
|
||||
if *min || *add != "" {
|
||||
log.Fatalf("--extra-small can't be combined with --min or --add")
|
||||
}
|
||||
*min = true
|
||||
*add = joinFeatures(featuretags.ExtraSmall)
|
||||
}
|
||||
|
||||
features := featuretags.Features
|
||||
|
||||
if *list {
|
||||
|
||||
+47
-10
@@ -608,21 +608,36 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
|
||||
}
|
||||
}()
|
||||
|
||||
if !buildfeatures.HasIPNBus {
|
||||
fmt.Fprintln(Stderr, "binary built with ts_omit_ipnbus; not waiting for completion")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start watching the IPN bus before we call Start() or StartLoginInteractive(),
|
||||
// or we could miss IPN notifications.
|
||||
//
|
||||
// In particular, if we're doing a force-reauth, we could miss the
|
||||
// notification with the auth URL we should print for the user.
|
||||
watcher, err := localClient.WatchIPNBus(watchCtx, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
//
|
||||
// Binaries built without the IPN bus instead poll tailscaled's status
|
||||
// once the loop below starts reading.
|
||||
var nextNotify func() (ipn.Notify, error)
|
||||
if buildfeatures.HasIPNBus {
|
||||
watcher, err := localClient.WatchIPNBus(watchCtx, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer watcher.Close()
|
||||
nextNotify = watcher.Next
|
||||
} else {
|
||||
var polled bool
|
||||
nextNotify = func() (ipn.Notify, error) {
|
||||
if polled {
|
||||
select {
|
||||
case <-watchCtx.Done():
|
||||
return ipn.Notify{}, watchCtx.Err()
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
polled = true
|
||||
return statusNotify(watchCtx)
|
||||
}
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
// Special case: bare "tailscale up" means to just start
|
||||
// running, if there's ever been a login.
|
||||
@@ -728,7 +743,7 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
|
||||
}
|
||||
|
||||
for {
|
||||
n, err := watcher.Next()
|
||||
n, err := nextNotify()
|
||||
if err != nil {
|
||||
watchErr <- err
|
||||
return
|
||||
@@ -833,6 +848,28 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE
|
||||
}
|
||||
}
|
||||
|
||||
// statusNotify returns an IPN notification synthesized from tailscaled's
|
||||
// current status, with the fields that runUp's wait loop reads: the
|
||||
// backend state, the auth URL, and the node key. It's how "tailscale up"
|
||||
// waits in binaries built without the IPN bus.
|
||||
func statusNotify(ctx context.Context) (ipn.Notify, error) {
|
||||
st, err := localClient.StatusWithoutPeers(ctx)
|
||||
if err != nil {
|
||||
return ipn.Notify{}, err
|
||||
}
|
||||
var n ipn.Notify
|
||||
if state, ok := ipn.StateFromString(st.BackendState); ok {
|
||||
n.State = &state
|
||||
}
|
||||
if st.AuthURL != "" {
|
||||
n.BrowseToURL = &st.AuthURL
|
||||
}
|
||||
if st.Self != nil {
|
||||
n.SelfChange = &tailcfg.Node{Key: st.Self.PublicKey}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func printDeviceApprovalInfo(printJson bool, prefs *ipn.Prefs, lastURLPrinted *string) {
|
||||
if printJson {
|
||||
printUpDoneJSON(ipn.NeedsMachineAuth, "")
|
||||
|
||||
@@ -66,6 +66,11 @@ func RequiredBy(ft FeatureTag) set.Set[FeatureTag] {
|
||||
return s
|
||||
}
|
||||
|
||||
// ExtraSmall is the set of features kept by build_dist.sh --extra-small
|
||||
// (via cmd/featuretags --extra-small), the smallest build that's still a
|
||||
// useful tailscaled. Pass it to [MinTags] to get its build tags.
|
||||
var ExtraSmall = []FeatureTag{"osrouter", "nattraversal"}
|
||||
|
||||
// MinTags returns the sorted Go build tags for a minimal build that
|
||||
// includes only the features in keep and the features they require.
|
||||
// Every other omittable feature in [Features] gets its ts_omit_ tag, and
|
||||
|
||||
@@ -24,6 +24,15 @@ natlab:
|
||||
../tool/go run build.go --build --app=natlabapp
|
||||
qemu-img convert -O qcow2 natlabapp.img natlabapp.qcow2
|
||||
|
||||
# Like natlab, but with tailscale and tailscaled built with the Go build
|
||||
# tags in $(TAGS), writing natlabapp-$(NAME).qcow2. tstest/natlab/vmtest
|
||||
# uses this for its minimal-feature gokrazy images (e.g. GokrazyExtraSmall).
|
||||
natlab-tagged:
|
||||
../tool/go run build.go --build --app=natlabapp --output=natlabapp-$(NAME) \
|
||||
--go-build-tags=tailscale.com/cmd/tailscale=$(TAGS) \
|
||||
--go-build-tags=tailscale.com/cmd/tailscaled=$(TAGS)
|
||||
qemu-img convert -O qcow2 natlabapp-$(NAME).img natlabapp-$(NAME).qcow2
|
||||
|
||||
# For natlab integration tests on macOS arm64:
|
||||
natlab-arm64:
|
||||
../tool/go run build.go --build --app=natlabapp.arm64
|
||||
|
||||
@@ -15,9 +15,11 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"tailscale.com/gokrazy/build"
|
||||
)
|
||||
@@ -29,8 +31,22 @@ var (
|
||||
gaf = flag.Bool("gaf", false, "if true, build a gokrazy archive format file instead of a full disk image")
|
||||
jsonOut = flag.Bool("json", false, "emit one machine-readable JSON result line to stdout")
|
||||
region = flag.String("region", "", "AWS region for import+register; default us-east-1 (honors $AWS_REGION)")
|
||||
output = flag.String("output", "", "base name of the .img or .gaf file to write; default is --app")
|
||||
|
||||
goBuildTags = map[string][]string{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.Func("go-build-tags", "PKG=TAG1,TAG2 to build Go package PKG with the given build tags, overriding config.json; may be repeated", func(v string) error {
|
||||
pkg, tags, ok := strings.Cut(v, "=")
|
||||
if !ok || pkg == "" {
|
||||
return errors.New("want PKG=TAG1,TAG2")
|
||||
}
|
||||
goBuildTags[pkg] = strings.Split(tags, ",")
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
@@ -57,6 +73,9 @@ func run(ctx context.Context) (build.Result, error) {
|
||||
App: *app,
|
||||
Bucket: *bucket,
|
||||
Region: build.ResolveRegion(*region, os.Getenv("AWS_REGION")),
|
||||
|
||||
Output: *output,
|
||||
GoBuildTags: goBuildTags,
|
||||
})
|
||||
if err != nil {
|
||||
return build.Result{App: *app, Error: err.Error()}, err
|
||||
|
||||
+77
-5
@@ -17,6 +17,7 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -80,6 +81,15 @@ type Config struct {
|
||||
// in. Empty means ResolveRegion("", $AWS_REGION).
|
||||
Region string
|
||||
|
||||
// Output is the base name (without extension) of the .img or .gaf
|
||||
// file written into Dir. Empty means App.
|
||||
Output string
|
||||
|
||||
// GoBuildTags, if non-empty, maps Go package import paths to the Go
|
||||
// build tags to build them with, replacing any GoBuildTags that the
|
||||
// app's config.json sets for those packages.
|
||||
GoBuildTags map[string][]string
|
||||
|
||||
// Logf receives human-readable progress. If nil, log.Printf is used.
|
||||
Logf logger.Logf
|
||||
// Stderr receives the output of subprocesses (monogok). If nil,
|
||||
@@ -306,15 +316,23 @@ func (b *Builder) buildImage(ctx context.Context, gaf bool) error {
|
||||
}
|
||||
|
||||
args := []string{"run", "github.com/bradfitz/monogok/cmd/monogok"}
|
||||
if len(b.GoBuildTags) > 0 {
|
||||
confPath, err := b.writeTaggedConfig(appDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(confPath)
|
||||
args = append(args, "--config", confPath)
|
||||
}
|
||||
if gaf {
|
||||
args = append(args,
|
||||
"overwrite",
|
||||
"--gaf", filepath.Join(b.Dir, b.App+".gaf"),
|
||||
"--gaf", b.outPath(".gaf"),
|
||||
)
|
||||
} else {
|
||||
args = append(args,
|
||||
"overwrite",
|
||||
"--full", filepath.Join(b.Dir, b.App+".img"),
|
||||
"--full", b.outPath(".img"),
|
||||
fmt.Sprintf("--target_storage_bytes=%d", imageSizeBytesFor(b.App)),
|
||||
)
|
||||
}
|
||||
@@ -327,11 +345,11 @@ func (b *Builder) buildImage(ctx context.Context, gaf bool) error {
|
||||
return err
|
||||
}
|
||||
if gaf {
|
||||
b.res.GAF = filepath.Join(b.Dir, b.App+".gaf")
|
||||
b.res.GAF = b.outPath(".gaf")
|
||||
return nil
|
||||
}
|
||||
|
||||
imgPath := filepath.Join(b.Dir, b.App+".img")
|
||||
imgPath := b.outPath(".img")
|
||||
f, err := os.OpenFile(imgPath, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open %s: %w", imgPath, err)
|
||||
@@ -345,6 +363,60 @@ func (b *Builder) buildImage(ctx context.Context, gaf bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// outPath returns the path in Dir of the build output with the given
|
||||
// extension (".img" or ".gaf").
|
||||
func (b *Builder) outPath(ext string) string {
|
||||
return filepath.Join(b.Dir, cmp.Or(b.Output, b.App)+ext)
|
||||
}
|
||||
|
||||
// writeTaggedConfig writes a copy of appDir's config.json with b.GoBuildTags
|
||||
// applied and returns its path, which the caller must remove. The copy is
|
||||
// written into appDir because monogok finds the module root by walking up
|
||||
// from the config file's directory.
|
||||
func (b *Builder) writeTaggedConfig(appDir string) (string, error) {
|
||||
confJSON, err := os.ReadFile(filepath.Join(appDir, "config.json"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Decode into generic maps so fields this package doesn't know about
|
||||
// are preserved.
|
||||
var conf map[string]any
|
||||
if err := json.Unmarshal(confJSON, &conf); err != nil {
|
||||
return "", fmt.Errorf("unmarshaling config.json: %w", err)
|
||||
}
|
||||
pkgConfs, _ := conf["PackageConfig"].(map[string]any)
|
||||
if pkgConfs == nil {
|
||||
pkgConfs = map[string]any{}
|
||||
conf["PackageConfig"] = pkgConfs
|
||||
}
|
||||
for pkg, tags := range b.GoBuildTags {
|
||||
pc, _ := pkgConfs[pkg].(map[string]any)
|
||||
if pc == nil {
|
||||
pc = map[string]any{}
|
||||
pkgConfs[pkg] = pc
|
||||
}
|
||||
pc["GoBuildTags"] = tags
|
||||
}
|
||||
out, err := json.MarshalIndent(conf, "", " ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := os.CreateTemp(appDir, ".config-*.json")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := f.Write(out); err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
return "", err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(f.Name())
|
||||
return "", err
|
||||
}
|
||||
return f.Name(), nil
|
||||
}
|
||||
|
||||
// UploadToS3 uploads the built image to s3://<Bucket>/<App>.img (a
|
||||
// concurrent multipart upload, progress on b.Stderr) and returns the URI
|
||||
// (also in Result.S3). Requires BuildImage first.
|
||||
@@ -356,7 +428,7 @@ func (b *Builder) UploadToS3(ctx context.Context) (s3URI string, err error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
imgPath := filepath.Join(b.Dir, b.App+".img")
|
||||
imgPath := b.res.Image
|
||||
f, err := os.Open(imgPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("open %s: %w", imgPath, err)
|
||||
|
||||
@@ -2,18 +2,23 @@
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// The natlabprep tool warms the local natlab vmtest cache by downloading
|
||||
// every cloud VM image natlab can boot. It is intended for CI prep steps
|
||||
// so a subsequent test run does not pay the per-image download cost.
|
||||
// every cloud VM image natlab can boot and, with --gokrazy, building every
|
||||
// gokrazy image. It is intended for CI prep steps so a subsequent test run
|
||||
// does not pay the per-image download or build cost.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log"
|
||||
|
||||
"tailscale.com/tstest/natlab/vmtest"
|
||||
)
|
||||
|
||||
var gokrazy = flag.Bool("gokrazy", false, "also build the gokrazy images (requires make and qemu-img)")
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
ctx := context.Background()
|
||||
for _, img := range vmtest.CloudImages() {
|
||||
log.Printf("ensuring %s ...", img.Name)
|
||||
@@ -21,4 +26,11 @@ func main() {
|
||||
log.Fatalf("ensuring %s: %v", img.Name, err)
|
||||
}
|
||||
}
|
||||
if *gokrazy {
|
||||
for _, img := range vmtest.GokrazyImages() {
|
||||
if err := vmtest.BuildGokrazyImage(ctx, img); err != nil {
|
||||
log.Fatalf("building %s: %v", img.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
"tailscale.com/feature/featuretags"
|
||||
)
|
||||
|
||||
// LinuxFamily classifies a Linux distro by the conventions that affect how we
|
||||
@@ -45,6 +46,23 @@ type OSImage struct {
|
||||
Family LinuxFamily // Linux distro family (affects cloud-init user-data); empty means Debian-like
|
||||
IsGokrazy bool // true for gokrazy images (different QEMU setup)
|
||||
IsMacOS bool // true for macOS images (launched via tailmac, not QEMU)
|
||||
|
||||
// gokrazyKeep, if non-nil, makes a gokrazy image a minimal build: its
|
||||
// tailscale and tailscaled are built with [featuretags.MinTags] of
|
||||
// these features.
|
||||
gokrazyKeep []featuretags.FeatureTag
|
||||
}
|
||||
|
||||
// gokrazyMakeArgs returns the arguments to make in the gokrazy directory
|
||||
// that build the gokrazy image img, and the name of the qcow2 file that
|
||||
// writes there.
|
||||
func (img OSImage) gokrazyMakeArgs() (args []string, qcow2 string) {
|
||||
if img.gokrazyKeep == nil {
|
||||
return []string{"natlab"}, "natlabapp.qcow2"
|
||||
}
|
||||
name := strings.TrimPrefix(img.Name, "gokrazy-")
|
||||
tags := strings.Join(featuretags.MinTags(img.gokrazyKeep...), ",")
|
||||
return []string{"natlab-tagged", "NAME=" + name, "TAGS=" + tags}, "natlabapp-" + name + ".qcow2"
|
||||
}
|
||||
|
||||
// GOOS returns the Go OS name for this image.
|
||||
@@ -85,6 +103,35 @@ var (
|
||||
MemoryMB: 384,
|
||||
}
|
||||
|
||||
// GokrazyExtraSmall is like [Gokrazy], but its tailscale and tailscaled
|
||||
// are built with the same feature-omitting build tags as
|
||||
// "build_dist.sh --extra-small". The tta test agent is built normally.
|
||||
GokrazyExtraSmall = OSImage{
|
||||
Name: "gokrazy-extrasmall",
|
||||
IsGokrazy: true,
|
||||
MemoryMB: 384,
|
||||
gokrazyKeep: featuretags.ExtraSmall,
|
||||
}
|
||||
|
||||
// GokrazyNoNATTraversal is like [GokrazyExtraSmall] but also without
|
||||
// NAT traversal (STUN, hole punching), so it only gets a direct path to
|
||||
// peers whose endpoints are directly reachable.
|
||||
GokrazyNoNATTraversal = OSImage{
|
||||
Name: "gokrazy-nonat",
|
||||
IsGokrazy: true,
|
||||
MemoryMB: 384,
|
||||
gokrazyKeep: []featuretags.FeatureTag{"osrouter", "udptransport"},
|
||||
}
|
||||
|
||||
// GokrazyDERPOnly is like [GokrazyExtraSmall] but also without UDP
|
||||
// transport, so all peer traffic goes over DERP.
|
||||
GokrazyDERPOnly = OSImage{
|
||||
Name: "gokrazy-derponly",
|
||||
IsGokrazy: true,
|
||||
MemoryMB: 384,
|
||||
gokrazyKeep: []featuretags.FeatureTag{"osrouter"},
|
||||
}
|
||||
|
||||
// Ubuntu2404 is Ubuntu 24.04 LTS (Noble Numbat) cloud image.
|
||||
Ubuntu2404 = OSImage{
|
||||
Name: "ubuntu-24.04",
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package vmtest_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/tstest/natlab/vmtest"
|
||||
"tailscale.com/tstest/natlab/vnet"
|
||||
)
|
||||
|
||||
// TestExtraSmall verifies that two gokrazy nodes running tailscale and
|
||||
// tailscaled built like "build_dist.sh --extra-small" (nearly every optional
|
||||
// feature omitted) can still send IP packets to each other over WireGuard,
|
||||
// and, since NAT traversal is kept, find a direct path through easy NATs.
|
||||
func TestExtraSmall(t *testing.T) {
|
||||
testMinimalBuild(t, vmtest.GokrazyExtraSmall, false, vmtest.PingRouteDirect)
|
||||
}
|
||||
|
||||
// TestNoNATTraversal is like [TestExtraSmall] but also without NAT
|
||||
// traversal. Peers behind separate NATs can't hole punch, so they must
|
||||
// stay on DERP.
|
||||
func TestNoNATTraversal(t *testing.T) {
|
||||
testMinimalBuild(t, vmtest.GokrazyNoNATTraversal, false, vmtest.PingRouteDERP)
|
||||
}
|
||||
|
||||
// TestNoNATTraversalSameLAN is like [TestNoNATTraversal], but with both
|
||||
// peers on one LAN, where their advertised local endpoints are directly
|
||||
// reachable without NAT traversal.
|
||||
func TestNoNATTraversalSameLAN(t *testing.T) {
|
||||
testMinimalBuild(t, vmtest.GokrazyNoNATTraversal, true, vmtest.PingRouteLocal)
|
||||
}
|
||||
|
||||
// TestDERPOnly is like [TestExtraSmall] but also without UDP transport, so
|
||||
// all traffic must go over DERP.
|
||||
func TestDERPOnly(t *testing.T) {
|
||||
testMinimalBuild(t, vmtest.GokrazyDERPOnly, false, vmtest.PingRouteDERP)
|
||||
}
|
||||
|
||||
// testMinimalBuild starts two nodes running img, either on one LAN or
|
||||
// behind separate easy NATs, and verifies that IP traffic flows between them
|
||||
// over WireGuard and that their disco pings settle on wantRoute.
|
||||
func testMinimalBuild(t *testing.T, img vmtest.OSImage, sameLAN bool, wantRoute vmtest.PingRoute) {
|
||||
env := vmtest.New(t)
|
||||
|
||||
aNet := env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT)
|
||||
bNet := aNet
|
||||
if !sameLAN {
|
||||
bNet = env.AddNetwork("2.2.2.2", "192.168.2.1/24", vnet.EasyNAT)
|
||||
}
|
||||
a := env.AddNode("a", aNet, vmtest.OS(img), vmtest.WebServer(8080))
|
||||
b := env.AddNode("b", bNet, vmtest.OS(img), vmtest.WebServer(8080))
|
||||
|
||||
env.Start()
|
||||
|
||||
if err := env.Ping(a, b, tailcfg.PingTSMP, 60*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Fetch each node's web server from the other over its Tailscale IP.
|
||||
// The web server echoes the client's source address, which must be the
|
||||
// client's Tailscale IP, proving the TCP connection went through the
|
||||
// TUN device and WireGuard.
|
||||
aIP := env.Status(a).Self.TailscaleIPs[0]
|
||||
bIP := env.Status(b).Self.TailscaleIPs[0]
|
||||
for _, tc := range []struct {
|
||||
from *vmtest.Node
|
||||
fromIP string
|
||||
to *vmtest.Node
|
||||
toIPPort string
|
||||
}{
|
||||
{a, aIP.String(), b, bIP.String() + ":8080"},
|
||||
{b, bIP.String(), a, aIP.String() + ":8080"},
|
||||
} {
|
||||
body := env.HTTPGet(tc.from, "http://"+tc.toIPPort+"/")
|
||||
want := fmt.Sprintf("Hello world I am %s from %s", tc.to.Name(), tc.fromIP)
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("HTTP GET from %s to %s: got %q; want %q", tc.from.Name(), tc.toIPPort, body, want)
|
||||
}
|
||||
}
|
||||
|
||||
if wantRoute != vmtest.PingRouteDERP {
|
||||
if err := env.PingExpect(a, b, wantRoute, 60*time.Second); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Seeing a DERP route once proves little, as every connection starts
|
||||
// on DERP, so give the nodes time to find a direct path and verify they
|
||||
// didn't.
|
||||
pr, err := env.PingSettle(a, b, 20*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pr.Endpoint != "" {
|
||||
t.Errorf("ping went via endpoint %v; want DERP only", pr.Endpoint)
|
||||
}
|
||||
}
|
||||
@@ -58,26 +58,21 @@ func hardwareAccelAvailable() bool {
|
||||
type gokrazyPlatform struct{}
|
||||
|
||||
func (gokrazyPlatform) planSteps(e *Env, n *Node) {
|
||||
e.Step("Build gokrazy image")
|
||||
e.Step(fmt.Sprintf("Build %s image", n.os.Name))
|
||||
e.Step("Launch QEMU: " + n.name)
|
||||
}
|
||||
|
||||
func (gokrazyPlatform) boot(ctx context.Context, e *Env, n *Node) error {
|
||||
e.gokrazyOnce.Do(func() {
|
||||
step := e.Step("Build gokrazy image")
|
||||
step.Begin()
|
||||
if err := e.ensureGokrazy(ctx); err != nil {
|
||||
step.End(err)
|
||||
e.t.Fatalf("ensureGokrazy: %v", err)
|
||||
}
|
||||
step.End(nil)
|
||||
})
|
||||
base, err := e.ensureGokrazy(ctx, n.os)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensureGokrazy(%s): %w", n.os.Name, err)
|
||||
}
|
||||
|
||||
e.ensureQEMUSocket()
|
||||
|
||||
vmStep := e.Step("Launch QEMU: " + n.name)
|
||||
vmStep.Begin()
|
||||
if err := e.startGokrazyQEMU(n); err != nil {
|
||||
if err := e.startGokrazyQEMU(n, base); err != nil {
|
||||
vmStep.End(err)
|
||||
return err
|
||||
}
|
||||
@@ -115,11 +110,12 @@ func (qemuCloudPlatform) boot(ctx context.Context, e *Env, n *Node) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// startGokrazyQEMU launches a QEMU process for a gokrazy node.
|
||||
// startGokrazyQEMU launches a QEMU process for a gokrazy node whose disk is
|
||||
// an overlay on the base qcow2 image at basePath.
|
||||
// This follows the same pattern as tstest/integration/nat/nat_test.go.
|
||||
func (e *Env) startGokrazyQEMU(n *Node) error {
|
||||
func (e *Env) startGokrazyQEMU(n *Node, basePath string) error {
|
||||
disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name))
|
||||
if err := createOverlay(e.gokrazyBase, disk); err != nil {
|
||||
if err := createOverlay(basePath, disk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+109
-45
@@ -18,6 +18,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
@@ -47,6 +48,7 @@ import (
|
||||
"tailscale.com/tstest/natlab/vnet"
|
||||
"tailscale.com/types/key"
|
||||
"tailscale.com/util/mak"
|
||||
"tailscale.com/util/set"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -74,9 +76,9 @@ type Env struct {
|
||||
// are downloaded from pkgs.tailscale.com instead of compiled from the tree.
|
||||
testVersion string
|
||||
|
||||
// gokrazy-specific paths
|
||||
gokrazyBase string // path to gokrazy base qcow2 image
|
||||
gokrazyKernel string // path to gokrazy kernel
|
||||
// gokrazyKernel is the path to the gokrazy kernel. It's set by
|
||||
// ensureGokrazy with gokrazyBuildMu held.
|
||||
gokrazyKernel string
|
||||
|
||||
// tailmac-specific paths (macOS VMs)
|
||||
tailmacDir string // path to tailmac bin/ directory containing Host.app
|
||||
@@ -96,12 +98,11 @@ type Env struct {
|
||||
|
||||
// Shared resource initialization (sync.Once for things multiple nodes share).
|
||||
vnetOnce sync.Once
|
||||
gokrazyOnce sync.Once
|
||||
qemuSockOnce sync.Once
|
||||
dgramSockOnce sync.Once
|
||||
compileMu sync.Mutex
|
||||
compileOnce map[string]*sync.Once // keyed by goos_goarch
|
||||
imageOnce map[string]*sync.Once // keyed by OSImage.Name
|
||||
compileOnce map[string]*sync.Once // keyed by goos_goarch
|
||||
imageOnce map[string]func() error // keyed by OSImage.Name (cloud and gokrazy images); see prepareImageOnce
|
||||
|
||||
// Web UI support.
|
||||
ctx context.Context // cancelled when test ends
|
||||
@@ -1956,22 +1957,30 @@ func (e *Env) ensureCompiled(ctx context.Context, goos, goarch string) {
|
||||
// ensureImage prepares the cloud image for os and returns any error from the
|
||||
// preparation. Safe for concurrent use; only prepares once per OS name.
|
||||
func (e *Env) ensureImage(ctx context.Context, os OSImage) error {
|
||||
return e.prepareImageOnce(os.Name, fmt.Sprintf("Prepare %s image", os.Name), func() error {
|
||||
return ensureImage(ctx, os)
|
||||
})
|
||||
}
|
||||
|
||||
// prepareImageOnce runs prepare as the web UI step stepName the first time
|
||||
// it's called for the image named imgName. Every call for that image,
|
||||
// concurrent or later, returns the error from that one run, so no node
|
||||
// boots from an image whose preparation failed.
|
||||
func (e *Env) prepareImageOnce(imgName, stepName string, prepare func() error) error {
|
||||
e.compileMu.Lock()
|
||||
once, ok := e.imageOnce[os.Name]
|
||||
f, ok := e.imageOnce[imgName]
|
||||
if !ok {
|
||||
once = new(sync.Once)
|
||||
mak.Set(&e.imageOnce, os.Name, once)
|
||||
f = sync.OnceValue(func() error {
|
||||
step := e.Step(stepName)
|
||||
step.Begin()
|
||||
err := prepare()
|
||||
step.End(err)
|
||||
return err
|
||||
})
|
||||
mak.Set(&e.imageOnce, imgName, f)
|
||||
}
|
||||
e.compileMu.Unlock()
|
||||
|
||||
var err error
|
||||
once.Do(func() {
|
||||
step := e.Step(fmt.Sprintf("Prepare %s image", os.Name))
|
||||
step.Begin()
|
||||
err = ensureImage(ctx, os)
|
||||
step.End(err)
|
||||
})
|
||||
return err
|
||||
return f()
|
||||
}
|
||||
|
||||
// registerBinaries registers compiled binaries with the vnet file server.
|
||||
@@ -2119,43 +2128,98 @@ func (e *Env) RecvTaildropFile(ctx context.Context, n *Node) (name string, conte
|
||||
return name, body
|
||||
}
|
||||
|
||||
var buildGokrazy sync.Once
|
||||
var (
|
||||
// gokrazyBuildMu serializes gokrazy image builds, which share the
|
||||
// gokrazy directory, and guards gokrazyBuilt.
|
||||
gokrazyBuildMu sync.Mutex
|
||||
|
||||
// ensureGokrazy builds the gokrazy base image (once per test process) and
|
||||
// locates the kernel. The build is fast (~4s) so we always rebuild to ensure
|
||||
// the baked-in binaries (tta, tailscale, tailscaled) match the current source.
|
||||
func (e *Env) ensureGokrazy(ctx context.Context) error {
|
||||
if e.gokrazyBase != "" {
|
||||
return nil // already found
|
||||
// gokrazyBuilt is the set of gokrazy image names that this process
|
||||
// has already built successfully.
|
||||
gokrazyBuilt set.Set[string]
|
||||
)
|
||||
|
||||
// GokrazyImages returns the gokrazy OS images that natlab builds from
|
||||
// source: [Gokrazy] and its minimal-feature variants. It is intended for
|
||||
// tooling such as a CI prep step that wants to prebuild them.
|
||||
func GokrazyImages() []OSImage {
|
||||
return []OSImage{Gokrazy, GokrazyExtraSmall, GokrazyNoNATTraversal, GokrazyDERPOnly}
|
||||
}
|
||||
|
||||
// BuildGokrazyImage builds the gokrazy image img from the current source
|
||||
// tree, writing its qcow2 file into the gokrazy directory, unless this
|
||||
// process has already built it. Build output goes to os.Stdout and
|
||||
// os.Stderr. It is safe for concurrent use.
|
||||
func BuildGokrazyImage(ctx context.Context, img OSImage) error {
|
||||
gokrazyBuildMu.Lock()
|
||||
defer gokrazyBuildMu.Unlock()
|
||||
return buildGokrazyImageLocked(ctx, img)
|
||||
}
|
||||
|
||||
func buildGokrazyImageLocked(ctx context.Context, img OSImage) error {
|
||||
if !img.IsGokrazy {
|
||||
return fmt.Errorf("%s is not a gokrazy image", img.Name)
|
||||
}
|
||||
if gokrazyBuilt.Contains(img.Name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
modRoot, err := findModRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var buildErr error
|
||||
buildGokrazy.Do(func() {
|
||||
e.t.Logf("building gokrazy natlab image...")
|
||||
cmd := exec.CommandContext(ctx, "make", "natlab")
|
||||
cmd.Dir = filepath.Join(modRoot, "gokrazy")
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
if err := cmd.Run(); err != nil {
|
||||
buildErr = fmt.Errorf("make natlab: %w", err)
|
||||
}
|
||||
})
|
||||
if buildErr != nil {
|
||||
return buildErr
|
||||
args, _ := img.gokrazyMakeArgs()
|
||||
if img.gokrazyKeep != nil {
|
||||
log.Printf("building %s image (keeping only features %v)...", img.Name, img.gokrazyKeep)
|
||||
} else {
|
||||
log.Printf("building %s image...", img.Name)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "make", args...)
|
||||
cmd.Dir = filepath.Join(modRoot, "gokrazy")
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("make %s: %w", args[0], err)
|
||||
}
|
||||
gokrazyBuilt.Make()
|
||||
gokrazyBuilt.Add(img.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
e.gokrazyBase = filepath.Join(modRoot, "gokrazy/natlabapp.qcow2")
|
||||
|
||||
kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod"))
|
||||
// ensureGokrazy builds the gokrazy image img (once per test process) and
|
||||
// locates the kernel. It returns the path to img's base qcow2 image.
|
||||
// The build is fast (~4s) so we always rebuild to ensure the baked-in
|
||||
// binaries (tta, tailscale, tailscaled) match the current source.
|
||||
// It is safe for concurrent use.
|
||||
func (e *Env) ensureGokrazy(ctx context.Context, img OSImage) (basePath string, err error) {
|
||||
modRoot, err := findModRoot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("finding kernel: %w", err)
|
||||
return "", err
|
||||
}
|
||||
if err := e.prepareImageOnce(img.Name, fmt.Sprintf("Build %s image", img.Name), func() error {
|
||||
return e.buildGokrazy(ctx, modRoot, img)
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, qcow2 := img.gokrazyMakeArgs()
|
||||
return filepath.Join(modRoot, "gokrazy", qcow2), nil
|
||||
}
|
||||
|
||||
// buildGokrazy builds the gokrazy image img, unless this process already
|
||||
// has, and sets e.gokrazyKernel.
|
||||
func (e *Env) buildGokrazy(ctx context.Context, modRoot string, img OSImage) error {
|
||||
gokrazyBuildMu.Lock()
|
||||
defer gokrazyBuildMu.Unlock()
|
||||
|
||||
if err := buildGokrazyImageLocked(ctx, img); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if e.gokrazyKernel == "" {
|
||||
kernel, err := findKernelPath(filepath.Join(modRoot, "go.mod"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("finding kernel: %w", err)
|
||||
}
|
||||
e.gokrazyKernel = kernel
|
||||
}
|
||||
e.gokrazyKernel = kernel
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user