diff --git a/.github/workflows/bumpdep.yml b/.github/workflows/bumpdep.yml index 9d6365ed1..b3ab7f106 100644 --- a/.github/workflows/bumpdep.yml +++ b/.github/workflows/bumpdep.yml @@ -7,9 +7,10 @@ on: deps: description: >- Comma-separated dependencies to bump. Special names: "go" (runs - ./pull-toolchain.sh), "wireguard-go" (github.com/tailscale/wireguard-go@tailscale), - "gvisor" (gvisor.dev/gvisor@go). Anything else is a Go module path - bumped to @latest, or to the version given in a "path@version" suffix. + ./pull-toolchain.sh), "wireguard-go", "gvisor". Anything else is a + Go module path, or a case-insensitive substring of one ("kernel" + bumps every gokrazy kernel), bumped to @latest, or to the version + given in a "path@version" suffix. See misc/bumpdeps. required: true issue: description: >- @@ -17,6 +18,12 @@ on: https://github.com/tailscale/tailscale/issues/12345. It becomes the "Updates #12345" line of the commit message. required: true + exclude-newer-than-days: + description: >- + Cooldown: ignore releases younger than this many days and use the + newest older one instead. 0 means take the latest. + required: false + default: "0" permissions: contents: read @@ -35,171 +42,16 @@ jobs: - name: Check out code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Validate issue URL - env: - ISSUE: ${{ inputs.issue }} - run: | - set -euo pipefail - issue=$(printf '%s' "$ISSUE" | tr -d '[:space:]') - if ! printf '%s\n' "$issue" | grep -Eq '^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/issues/[0-9]+$'; then - echo "::error::issue must be a GitHub issue URL like https://github.com/tailscale/tailscale/issues/12345; got '$ISSUE'" >&2 - exit 1 - fi - # Convert to the short form used in commit messages: "#123" for - # this repo, "owner/repo#123" for others. - path="${issue#https://github.com/}" - repo="${path%/issues/*}" - num="${path##*/}" - if [ "$repo" = "$GITHUB_REPOSITORY" ]; then - ref="#$num" - else - ref="$repo#$num" - fi - echo "ISSUE_REF=$ref" >>"$GITHUB_ENV" - + # misc/bumpdeps does the work: it validates the inputs, resolves + # versions, runs go get, make tidy, and make updatedeps, and writes + # the PR title, branch, and commit message to $GITHUB_OUTPUT. - name: Bump dependencies id: bump env: DEPS: ${{ inputs.deps }} - TRIGGERING_ACTOR: ${{ github.triggering_actor }} - run: | - set -euo pipefail - - # Build gocross if needed. - ./tool/go version - - # modver prints the current version of the given module, or nothing - # if it's not yet in go.mod. - modver() { - ./tool/go list -m -f '{{.Version}}' "$1" 2>/dev/null || true - } - - # goget bumps module $1 to version $2, appending a summary line - # to the PR body that links $3 (a GitHub repo URL) if given. - # It fails if go get downgraded anything: a downgrade means - # something went wrong and needs a human's rollback PR, not an - # unattended bump. - # - # It uses the default GOPROXY (proxy.golang.org). Callers that - # need the very latest commit of a tracked branch can set - # GOPROXY=direct, but only for repos that are cheap to clone; a - # direct fetch of something like github.com/gokrazy/kernel.amd64 - # clones gigabytes of kernel blobs and never finishes. - goget() { - local mod="$1" ver="$2" repo="${3:-}" - local before after log="$RUNNER_TEMP/go-get.log" - before=$(modver "$mod") - ./tool/go get "$mod@$ver" 2>&1 | tee "$log" - if grep -q '^go: downgraded ' "$log"; then - echo "::error::refusing to downgrade modules while bumping $mod@$ver:" >&2 - grep '^go: downgraded ' "$log" >&2 - exit 1 - fi - after=$(modver "$mod") - if [ "$before" = "$after" ]; then - echo "* $mod: already at $after" >>"$BODY" - elif [ -n "$repo" ] && [ -n "$before" ]; then - # Link a compare view; for pseudo-versions, the trailing - # component is the commit hash. - echo "* $mod: $repo/compare/${before##*-}...${after##*-}" >>"$BODY" - else - echo "* $mod: ${before:-(new)} to $after" >>"$BODY" - fi - } - - BODY="$RUNNER_TEMP/pr-body.md" - : >"$BODY" - - names=() # display names, for the title - slugs=() # cleaned input names, for the branch name - touch_gomod=0 - touch_toolchain=0 - - IFS=',' read -ra deps <<<"$DEPS" - for dep in "${deps[@]}"; do - dep="${dep#"${dep%%[![:space:]]*}"}" # strip leading whitespace - dep="${dep%"${dep##*[![:space:]]}"}" # strip trailing whitespace - [ -z "$dep" ] && continue - slugs+=("$dep") - case "$dep" in - go) - before=$(cat go.toolchain.rev) - ./pull-toolchain.sh - after=$(cat go.toolchain.rev) - if [ "$before" != "$after" ]; then - echo "* Go toolchain: https://github.com/tailscale/go/compare/${before}...${after}" >>"$BODY" - else - echo "* Go toolchain: already at ${after}" >>"$BODY" - fi - names+=("Go toolchain") - touch_toolchain=1 - ;; - wireguard-go) - # Our own small repo: fetch directly so a just-pushed - # commit isn't hidden behind the proxy's cache. - GOPROXY=direct goget github.com/tailscale/wireguard-go tailscale https://github.com/tailscale/wireguard-go - names+=("wireguard-go") - touch_gomod=1 - ;; - gvisor) - goget gvisor.dev/gvisor go https://github.com/google/gvisor - names+=("gvisor") - touch_gomod=1 - ;; - *) - mod="${dep%%@*}" - ver=latest - case "$dep" in - *@*) ver="${dep#*@}" ;; - esac - goget "$mod" "$ver" - names+=("$mod") - touch_gomod=1 - ;; - esac - done - - if [ "${#names[@]}" -eq 0 ]; then - echo "::error::no dependencies given" >&2 - exit 1 - fi - - prefix="" - [ "$touch_gomod" = 1 ] && prefix="go.mod" - if [ "$touch_toolchain" = 1 ]; then - prefix="${prefix:+$prefix,}go.toolchain.rev" - fi - title="$prefix: bump $(printf '%s, ' "${names[@]}" | sed 's/, $//')" - - # Branch name: actions/bumpdep/, restricted to characters - # that are safe in a git ref and bounded in length. - slug=$(IFS=,; printf '%s' "${slugs[*]}" | tr -c 'A-Za-z0-9._-' '-' | sed -E 's/\.+/./g; s/^[.-]+//' | cut -c1-80 | sed -E 's/[.-]+$//') - - echo "" >>"$BODY" - echo "Triggered by @${TRIGGERING_ACTOR} via the bumpdep workflow." >>"$BODY" - echo "" >>"$BODY" - echo "Updates $ISSUE_REF" >>"$BODY" - - { - echo "title=$title" - echo "branch=actions/bumpdep/$slug" - echo "commit-message<>"$GITHUB_OUTPUT" - - echo "body-path=$BODY" >>"$GITHUB_OUTPUT" - - echo "### $title" >>"$GITHUB_STEP_SUMMARY" - cat "$BODY" >>"$GITHUB_STEP_SUMMARY" - - - name: make tidy - run: make tidy - - - name: make updatedeps - run: make updatedeps + ISSUE: ${{ inputs.issue }} + COOLDOWN: ${{ inputs.exclude-newer-than-days }} + run: ./tool/go run ./misc/bumpdeps -github -issue "$ISSUE" -exclude-newer-than-days "${COOLDOWN:-0}" -- "$DEPS" - name: Show changes run: git --no-pager diff --stat diff --git a/.github/workflows/gokrazy-bump.yml b/.github/workflows/gokrazy-bump.yml new file mode 100644 index 000000000..ef24cfaa4 --- /dev/null +++ b/.github/workflows/gokrazy-bump.yml @@ -0,0 +1,74 @@ +name: gokrazy-bump + +# Weekly bump of the gokrazy modules (kernels, firmware, gokrazy itself) +# that the natlab VM tests boot. The PR carries the run-natlab-tests +# label so natlab-test.yml exercises the new kernels before merge. + +on: + schedule: + # Sunday night (Monday 05:17 UTC), off-the-hour to avoid GitHub's :00 + # cron-stampede window. + - cron: "17 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + gokrazy-bump: + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The substring matches every direct github.com/gokrazy/* module. + # See misc/bumpdeps for what the tool does and the outputs it sets. + - name: Bump gokrazy modules + id: bump + run: ./tool/go run ./misc/bumpdeps -github -issue https://github.com/tailscale/tailscale/issues/1866 -- github.com/gokrazy/ + + - name: Show changes + run: git --no-pager diff --stat + + - name: Get access token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + id: generate-token + with: + # Get token for app: https://github.com/apps/tailscale-code-updater + app-id: ${{ secrets.CODE_UPDATER_APP_ID }} + private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} + # Limit the token to only pushing a branch and opening a pull request. + permission-contents: write + permission-pull-requests: write + + # Nothing happens if go.mod didn't change: create-pull-request + # opens a PR only when there's a diff to commit. + - name: Send pull request + id: pull-request + uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 + with: + token: ${{ steps.generate-token.outputs.token }} + author: Dep Updater + committer: Dep Updater + branch: ${{ steps.bump.outputs.branch }} + commit-message: ${{ steps.bump.outputs.commit-message }} + title: ${{ steps.bump.outputs.title }} + body-path: ${{ steps.bump.outputs.body-path }} + labels: run-natlab-tests + signoff: true + delete-branch: true + # For scheduled runs this is whoever last edited this file. + reviewers: ${{ github.triggering_actor }} + + - name: Summary + if: ${{ steps.pull-request.outputs.pull-request-number }} + env: + PR_OPERATION: ${{ steps.pull-request.outputs.pull-request-operation }} + PR_URL: ${{ steps.pull-request.outputs.pull-request-url }} + run: echo "$PR_OPERATION $PR_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/misc/bumpdeps/bumpdeps.go b/misc/bumpdeps/bumpdeps.go index 85dd82ecd..346544535 100644 --- a/misc/bumpdeps/bumpdeps.go +++ b/misc/bumpdeps/bumpdeps.go @@ -1,18 +1,44 @@ // Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -// The bumpdeps program updates every direct dependency in go.mod to its -// latest version, or just the ones named on the command line. +// The bumpdeps program updates dependencies in go.mod (and, on request, +// the Go toolchain), tidies up afterwards, and writes a commit message +// describing what changed. With no arguments it bumps every direct +// dependency to its latest version. // -// It parses go.mod, asks the Go module proxy concurrently for the newest -// version of each required module, and then runs a single "go get" with -// the modules that have something newer. A few modules follow a branch -// rather than tagged releases; see specialBranches. +// It is also the brain of the bumpdep and gokrazy-bump GitHub workflows +// (.github/workflows/bumpdep.yml and gokrazy-bump.yml), which run it with +// -github and turn its outputs into a pull request. // -// Arguments, if any, are case-insensitive substrings of module paths: -// "bumpdeps gvisor wireguard" only considers modules whose path contains -// one of those. Explicitly named modules are considered even if they're -// only indirect dependencies. +// Each argument selects something to bump. Arguments may also be +// comma-separated, which is how the workflow passes them through. +// +// - "path@version" bumps (or adds) exactly that module at that +// version. The version is anything go get accepts: a tag, "latest", +// a branch name, or a commit hash. Only "latest" goes through the +// safety checks below; other versions are handed to go get as is. +// - "go" updates the Go toolchain by running ./pull-toolchain.sh. +// - "wireguard-go" and "gvisor" are aliases for the modules of the same +// name; see aliases. +// - An exact module path already in go.mod selects that module. +// - Anything else is a case-insensitive substring of module paths: +// "bumpdeps kernel" considers both gokrazy kernel modules. Indirect +// dependencies only match with -indirect; name them exactly to +// bump one on its own. +// - A module path that isn't in go.mod and matches nothing is added at +// its latest version. +// +// Versions come from the Go module proxy, asked concurrently. A few +// modules follow a branch rather than tagged releases; see +// specialBranches. Their head is found with git ls-remote so a +// just-pushed commit is picked up even if the proxy hasn't seen it yet, +// and the proxy then supplies the pseudo-version for that commit. +// Everything selected is bumped with a single "go get", followed by +// "make tidy" and "make updatedeps" unless -tidy=false. +// +// The program refuses to downgrade anything. A downgrade means something +// went wrong (a retracted release, a stray tag) and wants a human's +// rollback commit, not an unattended bump. // // The --exclude-newer-than-days flag is a cooldown, as described at // https://nesbitt.io/2026/03/04/package-managers-need-to-cool-down.html: @@ -24,10 +50,11 @@ // so they're held until their head is old enough. // // Indirect dependencies are only updated as far as the direct ones pull -// them, unless -indirect is set. Bumping them individually to @latest -// tends to break the build, since their importers haven't necessarily -// caught up. (github.com/gobwas/glob v1.0.0 removed packages that -// github.com/goreleaser/fileglob still imports, for instance.) +// them, unless -indirect is set or they're named explicitly. Bumping them +// individually to @latest tends to break the build, since their importers +// haven't necessarily caught up. (github.com/gobwas/glob v1.0.0 removed +// packages that github.com/goreleaser/fileglob still imports, for +// instance.) // // Modules with a replace directive are left alone, as are modules the proxy // doesn't know about (such as private modules) and modules whose latest @@ -35,18 +62,21 @@ // // # Running // -// From the repo root, run: `./tool/go run ./misc/bumpdeps` and then -// `make tidy && make updatedeps`. +// From the repo root: +// +// ./tool/go run ./misc/bumpdeps # all direct deps +// ./tool/go run ./misc/bumpdeps gvisor wireguard-go # just those two +// ./tool/go run ./misc/bumpdeps -issue https://github.com/tailscale/tailscale/issues/123 go +// +// The last form prints a commit message with an "Updates #123" line. package main import ( "cmp" "context" - "encoding/json" "errors" "flag" "fmt" - "io" "log" "net/http" "os" @@ -58,33 +88,52 @@ import ( "text/tabwriter" "time" - "golang.org/x/mod/modfile" "golang.org/x/mod/module" - "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" ) -// specialBranches maps module paths to the branch they should track -// instead of the proxy's notion of @latest. -var specialBranches = map[string]string{ - "gvisor.dev/gvisor": "go", // upstream convention for the Go-module-friendly branch - "github.com/tailscale/wireguard-go": "tailscale", // our fork's main branch - "github.com/tailscale/golang-x-crypto": "main", // our fork has stray upstream-style tags; see tempfork/acme +// branchInfo says which branch of which git repo a module tracks. +type branchInfo struct { + repo string // https URL of the git repo, for git ls-remote and compare links + branch string } +// specialBranches maps module paths to the branch they should track +// instead of the proxy's notion of @latest. +var specialBranches = map[string]branchInfo{ + // Upstream's convention for the Go-module-friendly branch. + "gvisor.dev/gvisor": {"https://github.com/google/gvisor", "go"}, + // Our fork's main branch. + "github.com/tailscale/wireguard-go": {"https://github.com/tailscale/wireguard-go", "tailscale"}, + // Our fork has stray upstream-style tags; see tempfork/acme. + "github.com/tailscale/golang-x-crypto": {"https://github.com/tailscale/golang-x-crypto", "main"}, +} + +// aliases maps short names accepted on the command line to module paths. +var aliases = map[string]string{ + "gvisor": "gvisor.dev/gvisor", + "wireguard-go": "github.com/tailscale/wireguard-go", +} + +// toolchainRepo is the git repo that go.toolchain.rev points into. +const toolchainRepo = "https://github.com/tailscale/go" + var ( - dryRun = flag.Bool("n", false, "print what would be updated without running go get") + dryRun = flag.Bool("n", false, "print what would be updated without changing anything") goBin = flag.String("go", "", "path to the go binary to run; defaults to ./tool/go if present, else go from $PATH") proxyURL = flag.String("proxy", "https://proxy.golang.org", "base URL of the Go module proxy to query") parallel = flag.Int("j", 32, "maximum number of concurrent proxy requests") indirect = flag.Bool("indirect", false, "also update modules marked // indirect; risky, since their importers may not build against newer versions") excludeNewerThanDays = flag.Int("exclude-newer-than-days", 0, "ignore versions younger than this many days and use the newest older one instead; 0 means no cooldown") + tidy = flag.Bool("tidy", true, "run \"make tidy\" and \"make updatedeps\" after updating") + issue = flag.String("issue", "", "GitHub issue motivating the bump, as a URL or #123 or owner/repo#123; becomes the \"Updates\" line of the commit message; required with -github") + github = flag.Bool("github", false, "run as the bumpdep GitHub workflow: read the actor and repo from the environment and write the title, branch, and commit message to $GITHUB_OUTPUT and $GITHUB_STEP_SUMMARY") ) func main() { log.SetFlags(0) flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "usage: bumpdeps [flags] [module-path-substring ...]\n") + fmt.Fprintf(flag.CommandLine.Output(), "usage: bumpdeps [flags] [dep ...]\n") flag.PrintDefaults() } flag.Parse() @@ -93,136 +142,193 @@ func main() { } } -// update describes one module that has a newer version available. +// update describes one module that has a newer version available, or +// that was asked for at an explicit version. type update struct { Path string - Current string - Latest string + Current string // version in go.mod before, or "" if it's being added + Latest string // version to bump to; after go get, the version go.mod ended up with } -// resolver looks up versions on the module proxy. -type resolver struct { - client *http.Client - base string // proxy URL without a trailing slash - cutoff time.Time // versions committed after this are too new; zero means no cooldown -} +func run(ctx context.Context, args []string) error { + var env *githubEnv + if *github { + var err error + if env, err = githubEnvFromOS(); err != nil { + return err + } + if *issue == "" { + return errors.New("-issue is required with -github") + } + } + var issueRef string + if *issue != "" { + repo := "tailscale/tailscale" + if env != nil { + repo = env.repo + } + var err error + if issueRef, err = parseIssueRef(*issue, repo); err != nil { + return err + } + } -func run(ctx context.Context, filters []string) error { - data, err := os.ReadFile("go.mod") + before, err := readGoMod() if err != nil { return err } - mf, err := modfile.Parse("go.mod", data, nil) + sel, err := parseArgs(before, args, *indirect) if err != nil { return err } - replaced := map[string]bool{} - for _, r := range mf.Replace { - replaced[r.Old.Path] = true - } r := &resolver{ - client: &http.Client{Timeout: 30 * time.Second}, - base: strings.TrimSuffix(*proxyURL, "/"), + client: &http.Client{Timeout: 2 * time.Minute}, + base: strings.TrimSuffix(*proxyURL, "/"), + lsRemote: gitLsRemote, } if *excludeNewerThanDays > 0 { r.cutoff = time.Now().Add(-time.Duration(*excludeNewerThanDays) * 24 * time.Hour) } + res, err := resolveAll(ctx, r, sel.lookups) + if err != nil { + return err + } + updates := append(res.updates, sel.explicit...) + slices.SortFunc(updates, func(a, b update) int { return cmp.Compare(a.Path, b.Path) }) + switch { + case len(updates) > 0: + tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "MODULE\tCURRENT\tLATEST") + for _, u := range updates { + fmt.Fprintf(tw, "%s\t%s\t%s\n", u.Path, cmp.Or(u.Current, "(new)"), u.Latest) + } + tw.Flush() + case len(res.held) > 0: + fmt.Printf("nothing to update; %d module(s) held by the %d-day cooldown\n", len(res.held), *excludeNewerThanDays) + case !sel.toolchain: + fmt.Println("all dependencies are up to date") + } + if *dryRun { + if sel.toolchain { + fmt.Println("would also run ./pull-toolchain.sh") + } + return res.err() + } + + if len(updates) > 0 { + if err := goGet(ctx, updates); err != nil { + return err + } + after, err := readGoMod() + if err != nil { + return err + } + if err := checkDowngrades(before, after); err != nil { + return err + } + for i := range updates { + if v, ok := after.versions[updates[i].Path]; ok { + updates[i].Latest = v + } + } + } + + var tc *toolchainBump + if sel.toolchain { + if tc, err = bumpToolchain(ctx); err != nil { + return err + } + } + + if *tidy && (len(updates) > 0 || tc.changed()) { + for _, target := range []string{"tidy", "updatedeps"} { + if err := runCommand(ctx, "make", target); err != nil { + return err + } + } + } + + rep := buildReport(reportInput{ + updated: updates, + unchanged: res.upToDate, + held: res.held, + requested: sel.names, + toolchain: tc, + issueRef: issueRef, + env: env, + now: time.Now(), + }) + if env != nil { + if err := rep.writeGitHubOutputs(env); err != nil { + return err + } + } else if len(updates) > 0 || tc.changed() { + fmt.Printf("\nSuggested commit message:\n\n%s", rep.commitMessage()) + } + return res.err() +} + +// resolveResult is what resolveAll learned about the modules it was asked +// to look up. +type resolveResult struct { + updates []update + upToDate []string // paths with nothing newer + held []heldInfo // paths whose only newer versions are inside the cooldown + failures int // lookups that errored, already logged +} + +// heldInfo describes a module held back by the cooldown. +type heldInfo struct { + Path string + Err *heldError +} + +// err returns an error if any lookups failed, so the program's exit +// status reflects them even though the successful updates were applied. +func (r *resolveResult) err() error { + if r.failures > 0 { + return fmt.Errorf("%d module lookups failed; see above", r.failures) + } + return nil +} + +// resolveAll asks the proxy about every module in mods concurrently. +func resolveAll(ctx context.Context, r *resolver, mods []module.Version) (*resolveResult, error) { var ( - mu sync.Mutex - updates []update - failures int - numHeld int - matched int + mu sync.Mutex + res resolveResult ) g, gctx := errgroup.WithContext(ctx) g.SetLimit(*parallel) - for _, req := range mf.Require { - if len(filters) > 0 { - if !matchesAny(req.Mod.Path, filters) { - continue - } - matched++ - } else if req.Indirect && !*indirect { - continue - } - if replaced[req.Mod.Path] { - log.Printf("skipping %s: has a replace directive", req.Mod.Path) - continue - } + for _, mod := range mods { g.Go(func() error { - latest, err := r.lookupNewer(gctx, req.Mod) + latest, err := r.lookupNewer(gctx, mod) mu.Lock() defer mu.Unlock() var held *heldError switch { case errors.As(err, &held): - numHeld++ - log.Printf("holding %s: %v", req.Mod.Path, err) + res.held = append(res.held, heldInfo{mod.Path, held}) + log.Printf("holding %s: %v", mod.Path, err) case err != nil: - failures++ - log.Printf("skipping %s: %v", req.Mod.Path, err) + res.failures++ + log.Printf("skipping %s: %v", mod.Path, err) case latest != "": - updates = append(updates, update{req.Mod.Path, req.Mod.Version, latest}) + res.updates = append(res.updates, update{mod.Path, mod.Version, latest}) + default: + res.upToDate = append(res.upToDate, mod.Path) } return nil }) } if err := g.Wait(); err != nil { - return err + return nil, err } - if len(filters) > 0 && matched == 0 { - return fmt.Errorf("no modules in go.mod match %q", filters) - } - slices.SortFunc(updates, func(a, b update) int { return cmp.Compare(a.Path, b.Path) }) - - if len(updates) == 0 { - if numHeld > 0 { - fmt.Printf("nothing to update; %d module(s) held by the %d-day cooldown\n", numHeld, *excludeNewerThanDays) - } else { - fmt.Println("all dependencies are up to date") - } - } else { - tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(tw, "MODULE\tCURRENT\tLATEST") - for _, u := range updates { - fmt.Fprintf(tw, "%s\t%s\t%s\n", u.Path, u.Current, u.Latest) - } - tw.Flush() - } - - if len(updates) > 0 && !*dryRun { - args := []string{"get"} - for _, u := range updates { - args = append(args, u.Path+"@"+u.Latest) - } - cmd := exec.CommandContext(ctx, goBinary(), args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - fmt.Fprintf(os.Stderr, "running: %s get ... (%d modules)\n", cmd.Path, len(updates)) - if err := cmd.Run(); err != nil { - return fmt.Errorf("go get: %w", err) - } - fmt.Fprintln(os.Stderr, "done; now run \"make tidy && make updatedeps\" (or \"go mod tidy\")") - } - - if failures > 0 { - return fmt.Errorf("%d module lookups failed; see above", failures) - } - return nil -} - -// matchesAny reports whether modPath contains any of the filters, -// ignoring case. -func matchesAny(modPath string, filters []string) bool { - lower := strings.ToLower(modPath) - for _, f := range filters { - if strings.Contains(lower, strings.ToLower(f)) { - return true - } - } - return false + slices.Sort(res.upToDate) + slices.SortFunc(res.held, func(a, b heldInfo) int { return cmp.Compare(a.Path, b.Path) }) + return &res, nil } // goBinary returns the go binary to run, honoring the -go flag. @@ -236,206 +342,58 @@ func goBinary() string { return "go" } -// heldError says a newer version exists but is younger than the cooldown. -type heldError struct { - version string - t time.Time +// goGet runs a single go get for all of updates. +func goGet(ctx context.Context, updates []update) error { + args := []string{"get"} + for _, u := range updates { + args = append(args, u.Path+"@"+u.Latest) + } + fmt.Fprintf(os.Stderr, "running: %s get ... (%d modules)\n", goBinary(), len(updates)) + if err := runCommand(ctx, goBinary(), args...); err != nil { + return fmt.Errorf("go get: %w", err) + } + return nil } -func (e *heldError) Error() string { - return fmt.Sprintf("%s is only %.1f days old", e.version, time.Since(e.t).Hours()/24) +// runCommand runs name with args, passing through its output. +func runCommand(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return nil } -// versionInfo is the proxy's @latest and @v/.info response. -type versionInfo struct { - Version string - Time time.Time +// toolchainBump records a run of ./pull-toolchain.sh. +type toolchainBump struct { + before, after string // contents of go.toolchain.rev } -// tooNew reports whether a version committed at t is younger than the -// cooldown. -func (r *resolver) tooNew(t time.Time) bool { - return !r.cutoff.IsZero() && t.After(r.cutoff) +// changed reports whether the toolchain moved. It's safe on a nil receiver +// so callers needn't check whether a bump was requested. +func (t *toolchainBump) changed() bool { + return t != nil && t.before != t.after } -// lookupNewer asks the module proxy for the newest version of mod, or for -// the head of its branch if it's in specialBranches. It returns "" if there -// is nothing newer than the version already in go.mod or if the newer -// version isn't usable under mod's path, and a *heldError if the only -// newer versions are younger than the cooldown. -func (r *resolver) lookupNewer(ctx context.Context, mod module.Version) (string, error) { - escaped, err := module.EscapePath(mod.Path) - if err != nil { - return "", err - } - base := r.base + "/" + escaped - - var info versionInfo - if branch, ok := specialBranches[mod.Path]; ok { - if err := fetchJSON(ctx, r.client, base+"/@v/"+branch+".info", &info); err != nil { - return "", err - } - } else { - if err := fetchJSON(ctx, r.client, base+"/@latest", &info); err != nil { - return "", err - } - if semver.Compare(info.Version, mod.Version) > 0 && r.tooNew(info.Time) { - // Fall back to the newest release that has aged enough, if any. - // Only tagged versions are listed, so branch heads (above) - // have nothing to fall back to and get held instead. - aged, err := r.newestAged(ctx, base, mod.Version) - if err != nil { - return "", err - } - if aged.Version == "" { - return "", &heldError{info.Version, info.Time} - } - info = aged - } - } - if !semver.IsValid(info.Version) { - return "", fmt.Errorf("proxy returned invalid version %q", info.Version) - } - - // Only ever move forward. The proxy's @latest is the newest tagged - // release, which can be older than a pseudo-version already in go.mod. - if semver.Compare(info.Version, mod.Version) <= 0 { - return "", nil - } - if r.tooNew(info.Time) { - return "", &heldError{info.Version, info.Time} - } - - // Also never move backward in time. Forks sometimes carry stray tags - // that sort above their real development branch (github.com/tailscale/ - // golang-x-crypto has a v0.91.0 from 2024, say), and the proxy happily - // reports those as @latest. - curTime, err := r.versionTime(ctx, base, mod.Version) - if err != nil { - return "", err - } - if !info.Time.After(curTime) { - return "", fmt.Errorf("%s (%s) is older than current %s (%s); probably a stray tag, so add it to specialBranches or bump by hand", - info.Version, info.Time.Format(time.DateOnly), mod.Version, curTime.Format(time.DateOnly)) - } - - // Modules sometimes move (github.com/imdario/mergo became - // dario.cat/mergo, say) and keep tagging releases under the new path. - // The proxy still reports those as @latest for the old path, but go - // get rejects them, so check the go.mod of the candidate version. - escapedVer, err := module.EscapeVersion(info.Version) - if err != nil { - return "", err - } - gomod, err := fetch(ctx, r.client, base+"/@v/"+escapedVer+".mod") - if err != nil { - return "", err - } - if got := modfile.ModulePath(gomod); got != mod.Path { - return "", fmt.Errorf("%s declares module path %s; update the import path by hand", info.Version, got) - } - return info.Version, nil -} - -// newestAged returns the newest tagged release of the module served at base -// that is both newer than cur and older than the cooldown, or a zero -// versionInfo if there is none. Prereleases are skipped, matching what -// @latest would pick. -func (r *resolver) newestAged(ctx context.Context, base, cur string) (versionInfo, error) { - list, err := fetch(ctx, r.client, base+"/@v/list") - if err != nil { - return versionInfo{}, err - } - var candidates []string - for _, v := range strings.Fields(string(list)) { - if semver.IsValid(v) && semver.Prerelease(v) == "" && semver.Compare(v, cur) > 0 { - candidates = append(candidates, v) - } - } - semver.Sort(candidates) - slices.Reverse(candidates) - for _, v := range candidates { - escapedVer, err := module.EscapeVersion(v) - if err != nil { - return versionInfo{}, err - } - var info versionInfo - if err := fetchJSON(ctx, r.client, base+"/@v/"+escapedVer+".info", &info); err != nil { - return versionInfo{}, err - } - if !r.tooNew(info.Time) { - return info, nil - } - } - return versionInfo{}, nil -} - -// versionTime returns the commit time of version ver of the module served -// at base. Pseudo-versions carry it in their name, so only tagged versions -// need a proxy round trip. -func (r *resolver) versionTime(ctx context.Context, base, ver string) (time.Time, error) { - if module.IsPseudoVersion(ver) { - return module.PseudoVersionTime(ver) - } - escapedVer, err := module.EscapeVersion(ver) - if err != nil { - return time.Time{}, err - } - var info versionInfo - if err := fetchJSON(ctx, r.client, base+"/@v/"+escapedVer+".info", &info); err != nil { - return time.Time{}, err - } - return info.Time, nil -} - -// statusError is a non-2xx response from the proxy. -type statusError struct { - code int - body string -} - -func (e *statusError) Error() string { - return fmt.Sprintf("proxy returned %d: %s", e.code, strings.TrimSpace(e.body)) -} - -// fetchJSON fetches url from the proxy and decodes its JSON body into dst. -func fetchJSON(ctx context.Context, client *http.Client, url string, dst any) error { - body, err := fetch(ctx, client, url) - if err != nil { - return err - } - return json.Unmarshal(body, dst) -} - -// fetch fetches url from the proxy and returns its body. It retries once -// on transient errors, since the proxy occasionally returns those under -// load. Client errors are final. -func fetch(ctx context.Context, client *http.Client, url string) ([]byte, error) { - for attempt := 0; ; attempt++ { - body, err := fetchOnce(ctx, client, url) - if err == nil { - return body, nil - } - var se *statusError - if attempt > 0 || (errors.As(err, &se) && se.code < 500) { - return nil, err - } - } -} - -func fetchOnce(ctx context.Context, client *http.Client, url string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) +// bumpToolchain runs ./pull-toolchain.sh and reports what it did to +// go.toolchain.rev. +func bumpToolchain(ctx context.Context) (*toolchainBump, error) { + const revFile = "go.toolchain.rev" + before, err := os.ReadFile(revFile) if err != nil { return nil, err } - res, err := client.Do(req) + if err := runCommand(ctx, "./pull-toolchain.sh"); err != nil { + return nil, err + } + after, err := os.ReadFile(revFile) if err != nil { return nil, err } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - body, _ := io.ReadAll(io.LimitReader(res.Body, 4<<10)) - return nil, &statusError{res.StatusCode, string(body)} - } - return io.ReadAll(res.Body) + return &toolchainBump{ + before: strings.TrimSpace(string(before)), + after: strings.TrimSpace(string(after)), + }, nil } diff --git a/misc/bumpdeps/bumpdeps_test.go b/misc/bumpdeps/bumpdeps_test.go new file mode 100644 index 000000000..2f8d6f63a --- /dev/null +++ b/misc/bumpdeps/bumpdeps_test.go @@ -0,0 +1,729 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +const testGoMod = `module example.com/app + +go 1.27 + +require ( + github.com/gokrazy/kernel.amd64 v0.0.0-20260101000000-aaaaaaaaaaaa + github.com/gokrazy/kernel.arm64 v0.0.0-20260101000000-bbbbbbbbbbbb + github.com/tailscale/wireguard-go v0.0.0-20260201000000-cccccccccccc + golang.org/x/net v0.50.0 + gvisor.dev/gvisor v0.0.0-20260301000000-dddddddddddd +) + +require github.com/gobwas/glob v0.2.3 // indirect + +require github.com/replaced/mod v1.0.0 + +replace github.com/replaced/mod => ../mod +` + +func testGoModParsed(t *testing.T) *goMod { + t.Helper() + gm, err := parseGoMod([]byte(testGoMod)) + if err != nil { + t.Fatal(err) + } + return gm +} + +func TestParseArgs(t *testing.T) { + gm := testGoModParsed(t) + paths := func(mods []module.Version) []string { + var out []string + for _, m := range mods { + out = append(out, m.Path) + } + return out + } + tests := []struct { + name string + args []string + indirect bool + wantLookups []string + wantExplicit []update + wantTC bool + wantNames []string + wantErr string + }{ + { + name: "no_args_selects_direct_deps", + wantLookups: []string{"github.com/gokrazy/kernel.amd64", "github.com/gokrazy/kernel.arm64", "github.com/tailscale/wireguard-go", "golang.org/x/net", "gvisor.dev/gvisor"}, + }, + { + name: "indirect_flag", + indirect: true, + wantLookups: []string{"github.com/gokrazy/kernel.amd64", "github.com/gokrazy/kernel.arm64", "github.com/tailscale/wireguard-go", "golang.org/x/net", "gvisor.dev/gvisor", "github.com/gobwas/glob"}, + }, + { + name: "substring_filter", + args: []string{"Kernel"}, + wantLookups: []string{"github.com/gokrazy/kernel.amd64", "github.com/gokrazy/kernel.arm64"}, + wantNames: []string{"Kernel"}, + }, + { + name: "substring_filter_skips_indirect", + args: []string{"glob"}, + wantErr: `no modules in go.mod match "glob"`, + }, + { + name: "substring_filter_matches_indirect_with_flag", + args: []string{"glob"}, + indirect: true, + wantLookups: []string{"github.com/gobwas/glob"}, + wantNames: []string{"glob"}, + }, + { + name: "exact_path_selects_indirect", + args: []string{"github.com/gobwas/glob"}, + wantLookups: []string{"github.com/gobwas/glob"}, + wantNames: []string{"github.com/gobwas/glob"}, + }, + { + name: "aliases_and_toolchain_comma_separated", + args: []string{"go, gvisor,wireguard-go"}, + wantLookups: []string{"gvisor.dev/gvisor", "github.com/tailscale/wireguard-go"}, + wantTC: true, + wantNames: []string{"Go toolchain", "gvisor", "wireguard-go"}, + }, + { + name: "exact_path", + args: []string{"golang.org/x/net"}, + wantLookups: []string{"golang.org/x/net"}, + wantNames: []string{"golang.org/x/net"}, + }, + { + name: "explicit_version", + args: []string{"github.com/gokrazy/kernel.amd64@main"}, + wantExplicit: []update{{"github.com/gokrazy/kernel.amd64", "v0.0.0-20260101000000-aaaaaaaaaaaa", "main"}}, + wantNames: []string{"github.com/gokrazy/kernel.amd64"}, + }, + { + name: "explicit_latest_is_a_lookup", + args: []string{"golang.org/x/net@latest"}, + wantLookups: []string{"golang.org/x/net"}, + wantNames: []string{"golang.org/x/net"}, + }, + { + name: "new_module", + args: []string{"github.com/new/mod"}, + wantLookups: []string{"github.com/new/mod"}, + wantNames: []string{"github.com/new/mod"}, + }, + { + name: "new_module_at_version", + args: []string{"github.com/new/mod@v1.2.3"}, + wantExplicit: []update{{"github.com/new/mod", "", "v1.2.3"}}, + wantNames: []string{"github.com/new/mod"}, + }, + { + name: "duplicate_selection_collapses", + args: []string{"gvisor", "gvisor.dev/gvisor", "gvisor"}, + wantLookups: []string{"gvisor.dev/gvisor"}, + wantNames: []string{"gvisor", "gvisor.dev/gvisor", "gvisor"}, + }, + { + name: "no_match", + args: []string{"nonesuch"}, + wantErr: `no modules in go.mod match "nonesuch"`, + }, + { + name: "replaced_explicit", + args: []string{"github.com/replaced/mod@v2.0.0"}, + wantErr: "replace directive", + }, + { + name: "empty_version", + args: []string{"golang.org/x/net@"}, + wantErr: "empty version", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sel, err := parseArgs(gm, tt.args, tt.indirect) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v; want containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if got := paths(sel.lookups); !reflect.DeepEqual(got, tt.wantLookups) { + t.Errorf("lookups = %q; want %q", got, tt.wantLookups) + } + if !reflect.DeepEqual(sel.explicit, tt.wantExplicit) { + t.Errorf("explicit = %+v; want %+v", sel.explicit, tt.wantExplicit) + } + if sel.toolchain != tt.wantTC { + t.Errorf("toolchain = %v; want %v", sel.toolchain, tt.wantTC) + } + if !reflect.DeepEqual(sel.names, tt.wantNames) { + t.Errorf("names = %q; want %q", sel.names, tt.wantNames) + } + }) + } +} + +func TestParseArgsKeepsCurrentVersions(t *testing.T) { + gm := testGoModParsed(t) + sel, err := parseArgs(gm, []string{"golang.org/x/net", "github.com/new/mod"}, false) + if err != nil { + t.Fatal(err) + } + want := []module.Version{ + {Path: "golang.org/x/net", Version: "v0.50.0"}, + {Path: "github.com/new/mod"}, + } + if !reflect.DeepEqual(sel.lookups, want) { + t.Errorf("lookups = %+v; want %+v", sel.lookups, want) + } +} + +func TestCheckDowngrades(t *testing.T) { + before := testGoModParsed(t) + after, err := parseGoMod([]byte(strings.NewReplacer( + "golang.org/x/net v0.50.0", "golang.org/x/net v0.49.0", + "gvisor.dev/gvisor v0.0.0-20260301000000-dddddddddddd", "gvisor.dev/gvisor v0.0.0-20260401000000-eeeeeeeeeeee", + ).Replace(testGoMod))) + if err != nil { + t.Fatal(err) + } + err = checkDowngrades(before, after) + if err == nil || !strings.Contains(err.Error(), "golang.org/x/net v0.50.0 => v0.49.0") { + t.Fatalf("err = %v; want x/net downgrade", err) + } + if strings.Contains(err.Error(), "gvisor") { + t.Errorf("upgrade reported as downgrade: %v", err) + } + if err := checkDowngrades(before, before); err != nil { + t.Errorf("unchanged go.mod: %v", err) + } +} + +func TestParseIssueRef(t *testing.T) { + tests := []struct { + in, want string + }{ + {"https://github.com/tailscale/tailscale/issues/12345", "#12345"}, + {" https://github.com/tailscale/tailscale/issues/12345\n", "#12345"}, + {"https://github.com/tailscale/corp/issues/7", "tailscale/corp#7"}, + {"#42", "#42"}, + {"tailscale/corp#42", "tailscale/corp#42"}, + {"tailscale/tailscale#42", "#42"}, + {"https://github.com/tailscale/tailscale/pull/12345", ""}, + {"12345", ""}, + {"", ""}, + } + for _, tt := range tests { + got, err := parseIssueRef(tt.in, "tailscale/tailscale") + if tt.want == "" { + if err == nil { + t.Errorf("%q: got %q, want error", tt.in, got) + } + continue + } + if err != nil { + t.Errorf("%q: %v", tt.in, err) + } else if got != tt.want { + t.Errorf("%q = %q; want %q", tt.in, got, tt.want) + } + } +} + +func TestDescribeChange(t *testing.T) { + tests := []struct { + u update + want string + }{ + { + update{"github.com/gokrazy/kernel.amd64", "v0.0.0-20260101000000-aaaaaaaaaaaa", "v0.0.0-20260201000000-bbbbbbbbbbbb"}, + "https://github.com/gokrazy/kernel.amd64/compare/aaaaaaaaaaaa...bbbbbbbbbbbb", + }, + { + update{"gvisor.dev/gvisor", "v0.0.0-20260101000000-aaaaaaaaaaaa", "v0.0.0-20260201000000-bbbbbbbbbbbb"}, + "https://github.com/google/gvisor/compare/aaaaaaaaaaaa...bbbbbbbbbbbb", + }, + { + update{"github.com/foo/bar", "v1.2.3", "v1.3.0"}, + "https://github.com/foo/bar/compare/v1.2.3...v1.3.0", + }, + { + update{"github.com/foo/bar/v2", "v2.0.0", "v2.1.0"}, + "https://github.com/foo/bar/compare/v2.0.0...v2.1.0", + }, + { + update{"github.com/foo/bar/sub", "v1.0.0", "v1.1.0"}, + "https://github.com/foo/bar/compare/sub/v1.0.0...sub/v1.1.0", + }, + { + update{"github.com/foo/bar/sub/v3", "v3.0.0", "v3.1.0"}, + "https://github.com/foo/bar/compare/sub/v3.0.0...sub/v3.1.0", + }, + { + update{"github.com/foo/bar", "v2.0.0+incompatible", "v2.1.0+incompatible"}, + "https://github.com/foo/bar/compare/v2.0.0...v2.1.0", + }, + { + update{"golang.org/x/net", "v0.50.0", "v0.51.0"}, + "v0.50.0 to v0.51.0", + }, + { + update{"golang.org/x/net", "", "v0.51.0"}, + "added at v0.51.0", + }, + } + for _, tt := range tests { + if got := describeChange(tt.u); got != tt.want { + t.Errorf("%+v = %q; want %q", tt.u, got, tt.want) + } + } +} + +func TestBranchName(t *testing.T) { + now := time.Date(2026, 9, 17, 15, 4, 5, 0, time.FixedZone("PDT", -7*3600)) + tests := []struct { + env githubEnv + want string + }{ + {githubEnv{workflow: "bumpdep", event: "workflow_dispatch", actor: "bradfitz"}, "actions/bumpdep/bradfitz/20260917-220405"}, + {githubEnv{workflow: "bumpdep", event: "workflow_dispatch", actor: "some one/../evil"}, "actions/bumpdep/some-one-evil/20260917-220405"}, + {githubEnv{workflow: "bumpdep", event: "workflow_dispatch"}, "actions/bumpdep/unknown/20260917-220405"}, + {githubEnv{event: "workflow_dispatch", actor: "bradfitz"}, "actions/bumpdep/bradfitz/20260917-220405"}, + {githubEnv{workflow: "gokrazy-bump", event: "schedule", actor: "bradfitz"}, "actions/gokrazy-bump/20260917-220405"}, + {githubEnv{workflow: "gokrazy-bump", event: "workflow_dispatch", actor: "bradfitz"}, "actions/gokrazy-bump/bradfitz/20260917-220405"}, + } + for _, tt := range tests { + if got := branchName(&tt.env, now); got != tt.want { + t.Errorf("branchName(%+v) = %q; want %q", tt.env, got, tt.want) + } + } +} + +func TestBuildReport(t *testing.T) { + now := time.Date(2026, 9, 17, 22, 4, 5, 0, time.UTC) + env := &githubEnv{workflow: "bumpdep", event: "workflow_dispatch", actor: "bradfitz", repo: "tailscale/tailscale"} + cronEnv := &githubEnv{workflow: "gokrazy-bump", event: "schedule", actor: "bradfitz", repo: "tailscale/tailscale"} + held := &heldError{"v0.52.0", time.Now().Add(-36 * time.Hour)} + tests := []struct { + name string + in reportInput + wantTitle string + wantBranch string + wantBody string + }{ + { + name: "workflow_run_with_everything", + in: reportInput{ + updated: []update{ + {"github.com/gokrazy/kernel.amd64", "v0.0.0-20260101000000-aaaaaaaaaaaa", "v0.0.0-20260201000000-bbbbbbbbbbbb"}, + {"github.com/new/mod", "", "v1.0.0"}, + }, + unchanged: []string{"github.com/gokrazy/kernel.arm64"}, + held: []heldInfo{{"golang.org/x/net", held}}, + toolchain: &toolchainBump{"1111111", "2222222"}, + issueRef: "#123", + env: env, + now: now, + }, + wantTitle: "go.mod,go.toolchain.rev: bump github.com/gokrazy/kernel.amd64, github.com/new/mod, Go toolchain", + wantBranch: "actions/bumpdep/bradfitz/20260917-220405", + wantBody: `* github.com/gokrazy/kernel.amd64: https://github.com/gokrazy/kernel.amd64/compare/aaaaaaaaaaaa...bbbbbbbbbbbb +* github.com/new/mod: added at v1.0.0 +* github.com/gokrazy/kernel.arm64: already current +* golang.org/x/net: held by the cooldown; v0.52.0 is only 1.5 days old +* Go toolchain: https://github.com/tailscale/go/compare/1111111...2222222 + +Triggered by @bradfitz via the bumpdep workflow. + +Updates #123 +`, + }, + { + name: "local_run_many_modules_no_issue", + in: reportInput{ + updated: []update{ + {"a.example/a", "v1.0.0", "v1.1.0"}, + {"b.example/b", "v1.0.0", "v1.1.0"}, + {"c.example/c", "v1.0.0", "v1.1.0"}, + {"d.example/d", "v1.0.0", "v1.1.0"}, + }, + now: now, + }, + wantTitle: "go.mod: bump 4 modules", + wantBody: `* a.example/a: v1.0.0 to v1.1.0 +* b.example/b: v1.0.0 to v1.1.0 +* c.example/c: v1.0.0 to v1.1.0 +* d.example/d: v1.0.0 to v1.1.0 +`, + }, + { + name: "scheduled_run", + in: reportInput{ + updated: []update{ + {"github.com/gokrazy/kernel.amd64", "v0.0.0-20260101000000-aaaaaaaaaaaa", "v0.0.0-20260201000000-bbbbbbbbbbbb"}, + }, + issueRef: "#1866", + env: cronEnv, + now: now, + }, + wantTitle: "go.mod: bump github.com/gokrazy/kernel.amd64", + wantBranch: "actions/gokrazy-bump/20260917-220405", + wantBody: `* github.com/gokrazy/kernel.amd64: https://github.com/gokrazy/kernel.amd64/compare/aaaaaaaaaaaa...bbbbbbbbbbbb + +Triggered by the gokrazy-bump workflow's schedule. + +Updates #1866 +`, + }, + { + name: "toolchain_only", + in: reportInput{ + toolchain: &toolchainBump{"1111111", "2222222"}, + issueRef: "tailscale/corp#9", + now: now, + }, + wantTitle: "go.toolchain.rev: bump Go toolchain", + wantBody: `* Go toolchain: https://github.com/tailscale/go/compare/1111111...2222222 + +Updates tailscale/corp#9 +`, + }, + { + name: "nothing_changed", + in: reportInput{ + unchanged: []string{"gvisor.dev/gvisor"}, + requested: []string{"gvisor", "Go toolchain"}, + toolchain: &toolchainBump{"1111111", "1111111"}, + env: env, + now: now, + }, + wantTitle: "go.mod: bump gvisor, Go toolchain", + wantBranch: "actions/bumpdep/bradfitz/20260917-220405", + wantBody: `* gvisor.dev/gvisor: already current +* Go toolchain: already at 1111111 + +Triggered by @bradfitz via the bumpdep workflow. +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := buildReport(tt.in) + if r.title != tt.wantTitle { + t.Errorf("title = %q; want %q", r.title, tt.wantTitle) + } + if r.branch != tt.wantBranch { + t.Errorf("branch = %q; want %q", r.branch, tt.wantBranch) + } + if r.body != tt.wantBody { + t.Errorf("body = %q; want %q", r.body, tt.wantBody) + } + }) + } +} + +func TestWriteGitHubOutputs(t *testing.T) { + dir := t.TempDir() + env := &githubEnv{ + actor: "bradfitz", + repo: "tailscale/tailscale", + outputFile: filepath.Join(dir, "output"), + summaryFile: filepath.Join(dir, "summary"), + tempDir: dir, + } + r := &report{ + title: "go.mod: bump gvisor", + branch: "actions/bumpdep/bradfitz/20260917-220405", + body: "* gvisor.dev/gvisor: v1 to v2\n\nUpdates #1\n", + } + if err := r.writeGitHubOutputs(env); err != nil { + t.Fatal(err) + } + out, err := os.ReadFile(env.outputFile) + if err != nil { + t.Fatal(err) + } + got := string(out) + for _, want := range []string{ + "title=go.mod: bump gvisor\n", + "branch=actions/bumpdep/bradfitz/20260917-220405\n", + "body-path=" + filepath.Join(dir, "bumpdeps-pr-body.md") + "\n", + "commit-message< 0 { + latest = v + } + } + json.NewEncoder(w).Encode(latest) + case rest == "v/list": + for _, v := range vers { + if !module.IsPseudoVersion(v.Version) { + fmt.Fprintln(w, v.Version) + } + } + case strings.HasSuffix(rest, ".info"): + q := strings.TrimSuffix(strings.TrimPrefix(rest, "v/"), ".info") + if info, ok := p.commits[modPath+"@"+q]; ok { + json.NewEncoder(w).Encode(info) + return + } + for _, v := range vers { + if v.Version == q { + json.NewEncoder(w).Encode(v) + return + } + } + http.Error(w, "not found: unknown revision", http.StatusNotFound) + case strings.HasSuffix(rest, ".mod"): + declared := modPath + if mp, ok := p.modPath[modPath]; ok { + declared = mp + } + fmt.Fprintf(w, "module %s\n", declared) + default: + http.NotFound(w, req) + } +} + +func compareVersions(a, b string) int { + return strings.Compare(a, b) // good enough for the single-digit versions in tests +} + +func TestLookupNewer(t *testing.T) { + now := time.Now() + day := 24 * time.Hour + proxy := &fakeProxy{ + versions: map[string][]versionInfo{ + "example.com/tagged": { + {"v1.0.0", now.Add(-30 * day)}, + {"v1.1.0", now.Add(-10 * day)}, + {"v1.2.0", now.Add(-2 * day)}, + {"v1.3.0-rc1", now.Add(-1 * day)}, + }, + "example.com/stray": { + {"v0.91.0", now.Add(-700 * day)}, + }, + "example.com/renamed": { + {"v1.0.0", now.Add(-100 * day)}, + {"v2.0.0", now.Add(-10 * day)}, + }, + "gvisor.dev/gvisor": { + {"v0.0.0-20260301000000-dddddddddddd", now.Add(-20 * day)}, + }, + }, + modPath: map[string]string{ + "example.com/renamed": "example.com/newname", + }, + commits: map[string]versionInfo{ + "gvisor.dev/gvisor@" + strings.Repeat("e", 40): {"v0.0.0-20260401000000-eeeeeeeeeeee", now.Add(-5 * day)}, + "gvisor.dev/gvisor@go": {"v0.0.0-20260301000000-dddddddddddd", now.Add(-20 * day)}, + }, + } + srv := httptest.NewServer(proxy) + defer srv.Close() + + newResolver := func(cooldownDays int, lsRemote func(ctx context.Context, repo, branch string) (string, error)) *resolver { + r := &resolver{client: srv.Client(), base: srv.URL, lsRemote: lsRemote} + if cooldownDays > 0 { + r.cutoff = now.Add(-time.Duration(cooldownDays) * day) + } + return r + } + headIsE := func(ctx context.Context, repo, branch string) (string, error) { + if repo != "https://github.com/google/gvisor" || branch != "go" { + return "", fmt.Errorf("unexpected ls-remote %s %s", repo, branch) + } + return strings.Repeat("e", 40), nil + } + lsRemoteFails := func(context.Context, string, string) (string, error) { + return "", errors.New("no git") + } + + tests := []struct { + name string + r *resolver + mod module.Version + want string + wantHeld bool + wantErr string + }{ + { + name: "newer_tagged_release", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/tagged", Version: "v1.1.0"}, + want: "v1.2.0", + }, + { + name: "already_latest", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/tagged", Version: "v1.2.0"}, + want: "", + }, + { + name: "new_module_gets_latest", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/tagged"}, + want: "v1.2.0", + }, + { + name: "cooldown_falls_back_to_aged_release", + r: newResolver(7, nil), + mod: module.Version{Path: "example.com/tagged", Version: "v1.0.0"}, + want: "v1.1.0", + }, + { + name: "cooldown_holds_when_nothing_aged_is_newer", + r: newResolver(7, nil), + mod: module.Version{Path: "example.com/tagged", Version: "v1.1.0"}, + wantHeld: true, + }, + { + name: "stray_old_tag_is_rejected", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/stray", Version: "v0.0.0-20260101000000-aaaaaaaaaaaa"}, + wantErr: "probably a stray tag", + }, + { + name: "renamed_module_is_rejected", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/renamed", Version: "v1.0.0"}, + wantErr: "declares module path example.com/newname", + }, + { + name: "unknown_module", + r: newResolver(0, nil), + mod: module.Version{Path: "example.com/private", Version: "v1.0.0"}, + wantErr: "proxy returned 404", + }, + { + name: "branch_head_via_ls-remote", + r: newResolver(0, headIsE), + mod: module.Version{Path: "gvisor.dev/gvisor", Version: "v0.0.0-20260301000000-dddddddddddd"}, + want: "v0.0.0-20260401000000-eeeeeeeeeeee", + }, + { + name: "branch_head_held_by_cooldown", + r: newResolver(7, headIsE), + mod: module.Version{Path: "gvisor.dev/gvisor", Version: "v0.0.0-20260301000000-dddddddddddd"}, + wantHeld: true, + }, + { + name: "branch_head_falls_back_to_proxy_when_git_fails", + r: newResolver(0, lsRemoteFails), + mod: module.Version{Path: "gvisor.dev/gvisor", Version: "v0.0.0-20260301000000-dddddddddddd"}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.r.lookupNewer(context.Background(), tt.mod) + var held *heldError + switch { + case tt.wantHeld: + if !errors.As(err, &held) { + t.Fatalf("err = %v; want heldError", err) + } + case tt.wantErr != "": + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v; want containing %q", err, tt.wantErr) + } + case err != nil: + t.Fatal(err) + case got != tt.want: + t.Errorf("got %q; want %q", got, tt.want) + } + }) + } +} + +func TestSplitArgs(t *testing.T) { + got := splitArgs([]string{"go, gvisor,,wireguard-go", " a@v1 \n", "", "b"}) + want := []string{"go", "gvisor", "wireguard-go", "a@v1", "b"} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %q; want %q", got, want) + } +} diff --git a/misc/bumpdeps/gomod.go b/misc/bumpdeps/gomod.go new file mode 100644 index 000000000..0f73a2229 --- /dev/null +++ b/misc/bumpdeps/gomod.go @@ -0,0 +1,189 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "fmt" + "os" + "slices" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// goMod is the parts of go.mod that bumpdeps cares about. +type goMod struct { + requires []*modfile.Require // in file order + versions map[string]string // module path to required version + replaced map[string]bool // module paths with a replace directive +} + +// readGoMod reads go.mod from the current directory. +func readGoMod() (*goMod, error) { + data, err := os.ReadFile("go.mod") + if err != nil { + return nil, err + } + return parseGoMod(data) +} + +func parseGoMod(data []byte) (*goMod, error) { + mf, err := modfile.Parse("go.mod", data, nil) + if err != nil { + return nil, err + } + gm := &goMod{ + requires: mf.Require, + versions: make(map[string]string, len(mf.Require)), + replaced: make(map[string]bool, len(mf.Replace)), + } + for _, req := range mf.Require { + gm.versions[req.Mod.Path] = req.Mod.Version + } + for _, r := range mf.Replace { + gm.replaced[r.Old.Path] = true + } + return gm, nil +} + +// selection is what the command line asked bumpdeps to do. +type selection struct { + toolchain bool // run ./pull-toolchain.sh + lookups []module.Version // modules to resolve via the proxy, with their current versions ("" if new) + explicit []update // modules to hand to go get at a given version, no lookup + names []string // what was asked for, for the report when nothing changed +} + +// splitArgs splits args on commas and whitespace and drops empty pieces. +// The GitHub workflow passes its comma-separated input as one argument. +func splitArgs(args []string) []string { + var out []string + for _, a := range args { + for _, f := range strings.FieldsFunc(a, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' || r == '\n' }) { + out = append(out, f) + } + } + return out +} + +// parseArgs turns the command line into a selection against gm. With no +// args, every direct dependency is selected (plus indirect ones if +// indirect is set). See the package comment for the argument forms. +func parseArgs(gm *goMod, args []string, indirect bool) (*selection, error) { + sel := &selection{} + args = splitArgs(args) + if len(args) == 0 { + for _, req := range gm.requires { + if req.Indirect && !indirect { + continue + } + sel.addLookup(gm, req.Mod) + } + return sel, nil + } + + seen := map[string]bool{} + for _, arg := range args { + if arg == "go" { + sel.toolchain = true + sel.names = append(sel.names, "Go toolchain") + continue + } + if path, ok := aliases[arg]; ok { + sel.names = append(sel.names, arg) + sel.addKnown(gm, path, seen) + continue + } + if path, ver, ok := strings.Cut(arg, "@"); ok { + if err := module.CheckPath(path); err != nil { + return nil, fmt.Errorf("bad module path in %q: %v", arg, err) + } + if ver == "" { + return nil, fmt.Errorf("empty version in %q", arg) + } + sel.names = append(sel.names, path) + if seen[path] { + return nil, fmt.Errorf("%s selected more than once", path) + } + seen[path] = true + if gm.replaced[path] { + return nil, fmt.Errorf("%s has a replace directive; edit go.mod by hand", path) + } + if ver == "latest" { + sel.lookups = append(sel.lookups, module.Version{Path: path, Version: gm.versions[path]}) + } else { + sel.explicit = append(sel.explicit, update{Path: path, Current: gm.versions[path], Latest: ver}) + } + continue + } + if _, ok := gm.versions[arg]; ok { + sel.names = append(sel.names, arg) + sel.addKnown(gm, arg, seen) + continue + } + var matched bool + for _, req := range gm.requires { + if req.Indirect && !indirect { + continue + } + if strings.Contains(strings.ToLower(req.Mod.Path), strings.ToLower(arg)) { + matched = true + sel.addKnown(gm, req.Mod.Path, seen) + } + } + if matched { + sel.names = append(sel.names, arg) + continue + } + // Nothing in go.mod matches. If it looks like a module path, + // add it at its latest version. + if strings.Contains(arg, "/") && module.CheckPath(arg) == nil { + sel.names = append(sel.names, arg) + seen[arg] = true + sel.lookups = append(sel.lookups, module.Version{Path: arg}) + continue + } + return nil, fmt.Errorf("no modules in go.mod match %q", arg) + } + return sel, nil +} + +// addKnown selects the module at path, which is required by gm, unless +// it was already selected or has a replace directive. +func (sel *selection) addKnown(gm *goMod, path string, seen map[string]bool) { + if seen[path] { + return + } + seen[path] = true + sel.addLookup(gm, module.Version{Path: path, Version: gm.versions[path]}) +} + +// addLookup queues mod for a proxy lookup, skipping replaced modules. +func (sel *selection) addLookup(gm *goMod, mod module.Version) { + if gm.replaced[mod.Path] { + fmt.Fprintf(os.Stderr, "skipping %s: has a replace directive\n", mod.Path) + return + } + sel.lookups = append(sel.lookups, mod) +} + +// checkDowngrades returns an error if any module required by both before +// and after has a lower version in after. A downgrade means something +// went wrong and needs a human's rollback commit, not an unattended bump. +func checkDowngrades(before, after *goMod) error { + var downgraded []string + for path, oldVer := range before.versions { + newVer, ok := after.versions[path] + if ok && semver.Compare(newVer, oldVer) < 0 { + downgraded = append(downgraded, fmt.Sprintf("%s %s => %s", path, oldVer, newVer)) + } + } + if len(downgraded) == 0 { + return nil + } + slices.Sort(downgraded) + return fmt.Errorf("refusing to downgrade modules (go.mod and go.sum are left modified; revert them):\n\t%s", strings.Join(downgraded, "\n\t")) +} diff --git a/misc/bumpdeps/report.go b/misc/bumpdeps/report.go new file mode 100644 index 000000000..38518a945 --- /dev/null +++ b/misc/bumpdeps/report.go @@ -0,0 +1,290 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "cmp" + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "golang.org/x/mod/module" +) + +// githubEnv is what the bumpdep workflow tells us through the environment +// GitHub Actions sets up for every step. +type githubEnv struct { + workflow string // GITHUB_WORKFLOW, the workflow's name ("bumpdep", "gokrazy-bump") + event string // GITHUB_EVENT_NAME: "workflow_dispatch", "schedule", ... + actor string // GITHUB_TRIGGERING_ACTOR, the user who dispatched the workflow + repo string // GITHUB_REPOSITORY, "owner/repo" + outputFile string // GITHUB_OUTPUT, where step outputs go + summaryFile string // GITHUB_STEP_SUMMARY, Markdown shown on the run page + tempDir string // RUNNER_TEMP +} + +// scheduled reports whether the workflow ran on its cron schedule rather +// than because somebody dispatched it. The actor is then whoever last +// touched the workflow file, which isn't worth naming. +func (env *githubEnv) scheduled() bool { + return env.event == "schedule" +} + +// githubEnvFromOS reads the GitHub Actions environment. +func githubEnvFromOS() (*githubEnv, error) { + env := &githubEnv{ + workflow: os.Getenv("GITHUB_WORKFLOW"), + event: os.Getenv("GITHUB_EVENT_NAME"), + actor: cmp.Or(os.Getenv("GITHUB_TRIGGERING_ACTOR"), os.Getenv("GITHUB_ACTOR")), + repo: os.Getenv("GITHUB_REPOSITORY"), + outputFile: os.Getenv("GITHUB_OUTPUT"), + summaryFile: os.Getenv("GITHUB_STEP_SUMMARY"), + tempDir: cmp.Or(os.Getenv("RUNNER_TEMP"), os.TempDir()), + } + for _, v := range []struct{ name, val string }{ + {"GITHUB_WORKFLOW", env.workflow}, + {"GITHUB_TRIGGERING_ACTOR", env.actor}, + {"GITHUB_REPOSITORY", env.repo}, + {"GITHUB_OUTPUT", env.outputFile}, + } { + if v.val == "" { + return nil, fmt.Errorf("-github: $%s is not set; are we running in GitHub Actions?", v.name) + } + } + return env, nil +} + +var issueURLRx = regexp.MustCompile(`^https://github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)/issues/([0-9]+)$`) +var issueShortRx = regexp.MustCompile(`^([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)?#([0-9]+)$`) + +// parseIssueRef turns an issue given as a GitHub URL, "#123", or +// "owner/repo#123" into the short form used in commit messages: "#123" +// for an issue in repo (an "owner/repo" string), "owner/repo#123" for +// one elsewhere. +func parseIssueRef(s, repo string) (string, error) { + s = strings.TrimSpace(s) + var owner, num string + if m := issueURLRx.FindStringSubmatch(s); m != nil { + owner, num = m[1], m[2] + } else if m := issueShortRx.FindStringSubmatch(s); m != nil { + owner, num = cmp.Or(m[1], repo), m[2] + } else { + return "", fmt.Errorf("issue must be a GitHub issue URL like https://github.com/tailscale/tailscale/issues/12345 (or #12345, or owner/repo#12345); got %q", s) + } + if strings.EqualFold(owner, repo) { + return "#" + num, nil + } + return owner + "#" + num, nil +} + +// reportInput is everything buildReport needs to describe a run. +type reportInput struct { + updated []update // modules that changed, with their final versions + unchanged []string // selected modules that were already current + held []heldInfo // selected modules held by the cooldown + requested []string // what the command line asked for, used when nothing changed + toolchain *toolchainBump // nil if not requested + issueRef string // "#123" or "owner/repo#123", or "" for none + env *githubEnv // nil outside the workflow + now time.Time +} + +// report is the commit message and branch for a run. +type report struct { + title string + branch string // only set when running as the workflow + body string // Markdown body, without the title +} + +// commitMessage returns the title and body as a commit message. +func (r *report) commitMessage() string { + return r.title + "\n\n" + r.body +} + +// maxTitleNames is how many changed things get named in the title before +// it falls back to a count. +const maxTitleNames = 3 + +// buildReport renders in as a report. +func buildReport(in reportInput) *report { + var prefixes, names []string + if len(in.updated) > 0 { + prefixes = append(prefixes, "go.mod") + if len(in.updated) > maxTitleNames { + names = append(names, fmt.Sprintf("%d modules", len(in.updated))) + } else { + for _, u := range in.updated { + names = append(names, u.Path) + } + } + } + if in.toolchain.changed() { + prefixes = append(prefixes, "go.toolchain.rev") + names = append(names, "Go toolchain") + } + if len(prefixes) == 0 { + // Nothing changed, so this only shows up in the step summary. + prefixes = append(prefixes, "go.mod") + names = append(names, in.requested...) + } + title := strings.Join(prefixes, ",") + ": bump " + strings.Join(names, ", ") + + var body strings.Builder + for _, u := range in.updated { + fmt.Fprintf(&body, "* %s: %s\n", u.Path, describeChange(u)) + } + for _, path := range in.unchanged { + fmt.Fprintf(&body, "* %s: already current\n", path) + } + for _, h := range in.held { + fmt.Fprintf(&body, "* %s: held by the cooldown; %v\n", h.Path, h.Err) + } + if tc := in.toolchain; tc != nil { + if tc.changed() { + fmt.Fprintf(&body, "* Go toolchain: %s/compare/%s...%s\n", toolchainRepo, tc.before, tc.after) + } else { + fmt.Fprintf(&body, "* Go toolchain: already at %s\n", tc.after) + } + } + if env := in.env; env != nil { + if env.scheduled() { + fmt.Fprintf(&body, "\nTriggered by the %s workflow's schedule.\n", env.workflow) + } else { + fmt.Fprintf(&body, "\nTriggered by @%s via the %s workflow.\n", env.actor, env.workflow) + } + } + if in.issueRef != "" { + fmt.Fprintf(&body, "\nUpdates %s\n", in.issueRef) + } + + r := &report{title: title, body: body.String()} + if in.env != nil { + r.branch = branchName(in.env, in.now) + } + return r +} + +// describeChange says how u moved, as a GitHub compare link if the +// module's repo is known and a plain "from to to" otherwise. +func describeChange(u update) string { + if u.Current == "" { + return "added at " + u.Latest + } + repo := repoURL(u.Path) + if repo == "" { + return u.Current + " to " + u.Latest + } + return fmt.Sprintf("%s/compare/%s...%s", repo, gitRef(u.Path, u.Current), gitRef(u.Path, u.Latest)) +} + +// repoURL returns the https URL of the GitHub repo that serves module +// path, or "" if it's not known. +func repoURL(path string) string { + if bi, ok := specialBranches[path]; ok { + return bi.repo + } + parts := strings.SplitN(path, "/", 4) + if len(parts) >= 3 && parts[0] == "github.com" { + return "https://github.com/" + parts[1] + "/" + parts[2] + } + return "" +} + +// gitRef returns something GitHub's compare view understands for version +// ver of module path: the commit hash of a pseudo-version, or the tag of a +// release. Modules in a subdirectory of their repo are tagged with the +// subdirectory as a prefix ("sub/v1.2.3"). +func gitRef(path, ver string) string { + if module.IsPseudoVersion(ver) { + if rev, err := module.PseudoVersionRev(ver); err == nil { + return rev + } + } + ver = strings.TrimSuffix(ver, "+incompatible") + prefix, _, ok := module.SplitPathVersion(path) + if !ok { + return ver + } + if parts := strings.SplitN(prefix, "/", 4); len(parts) == 4 && parts[0] == "github.com" { + return parts[3] + "/" + ver + } + return ver +} + +var unsafeRefChars = regexp.MustCompile(`[^A-Za-z0-9_-]+`) + +// branchName returns the branch the workflow should push to: +// actions///, or without the actor for +// scheduled runs. It's short and unique per run rather than descriptive: +// the old scheme of joining every module path produced names like +// actions/bumpdep/github.com-gokrazy-kernel.amd64-main-github.com-gokrazy-kernel.arm64-main-github. +func branchName(env *githubEnv, now time.Time) string { + clean := func(s, fallback string) string { + return cmp.Or(strings.Trim(unsafeRefChars.ReplaceAllString(s, "-"), "-"), fallback) + } + parts := []string{"actions", clean(env.workflow, "bumpdep")} + if !env.scheduled() { + parts = append(parts, clean(env.actor, "unknown")) + } + parts = append(parts, now.UTC().Format("20060102-150405")) + return strings.Join(parts, "/") +} + +// writeGitHubOutputs publishes r as step outputs (title, branch, +// commit-message, body-path) and appends it to the step summary. +func (r *report) writeGitHubOutputs(env *githubEnv) error { + bodyPath := filepath.Join(env.tempDir, "bumpdeps-pr-body.md") + if err := os.WriteFile(bodyPath, []byte(r.body), 0o644); err != nil { + return err + } + var out strings.Builder + for _, kv := range []struct{ k, v string }{ + {"title", r.title}, + {"branch", r.branch}, + {"body-path", bodyPath}, + {"commit-message", r.commitMessage()}, + } { + writeGitHubOutput(&out, kv.k, kv.v) + } + if err := appendFile(env.outputFile, out.String()); err != nil { + return err + } + if env.summaryFile != "" { + if err := appendFile(env.summaryFile, "### "+r.title+"\n\n"+r.body); err != nil { + return err + } + } + return nil +} + +// writeGitHubOutput appends "key=value" to w in the $GITHUB_OUTPUT file +// format, using the heredoc form for multi-line values. +func writeGitHubOutput(w *strings.Builder, key, value string) { + if !strings.Contains(value, "\n") { + fmt.Fprintf(w, "%s=%s\n", key, value) + return + } + var buf [8]byte + rand.Read(buf[:]) + delim := "BUMPDEPS_" + hex.EncodeToString(buf[:]) + fmt.Fprintf(w, "%s<<%s\n%s\n%s\n", key, delim, strings.TrimRight(value, "\n"), delim) +} + +// appendFile appends s to the file at path, creating it if needed. +func appendFile(path, s string) error { + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + if _, err := f.WriteString(s); err != nil { + f.Close() + return err + } + return f.Close() +} diff --git a/misc/bumpdeps/resolve.go b/misc/bumpdeps/resolve.go new file mode 100644 index 000000000..13090bb1d --- /dev/null +++ b/misc/bumpdeps/resolve.go @@ -0,0 +1,288 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os/exec" + "slices" + "strings" + "time" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// resolver looks up versions on the module proxy. +type resolver struct { + client *http.Client + base string // proxy URL without a trailing slash + cutoff time.Time // versions committed after this are too new; zero means no cooldown + + // lsRemote returns the commit hash at the head of branch in the git + // repo at repoURL. It's a field so tests can fake it. + lsRemote func(ctx context.Context, repoURL, branch string) (string, error) +} + +// heldError says a newer version exists but is younger than the cooldown. +type heldError struct { + version string + t time.Time +} + +func (e *heldError) Error() string { + return fmt.Sprintf("%s is only %.1f days old", e.version, time.Since(e.t).Hours()/24) +} + +// versionInfo is the proxy's @latest and @v/.info response. +type versionInfo struct { + Version string + Time time.Time +} + +// tooNew reports whether a version committed at t is younger than the +// cooldown. +func (r *resolver) tooNew(t time.Time) bool { + return !r.cutoff.IsZero() && t.After(r.cutoff) +} + +// lookupNewer asks the module proxy for the newest version of mod, or for +// the head of its branch if it's in specialBranches. It returns "" if there +// is nothing newer than the version already in go.mod or if the newer +// version isn't usable under mod's path, and a *heldError if the only +// newer versions are younger than the cooldown. +// +// mod.Version may be empty for a module that's not yet in go.mod, in +// which case anything the proxy offers is newer. +func (r *resolver) lookupNewer(ctx context.Context, mod module.Version) (string, error) { + escaped, err := module.EscapePath(mod.Path) + if err != nil { + return "", err + } + base := r.base + "/" + escaped + + var info versionInfo + if bi, ok := specialBranches[mod.Path]; ok { + info, err = r.branchHead(ctx, base, bi) + if err != nil { + return "", err + } + } else { + if err := fetchJSON(ctx, r.client, base+"/@latest", &info); err != nil { + return "", err + } + if isNewer(info.Version, mod.Version) && r.tooNew(info.Time) { + // Fall back to the newest release that has aged enough, if any. + // Only tagged versions are listed, so branch heads (above) + // have nothing to fall back to and get held instead. + aged, err := r.newestAged(ctx, base, mod.Version) + if err != nil { + return "", err + } + if aged.Version == "" { + return "", &heldError{info.Version, info.Time} + } + info = aged + } + } + if !semver.IsValid(info.Version) { + return "", fmt.Errorf("proxy returned invalid version %q", info.Version) + } + + // Only ever move forward. The proxy's @latest is the newest tagged + // release, which can be older than a pseudo-version already in go.mod. + if !isNewer(info.Version, mod.Version) { + return "", nil + } + if r.tooNew(info.Time) { + return "", &heldError{info.Version, info.Time} + } + + // Also never move backward in time. Forks sometimes carry stray tags + // that sort above their real development branch (github.com/tailscale/ + // golang-x-crypto has a v0.91.0 from 2024, say), and the proxy happily + // reports those as @latest. + if mod.Version != "" { + curTime, err := r.versionTime(ctx, base, mod.Version) + if err != nil { + return "", err + } + if !info.Time.After(curTime) { + return "", fmt.Errorf("%s (%s) is older than current %s (%s); probably a stray tag, so add it to specialBranches or bump by hand", + info.Version, info.Time.Format(time.DateOnly), mod.Version, curTime.Format(time.DateOnly)) + } + } + + // Modules sometimes move (github.com/imdario/mergo became + // dario.cat/mergo, say) and keep tagging releases under the new path. + // The proxy still reports those as @latest for the old path, but go + // get rejects them, so check the go.mod of the candidate version. + escapedVer, err := module.EscapeVersion(info.Version) + if err != nil { + return "", err + } + gomod, err := fetch(ctx, r.client, base+"/@v/"+escapedVer+".mod") + if err != nil { + return "", err + } + if got := modfile.ModulePath(gomod); got != mod.Path { + return "", fmt.Errorf("%s declares module path %s; update the import path by hand", info.Version, got) + } + return info.Version, nil +} + +// isNewer reports whether candidate is newer than cur, where an empty cur +// (module not yet in go.mod) is older than everything. +func isNewer(candidate, cur string) bool { + return cur == "" || semver.Compare(candidate, cur) > 0 +} + +// branchHead returns the version at the head of the branch that bi +// describes, for the module served at base. It asks the git repo directly +// for the head commit, so a just-pushed commit is seen even if the proxy +// hasn't noticed it yet, and then asks the proxy for the pseudo-version of +// that commit. If git isn't available it falls back to the proxy's idea of +// the branch head. +func (r *resolver) branchHead(ctx context.Context, base string, bi branchInfo) (versionInfo, error) { + var info versionInfo + rev, err := r.lsRemote(ctx, bi.repo, bi.branch) + if err != nil { + log.Printf("git ls-remote %s %s failed (%v); asking the proxy instead", bi.repo, bi.branch, err) + rev = bi.branch + } + if err := fetchJSON(ctx, r.client, base+"/@v/"+rev+".info", &info); err != nil { + return versionInfo{}, err + } + return info, nil +} + +// gitLsRemote returns the commit hash at the head of branch in repoURL. +func gitLsRemote(ctx context.Context, repoURL, branch string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, "git", "ls-remote", "--exit-code", repoURL, "refs/heads/"+branch).Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) && len(ee.Stderr) > 0 { + return "", fmt.Errorf("%w: %s", err, bytes.TrimSpace(ee.Stderr)) + } + return "", err + } + hash, _, _ := strings.Cut(strings.TrimSpace(string(out)), "\t") + if len(hash) != 40 && len(hash) != 64 { + return "", fmt.Errorf("unexpected ls-remote output %q", out) + } + return hash, nil +} + +// newestAged returns the newest tagged release of the module served at base +// that is both newer than cur and older than the cooldown, or a zero +// versionInfo if there is none. Prereleases are skipped, matching what +// @latest would pick. +func (r *resolver) newestAged(ctx context.Context, base, cur string) (versionInfo, error) { + list, err := fetch(ctx, r.client, base+"/@v/list") + if err != nil { + return versionInfo{}, err + } + var candidates []string + for _, v := range strings.Fields(string(list)) { + if semver.IsValid(v) && semver.Prerelease(v) == "" && isNewer(v, cur) { + candidates = append(candidates, v) + } + } + semver.Sort(candidates) + slices.Reverse(candidates) + for _, v := range candidates { + escapedVer, err := module.EscapeVersion(v) + if err != nil { + return versionInfo{}, err + } + var info versionInfo + if err := fetchJSON(ctx, r.client, base+"/@v/"+escapedVer+".info", &info); err != nil { + return versionInfo{}, err + } + if !r.tooNew(info.Time) { + return info, nil + } + } + return versionInfo{}, nil +} + +// versionTime returns the commit time of version ver of the module served +// at base. Pseudo-versions carry it in their name, so only tagged versions +// need a proxy round trip. +func (r *resolver) versionTime(ctx context.Context, base, ver string) (time.Time, error) { + if module.IsPseudoVersion(ver) { + return module.PseudoVersionTime(ver) + } + escapedVer, err := module.EscapeVersion(ver) + if err != nil { + return time.Time{}, err + } + var info versionInfo + if err := fetchJSON(ctx, r.client, base+"/@v/"+escapedVer+".info", &info); err != nil { + return time.Time{}, err + } + return info.Time, nil +} + +// statusError is a non-2xx response from the proxy. +type statusError struct { + code int + body string +} + +func (e *statusError) Error() string { + return fmt.Sprintf("proxy returned %d: %s", e.code, strings.TrimSpace(e.body)) +} + +// fetchJSON fetches url from the proxy and decodes its JSON body into dst. +func fetchJSON(ctx context.Context, client *http.Client, url string, dst any) error { + body, err := fetch(ctx, client, url) + if err != nil { + return err + } + return json.Unmarshal(body, dst) +} + +// fetch fetches url from the proxy and returns its body. It retries once +// on transient errors, since the proxy occasionally returns those under +// load. Client errors are final. +func fetch(ctx context.Context, client *http.Client, url string) ([]byte, error) { + for attempt := 0; ; attempt++ { + body, err := fetchOnce(ctx, client, url) + if err == nil { + return body, nil + } + var se *statusError + if attempt > 0 || (errors.As(err, &se) && se.code < 500) { + return nil, err + } + } +} + +func fetchOnce(ctx context.Context, client *http.Client, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(res.Body, 4<<10)) + return nil, &statusError{res.StatusCode, string(body)} + } + return io.ReadAll(res.Body) +}