diff --git a/kernel/linux/DESIGN.md b/kernel/linux/DESIGN.md index 450b26321..35dc4c812 100644 --- a/kernel/linux/DESIGN.md +++ b/kernel/linux/DESIGN.md @@ -560,9 +560,8 @@ Tools in the lean harness, for this kind of work: out-of-tree Rust modules need a kernel built with CONFIG_RUST and a matching rustc, which most distro kernels can't provide. Revisit later. * Changing the WireGuard wire protocol. tswg must interoperate with every - existing Tailscale peer running wireguard-go. (But see "Hybrid - post-quantum handshakes": wireguard-go is about to grow a handshake - tswg doesn't have.) + existing Tailscale peer running wireguard-go, including its hybrid + post-quantum handshakes (see "Hybrid post-quantum handshakes"). * Multipath or "spray" sending in the kernel. When disco hasn't yet trusted a direct path, tailscaled sprays from userspace (see "DERP and spray"). @@ -934,22 +933,41 @@ Other TODOs: ### Hybrid post-quantum handshakes -Found 2026-11-07: Tailscale's wireguard-go fork has a hybrid -post-quantum handshake (`wgcfg.PeerConfig.Hybrid`, different message -sizes), wired up in tailscale.com on 2026-10-29, with a control -"disable hybrid" capability for the initial rollout (2026-11-03). Nothing -sets `Hybrid` yet, but once it's in use between two nodes, a kernel-mode -node can't talk to a hybrid-only peer, since tswg only does classic -WireGuard. So the "wire-compatible with every Tailscale peer" assumption -no longer holds. Options: +Tailscale's wireguard-go has a hybrid post-quantum handshake, +`Noise_IKhfs+psk2_25519+MLKEM768_ChaChaPoly_BLAKE2s` +(`wgcfg.PeerConfig.Hybrid`, wired up in tailscale.com 2026-10-29, with +a control "disable hybrid" capability for the rollout). Both peers must +agree to use it, with no fallback to the classic handshake, so a +kernel-mode node that couldn't do it would be cut off from any peer that +asked for it. tswg does it (2026-11-08): -* a kernel-mode node advertises (to control) that it doesn't do hybrid - handshakes, so peers use classic handshakes with it; or -* implement the hybrid handshake in tswg (needs a kernel ML-KEM, or a - vendored one). - -For now the kernel data plane logs any peer that asks for a hybrid -handshake and uses a classic handshake. +* Wire format, as wireguard-go: the initiation carries an ephemeral + ML-KEM-768 encapsulation key, sealed with the key from `es`, before + the static key (which then uses nonce 1); the response carries a KEM + ciphertext, sealed with a key from `ee`, and the KEM's shared secret + is mixed into the chaining key before `se`. Messages grow from 148 and + 92 bytes to 1348 and 1196. Hybrid messages that are retries have bit + 31 of the type set (`TSWG_MESSAGE_RETRY`, a hint for routing them + over more reliable paths); tswg accepts it, sets it on its own + retries, and echoes it in responses, as wireguard-go does. +* ML-KEM-768 is ours (`src/mlkem.c`, FIPS 203, with its own Keccak): + Linux has none, and its SHA-3 is too new for the kernels we build for. + `tests/mlkem` checks it against Go's `crypto/mlkem`, which wireguard-go + uses: the same keys from the same seeds, decapsulation of each other's + ciphertexts, and the same implicit rejection of corrupted ones. +* `noise.c` keeps the classic message structs, with the sealed KEM field + passed separately; send.c and receive.c convert to and from the hybrid + layouts. The handshake's KEM key (our decapsulation key as initiator, + the peer's encapsulation key as responder) is on the heap, only while + a handshake is in progress. +* Per peer, set by `TSWGPEER_F_HYBRID`, which tailscaled sets from + `wgcfg.PeerConfig.Hybrid`. Like wireguard-go, a peer's handshakes are + all hybrid or all classic, and a change applies from the next one. +* Tested in natlab, with `TS_DEBUG_FORCE_HYBRID_HANDSHAKES=1` (nothing + else turns hybrid handshakes on yet): `TestTSWGTailscaledHybridTUN` + (kernel mode against wireguard-go, each initiating, with + wireguard-go's hybrid handshake metrics as proof) and + `TestTSWGTailscaledHybridKernel` (kernel mode on both sides). ### What doesn't need kernel work @@ -1049,6 +1067,8 @@ install the module. * `DESIGN.md`: this file. * `upstream/`: pristine upstream sources (see "Fork, not patches"). * `src/`: the tswg module sources, Kbuild and compat headers. +* `tests/mlkem/`: checks `src/mlkem.c` against Go's `crypto/mlkem` + (`go test` there). * `scripts/`: `probe.sh` (compat probes), `build.sh` (build tswg.ko against a KDIR in a private copy of `src/`), `ubuntu-headers.sh` and `build-for-ubuntu.sh` (Ubuntu kernels), `gokrazy-kdir.sh` and @@ -1166,8 +1186,8 @@ Everything else: (`TestTSWGTailscaledDirect`). - [ ] Kernel mode: packet filter (security; required before real use; deferred). -- [ ] Kernel mode: hybrid PQ handshakes (advertise no support, or - implement them in tswg). +- [x] Kernel mode: hybrid PQ handshakes (2026-11-08; implemented in + tswg, `TestTSWGTailscaledHybridTUN`). - [x] Kernel mode: TSMP (`TestTSWGTailscaledTSMP`). - [ ] Kernel mode: MagicDNS and the other netstack-provided services (deferred). diff --git a/kernel/linux/src/Kbuild b/kernel/linux/src/Kbuild index c712f266c..332f56b75 100644 --- a/kernel/linux/src/Kbuild +++ b/kernel/linux/src/Kbuild @@ -17,5 +17,6 @@ tswg-y += peerlookup.o tswg-y += allowedips.o tswg-y += ratelimiter.o tswg-y += cookie.o +tswg-y += mlkem.o tswg-y += netlink.o generated/netlink.o obj-m := tswg.o diff --git a/kernel/linux/src/generated/netlink.c b/kernel/linux/src/generated/netlink.c index 8379ea7bf..6975cbcd3 100644 --- a/kernel/linux/src/generated/netlink.c +++ b/kernel/linux/src/generated/netlink.c @@ -26,7 +26,7 @@ const struct nla_policy wireguard_wgallowedip_nl_policy[WGALLOWEDIP_A_FLAGS + 1] * attribute; see the TODO in DESIGN.md about forking the YNL spec. */ const struct nla_policy tswg_peer_nl_policy[TSWGPEER_A_MAX + 1] = { - [TSWGPEER_A_FLAGS] = NLA_POLICY_MASK(NLA_U32, TSWGPEER_F_NO_ROAM), + [TSWGPEER_A_FLAGS] = NLA_POLICY_MASK(NLA_U32, TSWGPEER_F_NO_ROAM | TSWGPEER_F_HYBRID), [TSWGPEER_A_CLEAR_SESSION] = { .type = NLA_FLAG, }, }; diff --git a/kernel/linux/src/include/uapi/linux/tswg.h b/kernel/linux/src/include/uapi/linux/tswg.h index e6473c3ef..11ecea045 100644 --- a/kernel/linux/src/include/uapi/linux/tswg.h +++ b/kernel/linux/src/include/uapi/linux/tswg.h @@ -102,6 +102,12 @@ enum tswgpeer_flag { * yet, the first authenticated packet still sets one. */ TSWGPEER_F_NO_ROAM = 1, + /* + * Do hybrid post-quantum handshakes with the peer + * (Noise_IKhfs+psk2_25519+MLKEM768, as Tailscale's wireguard-go + * does), and only those. Both peers must agree. + */ + TSWGPEER_F_HYBRID = 2, }; enum tswgpeer_attribute { diff --git a/kernel/linux/src/messages.h b/kernel/linux/src/messages.h index 208da7267..5e11bb000 100644 --- a/kernel/linux/src/messages.h +++ b/kernel/linux/src/messages.h @@ -10,6 +10,8 @@ #include #include +#include "mlkem.h" /* TSWG */ + #include #include #include @@ -95,6 +97,41 @@ struct message_handshake_response { struct message_macs macs; }; +/* + * TSWG: hybrid post-quantum handshakes (Noise_IKhfs+psk2_25519+MLKEM768), + * as in Tailscale's wireguard-go. The initiation carries an ephemeral + * ML-KEM-768 encapsulation key, and the response a ciphertext for it, + * each encrypted, after the ephemeral key. Both peers must agree to use + * them; there is no fallback. See noise.c. + */ +struct message_handshake_initiation_hybrid { + struct message_header header; + __le32 sender_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_kem[noise_encrypted_len(TSWG_MLKEM768_EK_LEN)]; + u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; + u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; + struct message_macs macs; +}; + +struct message_handshake_response_hybrid { + struct message_header header; + __le32 sender_index; + __le32 receiver_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_kem[noise_encrypted_len(TSWG_MLKEM768_CT_LEN)]; + u8 encrypted_nothing[noise_encrypted_len(0)]; + struct message_macs macs; +}; + +/* + * TSWG: wireguard-go sets TSWG_MESSAGE_RETRY in the type of hybrid + * handshake messages that are retries (a response is a retry if its + * initiation was), as a hint to send them over more reliable paths. The + * MACs cover it; otherwise it's ignored. + */ +#define TSWG_MESSAGE_RETRY (1U << 31) + struct message_handshake_cookie { struct message_header header; __le32 receiver_index; diff --git a/kernel/linux/src/mlkem.c b/kernel/linux/src/mlkem.c new file mode 100644 index 000000000..cabb232dc --- /dev/null +++ b/kernel/linux/src/mlkem.c @@ -0,0 +1,613 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TSWG: ML-KEM-768 (FIPS 203), for the hybrid post-quantum handshake of + * Tailscale's wireguard-go (Noise_IKhfs+psk2_25519+MLKEM768). Linux has + * no ML-KEM, and its SHA-3 is too new for the kernels we build for, so + * this has its own Keccak. It's written for clarity over speed: it runs + * a few times per handshake, in process context. Arithmetic is on + * coefficients reduced to [0, q) with "% q", which compilers turn into + * multiplications (no data-dependent division); nothing else branches on + * secrets, and decapsulation's implicit rejection is a constant-time + * select. + * + * Checked against Go's crypto/mlkem by tests/mlkem. + */ + +#include "mlkem.h" + +#include +#include +#include +#include +#include +#include +#include + +enum { + MLKEM_N = 256, + MLKEM_Q = 3329, + MLKEM_K = 3, + MLKEM_DU = 10, + MLKEM_DV = 4, + MLKEM_POLY_BYTES = 384, /* ByteEncode12 of a polynomial */ + MLKEM_PKE_DK_LEN = MLKEM_K * MLKEM_POLY_BYTES, + MLKEM_U_BYTES = 32 * MLKEM_DU, /* ByteEncode10 of a polynomial */ + MLKEM_PRF_LEN = 64 * 2 /* eta = 2, for both eta1 and eta2 */ +}; + +/* Keccak-f[1600] and the SHA-3 sponge (FIPS 202). */ + +static const u64 keccak_rc[24] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL +}; + +static const u8 keccak_rho[24] = { + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 +}; + +static const u8 keccak_pi[24] = { + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 +}; + +static void keccakf(u64 st[25]) +{ + u64 bc[5], t; + int i, j, r; + + for (r = 0; r < 24; r++) { + /* θ */ + for (i = 0; i < 5; i++) + bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; + for (i = 0; i < 5; i++) { + t = bc[(i + 4) % 5] ^ rol64(bc[(i + 1) % 5], 1); + for (j = 0; j < 25; j += 5) + st[j + i] ^= t; + } + /* ρ and π */ + t = st[1]; + for (i = 0; i < 24; i++) { + j = keccak_pi[i]; + bc[0] = st[j]; + st[j] = rol64(t, keccak_rho[i]); + t = bc[0]; + } + /* χ */ + for (j = 0; j < 25; j += 5) { + for (i = 0; i < 5; i++) + bc[i] = st[j + i]; + for (i = 0; i < 5; i++) + st[j + i] ^= ~bc[(i + 1) % 5] & bc[(i + 2) % 5]; + } + /* ι */ + st[0] ^= keccak_rc[r]; + } +} + +struct keccak { + u64 st[25]; + unsigned int rate; /* in bytes */ + unsigned int pos; /* next byte of the rate to absorb or squeeze */ +}; + +enum { + SHA3_256_RATE = 136, + SHA3_512_RATE = 72, + SHAKE128_RATE = 168, + SHAKE256_RATE = 136, + SHA3_DS = 0x06, /* domain separation and first padding bit */ + SHAKE_DS = 0x1f +}; + +static void keccak_init(struct keccak *k, unsigned int rate) +{ + memset(k->st, 0, sizeof(k->st)); + k->rate = rate; + k->pos = 0; +} + +static void keccak_xor_byte(struct keccak *k, unsigned int i, u8 b) +{ + k->st[i / 8] ^= (u64)b << (8 * (i % 8)); +} + +static void keccak_absorb(struct keccak *k, const u8 *in, size_t len) +{ + while (len--) { + keccak_xor_byte(k, k->pos++, *in++); + if (k->pos == k->rate) { + keccakf(k->st); + k->pos = 0; + } + } +} + +/* keccak_finish pads the input, after which k may only be squeezed. */ +static void keccak_finish(struct keccak *k, u8 ds) +{ + keccak_xor_byte(k, k->pos, ds); + keccak_xor_byte(k, k->rate - 1, 0x80); + keccakf(k->st); + k->pos = 0; +} + +static void keccak_squeeze(struct keccak *k, u8 *out, size_t len) +{ + while (len--) { + if (k->pos == k->rate) { + keccakf(k->st); + k->pos = 0; + } + *out++ = k->st[k->pos / 8] >> (8 * (k->pos % 8)); + k->pos++; + } +} + +/* + * keccak2 hashes the concatenation of a and b (b may be empty) into out, + * with the given rate and domain separation byte. + */ +static void keccak2(struct keccak *k, unsigned int rate, u8 ds, + const u8 *a, size_t a_len, const u8 *b, size_t b_len, + u8 *out, size_t out_len) +{ + keccak_init(k, rate); + keccak_absorb(k, a, a_len); + keccak_absorb(k, b, b_len); + keccak_finish(k, ds); + keccak_squeeze(k, out, out_len); +} + +/* Polynomials, the NTT and the encodings (FIPS 203, section 4). */ + +typedef u16 poly[MLKEM_N]; /* coefficients in [0, q) */ + +/* + * mlkem_zetas[i] is 17^BitRev7(i) mod q, and mlkem_gammas[i] is + * 17^(2 BitRev7(i) + 1) mod q (FIPS 203, appendix A), computed at init + * rather than transcribed. + */ +static u16 mlkem_zetas[128] __ro_after_init; +static u16 mlkem_gammas[128] __ro_after_init; + +static u32 mlkem_pow17(unsigned int e) +{ + u32 r = 1; + + while (e--) + r = r * 17 % MLKEM_Q; + return r; +} + +/* BitRev7 (not bitrev8, which needs CONFIG_BITREVERSE). */ +static unsigned int mlkem_bitrev7(unsigned int x) +{ + unsigned int i, r = 0; + + for (i = 0; i < 7; i++) + r |= (x >> i & 1) << (6 - i); + return r; +} + +void tswg_mlkem_init(void) +{ + unsigned int i, rev; + + for (i = 0; i < 128; i++) { + rev = mlkem_bitrev7(i); + mlkem_zetas[i] = mlkem_pow17(rev); + mlkem_gammas[i] = mlkem_pow17(2 * rev + 1); + } +} + +static u16 mod_q(u32 x) +{ + return x % MLKEM_Q; +} + +/* Algorithm 9, NTT. */ +static void poly_ntt(poly f) +{ + unsigned int len, start, j, i = 1; + u16 zeta, t; + + for (len = 128; len >= 2; len /= 2) { + for (start = 0; start < MLKEM_N; start += 2 * len) { + zeta = mlkem_zetas[i++]; + for (j = start; j < start + len; j++) { + t = mod_q((u32)zeta * f[j + len]); + f[j + len] = mod_q(f[j] + MLKEM_Q - t); + f[j] = mod_q(f[j] + t); + } + } + } +} + +/* Algorithm 10, NTT^-1. */ +static void poly_invntt(poly f) +{ + unsigned int len, start, j, i = 127; + u16 zeta, t; + + for (len = 2; len <= 128; len *= 2) { + for (start = 0; start < MLKEM_N; start += 2 * len) { + zeta = mlkem_zetas[i--]; + for (j = start; j < start + len; j++) { + t = f[j]; + f[j] = mod_q(t + f[j + len]); + f[j + len] = mod_q((u32)zeta * (f[j + len] + MLKEM_Q - t)); + } + } + } + for (j = 0; j < MLKEM_N; j++) + f[j] = mod_q((u32)f[j] * 3303); /* 3303 = 128^-1 mod q */ +} + +/* poly_mul_acc adds a * b (Algorithm 11, MultiplyNTTs) to acc. */ +static void poly_mul_acc(poly acc, const poly a, const poly b) +{ + unsigned int i; + u32 c0, c1; + + for (i = 0; i < 128; i++) { + /* Algorithm 12, BaseCaseMultiply. */ + c0 = (u32)a[2 * i] * b[2 * i] + + (u32)mod_q((u32)a[2 * i + 1] * b[2 * i + 1]) * mlkem_gammas[i]; + c1 = (u32)a[2 * i] * b[2 * i + 1] + (u32)a[2 * i + 1] * b[2 * i]; + acc[2 * i] = mod_q(acc[2 * i] + c0); + acc[2 * i + 1] = mod_q(acc[2 * i + 1] + c1); + } +} + +static void poly_add(poly f, const poly g) +{ + unsigned int i; + + for (i = 0; i < MLKEM_N; i++) + f[i] = mod_q(f[i] + g[i]); +} + +/* Algorithm 5, ByteEncode_d, for d <= 12. */ +static void poly_encode(u8 *out, const poly f, unsigned int d) +{ + unsigned int i, bits = 0; + u32 acc = 0; + + for (i = 0; i < MLKEM_N; i++) { + acc |= (u32)f[i] << bits; + bits += d; + while (bits >= 8) { + *out++ = acc; + acc >>= 8; + bits -= 8; + } + } +} + +/* Algorithm 6, ByteDecode_d, for d <= 12, without reducing mod q. */ +static void poly_decode(poly f, const u8 *in, unsigned int d) +{ + unsigned int i, bits = 0; + u32 acc = 0; + + for (i = 0; i < MLKEM_N; i++) { + while (bits < d) { + acc |= (u32)*in++ << bits; + bits += 8; + } + f[i] = acc & ((1U << d) - 1); + acc >>= d; + bits -= d; + } +} + +/* poly_decode12 is ByteDecode12, which reduces mod q. */ +static void poly_decode12(poly f, const u8 *in) +{ + unsigned int i; + + poly_decode(f, in, 12); + for (i = 0; i < MLKEM_N; i++) + f[i] = mod_q(f[i]); +} + +/* Compress_d and Decompress_d (section 4.2.1), in place. */ +static void poly_compress(poly f, unsigned int d) +{ + unsigned int i; + + for (i = 0; i < MLKEM_N; i++) + f[i] = (((u32)f[i] << d) + MLKEM_Q / 2) / MLKEM_Q & ((1U << d) - 1); +} + +static void poly_decompress(poly f, unsigned int d) +{ + unsigned int i; + + for (i = 0; i < MLKEM_N; i++) + f[i] = ((u32)f[i] * MLKEM_Q + (1U << (d - 1))) >> d; +} + +/* Algorithm 8, SamplePolyCBD_2, of PRF_2(seed, n) (SHAKE256). */ +static void poly_cbd2(struct keccak *k, poly f, const u8 seed[32], u8 n) +{ + u8 buf[MLKEM_PRF_LEN]; + unsigned int i, h; + u8 b; + + keccak2(k, SHAKE256_RATE, SHAKE_DS, seed, 32, &n, 1, buf, sizeof(buf)); + for (i = 0; i < MLKEM_N / 2; i++) { + b = buf[i]; + for (h = 0; h < 2; h++, b >>= 4) + f[2 * i + h] = mod_q(MLKEM_Q + (b & 1) + (b >> 1 & 1) - + (b >> 2 & 1) - (b >> 3 & 1)); + } + memzero_explicit(buf, sizeof(buf)); +} + +/* + * Algorithm 7, SampleNTT, of rho || x || y: Â[y][x] (FIPS 203 samples + * Â[i, j] from rho || j || i). + */ +static void poly_sample_ntt(struct keccak *k, poly f, const u8 rho[32], + u8 x, u8 y) +{ + u8 buf[SHAKE128_RATE]; + unsigned int i, j = 0; + u16 d1, d2; + + keccak_init(k, SHAKE128_RATE); + keccak_absorb(k, rho, 32); + keccak_absorb(k, &x, 1); + keccak_absorb(k, &y, 1); + keccak_finish(k, SHAKE_DS); + while (j < MLKEM_N) { + keccak_squeeze(k, buf, sizeof(buf)); + for (i = 0; i < sizeof(buf) && j < MLKEM_N; i += 3) { + d1 = buf[i] | (buf[i + 1] & 0xf) << 8; + d2 = buf[i + 1] >> 4 | buf[i + 2] << 4; + if (d1 < MLKEM_Q) + f[j++] = d1; + if (d2 < MLKEM_Q && j < MLKEM_N) + f[j++] = d2; + } + } +} + +/* K-PKE and ML-KEM (sections 5 and 6), with a heap workspace. */ + +struct mlkem_ws { + struct keccak k; + poly a; /* a sampled matrix entry */ + poly acc; + poly v[MLKEM_K]; /* ŝ; or t̂ */ + poly w[MLKEM_K]; /* ê then t̂; or ŷ; or û */ + poly e; /* e1[i], e2 */ + u8 g[64]; /* G's output */ + u8 m[32]; + u8 ct[TSWG_MLKEM768_CT_LEN]; /* re-encryption, for decapsulation */ +}; + +static struct mlkem_ws *mlkem_ws_alloc(void) +{ + return kmalloc(sizeof(struct mlkem_ws), GFP_KERNEL); +} + +static void mlkem_ws_free(struct mlkem_ws *ws) +{ + kfree_sensitive(ws); +} + +/* G is SHA3-512, H is SHA3-256, and J is SHAKE256 with 32 bytes out. */ +static void mlkem_g(struct mlkem_ws *ws, const u8 *a, size_t a_len, + const u8 *b, size_t b_len) +{ + keccak2(&ws->k, SHA3_512_RATE, SHA3_DS, a, a_len, b, b_len, ws->g, 64); +} + +static void mlkem_h(struct mlkem_ws *ws, u8 out[32], const u8 *in, size_t len) +{ + keccak2(&ws->k, SHA3_256_RATE, SHA3_DS, in, len, NULL, 0, out, 32); +} + +/* + * Algorithm 13, K-PKE.KeyGen, from d, writing ek (the K-PKE and ML-KEM + * encapsulation keys are the same) and the K-PKE decapsulation key. + */ +static void kpke_keygen(struct mlkem_ws *ws, u8 *ek, u8 *pke_dk, const u8 d[32]) +{ + const u8 k = MLKEM_K; + const u8 *rho = ws->g, *sigma = ws->g + 32; + unsigned int i, j; + + mlkem_g(ws, d, 32, &k, 1); + for (i = 0; i < MLKEM_K; i++) { + poly_cbd2(&ws->k, ws->v[i], sigma, i); + poly_ntt(ws->v[i]); + } + for (i = 0; i < MLKEM_K; i++) { + poly_cbd2(&ws->k, ws->w[i], sigma, MLKEM_K + i); + poly_ntt(ws->w[i]); + } + for (i = 0; i < MLKEM_K; i++) { + for (j = 0; j < MLKEM_K; j++) { + poly_sample_ntt(&ws->k, ws->a, rho, j, i); + poly_mul_acc(ws->w[i], ws->a, ws->v[j]); + } + poly_encode(ek + i * MLKEM_POLY_BYTES, ws->w[i], 12); + poly_encode(pke_dk + i * MLKEM_POLY_BYTES, ws->v[i], 12); + } + memcpy(ek + MLKEM_PKE_DK_LEN, rho, 32); +} + +/* Algorithm 14, K-PKE.Encrypt, of ws->m with randomness r. */ +static void kpke_encrypt(struct mlkem_ws *ws, u8 *ct, const u8 *ek, + const u8 r[32]) +{ + const u8 *rho = ek + MLKEM_PKE_DK_LEN; + unsigned int i, j; + + for (i = 0; i < MLKEM_K; i++) { + poly_decode12(ws->v[i], ek + i * MLKEM_POLY_BYTES); + poly_cbd2(&ws->k, ws->w[i], r, i); + poly_ntt(ws->w[i]); + } + /* u = NTT^-1(Âᵀ ∘ ŷ) + e1 */ + for (i = 0; i < MLKEM_K; i++) { + memset(ws->acc, 0, sizeof(ws->acc)); + for (j = 0; j < MLKEM_K; j++) { + poly_sample_ntt(&ws->k, ws->a, rho, i, j); + poly_mul_acc(ws->acc, ws->a, ws->w[j]); + } + poly_invntt(ws->acc); + poly_cbd2(&ws->k, ws->e, r, MLKEM_K + i); + poly_add(ws->acc, ws->e); + poly_compress(ws->acc, MLKEM_DU); + poly_encode(ct + i * MLKEM_U_BYTES, ws->acc, MLKEM_DU); + } + /* v = NTT^-1(t̂ᵀ ∘ ŷ) + e2 + Decompress_1(m) */ + memset(ws->acc, 0, sizeof(ws->acc)); + for (i = 0; i < MLKEM_K; i++) + poly_mul_acc(ws->acc, ws->v[i], ws->w[i]); + poly_invntt(ws->acc); + poly_cbd2(&ws->k, ws->e, r, 2 * MLKEM_K); + poly_add(ws->acc, ws->e); + poly_decode(ws->e, ws->m, 1); + poly_decompress(ws->e, 1); + poly_add(ws->acc, ws->e); + poly_compress(ws->acc, MLKEM_DV); + poly_encode(ct + MLKEM_K * MLKEM_U_BYTES, ws->acc, MLKEM_DV); +} + +/* Algorithm 15, K-PKE.Decrypt, into ws->m. */ +static void kpke_decrypt(struct mlkem_ws *ws, const u8 *pke_dk, const u8 *ct) +{ + unsigned int i, j; + + memset(ws->acc, 0, sizeof(ws->acc)); + for (i = 0; i < MLKEM_K; i++) { + poly_decode(ws->w[i], ct + i * MLKEM_U_BYTES, MLKEM_DU); + poly_decompress(ws->w[i], MLKEM_DU); + poly_ntt(ws->w[i]); + poly_decode12(ws->v[i], pke_dk + i * MLKEM_POLY_BYTES); + poly_mul_acc(ws->acc, ws->v[i], ws->w[i]); + } + poly_invntt(ws->acc); + poly_decode(ws->e, ct + MLKEM_K * MLKEM_U_BYTES, MLKEM_DV); + poly_decompress(ws->e, MLKEM_DV); + for (j = 0; j < MLKEM_N; j++) + ws->e[j] = mod_q(ws->e[j] + MLKEM_Q - ws->acc[j]); + poly_compress(ws->e, 1); + poly_encode(ws->m, ws->e, 1); +} + +/* Algorithm 16, ML-KEM.KeyGen_internal. */ +int tswg_mlkem768_keygen_seed(u8 ek[TSWG_MLKEM768_EK_LEN], + u8 dk[TSWG_MLKEM768_DK_LEN], + const u8 seed[TSWG_MLKEM768_SEED_LEN]) +{ + struct mlkem_ws *ws = mlkem_ws_alloc(); + u8 *p = dk + MLKEM_PKE_DK_LEN; + + if (!ws) + return -ENOMEM; + kpke_keygen(ws, ek, dk, seed); + /* dk = pke_dk || ek || H(ek) || z */ + memcpy(p, ek, TSWG_MLKEM768_EK_LEN); + p += TSWG_MLKEM768_EK_LEN; + mlkem_h(ws, p, ek, TSWG_MLKEM768_EK_LEN); + memcpy(p + 32, seed + 32, 32); + mlkem_ws_free(ws); + return 0; +} + +/* Algorithm 19, ML-KEM.KeyGen. */ +int tswg_mlkem768_keygen(u8 ek[TSWG_MLKEM768_EK_LEN], + u8 dk[TSWG_MLKEM768_DK_LEN]) +{ + u8 seed[TSWG_MLKEM768_SEED_LEN]; + int ret; + + get_random_bytes(seed, sizeof(seed)); + ret = tswg_mlkem768_keygen_seed(ek, dk, seed); + memzero_explicit(seed, sizeof(seed)); + return ret; +} + +/* + * mlkem_ek_valid is the modulus check of section 7.2: every coefficient + * of ek's t̂ must already be reduced. + */ +static bool mlkem_ek_valid(struct mlkem_ws *ws, const u8 *ek) +{ + unsigned int i, j; + + for (i = 0; i < MLKEM_K; i++) { + poly_decode(ws->a, ek + i * MLKEM_POLY_BYTES, 12); + for (j = 0; j < MLKEM_N; j++) + if (ws->a[j] >= MLKEM_Q) + return false; + } + return true; +} + +/* Algorithms 17 and 20, ML-KEM.Encaps(_internal). */ +int tswg_mlkem768_encaps(u8 ct[TSWG_MLKEM768_CT_LEN], + u8 ss[TSWG_MLKEM768_SS_LEN], + const u8 ek[TSWG_MLKEM768_EK_LEN]) +{ + struct mlkem_ws *ws = mlkem_ws_alloc(); + u8 h[32]; + + if (!ws) + return -ENOMEM; + if (!mlkem_ek_valid(ws, ek)) { + mlkem_ws_free(ws); + return -EINVAL; + } + get_random_bytes(ws->m, sizeof(ws->m)); + mlkem_h(ws, h, ek, TSWG_MLKEM768_EK_LEN); + mlkem_g(ws, ws->m, 32, h, 32); + memcpy(ss, ws->g, 32); + kpke_encrypt(ws, ct, ek, ws->g + 32); + mlkem_ws_free(ws); + return 0; +} + +/* + * Algorithms 18 and 21, ML-KEM.Decaps(_internal). dk is our own, so it + * skips the input checks of section 7.3. + */ +int tswg_mlkem768_decaps(u8 ss[TSWG_MLKEM768_SS_LEN], + const u8 ct[TSWG_MLKEM768_CT_LEN], + const u8 dk[TSWG_MLKEM768_DK_LEN]) +{ + const u8 *ek = dk + MLKEM_PKE_DK_LEN; + const u8 *h = ek + TSWG_MLKEM768_EK_LEN, *z = h + 32; + struct mlkem_ws *ws = mlkem_ws_alloc(); + u8 reject[32], mask; + unsigned int i; + + if (!ws) + return -ENOMEM; + kpke_decrypt(ws, dk, ct); + mlkem_g(ws, ws->m, 32, h, 32); + keccak2(&ws->k, SHAKE256_RATE, SHAKE_DS, z, 32, ct, + TSWG_MLKEM768_CT_LEN, reject, sizeof(reject)); + kpke_encrypt(ws, ws->ct, ek, ws->g + 32); + /* Implicit rejection: K̄ unless the ciphertext re-encrypts. */ + mask = -(u8)!!crypto_memneq(ct, ws->ct, TSWG_MLKEM768_CT_LEN); + for (i = 0; i < 32; i++) + ss[i] = ws->g[i] ^ (mask & (ws->g[i] ^ reject[i])); + memzero_explicit(reject, sizeof(reject)); + mlkem_ws_free(ws); + return 0; +} diff --git a/kernel/linux/src/mlkem.h b/kernel/linux/src/mlkem.h new file mode 100644 index 000000000..f2c48db5e --- /dev/null +++ b/kernel/linux/src/mlkem.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * TSWG: ML-KEM-768 (FIPS 203), for hybrid post-quantum handshakes; see + * mlkem.c. + */ + +#ifndef _TSWG_MLKEM_H +#define _TSWG_MLKEM_H + +#include + +enum tswg_mlkem768_lengths { + TSWG_MLKEM768_EK_LEN = 1184, /* encapsulation (public) key */ + TSWG_MLKEM768_DK_LEN = 2400, /* decapsulation (private) key */ + TSWG_MLKEM768_CT_LEN = 1088, /* ciphertext */ + TSWG_MLKEM768_SS_LEN = 32, /* shared secret */ + TSWG_MLKEM768_SEED_LEN = 64 /* d || z, as in Go's crypto/mlkem */ +}; + +void tswg_mlkem_init(void); + +/* + * The functions below return 0 or a negative errno: -ENOMEM, or -EINVAL + * for an encapsulation key that isn't valid. They may sleep. + */ +int tswg_mlkem768_keygen(u8 ek[TSWG_MLKEM768_EK_LEN], + u8 dk[TSWG_MLKEM768_DK_LEN]); +int tswg_mlkem768_keygen_seed(u8 ek[TSWG_MLKEM768_EK_LEN], + u8 dk[TSWG_MLKEM768_DK_LEN], + const u8 seed[TSWG_MLKEM768_SEED_LEN]); +int tswg_mlkem768_encaps(u8 ct[TSWG_MLKEM768_CT_LEN], + u8 ss[TSWG_MLKEM768_SS_LEN], + const u8 ek[TSWG_MLKEM768_EK_LEN]); +int tswg_mlkem768_decaps(u8 ss[TSWG_MLKEM768_SS_LEN], + const u8 ct[TSWG_MLKEM768_CT_LEN], + const u8 dk[TSWG_MLKEM768_DK_LEN]); + +#endif /* _TSWG_MLKEM_H */ diff --git a/kernel/linux/src/netlink.c b/kernel/linux/src/netlink.c index 0e9a41bfc..07594d67b 100644 --- a/kernel/linux/src/netlink.c +++ b/kernel/linux/src/netlink.c @@ -657,6 +657,7 @@ int tswg_set_peer(struct wg_peer *peer, struct nlattr *attr) u32 flags = nla_get_u32(tb[TSWGPEER_A_FLAGS]); WRITE_ONCE(peer->tswg_no_roam, !!(flags & TSWGPEER_F_NO_ROAM)); + tswg_noise_set_hybrid(&peer->handshake, flags & TSWGPEER_F_HYBRID); } if (tb[TSWGPEER_A_CLEAR_SESSION]) { /* As wg_pm_notification does for every peer before suspend. */ @@ -677,6 +678,8 @@ int tswg_get_peer(struct sk_buff *skb, struct wg_peer *peer) if (READ_ONCE(peer->tswg_no_roam)) flags |= TSWGPEER_F_NO_ROAM; + if (READ_ONCE(peer->handshake.hybrid)) + flags |= TSWGPEER_F_HYBRID; nest = nla_nest_start(skb, WGPEER_A_TSWG); if (!nest) return -EMSGSIZE; diff --git a/kernel/linux/src/noise.c b/kernel/linux/src/noise.c index 428d3afe3..77f0f073c 100644 --- a/kernel/linux/src/noise.c +++ b/kernel/linux/src/noise.c @@ -23,6 +23,19 @@ * ****** * -> e, es, s, ss, {t} * <- e, ee, se, psk, {} + * + * TSWG: and the hybrid post-quantum handshake of Tailscale's wireguard-go, + * Noise_IKhfs+psk2_25519+MLKEM768, which adds an ephemeral ML-KEM-768 + * key pair (ekem) to the initiation and a ciphertext for it to the + * response, encrypted with the keys of es and ee, and mixes the KEM's + * shared secret into the chaining key before se: + * + * -> e, es, {ekem}, s, ss, {t} + * <- e, ee, {ekem ciphertext}, kem, se, psk, {} + * + * The static key's encryption uses nonce 1, since ekem's used 0 with the + * same key. A peer's handshakes are all hybrid or all classic (see + * struct noise_handshake's hybrid), as both sides must agree. */ static const u8 handshake_name[37] __nonstring = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; @@ -31,16 +44,32 @@ static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; static atomic64_t keypair_counter = ATOMIC64_INIT(0); -void __init wg_noise_init(void) +/* TSWG: as above, for hybrid handshakes. */ +static const u8 hybrid_handshake_name[50] __nonstring = + "Noise_IKhfs+psk2_25519+MLKEM768_ChaChaPoly_BLAKE2s"; +static u8 hybrid_init_hash[NOISE_HASH_LEN] __ro_after_init; +static u8 hybrid_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; + +static void __init noise_init_hash(u8 hash[NOISE_HASH_LEN], + u8 chaining_key[NOISE_HASH_LEN], + const u8 *name, size_t name_len) { struct blake2s_ctx blake; - blake2s(NULL, 0, handshake_name, sizeof(handshake_name), - handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s(NULL, 0, name, name_len, chaining_key, NOISE_HASH_LEN); blake2s_init(&blake, NOISE_HASH_LEN); - blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s_update(&blake, chaining_key, NOISE_HASH_LEN); blake2s_update(&blake, identifier_name, sizeof(identifier_name)); - blake2s_final(&blake, handshake_init_hash); + blake2s_final(&blake, hash); +} + +void __init wg_noise_init(void) +{ + noise_init_hash(handshake_init_hash, handshake_init_chaining_key, + handshake_name, sizeof(handshake_name)); + noise_init_hash(hybrid_init_hash, hybrid_init_chaining_key, + hybrid_handshake_name, sizeof(hybrid_handshake_name)); + tswg_mlkem_init(); } /* Must hold peer->handshake.static_identity->lock */ @@ -83,6 +112,8 @@ static void handshake_zero(struct noise_handshake *handshake) memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); handshake->remote_index = 0; handshake->state = HANDSHAKE_ZEROED; + kfree_sensitive(handshake->kem); /* TSWG */ + handshake->kem = NULL; } void wg_noise_handshake_clear(struct noise_handshake *handshake) @@ -451,35 +482,67 @@ static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], memzero_explicit(temp_hash, NOISE_HASH_LEN); } +/* TSWG: mix_kem mixes a KEM shared secret into the chaining key. */ +static void mix_kem(u8 chaining_key[NOISE_HASH_LEN], + const u8 ss[TSWG_MLKEM768_SS_LEN]) +{ + kdf(chaining_key, NULL, NULL, ss, NOISE_HASH_LEN, 0, 0, + TSWG_MLKEM768_SS_LEN, chaining_key); +} + static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], - const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) + const u8 remote_static[NOISE_PUBLIC_KEY_LEN], + bool hybrid) { - memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); - memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); + memcpy(hash, hybrid ? hybrid_init_hash : handshake_init_hash, + NOISE_HASH_LEN); + memcpy(chaining_key, hybrid ? hybrid_init_chaining_key : + handshake_init_chaining_key, + NOISE_HASH_LEN); mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); } +/* + * TSWG: the nonce is zero but for a hybrid initiation's static key; see + * the top of the file. dst and src may be the same. + */ +static void message_encrypt_nonce(u8 *dst_ciphertext, const u8 *src_plaintext, + size_t src_len, u64 nonce, + u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, + NOISE_HASH_LEN, nonce, key); + mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); +} + static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], u8 hash[NOISE_HASH_LEN]) { - chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, - NOISE_HASH_LEN, - 0 /* Always zero for Noise_IK */, key); - mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); + message_encrypt_nonce(dst_ciphertext, src_plaintext, src_len, 0, key, + hash); +} + +static bool message_decrypt_nonce(u8 *dst_plaintext, const u8 *src_ciphertext, + size_t src_len, u64 nonce, + u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, + hash, NOISE_HASH_LEN, nonce, key)) + return false; + mix_hash(hash, src_ciphertext, src_len); + return true; } static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], u8 hash[NOISE_HASH_LEN]) { - if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, - hash, NOISE_HASH_LEN, - 0 /* Always zero for Noise_IK */, key)) - return false; - mix_hash(hash, src_ciphertext, src_len); - return true; + return message_decrypt_nonce(dst_plaintext, src_ciphertext, src_len, 0, + key, hash); } static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], @@ -515,10 +578,12 @@ static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) bool wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + u8 *dst_kem, bool *hybrid, struct noise_handshake *handshake) { u8 timestamp[NOISE_TIMESTAMP_LEN]; u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 *dk = NULL; /* TSWG */ bool ret = false; /* We need to wait for crng _before_ taking any locks, since @@ -526,6 +591,18 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, */ wait_for_random_bytes(); + /* + * TSWG: a hybrid initiation's ML-KEM key pair, made before taking + * the locks. The encapsulation key goes straight into dst_kem, to + * be encrypted in place. + */ + *hybrid = READ_ONCE(handshake->hybrid); + if (*hybrid) { + dk = kmalloc(TSWG_MLKEM768_DK_LEN, GFP_KERNEL); + if (!dk || tswg_mlkem768_keygen(dst_kem, dk)) + goto out_free; + } + down_read(&handshake->static_identity->lock); down_write(&handshake->lock); @@ -535,7 +612,7 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); handshake_init(handshake->chaining_key, handshake->hash, - handshake->remote_static); + handshake->remote_static, *hybrid); /* e */ curve25519_generate_secret(handshake->ephemeral_private); @@ -551,10 +628,16 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, handshake->remote_static)) goto out; + /* TSWG: {ekem} */ + if (*hybrid) + message_encrypt(dst_kem, dst_kem, TSWG_MLKEM768_EK_LEN, key, + handshake->hash); + /* s */ - message_encrypt(dst->encrypted_static, - handshake->static_identity->static_public, - NOISE_PUBLIC_KEY_LEN, key, handshake->hash); + message_encrypt_nonce(dst->encrypted_static, + handshake->static_identity->static_public, + NOISE_PUBLIC_KEY_LEN, *hybrid ? 1 : 0, key, + handshake->hash); /* ss */ if (!mix_precomputed_dh(handshake->chaining_key, key, @@ -571,18 +654,21 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, &handshake->entry); handshake->state = HANDSHAKE_CREATED_INITIATION; + swap(handshake->kem, dk); /* TSWG: any old one is freed below */ ret = true; out: up_write(&handshake->lock); up_read(&handshake->static_identity->lock); memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); +out_free: + kfree_sensitive(dk); return ret; } struct wg_peer * wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, - struct wg_device *wg) + const u8 *src_kem, struct wg_device *wg) { struct wg_peer *peer = NULL, *ret_peer = NULL; struct noise_handshake *handshake; @@ -594,12 +680,20 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, u8 e[NOISE_PUBLIC_KEY_LEN]; u8 t[NOISE_TIMESTAMP_LEN]; u64 initiation_consumption; + u8 *ek = NULL; /* TSWG: the peer's ekem */ + + if (src_kem) { + ek = kmalloc(TSWG_MLKEM768_DK_LEN, GFP_KERNEL); + if (!ek) + return NULL; + } down_read(&wg->static_identity.lock); if (unlikely(!wg->static_identity.has_identity)) goto out; - handshake_init(chaining_key, hash, wg->static_identity.static_public); + handshake_init(chaining_key, hash, wg->static_identity.static_public, + src_kem); /* e */ message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); @@ -608,9 +702,16 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) goto out; + /* TSWG: {ekem} */ + if (src_kem && !message_decrypt(ek, src_kem, + noise_encrypted_len(TSWG_MLKEM768_EK_LEN), + key, hash)) + goto out; + /* s */ - if (!message_decrypt(s, src->encrypted_static, - sizeof(src->encrypted_static), key, hash)) + if (!message_decrypt_nonce(s, src->encrypted_static, + sizeof(src->encrypted_static), + src_kem ? 1 : 0, key, hash)) goto out; /* Lookup which peer we're actually talking to */ @@ -619,6 +720,10 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, goto out; handshake = &peer->handshake; + /* TSWG: the peer must use the kind of handshake we expect. */ + if (!!src_kem != READ_ONCE(handshake->hybrid)) + goto out; + /* ss */ if (!mix_precomputed_dh(chaining_key, key, handshake->precomputed_static_static)) @@ -651,6 +756,7 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, if ((s64)(handshake->last_initiation_consumption - initiation_consumption) < 0) handshake->last_initiation_consumption = initiation_consumption; handshake->state = HANDSHAKE_CONSUMED_INITIATION; + swap(handshake->kem, ek); /* TSWG: any old one is freed below */ up_write(&handshake->lock); ret_peer = peer; @@ -661,13 +767,16 @@ out: up_read(&wg->static_identity.lock); if (!ret_peer) wg_peer_put(peer); + kfree_sensitive(ek); return ret_peer; } bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + u8 *dst_kem, bool *hybrid, struct noise_handshake *handshake) { u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 kem_ss[TSWG_MLKEM768_SS_LEN]; /* TSWG */ bool ret = false; /* We need to wait for crng _before_ taking any locks, since @@ -680,6 +789,8 @@ bool wg_noise_handshake_create_response(struct message_handshake_response *dst, if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) goto out; + /* TSWG: hybrid if the initiation we consumed was. */ + *hybrid = !!handshake->kem; dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); dst->receiver_index = handshake->remote_index; @@ -693,11 +804,24 @@ bool wg_noise_handshake_create_response(struct message_handshake_response *dst, dst->unencrypted_ephemeral, handshake->chaining_key, handshake->hash); - /* ee */ - if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, - handshake->remote_ephemeral)) + /* ee (TSWG: hybrid handshakes keep its key, for the ciphertext) */ + if (!mix_dh(handshake->chaining_key, *hybrid ? key : NULL, + handshake->ephemeral_private, handshake->remote_ephemeral)) goto out; + /* + * TSWG: {ekem ciphertext}, kem. Encapsulate to the initiator's + * ekem, with the ciphertext going straight into dst_kem, to be + * encrypted in place. + */ + if (*hybrid) { + if (tswg_mlkem768_encaps(dst_kem, kem_ss, handshake->kem)) + goto out; + message_encrypt(dst_kem, dst_kem, TSWG_MLKEM768_CT_LEN, key, + handshake->hash); + mix_kem(handshake->chaining_key, kem_ss); + } + /* se */ if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, handshake->remote_static)) @@ -721,12 +845,13 @@ out: up_write(&handshake->lock); up_read(&handshake->static_identity->lock); memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(kem_ss, TSWG_MLKEM768_SS_LEN); return ret; } struct wg_peer * wg_noise_handshake_consume_response(struct message_handshake_response *src, - struct wg_device *wg) + const u8 *src_kem, struct wg_device *wg) { enum noise_handshake_state state = HANDSHAKE_ZEROED; struct wg_peer *peer = NULL, *ret_peer = NULL; @@ -737,6 +862,20 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src, u8 e[NOISE_PUBLIC_KEY_LEN]; u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; + /* + * TSWG: for a hybrid response, a copy of our ekem decapsulation key, + * followed by the ciphertext, and the KEM's shared secret. + */ + u8 kem_ss[TSWG_MLKEM768_SS_LEN]; + bool kem_ok = !src_kem; + u8 *kem = NULL; + + if (src_kem) { + kem = kmalloc(TSWG_MLKEM768_DK_LEN + TSWG_MLKEM768_CT_LEN, + GFP_KERNEL); + if (!kem) + return NULL; + } down_read(&wg->static_identity.lock); @@ -757,18 +896,42 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src, NOISE_PUBLIC_KEY_LEN); memcpy(preshared_key, handshake->preshared_key, NOISE_SYMMETRIC_KEY_LEN); + /* + * TSWG: the response must be hybrid if and only if our initiation + * was, and we still want hybrid handshakes. + */ + if (src_kem && handshake->kem) { + memcpy(kem, handshake->kem, TSWG_MLKEM768_DK_LEN); + kem_ok = true; + } else if (handshake->kem) { + kem_ok = false; + } + if (!!src_kem != READ_ONCE(handshake->hybrid)) + kem_ok = false; up_read(&handshake->lock); - if (state != HANDSHAKE_CREATED_INITIATION) + if (state != HANDSHAKE_CREATED_INITIATION || !kem_ok) goto fail; /* e */ message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); /* ee */ - if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + if (!mix_dh(chaining_key, src_kem ? key : NULL, ephemeral_private, e)) goto fail; + /* TSWG: {ekem ciphertext}, kem */ + if (src_kem) { + u8 *ct = kem + TSWG_MLKEM768_DK_LEN; + + if (!message_decrypt(ct, src_kem, + noise_encrypted_len(TSWG_MLKEM768_CT_LEN), + key, hash) || + tswg_mlkem768_decaps(kem_ss, ct, kem)) + goto fail; + mix_kem(chaining_key, kem_ss); + } + /* se */ if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) goto fail; @@ -796,6 +959,8 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src, memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); handshake->remote_index = src->sender_index; handshake->state = HANDSHAKE_CONSUMED_RESPONSE; + kfree_sensitive(handshake->kem); /* TSWG: done with ekem */ + handshake->kem = NULL; up_write(&handshake->lock); ret_peer = peer; goto out; @@ -808,10 +973,22 @@ out: memzero_explicit(chaining_key, NOISE_HASH_LEN); memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); memzero_explicit(preshared_key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(kem_ss, TSWG_MLKEM768_SS_LEN); up_read(&wg->static_identity.lock); + kfree_sensitive(kem); return ret_peer; } +/* + * TSWG: tswg_noise_set_hybrid sets whether handshakes with the peer are + * hybrid. As in wireguard-go, it applies from the next handshake; one in + * progress fails. + */ +void tswg_noise_set_hybrid(struct noise_handshake *handshake, bool hybrid) +{ + WRITE_ONCE(handshake->hybrid, hybrid); +} + bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, struct noise_keypairs *keypairs) { diff --git a/kernel/linux/src/noise.h b/kernel/linux/src/noise.h index c527253db..0b83644a1 100644 --- a/kernel/linux/src/noise.h +++ b/kernel/linux/src/noise.h @@ -83,6 +83,17 @@ struct noise_handshake { u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; __le32 remote_index; + /* TSWG: whether to do hybrid handshakes with this peer; see noise.c. */ + bool hybrid; + /* + * TSWG: in a hybrid handshake, our ML-KEM decapsulation key while + * we're the initiator (HANDSHAKE_CREATED_INITIATION), or the peer's + * encapsulation key while we're the responder + * (HANDSHAKE_CONSUMED_INITIATION); otherwise NULL. Allocated with + * TSWG_MLKEM768_DK_LEN bytes. + */ + u8 *kem; + /* Protects all members except the immutable (after noise_handshake_ * init): remote_static, precomputed_static_static, static_identity. */ @@ -116,18 +127,28 @@ void wg_noise_set_static_identity_private_key( const u8 private_key[NOISE_PUBLIC_KEY_LEN]); void wg_noise_precompute_static_static(struct wg_peer *peer); +/* + * TSWG: the handshake functions take the classic messages, and for hybrid + * handshakes, separately, the encrypted_kem field of the hybrid message + * (NULL when consuming a classic one). The create functions decide which + * kind of handshake it is, reporting it in *hybrid; send.c and receive.c + * convert between the classic messages and the hybrid ones. + */ bool wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + u8 *dst_kem, bool *hybrid, struct noise_handshake *handshake); struct wg_peer * wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, - struct wg_device *wg); + const u8 *src_kem, struct wg_device *wg); bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + u8 *dst_kem, bool *hybrid, struct noise_handshake *handshake); struct wg_peer * wg_noise_handshake_consume_response(struct message_handshake_response *src, - struct wg_device *wg); + const u8 *src_kem, struct wg_device *wg); +void tswg_noise_set_hybrid(struct noise_handshake *handshake, bool hybrid); bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, struct noise_keypairs *keypairs); diff --git a/kernel/linux/src/peer.c b/kernel/linux/src/peer.c index ed87a422e..26f6b0c99 100644 --- a/kernel/linux/src/peer.c +++ b/kernel/linux/src/peer.c @@ -193,6 +193,7 @@ static void rcu_release(struct rcu_head *rcu) /* The final zeroing takes care of clearing any remaining handshake key * material and other potentially sensitive information. */ + kfree_sensitive(peer->handshake.kem); /* TSWG */ memzero_explicit(peer, sizeof(*peer)); kmem_cache_free(peer_cache, peer); } diff --git a/kernel/linux/src/queueing.h b/kernel/linux/src/queueing.h index 42d47a3f2..4778a4c9f 100644 --- a/kernel/linux/src/queueing.h +++ b/kernel/linux/src/queueing.h @@ -38,7 +38,7 @@ void wg_packet_decrypt_worker(struct work_struct *work); /* send.c APIs: */ void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, bool is_retry); -void wg_packet_send_handshake_response(struct wg_peer *peer); +void wg_packet_send_handshake_response(struct wg_peer *peer, bool retry); void wg_packet_send_handshake_cookie(struct wg_device *wg, struct sk_buff *initiating_skb, __le32 sender_index); diff --git a/kernel/linux/src/receive.c b/kernel/linux/src/receive.c index 703e55cef..204b11729 100644 --- a/kernel/linux/src/receive.c +++ b/kernel/linux/src/receive.c @@ -28,6 +28,19 @@ static void update_rx_stats(struct wg_peer *peer, size_t len) #define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) +/* + * TSWG: tswg_skb_type is SKB_TYPE_LE32 without TSWG_MESSAGE_RETRY, for + * handshake messages, which may have it; see validate_header_len. + */ +static __le32 tswg_skb_type(struct sk_buff *skb) +{ + __le32 type = SKB_TYPE_LE32(skb); + + if (type == cpu_to_le32(MESSAGE_DATA)) + return type; + return type & ~cpu_to_le32(TSWG_MESSAGE_RETRY); +} + static size_t validate_header_len(struct sk_buff *skb) { if (unlikely(skb->len < sizeof(struct message_header))) @@ -41,6 +54,13 @@ static size_t validate_header_len(struct sk_buff *skb) if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && skb->len == sizeof(struct message_handshake_response)) return sizeof(struct message_handshake_response); + /* TSWG: hybrid handshake messages, perhaps with the retry bit. */ + if (tswg_skb_type(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && + skb->len == sizeof(struct message_handshake_initiation_hybrid)) + return sizeof(struct message_handshake_initiation_hybrid); + if (tswg_skb_type(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && + skb->len == sizeof(struct message_handshake_response_hybrid)) + return sizeof(struct message_handshake_response_hybrid); if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && skb->len == sizeof(struct message_handshake_cookie)) return sizeof(struct message_handshake_cookie); @@ -92,9 +112,46 @@ static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) return 0; } +/* + * TSWG: tswg_classic_initiation and tswg_classic_response copy a hybrid + * handshake message's fields but encrypted_kem into the classic message + * dst, for noise.c; see wg_noise_handshake_*. + */ +static void tswg_classic_initiation(struct message_handshake_initiation *dst, + const struct message_handshake_initiation_hybrid *src) +{ + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); + dst->sender_index = src->sender_index; + memcpy(dst->unencrypted_ephemeral, src->unencrypted_ephemeral, + sizeof(dst->unencrypted_ephemeral)); + memcpy(dst->encrypted_static, src->encrypted_static, + sizeof(dst->encrypted_static)); + memcpy(dst->encrypted_timestamp, src->encrypted_timestamp, + sizeof(dst->encrypted_timestamp)); + dst->macs = src->macs; +} + +static void tswg_classic_response(struct message_handshake_response *dst, + const struct message_handshake_response_hybrid *src) +{ + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); + dst->sender_index = src->sender_index; + dst->receiver_index = src->receiver_index; + memcpy(dst->unencrypted_ephemeral, src->unencrypted_ephemeral, + sizeof(dst->unencrypted_ephemeral)); + memcpy(dst->encrypted_nothing, src->encrypted_nothing, + sizeof(dst->encrypted_nothing)); + dst->macs = src->macs; +} + static void wg_receive_handshake_packet(struct wg_device *wg, struct sk_buff *skb) { + /* TSWG: hybrid handshake messages, converted; see above. */ + struct message_handshake_initiation classic_initiation; + struct message_handshake_response classic_response; + const u8 *kem = NULL; + bool retry; enum cookie_mac_state mac_state; struct wg_peer *peer = NULL; /* This is global, so that our load calculation applies to the whole @@ -134,17 +191,26 @@ static void wg_receive_handshake_packet(struct wg_device *wg, return; } - switch (SKB_TYPE_LE32(skb)) { + retry = SKB_TYPE_LE32(skb) & cpu_to_le32(TSWG_MESSAGE_RETRY); + switch (tswg_skb_type(skb)) { case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { struct message_handshake_initiation *message = (struct message_handshake_initiation *)skb->data; + if (skb->len == sizeof(struct message_handshake_initiation_hybrid)) { + struct message_handshake_initiation_hybrid *hyb = + (void *)skb->data; + + tswg_classic_initiation(&classic_initiation, hyb); + message = &classic_initiation; + kem = hyb->encrypted_kem; + } if (packet_needs_cookie) { wg_packet_send_handshake_cookie(wg, skb, message->sender_index); return; } - peer = wg_noise_handshake_consume_initiation(message, wg); + peer = wg_noise_handshake_consume_initiation(message, kem, wg); if (unlikely(!peer)) { net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", wg->dev->name, skb); @@ -154,19 +220,27 @@ static void wg_receive_handshake_packet(struct wg_device *wg, net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", wg->dev->name, peer->internal_id, &peer->endpoint.addr); - wg_packet_send_handshake_response(peer); + wg_packet_send_handshake_response(peer, retry); break; } case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { struct message_handshake_response *message = (struct message_handshake_response *)skb->data; + if (skb->len == sizeof(struct message_handshake_response_hybrid)) { + struct message_handshake_response_hybrid *hyb = + (void *)skb->data; + + tswg_classic_response(&classic_response, hyb); + message = &classic_response; + kem = hyb->encrypted_kem; + } if (packet_needs_cookie) { wg_packet_send_handshake_cookie(wg, skb, message->sender_index); return; } - peer = wg_noise_handshake_consume_response(message, wg); + peer = wg_noise_handshake_consume_response(message, kem, wg); if (unlikely(!peer)) { net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", wg->dev->name, skb); @@ -918,7 +992,7 @@ void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) { if (unlikely(prepare_skb_header(skb, wg) < 0)) goto err; - switch (SKB_TYPE_LE32(skb)) { + switch (tswg_skb_type(skb)) { /* TSWG */ case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { diff --git a/kernel/linux/src/send.c b/kernel/linux/src/send.c index 540c2f06a..3a5bfbc1d 100644 --- a/kernel/linux/src/send.c +++ b/kernel/linux/src/send.c @@ -19,9 +19,51 @@ #include #include +/* + * TSWG: tswg_hybrid_initiation and tswg_hybrid_response lay out a hybrid + * handshake message, of which noise.c filled in the encrypted_kem field + * and the classic message src, from src; see wg_noise_handshake_*. If + * retry, they set TSWG_MESSAGE_RETRY, as wireguard-go does. + */ +static void tswg_hybrid_initiation(struct message_handshake_initiation_hybrid *dst, + const struct message_handshake_initiation *src, + bool retry) +{ + dst->header.type = src->header.type; + if (retry) + dst->header.type |= cpu_to_le32(TSWG_MESSAGE_RETRY); + dst->sender_index = src->sender_index; + memcpy(dst->unencrypted_ephemeral, src->unencrypted_ephemeral, + sizeof(dst->unencrypted_ephemeral)); + memcpy(dst->encrypted_static, src->encrypted_static, + sizeof(dst->encrypted_static)); + memcpy(dst->encrypted_timestamp, src->encrypted_timestamp, + sizeof(dst->encrypted_timestamp)); +} + +static void tswg_hybrid_response(struct message_handshake_response_hybrid *dst, + const struct message_handshake_response *src, + bool retry) +{ + dst->header.type = src->header.type; + if (retry) + dst->header.type |= cpu_to_le32(TSWG_MESSAGE_RETRY); + dst->sender_index = src->sender_index; + dst->receiver_index = src->receiver_index; + memcpy(dst->unencrypted_ephemeral, src->unencrypted_ephemeral, + sizeof(dst->unencrypted_ephemeral)); + memcpy(dst->encrypted_nothing, src->encrypted_nothing, + sizeof(dst->encrypted_nothing)); +} + static void wg_packet_send_handshake_initiation(struct wg_peer *peer) { struct message_handshake_initiation packet; + /* TSWG: the message sent, which for hybrid handshakes is in hyb. */ + struct message_handshake_initiation_hybrid *hyb; + void *msg = &packet; + size_t len = sizeof(packet); + bool hybrid; if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), REKEY_TIMEOUT)) @@ -32,16 +74,27 @@ static void wg_packet_send_handshake_initiation(struct wg_peer *peer) peer->device->dev->name, peer->internal_id, &peer->endpoint.addr); - if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { - wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + hyb = kmalloc(sizeof(*hyb), GFP_KERNEL); + if (!hyb) + return; + if (wg_noise_handshake_create_initiation(&packet, hyb->encrypted_kem, + &hybrid, &peer->handshake)) { + if (hybrid) { + /* timer_handshake_attempts counts retries. */ + tswg_hybrid_initiation(hyb, &packet, + peer->timer_handshake_attempts); + msg = hyb; + len = sizeof(*hyb); + } + wg_cookie_add_mac_to_packet(msg, len, peer); wg_timers_any_authenticated_packet_traversal(peer); wg_timers_any_authenticated_packet_sent(peer); atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); - wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), - HANDSHAKE_DSCP); + wg_socket_send_buffer_to_peer(peer, msg, len, HANDSHAKE_DSCP); wg_timers_handshake_initiated(peer); } + kfree(hyb); } void wg_packet_handshake_send_worker(struct work_struct *work) @@ -83,17 +136,31 @@ out: rcu_read_unlock_bh(); } -void wg_packet_send_handshake_response(struct wg_peer *peer) +void wg_packet_send_handshake_response(struct wg_peer *peer, bool retry) { struct message_handshake_response packet; + /* TSWG: as in wg_packet_send_handshake_initiation. */ + struct message_handshake_response_hybrid *hyb; + void *msg = &packet; + size_t len = sizeof(packet); + bool hybrid; atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", peer->device->dev->name, peer->internal_id, &peer->endpoint.addr); - if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { - wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + hyb = kmalloc(sizeof(*hyb), GFP_KERNEL); + if (!hyb) + return; + if (wg_noise_handshake_create_response(&packet, hyb->encrypted_kem, + &hybrid, &peer->handshake)) { + if (hybrid) { + tswg_hybrid_response(hyb, &packet, retry); + msg = hyb; + len = sizeof(*hyb); + } + wg_cookie_add_mac_to_packet(msg, len, peer); if (wg_noise_handshake_begin_session(&peer->handshake, &peer->keypairs)) { wg_timers_session_derived(peer); @@ -101,11 +168,11 @@ void wg_packet_send_handshake_response(struct wg_peer *peer) wg_timers_any_authenticated_packet_sent(peer); atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); - wg_socket_send_buffer_to_peer(peer, &packet, - sizeof(packet), + wg_socket_send_buffer_to_peer(peer, msg, len, HANDSHAKE_DSCP); } } + kfree(hyb); } void wg_packet_send_handshake_cookie(struct wg_device *wg, diff --git a/kernel/linux/src/socket.c b/kernel/linux/src/socket.c index 2aa819605..f2effea39 100644 --- a/kernel/linux/src/socket.c +++ b/kernel/linux/src/socket.c @@ -342,7 +342,7 @@ void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) * a WireGuard message: a little-endian 32-bit type of 1 through 4. Nothing * else Tailscale sends on the socket matches: disco starts with "TS", STUN * with 0x00 or 0x01 followed by a nonzero byte, and Geneve-encapsulated peer - * relay traffic with 0x00. + * relay traffic with 0x00. Handshake messages may have TSWG_MESSAGE_RETRY. */ static bool tswg_is_wg_message(struct sk_buff *skb) { @@ -354,7 +354,9 @@ static bool tswg_is_wg_message(struct sk_buff *skb) if (!h) return false; return h[0] >= MESSAGE_HANDSHAKE_INITIATION && h[0] <= MESSAGE_DATA && - !h[1] && !h[2] && !h[3]; + !h[1] && !h[2] && + (!h[3] || (h[3] == TSWG_MESSAGE_RETRY >> 24 && + h[0] <= MESSAGE_HANDSHAKE_RESPONSE)); } /* diff --git a/kernel/linux/tests/mlkem/mlkem_test.go b/kernel/linux/tests/mlkem/mlkem_test.go new file mode 100644 index 000000000..c2d28a6c5 --- /dev/null +++ b/kernel/linux/tests/mlkem/mlkem_test.go @@ -0,0 +1,171 @@ +// Package mlkem checks tswg's ML-KEM-768 (src/mlkem.c) against Go's +// crypto/mlkem, which Tailscale's wireguard-go uses for its hybrid +// handshakes. It builds src/mlkem.c for userspace (with the stand-in +// kernel headers in testdata/shim) into a driver it talks to over a pipe. +// +// Run it with "go test" in this directory; it needs a C compiler. +package mlkem + +import ( + "bufio" + "bytes" + "crypto/mlkem" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// driver is a running driver.c. +type driver struct { + t *testing.T + in io.Writer + out *bufio.Scanner +} + +func startDriver(t *testing.T) *driver { + bin := filepath.Join(t.TempDir(), "driver") + cc := exec.Command("cc", "-O2", "-std=gnu11", "-Wall", "-Wno-unused-function", + "-Itestdata/shim", "-o", bin, "testdata/driver.c") + if out, err := cc.CombinedOutput(); err != nil { + t.Fatalf("building driver: %v\n%s", err, out) + } + cmd := exec.Command(bin) + in, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + in.Close() + cmd.Wait() + }) + sc := bufio.NewScanner(out) + sc.Buffer(nil, 1<<20) + return &driver{t: t, in: in, out: sc} +} + +// call runs a driver command, returning its results, or nil for "err". +func (d *driver) call(cmd string, args ...[]byte) [][]byte { + d.t.Helper() + line := cmd + for _, a := range args { + line += " " + hex.EncodeToString(a) + } + if _, err := fmt.Fprintln(d.in, line); err != nil { + d.t.Fatal(err) + } + if !d.out.Scan() { + d.t.Fatalf("driver: %v", d.out.Err()) + } + if d.out.Text() == "err" { + return nil + } + var res [][]byte + for _, f := range strings.Fields(d.out.Text()) { + b, err := hex.DecodeString(f) + if err != nil { + d.t.Fatal(err) + } + res = append(res, b) + } + return res +} + +const iterations = 200 + +func TestMLKEM768(t *testing.T) { + d := startDriver(t) + for i := range iterations { + seed := make([]byte, mlkem.SeedSize) + rand.Read(seed) + goDK, err := mlkem.NewDecapsulationKey768(seed) + if err != nil { + t.Fatal(err) + } + goEK := goDK.EncapsulationKey() + + // Key generation from the same seed gives the same public key. + r := d.call("keygen", seed) + if r == nil { + t.Fatal("keygen failed") + } + ek, dk := r[0], r[1] + if !bytes.Equal(ek, goEK.Bytes()) { + t.Fatalf("%d: encapsulation key differs from Go's", i) + } + + // Go encapsulates, we decapsulate. + goSS, goCT := goEK.Encapsulate() + r = d.call("decaps", dk, goCT) + if r == nil || !bytes.Equal(r[0], goSS) { + t.Fatalf("%d: decapsulating Go's ciphertext: got %x, want %x", i, r, goSS) + } + + // We encapsulate, Go decapsulates. + r = d.call("encaps", ek) + if r == nil { + t.Fatalf("%d: encaps failed", i) + } + ct, ss := r[0], r[1] + want, err := goDK.Decapsulate(ct) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(ss, want) { + t.Fatalf("%d: Go decapsulates our ciphertext to %x, we said %x", i, want, ss) + } + + // A corrupted ciphertext is implicitly rejected, the same way. + bad := bytes.Clone(goCT) + bad[i%len(bad)] ^= 1 << (i % 8) + want, err = goDK.Decapsulate(bad) + if err != nil { + t.Fatal(err) + } + r = d.call("decaps", dk, bad) + if r == nil || !bytes.Equal(r[0], want) { + t.Fatalf("%d: implicit rejection: got %x, want %x", i, r, want) + } + if bytes.Equal(want, goSS) { + t.Fatalf("%d: corrupted ciphertext gave the real shared secret", i) + } + } +} + +// TestInvalidEncapsulationKey checks that encapsulation rejects keys whose +// coefficients aren't reduced mod q, as Go does. +func TestInvalidEncapsulationKey(t *testing.T) { + d := startDriver(t) + seed := make([]byte, mlkem.SeedSize) + rand.Read(seed) + r := d.call("keygen", seed) + if r == nil { + t.Fatal("keygen failed") + } + for _, off := range []int{0, 384, 1149} { + ek := bytes.Clone(r[0]) + // Set a 12-bit coefficient starting at byte off (even + // coefficients start on byte boundaries) to 4095. + ek[off] = 0xff + ek[off+1] |= 0x0f + if _, err := mlkem.NewEncapsulationKey768(ek); err == nil { + t.Fatalf("Go accepted the key corrupted at %d", off) + } + if d.call("encaps", ek) != nil { + t.Errorf("encaps accepted the key corrupted at %d", off) + } + } +} diff --git a/kernel/linux/tests/mlkem/testdata/driver.c b/kernel/linux/tests/mlkem/testdata/driver.c new file mode 100644 index 000000000..09e1280f7 --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/driver.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * driver runs src/mlkem.c in userspace for mlkem_test.go. It reads + * commands, one per line, with arguments in hex, and writes each result + * as a line of space-separated hex, or "err": + * + * keygen SEED -> EK DK + * encaps EK -> CT SS + * decaps DK CT -> SS + */ + +#include +#include +#include + +#include "../../../src/mlkem.c" + +static int unhex(u8 *dst, size_t n, const char *s) +{ + size_t i; + + if (strlen(s) != 2 * n) + return -1; + for (i = 0; i < n; i++) + if (sscanf(s + 2 * i, "%2hhx", &dst[i]) != 1) + return -1; + return 0; +} + +static void puthex(const u8 *p, size_t n) +{ + while (n--) + printf("%02x", *p++); +} + +int main(void) +{ + static u8 ek[TSWG_MLKEM768_EK_LEN], dk[TSWG_MLKEM768_DK_LEN]; + static u8 ct[TSWG_MLKEM768_CT_LEN], ss[TSWG_MLKEM768_SS_LEN]; + static u8 seed[TSWG_MLKEM768_SEED_LEN]; + static char line[16384]; + char *cmd, *a, *b; + + setvbuf(stdout, NULL, _IOLBF, 0); + tswg_mlkem_init(); + while (fgets(line, sizeof(line), stdin)) { + line[strcspn(line, "\n")] = 0; + cmd = strtok(line, " "); + a = strtok(NULL, " "); + b = strtok(NULL, " "); + if (!cmd || !a) { + puts("err"); + } else if (!strcmp(cmd, "keygen")) { + if (unhex(seed, sizeof(seed), a) || + tswg_mlkem768_keygen_seed(ek, dk, seed)) { + puts("err"); + continue; + } + puthex(ek, sizeof(ek)); + putchar(' '); + puthex(dk, sizeof(dk)); + putchar('\n'); + } else if (!strcmp(cmd, "encaps")) { + if (unhex(ek, sizeof(ek), a) || + tswg_mlkem768_encaps(ct, ss, ek)) { + puts("err"); + continue; + } + puthex(ct, sizeof(ct)); + putchar(' '); + puthex(ss, sizeof(ss)); + putchar('\n'); + } else if (!strcmp(cmd, "decaps") && b) { + if (unhex(dk, sizeof(dk), a) || unhex(ct, sizeof(ct), b) || + tswg_mlkem768_decaps(ss, ct, dk)) { + puts("err"); + continue; + } + puthex(ss, sizeof(ss)); + putchar('\n'); + } else { + puts("err"); + } + } + return 0; +} diff --git a/kernel/linux/tests/mlkem/testdata/shim/crypto/utils.h b/kernel/linux/tests/mlkem/testdata/shim/crypto/utils.h new file mode 100644 index 000000000..91d1e5166 --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/crypto/utils.h @@ -0,0 +1,9 @@ +#include +static inline unsigned long crypto_memneq(const void *a, const void *b, size_t n) +{ + const u8 *x = a, *y = b; + unsigned long r = 0; + while (n--) + r |= *x++ ^ *y++; + return r; +} diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/bitops.h b/kernel/linux/tests/mlkem/testdata/shim/linux/bitops.h new file mode 100644 index 000000000..08bcca44c --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/bitops.h @@ -0,0 +1,2 @@ +#include +static inline u64 rol64(u64 w, unsigned int s) { return w << s | w >> ((64 - s) & 63); } diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/cache.h b/kernel/linux/tests/mlkem/testdata/shim/linux/cache.h new file mode 100644 index 000000000..f8b9341ab --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/cache.h @@ -0,0 +1 @@ +#define __ro_after_init diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/errno.h b/kernel/linux/tests/mlkem/testdata/shim/linux/errno.h new file mode 100644 index 000000000..70f2bd34e --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/errno.h @@ -0,0 +1 @@ +#include diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/random.h b/kernel/linux/tests/mlkem/testdata/shim/linux/random.h new file mode 100644 index 000000000..97e137273 --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/random.h @@ -0,0 +1,7 @@ +#include +#include +static inline void get_random_bytes(void *p, size_t n) +{ + if (getrandom(p, n, 0) != (ssize_t)n) + abort(); +} diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/slab.h b/kernel/linux/tests/mlkem/testdata/shim/linux/slab.h new file mode 100644 index 000000000..dec06ba53 --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/slab.h @@ -0,0 +1,4 @@ +#include +#define GFP_KERNEL 0 +static inline void *kmalloc(size_t n, int flags) { return malloc(n); } +static inline void kfree_sensitive(void *p) { free(p); } diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/string.h b/kernel/linux/tests/mlkem/testdata/shim/linux/string.h new file mode 100644 index 000000000..9209d16b5 --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/string.h @@ -0,0 +1,2 @@ +#include +static inline void memzero_explicit(void *p, size_t n) { explicit_bzero(p, n); } diff --git a/kernel/linux/tests/mlkem/testdata/shim/linux/types.h b/kernel/linux/tests/mlkem/testdata/shim/linux/types.h new file mode 100644 index 000000000..58da7604a --- /dev/null +++ b/kernel/linux/tests/mlkem/testdata/shim/linux/types.h @@ -0,0 +1,8 @@ +/* Userspace stand-ins for the kernel headers mlkem.c uses; see mlkem_test.go. */ +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64;