Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7cd85e085c | ||
|
|
ccd0e7af78 |
No files matched your search
@@ -26,7 +26,29 @@ When an AI agent or developer is executing CLI commands via tools in this projec
|
||||
- **Rule**: NEVER use the `&&` operator to chain commands in PowerShell. It will result in parser errors.
|
||||
- **Resolution**: Use sequential execution, run commands separately, or use PowerShell-safe chaining like `; if ($?) { }`.
|
||||
|
||||
## 5. Next.js vs. Undici Fetch Type Mismatches
|
||||
## 5. UTF-8 BOM in `package.json` Crashing Node Scripts
|
||||
If `pnpm dev` (or any workspace script) dies with `SyntaxError: Unexpected token '', "{ ..." is not valid JSON`, the offending file has a UTF-8 BOM (`EF BB BF`) at byte 0.
|
||||
- **Root Cause**: Windows PowerShell 5.1 `Set-Content -Encoding UTF8` and Notepad's "Save as UTF-8" both emit a BOM. Node's `fs.readFileSync(p, 'utf-8')` does NOT strip it, so `JSON.parse` rejects the leading ``. pnpm/npm strip BOMs silently, which is why this only blows up in custom workspace scripts.
|
||||
- **Diagnosis** (PowerShell):
|
||||
```powershell
|
||||
Get-ChildItem local-plugins -Directory | ForEach-Object {
|
||||
$p = Join-Path $_.FullName 'package.json'
|
||||
if (Test-Path $p) {
|
||||
$b = [System.IO.File]::ReadAllBytes($p) | Select-Object -First 3
|
||||
if ($b[0] -eq 0xEF -and $b[1] -eq 0xBB -and $b[2] -eq 0xBF) { $_.Name }
|
||||
}
|
||||
}
|
||||
```
|
||||
- **Fix at the seam (preferred)**: In any script that parses externally-authored JSON, strip the BOM before parsing. See `readJsonFile()` in [`scripts/sync-local-plugins.mjs`](../../scripts/sync-local-plugins.mjs) as the canonical helper:
|
||||
```js
|
||||
let text = fs.readFileSync(filePath, "utf-8");
|
||||
if (text.charCodeAt(0) === 0xFEFF) text = text.slice(1);
|
||||
return JSON.parse(text);
|
||||
```
|
||||
- **Clean existing files** (one-time): rewrite each file without its first 3 bytes via `[System.IO.File]::WriteAllBytes($p, $bytes[3..($bytes.Length - 1)])`.
|
||||
- **Prevention**: When authoring files via PowerShell, use `Out-File -Encoding utf8NoBOM` (PS 7+) or `[System.IO.File]::WriteAllText($p, $text, [System.Text.UTF8Encoding]::new($false))` (any PS). Never edit `package.json` in Notepad.
|
||||
|
||||
## 6. Next.js vs. Undici Fetch Type Mismatches
|
||||
When implementing SSRF protections (e.g., DNS IP Pinning via custom `Dispatcher`), you will likely use `undici.Agent`.
|
||||
- **Root Cause**: Passing global Next.js `RequestInit` options directly into `undici.fetch` will throw strict TypeScript errors because the `BodyInit` and `RequestCache` signatures differ, and `dns.lookup` returns a generic `number` instead of exactly `4 | 6` for IP families.
|
||||
- **Resolution**: Strip out Next.js specific keys, explicitly cast `body` as `any`, and explicitly type `resolvedFamily: number` when interfacing between Next.js request streams and the `undici` library.
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "worldwideview",
|
||||
"version": "2.16.0",
|
||||
"version": "2.16.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "[email protected]",
|
||||
|
||||
Generated
+1112
-6
File diff suppressed because it is too large.
Load diff
+120
-10
@@ -7,6 +7,7 @@
|
||||
*/
|
||||
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { build } from "vite";
|
||||
import { fileURLToPath } from "url";
|
||||
@@ -73,9 +74,25 @@ function wwvHostRedirectPlugin() {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @function readJsonFile
|
||||
* @description Reads a UTF-8 JSON file, tolerating a leading BOM (U+FEFF).
|
||||
* Node's `fs.readFileSync(p, 'utf-8')` preserves the BOM byte, which `JSON.parse`
|
||||
* then rejects with "Unexpected token". Scaffolders and editors on Windows
|
||||
* occasionally emit BOMs into `package.json`; pnpm/npm strip them silently,
|
||||
* so we match that behavior here to avoid breaking dev on otherwise-valid files.
|
||||
* @param {string} filePath
|
||||
* @returns {any}
|
||||
*/
|
||||
function readJsonFile(filePath) {
|
||||
let text = fs.readFileSync(filePath, "utf-8");
|
||||
if (text.charCodeAt(0) === 0xFEFF) text = text.slice(1);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* @function discoverLocalPlugins
|
||||
* @description Finds all directories in `local-plugins/` that contain a valid
|
||||
* @description Finds all directories in `local-plugins/` that contain a valid
|
||||
* `package.json` with a `worldwideview` manifest block.
|
||||
* @returns {Array<{dir: string, manifest: any, pluginDir: string}>}
|
||||
*/
|
||||
@@ -87,12 +104,12 @@ export function discoverLocalPlugins() {
|
||||
if (dir.startsWith(".")) return false;
|
||||
const pkgPath = path.join(LOCAL_PLUGINS_DIR, dir, "package.json");
|
||||
if (!fs.existsSync(pkgPath)) return false;
|
||||
const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
|
||||
const pkg = readJsonFile(pkgPath);
|
||||
return !!pkg.worldwideview;
|
||||
})
|
||||
.map(dir => {
|
||||
const pkgPath = path.join(LOCAL_PLUGINS_DIR, dir, "package.json");
|
||||
const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
|
||||
const pkg = readJsonFile(pkgPath);
|
||||
const manifest = pkg.worldwideview;
|
||||
manifest.version = pkg.version;
|
||||
manifest.name = pkg.name;
|
||||
@@ -140,12 +157,71 @@ function resolvePluginEntry(pluginDir, manifest, pkg) {
|
||||
return candidates.find(f => fs.existsSync(f)) ?? null;
|
||||
}
|
||||
|
||||
const SKIP_DIRS = new Set(["dist", "node_modules", ".git", ".turbo", ".vite"]);
|
||||
|
||||
/**
|
||||
* @function latestSourceMtime
|
||||
* @description Walks a plugin directory and returns the newest mtime (ms) across
|
||||
* source files. Used to decide whether a cached `dist/frontend.mjs` is still valid.
|
||||
* Skips build/output dirs so this is fast even on cold disks.
|
||||
* @param {string} rootDir
|
||||
* @returns {number} newest mtime in ms, or 0 if no files found
|
||||
*/
|
||||
function latestSourceMtime(rootDir) {
|
||||
let newest = 0;
|
||||
/** @param {string} dir */
|
||||
function walk(dir) {
|
||||
let entries;
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith(".") && entry.name !== ".env") continue;
|
||||
if (SKIP_DIRS.has(entry.name)) continue;
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(full);
|
||||
} else if (entry.isFile()) {
|
||||
const m = fs.statSync(full).mtimeMs;
|
||||
if (m > newest) newest = m;
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(rootDir);
|
||||
return newest;
|
||||
}
|
||||
|
||||
/**
|
||||
* @function isBuildFresh
|
||||
* @description Returns true if the plugin's existing `dist/frontend.mjs` is newer
|
||||
* than every source file under its directory — meaning Vite would produce a
|
||||
* byte-identical bundle and the build can be skipped.
|
||||
* @param {string} pluginDir
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isBuildFresh(pluginDir) {
|
||||
const distFile = path.join(pluginDir, "dist", "frontend.mjs");
|
||||
if (!fs.existsSync(distFile)) return false;
|
||||
const distMtime = fs.statSync(distFile).mtimeMs;
|
||||
const sourceMtime = latestSourceMtime(pluginDir);
|
||||
return distMtime >= sourceMtime;
|
||||
}
|
||||
|
||||
/**
|
||||
* @function buildPlugin
|
||||
* @description Invokes the Vite build engine to compile a plugin's source
|
||||
* into a single ES module, externalizing shared host dependencies.
|
||||
*
|
||||
* Returns `"fresh"` if an up-to-date `dist/frontend.mjs` already exists (no build run),
|
||||
* `true` if Vite produced a new bundle, or `false` on failure.
|
||||
*/
|
||||
export async function buildPlugin({ dir, manifest, pkg, pluginDir }) {
|
||||
if (isBuildFresh(pluginDir)) {
|
||||
return "fresh";
|
||||
}
|
||||
|
||||
const entryFile = resolvePluginEntry(pluginDir, manifest, pkg ?? {});
|
||||
|
||||
if (!entryFile) {
|
||||
@@ -200,7 +276,7 @@ export async function buildPlugin({ dir, manifest, pkg, pluginDir }) {
|
||||
}
|
||||
}
|
||||
|
||||
export function syncToPublic({ dir, manifest, pluginDir }) {
|
||||
export function syncToPublic({ dir, manifest, pluginDir }, { quiet = false } = {}) {
|
||||
const publicName = manifest.id || dir.replace("wwv-plugin-", "");
|
||||
const targetDir = path.join(OUTPUT_DIR, publicName);
|
||||
const distFile = path.join(pluginDir, "dist", "frontend.mjs");
|
||||
@@ -237,7 +313,7 @@ export function syncToPublic({ dir, manifest, pluginDir }) {
|
||||
JSON.stringify(pluginJson, null, 2)
|
||||
);
|
||||
|
||||
console.log(`[sync] ✅ ${publicName} → public/plugins-local/${publicName}/`);
|
||||
if (!quiet) console.log(`[sync] ✅ ${publicName} → public/plugins-local/${publicName}/`);
|
||||
}
|
||||
|
||||
// Clean stale plugins from public/plugins-local/ that no longer exist in local-plugins/
|
||||
@@ -252,6 +328,32 @@ function cleanStale(activeIds) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @function runWithConcurrency
|
||||
* @description Runs an async worker over `items` with a fixed concurrency cap.
|
||||
* Vite builds are CPU-bound; running all ~30 in parallel thrashes the scheduler
|
||||
* and balloons RSS. A cap around half the logical cores keeps wall-clock low
|
||||
* without starving the rest of the dev environment (Next.js, the data engine).
|
||||
* @template T, R
|
||||
* @param {T[]} items
|
||||
* @param {number} concurrency
|
||||
* @param {(item: T) => Promise<R>} worker
|
||||
* @returns {Promise<R[]>}
|
||||
*/
|
||||
async function runWithConcurrency(items, concurrency, worker) {
|
||||
const results = new Array(items.length);
|
||||
let cursor = 0;
|
||||
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
||||
while (true) {
|
||||
const i = cursor++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await worker(items[i]);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function syncAll() {
|
||||
const plugins = discoverLocalPlugins();
|
||||
|
||||
@@ -261,12 +363,20 @@ export async function syncAll() {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[sync] Found ${plugins.length} local plugin(s): ${plugins.map(p => p.dir).join(", ")}`);
|
||||
const concurrency = Math.max(2, Math.min(plugins.length, Math.ceil(os.cpus().length / 2)));
|
||||
console.log(`[sync] Found ${plugins.length} local plugin(s); building with concurrency=${concurrency}`);
|
||||
const started = Date.now();
|
||||
|
||||
for (const plugin of plugins) {
|
||||
const ok = await buildPlugin(plugin);
|
||||
if (ok) syncToPublic(plugin);
|
||||
}
|
||||
let freshCount = 0;
|
||||
let builtCount = 0;
|
||||
await runWithConcurrency(plugins, concurrency, async (plugin) => {
|
||||
const result = await buildPlugin(plugin);
|
||||
if (result === "fresh") freshCount++;
|
||||
else if (result) builtCount++;
|
||||
if (result) syncToPublic(plugin, { quiet: result === "fresh" });
|
||||
});
|
||||
|
||||
console.log(`[sync] Done in ${((Date.now() - started) / 1000).toFixed(1)}s (${builtCount} built, ${freshCount} cached)`);
|
||||
|
||||
const activeIds = plugins.map(p => p.manifest.id || p.dir.replace("wwv-plugin-", ""));
|
||||
cleanStale(activeIds);
|
||||
|
||||
@@ -15,40 +15,66 @@ if (!fs.existsSync(LOCAL_PLUGINS_DIR)) {
|
||||
let debounceTimeout = null;
|
||||
let isSyncing = false;
|
||||
let pendingSync = false;
|
||||
/** Files changed since last sync completed — collected during the debounce window */
|
||||
const pendingChanges = new Set();
|
||||
|
||||
function handleFileChange(eventType, filename) {
|
||||
if (filename && (
|
||||
filename.includes("dist") ||
|
||||
filename.includes("node_modules") ||
|
||||
if (!filename) return;
|
||||
if (
|
||||
filename.includes("dist") ||
|
||||
filename.includes("node_modules") ||
|
||||
filename.endsWith(".map") ||
|
||||
filename.includes(".git")
|
||||
)) return; // ignore build artifacts and git internals
|
||||
|
||||
) return; // ignore build artifacts and git internals
|
||||
|
||||
pendingChanges.add(filename);
|
||||
|
||||
if (debounceTimeout) {
|
||||
clearTimeout(debounceTimeout);
|
||||
}
|
||||
|
||||
|
||||
debounceTimeout = setTimeout(async () => {
|
||||
if (isSyncing) {
|
||||
pendingSync = true;
|
||||
return;
|
||||
}
|
||||
await runSync(filename);
|
||||
await runSync();
|
||||
}, 500);
|
||||
}
|
||||
|
||||
async function runSync(filename) {
|
||||
/**
|
||||
* Pulls plugin slugs out of changed file paths so we can name what triggered the rebuild.
|
||||
* fs.watch on `local-plugins/` emits paths like `aviation/src/index.ts` (POSIX or Windows).
|
||||
*/
|
||||
function summarizeChangedPlugins(files) {
|
||||
const slugs = new Set();
|
||||
for (const f of files) {
|
||||
const head = f.split(/[\\/]/)[0];
|
||||
if (head) slugs.add(head);
|
||||
}
|
||||
return [...slugs];
|
||||
}
|
||||
|
||||
async function runSync() {
|
||||
isSyncing = true;
|
||||
console.log(`\n[watch] Change detected in local plugins (file: ${filename}). Syncing...`);
|
||||
const changed = summarizeChangedPlugins(pendingChanges);
|
||||
pendingChanges.clear();
|
||||
const label = changed.length === 0
|
||||
? "pending changes"
|
||||
: changed.length <= 3
|
||||
? changed.join(", ")
|
||||
: `${changed.slice(0, 3).join(", ")} (+${changed.length - 3} more)`;
|
||||
console.log(`\n[watch] 🔄 Change detected in: ${label}`);
|
||||
try {
|
||||
await syncAll();
|
||||
console.log(`[watch] ✨ Hot-reload ready`);
|
||||
} catch (err) {
|
||||
console.error(`[watch] Sync failed:`, err);
|
||||
} finally {
|
||||
isSyncing = false;
|
||||
if (pendingSync) {
|
||||
pendingSync = false;
|
||||
await runSync("pending changes");
|
||||
await runSync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,8 @@ describe("WsClient", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
// Pin Math.random to 0 so jitter is deterministic (delay = base only)
|
||||
vi.spyOn(Math, "random").mockReturnValue(0);
|
||||
|
||||
// Mock WebSocket global
|
||||
mockWs = {
|
||||
@@ -85,7 +87,7 @@ describe("WsClient", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("should attempt to reconnect after 5s if the connection is closed while subscriptions exist", () => {
|
||||
it("should attempt to reconnect after base delay if the connection is closed while subscriptions exist", () => {
|
||||
wsClient.subscribe("plugin-a", "ws://engine-1/stream");
|
||||
mockWs.onclose();
|
||||
|
||||
|
||||
@@ -9,9 +9,16 @@ interface EngineConnection {
|
||||
subscriptions: Set<string>;
|
||||
/** Grace period timer — closes the connection if no plugins remain subscribed */
|
||||
cleanupTimer: NodeJS.Timeout | null;
|
||||
/** Backoff attempt counter — resets after a stable connection (>5s open) */
|
||||
reconnectAttempts: number;
|
||||
/** Timer that resets the backoff counter once a connection has been stable */
|
||||
stableConnectionTimer: NodeJS.Timeout | null;
|
||||
}
|
||||
|
||||
const RECONNECT_DELAY_MS = 5000;
|
||||
const RECONNECT_BASE_MS = 5000;
|
||||
const RECONNECT_MAX_MS = 60000; // Cap at 1 minute
|
||||
const RECONNECT_JITTER_MS = 4000;
|
||||
const STABLE_CONNECTION_MS = 5000; // Reset backoff after 5s of stable connection
|
||||
const CLEANUP_GRACE_MS = 30000;
|
||||
|
||||
/** Normalizes underscore-based pluginIds to kebab-case (e.g. `my_plugin` → `my-plugin`). */
|
||||
@@ -30,6 +37,8 @@ class WebSocketClient {
|
||||
reconnectTimer: null,
|
||||
subscriptions: new Set(),
|
||||
cleanupTimer: null,
|
||||
reconnectAttempts: 0,
|
||||
stableConnectionTimer: null,
|
||||
};
|
||||
this.engines.set(engineUrl, engine);
|
||||
}
|
||||
@@ -48,6 +57,12 @@ class WebSocketClient {
|
||||
|
||||
engine.ws.onopen = () => {
|
||||
console.debug(`[WSClient] 🟢 Connected to ${engineUrl}. WS Handshake took ${(performance.now() - wsStart).toFixed(2)}ms`);
|
||||
// Only reset backoff if the connection stays open for a non-trivial time —
|
||||
// an immediate close (e.g. server-side rejection) shouldn't be treated as success.
|
||||
if (engine.stableConnectionTimer) clearTimeout(engine.stableConnectionTimer);
|
||||
engine.stableConnectionTimer = setTimeout(() => {
|
||||
engine.reconnectAttempts = 0;
|
||||
}, STABLE_CONNECTION_MS);
|
||||
// Resubscribe to all active plugins on this engine
|
||||
for (const pluginId of engine.subscriptions) {
|
||||
this.send(engine, { action: "subscribe", pluginId });
|
||||
@@ -79,12 +94,25 @@ class WebSocketClient {
|
||||
};
|
||||
|
||||
engine.ws.onclose = () => {
|
||||
console.warn(`[WSClient] Disconnected from ${engineUrl}. Reconnecting in 5s...`);
|
||||
engine.ws = null;
|
||||
if (engine.stableConnectionTimer) {
|
||||
clearTimeout(engine.stableConnectionTimer);
|
||||
engine.stableConnectionTimer = null;
|
||||
}
|
||||
if (engine.reconnectTimer) clearTimeout(engine.reconnectTimer);
|
||||
// Only reconnect if there are still active subscriptions
|
||||
if (engine.subscriptions.size > 0) {
|
||||
engine.reconnectTimer = setTimeout(() => this.connectEngine(engineUrl), RECONNECT_DELAY_MS);
|
||||
// Exponential backoff with jitter to prevent thundering herd on engine restart.
|
||||
// 5s -> 10s -> 20s -> 40s -> 60s (cap), plus ±4s of jitter so simultaneous
|
||||
// sessions don't all reconnect at the same instant.
|
||||
const expDelay = Math.min(
|
||||
RECONNECT_BASE_MS * Math.pow(2, engine.reconnectAttempts),
|
||||
RECONNECT_MAX_MS
|
||||
);
|
||||
const delay = expDelay + Math.random() * RECONNECT_JITTER_MS;
|
||||
engine.reconnectAttempts++;
|
||||
console.warn(`[WSClient] Disconnected from ${engineUrl}. Reconnecting in ${Math.round(delay / 1000)}s (attempt ${engine.reconnectAttempts})...`);
|
||||
engine.reconnectTimer = setTimeout(() => this.connectEngine(engineUrl), delay);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -149,6 +177,7 @@ class WebSocketClient {
|
||||
if (engine.subscriptions.size === 0) {
|
||||
console.log(`[WSClient] No subscriptions remain for ${engineUrl}. Closing connection.`);
|
||||
if (engine.reconnectTimer) clearTimeout(engine.reconnectTimer);
|
||||
if (engine.stableConnectionTimer) clearTimeout(engine.stableConnectionTimer);
|
||||
engine.ws?.close();
|
||||
this.engines.delete(engineUrl);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user