diff --git a/.agents/rules/application-architecture.md b/.agents/rules/application-architecture.md new file mode 100644 index 00000000..37a0f2f9 --- /dev/null +++ b/.agents/rules/application-architecture.md @@ -0,0 +1,169 @@ +--- +trigger: model_decision +description: Architecture rules and conventions for the WorldwideView Next.js 16 frontend application, state management, and CesiumJS integration. +--- + +# WorldwideView Application Architecture + +## 1. Core Framework +- Built on **Next.js 16 (App Router)** with `output: "standalone"`. +- Uses **React 19** and **TypeScript 5** (strict mode). +- **Styling**: Strictly Vanilla CSS. No Tailwind or CSS-in-JS. +- **Component File Size**: Max 150 lines per file. Split components, extract helpers, and use hooks for anything larger. + +## 2. State & Event Bus +- **Zustand**: Slices handle global state (`globe`, `layers`, `timeline`, `ui`, `filter`, `data`, `config`, `favorites`, `geojson`). +- **DataBus**: Custom typed pub/sub singleton (`DataBus.getInstance().emit()`) decoupling React components from background WebSocket streams. + +## 3. Edition System +Features are controlled by `NEXT_PUBLIC_WWV_EDITION`: +- `local`: Self-hosted, full features, auth. +- `cloud`: Managed instance. +- `demo`: Public demo, no auth. + +## Source Layout + + +``` +worldwideview/ +├── .agents/ # AI agent workflows, rules, memories, skills +├── docs/ # Public-facing documentation +├── artifacts/ +│ └── context/ # ← THIS FOLDER — private source of truth (not committed) +├── packages/ # Plugin packages (workspaces) +├── public/ # Static assets (Cesium assets, GeoJSON data files) +├── scripts/ # Build/utility scripts (copy-cesium.mjs, scaffold-osm-plugin.mjs, etc.) +└── src/ + ├── app/ # Next.js App Router (layouts, pages, API routes) + ├── components/ # React UI components (HUD, LayersPanel, Panels, Sidebar) + ├── core/ + │ ├── data/ # DataBus, PollingManager, CacheLayer + │ ├── globe/ # Cesium hooks, viewer lifecycle, primitive collections + │ └── plugins/ # Plugin registry, lifecycle, loaders + ├── lib/ # Shared utilities, server-side logic + │ └── marketplace/ # Marketplace bridge (auth, token, CORS, repository) + └── plugins/ # Built-in feature plugins (aviation, maritime, borders, etc.) +``` + +--- + +## Module Breakdown + +### `src/core/data/` — The Heartbeat + +- **DataBus**: Decentralized event pipeline for all system actions. Plugins emit, components subscribe. +- **PollingManager**: Intelligent scheduler for external API calls with exponential backoff. +- **CacheLayer**: 2-stage persistent caching — In-Memory + IndexedDB (`worldwideview-cache`). + +### `src/core/globe/` — Rendering Engine + +- **Cesium Integration**: Low-level control over the Cesium Viewer and Primitive Collections. +- **EntityRenderer**: High-performance Primitive-based renderer. See `Primitives vs. Entities` below. +- **Spiderifier & Clustering Engine** (`StackManager.ts` & `stackAnimation.ts`): + - Automatically identifies overlapping entities and groups them into geometrically balanced clusters (concentric rings for <18 density, Fermat's golden spiral for massive stacks). + - **Mobile Layout Constraints**: Integrates responsive coordinate geometry. Modifies logical icon dimensions via `iconUpscaler.ts` (32px mobile / 48px desktop) and scales the radial expansion offsets proportionally by 70% on mobile displays to ensure tight, aesthetically pleasing clusters without massive gaps. + - **Z-Fighting Mitigation**: Clustering involves highly frequent stack rebuild cycles on any zoom/pan delta. To prevent chaotic render flickering, the global `AnimationLoop` explicitly yields visibility management to `stackAnimation.ts` via `isEntityClustered()`. Items transitioning into the `collapsed` state intentionally bypass animation lerping (to prevent 220ms ghost visibility bleed) and mathematically cloak themselves instantly. +- Globe is initialized via `GlobeView.tsx`. Viewer instance passed via `CesiumContext`. + +### `src/core/plugins/` — Plugin System + +- **PluginManager**: Registers, boots, and destroys plugins. Enforces capability checks. +- **Loaders**: Plugins are dynamically imported at runtime as ES module bundles (`loadPluginFromManifest`). The legacy `StaticDataPlugin` and `DeclarativePlugin` runtimes are fully deprecated. +- **Manifest Requirement**: All plugins MUST include a `"worldwideview"` manifest block in their `package.json` (containing `id`, `icon`, `category`, and `format`). This is strictly required by the Marketplace in order to parse display attributes correctly. + +--- + +## Performance: Primitives vs. Entities + +**Core design decision:** Use Cesium `Primitive` API instead of the high-level `Entity` API for high-count datasets. + +| Feature | Entity API | Primitive API (WWV) | +|---|---|---| +| **Abstraction** | High (easy to use) | Low (direct GPU access) | +| **JS Overhead** | Significant per-entity objects | Minimal (batched draw calls) | +| **Max Capacity** | ~1,000 entities | **100,000+ points/billboards** | +| **WWV Use Case** | Info window / detail panel content | Live data point visualization | + +Specifically: `PointPrimitiveCollection` and `BillboardCollection` ensure 60FPS even with dense global data. The full migration from Entity-based rendering to Primitives was completed to eliminate severe FPS drops. + +--- + +## Globe Occlusion & Depth Testing + +- **Correct approach:** `depthTestAgainstTerrain = false` — uses GPU to depth-test against the smooth WGS84 ellipsoid. Icons visible on the near hemisphere, correctly occluded behind Earth. +- **Avoid:** `eyeOffset`, `disableDepthTestDistance` hacks that push entities in front of everything. These cause icons to bleed through the globe. +- The Plugin SDK's `CesiumEntityOptions` exposes this via `disableDepthTestDistance` when absolutely needed for specific usecases. + +--- + +## Data Pipeline (Example: Aviation) + +``` +Remote API (e.g. OpenSky) + ↓ [raw JSON, every ~30s via PollingManager] +Plugin (e.g. AviationPlugin) + ↓ [parse, filter, sanitize → GeoEntity[]] +DataBus.emit("dataUpdated", { entities }) + ↓ +EntityRenderer [batched Primitive update] + ↓ +GPU (CesiumJS) → 60FPS visualization +``` + +--- + +## Database (Unified PostgreSQL) + +**Location:** We have migrated from a fragmented SQLite + PostgreSQL setup to a unified **PostgreSQL** architecture using Prisma 7 with the `@prisma/adapter-pg` driver. This enables true multi-tenant cloud capability, real-time sync, and scalable connection pooling via `DATABASE_URL`. + +**Next.js Core Tables:** +| Model | Table | Purpose | +|---|---|---| +| `User` | `users` | Auth accounts (email, bcrypt password, role) | +| `InstalledPlugin` | `installed_plugins` | Marketplace plugin install records | +| `Setting` | `settings` | Key-value app config (Cesium token, map style, etc.) | +| `Tenant` | `tenants` | Multi-tenant organization boundaries | + +**Data Engine Tables:** +Managed explicitly via optimized queries in `packages/wwv-data-engine/src/db.ts` targeting PostgreSQL to maximize ingest performance. +Includes tables like `iranwar_events`, `earthquakes`, `wildfires`, and `maritime_history`. + +> [!WARNING] +> **Catastrophic Time-Series Polling (The Lookback Trap):** +> High-frequency polling (e.g. 6,000 aircraft every 10s) accumulates millions of rows rapidly. +> - **DO NOT** query unindexed columns (like `ts`) for lookback/freshness without a dedicated index, otherwise you will trigger full table scans and 500 Route timeout errors. +> - **REQUIRED:** Implement aggressive data retention / hourly pruning via `scheduler.ts` to prevent infinite table growth. +> - **PREFERRED:** Keep real-time / spatial lookbacks in Redis (like `getLiveSnapshot()`), and leave PostgreSQL strictly for historical chronological replays that hit standard indices. + +**Hot-Caching (Redis) & Connection Rules:** +The `wwv-data-engine` polls external APIs, dumps raw payloads to PostgreSQL for historical persistence, and broadcasts active snapshots via WebSockets while persisting to Upstash Redis. The Next.js frontend fetches initial state from Redis to ensure horizontal scalability. +*TLS Connection Requirement:* The `REDIS_URL` mapped in Coolify and `.env.local` must use the `rediss://` protocol for TLS encryption. Attempting to use plain `redis://` with Upstash will cause the connection to drop (`ECONNRESET`) and throw a `MaxRetriesPerRequestError (which is 3)` inside the data engine. +*Upstash Request Limit Throttling:* Due to Upstash Serverless billing (capped at 10k requests/day or 500k/month on free tier), the data engine decouples WebSocket live broadcasting from Redis persistence. High-frequency scrapers (like Aviation) run every 15 seconds to push WebSockets, but the backend forces a **5-minute throttle** before saving the next snapshot to Upstash using `zlib` compression. This executes just 2 commands (`SET` and `EXPIRE`) per 5 minutes per scraper, completely eliminating `max requests limit exceeded` errors while retaining real-time frontend updates. + +**Resilience (Data Self-Healing):** +Even with persistent PostgreSQL, critical history (like `iranwarlive` events) should be bundled as `seed.json` or `history.csv` artifacts. If the Data Engine detects an empty table, it MUST parse its built-in fallback seed to hydrate the database and Redis, ensuring 100% data recovery without manual intervention. + +**Dynamic Engine URL Resolution:** +The decoupled Data Engine lives at `http://localhost:5001` during local development, and deploys to `https://dataengine.worldwideview.dev` in production. The Next.js frontend injects this context directly into the plugin via the `ctx.env` object (specifically reading from `NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL` environment variables). Plugins resolve their required backend natively by parsing `this.context.env.DATA_ENGINE_URL`. The legacy `globalThis.__WWV_ENGINE_URL__` proxy approach has been fully deprecated. + +--- + +## Settings Architecture + +User-configurable values live in the `settings` table (not `.env`). They can be updated without rebuilds. + +| Setting Type | Where | Example | +|---|---|---| +| **Infrastructure** | `.env` | `DATABASE_URL`, `NEXT_PUBLIC_WWV_EDITION` | +| **User settings** | `settings` table (DB) | Cesium token, map style, API keys | + +**Setup Wizard:** If `cesium_token` is missing from the DB on first load, the app shows a setup wizard. Plugin-level wizards work the same way — plugin declares required settings in its manifest; if missing, it renders a setup form instead of data. + +--- + +## Design Principles + +- **Single Responsibility:** Plugins only handle data mapping — they don't know about UI or cache. +- **Dependency Inversion:** `PluginManager` communicates via the `WorldPlugin` interface — new plugins don't require core changes. +- **Event-Driven:** Components subscribe to what they need via DataBus — no prop-drilling. +- **Defensive Programming:** All external calls wrapped in error boundaries + `PollingManager` backoff. diff --git a/.agents/rules/cloud-auth-architecture.md b/.agents/rules/cloud-auth-architecture.md new file mode 100644 index 00000000..16105bbf --- /dev/null +++ b/.agents/rules/cloud-auth-architecture.md @@ -0,0 +1,217 @@ +--- +trigger: model_decision +description: Cloud edition implementation details, PostgreSQL Row-Level Security (RLS) isolation, cross-subdomain auth, and cryptographic license keys. +--- + +# WorldWideView — Cloud & Auth Architecture + + +> [!NOTE] +> **This file contains the master plan and advanced implementation details for the future Cloud edition.** +> It covers single-container multi-tenancy, PostgreSQL Row-Level Security (RLS) isolation, cross-subdomain auth, and cryptographic license keys. It is intended for AI reference when building multi-tenant or auth-related features. + +--- + +## Auth Strategy + +**Auth.js (NextAuth)** is the single auth API across all editions. `app.worldwideview.dev` owns **all auth UI** — login, registration, password reset. No other subdomain ever shows a login or registration form. + +```typescript +// src/lib/auth.ts +const providers = isCloud + ? [SupabaseProvider({ /* delegates to Supabase Auth (GoTrue) */ })] + : [Credentials({ /* local bcrypt + PostgreSQL */ })]; + +export const { auth, signIn, signOut } = NextAuth({ providers }); +``` + +Application code is **identical** across editions — always calls `auth()`, `signIn()`, `signOut()`. + +**One account for the entire ecosystem:** +- Cloud instance (`[user].app.worldwideview.dev`) +- Marketplace (browse, install, publish) — via SSO redirect +- Local instance Bridge API connection + +**Marketplace never owns auth UI** — it redirects to `app.worldwideview.dev/login?redirect_to=marketplace...` for all sign-in. + +--- + +## Registration & Auto-Provisioning + +Registration: `app.worldwideview.dev/register` + +``` +1. User fills: email, password, username (= subdomain) +2. Account created (Supabase Auth) +3. Tenant row inserted instantly → [username].app.worldwideview.dev is live +4. User lands on their instance → Setup Wizard (Cesium token) +``` + +**Subdomain validation:** alphanumeric + hyphens, min 3 chars, not already taken. +**Reserved names:** `www`, `app`, `api`, `demo`, `marketplace`, `docs`, `status`, `admin`, `cdn`, `blog`, `changelog` + +"Provisioning" is just inserting a database row — no container to spin up (multi-tenant shared app). + +--- + +## Multi-Tenant Architecture + +**One Docker container, RLS isolation.** All cloud tenants share one Next.js deployment. Data isolation enforced at the **PostgreSQL database level** via Row-Level Security: + +``` +nginx (wildcard *.app.worldwideview.dev) + → reads subdomain → passes X-Tenant-ID header + → single WWV Docker container (Next.js) + → middleware reads X-Tenant-ID + → sets app.tenant_id in DB connection + → RLS scopes ALL queries to that tenant +``` + +```sql +ALTER TABLE installed_plugins ENABLE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON installed_plugins + USING (tenant_id = current_setting('app.tenant_id')::uuid); +``` + +PostgreSQL **physically refuses** to return other tenants' rows — even application bugs cannot leak data. + +> Schema-per-tenant (stronger isolation) can be offered later for enterprise. The `tenant_id` design makes migration straightforward. + +### Security Analysis + +| Concern | Risk | Mitigation | +|---|---|---| +| Data bleed between tenants | 🔴 Critical | RLS enforced at PostgreSQL level | +| One tenant crashes the app | 🟡 Medium | Next.js request isolation. Rate limit per tenant at nginx | +| One tenant hammers CPU | 🟡 Medium | Rate limiting at nginx (req/min per subdomain) | +| Plugin code between tenants | 🟢 Low | Plugins run in the user's **browser**, not the server | +| Server crash from users | 🟢 Low | Users can't run arbitrary server code. Rate limits handle abuse | + +--- + +## Storage & Database + +| Feature | Local | Cloud | +|---|---|---| +| **Database queries** | Prisma → PostgreSQL | Prisma → PostgreSQL (Supabase) | +| **Auth** | None (single owner) | Supabase Auth (GoTrue) | +| **File storage** | Local filesystem (`data/`) | Supabase Storage | +| **Storage limit** | ∞ (your disk) | Quota enforced per tier | + +**Prisma is write-once, deploy-anywhere** — the same queries and schema work on both databases. `DATABASE_URL` in `.env` is the only difference. + +--- + +## Tier Matrix + +| Deployment | `NEXT_PUBLIC_WWV_EDITION` | License Key | Users | Storage | +|---|---|---|---|---| +| **Demo** | `demo` | none | ∞ (anonymous, read-only) | N/A | +| Local Free | `local` | none | 1 (owner) | ∞ | +| Local Pro | `local` | Pro key | 1 (owner) | ∞ | +| Local Team | `local` | Team key | Unlimited | ∞ | +| Cloud Free | `cloud` | none | 3 | 500 MB | +| Cloud Pro | `cloud` | Pro key | 20 | 5 GB | +| Cloud Enterprise | `cloud` | Enterprise key | Unlimited | 50 GB+ | + +### Cryptographic License Keys + +Paid features gated by **RSA-signed JWTs** — users cannot forge them by flipping flags: + +```typescript +// Marketplace signs (PRIVATE key — only you have this): +const licenseKey = jwt.sign( + { tier: "pro", exp: "2027-01-01", org: "acme-corp" }, + PRIVATE_KEY, { algorithm: "RS256" } +); + +// WWV verifies (PUBLIC key — hardcoded in codebase): +const license = jwt.verify(process.env.WWV_LICENSE_KEY, PUBLIC_KEY); +// license.tier = "free" | "pro" | "team" +``` + +### Storage Quota (Cloud Only) + +- **Counts:** Imported GeoJSON, plugin data history, uploaded assets +- **NOT counted:** Built-in plugin real-time data (transient) +- **Quota exceeded:** Upgrade prompt, reject new imports, allow read access + +--- + +## Environment Variables + +### Local `.env` +```env +NEXT_PUBLIC_WWV_EDITION=local +DATABASE_URL=postgresql://postgres:postgres@localhost:5432/wwv?schema=public +``` + +### Cloud `.env` +```env +NEXT_PUBLIC_WWV_EDITION=cloud +DATABASE_URL=postgresql://... +NEXT_PUBLIC_SUPABASE_URL=https://[project].supabase.co +NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJ... +SUPABASE_SERVICE_ROLE_KEY=eyJ... +WWV_TENANT_MODE=rls +WWV_STORAGE_PROVIDER=supabase +``` + +### Demo `.env` +```env +NEXT_PUBLIC_WWV_EDITION=demo +DATABASE_URL=postgresql://... +``` + +`.env` is **infrastructure only** — things the operator sets once. Everything user-configurable lives in the `settings` DB table. + +--- + +## Edition Adapters + +Three thin files are all that differs between editions — everything else is edition-unaware: + +| Adapter | Local | Cloud | Demo | +|---|---|---|---| +| `src/core/auth.ts` | Auth.js `Credentials` (bcrypt + PostgreSQL) | Auth.js `@auth/supabase-adapter` | Disabled | +| `src/core/storage.ts` | `fs.writeFile()` to `data/` | Supabase Storage SDK | Read-only | +| `src/core/tenant.ts` | No-op (single tenant) | Sets `app.tenant_id` for RLS | No-op | + +--- + +## Bot Prevention (Layered) + +| Layer | What | Cost | +|---|---|---| +| **Email verification** | Supabase Auth built-in | Free | +| **Rate limiting** | 3 signups per IP per hour (nginx) | Free | +| **Cloudflare Turnstile** | Invisible CAPTCHA on signup | Free | +| **Cloudflare Bot Fight Mode** | Auto-enabled on free tier | Free | + +--- + +## CI/CD Pipeline + +``` +Push to silvertakana/worldwideview + → GitHub Actions: build Docker image + → Push to ghcr.io/silvertakana/worldwideview:latest + → Coolify webhook triggers + → Rolling restart (zero downtime) +``` + +### `worldwideview-cloud/docker-compose.yml` (concept) +```yaml +services: + wwv-cloud: + image: ghcr.io/silvertakana/worldwideview:latest + environment: + NEXT_PUBLIC_WWV_EDITION: cloud + DATABASE_URL: ${SUPABASE_PG_URL} + SUPABASE_URL: ${SUPABASE_URL} + SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY} + labels: + - "traefik.http.routers.wwv.rule=HostRegexp(`{user:[a-z0-9-]+}.app.worldwideview.dev`)" +``` + +The cloud repo (`worldwideview-cloud`) holds **no application logic** — it is purely a thin deployment configuration layer that pulls versioned Docker images. diff --git a/.agents/rules/context-and-memory.md b/.agents/rules/context-and-memory.md index 958bc6b0..5c71a247 100644 --- a/.agents/rules/context-and-memory.md +++ b/.agents/rules/context-and-memory.md @@ -10,7 +10,7 @@ This specification ensures that any agent maintains persistent, accurate project ## 2. Synchronization Target The agent MUST orient itself by immediately reading: 1. `c:\dev\worldwideview\AGENTS.md` (The root index of all technical stack details) -2. `c:\dev\worldwideview\.agents\context\01-platform-overview.md` (The high-level platform goals) +2. `c:\dev\worldwideview\.agents\rules\platform-architecture.md` (The high-level platform goals) If deep technical rules are required for a specific domain (e.g., UI state, Map Rendering, Monorepo management), the agent MUST consult the relevant file in `c:\dev\worldwideview\.agents\rules\`. diff --git a/.agents/rules/continuous-improvement.md b/.agents/rules/continuous-improvement.md new file mode 100644 index 00000000..c25563b5 --- /dev/null +++ b/.agents/rules/continuous-improvement.md @@ -0,0 +1,28 @@ +--- +trigger: always_on +description: Guidelines for ongoing project maintenance, cleanliness, and self-correction during everyday development tasks. +--- + +# Continuous Improvement & Maintenance + +As an AI agent working within the WorldWideView project, you are expected to practice the **Boy Scout Rule**: always leave the codebase (and its context) cleaner than you found it. + +While executing your primary programming tasks, you MUST actively abide by these maintenance protocols: + +## 1. Documentation & Rule Parity +- **Self-Correction**: If you notice that an architectural choice you are implementing contradicts an existing rule in `.agents/rules/`, you must halt and update the rule file to reflect the new paradigm. +- **Dead Context**: If you encounter context, plans, or architectural files referencing deprecated concepts (e.g., legacy microservices, `StaticDataPlugin`), proactively prune or update them. + +## 2. Codebase Cleanliness +- **Dead Code**: If you refactor a file and leave behind unused functions, imports, or CSS classes, clean them up immediately. Do not leave "commented out" code unless explicitly requested for future use. +- **Console Logs**: Remove debugging `console.log` statements before finalizing a feature, except for structured error logging or critical startup info. +- **Sandbox Hygiene**: Any throwaway scripts or debugging tools placed in `/local-scripts/` should be deleted once the underlying issue is resolved, unless they provide permanent utility. + +## 3. Dependency & Typings Checks +- **Type Safety**: Never use `any` or `@ts-ignore` as a shortcut when refactoring. Always build the proper TypeScript interfaces. +- **Manifest Validation**: Whenever you touch a plugin, ensure its `package.json` `"worldwideview"` manifest block is completely accurate and synced with its capabilities. + +## 4. The "See Something, Say Something" Rule +If you spot an unrelated but obvious bug, security vulnerability, or architectural anti-pattern in the file you are editing: +1. Fix it if it's a minor, isolated issue (and document the fix in your summary). +2. For larger issues, explicitly call it out to the user so it can be added to the technical debt backlog. diff --git a/.agents/rules/data-engine-architecture.md b/.agents/rules/data-engine-architecture.md new file mode 100644 index 00000000..1b0c4d58 --- /dev/null +++ b/.agents/rules/data-engine-architecture.md @@ -0,0 +1,31 @@ +--- +trigger: model_decision +description: Architecture rules and dependency management guidelines for wwv-data-engine seeders. +--- + +# Data Engine V2 Architecture & Dependency Loading + +## 1. Engine Versions: V2 vs Legacy +The current and only active engine is **`wwv-data-engine-v2`**. +- **Legacy Engine**: Previously, data sources ran as isolated, standalone microservices running their own Node environments. +- **V2 Engine**: Acts as a unified **"Host Environment" runner**. It is a single Node.js process that dynamically downloads and executes multiple independent seeder bundles (both public community seeders and private seeders). + +## 2. Deployment Architecture +- **Single Docker Image**: V2 is deployed via Coolify as a single Docker image, natively mapping environment variables continuously into the container shell. +- **Automated Restarts**: Code pushes to the `wwv-data-engine-public` repository automatically trigger a service restart via a secure `COOLIFY_API_TOKEN` configured in the CI/CD pipeline (replacing unreliable generic webhooks). +- **Dual-Output Engine**: Seeders in V2 expose both a WebSocket stream (`/stream`) for real-time instantaneous updates, and a REST API endpoint (`/api/:id`) for fetching live data snapshots directly from Redis. + +## 3. Pnpm Workspaces & Dynamic Dependency Loading +To prevent seeders from ballooning in size by bundling a massive amount of dependencies, the system heavily relies on `pnpm` workspaces. +- **Host Environment**: The V2 engine natively provides common geospatial and utility packages (e.g., `zod`, `ws`, `node-cron`, `undici`, `satellite.js`, `geoip-lite`). Seeders **MUST NOT** bundle these standard dependencies, and should leave them external in `tsup`. +- **Dynamic Workspaces**: When V2 loads a compiled `dist/index.mjs` via `node`, it dynamically resolves imports by traversing up to the workspace `node_modules`. + +## 4. Production Deployment & Downloader +In production, the `wwv-data-engine-v2` does NOT mount the local file system. Instead: +1. `download-seeders.ts` runs on container startup. +2. It fetches the latest compiled `seeders.zip` from GitHub Releases for both community and private repositories. +3. **CRITICAL STEP:** After unzipping the workspaces into a staging directory (`/app/seeders`), the script dynamically generates a root `package.json` and `pnpm-workspace.yaml`. +4. It then runs a single **workspace-aware `pnpm install --prod`**. This ensures all custom unbundled dependencies declared by the extracted plugins are downloaded directly into the container and properly linked together. + +## 5. Namespace Separation Rule +To prevent collisions and `404` or `ERR_MODULE_NOT_FOUND` errors, seeders **MUST NOT** exist simultaneously in both the community (`wwv-seeders-community`) and private (`wwv-seeders-private`) repositories. Namespace overlaps will cause module resolution failures when the V2 engine attempts to load them. Private seeders have priority. diff --git a/.agents/rules/deployment-and-testing.md b/.agents/rules/deployment-and-testing.md new file mode 100644 index 00000000..64e9f00c --- /dev/null +++ b/.agents/rules/deployment-and-testing.md @@ -0,0 +1,304 @@ +# Deployment, Testing, and Operations + +## Development Commands + +```bash +pnpm install # Install all workspace dependencies +pnpm run setup # Generate .env.local with AUTH_SECRET (first-time setup) +pnpm dev # Next.js frontend only (auto-runs prisma db push + copy-cesium) +pnpm dev:all # Frontend + wwv-data-engine (via Docker Compose) concurrently +pnpm dev:backends # Starts the local data engine + Redis via Docker Compose +pnpm build # Production build +pnpm test # Run all Vitest tests (scoped to src/lib, src/core, src/plugins) +pnpm db:reset # Reset and re-migrate the frontend database (destructive) +pnpm start:backends # Legacy command for standalone Fastify backends +pnpm clean:backends # Wipe all plugin database records +pnpm run scaffold-osm-plugin # Generate a new plugin from scaffold +pnpm dev:plugins # File watcher for local-plugins/ directory (runs automatically in dev) +node packages/wwv-cli/dist/index.js create --local # Scaffold a new local plugin +node packages/wwv-cli/dist/index.js link # Promote a local plugin to packages/ +``` + +Frontend runs at `http://localhost:3000`. + +## Deployment + +- **Docker**: Multi-stage Dockerfile using the Extractor Pattern (`deps` → `builder` → `runner`). The `node_modules` folders must be explicitly untracked in from `git` across the workspace to prevent corrupted BuildKit contextual cache overlaps during the `COPY . .` stage. +- **Standalone output**: `next.config.ts` uses `output: "standalone"`. +- **Cesium assets**: Copied to `public/cesium/` via `scripts/copy-cesium.mjs` at build time, excluded from output tracing. +- **Prisma Configuration**: `prisma.config.ts` must export a native javascript object instead of dynamically importing CLI wrapper binaries (`prisma/config` or `dotenv`). The standalone Next.js tracer strips CLI devDependencies during the build, which will cause fatal runtime container crashes if imported. +- **Data Engine**: The single runner container defined in `docker-compose.yml`, proxied via `next.config.ts` rewrites. +- **Coolify**: Deployed via Dockerfile builder natively mapping environment variables continuously into the container shell. +- **Docker volumes**: Ensure PostgreSQL data and Redis volumes are mounted for persistence. + +## Testing Strategy + +- **Framework**: Vitest with jsdom environment +- **Coverage**: `src/lib/**`, `src/core/**`, `src/plugins/**` +- **Run**: `pnpm test` (or `vitest run`) +- **Key test files**: `rateLimit.test.ts`, `edition.test.ts`, `demoAdmin.test.ts`, `DeclarativePlugin.test.ts`, `cors.test.ts`, `repository.test.ts`, `marketplaceToken.test.ts` + +## Security Headers + +Configured in `next.config.ts` `headers()`: +- **CSP**: Restrictive with exceptions for CesiumJS (`unsafe-eval`, `unsafe-inline`), camera streams (`http: https:`), and analytics +- **X-Frame-Options**: DENY +- **X-Content-Type-Options**: nosniff +- **Referrer-Policy**: strict-origin-when-cross-origin +- **Permissions-Policy**: camera/microphone disabled, geolocation self-only + + + + +--- + +## Package Manager + +The project uses **pnpm workspaces**. All `npm` commands in older docs/scripts should be treated as `pnpm`. + +```bash +pnpm install # install all workspace deps +pnpm run dev # start dev server ONLY (worldwideview/) +pnpm run dev:all # start ALL services concurrently (worldwideview, data-engine, marketplace) +pnpm run build # production build +``` + +--- + +## Local Dev Setup + +``` +1. Clone repo +2. pnpm install +3. pnpm run setup # generates .env.local with AUTH_SECRET, copies .env.example +4. pnpm run dev:all # predev: prisma migrate deploy + copy-cesium.mjs → then starts all services +5. Open http://localhost:3000 +``` + +**Marketplace:** `cd worldwideview-marketplace && pnpm run dev` → http://localhost:3001 + +**Database auto-creates** on first run via `predev` script — no manual DB setup required. + +--- + +## Database Operations + +**All SQL from `c:\dev\worldwideview`:** + +```powershell +npx prisma db execute --sql "" +``` + +> Stop the dev server first to avoid "database is locked" errors. + +### Common Commands + +```powershell +# View row counts (all tables) +npx prisma db execute --sql "SELECT 'users' AS t, COUNT(*) AS n FROM users UNION ALL SELECT 'aviation_history', COUNT(*) FROM aviation_history UNION ALL SELECT 'installed_plugins', COUNT(*) FROM installed_plugins UNION ALL SELECT 'settings', COUNT(*) FROM settings;" + +# List installed plugins +npx prisma db execute --sql "SELECT pluginId, version, installedAt FROM installed_plugins ORDER BY installedAt DESC;" + +# Clear all installed plugins +npx prisma db execute --sql "DELETE FROM installed_plugins;" + +# List settings +npx prisma db execute --sql "SELECT key, value FROM settings;" + +# Set/update a setting +npx prisma db execute --sql "INSERT INTO settings (key, value) VALUES ('', '') ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;" + +# Prune aviation history older than 7 days +npx prisma db execute --sql "DELETE FROM aviation_history WHERE timestamp < NOW() - INTERVAL '7 days';" + +# Vacuum (reclaim disk space after pruning) +npx prisma db execute --sql "VACUUM FULL;" +``` + +--- + +## Data Engine & Data Recovery + +The secondary backend (Data Engine) handles high-volume API ingestion and pushes active snapshots to Upstash Redis. Because its PostgreSQL database is completely wiped on fresh Coolify deployments if volumes aren't preserved, it uses **Self-Healing Architecture**. + +**How Recovery Works:** +1. Upon startup, or during the 60-second cron cycle, the seeder checks if the target table (e.g., `iranwar_events`) is empty. +2. If empty, the engine instantly hydrates the PostgreSQL database and the Upstash Redis cache from the committed fallback payload (e.g., `data/fallback/iranwar_seed.json`). +3. To **force a rapid recovery** locally (such as when testing new scraper data), simply wipe the table and let the minute-tick refill it via `npx prisma db execute --sql "DELETE FROM iranwar_events;"`. + +**OSINT Scraping & Hydration Best Practices:** +Utility scripts (like `hydrate.ts`) exist to repair or enrich the fallback seed block offline. Important takeaways from building these scrapers: +- **Location Mapping:** Always append structured `_osint_meta.coordinates: {lat, lng}`. Without exact coordinates, the SDK defaults to Null Island or scatters points uniformly across the globe (declutter grid effect). +- **Tracking URL Resolution:** News aggregators (like Bing News RSS) wrap URLs in tracking redirect pages (`apiclick.aspx`). Never serve these directly to the frontend. Use `cheerio` to fetch the tracking page, parse the `` node, extract the *actual* news URL, and then scrape the destination article for `og:image` and `og:video` tags. +- **Frontend Sync:** Ensure Next.js schema mapping in the plugin (`wwv-plugin-xyz/src/index.ts`) perfectly mirrors the enriched fields so they render correctly in the UI. + +### Schema Changes + +```powershell +npx prisma db push # sync schema changes to DB (non-destructive) +npx prisma migrate dev --name "" # create + apply a migration +npx prisma generate # regenerate Prisma client types after schema change +npx prisma studio # visual DB browser at http://localhost:5555 +``` + +### Full Reset + +```powershell +# Stop dev server first +Remove-Item c:\dev\worldwideview\data\wwv.db +npx prisma db push +pnpm run dev +``` + +### Browser Cache Clearing + +```javascript +// Clear IndexedDB entity cache +indexedDB.deleteDatabase("worldwideview-cache"); + +// Clear localStorage user API keys +Object.keys(localStorage) + .filter(k => k.startsWith("wwv_")) + .forEach(k => localStorage.removeItem(k)); +``` + +Known `wwv_*` localStorage keys (defined in `src/lib/userApiKeys.ts`): +- `wwv_key_google_maps` +- `wwv_key_nasa_firms` + +Hard refresh: `Ctrl+Shift+R` + +--- + +## DB Troubleshooting + +| Error | Fix | +|---|---| +| "Database is locked" | Stop the dev server first | +| "Table does not exist" | Run `npx prisma db push` | +| "Plugin still shows after clearing" | Hard-refresh (`Ctrl+Shift+R`) or clear IndexedDB | +| DB file is very large | Prune aviation history + run `VACUUM` | +| Prisma client types are stale | Run `npx prisma generate` | + +--- + +## Creating Static OSM Plugins + +See full guide: `.agents/skills/osm-static-plugin-creation.md` + +**Quick start:** +```bash +node scripts/scaffold-osm-plugin.mjs '{ + "name": "volcanoes", + "displayName": "Volcanoes", + "osmTag": "natural=volcano", + "icon": "Mountain", + "color": "#ef4444", + "category": "Natural Disaster" +}' +``` + +**Manual Overpass API query (PowerShell):** +```powershell +$tag = '"natural"="volcano"' +$body = "data=[out:json][timeout:300];(node[$tag];way[$tag];relation[$tag];);out center;" +Invoke-RestMethod -Uri 'https://overpass-api.de/api/interpreter' -Method Post -Body $body -OutFile 'tmp/raw_data.json' +``` + +**Note:** OSM data is licensed under **ODbL** — include attribution in plugin READMEs. + +--- + +## npm Plugin Publishing + +See full guide: `.agents/skills/version-control/npm-plugin-publishing.md` + +```bash +# First-time publish +pnpm publish --workspace=packages/wwv-plugin- --access public + +# Future versions — via Trusted Publishing (GitHub Actions CI/CD) +``` + +--- + +## Git / Version Control + +See full guide: `.agents/skills/version-control/git-version-control.md` + +**Commit convention:** `feat:`, `fix:`, `chore:`, `docs:` prefixes. +**Commit when:** A new feature is completed (per project rules). + +--- + +## Security Checks + +See full guide: `.agents/skills/security/security-check.md` + +--- + +## Deployment + +**Host Server:** +The production Coolify instance runs on a local server at `192.168.68.64`. Direct SSH access: `ssh root@192.168.68.64`. + +**CI/CD Flow:** +1. Push to `silvertakana/worldwideview` +2. Coolify webhook triggers → automated BuildKit deployment on the host server. + +**Coolify Deployment Rules & Gotchas (The "Redis 500k" Rule):** +- **Coolify Pulls from GitHub:** Coolify does *not* deploy your local workspace files. It strictly clones the repository from GitHub. +- **Push Before You Deploy:** If you make a critical infrastructure or optimization fix locally (e.g., swapping a massive `HSET` pipeline for a compressed 5-min throttled `setLiveSnapshot`), you **must** `git commit` and `git push` those changes to the `main` branch before triggering a Coolify deployment. +- **The "Rollback DDOS" Risk:** If you update environmental variables (like a new Upstash Redis URL) inside Coolify and deploy *without* pushing your local optimized code to GitHub, Coolify will pull the *older, unoptimized* repository code and aim it at your new endpoints. This will immediately exhaust the 500,000 requests/month Upstash free limit. +- **Redis Connection Errors:** If the data engine crashes on startup with `MaxRetriesPerRequestError: Reached the max retries per request limit (which is 3)` and `ECONNRESET`, the `REDIS_URL` in your Coolify environment variables is incorrectly using `redis://` instead of the TLS-required `rediss://`. +- **Immediate Rollback on Failure:** Whenever a Coolify deployment fails or introduces a critical bug (such as rate-limit exhaustion), **immediately select the last known working deployment in the Coolify dashboard and redeploy it.** Do not leave a broken deployment hanging while trying to debug directly in production. Roll back to stability first, debug locally, commit the verified fix, and then deploy forward. + +**Local verification:** +Before pushing infrastructure changes, run `docker build -t wwv-test .` to verify that the Next.js `standalone` isolated multi-stage build works correctly. + +**Critical Docker/BuildKit Patterns:** +- **No Tracked `node_modules`:** Never track `node_modules` in git. It will severely break the Docker `COPY . .` BuildKit cache stage, as it attempts to overwrite the internally symlinked modules from the `deps` stage with the non-symlinked folders from git, causing cryptic `cannot copy to non-directory` errors. +- **Prisma 7 in Standalone:** Prisma v7 strictly forbids `url = env("DATABASE_URL")` in `schema.prisma` if `prisma.config.ts` exists. You *must* provide the `url` property in `prisma.config.ts`. +- **The `dotenv` Paradox:** Next.js `standalone` output does NOT automatically trace CLI tool devDependencies (like `dotenv`). If you import `dotenv` in `prisma.config.ts`, the production runner container will crash in a reboot loop. *Solution:* `prisma.config.ts` must use native Node `fs` and `path` to gracefully read from `.env` as a fallback, exporting a plain Javascript object without CLI wrappers. +- **Standalone Scripts & Next.js Typechecking:** Next.js `pnpm build` triggers `tsc`. By default, this will aggressively typecheck the `scripts/` directory. One-off migration scripts that import backend-only dependencies (like `pg`) will fail the global build. Always ensure `"scripts"` is added to the `"exclude"` array in `tsconfig.json`. +- **WSL2 / Docker Memory Bloat:** Next.js `standalone` + `pnpm workspaces` caching takes roughly 7GB to 10GB of virtual disk space and maxes out WSL2 memory during `docker-compose build`. Avoid building complex monorepos locally unless necessary. To reclaim local BuildKit disk space: `docker builder prune -a -f`. To reclaim WSL RAM: `wsl --shutdown`. +- **TypeScript NodeNext ESM Extensions:** By default in strict NodeNext ESM, backend scripts compiled via TypeScript (like `server.ts`) require dynamic imports to explicitly list the extension. You MUST write `await import('./routes/index.js')`, even if the underlying code is a `.ts` file, or the Docker compilation will fail with `TS2834`. +- **Git vs Docker Context Bloat:** Ensure `.dockerignore` ruthlessly excludes generated artifacts (e.g., `artifacts/`, `performance/`). Ensure `.gitignore` strips large binary caches like heavy database seed files inside `packages/*/data/` to prevent GitHub from bouncing commits exceeding the 100MB chunk limit. +- **Host Resource Exhaustion & Implicit Failures:** Complex multi-stage pnpm builds generate massive amounts of intermediate layer cache. If the Coolify host server reaches 100% disk capacity during an active deployment, the build logs will silently truncate and die abruptly with no explicit `pnpm` error. Worse, services like PostgreSQL (`coolify-db`) will instantly crash because they cannot write lock files, causing total system outages. Always maintain at least 30-40GB of free space on deployment hosts before auto-deploying via Coolify. + + +**Local update:** `git pull && pnpm install && pnpm run dev` + +--- + +## Disk Space Management & Monitoring + +The Docker build uses `standalone` output to minimize image size. The `copy-cesium.mjs` script copies only the required Cesium assets into `public/` — avoids shipping the entire Cesium package (300MB+) in the build. + +**Host Server Maintenance:** +On Coolify deployment targets (e.g., `192.168.68.64`), dangling BuildKit caches and orphaned images will eventually consume 100% of the disk. +- To recover a crashed server: SSH into the host and run `docker system prune -a -f`. +- **Proactive Monitoring:** Use the `netdata-remote` MCP server to poll or setup alerts for the host disk space. It is highly recommended to set a warning alert when disk usage exceeds 80%. + +**Application Database Growth:** +Aviation history can grow large. Prune regularly: +```powershell +npx prisma db execute --sql "DELETE FROM aviation_history WHERE timestamp < datetime('now', '-7 days');" +npx prisma db execute --sql "VACUUM;" +``` + +--- + +## Terminology Quick Reference + +| Term | Definition | +|---|---| +| **Edition** | `local`, `cloud`, or `demo` — set via `NEXT_PUBLIC_WWV_EDITION`. Drives auth/storage/tenant adapters | +| **Bridge API** | Routes in WWV (`/api/marketplace/*`) that the marketplace calls to install/manage plugins | +| **Setup Wizard** | First-run config UI — app-level (Cesium token) and plugin-level (plugin API keys) | +| **Runtime Settings Store** | `settings` DB table — replaces `.env` for everything user-facing | +| **RLS** | Row-Level Security — PostgreSQL feature enforcing tenant data isolation at the DB level | +| **Snapshot Capture** | WWV periodically stores plugin data per tenant for history/timeline. Pro/Local only | +| **Trust Stamping** | WWV always overwrites `manifest.trust` server-side — no client claims are accepted | +| **NpmCache** | DB table caching NPM registry metadata to decouple from live NPM polling | diff --git a/.agents/rules/directory-structure.md b/.agents/rules/directory-structure.md new file mode 100644 index 00000000..4a5c00c7 --- /dev/null +++ b/.agents/rules/directory-structure.md @@ -0,0 +1,69 @@ +# Directory Structure & Repositories + +## Directory Structure + +``` +worldwideview/ +├── src/ +│ ├── app/ # Next.js App Router (pages, API routes, layouts) +│ │ ├── api/ # Server-side API routes (auth, aviation, camera, etc.) +│ │ ├── login/ # Login page +│ │ ├── setup/ # First-time setup page +│ │ └── globals.css # Root stylesheet +│ ├── components/ +│ │ ├── common/ # Shared UI: BootOverlay, FloatingWindow, PluginIcon +│ │ ├── layout/ # AppShell, Header, SearchBar, DataBusSubscriber +│ │ ├── panels/ # LayerPanel, EntityInfoCard, FilterPanel, GraphicsSettings +│ │ ├── timeline/ # Timeline component +│ │ ├── marketplace/ # Plugin install/unverified dialogs +│ │ ├── ui/ # Tooltip, ReloadToast +│ │ └── video/ # Floating video manager +│ ├── core/ +│ │ ├── data/ # DataBus, PollingManager, CacheLayer, SmartFetcher +│ │ ├── filters/ # filterEngine (applies plugin filters to entities) +│ │ ├── globe/ # GlobeView, EntityRenderer, AnimationLoop, StackManager, +│ │ │ │ CameraController, InteractionHandler, SelectionHandler, +│ │ │ │ ModelManager, ImageryProviderFactory +│ │ │ └── hooks/ # useCameraActions, useEntityRendering, useModelRendering, etc. +│ │ ├── hooks/ # useBootSequence, useIsMobile, useMarketplaceSync +│ │ ├── plugins/ # PluginManager, PluginRegistry, PluginManifest, +│ │ │ │ loadPluginFromManifest, validateManifest, InstalledPluginsLoader +│ │ │ └── loaders/ # mapJsonToEntities (Legacy loaders like DeclarativePlugin are deprecated) +│ │ └── state/ # Zustand store + slices (config, data, globe, layers, timeline, ui, etc.) +│ ├── lib/ # auth, db, rateLimit, analytics, AIS stream, marketplace APIs +│ ├── plugins/ # GeoJSON plugin registrations +│ ├── styles/ # HUD animations CSS +│ ├── types/ # GeoJSON types, Umami types +│ └── generated/ # Prisma generated client (gitignored) +├── local-plugins/ # Local sandbox for developing plugins (gitignored) +├── packages/ # pnpm monorepo workspace packages +│ ├── wwv-plugin-sdk/ # Plugin SDK: type definitions, manifest schema +│ ├── wwv-plugin-aviation/ +│ ├── wwv-plugin-maritime/ +│ ├── wwv-plugin-wildfire/ +│ ├── wwv-plugin-borders/ +│ ├── wwv-plugin-camera/ +│ ├── wwv-plugin-military-aviation/ +│ ├── wwv-plugin-satellite/ +│ ├── wwv-plugin-iranwarlive/ # Standalone plugin with custom endpoints +│ └── wwv-plugin-{airports,embassies,lighthouses,nuclear,seaports,spaceports,volcanoes}/ +├── prisma/ # schema.prisma, migrations/ +├── public/ # Static assets, Cesium workers, plugin GeoJSON data +├── scripts/ # Build scripts (copy-cesium, scaffold-osm-plugin, setup) +├── data/ # PostgreSQL data volume (gitignored) +├── Dockerfile # Multi-stage production build +├── docker-compose.yml # Main app + data engine runner + local seeders +└── .agents/ # Agent documentation, rules, skills, workflows +``` + +## Related Repositories + +| Repo | Purpose | +|---|---| +| `worldwideview` | Main application (this repo) | +| `wwv-data-engine` | Generic data engine runner (PUBLIC, runs via Docker) | +| `wwv-seeders-community` | Open-source community seeders (PUBLIC) | +| `wwv-seeders-private` | Proprietary seeder scripts (PRIVATE, downloaded in prod) | +| `worldwideview-marketplace` | Plugin marketplace web app | +| `worldwideview-plugins` | Published npm plugin packages (frontend source) | +| `worldwideview-web` | Marketing / landing page | diff --git a/.agents/rules/environment-config.md b/.agents/rules/environment-config.md new file mode 100644 index 00000000..2381f576 --- /dev/null +++ b/.agents/rules/environment-config.md @@ -0,0 +1,17 @@ +# Environment Variables + +| Variable | Required | Description | +|---|---|---| +| `DATABASE_URL` | Yes | PostgreSQL connection string | +| `AUTH_SECRET` | Yes | JWT signing secret (generate with `openssl rand -hex 32`) | +| `NEXT_PUBLIC_CESIUM_ION_TOKEN` | No | Cesium Ion access token | +| `NEXT_PUBLIC_BING_MAPS_KEY` | No | Bing Maps imagery | +| `NEXT_PUBLIC_GOOGLE_MAPS_API_KEY` | No | Google 3D Tiles | +| `NEXT_PUBLIC_WWV_EDITION` | No | `local` / `cloud` / `demo` (default: `local`) | +| `NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL` | No | Override engine WebSocket URL (default: cloud) | +| `OPENSKY_CREDENTIALS` | No | Comma-separated `id:secret` pairs for credential rotation | +| `WWV_BRIDGE_TOKEN` | No | Shared secret for marketplace → WWV install bridge | +| `WWV_DEMO_ADMIN_SECRET` | No | Demo edition admin password | +| `IRANWARLIVE_BACKEND_URL` | No | Override for IranWarLive custom backend URL | + +Secrets go in `.env.local` (gitignored). Non-secrets go in `.env` (committed). diff --git a/.agents/rules/marketplace-architecture.md b/.agents/rules/marketplace-architecture.md new file mode 100644 index 00000000..500e606a --- /dev/null +++ b/.agents/rules/marketplace-architecture.md @@ -0,0 +1,222 @@ +--- +trigger: model_decision +description: Guidelines for the WorldwideView plugin marketplace, database interactions, and dynamic plugin installation. +--- + +# Marketplace Architecture + +## 1. InstalledPluginsLoader +Marketplace plugins are dynamically loaded via `InstalledPluginsLoader`. At startup, this service queries the PostgreSQL database (`installed_plugins` table) for dynamically installed manifests. + +## 2. Dynamic Bundle Loading +Plugins from the marketplace use the `bundle` format and are imported at runtime from CDN endpoints (e.g., unpkg) via `loadPluginFromManifest()`. + +## 3. Database Sync +- Marketplace installations mutate the Prisma database. +- `pluginManager.loadFromManifest()` validates and registers these plugins. +- Ensure any schema changes to the marketplace tables follow the `database-migrations.md` rule. + +## Current Implementation State + + +- **Stage 4 (Bridge API + One-Click Install)** ✅ COMPLETE +- Plugin data currently served from a volume-backed **PostgreSQL database** — plugins are submitted by entering an NPM package name in the Admin UI +- The marketplace auto-fetches metadata from the NPM registry and parses the `"worldwideview"` block in `package.json` +- A background `NpmCache` table caches NPM registry metadata to decouple from live NPM dependency +- Future: Supabase DB for full publisher workflow (Stage 5) + +--- + +## Plugin Submission Flow (Current) + +1. Developer publishes their plugin to npm with a `"worldwideview"` block in `package.json` +2. Admin opens the Marketplace Admin UI +3. Admin enters the npm package name (e.g. `@worldwideview/wwv-plugin-my-data`) +4. Marketplace fetches metadata from NPM registry, parses the manifest block, stores in PostgreSQL DB +5. Plugin appears in the browse catalog + +> Static plugins still need their GeoJSON files placed in the host application's `public/data/` directory. + +--- + +## One-Click Install Flow (Redirect-Based) + +**No static API keys.** Auth is entirely session-based. The marketplace redirects the browser to WWV, which checks the user's existing login session. + +``` +User clicks "Install" on marketplace + ↓ +Marketplace: check localStorage for instance URL + ↓ (if not configured) +Show "Connect Your Instance" modal → user enters http://localhost:3000 + ↓ +MP → GET /api/marketplace/grant-token?redirectTo=... + ↓ +WWV: check Auth.js session → issue 30-day JWT → redirect ?token= + ↓ +MP stores JWT in localStorage + ↓ +MP: Navigate browser to /api/marketplace/install-redirect + ?pluginId=borders&manifest=&redirectTo=... + ↓ +WWV: check session → upsertPlugin() → issueMarketplaceToken() + ↓ +Redirect back ?installed=borders&token= + ↓ +MP: Show ✓ Installed — store JWT for future Manage-page calls +``` + +### Token Flow + +After successful install, WWV issues a **30-day HS256 JWT** (`scope: "marketplace"`, signed with `AUTH_SECRET`). The marketplace stores it in `localStorage` and sends it as `Authorization: Bearer ` on all subsequent API calls. + +`validateMarketplaceAuth` accepts auth in priority order: +1. Active Auth.js session (same-origin browser) +2. Marketplace JWT Bearer token (cross-origin Manage page) +3. Legacy static `WWV_BRIDGE_TOKEN` (backward compat) + +--- + +## WWV Bridge API Routes (Implemented) + +| Route | Method | Purpose | +|---|---|---| +| `/api/marketplace/install-redirect` | GET | Browser redirect install — checks session, installs plugin, issues JWT | +| `/api/marketplace/grant-token` | GET | Issues marketplace JWT without install (for Manage page) | +| `/api/marketplace/status` | GET | Returns all installed plugins (session or JWT auth) | +| `/api/marketplace/uninstall` | POST | Deletes an installed plugin record | +| `/api/marketplace/load` | GET | Returns valid non-built-in manifests for client startup (no auth) | +| `/api/auth/setup-status` | GET | CORS-enabled connection test endpoint | + +### Auth Module: `src/lib/marketplace/` + +| File | Purpose | +|---|---| +| `auth.ts` | `validateMarketplaceAuth()` — session → JWT → legacy token | +| `marketplaceToken.ts` | `issueMarketplaceToken()` / `verifyMarketplaceToken()` — HS256 JWT via jose | +| `cors.ts` | CORS headers for marketplace bridge routes | +| `repository.ts` | `upsertPlugin()`, `uninstallPlugin()`, `getInstalledPlugins()` | +| `registryClient.ts` | Fetches and Ed25519-verifies the signed plugin registry. 5-min in-memory cache. Stamps trust at install time | +| `trustedPlugins.ts` | `localStorage` helpers for tracking user-approved unverified plugins | + +--- + +## Marketplace API Endpoints (Target State) + +### Public +| Method | Path | Description | +|---|---|---| +| GET | `/api/plugins` | List/search plugins | +| GET | `/api/plugins/:id` | Plugin detail | +| GET | `/api/plugins/:id/bundle` | Download plugin bundle | +| GET | `/api/categories` | List categories | +| GET | `/api/registry` | Ed25519-signed verified-plugins registry — fetched by WWV at install time | + +### Authenticated (Consumer) +| Method | Path | Description | +|---|---|---| +| POST | `/api/instances` | Register a WWV instance | +| PUT | `/api/instances/:id/default` | Set default instance | +| POST | `/api/plugins/:id/install` | Trigger install to default instance | +| GET | `/api/me/plugins` | List user's installed plugins | + +### Authenticated (Publisher) +| Method | Path | Description | +|---|---|---| +| POST | `/api/publisher/plugins` | Submit a new plugin | +| PUT | `/api/publisher/plugins/:id` | Update plugin | +| POST | `/api/publisher/plugins/:id/versions` | Publish new version | +| GET | `/api/publisher/analytics` | Publisher analytics | + +--- + +## Data Model + +### WWV Local DB (Prisma — current) + +```prisma +model InstalledPlugin { + id String @id @default(uuid()) + pluginId String + version String + config String // Full manifest JSON + installedAt DateTime @default(now()) +} + +model NpmCache { + id String @id @default(uuid()) + packageName String @unique + metadata String // JSON — cached NPM registry response + lastFetched DateTime @default(now()) +} +``` + +### Marketplace DB (Supabase — target, Stage 5+) + +``` +users ─── published_plugins ─── plugin_versions + │ + reviews +``` + +Key fields on `published_plugins`: `slug`, `publisher_namespace` (`publisher.plugin-name`), `status` (`draft | review | published | rejected`), `manifest` (jsonb). + +### Cloud DB (Supabase PostgreSQL — RLS-scoped) + +``` +tenants ─── installed_plugins + └── settings +``` + +--- + +## Security + +| Concern | Mitigation | +|---|---| +| No static API keys | Install flow uses session redirects — no token to leak | +| Marketplace JWT | 30-day HS256, scope-locked, verified via `jose` | +| CSRF | Install/grant-token are GET redirects — user explicitly navigates | +| Open redirect | `redirectTo` validated against allowlist | +| Manifest tampering | Base64 only — no code execution. Validated via `validateManifest()` | +| Malicious plugins | Sandboxed in Web Workers (unverified). CSP headers. Code review (verified) | +| Supply chain | Plugin bundles checksummed (SHA-256), integrity verified | +| Plugin API keys | Stored encrypted in `settings` table, never exposed to browser | +| Trust forgery | Registry signed with Ed25519 — `verified` cannot be claimed for unlisted plugins | +| Trust stamping | WWV always overwrites `manifest.trust` server-side — client/marketplace input ignored | +| Unverified consent | Warning dialog before any unverified plugin loads; approval stored per-plugin in `localStorage` | + +--- + +## API Key Proxy & Rate Limiting (For `apikey` plugins) + +Response caching makes server-as-proxy viable at scale: + +``` +1000 users viewing earthquake plugin: +→ 1000 requests hit your proxy +→ Proxy: "I fetched this URL 3s ago, cache still valid" +→ Returns cached response to all 1000 +→ 1 actual upstream request to USGS +``` + +| Tier | Proxy req/min | Cache TTL | +|---|---|---| +| Free | 60 | 30s | +| Pro | 300 | 10s | +| Enterprise | Unlimited | 5s | + +--- + +## History & Timeline + +**Third-party history (`historyUrl`):** Plugin manifest declares a history URL with `%t` placeholder. WWV replaces it with the scrubber timestamp — third-party provides the endpoint. No storage on your side. Available on all tiers. + +**WWV Snapshot Capture:** Your server periodically stores plugin data per tenant. Counts toward storage quota. Pro/Local only. + +| Feature | Free (cloud) | Pro / Local | +|---|---|---| +| Live data (declarative `url`) | ✅ | ✅ | +| Third-party history (`historyUrl`) | ✅ If supported by API | ✅ | +| WWV snapshot capture | ❌ | ✅ Counts toward quota | +| Built-in aviation/maritime history | ✅ Limited (24h) | ✅ Full | diff --git a/.agents/rules/platform-architecture.md b/.agents/rules/platform-architecture.md new file mode 100644 index 00000000..e15b9928 --- /dev/null +++ b/.agents/rules/platform-architecture.md @@ -0,0 +1,120 @@ +--- +trigger: model_decision +description: High-level platform goals, product vision, business model, multi-repository layout, and the Edition System. +--- + +# WorldWideView — Platform Architecture + + +> **This folder is the canonical source of truth for the project. It is NOT committed to git.** +> It contains live architectural decisions, future plans, and operational knowledge. + +--- + +> [!TIP] +> **For the fundamental system architecture, component layout, and data pipelines, please read [`docs/ARCHITECTURE.md`](../../../docs/ARCHITECTURE.md) first.** +> This file focuses strictly on the advanced business vision, multi-repository layout, and the Edition System. + +## Product Vision & Business Model + +WorldWideView operates on a **VS Code distribution model**. It is a visualization platform, not a compute provider. Plugin authors bring their own backends to display real-time geospatial intelligence. + +--- + +## Repository Structure + +Five repositories, modelled after the n8n pattern: + +| Repo | License | Purpose | +|---|---|---| +| `silvertakana/worldwideview` | Elastic License 2.0 | Core 3D globe app (Next.js + CesiumJS + Prisma) | +| `silvertakana/wwv-data-engine` | Open Source | Generic runner (Fastify + Redis + WS) | +| `silvertakana/wwv-seeders-community` | Open Source | Open-source community seeders | +| `silvertakana/wwv-seeders-private` | **Private** | Proprietary seeder scripts (aviation, maritime, etc.) | +| `silvertakana/worldwideview-marketplace` | Open Source | Plugin catalog, signed registry, admin UI | +| `silvertakana/worldwideview-plugins` | Open Source | Published plugin frontend source | +| `silvertakana/worldwideview-web` | Open Source | Landing page, docs, static content | +| `silvertakana/worldwideview-cloud` | **Private** | Cloud deployment config — Docker Compose, nginx, provisioning scripts | +| `silvertakana/plugin-sdk` | MIT | `@worldwideview/plugin-sdk` — shared types for repos + 3rd-party authors | + +**n8n Parallel:** +| n8n | WorldWideView | +|---|---| +| `n8n-io/n8n` | `silvertakana/worldwideview` | +| `n8n-io/n8n-cloud` (private) | `silvertakana/worldwideview-cloud` (private) | +| `n8n.io` landing | `worldwideview.dev` (static on Vercel) | +| `app.n8n.cloud` dashboard | `app.worldwideview.dev` | +| `[user].app.n8n.cloud` | `[user].app.worldwideview.dev` | +| `@n8n/workflow` npm | `@worldwideview/plugin-sdk` npm | + +--- + +## Domain Architecture + +| URL | What | Stack | +|---|---|---| +| `worldwideview.dev` | Static marketing/landing | Next.js (static export) on Vercel | +| `app.worldwideview.dev` | Auth, accounts, dashboard | Next.js + Supabase Auth | +| `[user].app.worldwideview.dev` | User's cloud WWV instance | Next.js (multi-tenant, RLS) | +| `marketplace.worldwideview.dev` | Plugin catalog, publishing, install | Next.js (SSO via WorldWideView auth) | +| `demo.worldwideview.dev` | Public demo — pre-configured, read-only | Next.js (WWV app) | +| `docs.worldwideview.dev` | Documentation | TBD (Mintlify/Nextra) | +| `status.worldwideview.dev` | Uptime / incident | Uptime Kuma | +| `api.worldwideview.dev` | Built-in data services (aviation, maritime) | Next.js API routes or standalone | +| `cdn.worldwideview.dev` | Plugin bundle storage | Cloudflare R2 | + +**Reserved subdomains:** `www`, `app`, `api`, `demo`, `marketplace`, `docs`, `status`, `admin`, `cdn`, `blog`, `changelog` + +--- + +## Infrastructure + +All on **Coolify** running on a repurposed **ThinkPad T480s** (i5-8350U, 16GB RAM, 256GB NVMe, Ubuntu). Zero monthly hosting cost. + +``` +ThinkPad T480s (home server, $0/mo) Supabase (hosted) +├── Coolify ──→ ├── PostgreSQL (cloud DB) +│ ├── worldwideview.dev (landing, Vercel) ├── Auth (GoTrue) +│ ├── app.worldwideview.dev (auth/dashboard) └── Storage (file uploads) +│ ├── WWV Cloud App (Docker) +│ ├── Marketplace App (Docker) Cloudflare (free) +│ ├── Demo Instance (Docker) ├── DNS + wildcard routing +│ ├── api.worldwideview.dev ├── CDN caching +│ ├── Uptime Kuma ├── DDoS protection +│ └── Plausible Analytics └── R2 (plugin bundle CDN) +└── Ubuntu Server +``` + +**CI/CD:** Push to `silvertakana/worldwideview` → GitHub Actions builds Docker image → pushes to `ghcr.io/silvertakana/worldwideview:latest` → Coolify webhook triggers rolling restart. + +**Monitoring:** Uptime Kuma (all subdomains), Plausible/Umami analytics. + +--- + +## Business Model + +**Distribution platform, not compute provider.** Plugin authors handle their own backends. + +| Revenue Stream | How | +|---|---| +| **Cloud hosting** | `[user].app.worldwideview.dev` — free/pro/enterprise tiers | +| **Marketplace cut** | % of paid plugin subscriptions via Stripe Connect | +| **Verified tier** | Plugin authors pay for review + verified badge | +| **Pro/Enterprise license** | More quota, faster cache TTL, advanced features | +| **Extension packs** | Curated bundles of related plugins | + +Plugin authors can monetize independently (VS Code + GitHub Copilot model). Selling through the marketplace is **incentivized** (managed billing, featured placement, analytics) but not mandatory. + +--- + +## Editions + +Three editions; same codebase, gated by `NEXT_PUBLIC_WWV_EDITION`: + +| Edition | Value | Database | Auth | Plugin Install | +|---|---|---|---|---| +| **Local** | `local` | PostgreSQL | Auth.js Credentials | Yes | +| **Cloud** | `cloud` | PostgreSQL (Supabase) | Auth.js + Supabase | Yes | +| **Demo** | `demo` | PostgreSQL (read-only) | Disabled | No | + +Three thin adapter files handle edition differences: `src/core/auth.ts`, `src/core/storage.ts`, `src/core/tenant.ts`. Everything else is edition-unaware. diff --git a/.agents/rules/plugin-architecture.md b/.agents/rules/plugin-architecture.md index bb3f7630..a87daf83 100644 --- a/.agents/rules/plugin-architecture.md +++ b/.agents/rules/plugin-architecture.md @@ -89,3 +89,305 @@ To rapidly iterate on plugins without polluting the core monorepo `packages/` di ## When to Apply When writing `fetch` implementations for plugins, or tracing why an entity dropped off the map. Ensure missing data correctly triggers the cache fallback via the DataBus. Ensure all new external plugins use the `bundle` format and exist in the marketplace registry to avoid CDN 404 hydrating crashes. + + +## Capability Declarations + + +Every plugin declares what it can access. Undeclared capabilities are **blocked at runtime**. + +```typescript +type PluginCapability = + | "data:own" // Fetch and render its own data layer + | "data:read:" // Read entities from another plugin + | "ui:detail-panel" // Inject UI into entity detail panel + | "ui:sidebar" // Add a sidebar section + | "ui:toolbar" // Add toolbar buttons + | "ui:settings" // Add a settings page + | "globe:overlay" // Draw on the globe (polylines, polygons, heatmaps) + | "globe:camera" // Control camera movement + | "storage:read" // Read from local/cloud storage + | "storage:write" // Write to local/cloud storage + | "network:fetch" // Make external HTTP requests +``` + +--- + +## Trust Tiers + +| Trust Tier | Who | Capabilities Allowed | How Determined | +|---|---|---|---| +| **Built-in** | Ships with WWV | All | Hardcoded in `AppShell` at build time | +| **Verified** | WorldWideView-reviewed | Any declared capability | Plugin ID in the **signed registry** | +| **Unverified** | Community / 3rd-party | `data:own`, `ui:settings` only | Plugin ID NOT in the registry | + +### Signed Plugin Registry + +Trust is determined by a cryptographically signed JSON file: + +``` +GET marketplace.worldwideview.dev/api/registry +→ { plugins: ["aviation", "maritime", ...], issuedAt, signature } +``` + +- Signed with **Ed25519** private key (held only by registry maintainer) +- WWV instances have the **public key hardcoded** — nobody can forge a fake verified list +- WWV fetches and **verifies signature server-side** at install time +- Trust is **always stamped server-side** — no client-side trust claims accepted +- Registry response cached for **5 minutes** (in-memory on each WWV instance) +- If registry unreachable, last cached result used; new unknowns default to `unverified` +- **Managing:** Edit `src/data/verifiedPlugins.ts` in marketplace repo and redeploy +- **Revoking:** Remove plugin ID and redeploy — WWV instances drop trust on next cache refresh + +### Unverified Plugin Warning + +When an unverified manifest is encountered: +1. Plugin held in pending queue (not loaded) +2. `UnverifiedPluginDialog` shown: "This plugin is unverified. Install at your own risk." +3. User can Allow (stores approval in `localStorage`) or Deny +4. Once approved, dialog won't reappear for that plugin ID + +--- + +## VS Code Design Patterns Applied + +**Activation events** — plugins declare when they activate, not at startup: +```json +"activationEvents": ["onView:aviationLayer", "onCommand:wwv.showCockpit"] +``` + +**Contribution points** — declarative UI, no code needed for sidebar/menu entries: +```json +"contributes": { + "commands": [{ "command": "wwv.enterCockpit", "title": "Enter Cockpit View" }], + "settings": [{ "key": "refreshInterval", "type": "number", "default": 10 }] +} +``` + +**Extension dependencies** — plugins can require other plugins (auto-installed if missing): +```json +"extensionDependencies": ["worldwideview.aviation"] +``` + +**Publisher namespaces** — `publisher.plugin-name` (e.g., `silvertakana.aviation`). Prevents naming conflicts. + +**Extension packs** — bundles of related plugins: +```json +{ "id": "worldwideview.military-suite", "extensionPack": ["aviation", "military-bases", "radar-overlay"] } +``` + +--- + +## Plugin Auth Types + +Plugins declare auth type in the manifest: + +| Auth Type | Who Calls the API | Server Role | +|---|---|---| +| `none` | Browser, directly | Nothing | +| `apikey` | Server proxy | Injects key from DB, caches response | +| `token-exchange` | Browser, directly (after exchange) | One-time key→token swap | +| `oauth` | Browser, directly (after OAuth) | OAuth callback handler, stores token | + +**Recommended for scale:** `token-exchange` — your server is never in the hot path for data requests. + +--- + +## Plugin Development Workflows + +Starting in 2026, WorldWideView adopted the **Architect and Toolbox** paradigm using NPM global tools, ensuring a fully decoupled developer experience. Plugin repositories contain *pure code* with absolutely no platform hosting boilerplate (e.g. no embedded docker-compose files). + +### The Developer Experience +1. **Scaffold**: `node packages/wwv-cli/dist/index.js create --local` (Creates sandbox inside `local-plugins/`) +2. **Install**: `pnpm install` (Links dependencies via workspace) +3. **Develop**: `pnpm dev:all` (Starts frontend, data engine, and plugin watcher dynamically) + +### Local vs Docker Development +The marketplace API natively supports hot-reloading for local plugins without rebuilding the core. + +**Docker-Hosted Mode (Recommended for external devs):** +If a developer wants to build a plugin without cloning the 1GB+ WorldWideView monorepo, they can pull the official WWV Docker image and mount their compiled plugin code: +1. Run WWV with the development flag: `WWV_PLUGIN_DEV=true` +2. Mount the local plugin's `dist/` folder into the container's `/app/local-plugins/` directory. +3. Because the plugin is strictly code, Vite rebuilds in <100ms, and a browser refresh instantly loads the new plugin from the Docker volume. + +**Core Contributor Mode:** +If working inside a cloned `worldwideview` directory, scaffolding the plugin into `local-plugins/` makes it a pnpm workspace member. Running `pnpm dev:all` automatically spawns `pnpm dev:plugins`, which continuously transpiles local plugins so the Next.js frontend can hot-reload them without complex linking. + +--- + +DB stores **references only** — never plugin files: + +```prisma +model InstalledPlugin { + pluginId String + version String + manifestUrl String // "https://marketplace.../api/plugins/aviation/manifest" + bundleUrl String? // local path or CDN URL (differs by edition) +} +``` + +| Edition | `bundleUrl` | How served | +|---|---|---| +| **Local** | `file:./plugins/cockpit-view/bundle.js` | Local filesystem (downloaded at install) | +| **Cloud** | `https://cdn.worldwideview.dev/plugins/cockpit-view/bundle.js` | CDN at runtime | + +--- + +## Environment Variables & Execution Agnosticism + +WWV enforces a strictly decoupled execution context, allowing plugins to function reliably whether they are loading locally (`plugins-local`) or from a public CDN (`unpkg.com`). + +### Dynamic Assets (`import.meta.url`) +For file-based assets bundled alongside the plugin (such as JSON datasets or 3D GLTF models), the runtime relies on native ECMAScript relative resolution instead of hardcoded framework paths. +- Plugins securely fetch their assets using `new URL('data/data.json', import.meta.url).href`. +- **Impact:** Assets resolve properly directly from the domain they were executed on. No CORS bypasses or CDN fallbacks required. + +### .env Passthrough Injection +The WorldWideView core engine functions as a "blind pipe" for developer environment variables, allowing organizations to securely route plugins to custom backend microservices (local or cloud) without modifying the platform. +1. The developer adds `NEXT_PUBLIC_WWV_PLUGIN_EXAMPLE_URL="http://localhost:5001"` to their `.env.local`. +2. The engine detects the `NEXT_PUBLIC_WWV_PLUGIN_` prefix, strips it, and dynamically hydrates the `PluginContext`. +3. The plugin natively retrieves `ctx.env.EXAMPLE_URL`. +4. *Security Clause:* Variables accessible via `ctx.env` are exposed directly to the **client-side bundle**. Core secrets/API tokens must never be used here; they must exclusively use the server-side `ctx.getSecret()` workflow. + +--- + +## Lazy Loading + +Plugins load their code **only when activated**, not at startup: + +``` +Startup: load manifests from DB (one query) → globe with built-in plugins only +Toggle on "Aviation" → import() aviation bundle → register → fetch data +Toggle on "Cockpit View" → import() cockpit bundle → register +``` + +--- + +## Implementation Stack + +| Piece | Library | +|---|---| +| Bundle loading | Native `import()` or `@module-federation/enhanced/runtime` | +| Manifest validation | `zod` | +| Sandboxing (unverified plugins) | `comlink` + Web Workers | +| Dependency resolution | `toposort` | +| Event system | `mitt` (200 bytes) | +| Plugin registry | Custom `PluginManager` class | + +--- + +## Built-In Plugins (Current) + +| Plugin | Format | Data Source | +|---|---|---| +| Aviation | bundle | OpenSky Network (ADS-B) | +| Maritime | bundle | AIS (maritime tracking) | +| Borders | bundle | GeoJSON from public API | +| Wildfire | bundle | NASA FIRMS | +| Military Bases | bundle (compiled) | OpenStreetMap (via Overpass) | +| Nuclear Facilities | bundle (compiled) | OpenStreetMap | +| Camera/CCTV | bundle | IP camera streams | +| GeoJSON | bundle | User-supplied GeoJSON files | + +Future direction: Aviation and maritime redesigned as **declarative plugins** pointing to `api.worldwideview.dev` — your own API server handles polling, persistence, and history for all tenants. + +--- + +## Creating Static OSM Plugins + +For static plugins built from OpenStreetMap data, use the scaffold script: + +```bash +node scripts/scaffold-osm-plugin.mjs '{ + "name": "volcanoes", + "displayName": "Volcanoes", + "description": "Active and dormant volcanoes worldwide from OSM", + "osmTag": "natural=volcano", + "icon": "Mountain", + "color": "#ef4444", + "category": "Natural Disaster" +}' +``` + +This queries Overpass API, converts to GeoJSON, scaffolds the package, and prints marketplace snippets. + +**Common OSM tags:** + +| Dataset | OSM Tag | ~Count | +|---------|---------|--------| +| Airports | `aeroway=aerodrome` | 40k | +| Seaports | `harbour=yes` | 15k | +| Nuclear facilities | `generator:source=nuclear` | 758 | +| Volcanoes | `natural=volcano` | 1.5k | +| Lighthouses | `man_made=lighthouse` | 20k | +| Embassies | `amenity=embassy` | 10k | +| Helipads | `aeroway=helipad` | 25k | +| Power plants | `power=plant` | 15k | +| Wind farms | `generator:source=wind` | 50k | + +See skill: `.agents/skills/osm-static-plugin-creation.md` + +--- + +## The Data Engine (Backend & Seeders) + +When a plugin requires 24/7 background polling, long-term history (PostgreSQL), or high-frequency caching (Redis), it should leverage the **Data Engine**. The WWV architecture splits the frontend UI package and the backend data collection into two distinct tiers. + +### 1. First-Party / Built-In Plugins +For plugins developed directly inside the WWV monorepo: +- **Seeder Module**: Developers write a single seeder (e.g. `seeders/aviation.ts`) inside the `wwv-data-engine` package. +- **Unified Container**: All first-party seeders run inside one singular `wwv-data-engine` Docker container. This prevents background polling from being destroyed by Next.js edge isolation or HMR. + +### 2. Third-Party / Community Plugins (The VPS Problem) +Currently, **there is a high DX friction for third-party developers hosting their own heavy plugins.** +- The `wwv-data-engine` is deeply coupled to the WWV host. Third-party developers cannot push their backend seeders into WWV's actual servers. +- The `wwv-plugin-sdk` only assists with the frontend `WorldPlugin` interface. +- Consequently, a sole developer with a VPS must write their own Node.js backend entirely from scratch to serve their custom data feeds, handle their own caching, and manage multiple Node instances if they create multiple plugins. + +### 3. The Future: Distributable Data Engine Image +To solve this friction and make third-party plugin hosting highly intuitive, the WWV roadmap includes publishing a **WWV Mini-Engine** as a standalone Docker image: +1. **Lightweight Deployment:** Developers run `docker run worldwideview/mini-engine` on their VPS. +2. **Backend SDK:** They write simple `.ts` fetch scripts returning `GeoEntity[]` and drop them into a mounted `/seeders` volume. +3. **Zero Configuration:** The Docker engine hot-reloads these seeders, handles the Redis caching, and exposes standardized REST or WebSocket endpoints automatically. The developer focuses only on fetching data, avoiding all backend infrastructure setup. + +See skill: `.agents/skills/data-engine-seeder-creation.md` + +--- + +## Architectural Differences: WorldWideView (WWV) vs WorldMonitor (WM) + +While WorldWideView draws UI/UX and feature inspiration from WorldMonitor as a baseline, the underlying plugin architectures are fundamentally different. + +### WorldMonitor (The Inspiration) +- **Monolithic Frontend**: Layers and datasets are typically hardcoded directly into the application's core logic. +- **Custom Implementations**: Each data source usually requires its own bespoke API fetching, state management, and map rendering logic (e.g., raw Mapbox/Cesium objects). +- **Coupled Backend**: Data ingestion is often tightly coupled to the monolithic application or spread across scattered, heterogeneous microservices without a unified cache. + +### WorldWideView (The Reality) +- **Extensible VS Code-Style Runtime**: Plugins are wholly independent, sandboxable modules loaded dynamically via activation events. The core map knows nothing about aviation or wildfires. +- **Unified Data Contract (`GeoEntity`)**: All data layers map their bespoke formats into a strict, unified `GeoEntity` system. The core engine handles clustering, filtering, and timeline playback generically—no custom rendering trickery needed per dataset. +- **Shared Data Engine**: WWV enforces a strict separation of concerns. All 24/7 API polling, data normalizations, and history retention happen off-thread in the singular `wwv-data-engine` (utilizing Redis hot cache + PostgreSQL history retention), providing a clean, unified REST interface for the plugin frontend. +- **Cryptographic Trust**: WWV incorporates a signed marketplace registry and fine-grained capability declarations (`data:read`, `ui:sidebar`), establishing an enterprise-ready security posture for third-party extensions. + +### Planned Features (Inspired by Monolithic Systems / WM) +While avoiding monolithic coupling, WWV plans to implement the following high-UX features natively or via generic plugin capabilities: + +1. **Global WebSocket Firehose (Unified Data Pipeline)**: Instead of the browser running individual `setInterval` HTTP polling loops for every activated plugin (which degrades performance and fragments data timing), WWV will shift to a single WebSocket connection. When a user activates a plugin, the frontend sends a `subscribe` message. The Data Engine then handles all background polling and simply streams the parsed updates down the singular socket. This dramatically simplifies the developer mental model—plugin developers won't write `fetch()` loops; they just provide a render configuration and respond to incoming data events. +2. **Cross-Layer Geofencing & Alerts**: A new `User Alert System` designed explicitly as a **plugin**. Utilizing the unified `GeoEntity` outputs, users can draw a polygon on the map and trigger webhooks/notifications if *any* entity from *any* active plugin crosses the boundary. +3. **Advanced Dead-Reckoning Hooks**: Building a `useDeadReckoning()` hook directly into the `@worldwideview/wwv-plugin-sdk`. It will automatically use standard `speed` and `heading` props to smoothly glide vehicle entities across the Cesium globe at 60 FPS between data pings, preventing plugin authors from rewriting complex math logic. +4. **Curated Workspaces / Scenarios**: The ability for users to save their current state as a "Workspace URL." This generates a shareable link that encodes the exact camera position, timeline bounds, activated plugins, and their respective layer configurations/filters. + + +## Local Plugins & Seeders + +### 1. Local Sandbox Workflow +- **Scaffold**: `node packages/wwv-cli/dist/index.js create --local` (creates inside `local-plugins/wwv-plugin-`). +- **Develop**: `pnpm dev` triggers hot-reloading. +- **Promote**: `node packages/wwv-cli/dist/index.js link ` moves it to `packages/`. + +### 2. Data Engine Seeders +Seeders in `wwv-data-engine/src/seeders/` provide the backend data. +- **Cron Seeder**: Uses `registerSeeder({ name: "plugin-id", cron: "..." })` and pushes to Redis via `setLiveSnapshot()`. +- **Init Seeder**: High-frequency or persistent websockets via `init: () => void`. +- **Constraint**: Seeder `name` MUST exactly match the frontend plugin `id`. Do not bundle workspace dependencies like `ws` or `zod` into the seeder `dist`. diff --git a/.agents/rules/server-management.md b/.agents/rules/server-management.md new file mode 100644 index 00000000..e70fd1ef --- /dev/null +++ b/.agents/rules/server-management.md @@ -0,0 +1,25 @@ +--- +trigger: model_decision +description: Server development and debugging rules for the production/homelab server using SSH and Coolify MCP. +--- + +# Server Management & Debugging + +## 1. Coolify MCP (High-Level) +Use the `coolify` MCP server for managed actions: +- **Discovery**: `mcp_coolify_projects`, `mcp_coolify_environments` +- **Inspection**: `mcp_coolify_application_logs`, `mcp_coolify_get_application` +- **Actions**: `mcp_coolify_control` (restart, stop) + +## 2. Direct SSH (Low-Level) +For filesystem or network issues, use direct SSH to the production server: +`ssh root@192.168.68.69 ""` + +**Common Commands**: +- `docker ps` +- `docker logs --tail 100 ` +- `df -h` +- `top -b -n 1` (Never run interactive `top` or `nano` via agent commands). + +## 3. Safety Precautions +Always use non-interactive flags for SSH. Communicate clearly before executing high-risk actions like volume deletion or critical infrastructure restarts. diff --git a/.agents/rules/stakeholders-and-human-centered-design.md b/.agents/rules/stakeholders-and-human-centered-design.md new file mode 100644 index 00000000..b6b75b35 --- /dev/null +++ b/.agents/rules/stakeholders-and-human-centered-design.md @@ -0,0 +1,378 @@ +--- +trigger: model_decision +description: Human-centered design principles, stakeholder map, and decision matrix. +--- + +# Stakeholders & Human-Centered Design + + +> [!IMPORTANT] +> **This document is mandatory reading before any feature, UI, API, or architectural decision.** +> Every line of code in this project ultimately serves a human being. The AI agent MUST identify which stakeholder(s) a change affects and design the solution around their needs, abilities, and constraints — not the other way around. + +--- + +## Why This Document Exists + +Software that doesn't consider its humans fails. Not technically — socially. A plugin API that's elegant internally but hostile to external developers will die with zero third-party plugins. A monetization tier that ignores what users actually value will generate zero revenue. A contribution process that assumes PhD-level Cesium knowledge will attract zero contributors. + +**The directive is simple:** Before you write code, ask *"Who is this for, what do they need, and what will they actually experience?"* + +--- + +## The Stakeholder Universe + +WorldWideView is not a single-user tool. It is a **platform ecosystem** with at least nine distinct human roles, each with different goals, different technical abilities, different emotional states when they interact with the product, and different definitions of "success." + +``` + ┌──────────────────┐ + │ Owner / Founder │ + │ (silvertakana) │ + └────────┬─────────┘ + │ vision, architecture, deployment + ┌───────────────────┼───────────────────┐ + │ │ │ + ┌──────┴──────┐ ┌───────┴───────┐ ┌───────┴───────┐ + │ Core Devs │ │ Contributors │ │ Plugin Devs │ + │ (team) │ │ (community) │ │ (3rd party) │ + └──────┬──────┘ └───────┬───────┘ └───────┬───────┘ + │ │ │ + └───────────┬───────┘ │ + │ build the product │ extend the product + ┌───────────┴────────────────────────────┘ + │ + ┌──────┴──────────────────────────────────┐ + │ The Application │ + │ ┌────────┐ ┌────────┐ ┌───────────┐ │ + │ │ Demo │ │ Local │ │ Cloud │ │ + │ │Instance│ │Instance│ │ Instance │ │ + │ └───┬────┘ └───┬────┘ └─────┬─────┘ │ + └──────┼───────────┼─────────────┼────────┘ + │ │ │ + ┌──────┴──┐ ┌─────┴────┐ ┌────┴─────┐ + │Casual │ │Power User│ │Paying │ + │Visitor │ │(self-host)│ │Customer │ + └─────────┘ └──────────┘ └────┬─────┘ + │ + ┌────────┴────────┐ + │ Advertisers │ + │ (demo edition) │ + └─────────────────┘ +``` + +--- + +## Stakeholder Profiles + +### 1. Owner / Founder (silvertakana) + +| Attribute | Description | +|---|---| +| **Role** | Solo founder, architect, deployer, and operator of the entire ecosystem | +| **Goal** | Build a sustainable, revenue-generating geospatial intelligence platform that rivals WorldMonitor | +| **Motivations** | Technical excellence, creative ownership, financial independence, community building | +| **Technical Ability** | Expert — TypeScript, Next.js, CesiumJS, Docker, infrastructure, Prisma, AI-assisted development | +| **Emotional State** | Ambitious but resource-constrained; every wasted hour counts double | +| **Definition of Success** | Growing user base, marketplace with active plugin developers, recurring revenue, industry recognition | +| **Pain Points** | Single point of failure for ops, deployment, and architecture decisions; time pressure; hosting cost sensitivity ($0/mo target) | +| **Interaction Surface** | `.agents/` rules, ROADMAP.md, Coolify dashboard, code reviews, AI pair-programming sessions | + +**Design implications:** +- Automation is critical — CI/CD, self-healing databases, zero-config setup +- AI agent rules must preserve the owner's vision even when the owner isn't in the conversation +- Cost-efficient architecture (PostgreSQL, ThinkPad server, free-tier services) is a feature, not a compromise +- Every architectural decision must consider the owner's ability to maintain it solo + +--- + +### 2. Core Development Team (WWV Devs) + +| Attribute | Description | +|---|---| +| **Role** | Trusted developers with direct commit access to the monorepo | +| **Goal** | Ship features quickly without breaking existing functionality | +| **Motivations** | Clean code, clear architecture, fast iteration, not getting paged at 2am | +| **Technical Ability** | High — comfortable with TypeScript, Next.js App Router, Zustand, pnpm workspaces | +| **Emotional State** | Productive when patterns are clear; frustrated by undocumented gotchas | +| **Definition of Success** | PRs merged cleanly, features work on first deploy, zero regressions | +| **Pain Points** | File bloat (>150 lines), implicit state dependencies, CesiumJS rendering quirks, Docker build complexity | +| **Interaction Surface** | `src/core/`, `src/components/`, `packages/`, PR reviews, `pnpm dev` | + +**Design implications:** +- 150-line file limit is sacred — enforce it in every code change +- Clear slice boundaries in Zustand store prevent cross-team conflicts +- `AGENTS.md` and `.agents/` context must always be current so that AI-augmented devs have accurate context +- Rendering rules (billboard vs. point, never mix properties) must be enforced systematically +- Docker gotchas documented in `06-dev-workflows.md` prevent time-sink debugging + +--- + +### 3. Open Source Contributors + +| Attribute | Description | +|---|---| +| **Role** | External developers who fork, fix bugs, add features, or create plugins via PRs | +| **Goal** | Contribute meaningfully without needing to understand the entire 50-package monorepo | +| **Motivations** | Portfolio building, learning CesiumJS/geospatial tech, improving a tool they use, open source goodwill | +| **Technical Ability** | **Variable** — ranges from junior devs submitting typo fixes to senior engineers contributing optimizations | +| **Emotional State** | Enthusiastic but easily discouraged by onboarding friction; will abandon contribution if setup takes >15 minutes | +| **Definition of Success** | PR accepted and merged; their name in the contributors list; feature actually ships | +| **Pain Points** | Complex monorepo, CesiumJS learning curve, Elastic License 2.0 implications, unclear "where to start" | +| **Interaction Surface** | `CONTRIBUTING.md`, `docs/SETUP.md`, `docs/PLUGIN_GUIDE.md`, GitHub Issues, PR templates | + +**Design implications:** +- **One-command setup** (`pnpm run setup && pnpm dev:all`) is non-negotiable — if it breaks, contributors leave +- `CONTRIBUTING.md` must use `pnpm` (currently shows `npm` in places — this is a bug) +- "Good first issue" labels on GitHub attract newcomers +- Plugin creation is the #1 entry point for contributors — the scaffold script and Plugin Guide must be flawless +- The Elastic License 2.0 must be clearly communicated (it's NOT fully open source — contributors need to understand what they're signing up for) + +--- + +### 4. Third-Party Plugin Developers + +| Attribute | Description | +|---|---| +| **Role** | External developers who build and publish plugins to the WorldWideView Marketplace | +| **Goal** | Create a plugin that integrates their data source, potentially monetize it | +| **Motivations** | Revenue (marketplace cut), reach (installed on many WWV instances), reputation, data evangelism | +| **Technical Ability** | **Mid-to-high** — comfortable with TypeScript and npm, but NOT Cesium experts. They should never need to learn Cesium internals. | +| **Emotional State** | Evaluating whether the platform is worth investing development time in. Extremely sensitive to DX friction. | +| **Definition of Success** | Plugin published, visible in marketplace, installed by users, data renders correctly, optional revenue stream | +| **Pain Points** | (1) Backend hosting burden — they need a VPS for active plugins (the "VPS Problem"), (2) No distributable mini-engine yet, (3) SDK docs may not cover edge cases, (4) Rendering bugs with billboard/point property mixing | +| **Interaction Surface** | `@worldwideview/wwv-plugin-sdk`, `docs/PLUGIN_GUIDE.md`, `docs/creating-a-plugin.md`, marketplace submission UI, npm publishing | + +**Design implications:** +- The Plugin SDK is the **#1 most important public API surface** — it must be impeccably documented, versioned, and stable +- `WorldPlugin` interface changes are breaking changes that affect real people with real published packages +- Three plugin formats (Declarative, Static, Code Bundle) exist specifically to lower the barrier: + - **Declarative** = zero code (JSON only) — anyone with a data URL can create a plugin + - **Static** = GeoJSON file + manifest — data analysts who aren't developers + - **Code Bundle** = full TypeScript — professional developers +- The future **Mini-Engine Docker image** (§3 of `03-plugin-system.md`) is critical for reducing the VPS burden +- Rendering must be abstracted — plugin devs define `renderEntity()`, and the engine handles GPU batching +- Error messages from the plugin lifecycle must be clear, not cryptic Cesium stack traces +- Marketplace analytics (install counts, usage) are what keep plugin developers motivated + +--- + +### 5. Casual Visitors (Demo Users) + +| Attribute | Description | +|---|---| +| **Role** | Anonymous visitors who land on `demo.worldwideview.dev` — no account, no setup | +| **Goal** | Explore the globe, see cool real-time data, understand what WorldWideView is | +| **Motivations** | Curiosity, news events (war, natural disasters, flight tracking), linked from social media or Reddit | +| **Technical Ability** | **Low to none** — they don't know what ADS-B or GeoJSON means. They just want to see planes on a globe. | +| **Emotional State** | Impatient. They will leave within 10 seconds if the globe doesn't load or the UI is confusing. | +| **Definition of Success** | "Whoa, this is cool" → bookmark/share → return later → potentially convert to self-host or cloud user | +| **Pain Points** | Slow boot time (Cesium asset loading), confusing UX, data not loading (API rate limits), ads disrupting experience | +| **Interaction Surface** | Boot overlay, 3D globe, layer toggle panel, entity hover/click, info cards, timeline, ad strip | + +**Design implications:** +- **First 5 seconds are everything** — the boot overlay must feel cinematic, not broken +- Pre-activate compelling default layers (aviation is the universal crowd-pleaser) +- Globe interaction must work without instructions — left-click drag, scroll zoom, click entity +- The "Why ads?" banner (`DemoAdStrip`) is a trust-building honesty moment — keep it human and transparent +- Mobile layout must be flawless — many demo visitors come from social media on phones +- No login walls, no setup wizards, no configuration prompts — demo is zero-friction +- Performance on commodity hardware (integrated GPUs, 2017 laptops) must be acceptable +- Error states must be invisible — if OpenSky is down, the layer just shows fewer planes, not an error dialog + +--- + +### 6. Self-Hosted Power Users (Local Edition) + +| Attribute | Description | +|---|---| +| **Role** | Technical users who run WorldWideView on their own hardware (Raspberry Pi, home server, VPS) | +| **Goal** | Full control over their own geospatial intelligence dashboard, no cloud dependency | +| **Motivations** | Privacy, customization, data sovereignty, OSINT hobbyism, professional intelligence work | +| **Technical Ability** | **High** — comfortable with Docker, environment variables, CLI tools, possibly contributing plugins | +| **Emotional State** | Empowered but demanding. They chose self-hosting for a reason — they expect it to work without phoning home. | +| **Definition of Success** | Instance running 24/7, plugins installed and polling reliably, data persisted locally, zero external dependencies | +| **Pain Points** | Docker build complexity, PostgreSQL connection pooling issues, Cesium token setup, Prisma migration gotchas, plugin data not loading | +| **Interaction Surface** | `docker-compose.yml`, `.env.local`, Setup Wizard, Layers Panel, Config Panel, Marketplace Bridge, `pnpm run dev:all` | + +**Design implications:** +- **Zero-config first run** — `pnpm run setup && pnpm dev` or `docker-compose up` must "just work" +- The Setup Wizard (Cesium token prompt) is the ONLY mandatory configuration step — minimize it +- PostgreSQL must handle concurrent reads from the app + data engine efficiently +- Self-hosted users are the most likely to become contributors — treat their issues as high-priority +- Data engine self-healing (fallback seeds) ensures restarts don't lose data +- Offline capability is a differentiator — cached entities in IndexedDB survive network outages +- `NEXT_PUBLIC_WWV_EDITION=local` must be the default, most-tested path + +--- + +### 7. Paying Customers (Cloud Pro/Enterprise) + +| Attribute | Description | +|---|---| +| **Role** | Users or organizations paying for `[user].app.worldwideview.dev` — the revenue engine | +| **Goal** | Reliable, always-on geospatial intelligence with history, higher rate limits, and team access | +| **Motivations** | Professional OSINT, military/defense analysis, logistics monitoring, journalism, corporate security | +| **Technical Ability** | **Variable** — from journalists who just need a globe to security analysts who want API access | +| **Emotional State** | **Paying customers have the lowest tolerance for bugs.** They expect enterprise-grade uptime and support. | +| **Definition of Success** | Data is live, history works, team members can access, exports work, it doesn't go down | +| **Pain Points** | Downtime, stale data, missing features compared to competitors, unclear what "Pro" actually gives them | +| **Interaction Surface** | Cloud dashboard, license key input, plugin marketplace, support channels, billing UI | + +**Design implications:** +- **Uptime is king** — Uptime Kuma monitoring, error alerting, and rollback procedures exist for this reason +- Tier differentiation must be clear and valuable: + - Free: 3 users, 500MB, 24h history + - Pro: 20 users, 5GB, full history, snapshot capture + - Enterprise: unlimited, 50GB+, custom domains +- License key verification (RSA-signed JWT) prevents forgery but must not be onerous UX +- RLS tenant isolation is non-negotiable — data leaks between tenants are existential +- Response caching at the API proxy layer makes the $0/mo server viable for multiple tenants +- The cloud edition must feel **identical** to local — same UI, same plugins, same experience + +--- + +### 8. Advertisers (Demo Edition Revenue) + +| Attribute | Description | +|---|---| +| **Role** | Google AdSense (programmatic) — automated ad placement on the demo instance | +| **Goal** | Display relevant ads to engaged visitors, generate clicks/impressions | +| **Motivations** | ROI on ad spend; relevant audience targeting | +| **Technical Ability** | N/A — they interact through Google's ad platform, not with WWV directly | +| **Emotional State** | N/A — algorithmic. But their platform (AdSense) has strict technical requirements. | +| **Definition of Success** | Ads render correctly, non-intrusively, on viewable screen real estate | +| **Pain Points** | AdSense script crashes on resize, ads failing to fill on localhost/tunnels, ads overlapping interactive content | +| **Interaction Surface** | `DemoAdStrip.tsx`, `AdUnit.tsx`, Google AdSense publisher dashboard | + +**Design implications:** +- Ad strip is a **structurally separate sibling container** — never overlaps the globe viewport +- Fixed-position elements (header, timeline, sidebars) use `--ad-strip-inset` CSS variable to avoid overlap +- AdSense `push()` errors on resize are handled by fully unmounting ad units below 850px height +- Ads are **demo edition only** — they never appear in local or cloud editions +- The "Why ads?" banner is a conscious UX choice that builds trust with users and is dismissible +- Ad revenue is a temporary bridge to sustainability, not the long-term business model + +--- + +### 9. Data Source Providers (Upstream APIs) + +| Attribute | Description | +|---|---| +| **Role** | External services that provide the raw data: OpenSky, AIS providers, NASA FIRMS, ACLED, CelesTrak, GPSJam, etc. | +| **Goal** | Serve data within their rate limits and terms of service | +| **Motivations** | Open data mission (NASA, CelesTrak), freemium conversion (OpenSky), research community | +| **Technical Ability** | N/A — they expose APIs; WWV consumes them | +| **Emotional State** | Hostile if rate-limited. Neutral if respected. | +| **Definition of Success** | API consumers stay within rate limits, attribute correctly, and don't DDOS their endpoints | +| **Pain Points** | Credential rotation failures (OpenSky), rate limit exhaustion, unattributed scraping | +| **Interaction Surface** | Data Engine seeders, API route proxies, `OPENSKY_CREDENTIALS` rotation, `pollingIntervalMs` settings | + +**Design implications:** +- **Respect upstream rate limits** — this is an ethical and practical obligation +- Credential rotation (`OPENSKY_CREDENTIALS` comma-separated pairs) prevents per-key throttling +- The Data Engine's 5-minute Redis throttle (`setLiveSnapshot`) is specifically designed to stay within Upstash free-tier limits +- API proxy caching (1000 users → 1 upstream request) is architecturally essential +- Attribution (e.g., "Data: OpenSky Network") should be visible in entity detail panels +- Graceful degradation when APIs go down — show cached data, not error modals + +--- + +## The Stakeholder Decision Matrix + +When making any decision, use this matrix to identify impact: + +| Decision Area | Owner | Core Devs | Contributors | Plugin Devs | Casual Visitors | Power Users | Paying Customers | Advertisers | Data Providers | +|---|---|---|---|---|---|---|---|---|---| +| **Plugin SDK change** | ⚠️ | ⚠️ | ⬜ | 🔴 | ⬜ | ⬜ | ⬜ | ⬜ | ⬜ | +| **UI/UX redesign** | ⚠️ | ⬜ | ⬜ | ⬜ | 🔴 | ⚠️ | 🔴 | ⚠️ | ⬜ | +| **Rendering pipeline change** | ⚠️ | 🔴 | ⬜ | ⚠️ | ⚠️ | ⚠️ | ⚠️ | ⬜ | ⬜ | +| **Database schema migration** | ⚠️ | ⚠️ | ⬜ | ⬜ | ⬜ | 🔴 | 🔴 | ⬜ | ⬜ | +| **New data source plugin** | ✅ | ⬜ | ✅ | ⬜ | ✅ | ✅ | ✅ | ⬜ | ⚠️ | +| **Pricing/tier change** | 🔴 | ⬜ | ⬜ | ⚠️ | ⬜ | ⚠️ | 🔴 | ⬜ | ⬜ | +| **Ad placement change** | ⚠️ | ⬜ | ⬜ | ⬜ | ⚠️ | ⬜ | ⬜ | 🔴 | ⬜ | +| **API rate limit adjustment** | ⬜ | ⬜ | ⬜ | ⬜ | ⬜ | ⬜ | ⚠️ | ⬜ | 🔴 | +| **Marketplace policy** | 🔴 | ⬜ | ⬜ | 🔴 | ⬜ | ⬜ | ⚠️ | ⬜ | ⬜ | +| **Docker/deployment change** | 🔴 | ⚠️ | ⬜ | ⬜ | ⬜ | 🔴 | ⚠️ | ⬜ | ⬜ | +| **License change** | 🔴 | ⚠️ | 🔴 | 🔴 | ⬜ | ⚠️ | ⬜ | ⬜ | ⬜ | + +**Legend:** 🔴 = Primary impact (design for this person) | ⚠️ = Secondary impact (consider carefully) | ✅ = Positive impact | ⬜ = Minimal/no impact + +--- + +## The Golden Rules + +### 1. Every Feature Has a Human + +Before writing a single line of code, the AI agent MUST answer: +- **WHO** is this for? (Name the stakeholder) +- **WHY** do they need it? (What problem does it solve for them?) +- **HOW** will they discover/use it? (What's their journey?) +- **WHAT** happens when it breaks? (Graceful degradation plan) + +### 2. Reduce Friction, Not Features + +The most dangerous failure mode is building powerful systems with hostile interfaces. Apply the **Principle of Least Effort**: +- Contributors: one-command setup, not a wiki page of prerequisites +- Plugin devs: `renderEntity()` returns an object, not a Cesium Primitive +- Casual visitors: zero onboarding, zero configuration +- Power users: sensible defaults, optional configuration +- Paying customers: it just works, always + +### 3. Design for the Least Technical User in Each Cohort + +| Cohort | Least Technical Member | Design Target | +|---|---|---| +| Plugin Developers | Data analyst who can write JSON | Declarative plugin format | +| Contributors | Junior dev on their first open source PR | Scaffold scripts + clear CONTRIBUTING.md | +| Casual Visitors | Non-technical person who followed a Twitter link | Auto-loading layers, intuitive globe controls | +| Power Users | Sysadmin who can `docker-compose up` but doesn't know Next.js | Pre-built Docker images, `.env` only config | +| Paying Customers | Journalist tracking a conflict | Clear UI, no jargon, exportable data | + +### 4. Protect the Ecosystem Trust + +The trust hierarchy is cryptographically enforced (Ed25519 signed registry), but trust is also social: +- **Built-in plugins** must always work — they're the first impression +- **Verified plugins** carry WorldWideView's reputation — review them seriously +- **Unverified plugins** get a warning dialog — users make informed decisions +- **Plugin developers** who see installs and analytics stay motivated +- **Data providers** who aren't rate-limit abused will keep serving data + +### 5. Revenue Serves Sustainability, Not Extraction + +The business model exists to keep the project alive: +- Ads on demo = temporary hosting cost recovery (transparent "Why ads?" banner) +- Cloud Pro = value exchange (history, higher limits, team access) +- Marketplace cut = sustainable ecosystem (Stripe Connect revenue split) +- Plugin developer monetization = VS Code model (developers set their own prices) + +Never design a feature that extracts value from users without returning it. The demo must always be useful, not crippled. + +--- + +## Applying This Document + +### For the AI Agent + +When the agent is asked to implement a feature, modify architecture, or solve a bug, it MUST: + +1. **Identify affected stakeholders** from the list above +2. **Consider the least-technical user** in each affected cohort +3. **Check the decision matrix** for impact severity +4. **Apply the golden rules** to the implementation approach +5. **Document stakeholder considerations** in PR descriptions and implementation plans + +### For Human Developers + +When reviewing PRs or proposing features, ask: +- "Did we consider how this affects plugin developers?" +- "Will a first-time contributor understand this?" +- "Does this work on the demo without configuration?" +- "Will this survive a Docker restart on the ThinkPad?" + +--- + +## Version History + +| Date | Change | +|---|---| +| 2026-04-05 | Initial stakeholder map created from comprehensive project research | diff --git a/.agents/rules/troubleshooting-and-debugging.md b/.agents/rules/troubleshooting-and-debugging.md new file mode 100644 index 00000000..5c1ff482 --- /dev/null +++ b/.agents/rules/troubleshooting-and-debugging.md @@ -0,0 +1,22 @@ +--- +trigger: model_decision +description: Troubleshooting and debugging guidelines for WorldWideView, covering latency, logging, and namespace collisions. +--- + +# Troubleshooting and Debugging + +## 1. Plugin Data Latency +If there is a noticeable delay (e.g., ~1 second) between enabling a plugin and the initial data arriving on the globe, **do not assume the UI is blocking**. +- **WebSocket Handshake**: Initial latency is often caused by the time it takes to establish a new WebSocket connection to the engine's `/stream`. +- **Upstream Cooldowns**: Plugins querying external providers (like OpenSky for `aviation`) have strict rate limits. If a snapshot returns a `404`, it might mean the seeder is in a cooldown period or hasn't successfully fetched data yet. This is expected behavior. +- **Diagnosis**: Instrument the `DataBusSubscriber` and `WsClient` with timing logs (`performance.now()`) to track exactly where the bottleneck occurs (manifest discovery vs connection establishment vs first payload). + +## 2. Log Verbosity and Structured Logging +We employ a **structured logging strategy** in the frontend to reduce noise and improve readability. +- **Rule**: Do not add excessive `console.log` statements that dump raw JSON payloads or stream contents directly to the console in core files like `WsClient.ts` or `PluginManager.ts`. +- **Implementation**: If detailed logs are needed, use conditional debug levels or summarize the output (e.g., logging payload sizes or entity counts instead of the full object array). This ensures critical diagnostic information remains visible without overflowing the terminal. + +## 3. Namespace Collisions +If a plugin fails to load in the Data Engine with a `404` or `Error [ERR_MODULE_NOT_FOUND]` error during startup: +- **Root Cause**: The seeder may exist simultaneously in both the `wwv-seeders-community` and `wwv-seeders-private` repositories. +- **Resolution**: Seeders must have a unique namespace. Ensure that private plugins (e.g., `aviation`, `maritime`, `military-aviation`) are explicitly removed from the community repository to prevent workspace-level module resolution failures when the V2 engine attempts to load them. Private seeders take priority, but overlapping names will cause dependency conflicts. diff --git a/.agents/settings.local.json b/.agents/settings.local.json new file mode 100644 index 00000000..70ffd1ee --- /dev/null +++ b/.agents/settings.local.json @@ -0,0 +1,17 @@ +{ + "env": { + "CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS": "1" + }, + "permissions": { + "allow": [ + "Read(src/**)", + "Bash(npm test)" + ], + "deny": [ + "Read(.env)", + "Write(config.json)" + ], + "defaultMode": "bypassPermissions" + }, + "prefersReducedMotion": false +} diff --git a/.agents/skills/migrate-legacy-plugin/SKILL.md b/.agents/skills/migrate-legacy-plugin/SKILL.md new file mode 100644 index 00000000..be9c1532 --- /dev/null +++ b/.agents/skills/migrate-legacy-plugin/SKILL.md @@ -0,0 +1,257 @@ +--- +name: migrate-legacy-plugin +description: Use when moving a WorldWideView plugin from packages/ (or ../worldwideview-plugins/packages/) to local-plugins/, converting legacy plugins to the new Engine & Payload architecture, or when a plugin hardcodes engine URLs, bundles Node.js built-ins, or uses import.meta.url for file paths inside Docker containers +--- + +# Migrate Legacy Plugin to Modern Architecture + +## Overview + +Moves a WorldWideView plugin from the old `packages/` monorepo location (or the external `worldwideview-plugins` repository) into the `local-plugins/` directory (which is itself a publishable Git repository). Simultaneously fixes two **independent** concerns that legacy plugins always have: + +1. **Frontend routing** — hardcoded engine URLs must be replaced with `resolveEngineUrl()` +2. **Backend seeder build** — ESM bundling and container-aware file paths + +**Core principle:** These are two separate problems in two separate codebases. Never fix one and forget the other. + +## When to Use + +- Moving a plugin from `packages/wwv-plugin-/` (or `../worldwideview-plugins/packages/wwv-plugin-/`) to `local-plugins/wwv-plugin-/` +- Plugin's `fetch()` hardcodes a URL like `https://dataengine.worldwideview.dev` +- Plugin uses `this.context?.apiBaseUrl` instead of `resolveEngineUrl()` +- Backend seeder crashes with `Dynamic require` or `ERR_INVALID_FILE_URL_PATH` in Docker +- Seeder uses `fileURLToPath(import.meta.url)` for locating seed data files +- Plugin builds with `tsc` instead of `tsup`/`vite` + +**Do NOT use for:** +- Creating a brand-new plugin from scratch → use `worldwideview-plugin-creation` skill +- Modifying rendering or Cesium primitives → see `cesium-rendering` rule +- Debugging state management → see `state-management` rule + +## The Two Independent Concerns + +> [!TIP] +> **Automated Migration Script** +> A new automation script is available! Before doing this manually, simply run: +> ```bash +> node scripts/migrate-plugin.mjs +> ``` +> Example: `node scripts/migrate-plugin.mjs gps-jamming` +> +> The script will automatically copy the frontend plugin, inject peerDependencies, refactor `index.tsx` routing, fix the backend seeder's build settings and `SEEDERS_DIR` paths, and register the plugin in `defaultPlugins.ts`. +> **You still need to run `pnpm install` and verify the builds manually as described below.** + +> [!IMPORTANT] +> Legacy plugins always have BOTH of these problems. They are independent — you can fix one without the other, but you MUST fix both. Do not get tunnel-visioned into only fixing one. + +| Concern | Where | Symptom | Fix | +|---|---|---|---| +| **Frontend routing** | `local-plugins/wwv-plugin-/src/index.ts` | Plugin always hits cloud engine, ignores local dev engine | Use `context.getEngineUrl()` | +| **Backend build** | `local-seeders//packages//` | Container crashes on startup with ESM errors | `tsup --format esm --target es2022`, use `SEEDERS_DIR` | + +--- + +## Step 1: Move the Plugin Directory + +Copy the plugin from `packages/` (or the external plugins repo) to `local-plugins/`: + +```bash +# From project root (if in local packages/) +cp -r packages/wwv-plugin- local-plugins/wwv-plugin- + +# OR from external plugins repository: +cp -r ../worldwideview-plugins/packages/wwv-plugin- local-plugins/wwv-plugin- +``` + +The `local-plugins/` directory is: +- Part of the pnpm workspace (`pnpm-workspace.yaml` includes `local-plugins/*`) +- Gitignored from the main worldwideview repo (it IS its own Git repo — `worldwideview-plugins`) +- Auto-discovered by the `dev:plugins` watcher during `pnpm dev` + +## Step 2: Fix Frontend Routing (getEngineUrl) + +This is the fix that gets missed most often. Legacy plugins typically hardcode their engine URL or use messy dynamic imports. + +```typescript +// ❌ LEGACY — always hits cloud, ignores local engine +const baseUrl = this.context?.apiBaseUrl || "https://dataengine.worldwideview.dev"; +const res = await fetch(`${baseUrl}/api/myplugin`); + +// ❌ BAD (Previous Workaround) — creates bundling chunks and causes 404s +const { resolveEngineUrl } = await import("@/core/data/resolveEngineUrl"); +const wsUrl = resolveEngineUrl(this.id); +const engineBase = wsUrl.replace(/^ws/, 'http').replace(/\/stream$/, ''); +const res = await fetch(`${engineBase}/api/myplugin`); +``` + +The correct approach uses `this.context.getEngineUrl()` provided by the Host when the plugin is initialized. The Host automatically handles checking `localhost:5000/manifest`, applying `getServerConfig().streamUrl`, reading `plugin.json`, or falling back to the cloud. + +```typescript +// ✅ MODERN — resolves local-first, respects split-routing, clean code +async fetch(_timeRange: TimeRange): Promise { + try { + const engineBase = this.context!.getEngineUrl(); + const res = await globalThis.fetch(`${engineBase}/api/${this.id}`); + if (!res.ok) throw new Error(`Backend returned ${res.status}`); + + const data = await res.json(); + return this.mapPayloadToEntities(data); + } catch (err) { + console.error(`[${this.id}] Fetch error:`, err); + return []; + } +} +``` + +If the plugin is WebSocket-only (`getPollingInterval()` returns 0 and `fetch()` returns `[]`), the frontend routing is handled automatically by `DataBusSubscriber` and `WsClient`. No changes needed to the plugin class — but verify the seeder ID matches. + +### Why This Matters + +Without this fix, running `pnpm dev:all` starts a local engine at `localhost:5000` with your seeder, but the frontend plugin ignores it and hits the cloud. You'll see stale or missing data and waste time debugging the wrong layer. + +## Step 3: Update package.json + +Ensure the plugin's `package.json` follows the modern format: + +```json +{ + "name": "@worldwideview/wwv-plugin-", + "version": "1.0.0", + "main": "dist/frontend.mjs", + "types": "src/index.ts", + "type": "module", + "module": "dist/frontend.mjs", + "files": ["src"], + "worldwideview": { + "id": "", + "icon": "IconName", + "category": "category-name", + "format": "bundle", + "capabilities": ["layer"] + }, + "peerDependencies": { + "@worldwideview/wwv-plugin-sdk": "^1.4.10" + }, + "scripts": { + "build": "vite build" + } +} +``` + +Key differences from legacy: +- `"main"` points to `dist/frontend.mjs` (not `src/index.ts`) +- Has `"type": "module"` and `"module"` field +- Uses `peerDependencies` (not `dependencies`) for SDK +- Uses `"workspace:*"` only for libraries that must be linked locally (e.g., `wwv-lib-incidents`) + +## Step 4: Ensure Vite Build Config + +Every `local-plugins/` plugin needs this `vite.config.ts`: + +```typescript +import { defineConfig } from "vite"; +import { wwvPluginGlobals } from "@worldwideview/wwv-plugin-sdk"; + +export default defineConfig({ + plugins: [wwvPluginGlobals()], + build: { + lib: { + entry: "src/index.ts", + formats: ["es"], + fileName: () => "frontend.mjs", + }, + minify: true, + sourcemap: false, + }, +}); +``` + +`wwvPluginGlobals()` externalizes React, Cesium, Resium, zustand, and the SDK so the plugin doesn't bundle its own copies. + +## Step 5: Fix Backend Seeder (if applicable) + +If the plugin has a corresponding seeder in `local-seeders/`, fix two things: + +### 5a. ESM Build + +Legacy seeders built with `tsc` or bundled Node built-ins. Fix: + +```json +"scripts": { + "build": "tsup src/index.ts --format esm --target es2022 --clean" +} +``` + +This produces clean ESM that the data engine can `import()` without `Dynamic require` errors. + +### 5b. Container-Aware File Paths + +Legacy seeders used `fileURLToPath(import.meta.url)` to locate seed data. This breaks inside Docker because the transpiled path doesn't match the container filesystem. + +```typescript +// ❌ LEGACY — breaks in Docker container +import { fileURLToPath } from "url"; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const dataPath = path.join(__dirname, "data", "seed.json"); + +// ✅ MODERN — works everywhere +const dataPath = process.env.SEEDERS_DIR + ? path.join(process.env.SEEDERS_DIR, "community", "packages", "", "data", "seed.json") + : path.resolve(process.cwd(), "data", "seed.json"); +``` + +`SEEDERS_DIR` is set by the Docker container to `/app/seeders`. When running locally outside Docker, the fallback uses `process.cwd()`. + +### 5c. Seeder Directory Naming + +The folder name inside `local-seeders/community/packages/` or `local-seeders/private/packages/` **must exactly match** the plugin's string ID (`this.id` in `index.tsx`). + +The `wwv-data-engine`'s auto-discovery logic (`seeder-loader.ts`) derives the internal plugin ID from the directory name. If the folder is named `gpsjam` but the frontend plugin expects `gps-jamming`, the frontend will not find it in the `/manifest` and will silently fall back to the cloud engine, breaking local testing. + +```bash +# ❌ INCORRECT (Frontend expects "gps-jamming") +local-seeders/community/packages/gpsjam/ + +# ✅ CORRECT +local-seeders/community/packages/gps-jamming/ +``` + +## Step 6: Clean Up packages/ References + +After moving to `local-plugins/`, remove the old references: + +- [ ] Delete `packages/wwv-plugin-/` directory (or the source directory in `../worldwideview-plugins/packages/`) +- [ ] Remove from `transpilePackages` in `next.config.ts` (if present) +- [ ] Remove path alias from `tsconfig.json` `paths` (if present) +- [ ] Remove any registration in `AppShell.tsx` (now loaded dynamically) +- [ ] Run `pnpm install` from project root + +## Step 7: Verify + +```bash +# 1. Start everything +pnpm dev:all + +# 2. Check engine discovered the seeder (if applicable) +curl http://localhost:5000/manifest +# Should include your plugin ID + +# 3. Check frontend console — should show: +# [EngineManifest] Local engine detected: N seeders ["your-plugin", ...] +# NOT: fetching from https://dataengine.worldwideview.dev + +# 4. Toggle the layer on the globe — entities should render +``` + +## Migration Checklist + +- [ ] Plugin copied from `packages/` (or external repo) to `local-plugins/` +- [ ] Frontend `fetch()` uses `this.context.getEngineUrl()` (not hardcoded URLs or dynamic imports) +- [ ] `package.json` has `"type": "module"`, `"main": "dist/frontend.mjs"` +- [ ] `vite.config.ts` uses `wwvPluginGlobals()` externalization +- [ ] Backend seeder builds with `tsup --format esm --target es2022` (if applicable) +- [ ] Backend seeder uses `process.env.SEEDERS_DIR` for file paths (if applicable) +- [ ] Old source directory deleted +- [ ] Old `transpilePackages`/`tsconfig paths` entries removed +- [ ] `pnpm install` run from root +- [ ] `pnpm dev:all` → local engine detected → data renders on globe diff --git a/.agents/skills/supabase-postgres-best-practices/SKILL.md b/.agents/skills/supabase-postgres-best-practices/SKILL.md new file mode 100644 index 00000000..d9ef1949 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/SKILL.md @@ -0,0 +1,64 @@ +--- +name: supabase-postgres-best-practices +description: Postgres performance optimization and best practices from Supabase. Use this skill when writing, reviewing, or optimizing Postgres queries, schema designs, or database configurations. +license: MIT +metadata: + author: supabase + version: "1.1.1" + organization: Supabase + date: January 2026 + abstract: Comprehensive Postgres performance optimization guide for developers using Supabase and Postgres. Contains performance rules across 8 categories, prioritized by impact from critical (query performance, connection management) to incremental (advanced features). Each rule includes detailed explanations, incorrect vs. correct SQL examples, query plan analysis, and specific performance metrics to guide automated optimization and code generation. +--- + +# Supabase Postgres Best Practices + +Comprehensive performance optimization guide for Postgres, maintained by Supabase. Contains rules across 8 categories, prioritized by impact to guide automated query optimization and schema design. + +## When to Apply + +Reference these guidelines when: +- Writing SQL queries or designing schemas +- Implementing indexes or query optimization +- Reviewing database performance issues +- Configuring connection pooling or scaling +- Optimizing for Postgres-specific features +- Working with Row-Level Security (RLS) + +## Rule Categories by Priority + +| Priority | Category | Impact | Prefix | +|----------|----------|--------|--------| +| 1 | Query Performance | CRITICAL | `query-` | +| 2 | Connection Management | CRITICAL | `conn-` | +| 3 | Security & RLS | CRITICAL | `security-` | +| 4 | Schema Design | HIGH | `schema-` | +| 5 | Concurrency & Locking | MEDIUM-HIGH | `lock-` | +| 6 | Data Access Patterns | MEDIUM | `data-` | +| 7 | Monitoring & Diagnostics | LOW-MEDIUM | `monitor-` | +| 8 | Advanced Features | LOW | `advanced-` | + +## How to Use + +Read individual rule files for detailed explanations and SQL examples: + +``` +references/query-missing-indexes.md +references/query-partial-indexes.md +references/_sections.md +``` + +Each rule file contains: +- Brief explanation of why it matters +- Incorrect SQL example with explanation +- Correct SQL example with explanation +- Optional EXPLAIN output or metrics +- Additional context and references +- Supabase-specific notes (when applicable) + +## References + +- https://www.postgresql.org/docs/current/ +- https://supabase.com/docs +- https://wiki.postgresql.org/wiki/Performance_Optimization +- https://supabase.com/docs/guides/database/overview +- https://supabase.com/docs/guides/auth/row-level-security diff --git a/.agents/skills/supabase-postgres-best-practices/references/_contributing.md b/.agents/skills/supabase-postgres-best-practices/references/_contributing.md new file mode 100644 index 00000000..1c055aff --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_contributing.md @@ -0,0 +1,170 @@ +# Writing Guidelines for Postgres References + +This document provides guidelines for creating effective Postgres best +practice references that work well with AI agents and LLMs. + +## Key Principles + +### 1. Concrete Transformation Patterns + +Show exact SQL rewrites. Avoid philosophical advice. + +**Good:** "Use `WHERE id = ANY(ARRAY[...])` instead of +`WHERE id IN (SELECT ...)`" **Bad:** "Design good schemas" + +### 2. Error-First Structure + +Always show the problematic pattern first, then the solution. This trains agents +to recognize anti-patterns. + +```markdown +**Incorrect (sequential queries):** [bad example] + +**Correct (batched query):** [good example] +``` + +### 3. Quantified Impact + +Include specific metrics. Helps agents prioritize fixes. + +**Good:** "10x faster queries", "50% smaller index", "Eliminates N+1" +**Bad:** "Faster", "Better", "More efficient" + +### 4. Self-Contained Examples + +Examples should be complete and runnable (or close to it). Include `CREATE TABLE` +if context is needed. + +```sql +-- Include table definition when needed for clarity +CREATE TABLE users ( + id bigint PRIMARY KEY, + email text NOT NULL, + deleted_at timestamptz +); + +-- Now show the index +CREATE INDEX users_active_email_idx ON users(email) WHERE deleted_at IS NULL; +``` + +### 5. Semantic Naming + +Use meaningful table/column names. Names carry intent for LLMs. + +**Good:** `users`, `email`, `created_at`, `is_active` +**Bad:** `table1`, `col1`, `field`, `flag` + +--- + +## Code Example Standards + +### SQL Formatting + +```sql +-- Use lowercase keywords, clear formatting +CREATE INDEX CONCURRENTLY users_email_idx + ON users(email) + WHERE deleted_at IS NULL; + +-- Not cramped or ALL CAPS +CREATE INDEX CONCURRENTLY USERS_EMAIL_IDX ON USERS(EMAIL) WHERE DELETED_AT IS NULL; +``` + +### Comments + +- Explain _why_, not _what_ +- Highlight performance implications +- Point out common pitfalls + +### Language Tags + +- `sql` - Standard SQL queries +- `plpgsql` - Stored procedures/functions +- `typescript` - Application code (when needed) +- `python` - Application code (when needed) + +--- + +## When to Include Application Code + +**Default: SQL Only** + +Most references should focus on pure SQL patterns. This keeps examples portable. + +**Include Application Code When:** + +- Connection pooling configuration +- Transaction management in application context +- ORM anti-patterns (N+1 in Prisma/TypeORM) +- Prepared statement usage + +**Format for Mixed Examples:** + +````markdown +**Incorrect (N+1 in application):** + +```typescript +for (const user of users) { + const posts = await db.query("SELECT * FROM posts WHERE user_id = $1", [ + user.id, + ]); +} +``` +```` + +**Correct (batch query):** + +```typescript +const posts = await db.query("SELECT * FROM posts WHERE user_id = ANY($1)", [ + userIds, +]); +``` + +--- + +## Impact Level Guidelines + +| Level | Improvement | Use When | +|-------|-------------|----------| +| **CRITICAL** | 10-100x | Missing indexes, connection exhaustion, sequential scans on large tables | +| **HIGH** | 5-20x | Wrong index types, poor partitioning, missing covering indexes | +| **MEDIUM-HIGH** | 2-5x | N+1 queries, inefficient pagination, RLS optimization | +| **MEDIUM** | 1.5-3x | Redundant indexes, query plan instability | +| **LOW-MEDIUM** | 1.2-2x | VACUUM tuning, configuration tweaks | +| **LOW** | Incremental | Advanced patterns, edge cases | + +--- + +## Reference Standards + +**Primary Sources:** + +- Official Postgres documentation +- Supabase documentation +- Postgres wiki +- Established blogs (2ndQuadrant, Crunchy Data) + +**Format:** + +```markdown +Reference: +[Postgres Indexes](https://www.postgresql.org/docs/current/indexes.html) +``` + +--- + +## Review Checklist + +Before submitting a reference: + +- [ ] Title is clear and action-oriented +- [ ] Impact level matches the performance gain +- [ ] impactDescription includes quantification +- [ ] Explanation is concise (1-2 sentences) +- [ ] Has at least 1 **Incorrect** SQL example +- [ ] Has at least 1 **Correct** SQL example +- [ ] SQL uses semantic naming +- [ ] Comments explain _why_, not _what_ +- [ ] Trade-offs mentioned if applicable +- [ ] Reference links included +- [ ] `pnpm test` passes diff --git a/.agents/skills/supabase-postgres-best-practices/references/_sections.md b/.agents/skills/supabase-postgres-best-practices/references/_sections.md new file mode 100644 index 00000000..8ba57c23 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_sections.md @@ -0,0 +1,39 @@ +# Section Definitions + +This file defines the rule categories for Postgres best practices. Rules are automatically assigned to sections based on their filename prefix. + +Take the examples below as pure demonstrative. Replace each section with the actual rule categories for Postgres best practices. + +--- + +## 1. Query Performance (query) +**Impact:** CRITICAL +**Description:** Slow queries, missing indexes, inefficient query plans. The most common source of Postgres performance issues. + +## 2. Connection Management (conn) +**Impact:** CRITICAL +**Description:** Connection pooling, limits, and serverless strategies. Critical for applications with high concurrency or serverless deployments. + +## 3. Security & RLS (security) +**Impact:** CRITICAL +**Description:** Row-Level Security policies, privilege management, and authentication patterns. + +## 4. Schema Design (schema) +**Impact:** HIGH +**Description:** Table design, index strategies, partitioning, and data type selection. Foundation for long-term performance. + +## 5. Concurrency & Locking (lock) +**Impact:** MEDIUM-HIGH +**Description:** Transaction management, isolation levels, deadlock prevention, and lock contention patterns. + +## 6. Data Access Patterns (data) +**Impact:** MEDIUM +**Description:** N+1 query elimination, batch operations, cursor-based pagination, and efficient data fetching. + +## 7. Monitoring & Diagnostics (monitor) +**Impact:** LOW-MEDIUM +**Description:** Using pg_stat_statements, EXPLAIN ANALYZE, metrics collection, and performance diagnostics. + +## 8. Advanced Features (advanced) +**Impact:** LOW +**Description:** Full-text search, JSONB optimization, PostGIS, extensions, and advanced Postgres features. diff --git a/.agents/skills/supabase-postgres-best-practices/references/_template.md b/.agents/skills/supabase-postgres-best-practices/references/_template.md new file mode 100644 index 00000000..91ace90e --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_template.md @@ -0,0 +1,34 @@ +--- +title: Clear, Action-Oriented Title (e.g., "Use Partial Indexes for Filtered Queries") +impact: MEDIUM +impactDescription: 5-20x query speedup for filtered queries +tags: indexes, query-optimization, performance +--- + +## [Rule Title] + +[1-2 sentence explanation of the problem and why it matters. Focus on performance impact.] + +**Incorrect (describe the problem):** + +```sql +-- Comment explaining what makes this slow/problematic +CREATE INDEX users_email_idx ON users(email); + +SELECT * FROM users WHERE email = 'user@example.com' AND deleted_at IS NULL; +-- This scans deleted records unnecessarily +``` + +**Correct (describe the solution):** + +```sql +-- Comment explaining why this is better +CREATE INDEX users_active_email_idx ON users(email) WHERE deleted_at IS NULL; + +SELECT * FROM users WHERE email = 'user@example.com' AND deleted_at IS NULL; +-- Only indexes active users, 10x smaller index, faster queries +``` + +[Optional: Additional context, edge cases, or trade-offs] + +Reference: [Postgres Docs](https://www.postgresql.org/docs/current/) diff --git a/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md b/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md new file mode 100644 index 00000000..582cbeaa --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md @@ -0,0 +1,55 @@ +--- +title: Use tsvector for Full-Text Search +impact: MEDIUM +impactDescription: 100x faster than LIKE, with ranking support +tags: full-text-search, tsvector, gin, search +--- + +## Use tsvector for Full-Text Search + +LIKE with wildcards can't use indexes. Full-text search with tsvector is orders of magnitude faster. + +**Incorrect (LIKE pattern matching):** + +```sql +-- Cannot use index, scans all rows +select * from articles where content like '%postgresql%'; + +-- Case-insensitive makes it worse +select * from articles where lower(content) like '%postgresql%'; +``` + +**Correct (full-text search with tsvector):** + +```sql +-- Add tsvector column and index +alter table articles add column search_vector tsvector + generated always as (to_tsvector('english', coalesce(title,'') || ' ' || coalesce(content,''))) stored; + +create index articles_search_idx on articles using gin (search_vector); + +-- Fast full-text search +select * from articles +where search_vector @@ to_tsquery('english', 'postgresql & performance'); + +-- With ranking +select *, ts_rank(search_vector, query) as rank +from articles, to_tsquery('english', 'postgresql') query +where search_vector @@ query +order by rank desc; +``` + +Search multiple terms: + +```sql +-- AND: both terms required +to_tsquery('postgresql & performance') + +-- OR: either term +to_tsquery('postgresql | mysql') + +-- Prefix matching +to_tsquery('post:*') +``` + +Reference: [Full Text Search](https://supabase.com/docs/guides/database/full-text-search) diff --git a/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md b/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md new file mode 100644 index 00000000..e3d261ea --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md @@ -0,0 +1,49 @@ +--- +title: Index JSONB Columns for Efficient Querying +impact: MEDIUM +impactDescription: 10-100x faster JSONB queries with proper indexing +tags: jsonb, gin, indexes, json +--- + +## Index JSONB Columns for Efficient Querying + +JSONB queries without indexes scan the entire table. Use GIN indexes for containment queries. + +**Incorrect (no index on JSONB):** + +```sql +create table products ( + id bigint primary key, + attributes jsonb +); + +-- Full table scan for every query +select * from products where attributes @> '{"color": "red"}'; +select * from products where attributes->>'brand' = 'Nike'; +``` + +**Correct (GIN index for JSONB):** + +```sql +-- GIN index for containment operators (@>, ?, ?&, ?|) +create index products_attrs_gin on products using gin (attributes); + +-- Now containment queries use the index +select * from products where attributes @> '{"color": "red"}'; + +-- For specific key lookups, use expression index +create index products_brand_idx on products ((attributes->>'brand')); +select * from products where attributes->>'brand' = 'Nike'; +``` + +Choose the right operator class: + +```sql +-- jsonb_ops (default): supports all operators, larger index +create index idx1 on products using gin (attributes); + +-- jsonb_path_ops: only @> operator, but 2-3x smaller index +create index idx2 on products using gin (attributes jsonb_path_ops); +``` + +Reference: [JSONB Indexes](https://www.postgresql.org/docs/current/datatype-json.html#JSON-INDEXING) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md b/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md new file mode 100644 index 00000000..40b9cc50 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md @@ -0,0 +1,46 @@ +--- +title: Configure Idle Connection Timeouts +impact: HIGH +impactDescription: Reclaim 30-50% of connection slots from idle clients +tags: connections, timeout, idle, resource-management +--- + +## Configure Idle Connection Timeouts + +Idle connections waste resources. Configure timeouts to automatically reclaim them. + +**Incorrect (connections held indefinitely):** + +```sql +-- No timeout configured +show idle_in_transaction_session_timeout; -- 0 (disabled) + +-- Connections stay open forever, even when idle +select pid, state, state_change, query +from pg_stat_activity +where state = 'idle in transaction'; +-- Shows transactions idle for hours, holding locks +``` + +**Correct (automatic cleanup of idle connections):** + +```sql +-- Terminate connections idle in transaction after 30 seconds +alter system set idle_in_transaction_session_timeout = '30s'; + +-- Terminate completely idle connections after 10 minutes +alter system set idle_session_timeout = '10min'; + +-- Reload configuration +select pg_reload_conf(); +``` + +For pooled connections, configure at the pooler level: + +```ini +# pgbouncer.ini +server_idle_timeout = 60 +client_idle_timeout = 300 +``` + +Reference: [Connection Timeouts](https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-IDLE-IN-TRANSACTION-SESSION-TIMEOUT) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md b/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md new file mode 100644 index 00000000..cb3e400c --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md @@ -0,0 +1,44 @@ +--- +title: Set Appropriate Connection Limits +impact: CRITICAL +impactDescription: Prevent database crashes and memory exhaustion +tags: connections, max-connections, limits, stability +--- + +## Set Appropriate Connection Limits + +Too many connections exhaust memory and degrade performance. Set limits based on available resources. + +**Incorrect (unlimited or excessive connections):** + +```sql +-- Default max_connections = 100, but often increased blindly +show max_connections; -- 500 (way too high for 4GB RAM) + +-- Each connection uses 1-3MB RAM +-- 500 connections * 2MB = 1GB just for connections! +-- Out of memory errors under load +``` + +**Correct (calculate based on resources):** + +```sql +-- Formula: max_connections = (RAM in MB / 5MB per connection) - reserved +-- For 4GB RAM: (4096 / 5) - 10 = ~800 theoretical max +-- But practically, 100-200 is better for query performance + +-- Recommended settings for 4GB RAM +alter system set max_connections = 100; + +-- Also set work_mem appropriately +-- work_mem * max_connections should not exceed 25% of RAM +alter system set work_mem = '8MB'; -- 8MB * 100 = 800MB max +``` + +Monitor connection usage: + +```sql +select count(*), state from pg_stat_activity group by state; +``` + +Reference: [Database Connections](https://supabase.com/docs/guides/platform/performance#connection-management) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md b/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md new file mode 100644 index 00000000..e2ebd581 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md @@ -0,0 +1,41 @@ +--- +title: Use Connection Pooling for All Applications +impact: CRITICAL +impactDescription: Handle 10-100x more concurrent users +tags: connection-pooling, pgbouncer, performance, scalability +--- + +## Use Connection Pooling for All Applications + +Postgres connections are expensive (1-3MB RAM each). Without pooling, applications exhaust connections under load. + +**Incorrect (new connection per request):** + +```sql +-- Each request creates a new connection +-- Application code: db.connect() per request +-- Result: 500 concurrent users = 500 connections = crashed database + +-- Check current connections +select count(*) from pg_stat_activity; -- 487 connections! +``` + +**Correct (connection pooling):** + +```sql +-- Use a pooler like PgBouncer between app and database +-- Application connects to pooler, pooler reuses a small pool to Postgres + +-- Configure pool_size based on: (CPU cores * 2) + spindle_count +-- Example for 4 cores: pool_size = 10 + +-- Result: 500 concurrent users share 10 actual connections +select count(*) from pg_stat_activity; -- 10 connections +``` + +Pool modes: + +- **Transaction mode**: connection returned after each transaction (best for most apps) +- **Session mode**: connection held for entire session (needed for prepared statements, temp tables) + +Reference: [Connection Pooling](https://supabase.com/docs/guides/database/connecting-to-postgres#connection-pooler) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md b/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md new file mode 100644 index 00000000..555547d8 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md @@ -0,0 +1,46 @@ +--- +title: Use Prepared Statements Correctly with Pooling +impact: HIGH +impactDescription: Avoid prepared statement conflicts in pooled environments +tags: prepared-statements, connection-pooling, transaction-mode +--- + +## Use Prepared Statements Correctly with Pooling + +Prepared statements are tied to individual database connections. In transaction-mode pooling, connections are shared, causing conflicts. + +**Incorrect (named prepared statements with transaction pooling):** + +```sql +-- Named prepared statement +prepare get_user as select * from users where id = $1; + +-- In transaction mode pooling, next request may get different connection +execute get_user(123); +-- ERROR: prepared statement "get_user" does not exist +``` + +**Correct (use unnamed statements or session mode):** + +```sql +-- Option 1: Use unnamed prepared statements (most ORMs do this automatically) +-- The query is prepared and executed in a single protocol message + +-- Option 2: Deallocate after use in transaction mode +prepare get_user as select * from users where id = $1; +execute get_user(123); +deallocate get_user; + +-- Option 3: Use session mode pooling (port 5432 vs 6543) +-- Connection is held for entire session, prepared statements persist +``` + +Check your driver settings: + +```sql +-- Many drivers use prepared statements by default +-- Node.js pg: { prepare: false } to disable +-- JDBC: prepareThreshold=0 to disable +``` + +Reference: [Prepared Statements with Pooling](https://supabase.com/docs/guides/database/connecting-to-postgres#connection-pool-modes) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md b/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md new file mode 100644 index 00000000..997947cb --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md @@ -0,0 +1,54 @@ +--- +title: Batch INSERT Statements for Bulk Data +impact: MEDIUM +impactDescription: 10-50x faster bulk inserts +tags: batch, insert, bulk, performance, copy +--- + +## Batch INSERT Statements for Bulk Data + +Individual INSERT statements have high overhead. Batch multiple rows in single statements or use COPY. + +**Incorrect (individual inserts):** + +```sql +-- Each insert is a separate transaction and round trip +insert into events (user_id, action) values (1, 'click'); +insert into events (user_id, action) values (1, 'view'); +insert into events (user_id, action) values (2, 'click'); +-- ... 1000 more individual inserts + +-- 1000 inserts = 1000 round trips = slow +``` + +**Correct (batch insert):** + +```sql +-- Multiple rows in single statement +insert into events (user_id, action) values + (1, 'click'), + (1, 'view'), + (2, 'click'), + -- ... up to ~1000 rows per batch + (999, 'view'); + +-- One round trip for 1000 rows +``` + +For large imports, use COPY: + +```sql +-- COPY is fastest for bulk loading +copy events (user_id, action, created_at) +from '/path/to/data.csv' +with (format csv, header true); + +-- Or from stdin in application +copy events (user_id, action) from stdin with (format csv); +1,click +1,view +2,click +\. +``` + +Reference: [COPY](https://www.postgresql.org/docs/current/sql-copy.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md b/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md new file mode 100644 index 00000000..2109186f --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md @@ -0,0 +1,53 @@ +--- +title: Eliminate N+1 Queries with Batch Loading +impact: MEDIUM-HIGH +impactDescription: 10-100x fewer database round trips +tags: n-plus-one, batch, performance, queries +--- + +## Eliminate N+1 Queries with Batch Loading + +N+1 queries execute one query per item in a loop. Batch them into a single query using arrays or JOINs. + +**Incorrect (N+1 queries):** + +```sql +-- First query: get all users +select id from users where active = true; -- Returns 100 IDs + +-- Then N queries, one per user +select * from orders where user_id = 1; +select * from orders where user_id = 2; +select * from orders where user_id = 3; +-- ... 97 more queries! + +-- Total: 101 round trips to database +``` + +**Correct (single batch query):** + +```sql +-- Collect IDs and query once with ANY +select * from orders where user_id = any(array[1, 2, 3, ...]); + +-- Or use JOIN instead of loop +select u.id, u.name, o.* +from users u +left join orders o on o.user_id = u.id +where u.active = true; + +-- Total: 1 round trip +``` + +Application pattern: + +```sql +-- Instead of looping in application code: +-- for user in users: db.query("SELECT * FROM orders WHERE user_id = $1", user.id) + +-- Pass array parameter: +select * from orders where user_id = any($1::bigint[]); +-- Application passes: [1, 2, 3, 4, 5, ...] +``` + +Reference: [N+1 Query Problem](https://supabase.com/docs/guides/database/query-optimization) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md b/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md new file mode 100644 index 00000000..633d8393 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md @@ -0,0 +1,50 @@ +--- +title: Use Cursor-Based Pagination Instead of OFFSET +impact: MEDIUM-HIGH +impactDescription: Consistent O(1) performance regardless of page depth +tags: pagination, cursor, keyset, offset, performance +--- + +## Use Cursor-Based Pagination Instead of OFFSET + +OFFSET-based pagination scans all skipped rows, getting slower on deeper pages. Cursor pagination is O(1). + +**Incorrect (OFFSET pagination):** + +```sql +-- Page 1: scans 20 rows +select * from products order by id limit 20 offset 0; + +-- Page 100: scans 2000 rows to skip 1980 +select * from products order by id limit 20 offset 1980; + +-- Page 10000: scans 200,000 rows! +select * from products order by id limit 20 offset 199980; +``` + +**Correct (cursor/keyset pagination):** + +```sql +-- Page 1: get first 20 +select * from products order by id limit 20; +-- Application stores last_id = 20 + +-- Page 2: start after last ID +select * from products where id > 20 order by id limit 20; +-- Uses index, always fast regardless of page depth + +-- Page 10000: same speed as page 1 +select * from products where id > 199980 order by id limit 20; +``` + +For multi-column sorting: + +```sql +-- Cursor must include all sort columns +select * from products +where (created_at, id) > ('2024-01-15 10:00:00', 12345) +order by created_at, id +limit 20; +``` + +Reference: [Pagination](https://supabase.com/docs/guides/database/pagination) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md b/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md new file mode 100644 index 00000000..bc95e230 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md @@ -0,0 +1,50 @@ +--- +title: Use UPSERT for Insert-or-Update Operations +impact: MEDIUM +impactDescription: Atomic operation, eliminates race conditions +tags: upsert, on-conflict, insert, update +--- + +## Use UPSERT for Insert-or-Update Operations + +Using separate SELECT-then-INSERT/UPDATE creates race conditions. Use INSERT ... ON CONFLICT for atomic upserts. + +**Incorrect (check-then-insert race condition):** + +```sql +-- Race condition: two requests check simultaneously +select * from settings where user_id = 123 and key = 'theme'; +-- Both find nothing + +-- Both try to insert +insert into settings (user_id, key, value) values (123, 'theme', 'dark'); +-- One succeeds, one fails with duplicate key error! +``` + +**Correct (atomic UPSERT):** + +```sql +-- Single atomic operation +insert into settings (user_id, key, value) +values (123, 'theme', 'dark') +on conflict (user_id, key) +do update set value = excluded.value, updated_at = now(); + +-- Returns the inserted/updated row +insert into settings (user_id, key, value) +values (123, 'theme', 'dark') +on conflict (user_id, key) +do update set value = excluded.value +returning *; +``` + +Insert-or-ignore pattern: + +```sql +-- Insert only if not exists (no update) +insert into page_views (page_id, user_id) +values (1, 123) +on conflict (page_id, user_id) do nothing; +``` + +Reference: [INSERT ON CONFLICT](https://www.postgresql.org/docs/current/sql-insert.html#SQL-ON-CONFLICT) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md b/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md new file mode 100644 index 00000000..572eaf0d --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md @@ -0,0 +1,56 @@ +--- +title: Use Advisory Locks for Application-Level Locking +impact: MEDIUM +impactDescription: Efficient coordination without row-level lock overhead +tags: advisory-locks, coordination, application-locks +--- + +## Use Advisory Locks for Application-Level Locking + +Advisory locks provide application-level coordination without requiring database rows to lock. + +**Incorrect (creating rows just for locking):** + +```sql +-- Creating dummy rows to lock on +create table resource_locks ( + resource_name text primary key +); + +insert into resource_locks values ('report_generator'); + +-- Lock by selecting the row +select * from resource_locks where resource_name = 'report_generator' for update; +``` + +**Correct (advisory locks):** + +```sql +-- Session-level advisory lock (released on disconnect or unlock) +select pg_advisory_lock(hashtext('report_generator')); +-- ... do exclusive work ... +select pg_advisory_unlock(hashtext('report_generator')); + +-- Transaction-level lock (released on commit/rollback) +begin; +select pg_advisory_xact_lock(hashtext('daily_report')); +-- ... do work ... +commit; -- Lock automatically released +``` + +Try-lock for non-blocking operations: + +```sql +-- Returns immediately with true/false instead of waiting +select pg_try_advisory_lock(hashtext('resource_name')); + +-- Use in application +if (acquired) { + -- Do work + select pg_advisory_unlock(hashtext('resource_name')); +} else { + -- Skip or retry later +} +``` + +Reference: [Advisory Locks](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md b/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md new file mode 100644 index 00000000..974da5ed --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md @@ -0,0 +1,68 @@ +--- +title: Prevent Deadlocks with Consistent Lock Ordering +impact: MEDIUM-HIGH +impactDescription: Eliminate deadlock errors, improve reliability +tags: deadlocks, locking, transactions, ordering +--- + +## Prevent Deadlocks with Consistent Lock Ordering + +Deadlocks occur when transactions lock resources in different orders. Always +acquire locks in a consistent order. + +**Incorrect (inconsistent lock ordering):** + +```sql +-- Transaction A -- Transaction B +begin; begin; +update accounts update accounts +set balance = balance - 100 set balance = balance - 50 +where id = 1; where id = 2; -- B locks row 2 + +update accounts update accounts +set balance = balance + 100 set balance = balance + 50 +where id = 2; -- A waits for B where id = 1; -- B waits for A + +-- DEADLOCK! Both waiting for each other +``` + +**Correct (lock rows in consistent order first):** + +```sql +-- Explicitly acquire locks in ID order before updating +begin; +select * from accounts where id in (1, 2) order by id for update; + +-- Now perform updates in any order - locks already held +update accounts set balance = balance - 100 where id = 1; +update accounts set balance = balance + 100 where id = 2; +commit; +``` + +Alternative: use a single statement to update atomically: + +```sql +-- Single statement acquires all locks atomically +begin; +update accounts +set balance = balance + case id + when 1 then -100 + when 2 then 100 +end +where id in (1, 2); +commit; +``` + +Detect deadlocks in logs: + +```sql +-- Check for recent deadlocks +select * from pg_stat_database where deadlocks > 0; + +-- Enable deadlock logging +set log_lock_waits = on; +set deadlock_timeout = '1s'; +``` + +Reference: +[Deadlocks](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-DEADLOCKS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md b/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md new file mode 100644 index 00000000..e6b8ef26 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md @@ -0,0 +1,50 @@ +--- +title: Keep Transactions Short to Reduce Lock Contention +impact: MEDIUM-HIGH +impactDescription: 3-5x throughput improvement, fewer deadlocks +tags: transactions, locking, contention, performance +--- + +## Keep Transactions Short to Reduce Lock Contention + +Long-running transactions hold locks that block other queries. Keep transactions as short as possible. + +**Incorrect (long transaction with external calls):** + +```sql +begin; +select * from orders where id = 1 for update; -- Lock acquired + +-- Application makes HTTP call to payment API (2-5 seconds) +-- Other queries on this row are blocked! + +update orders set status = 'paid' where id = 1; +commit; -- Lock held for entire duration +``` + +**Correct (minimal transaction scope):** + +```sql +-- Validate data and call APIs outside transaction +-- Application: response = await paymentAPI.charge(...) + +-- Only hold lock for the actual update +begin; +update orders +set status = 'paid', payment_id = $1 +where id = $2 and status = 'pending' +returning *; +commit; -- Lock held for milliseconds +``` + +Use `statement_timeout` to prevent runaway transactions: + +```sql +-- Abort queries running longer than 30 seconds +set statement_timeout = '30s'; + +-- Or per-session +set local statement_timeout = '5s'; +``` + +Reference: [Transaction Management](https://www.postgresql.org/docs/current/tutorial-transactions.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md b/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md new file mode 100644 index 00000000..77bdbb97 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md @@ -0,0 +1,54 @@ +--- +title: Use SKIP LOCKED for Non-Blocking Queue Processing +impact: MEDIUM-HIGH +impactDescription: 10x throughput for worker queues +tags: skip-locked, queue, workers, concurrency +--- + +## Use SKIP LOCKED for Non-Blocking Queue Processing + +When multiple workers process a queue, SKIP LOCKED allows workers to process different rows without waiting. + +**Incorrect (workers block each other):** + +```sql +-- Worker 1 and Worker 2 both try to get next job +begin; +select * from jobs where status = 'pending' order by created_at limit 1 for update; +-- Worker 2 waits for Worker 1's lock to release! +``` + +**Correct (SKIP LOCKED for parallel processing):** + +```sql +-- Each worker skips locked rows and gets the next available +begin; +select * from jobs +where status = 'pending' +order by created_at +limit 1 +for update skip locked; + +-- Worker 1 gets job 1, Worker 2 gets job 2 (no waiting) + +update jobs set status = 'processing' where id = $1; +commit; +``` + +Complete queue pattern: + +```sql +-- Atomic claim-and-update in one statement +update jobs +set status = 'processing', worker_id = $1, started_at = now() +where id = ( + select id from jobs + where status = 'pending' + order by created_at + limit 1 + for update skip locked +) +returning *; +``` + +Reference: [SELECT FOR UPDATE SKIP LOCKED](https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md new file mode 100644 index 00000000..542978c3 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md @@ -0,0 +1,45 @@ +--- +title: Use EXPLAIN ANALYZE to Diagnose Slow Queries +impact: LOW-MEDIUM +impactDescription: Identify exact bottlenecks in query execution +tags: explain, analyze, diagnostics, query-plan +--- + +## Use EXPLAIN ANALYZE to Diagnose Slow Queries + +EXPLAIN ANALYZE executes the query and shows actual timings, revealing the true performance bottlenecks. + +**Incorrect (guessing at performance issues):** + +```sql +-- Query is slow, but why? +select * from orders where customer_id = 123 and status = 'pending'; +-- "It must be missing an index" - but which one? +``` + +**Correct (use EXPLAIN ANALYZE):** + +```sql +explain (analyze, buffers, format text) +select * from orders where customer_id = 123 and status = 'pending'; + +-- Output reveals the issue: +-- Seq Scan on orders (cost=0.00..25000.00 rows=50 width=100) (actual time=0.015..450.123 rows=50 loops=1) +-- Filter: ((customer_id = 123) AND (status = 'pending'::text)) +-- Rows Removed by Filter: 999950 +-- Buffers: shared hit=5000 read=15000 +-- Planning Time: 0.150 ms +-- Execution Time: 450.500 ms +``` + +Key things to look for: + +```sql +-- Seq Scan on large tables = missing index +-- Rows Removed by Filter = poor selectivity or missing index +-- Buffers: read >> hit = data not cached, needs more memory +-- Nested Loop with high loops = consider different join strategy +-- Sort Method: external merge = work_mem too low +``` + +Reference: [EXPLAIN](https://supabase.com/docs/guides/database/inspect) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md new file mode 100644 index 00000000..d7e82f1a --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md @@ -0,0 +1,55 @@ +--- +title: Enable pg_stat_statements for Query Analysis +impact: LOW-MEDIUM +impactDescription: Identify top resource-consuming queries +tags: pg-stat-statements, monitoring, statistics, performance +--- + +## Enable pg_stat_statements for Query Analysis + +pg_stat_statements tracks execution statistics for all queries, helping identify slow and frequent queries. + +**Incorrect (no visibility into query patterns):** + +```sql +-- Database is slow, but which queries are the problem? +-- No way to know without pg_stat_statements +``` + +**Correct (enable and query pg_stat_statements):** + +```sql +-- Enable the extension +create extension if not exists pg_stat_statements; + +-- Find slowest queries by total time +select + calls, + round(total_exec_time::numeric, 2) as total_time_ms, + round(mean_exec_time::numeric, 2) as mean_time_ms, + query +from pg_stat_statements +order by total_exec_time desc +limit 10; + +-- Find most frequent queries +select calls, query +from pg_stat_statements +order by calls desc +limit 10; + +-- Reset statistics after optimization +select pg_stat_statements_reset(); +``` + +Key metrics to monitor: + +```sql +-- Queries with high mean time (candidates for optimization) +select query, mean_exec_time, calls +from pg_stat_statements +where mean_exec_time > 100 -- > 100ms average +order by mean_exec_time desc; +``` + +Reference: [pg_stat_statements](https://supabase.com/docs/guides/database/extensions/pg_stat_statements) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md new file mode 100644 index 00000000..e0e8ea0b --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md @@ -0,0 +1,55 @@ +--- +title: Maintain Table Statistics with VACUUM and ANALYZE +impact: MEDIUM +impactDescription: 2-10x better query plans with accurate statistics +tags: vacuum, analyze, statistics, maintenance, autovacuum +--- + +## Maintain Table Statistics with VACUUM and ANALYZE + +Outdated statistics cause the query planner to make poor decisions. VACUUM reclaims space, ANALYZE updates statistics. + +**Incorrect (stale statistics):** + +```sql +-- Table has 1M rows but stats say 1000 +-- Query planner chooses wrong strategy +explain select * from orders where status = 'pending'; +-- Shows: Seq Scan (because stats show small table) +-- Actually: Index Scan would be much faster +``` + +**Correct (maintain fresh statistics):** + +```sql +-- Manually analyze after large data changes +analyze orders; + +-- Analyze specific columns used in WHERE clauses +analyze orders (status, created_at); + +-- Check when tables were last analyzed +select + relname, + last_vacuum, + last_autovacuum, + last_analyze, + last_autoanalyze +from pg_stat_user_tables +order by last_analyze nulls first; +``` + +Autovacuum tuning for busy tables: + +```sql +-- Increase frequency for high-churn tables +alter table orders set ( + autovacuum_vacuum_scale_factor = 0.05, -- Vacuum at 5% dead tuples (default 20%) + autovacuum_analyze_scale_factor = 0.02 -- Analyze at 2% changes (default 10%) +); + +-- Check autovacuum status +select * from pg_stat_progress_vacuum; +``` + +Reference: [VACUUM](https://supabase.com/docs/guides/database/database-size#vacuum-operations) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md new file mode 100644 index 00000000..fea64523 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md @@ -0,0 +1,44 @@ +--- +title: Create Composite Indexes for Multi-Column Queries +impact: HIGH +impactDescription: 5-10x faster multi-column queries +tags: indexes, composite-index, multi-column, query-optimization +--- + +## Create Composite Indexes for Multi-Column Queries + +When queries filter on multiple columns, a composite index is more efficient than separate single-column indexes. + +**Incorrect (separate indexes require bitmap scan):** + +```sql +-- Two separate indexes +create index orders_status_idx on orders (status); +create index orders_created_idx on orders (created_at); + +-- Query must combine both indexes (slower) +select * from orders where status = 'pending' and created_at > '2024-01-01'; +``` + +**Correct (composite index):** + +```sql +-- Single composite index (leftmost column first for equality checks) +create index orders_status_created_idx on orders (status, created_at); + +-- Query uses one efficient index scan +select * from orders where status = 'pending' and created_at > '2024-01-01'; +``` + +**Column order matters** - place equality columns first, range columns last: + +```sql +-- Good: status (=) before created_at (>) +create index idx on orders (status, created_at); + +-- Works for: WHERE status = 'pending' +-- Works for: WHERE status = 'pending' AND created_at > '2024-01-01' +-- Does NOT work for: WHERE created_at > '2024-01-01' (leftmost prefix rule) +``` + +Reference: [Multicolumn Indexes](https://www.postgresql.org/docs/current/indexes-multicolumn.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md new file mode 100644 index 00000000..9d2a4947 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md @@ -0,0 +1,40 @@ +--- +title: Use Covering Indexes to Avoid Table Lookups +impact: MEDIUM-HIGH +impactDescription: 2-5x faster queries by eliminating heap fetches +tags: indexes, covering-index, include, index-only-scan +--- + +## Use Covering Indexes to Avoid Table Lookups + +Covering indexes include all columns needed by a query, enabling index-only scans that skip the table entirely. + +**Incorrect (index scan + heap fetch):** + +```sql +create index users_email_idx on users (email); + +-- Must fetch name and created_at from table heap +select email, name, created_at from users where email = 'user@example.com'; +``` + +**Correct (index-only scan with INCLUDE):** + +```sql +-- Include non-searchable columns in the index +create index users_email_idx on users (email) include (name, created_at); + +-- All columns served from index, no table access needed +select email, name, created_at from users where email = 'user@example.com'; +``` + +Use INCLUDE for columns you SELECT but don't filter on: + +```sql +-- Searching by status, but also need customer_id and total +create index orders_status_idx on orders (status) include (customer_id, total); + +select status, customer_id, total from orders where status = 'shipped'; +``` + +Reference: [Index-Only Scans](https://www.postgresql.org/docs/current/indexes-index-only-scans.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md b/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md new file mode 100644 index 00000000..93b32590 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md @@ -0,0 +1,48 @@ +--- +title: Choose the Right Index Type for Your Data +impact: HIGH +impactDescription: 10-100x improvement with correct index type +tags: indexes, btree, gin, gist, brin, hash, index-types +--- + +## Choose the Right Index Type for Your Data + +Different index types excel at different query patterns. The default B-tree isn't always optimal. + +**Incorrect (B-tree for JSONB containment):** + +```sql +-- B-tree cannot optimize containment operators +create index products_attrs_idx on products (attributes); +select * from products where attributes @> '{"color": "red"}'; +-- Full table scan - B-tree doesn't support @> operator +``` + +**Correct (GIN for JSONB):** + +```sql +-- GIN supports @>, ?, ?&, ?| operators +create index products_attrs_idx on products using gin (attributes); +select * from products where attributes @> '{"color": "red"}'; +``` + +Index type guide: + +```sql +-- B-tree (default): =, <, >, BETWEEN, IN, IS NULL +create index users_created_idx on users (created_at); + +-- GIN: arrays, JSONB, full-text search +create index posts_tags_idx on posts using gin (tags); + +-- GiST: geometric data, range types, nearest-neighbor (KNN) queries +create index locations_idx on places using gist (location); + +-- BRIN: large time-series tables (10-100x smaller) +create index events_time_idx on events using brin (created_at); + +-- Hash: equality-only (slightly faster than B-tree for =) +create index sessions_token_idx on sessions using hash (token); +``` + +Reference: [Index Types](https://www.postgresql.org/docs/current/indexes-types.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md new file mode 100644 index 00000000..e6daace7 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md @@ -0,0 +1,43 @@ +--- +title: Add Indexes on WHERE and JOIN Columns +impact: CRITICAL +impactDescription: 100-1000x faster queries on large tables +tags: indexes, performance, sequential-scan, query-optimization +--- + +## Add Indexes on WHERE and JOIN Columns + +Queries filtering or joining on unindexed columns cause full table scans, which become exponentially slower as tables grow. + +**Incorrect (sequential scan on large table):** + +```sql +-- No index on customer_id causes full table scan +select * from orders where customer_id = 123; + +-- EXPLAIN shows: Seq Scan on orders (cost=0.00..25000.00 rows=100 width=85) +``` + +**Correct (index scan):** + +```sql +-- Create index on frequently filtered column +create index orders_customer_id_idx on orders (customer_id); + +select * from orders where customer_id = 123; + +-- EXPLAIN shows: Index Scan using orders_customer_id_idx (cost=0.42..8.44 rows=100 width=85) +``` + +For JOIN columns, always index the foreign key side: + +```sql +-- Index the referencing column +create index orders_customer_id_idx on orders (customer_id); + +select c.name, o.total +from customers c +join orders o on o.customer_id = c.id; +``` + +Reference: [Query Optimization](https://supabase.com/docs/guides/database/query-optimization) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md new file mode 100644 index 00000000..3e61a341 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md @@ -0,0 +1,45 @@ +--- +title: Use Partial Indexes for Filtered Queries +impact: HIGH +impactDescription: 5-20x smaller indexes, faster writes and queries +tags: indexes, partial-index, query-optimization, storage +--- + +## Use Partial Indexes for Filtered Queries + +Partial indexes only include rows matching a WHERE condition, making them smaller and faster when queries consistently filter on the same condition. + +**Incorrect (full index includes irrelevant rows):** + +```sql +-- Index includes all rows, even soft-deleted ones +create index users_email_idx on users (email); + +-- Query always filters active users +select * from users where email = 'user@example.com' and deleted_at is null; +``` + +**Correct (partial index matches query filter):** + +```sql +-- Index only includes active users +create index users_active_email_idx on users (email) +where deleted_at is null; + +-- Query uses the smaller, faster index +select * from users where email = 'user@example.com' and deleted_at is null; +``` + +Common use cases for partial indexes: + +```sql +-- Only pending orders (status rarely changes once completed) +create index orders_pending_idx on orders (created_at) +where status = 'pending'; + +-- Only non-null values +create index products_sku_idx on products (sku) +where sku is not null; +``` + +Reference: [Partial Indexes](https://www.postgresql.org/docs/current/indexes-partial.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md b/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md new file mode 100644 index 00000000..1d2ef8f9 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md @@ -0,0 +1,80 @@ +--- +title: Add Constraints Safely in Migrations +impact: HIGH +impactDescription: Prevents migration failures and enables idempotent schema changes +tags: constraints, migrations, schema, alter-table +--- + +## Add Constraints Safely in Migrations + +PostgreSQL does not support `ADD CONSTRAINT IF NOT EXISTS`. Migrations using this syntax will fail. + +**Incorrect (causes syntax error):** + +```sql +-- ERROR: syntax error at or near "not" (SQLSTATE 42601) +alter table public.profiles +add constraint if not exists profiles_birthchart_id_unique unique (birthchart_id); +``` + +**Correct (idempotent constraint creation):** + +```sql +-- Use DO block to check before adding +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'profiles_birthchart_id_unique' + and conrelid = 'public.profiles'::regclass + ) then + alter table public.profiles + add constraint profiles_birthchart_id_unique unique (birthchart_id); + end if; +end $$; +``` + +For all constraint types: + +```sql +-- Check constraints +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'check_age_positive' + ) then + alter table users add constraint check_age_positive check (age > 0); + end if; +end $$; + +-- Foreign keys +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'profiles_birthchart_id_fkey' + ) then + alter table profiles + add constraint profiles_birthchart_id_fkey + foreign key (birthchart_id) references birthcharts(id); + end if; +end $$; +``` + +Check if constraint exists: + +```sql +-- Query to check constraint existence +select conname, contype, pg_get_constraintdef(oid) +from pg_constraint +where conrelid = 'public.profiles'::regclass; + +-- contype values: +-- 'p' = PRIMARY KEY +-- 'f' = FOREIGN KEY +-- 'u' = UNIQUE +-- 'c' = CHECK +``` + +Reference: [Constraints](https://www.postgresql.org/docs/current/ddl-constraints.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md b/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md new file mode 100644 index 00000000..f253a581 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md @@ -0,0 +1,46 @@ +--- +title: Choose Appropriate Data Types +impact: HIGH +impactDescription: 50% storage reduction, faster comparisons +tags: data-types, schema, storage, performance +--- + +## Choose Appropriate Data Types + +Using the right data types reduces storage, improves query performance, and prevents bugs. + +**Incorrect (wrong data types):** + +```sql +create table users ( + id int, -- Will overflow at 2.1 billion + email varchar(255), -- Unnecessary length limit + created_at timestamp, -- Missing timezone info + is_active varchar(5), -- String for boolean + price varchar(20) -- String for numeric +); +``` + +**Correct (appropriate data types):** + +```sql +create table users ( + id bigint generated always as identity primary key, -- 9 quintillion max + email text, -- No artificial limit, same performance as varchar + created_at timestamptz, -- Always store timezone-aware timestamps + is_active boolean default true, -- 1 byte vs variable string length + price numeric(10,2) -- Exact decimal arithmetic +); +``` + +Key guidelines: + +```sql +-- IDs: use bigint, not int (future-proofing) +-- Strings: use text, not varchar(n) unless constraint needed +-- Time: use timestamptz, not timestamp +-- Money: use numeric, not float (precision matters) +-- Enums: use text with check constraint or create enum type +``` + +Reference: [Data Types](https://www.postgresql.org/docs/current/datatype.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md new file mode 100644 index 00000000..6c3d6ff6 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md @@ -0,0 +1,59 @@ +--- +title: Index Foreign Key Columns +impact: HIGH +impactDescription: 10-100x faster JOINs and CASCADE operations +tags: foreign-key, indexes, joins, schema +--- + +## Index Foreign Key Columns + +Postgres does not automatically index foreign key columns. Missing indexes cause slow JOINs and CASCADE operations. + +**Incorrect (unindexed foreign key):** + +```sql +create table orders ( + id bigint generated always as identity primary key, + customer_id bigint references customers(id) on delete cascade, + total numeric(10,2) +); + +-- No index on customer_id! +-- JOINs and ON DELETE CASCADE both require full table scan +select * from orders where customer_id = 123; -- Seq Scan +delete from customers where id = 123; -- Locks table, scans all orders +``` + +**Correct (indexed foreign key):** + +```sql +create table orders ( + id bigint generated always as identity primary key, + customer_id bigint references customers(id) on delete cascade, + total numeric(10,2) +); + +-- Always index the FK column +create index orders_customer_id_idx on orders (customer_id); + +-- Now JOINs and cascades are fast +select * from orders where customer_id = 123; -- Index Scan +delete from customers where id = 123; -- Uses index, fast cascade +``` + +Find missing FK indexes: + +```sql +select + conrelid::regclass as table_name, + a.attname as fk_column +from pg_constraint c +join pg_attribute a on a.attrelid = c.conrelid and a.attnum = any(c.conkey) +where c.contype = 'f' + and not exists ( + select 1 from pg_index i + where i.indrelid = c.conrelid and a.attnum = any(i.indkey) + ); +``` + +Reference: [Foreign Keys](https://www.postgresql.org/docs/current/ddl-constraints.html#DDL-CONSTRAINTS-FK) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md b/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md new file mode 100644 index 00000000..f0072940 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md @@ -0,0 +1,55 @@ +--- +title: Use Lowercase Identifiers for Compatibility +impact: MEDIUM +impactDescription: Avoid case-sensitivity bugs with tools, ORMs, and AI assistants +tags: naming, identifiers, case-sensitivity, schema, conventions +--- + +## Use Lowercase Identifiers for Compatibility + +PostgreSQL folds unquoted identifiers to lowercase. Quoted mixed-case identifiers require quotes forever and cause issues with tools, ORMs, and AI assistants that may not recognize them. + +**Incorrect (mixed-case identifiers):** + +```sql +-- Quoted identifiers preserve case but require quotes everywhere +CREATE TABLE "Users" ( + "userId" bigint PRIMARY KEY, + "firstName" text, + "lastName" text +); + +-- Must always quote or queries fail +SELECT "firstName" FROM "Users" WHERE "userId" = 1; + +-- This fails - Users becomes users without quotes +SELECT firstName FROM Users; +-- ERROR: relation "users" does not exist +``` + +**Correct (lowercase snake_case):** + +```sql +-- Unquoted lowercase identifiers are portable and tool-friendly +CREATE TABLE users ( + user_id bigint PRIMARY KEY, + first_name text, + last_name text +); + +-- Works without quotes, recognized by all tools +SELECT first_name FROM users WHERE user_id = 1; +``` + +Common sources of mixed-case identifiers: + +```sql +-- ORMs often generate quoted camelCase - configure them to use snake_case +-- Migrations from other databases may preserve original casing +-- Some GUI tools quote identifiers by default - disable this + +-- If stuck with mixed-case, create views as a compatibility layer +CREATE VIEW users AS SELECT "userId" AS user_id, "firstName" AS first_name FROM "Users"; +``` + +Reference: [Identifiers and Key Words](https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md b/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md new file mode 100644 index 00000000..13137a03 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md @@ -0,0 +1,55 @@ +--- +title: Partition Large Tables for Better Performance +impact: MEDIUM-HIGH +impactDescription: 5-20x faster queries and maintenance on large tables +tags: partitioning, large-tables, time-series, performance +--- + +## Partition Large Tables for Better Performance + +Partitioning splits a large table into smaller pieces, improving query performance and maintenance operations. + +**Incorrect (single large table):** + +```sql +create table events ( + id bigint generated always as identity, + created_at timestamptz, + data jsonb +); + +-- 500M rows, queries scan everything +select * from events where created_at > '2024-01-01'; -- Slow +vacuum events; -- Takes hours, locks table +``` + +**Correct (partitioned by time range):** + +```sql +create table events ( + id bigint generated always as identity, + created_at timestamptz not null, + data jsonb +) partition by range (created_at); + +-- Create partitions for each month +create table events_2024_01 partition of events + for values from ('2024-01-01') to ('2024-02-01'); + +create table events_2024_02 partition of events + for values from ('2024-02-01') to ('2024-03-01'); + +-- Queries only scan relevant partitions +select * from events where created_at > '2024-01-15'; -- Only scans events_2024_01+ + +-- Drop old data instantly +drop table events_2023_01; -- Instant vs DELETE taking hours +``` + +When to partition: + +- Tables > 100M rows +- Time-series data with date-based queries +- Need to efficiently drop old data + +Reference: [Table Partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md b/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md new file mode 100644 index 00000000..fb0fbb16 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md @@ -0,0 +1,61 @@ +--- +title: Select Optimal Primary Key Strategy +impact: HIGH +impactDescription: Better index locality, reduced fragmentation +tags: primary-key, identity, uuid, serial, schema +--- + +## Select Optimal Primary Key Strategy + +Primary key choice affects insert performance, index size, and replication +efficiency. + +**Incorrect (problematic PK choices):** + +```sql +-- identity is the SQL-standard approach +create table users ( + id serial primary key -- Works, but IDENTITY is recommended +); + +-- Random UUIDs (v4) cause index fragmentation +create table orders ( + id uuid default gen_random_uuid() primary key -- UUIDv4 = random = scattered inserts +); +``` + +**Correct (optimal PK strategies):** + +```sql +-- Use IDENTITY for sequential IDs (SQL-standard, best for most cases) +create table users ( + id bigint generated always as identity primary key +); + +-- For distributed systems needing UUIDs, use UUIDv7 (time-ordered) +-- Requires pg_uuidv7 extension: create extension pg_uuidv7; +create table orders ( + id uuid default uuid_generate_v7() primary key -- Time-ordered, no fragmentation +); + +-- Alternative: time-prefixed IDs for sortable, distributed IDs (no extension needed) +create table events ( + id text default concat( + to_char(now() at time zone 'utc', 'YYYYMMDDHH24MISSMS'), + gen_random_uuid()::text + ) primary key +); +``` + +Guidelines: + +- Single database: `bigint identity` (sequential, 8 bytes, SQL-standard) +- Distributed/exposed IDs: UUIDv7 (requires pg_uuidv7) or ULID (time-ordered, no + fragmentation) +- `serial` works but `identity` is SQL-standard and preferred for new + applications +- Avoid random UUIDs (v4) as primary keys on large tables (causes index + fragmentation) + +Reference: +[Identity Columns](https://www.postgresql.org/docs/current/sql-createtable.html#SQL-CREATETABLE-PARMS-GENERATED-IDENTITY) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md b/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md new file mode 100644 index 00000000..448ec345 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md @@ -0,0 +1,54 @@ +--- +title: Apply Principle of Least Privilege +impact: MEDIUM +impactDescription: Reduced attack surface, better audit trail +tags: privileges, security, roles, permissions +--- + +## Apply Principle of Least Privilege + +Grant only the minimum permissions required. Never use superuser for application queries. + +**Incorrect (overly broad permissions):** + +```sql +-- Application uses superuser connection +-- Or grants ALL to application role +grant all privileges on all tables in schema public to app_user; +grant all privileges on all sequences in schema public to app_user; + +-- Any SQL injection becomes catastrophic +-- drop table users; cascades to everything +``` + +**Correct (minimal, specific grants):** + +```sql +-- Create role with no default privileges +create role app_readonly nologin; + +-- Grant only SELECT on specific tables +grant usage on schema public to app_readonly; +grant select on public.products, public.categories to app_readonly; + +-- Create role for writes with limited scope +create role app_writer nologin; +grant usage on schema public to app_writer; +grant select, insert, update on public.orders to app_writer; +grant usage on sequence orders_id_seq to app_writer; +-- No DELETE permission + +-- Login role inherits from these +create role app_user login password 'xxx'; +grant app_writer to app_user; +``` + +Revoke public defaults: + +```sql +-- Revoke default public access +revoke all on schema public from public; +revoke all on all tables in schema public from public; +``` + +Reference: [Roles and Privileges](https://supabase.com/blog/postgres-roles-and-privileges) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md b/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md new file mode 100644 index 00000000..c61e1a85 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md @@ -0,0 +1,50 @@ +--- +title: Enable Row Level Security for Multi-Tenant Data +impact: CRITICAL +impactDescription: Database-enforced tenant isolation, prevent data leaks +tags: rls, row-level-security, multi-tenant, security +--- + +## Enable Row Level Security for Multi-Tenant Data + +Row Level Security (RLS) enforces data access at the database level, ensuring users only see their own data. + +**Incorrect (application-level filtering only):** + +```sql +-- Relying only on application to filter +select * from orders where user_id = $current_user_id; + +-- Bug or bypass means all data is exposed! +select * from orders; -- Returns ALL orders +``` + +**Correct (database-enforced RLS):** + +```sql +-- Enable RLS on the table +alter table orders enable row level security; + +-- Create policy for users to see only their orders +create policy orders_user_policy on orders + for all + using (user_id = current_setting('app.current_user_id')::bigint); + +-- Force RLS even for table owners +alter table orders force row level security; + +-- Set user context and query +set app.current_user_id = '123'; +select * from orders; -- Only returns orders for user 123 +``` + +Policy for authenticated role: + +```sql +create policy orders_user_policy on orders + for all + to authenticated + using (user_id = auth.uid()); +``` + +Reference: [Row Level Security](https://supabase.com/docs/guides/database/postgres/row-level-security) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md b/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md new file mode 100644 index 00000000..b32d92f7 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md @@ -0,0 +1,57 @@ +--- +title: Optimize RLS Policies for Performance +impact: HIGH +impactDescription: 5-10x faster RLS queries with proper patterns +tags: rls, performance, security, optimization +--- + +## Optimize RLS Policies for Performance + +Poorly written RLS policies can cause severe performance issues. Use subqueries and indexes strategically. + +**Incorrect (function called for every row):** + +```sql +create policy orders_policy on orders + using (auth.uid() = user_id); -- auth.uid() called per row! + +-- With 1M rows, auth.uid() is called 1M times +``` + +**Correct (wrap functions in SELECT):** + +```sql +create policy orders_policy on orders + using ((select auth.uid()) = user_id); -- Called once, cached + +-- 100x+ faster on large tables +``` + +Use security definer functions for complex checks: + +```sql +-- Create helper function (runs as definer, bypasses RLS) +create or replace function is_team_member(team_id bigint) +returns boolean +language sql +security definer +set search_path = '' +as $$ + select exists ( + select 1 from public.team_members + where team_id = $1 and user_id = (select auth.uid()) + ); +$$; + +-- Use in policy (indexed lookup, not per-row check) +create policy team_orders_policy on orders + using ((select is_team_member(team_id))); +``` + +Always add indexes on columns used in RLS policies: + +```sql +create index orders_user_id_idx on orders (user_id); +``` + +Reference: [RLS Performance](https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations) diff --git a/.agents/skills/supabase/SKILL.md b/.agents/skills/supabase/SKILL.md new file mode 100644 index 00000000..53b80566 --- /dev/null +++ b/.agents/skills/supabase/SKILL.md @@ -0,0 +1,112 @@ +--- +name: supabase +description: "Use when doing ANY task involving Supabase. Triggers: Supabase products (Database, Auth, Edge Functions, Realtime, Storage, Vectors, Cron, Queues); client libraries and SSR integrations (supabase-js, @supabase/ssr) in Next.js, React, SvelteKit, Astro, Remix; auth issues (login, logout, sessions, JWT, cookies, getSession, getUser, getClaims, RLS); Supabase CLI or MCP server; schema changes, migrations, security audits, Postgres extensions (pg_graphql, pg_cron, pg_vector)." +metadata: + author: supabase + version: "0.1.2" +--- + +# Supabase + +## Core Principles + +**1. Supabase changes frequently — verify against changelog and current docs before implementing.** +Do not rely on training data for Supabase features. Function signatures, config.toml settings, and API conventions change between versions. + +First, fetch `https://supabase.com/changelog.md` (a lightweight summary index — not a heavy pull), scan for `breaking-change` tags relevant to your task, and follow the linked page for any that apply. Then look up the relevant topic using the documentation access methods below. + +**2. Verify your work.** +After implementing any fix, run a test query to confirm the change works. A fix without verification is incomplete. + +**3. Recover from errors, don't loop.** +If an approach fails after 2-3 attempts, stop and reconsider. Try a different method, check documentation, inspect the error more carefully, and review relevant logs when available. Supabase issues are not always solved by retrying the same command, and the answer is not always in the logs, but logs are often worth checking before proceeding. + +**4. Exposing tables to the Data API:** Depending on the user's [Data API settings](https://supabase.com/dashboard/project//integrations/data_api/settings), newly created tables may not be automatically exposed via the Data (REST) API. If this is the case, `anon` and `authenticated` roles will need to be explicitly granted access. + +> Note that this is separate from RLS, which controls which _rows_ are visible once a table is accessible, not whether the table is accessible at all. + +When a user reports a SQL-created table is unexpectedly inaccessible, check their Data API settings and whether the roles have been granted access via explicit `GRANT` SQL. When granting public (`anon`/`authenticated`) access, always enable RLS too. See [Exposing a Table to the Data API](https://supabase.com/docs/guides/api/securing-your-api.md) for the full setup workflow. + +**5. RLS in exposed schemas.** +Enable RLS on every table in any exposed schema, which includes `public` by default. This is critical in Supabase because tables in exposed schemas can be reachable through the Data API when the `anon`/`authenticated` roles have access (see [Exposing a Table to the Data API](https://supabase.com/docs/guides/api/securing-your-api.md)). For private schemas, prefer RLS as defense in depth. After enabling RLS, create policies that match the actual access model rather than defaulting every table to the same `auth.uid()` pattern. + +**6. Security checklist.** +When working on any Supabase task that touches auth, RLS, views, storage, or user data, run through this checklist. These are Supabase-specific security traps that silently create vulnerabilities: + +- **Auth and session security** + - **Never use `user_metadata` claims in JWT-based authorization decisions.** In Supabase, `raw_user_meta_data` is user-editable and can appear in `auth.jwt()`, so it is unsafe for RLS policies or any other authorization logic. Store authorization data in `raw_app_meta_data` / `app_metadata` instead. + - **Deleting a user does not invalidate existing access tokens.** Sign out or revoke sessions first, keep JWT expiry short for sensitive apps, and for strict guarantees validate `session_id` against `auth.sessions` on sensitive operations. + - **If you use `app_metadata` or `auth.jwt()` for authorization, remember JWT claims are not always fresh until the user's token is refreshed.** + +- **API key and client exposure** + - **Never expose the `service_role` or secret key in public clients.** Prefer publishable keys for frontend code. Legacy `anon` keys are only for compatibility. In Next.js, any `NEXT_PUBLIC_` env var is sent to the browser. + +- **RLS, views, and privileged database code** + - **Views bypass RLS by default.** In Postgres 15 and above, use `CREATE VIEW ... WITH (security_invoker = true)`. In older versions of Postgres, protect your views by revoking access from the `anon` and `authenticated` roles, or by putting them in an unexposed schema. + - **UPDATE requires a SELECT policy.** In Postgres RLS, an UPDATE needs to first SELECT the row. Without a SELECT policy, updates silently return 0 rows — no error, just no change. + - **Do not put `security definer` functions in an exposed schema.** Keep them in a private or otherwise unexposed schema. + +- **Storage access control** + - **Storage upsert requires INSERT + SELECT + UPDATE.** Granting only INSERT allows new uploads but file replacement (upsert) silently fails. You need all three. + +For any security concern not covered above, fetch the Supabase product security index: `https://supabase.com/docs/guides/security/product-security.md` + +## Supabase CLI + +Always discover commands via `--help` — never guess. The CLI structure changes between versions. + +```bash +supabase --help # All top-level commands +supabase --help # Subcommands (e.g., supabase db --help) +supabase --help # Flags for a specific command +``` + +**Supabase CLI Known gotchas:** + +- `supabase db query` requires **CLI v2.79.0+** → use MCP `execute_sql` or `psql` as fallback +- `supabase db advisors` requires **CLI v2.81.3+** → use MCP `get_advisors` as fallback +- When you need a new migration SQL file, **always** create it with `supabase migration new ` first. Never invent a migration filename or rely on memory for the expected format. + +**Version check and upgrade:** Run `supabase --version` to check. For CLI changelogs and version-specific features, consult the [CLI documentation](https://supabase.com/docs/reference/cli/introduction) or [GitHub releases](https://github.com/supabase/cli/releases). + +## Supabase MCP Server + +For setup instructions, server URL, and configuration, see the [MCP setup guide](https://supabase.com/docs/guides/getting-started/mcp). + +**Troubleshooting connection issues** — follow these steps in order: + +1. **Check if the server is reachable:** + `curl -so /dev/null -w "%{http_code}" https://mcp.supabase.com/mcp` + A `401` is expected (no token) and means the server is up. Timeout or "connection refused" means it may be down. + +2. **Check `.mcp.json` configuration:** + Verify the project root has a valid `.mcp.json` with the correct server URL. If missing, create one pointing to `https://mcp.supabase.com/mcp`. + +3. **Authenticate the MCP server:** + If the server is reachable and `.mcp.json` is correct but tools aren't visible, the user needs to authenticate. The Supabase MCP server uses OAuth 2.1 — tell the user to trigger the auth flow in their agent, complete it in the browser, and reload the session. + +## Supabase Documentation + +Before implementing any Supabase feature, find the relevant documentation. Use these methods in priority order: + +1. **MCP `search_docs` tool** (preferred — returns relevant snippets directly) +2. **Fetch docs pages as markdown** — any docs page can be fetched by appending `.md` to the URL path. +3. **Web search** for Supabase-specific topics when you don't know which page to look at. + +## Making and Committing Schema Changes + +**To make schema changes, use `execute_sql` (MCP) or `supabase db query` (CLI).** These run SQL directly on the database without creating migration history entries, so you can iterate freely and generate a clean migration when ready. + +Do NOT use `apply_migration` to change a local database schema — it writes a migration history entry on every call, which means you can't iterate, and `supabase db diff` / `supabase db pull` will produce empty or conflicting diffs. If you use it, you'll be stuck with whatever SQL you passed on the first try. + +**When ready to commit** your changes to a migration file: + +1. **Run advisors** → `supabase db advisors` (CLI v2.81.3+) or MCP `get_advisors`. Fix any issues. +2. **Review the Security Checklist above** if your changes involve views, functions, triggers, or storage. +3. **Generate the migration** → `supabase db pull --local --yes` +4. **Verify** → `supabase migration list --local` + +## Reference Guides + +- **Skill Feedback** → [references/skill-feedback.md](references/skill-feedback.md) + **MUST read when** the user reports that this skill gave incorrect guidance or is missing information. diff --git a/.agents/skills/supabase/assets/feedback-issue-template.md b/.agents/skills/supabase/assets/feedback-issue-template.md new file mode 100644 index 00000000..e8108253 --- /dev/null +++ b/.agents/skills/supabase/assets/feedback-issue-template.md @@ -0,0 +1,17 @@ +## What happened + +**Task:** + +**Skill said:** + +**Expected:** + +## Source + +**File:** + +**Section:** + +## Fix suggestion + + diff --git a/.agents/skills/supabase/references/skill-feedback.md b/.agents/skills/supabase/references/skill-feedback.md new file mode 100644 index 00000000..9c047d5a --- /dev/null +++ b/.agents/skills/supabase/references/skill-feedback.md @@ -0,0 +1,17 @@ +# Skill Feedback + +Use this when the user reports that the skill gave incorrect guidance, is missing information, or could be improved. This is about the skill (agent instructions), not about Supabase the product. + +## Steps + +1. **Ask permission** — Ask the user if they'd like to submit feedback to the skill maintainers. If they decline, move on. + +2. **Draft the issue** — Use the template at [assets/feedback-issue-template.md](../assets/feedback-issue-template.md) to structure the feedback. Fill in the fields based on the conversation. Always identify which specific reference file and section caused the problem. + +3. **Submit** — Create a GitHub Issue on the `supabase/agent-skills` repository using the draft as the issue body. The title must follow this format: `user-feedback: `. + +4. **Share the result** — Share the issue URL with the user after submission. If submission fails, give the user this link to create the issue manually: + +``` +https://github.com/supabase/agent-skills/issues/new +``` diff --git a/.agents/workflows/researching-plugins/SKILL.md b/.agents/workflows/researching-plugins/SKILL.md new file mode 100644 index 00000000..4cca1f30 --- /dev/null +++ b/.agents/workflows/researching-plugins/SKILL.md @@ -0,0 +1,101 @@ +--- +name: researching-plugins +description: Use when evaluating new data sources, researching APIs, or planning a new plugin structure, before writing any implementation code +--- + +# Researching Plugins + +## Overview +This skill defines the rigorous process for researching, evaluating, and planning new data source plugins for WorldWideView. It combines data source validation (evaluating free tiers, rate limits, and comprehensiveness) with the generation of a placeholder-free, TDD-focused implementation plan. + +## When to Use +- The user requests a new data source or layer (e.g., "add weather data", "track global earthquakes") +- You need to evaluate an API or dataset for continuous polling and 3D globe visualization +- You are formulating the implementation steps for a new plugin +- *Do not use* if the plugin is already fully planned and you are just executing tasks + +## 1. Research & Evaluation Process + +### Step 1: Take Inspiration from WorldMonitor +Before searching the web blindly, check if the `koala73/worldmonitor` repository already implements a similar feature. +- Review their data sources, API endpoints, and polling intervals. +- Analyze which data fields they map for visualization. +- Use this as a baseline for what is possible and expected. + +### Step 2: Validate Data Sources +Search for APIs that meet the following strict criteria. If an API fails these, keep searching or ask the user for guidance: +- **Cost**: Prioritize highly robust free tiers or completely open data. +- **Comprehensiveness**: Does it provide global coverage? Does it return coordinates (lat/lon) and identifying metadata? +- **Rate Limits**: Can it support the required polling interval? (Real-time data needs high rate limits). +- **Format**: JSON or GeoJSON is preferred. + +## 2. Draft the Implementation Plan + +Once the data source and API are validated, you MUST write a comprehensive implementation plan using the standards from the `writing-plans` skill. + +Save the research and plan to `.agents/context/YYYY-MM-DD--research.md`. + +### Required Plan Header +```markdown +# [Plugin Name] Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** [One sentence describing what this plugin does] +**Data Source:** [API Name, API URL, Auth Method, Rate Limits] +**Architecture:** [Proxy route, Static Data, or Microservice?] + +--- +``` + +### Bite-Sized Task Structure +Each task in the plan MUST follow a strict TDD, action-by-action sequence. + +**Required Task Format:** +```markdown +### Task N: [Component Name] + +**Files:** +- Create: `packages/wwv-plugin-[name]/src/index.ts` +- Test: `packages/wwv-plugin-[name]/test/index.test.ts` + +- [ ] **Step 1: Write the failing test** +[Include full test code block here] + +- [ ] **Step 2: Run test to verify it fails** +Run: `pnpm test` +Expected: FAIL + +- [ ] **Step 3: Write minimal implementation** +[Include full implementation code block here] + +- [ ] **Step 4: Run test to verify it passes** +Run: `pnpm test` +Expected: PASS + +- [ ] **Step 5: Commit** +[Include git commit command] +``` + +## The Iron Law of Plan Writing + +1. **No Placeholders:** You MUST provide the exact file paths and complete code blocks for every step. Do not write "TODO: Add error handling" or "Add validation here." +2. **Exact File Paths Always:** Define exactly where files live (e.g., `src/app/api/...`). +3. **DRY & YAGNI:** Do not add speculative features or abstract prematurely. +4. **Define the Types First:** Ensure the API response interfaces and the plugin manifest types are defined early in the plan. + +## Common Mistakes & Rationalizations + +| Excuse / Rationalization | Reality / Fix | +|--------------------------|---------------| +| "I'll propose a paid API because the free ones are hard to find" | WorldWideView is a free tool; prioritize open data. Dig deeper into governmental or open-source datasets. | +| "I left TODOs in the plan so the execution agent can fill them in" | Execution agents fail when given placeholders. The plan MUST contain the exact code to be written. | +| "I skipped the test steps because it's just an API wrapper" | Every feature requires tests. Draft a test that mocks the API response. | +| "I forgot to specify the architecture type" | You must decide if it's a Static, Proxy, or Microservice plugin during the research phase. | + +## Red Flags - STOP and Start Over +If you find yourself doing any of the following, delete your work and start over: +- Proposing an API with a 50 requests/day limit for real-time tracking. +- Writing a plan step that says "Implement similar to Task 2." +- Omitting exact file paths or complete code blocks in your tasks. +- Failing to verify if WorldMonitor has already solved this problem. diff --git a/.agents/workflows/worldwideview-plugin-creation/SKILL.md b/.agents/workflows/worldwideview-plugin-creation/SKILL.md new file mode 100644 index 00000000..ce8459c6 --- /dev/null +++ b/.agents/workflows/worldwideview-plugin-creation/SKILL.md @@ -0,0 +1,551 @@ +--- +name: worldwideview-plugin-creation +description: Use when creating a new plugin, adding a new data source, or debugging missing plugin data in the WorldWideView project. Triggers on seeder creation, WebSocket streaming issues, plugin registration failures, manifest validation errors, GeoEntity rendering problems, or data engine integration +--- + +# WorldWideView Plugin Creation + +## Overview + +A WorldWideView plugin is a self-contained data source that renders geospatial entities on a 3D Cesium globe. Every plugin implements the `WorldPlugin` interface from the SDK and connects to a **data engine** (Fastify + Redis) that streams live data over WebSocket. + +**Core principle:** Plugins are code bundles. The engine pushes data. The frontend renders it. There is no REST polling between frontend and engine — only WebSocket streaming. + +## When to Use + +- Creating a new data layer plugin (any architecture) +- Building a new seeder in the data engine +- Plugin data doesn't appear on the globe +- WebSocket subscription or split-routing issues +- Build failures after adding a plugin package +- Debugging `renderEntity` or entity clipping artifacts +- Marketplace plugin installation/validation issues + +**Do NOT use for:** +- Modifying globe rendering primitives → see `cesium-rendering` rule +- Zustand state management → see `state-management` rule +- Prisma schema changes → see `database-migrations` rule +- Monorepo/pnpm workspace issues → see `monorepo-workflow` rule + +## Architecture Decision + +```dot +digraph plugin_type { + rankdir=TB; + node [shape=diamond]; + + "Does it need live\nexternal API data?" -> "Is polling\nfrequency > 30s?" [label="yes"]; + "Does it need live\nexternal API data?" -> "Static GeoJSON\n(wwvStaticCompiler)" [label="no"]; + "Is polling\nfrequency > 30s?" -> "Cron Seeder\n(engine)" [label="yes"]; + "Is polling\nfrequency > 30s?" -> "Init Seeder\n(engine)" [label="no"]; + + "Static GeoJSON\n(wwvStaticCompiler)" [shape=box]; + "Cron Seeder\n(engine)" [shape=box]; + "Init Seeder\n(engine)" [shape=box]; +} +``` + +--- + +## Quick Reference + +| Piece | Location | Purpose | +|---|---|---| +| Sandbox plugin | `local-plugins/wwv-plugin-/` | Local development workspace for new plugins | +| Plugin class | `packages/wwv-plugin-/src/index.ts` | Implements `WorldPlugin` interface | +| Package metadata | `packages/wwv-plugin-/package.json` | `worldwideview` block with id, type, format, category | +| Engine seeder | `wwv-data-engine/src/seeders/.ts` | Fetches external data → Redis → WS broadcast | +| Build config | `next.config.ts` → `transpilePackages` | Required for monorepo packages | +| TS alias | `tsconfig.json` → `paths` | Required for monorepo packages | +| Registration | `src/core/plugins/PluginRegistry.ts` | Register plugin at boot | +| WS client | `src/core/data/WsClient.ts` | Client-side WebSocket subscriptions | +| URL resolution | `src/core/data/resolveEngineUrl.ts` | Per-plugin local vs cloud routing | +| Manifest validation | `src/core/plugins/validateManifest.ts` | Checks required fields for CDN bundles | +| Host globals | `src/core/plugins/hostGlobals.ts` | Injects shared deps for CDN plugins | + +--- + +## Part 1: Frontend Plugin (WorldPlugin) + +### 1.0 Development Workflow (Local Sandbox) + +The standard way to develop a new plugin is using the **Local Sandbox** (`local-plugins/` directory). This keeps experimental code out of the main monorepo packages until it is ready for production. + +1. **Scaffold**: Run `node packages/wwv-cli/dist/index.js create --local` from the project root. This generates a boilerplate plugin in `local-plugins/wwv-plugin-`. +2. **Develop**: Run `pnpm dev`. The built-in file watcher (`pnpm dev:plugins`) automatically rebuilds and syncs your local plugin to `public/plugins-local/` whenever you save a file. +3. **Promote**: When stable, run `node packages/wwv-cli/dist/index.js link `. This promotes the plugin to the `packages/` directory and cleans up the sandbox artifacts. +4. **Install**: Run `pnpm install` from the root to update workspace symlinks. + +### 1.1 Implement the Interface + +Every plugin **must** implement `WorldPlugin` from `@worldwideview/wwv-plugin-sdk`: + +```typescript +import type { + WorldPlugin, PluginContext, GeoEntity, + CesiumEntityOptions, TimeRange, LayerConfig, PluginCategory +} from "@worldwideview/wwv-plugin-sdk"; +import { Flame } from "lucide-react"; +import pkg from "../package.json"; + +export default class WildfiresPlugin implements WorldPlugin { + id = "wildfires"; + name = "Wildfires"; + description = "Live wildfire incidents from NASA FIRMS"; + icon = Flame; + category: PluginCategory = "natural-disaster"; + version = pkg.version; // NEVER hardcode — always import from package.json + + async initialize(ctx: PluginContext): Promise { + // Store context for later use (env vars, edition, callbacks) + } + + destroy(): void { + // Cleanup timers, WebSocket connections, etc. + } + + async fetch(timeRange: TimeRange): Promise { + // For WS-streamed plugins, return [] — data arrives via WsClient + return []; + } + + getPollingInterval(): number { + return 0; // 0 = no REST polling (WS-only) + } + + getLayerConfig(): LayerConfig { + return { + color: "#ef4444", + clusterEnabled: true, + clusterDistance: 50, + maxEntities: 5000, + }; + } + + renderEntity(entity: GeoEntity): CesiumEntityOptions { + return { + type: "point", + color: "#ef4444", + size: 6, + outlineColor: "#ffffff", + outlineWidth: 1, + }; + } +} +``` + +### 1.2 Required vs Optional Methods + +**Required:** + +| Method | Purpose | +|---|---| +| `initialize(ctx)` | Receive `PluginContext` (env, edition, callbacks) | +| `destroy()` | Cleanup on shutdown | +| `fetch(timeRange)` | Return entities (REST path). Return `[]` for WS-only | +| `getPollingInterval()` | Polling frequency in ms. Return `0` for WS-only | +| `getLayerConfig()` | Layer appearance (color, clustering, max entities) | +| `renderEntity(entity)` | Per-entity Cesium rendering options | + +**Optional:** + +| Method | Purpose | +|---|---| +| `getServerConfig()` | REST/WS endpoint config (`apiBasePath`, `streamUrl`) | +| `getSelectionBehavior(entity)` | Trail/fly-to on entity click | +| `getFilterDefinitions()` | User-facing filter controls | +| `getLegend()` | Legend entries for layer panel | +| `getSidebarComponent()` | Custom sidebar React component | +| `getDetailComponent()` | Custom detail panel for selected entity | +| `getSettingsComponent()` | Plugin settings UI | +| `getGlobeComponent()` | React component injected into globe view | +| `mapWebsocketPayload(payload, existing)` | Transform raw WS data into `GeoEntity[]` | +| `requiresConfiguration(settings)` | Return `true` if plugin needs setup first | + +### 1.3 Package Metadata + +The `package.json` **must** include a `worldwideview` block: + +```json +{ + "name": "@worldwideview/wwv-plugin-wildfires", + "version": "1.0.0", + "main": "src/index.ts", + "worldwideview": { + "id": "wildfires", + "type": "data-layer", + "format": "bundle", + "category": "natural-disaster", + "icon": "Flame", + "capabilities": ["data:own", "globe:overlay"] + }, + "dependencies": { + "@worldwideview/wwv-plugin-sdk": "workspace:*" + } +} +``` + +**Valid categories** (lowercase only): `aviation` · `maritime` · `military` · `conflict` · `natural-disaster` · `infrastructure` · `space` · `cyber` · `economic` · `intelligence` · `custom` + +**Valid capabilities**: `data:own` · `data:read:{plugin}` · `ui:detail-panel` · `ui:sidebar` · `ui:toolbar` · `ui:settings` · `globe:overlay` · `globe:camera` · `storage:read` · `storage:write` · `network:fetch` + +### 1.4 Rendering Rules + +| Entity Type | Properties | Notes | +|---|---|---| +| `"point"` | `color`, `size`, `outlineColor`, `outlineWidth` | Simple colored dots | +| `"billboard"` | `iconUrl`, `color`, `iconScale` | SVG/PNG icons | +| `"model"` | `modelUrl`, `modelScale`, `modelMinPixelSize` | 3D glTF models | +| `"label"` | `labelText`, `labelFont`, `color` | Text labels | + +> **CRITICAL:** NEVER mix point and billboard properties. Using `size`/`outlineWidth` on a billboard entity causes GPU clipping artifacts. This is a hard rendering constraint. + +Use `createSvgIconUrl()` from the SDK for billboard icons: + +```typescript +import { createSvgIconUrl } from "@worldwideview/wwv-plugin-sdk"; +import { Plane } from "lucide-react"; +const iconUrl = createSvgIconUrl(Plane, { color: "#3b82f6" }); +``` + +### 1.5 Registration (3 Paths) + +**A. Built-in (monorepo package):** + +```typescript +import { pluginRegistry } from "@/core/plugins/PluginRegistry"; +import { pluginManager } from "@/core/plugins/PluginManager"; +import WildfiresPlugin from "@worldwideview/wwv-plugin-wildfires"; + +const plugin = new WildfiresPlugin(); +pluginRegistry.register(plugin); +await pluginManager.registerPlugin(plugin); +``` + +`AppShell.tsx` iterates `pluginRegistry.getAll()` at startup. + +**B. Marketplace (database-installed):** +`InstalledPluginsLoader` reads from the `installed_plugins` PostgreSQL table and calls `pluginManager.loadFromManifest()`. No code changes needed. + +**C. Dynamic import (runtime):** +For user-imported GeoJSON layers, call `pluginManager.loadFromManifest(manifest)` directly. + +### 1.6 Build Configuration (Promoted Monorepo Packages Only) + +Once a plugin is promoted from the local sandbox via the `link` command, it becomes a formal monorepo package. You must manually add it to the build pipeline: + +Add to `next.config.ts`: + +```typescript +transpilePackages: [ + "@worldwideview/wwv-plugin-sdk", + "@worldwideview/wwv-plugin-wildfires", // ADD +], +``` + +Add to `tsconfig.json`: + +```json +"paths": { + "@worldwideview/wwv-plugin-wildfires": ["./packages/wwv-plugin-wildfires/src"] +} +``` + +Run `pnpm install` from project root after creating the package. + +--- + +## Part 2: Data Engine Seeder + +The data engine is a **content-agnostic runner** (`wwv-data-engine` public, deployed via Docker). It runs Fastify on port 5000 with Redis caching and WebSocket streaming. Seeders are volume-mounted from `local-seeders/` (dev) or downloaded from GitHub Releases (production). + +> [!IMPORTANT] +> If you are **migrating** an existing plugin from `packages/` to `local-plugins/`, use the `migrate-legacy-plugin` skill instead. It covers both frontend routing fixes AND backend seeder build fixes as two independent concerns. + +### 2.1 Engine Architecture + +``` +External API → Seeder (fetch + transform) + → setLiveSnapshot(pluginId, payload, ttlSeconds) + → broadcastPluginData() → Active WS subscribers (immediate) + → Redis SET data:{pluginId}:live (throttled to every 5 min) + → SQLite history (optional, via better-sqlite3) +``` + +**Key components:** +- `src/scheduler.ts` — Seeder registry and cron scheduling +- `src/redis.ts` — Redis client, `setLiveSnapshot()` and `getLiveSnapshot()` +- `src/websocket.ts` — WS handler, `broadcastPluginData()` +- `src/seed-utils.ts` — `withRetry()`, `fetchWithTimeout()`, `haversineKm()` +- `src/db.ts` — SQLite (better-sqlite3) for local history storage +- `src/seeders/index.ts` — Auto-discovery, dynamically requires all seeder files + +### 2.2 Two Seeder Patterns + +**Cron Seeder** (periodic polling — most common): + +```typescript +import { db } from "../db"; +import { setLiveSnapshot } from "../redis"; +import { fetchWithTimeout, withRetry } from "../seed-utils"; +import { registerSeeder } from "../scheduler"; + +// Optional: SQLite history table insert +const insertRow = db.prepare( + "INSERT OR IGNORE INTO my_data (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)" +); + +async function seedMyPlugin() { + console.log("[MyPlugin] Polling external API..."); + + const res = await withRetry(() => + fetchWithTimeout("https://api.example.com/data") + ); + const data = await res.json(); + + const items = data.features.map((f: any) => ({ + id: f.id, + lat: f.geometry.coordinates[1], + lon: f.geometry.coordinates[0], + name: f.properties.name, + // ... transform to your schema + })); + + // Optional: save to SQLite + for (const item of items) { + insertRow.run({ + id: item.id, + payload: JSON.stringify(item), + source_ts: Date.now(), + fetched_at: Date.now(), + }); + } + + // REQUIRED: Push to Redis + broadcast to WS subscribers + await setLiveSnapshot("my-plugin", { + source: "my-plugin", + fetchedAt: new Date().toISOString(), + items, + totalCount: items.length, + }, 3600); // TTL in seconds +} + +registerSeeder({ + name: "my-plugin", // MUST match the plugin's id on the frontend + cron: "*/15 * * * *", // Every 15 minutes + fn: seedMyPlugin, +}); +``` + +**Init Seeder** (persistent/high-frequency — e.g., ISS every 5s): + +```typescript +import { setLiveSnapshot } from "../redis"; +import { registerSeeder } from "../scheduler"; + +function startMySeeder() { + console.log("[MyPlugin] Starting persistent seeder..."); + + async function poll() { + const res = await fetch("https://api.example.com/live"); + const data = await res.json(); + await setLiveSnapshot("my-plugin", data, 60); + } + + poll(); // Initial fetch + setInterval(poll, 5000); // Every 5 seconds +} + +registerSeeder({ + name: "my-plugin", + init: startMySeeder, // No cron — uses init for persistent polling +}); +``` + +### 2.3 SeederDefinition Interface + +```typescript +interface SeederDefinition { + name: string; // Plugin ID — MUST match frontend plugin id + cron?: string; // Cron expression (e.g., "0 * * * *" = hourly) + fn?: () => Promise; // Function called on each cron tick + init?: () => void; // Initialization for persistent seeders (websockets, intervals) +} +``` + +### 2.4 Auto-Discovery + +`src/seeders/index.ts` dynamically `require()`s every `.ts`/`.js` file in the seeders directory (except `index.ts` and test files). **You do not need to manually import your seeder** — just create the file and call `registerSeeder()` at module scope. + +### 2.5 SQLite History (Optional) + +For data that benefits from historical queries (playback mode), add a table in `src/db.ts`: + +```typescript +db.exec(` + CREATE TABLE IF NOT EXISTS my_data ( + id TEXT PRIMARY KEY, + payload JSON NOT NULL, + source_ts INTEGER NOT NULL, + fetched_at INTEGER NOT NULL + ) +`); +``` + +### 2.6 Utility Functions + +| Function | Purpose | +|---|---| +| `withRetry(fn, maxRetries, delayMs)` | Exponential backoff retry wrapper | +| `fetchWithTimeout(url, options, timeoutMs)` | Fetch with abort signal (default 15s) | +| `haversineKm(lat1, lon1, lat2, lon2)` | Distance between coordinates in km | +| `sleep(ms)` | Promise-based delay | +| `CHROME_UA` | Chrome User-Agent string for scraping | + +--- + +## Part 3: Data Pipeline (Frontend ↔ Engine) + +### 3.1 WebSocket Protocol + +**Client → Engine:** +```json +{ "action": "subscribe", "pluginId": "earthquakes" } +{ "action": "unsubscribe", "pluginId": "earthquakes" } +``` + +**Engine → Client:** +```json +{ "type": "welcome", "engine": "wwv-data-engine", "plugins": ["earthquakes", "wildfires", ...] } +{ "type": "data", "pluginId": "earthquakes", "payload": { "items": [...], "totalCount": 42 } } +``` + +On subscribe, the engine immediately pushes the latest cached Redis snapshot — no waiting for the next cron tick. + +### 3.2 Split-Routing (resolveEngineUrl) + +`resolveEngineUrl(pluginId)` determines which engine serves each plugin: + +1. **Local engine** (`localhost:5000`) if running and its `/manifest` includes this plugin ← **HIGHEST PRIORITY** +2. Plugin's `getServerConfig().streamUrl` (code override) +3. Plugin's `PluginManifest.dataSource.streamUrl` (manifest override) +4. `NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL` env var +5. Fallback: `wss://dataengine.worldwideview.dev/stream` (cloud) + +### 3.3 Full Data Flow + +``` +Engine seeder calls setLiveSnapshot(pluginId, payload, ttl) + → broadcastPluginData() fans out to subscribed WS connections + → WsClient receives { type: "data", pluginId, payload } + → WsClient calls plugin.mapWebsocketPayload() if defined + → DataBus.emit("dataUpdated", { pluginId, entities }) + → DataBusSubscriber → Zustand store.entitiesByPlugin + → GlobeView (memoized visible entities) + → EntityRenderer (billboard/point/label primitives) +``` + +### 3.4 Host Globals (CDN Plugins) + +CDN-loaded plugins share the host app's dependencies via `globalThis.__WWV_HOST__`: +- `react`, `react-dom`, `react/jsx-runtime` +- `cesium`, `resium` +- `zustand` +- `@worldwideview/wwv-plugin-sdk` + +Use `wwvPluginGlobals()` in your Vite config to externalize automatically: + +```typescript +import { defineConfig } from "vite"; +import { wwvPluginGlobals } from "@worldwideview/wwv-plugin-sdk"; + +export default defineConfig({ + plugins: [wwvPluginGlobals()], + build: { + lib: { entry: "./src/index.ts", formats: ["es"], fileName: "frontend" }, + rollupOptions: { output: { entryFileNames: "frontend.mjs" } }, + }, +}); +``` + +--- + +## Part 4: Static GeoJSON Plugins + +For plugins with static data (no live API), use `wwvStaticCompiler`: + +```typescript +import { defineConfig } from "vite"; +import { wwvPluginGlobals, wwvStaticCompiler } from "@worldwideview/wwv-plugin-sdk"; + +export default defineConfig({ + plugins: [wwvStaticCompiler(), wwvPluginGlobals()], + build: { + lib: { entry: "./src/index.ts", formats: ["es"], fileName: "frontend" }, + }, +}); +``` + +Place GeoJSON in `data/data.json`. The compiler auto-generates a `WorldPlugin` class from `package.json` metadata at build time. No `src/index.ts` needed. + +--- + +## Common Mistakes + +### "Sandbox Plugin Breaking Next.js Build" +1. **Premature Configuration** — Modifying `next.config.ts` (`transpilePackages`) or `tsconfig.json` (`paths`) while a plugin is still in `local-plugins/`. The `local-plugins` sandbox relies on dynamic hot-loading in the browser via `pnpm dev:plugins`. Do **not** touch monorepo configs until the plugin is formally promoted using `wwv link `. +2. **Manual Registration** — Manually adding a sandbox plugin to `AppShell.tsx`. Let the local hot-loading manifest handle sandbox plugins dynamically. + +### "Missing Boilerplate / Errors in package.json" +1. **Bypassing the CLI** — Creating a plugin manually instead of using `node packages/wwv-cli/dist/index.js create --local`. Always use the CLI to ensure the correct metadata, type definitions, and test scaffolding are generated. + +### "No data on the globe" +1. **Seeder name ≠ plugin id** — The `name` in `registerSeeder()` MUST match the `id` property of the frontend `WorldPlugin` class +2. **Engine `/manifest` missing plugin** — Check `GET localhost:5000/manifest` → `plugins` array +3. **WsClient not subscribing** — Add `console.log` in `resolveEngineUrl()` to verify URL resolution +4. **Plugin hardcodes engine URL** — Legacy plugins use `this.context?.apiBaseUrl` instead of `resolveEngineUrl()`. See `migrate-legacy-plugin` skill +4. **`mapWebsocketPayload` missing** — If the engine sends a non-standard payload shape, the plugin must implement `mapWebsocketPayload()` or data silently drops + +### "Build crash / Module not found" +1. Missing `transpilePackages` entry in `next.config.ts` → **immediate runtime crash** +2. Missing `tsconfig.json` path alias → TypeScript compilation error +3. Forgot `pnpm install` after creating package → dependency resolution failure + +### "Invalid Hook Call" (CDN plugins) +Plugin bundles its own React instead of using host globals. Fix: add `wwvPluginGlobals()` to Vite config. + +### "Manifest validation failed" +Required fields: `id`, `name`, `version`, `entry`, `trust`. Entry URL must be from allowed domains (CDN, localhost, `worldwideview.dev`, or relative path). `capabilities` must be non-empty array. + +### "GPU clipping / entity disappearing" +Using `size`/`outlineWidth`/`outlineColor` on a `"billboard"` entity. These properties are **point-only**. Billboard uses `iconUrl`/`iconScale`/`color`. + +### "Redis snapshot not updating" +`setLiveSnapshot()` throttles Redis writes to every 5 minutes to avoid exceeding Upstash request limits. WebSocket broadcasts are **not** throttled. Check WS delivery first. + +### "Category type error" +Categories are **lowercase**: `"aviation"` not `"Aviation"`, `"natural-disaster"` not `"NaturalDisaster"`. Must match the `PluginCategory` union in the SDK. + +--- + +## End-to-End Checklist + +- [ ] **Development:** Scaffolded via `wwv create --local` and developed in `local-plugins/` +- [ ] **Promotion:** Promoted to monorepo via `wwv link ` when stable +- [ ] **Frontend:** Plugin class implements `WorldPlugin` with correct `id` +- [ ] **Frontend:** `package.json` has `worldwideview` block with matching `id` +- [ ] **Frontend:** Version imported from `package.json` (never hardcoded) +- [ ] **Frontend:** Added to `transpilePackages` in `next.config.ts` +- [ ] **Frontend:** Added path alias in `tsconfig.json` +- [ ] **Frontend:** Registered via `pluginRegistry` + `pluginManager` in `AppShell.tsx` +- [ ] **Frontend:** `renderEntity()` doesn't mix point/billboard properties +- [ ] **Engine:** Seeder file in `src/seeders/.ts` +- [ ] **Engine:** `registerSeeder({ name })` matches frontend plugin `id` +- [ ] **Engine:** Calls `setLiveSnapshot()` with correct plugin ID key +- [ ] **Engine:** Data shape is an object with `items` array (or plugin implements `mapWebsocketPayload`) +- [ ] **Engine:** SQLite table created in `db.ts` if historical data needed +- [ ] **Verification:** `GET localhost:5000/manifest` includes the plugin ID +- [ ] **Verification:** `GET localhost:5000/health` shows seeder last-run timestamp +- [ ] **Verification:** WebSocket subscribe returns data snapshot +- [ ] **Workspace:** Ran `pnpm install` from project root +- [ ] **Workspace:** Version bump follows `/commit` workflow diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8acc4770..b6d0ebf7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,29 +1,28 @@ -name: Docker Build and Publish +name: Docker Image CI on: push: - branches: [ "main", "feat/docker-ci" ] + branches: [ "main" ] pull_request: branches: [ "main" ] - workflow_dispatch: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: + build-and-push-image: runs-on: ubuntu-latest permissions: contents: read packages: write - + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@v3 - name: Log in to the Container registry uses: docker/login-action@v3 @@ -34,30 +33,30 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=sha + type=sha,format=short - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . - # Push to GHCR only when the workflow has write access to the - # registry: any push event, workflow_dispatch, or PRs whose head - # branch lives in this repo (not a fork). Fork PRs still build — - # so the Dockerfile is exercised and the image is verified — but - # skip the push step that would 403 with - # "denied: installation not allowed to Write organization package". - push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - # GHA cache writes also require write permission, so gate it on - # the same condition. (Reads are fine for everyone.) - cache-to: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'type=gha,mode=max' || '' }} + cache-to: type=gha,mode=max + build-args: | + NEXT_PUBLIC_CESIUM_ION_TOKEN=${{ secrets.NEXT_PUBLIC_CESIUM_ION_TOKEN }} + NEXT_PUBLIC_GOOGLE_MAPS_API_KEY=${{ secrets.NEXT_PUBLIC_GOOGLE_MAPS_API_KEY }} + NEXT_PUBLIC_BING_MAPS_KEY=${{ secrets.NEXT_PUBLIC_BING_MAPS_KEY }} + NEXT_PUBLIC_WWV_EDITION=${{ vars.NEXT_PUBLIC_WWV_EDITION || 'demo' }} + NEXT_PUBLIC_WS_ENGINE_URL=${{ vars.NEXT_PUBLIC_WS_ENGINE_URL }} + NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL=${{ vars.NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL }} + NEXT_PUBLIC_ADSENSE_CLIENT_ID=${{ secrets.NEXT_PUBLIC_ADSENSE_CLIENT_ID }} + NEXT_PUBLIC_SUPABASE_URL=${{ vars.NEXT_PUBLIC_SUPABASE_URL }} + NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }} diff --git a/.gitignore b/.gitignore index 923d5485..59e7c74a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,9 @@ node_modules/ /out/ # agents -/.agents/ +/.agents/plans +/.agents/context +/.agents/research/ # claude code IDE config /.claude/ @@ -103,4 +105,16 @@ build_output*.txt # Seeder repositories /local-seeders/ -/.local-seeders-dist/ \ No newline at end of file +/.local-seeders-dist/ + +# WorldwideView Ecosystem Repositories (Development Structure) +# The following ignores represent the different repositories related to the entire setup found in C:\dev +/worldwideview-marketplace/ +/worldwideview-plugins/ +/worldwideview-web/ +/wwv-data-engine/ +/wwv-data-engine-public/ +/wwv-seeders/ +/wwv-seeders-private/ +/wwv-test/ +worldwideview.code-workspace \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 9015bd6b..92cf3e4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,59 +30,8 @@ Our primary design, feature-set, and operational layout goal is to mimic the str ## 3. Directory Structure -``` -worldwideview/ -├── src/ -│ ├── app/ # Next.js App Router (pages, API routes, layouts) -│ │ ├── api/ # Server-side API routes (auth, aviation, camera, etc.) -│ │ ├── login/ # Login page -│ │ ├── setup/ # First-time setup page -│ │ └── globals.css # Root stylesheet -│ ├── components/ -│ │ ├── common/ # Shared UI: BootOverlay, FloatingWindow, PluginIcon -│ │ ├── layout/ # AppShell, Header, SearchBar, DataBusSubscriber -│ │ ├── panels/ # LayerPanel, EntityInfoCard, FilterPanel, GraphicsSettings -│ │ ├── timeline/ # Timeline component -│ │ ├── marketplace/ # Plugin install/unverified dialogs -│ │ ├── ui/ # Tooltip, ReloadToast -│ │ └── video/ # Floating video manager -│ ├── core/ -│ │ ├── data/ # DataBus, PollingManager, CacheLayer, SmartFetcher -│ │ ├── filters/ # filterEngine (applies plugin filters to entities) -│ │ ├── globe/ # GlobeView, EntityRenderer, AnimationLoop, StackManager, -│ │ │ │ CameraController, InteractionHandler, SelectionHandler, -│ │ │ │ ModelManager, ImageryProviderFactory -│ │ │ └── hooks/ # useCameraActions, useEntityRendering, useModelRendering, etc. -│ │ ├── hooks/ # useBootSequence, useIsMobile, useMarketplaceSync -│ │ ├── plugins/ # PluginManager, PluginRegistry, PluginManifest, -│ │ │ │ loadPluginFromManifest, validateManifest, InstalledPluginsLoader -│ │ │ └── loaders/ # DeclarativePlugin, StaticDataPlugin, mapJsonToEntities -│ │ └── state/ # Zustand store + slices (config, data, globe, layers, timeline, ui, etc.) -│ ├── lib/ # auth, db, rateLimit, analytics, AIS stream, marketplace APIs -│ ├── plugins/ # GeoJSON plugin registrations -│ ├── styles/ # HUD animations CSS -│ ├── types/ # GeoJSON types, Umami types -│ └── generated/ # Prisma generated client (gitignored) -├── local-plugins/ # Local sandbox for developing plugins (gitignored) -├── packages/ # pnpm monorepo workspace packages -│ ├── wwv-plugin-sdk/ # Plugin SDK: type definitions, manifest schema -│ ├── wwv-plugin-aviation/ -│ ├── wwv-plugin-maritime/ -│ ├── wwv-plugin-wildfire/ -│ ├── wwv-plugin-borders/ -│ ├── wwv-plugin-camera/ -│ ├── wwv-plugin-military-aviation/ -│ ├── wwv-plugin-satellite/ -│ ├── wwv-plugin-iranwarlive/ # Standalone plugin with custom endpoints -│ └── wwv-plugin-{airports,embassies,lighthouses,nuclear,seaports,spaceports,volcanoes}/ -├── prisma/ # schema.prisma, migrations/ -├── public/ # Static assets, Cesium workers, plugin GeoJSON data -├── scripts/ # Build scripts (copy-cesium, scaffold-osm-plugin, setup) -├── data/ # PostgreSQL data volume (gitignored) -├── Dockerfile # Multi-stage production build -├── docker-compose.yml # Main app + data engine runner + local seeders -└── .agents/ # Agent documentation, rules, skills, workflows -``` +> [!NOTE] +> See `.agents/rules/directory-structure.md` for the full project directory map and related repositories. --- @@ -217,94 +166,24 @@ Built-in plugins are instantiated in `AppShell.tsx` and registered via `PluginRe ### 5.8 Workspace Hygiene Whenever agents generate temporary debugging scripts, test REST endpoints via `.mjs`, or dump traces/JSON outputs, they **MUST** save these exclusively inside `/local-scripts/`. The root directory is strictly for production configuration files. ---- - -## 6. Environment Variables - -| Variable | Required | Description | -|---|---|---| -| `DATABASE_URL` | Yes | PostgreSQL connection string | -| `AUTH_SECRET` | Yes | JWT signing secret (generate with `openssl rand -hex 32`) | -| `NEXT_PUBLIC_CESIUM_ION_TOKEN` | No | Cesium Ion access token | -| `NEXT_PUBLIC_BING_MAPS_KEY` | No | Bing Maps imagery | -| `NEXT_PUBLIC_GOOGLE_MAPS_API_KEY` | No | Google 3D Tiles | -| `NEXT_PUBLIC_WWV_EDITION` | No | `local` / `cloud` / `demo` (default: `local`) | -| `NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL` | No | Override engine WebSocket URL (default: cloud) | -| `OPENSKY_CREDENTIALS` | No | Comma-separated `id:secret` pairs for credential rotation | -| `WWV_BRIDGE_TOKEN` | No | Shared secret for marketplace → WWV install bridge | -| `WWV_DEMO_ADMIN_SECRET` | No | Demo edition admin password | -| `IRANWARLIVE_BACKEND_URL` | No | Override for IranWarLive custom backend URL | - -Secrets go in `.env.local` (gitignored). Non-secrets go in `.env` (committed). +### 5.9 Rule Adherence & Proactive Updates +> [!CAUTION] +> **Agents will trust the `.agents/rules/` files unconditionally.** +> Whenever you execute a major architectural shift or make a change that invalidates an existing rule, you **MUST** immediately update the corresponding rule file(s) and/or `AGENTS.md` to reflect the new reality. Do not leave rules untouched after a change; operating on outdated information will cause future agents to break the codebase. --- -## 7. Development Commands +## 6. Environment & Configuration -```bash -pnpm install # Install all workspace dependencies -pnpm run setup # Generate .env.local with AUTH_SECRET (first-time setup) -pnpm dev # Next.js frontend only (auto-runs prisma db push + copy-cesium) -pnpm dev:all # Frontend + wwv-data-engine (via Docker Compose) concurrently -pnpm dev:backends # Starts the local data engine + Redis via Docker Compose -pnpm build # Production build -pnpm test # Run all Vitest tests (scoped to src/lib, src/core, src/plugins) -pnpm db:reset # Reset and re-migrate the frontend database (destructive) -pnpm start:backends # Legacy command for standalone Fastify backends -pnpm clean:backends # Wipe all plugin database records -pnpm run scaffold-osm-plugin # Generate a new plugin from scaffold -pnpm dev:plugins # File watcher for local-plugins/ directory (runs automatically in dev) -node packages/wwv-cli/dist/index.js create --local # Scaffold a new local plugin -node packages/wwv-cli/dist/index.js link # Promote a local plugin to packages/ -``` - -Frontend runs at `http://localhost:3000`. +> [!NOTE] +> See `.agents/rules/environment-config.md` for required environment variables and secrets. --- -## 8. Deployment +## 7. Development, Deployment & Testing -- **Docker**: Multi-stage Dockerfile using the Extractor Pattern (`deps` → `builder` → `runner`). The `node_modules` folders must be explicitly untracked in from `git` across the workspace to prevent corrupted BuildKit contextual cache overlaps during the `COPY . .` stage. -- **Standalone output**: `next.config.ts` uses `output: "standalone"`. -- **Cesium assets**: Copied to `public/cesium/` via `scripts/copy-cesium.mjs` at build time, excluded from output tracing. -- **Prisma Configuration**: `prisma.config.ts` must export a native javascript object instead of dynamically importing CLI wrapper binaries (`prisma/config` or `dotenv`). The standalone Next.js tracer strips CLI devDependencies during the build, which will cause fatal runtime container crashes if imported. -- **Data Engine**: The single runner container defined in `docker-compose.yml`, proxied via `next.config.ts` rewrites. -- **Coolify**: Deployed via Dockerfile builder natively mapping environment variables continuously into the container shell. -- **Docker volumes**: Ensure PostgreSQL data and Redis volumes are mounted for persistence. - ---- - -## 9. Testing Strategy - -- **Framework**: Vitest with jsdom environment -- **Coverage**: `src/lib/**`, `src/core/**`, `src/plugins/**` -- **Run**: `pnpm test` (or `vitest run`) -- **Key test files**: `rateLimit.test.ts`, `edition.test.ts`, `demoAdmin.test.ts`, `DeclarativePlugin.test.ts`, `cors.test.ts`, `repository.test.ts`, `marketplaceToken.test.ts` - ---- - -## 10. Security Headers - -Configured in `next.config.ts` `headers()`: -- **CSP**: Restrictive with exceptions for CesiumJS (`unsafe-eval`, `unsafe-inline`), camera streams (`http: https:`), and analytics -- **X-Frame-Options**: DENY -- **X-Content-Type-Options**: nosniff -- **Referrer-Policy**: strict-origin-when-cross-origin -- **Permissions-Policy**: camera/microphone disabled, geolocation self-only - ---- - -## 11. Related Repositories - -| Repo | Purpose | -|---|---| -| `worldwideview` | Main application (this repo) | -| `wwv-data-engine` | Generic data engine runner (PUBLIC, runs via Docker) | -| `wwv-seeders-community` | Open-source community seeders (PUBLIC) | -| `wwv-seeders-private` | Proprietary seeder scripts (PRIVATE, downloaded in prod) | -| `worldwideview-marketplace` | Plugin marketplace web app | -| `worldwideview-plugins` | Published npm plugin packages (frontend source) | -| `worldwideview-web` | Marketing / landing page | +> [!NOTE] +> See `.agents/rules/deployment-and-testing.md` for development commands, Docker architecture, Vitest strategy, and CSP security headers. --- @@ -314,14 +193,24 @@ Read the relevant rule file when working in that domain: | Rule | When to use | Path | |---|---|---| +| `platform-architecture` | High-level platform goals, product vision, business model, and Edition System | `.agents/rules/platform-architecture.md` | +| `application-architecture` | Next.js frontend, Zustand state management, and CesiumJS integration | `.agents/rules/application-architecture.md` | +| `plugin-architecture` | Creating/modifying plugins, lifecycle, capability declarations, and seeders | `.agents/rules/plugin-architecture.md` | +| `marketplace-architecture` | Dynamic plugin installation, DB sync, and CDN loading | `.agents/rules/marketplace-architecture.md` | +| `cloud-auth-architecture` | Cloud edition, PostgreSQL RLS, multi-tenancy, and licensing | `.agents/rules/cloud-auth-architecture.md` | +| `server-management` | Server development and debugging using SSH and Coolify MCP | `.agents/rules/server-management.md` | +| `stakeholders-and-human-centered-design` | Human-centered design principles and stakeholder map | `.agents/rules/stakeholders-and-human-centered-design.md` | +| `directory-structure` | Project structure and related repositories | `.agents/rules/directory-structure.md` | +| `deployment-and-testing` | Docker build patterns, testing strategies, security headers | `.agents/rules/deployment-and-testing.md` | +| `environment-config` | Secrets and environment variables | `.agents/rules/environment-config.md` | | `monorepo-workflow` | pnpm commands, adding packages, workspace config | `.agents/rules/monorepo-workflow.md` | -| `plugin-architecture` | Creating/modifying plugins, lifecycle, registration | `.agents/rules/plugin-architecture.md` | | `data-engine-architecture` | Data Engine backend seeder loading, pnpm workspace dependencies | `.agents/rules/data-engine-architecture.md` | | `cesium-rendering` | Globe rendering, entity types, primitives, LOD, culling | `.agents/rules/cesium-rendering.md` | | `state-management` | Zustand slices, store access, plugin settings | `.agents/rules/state-management.md` | | `database-migrations` | Prisma schema changes, migrations, PostgreSQL | `.agents/rules/database-migrations.md` | | `continuous-improvement` | When to create/update rules, skills, or workflows | `.agents/rules/continuous-improvement.md` | | `context-and-memory` | How to orient and maintain project context between sessions | `.agents/rules/context-and-memory.md` | +| `troubleshooting-and-debugging` | Resolving plugin latency, namespace collisions, or deployment issues | `.agents/rules/troubleshooting-and-debugging.md` | --- diff --git a/package.json b/package.json index f7f4f4f2..c62927f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "worldwideview", - "version": "2.7.5", + "version": "2.7.6", "private": true, "type": "module", "packageManager": "pnpm@9.15.4", diff --git a/src/components/layout/DataBusSubscriber.tsx b/src/components/layout/DataBusSubscriber.tsx index 73c5ff0f..ee0396f7 100644 --- a/src/components/layout/DataBusSubscriber.tsx +++ b/src/components/layout/DataBusSubscriber.tsx @@ -44,14 +44,15 @@ export function DataBusSubscriber() { }); const unsubToggle = dataBus.on("layerToggled", ({ pluginId, enabled }) => { - console.log(`[DataBusSubscriber] layerToggled event received for ${pluginId}, enabled: ${enabled}`); + const t0 = performance.now(); + console.debug(`[DataBusSubscriber] layerToggled event received for ${pluginId}, enabled: ${enabled}`); const engineUrl = resolveEngineUrl(pluginId); - console.log(`[DataBusSubscriber] Resolved engine URL for ${pluginId} to ${engineUrl}`); + console.debug(`[DataBusSubscriber] Resolved engine URL for ${pluginId} to ${engineUrl}. Took ${(performance.now() - t0).toFixed(2)}ms`); if (enabled) { - console.log(`[DataBusSubscriber] Calling wsClient.subscribe(${pluginId}, ${engineUrl})`); + console.debug(`[DataBusSubscriber] Calling wsClient.subscribe(${pluginId}, ${engineUrl})`); wsClient.subscribe(pluginId, engineUrl); } else { - console.log(`[DataBusSubscriber] Calling wsClient.unsubscribe(${pluginId}, ${engineUrl})`); + console.debug(`[DataBusSubscriber] Calling wsClient.unsubscribe(${pluginId}, ${engineUrl})`); wsClient.unsubscribe(pluginId, engineUrl); } }); diff --git a/src/core/data/WsClient.ts b/src/core/data/WsClient.ts index 82a9063e..49cb352c 100644 --- a/src/core/data/WsClient.ts +++ b/src/core/data/WsClient.ts @@ -38,10 +38,11 @@ class WebSocketClient { return; } + const wsStart = performance.now(); engine.ws = new WebSocket(engineUrl); engine.ws.onopen = () => { - console.log(`[WSClient] 🟢 Connected to ${engineUrl}`); + console.debug(`[WSClient] 🟢 Connected to ${engineUrl}. WS Handshake took ${(performance.now() - wsStart).toFixed(2)}ms`); // Resubscribe to all active plugins on this engine for (const pluginId of engine.subscriptions) { this.send(engine, { action: "subscribe", pluginId }); @@ -50,12 +51,13 @@ class WebSocketClient { engine.ws.onmessage = (event) => { try { - console.log(`[WSClient] 📥 Received raw message:`, event.data.substring(0, 150) + (event.data.length > 150 ? '...' : '')); + const msgTime = performance.now(); + console.debug(`[WSClient] 📥 Received raw message at +${(msgTime - wsStart).toFixed(2)}ms from start:`, event.data.substring(0, 150) + (event.data.length > 150 ? '...' : '')); const data = JSON.parse(event.data); // Handle welcome message (informational, no action needed) if (data.type === "welcome") { - console.log(`[WSClient] 👋 Engine ${engineUrl} serves: ${data.plugins?.join(", ")}`); + console.debug(`[WSClient] 👋 Engine ${engineUrl} serves: ${data.plugins?.join(", ")}`); return; } @@ -99,7 +101,7 @@ class WebSocketClient { })); } - console.log(`[WSClient] 🔄 Dispatching ${finalEntities.length} entities for ${data.pluginId} to DataBus`); + console.debug(`[WSClient] 🔄 Dispatching ${finalEntities.length} entities for ${data.pluginId} to DataBus`); dataBus.emit("dataUpdated", { pluginId: data.pluginId!, @@ -114,7 +116,7 @@ class WebSocketClient { } public subscribe(pluginId: string, engineUrl: string) { - console.log(`[WSClient] 📡 Subscribing to ${pluginId} at ${engineUrl}`); + console.debug(`[WSClient] 📡 Subscribing to ${pluginId} at ${engineUrl}`); const engine = this.getOrCreateEngine(engineUrl); // Cancel any pending cleanup diff --git a/src/core/data/engineManifest.ts b/src/core/data/engineManifest.ts index cfa857e3..da34ca33 100644 --- a/src/core/data/engineManifest.ts +++ b/src/core/data/engineManifest.ts @@ -20,7 +20,8 @@ export async function fetchLocalEngineManifest(): Promise { try { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 2000); + // 500ms is more than enough for a localhost connection. + const timeout = setTimeout(() => controller.abort(), 500); const res = await fetch(`${getLocalEngineBase()}/manifest`, { signal: controller.signal, @@ -38,7 +39,9 @@ export async function fetchLocalEngineManifest(): Promise { return localManifest; } catch { console.log("[EngineManifest] No local engine detected, using cloud."); - manifestFetched = false; + // We intentionally leave manifestFetched = true here. + // This caches the failure so we don't incur this timeout penalty + // every single time a plugin is toggled. return null; } } diff --git a/src/core/hooks/useMarketplaceSync.ts b/src/core/hooks/useMarketplaceSync.ts index a8e44819..261986fa 100644 --- a/src/core/hooks/useMarketplaceSync.ts +++ b/src/core/hooks/useMarketplaceSync.ts @@ -105,7 +105,7 @@ export function useMarketplaceSync(hostReady: boolean) { try { const res = await fetch("/api/marketplace/load"); const json = await res.json(); - console.log(`[MarketplaceSync] Received raw JSON from /api/marketplace/load:`, JSON.stringify(json, null, 2)); + console.debug(`[MarketplaceSync] Received marketplace manifest json`); if (!res.ok) { throw new Error(json.error || `Failed to fetch marketplace configuration (Status ${res.status})`); diff --git a/src/core/plugins/PluginManager.ts b/src/core/plugins/PluginManager.ts index c74029b0..4aacdbb7 100644 --- a/src/core/plugins/PluginManager.ts +++ b/src/core/plugins/PluginManager.ts @@ -148,10 +148,11 @@ class PluginManager { } async enablePlugin(pluginId: string): Promise { - console.log(`[PluginManager] enablePlugin called for ${pluginId}`); + const start = performance.now(); + console.debug(`[PluginManager] enablePlugin called for ${pluginId}`); // Ensure local manifest is fetched so we don't accidentally fall back to cloud if toggled too fast await fetchLocalEngineManifest(); - console.log(`[PluginManager] Manifest fetched for ${pluginId}`); + console.debug(`[PluginManager] Manifest fetched for ${pluginId}. Took ${(performance.now() - start).toFixed(2)}ms`); const managed = this.plugins.get(pluginId); if (!managed) { @@ -176,12 +177,12 @@ class PluginManager { } pollingManager.start(pluginId); - console.log(`[PluginManager] Emitting layerToggled true for ${pluginId}`); + console.debug(`[PluginManager] Emitting layerToggled true for ${pluginId}. Total setup took ${(performance.now() - start).toFixed(2)}ms`); dataBus.emit("layerToggled", { pluginId, enabled: true }); } disablePlugin(pluginId: string): void { - console.log(`[PluginManager] disablePlugin called for ${pluginId}`); + console.debug(`[PluginManager] disablePlugin called for ${pluginId}`); const managed = this.plugins.get(pluginId); if (!managed) { console.error(`[PluginManager] Plugin ${pluginId} not found during disable`); @@ -190,7 +191,7 @@ class PluginManager { managed.enabled = false; managed.entities = []; pollingManager.stop(pluginId); - console.log(`[PluginManager] Emitting layerToggled false for ${pluginId}`); + console.debug(`[PluginManager] Emitting layerToggled false for ${pluginId}`); dataBus.emit("layerToggled", { pluginId, enabled: false }); dataBus.emit("dataUpdated", { pluginId, entities: [] }); }