Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
107625d0f3 | ||
|
|
24e75a8f6c |
No files matched your search
@@ -12,22 +12,6 @@ paths:
|
||||
## Purpose
|
||||
The standard operating procedure for instantiating, modifying, and registering a new data source plugin within the engine.
|
||||
|
||||
## Data Engine Agnosticism (Separation of Concern)
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **WWV is agnostic to which data engine a plugin uses.** Each plugin MUST declare its own `streamUrl` in `getServerConfig()`. The WWV application never controls or overrides plugin stream URLs.
|
||||
|
||||
This is intentional architectural separation: a user can have 30 plugins pointing to 30 different custom backend servers, and WWV opens 30 separate WebSocket connections without caring. The WWV app does not have a "the data engine" — each plugin has its own.
|
||||
|
||||
**Practical rules:**
|
||||
- Plugins hardcode their cloud engine URL (e.g. `wss://dataenginev2.worldwideview.dev/stream`) — this is correct and by design.
|
||||
- Do not add env vars to let WWV override a plugin's `streamUrl` globally — that breaks the separation.
|
||||
- When editing a plugin's fallback fetch URL, use `https://dataenginev2.worldwideview.dev` (note the `v2`).
|
||||
- `NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL` is a local dev only var (`.env.local` only, never production) that routes locally-developed plugin seeders to a local engine instance. It is NOT injected into plugin stream URLs.
|
||||
- `WWV_DATA_ENGINE_URL` is the production server-side var (docker-compose / Coolify) for the WWV app's own REST calls (MCP tools, entity search). Never set `NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL` in production.
|
||||
|
||||
---
|
||||
|
||||
## The WorldPlugin Contract
|
||||
|
||||
All data ingest flows through `WorldPlugin` (defined entirely within `@worldwideview/wwv-plugin-sdk`).
|
||||
|
||||
@@ -96,19 +96,6 @@ PROXY_HOST_ALLOWLIST=*
|
||||
# Comma-separated list of plugin IDs
|
||||
NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS=
|
||||
|
||||
# Local Engine URL (local dev only — set in .env.local, NEVER in production)
|
||||
# Points at your local data engine instance for per-plugin WS stream routing.
|
||||
# Each worktree should use a different port so their engines don't collide.
|
||||
# Local dev (engine on host): http://localhost:5000
|
||||
NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL=
|
||||
|
||||
# Data Engine REST URL (production / Docker Compose — server-side only)
|
||||
# Used by MCP data tools and server-side entity queries to reach the engine REST API.
|
||||
# NOT read by the browser. Set this in docker-compose or Coolify, not in .env.local.
|
||||
# Docker Compose (same network): http://wwv-data-engine:5000
|
||||
# External engine: http://<engine-host-ip>:5002
|
||||
# WWV_DATA_ENGINE_URL=
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# Trigger model: merge to main with a version bump in the relevant package.json is the
|
||||
# intended signal. This workflow checks npm for the exact version before publishing, so
|
||||
# re-running on an already-published version is safe and idempotent.
|
||||
#
|
||||
# Auth: a single npm granular token scoped to @worldwideview/*, stored as the
|
||||
# org-level GitHub secret NPM_TOKEN. One secret covers every @worldwideview package
|
||||
# (these libs + all wwv-plugins), so no per-package setup is needed. `actions/setup-node`
|
||||
# (with registry-url) writes an .npmrc that reads NODE_AUTH_TOKEN. We still request
|
||||
# `id-token: write` and pass `--provenance` so npm records a signed build-provenance
|
||||
# attestation (provenance works with token auth; it does NOT require per-package config).
|
||||
#
|
||||
# Publish tool: we use `npm publish` (not `pnpm publish`). Correct ONLY while these packages
|
||||
# have no `workspace:*` dependencies (npm does not rewrite the workspace: protocol). The 3
|
||||
# packages here have none (sdk -> react peer; libs -> sdk peer via a semver range). If any
|
||||
# package later adds a `workspace:*` dependency, switch its publish step to
|
||||
# `pnpm publish --no-git-checks`.
|
||||
|
||||
name: Publish Packages to npm
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
# Serialize runs so two fast-follow merges can't both pass the version-check and then
|
||||
# race to publish the same version (npm would 403 the loser). Never cancel an in-flight
|
||||
# publish.
|
||||
concurrency:
|
||||
group: publish-packages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
# id-token: write is for build-provenance attestation (not auth); scope to this job.
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
# Auth for every npm publish step in this job (setup-node reads NODE_AUTH_TOKEN).
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: https://registry.npmjs.org
|
||||
cache: pnpm
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build wwv-plugin-sdk
|
||||
run: pnpm --filter @worldwideview/wwv-plugin-sdk run build
|
||||
|
||||
- name: Publish @worldwideview/wwv-plugin-sdk
|
||||
run: |
|
||||
LOCAL_VERSION=$(node -p "require('./packages/wwv-plugin-sdk/package.json').version")
|
||||
# Equality check (not exit-code): only skip when that EXACT version is already on
|
||||
# npm. An unpublished-yet-existing package returns empty here -> we publish. --silent
|
||||
# keeps npm notices off stdout so the comparison stays clean.
|
||||
NPM_VERSION=$(npm view "@worldwideview/wwv-plugin-sdk@${LOCAL_VERSION}" version --silent 2>/dev/null || true)
|
||||
if [ "$NPM_VERSION" = "$LOCAL_VERSION" ]; then
|
||||
echo "@worldwideview/wwv-plugin-sdk@${LOCAL_VERSION} already published — skipping"
|
||||
else
|
||||
echo "Publishing @worldwideview/wwv-plugin-sdk@${LOCAL_VERSION}"
|
||||
npm publish ./packages/wwv-plugin-sdk --provenance --access public
|
||||
fi
|
||||
|
||||
- name: Publish @worldwideview/wwv-lib-incidents
|
||||
run: |
|
||||
LOCAL_VERSION=$(node -p "require('./packages/wwv-lib-incidents/package.json').version")
|
||||
NPM_VERSION=$(npm view "@worldwideview/wwv-lib-incidents@${LOCAL_VERSION}" version --silent 2>/dev/null || true)
|
||||
if [ "$NPM_VERSION" = "$LOCAL_VERSION" ]; then
|
||||
echo "@worldwideview/wwv-lib-incidents@${LOCAL_VERSION} already published — skipping"
|
||||
else
|
||||
echo "Publishing @worldwideview/wwv-lib-incidents@${LOCAL_VERSION}"
|
||||
npm publish ./packages/wwv-lib-incidents --provenance --access public
|
||||
fi
|
||||
|
||||
- name: Publish @worldwideview/wwv-lib-aviation
|
||||
run: |
|
||||
LOCAL_VERSION=$(node -p "require('./packages/wwv-lib-aviation/package.json').version")
|
||||
NPM_VERSION=$(npm view "@worldwideview/wwv-lib-aviation@${LOCAL_VERSION}" version --silent 2>/dev/null || true)
|
||||
if [ "$NPM_VERSION" = "$LOCAL_VERSION" ]; then
|
||||
echo "@worldwideview/wwv-lib-aviation@${LOCAL_VERSION} already published — skipping"
|
||||
else
|
||||
echo "Publishing @worldwideview/wwv-lib-aviation@${LOCAL_VERSION}"
|
||||
npm publish ./packages/wwv-lib-aviation --provenance --access public
|
||||
fi
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Publish Plugin to NPM
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "wwv-plugin-*-v*" # e.g. wwv-plugin-aviation-v1.0.1
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # Required for OIDC trusted publishing (no NPM_TOKEN needed)
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Detect package name from tag
|
||||
id: pkg
|
||||
run: |
|
||||
TAG="${{ github.ref_name }}"
|
||||
# Strips the -vX.Y.Z suffix: "wwv-plugin-aviation-v1.0.1" → "wwv-plugin-aviation"
|
||||
PKG=$(echo "$TAG" | sed 's/-v[0-9].*//')
|
||||
echo "package=$PKG" >> $GITHUB_OUTPUT
|
||||
echo "Publishing: $PKG"
|
||||
|
||||
- name: Build package
|
||||
run: npm run build -w packages/${{ steps.pkg.outputs.package }}
|
||||
|
||||
- name: Publish to NPM (OIDC)
|
||||
run: npm publish packages/${{ steps.pkg.outputs.package }} --access public --provenance
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "worldwideview",
|
||||
"version": "2.48.24",
|
||||
"version": "2.48.25",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "[email protected]",
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
import type { PluginDataSnapshot } from "@/lib/data-query/types";
|
||||
import type { RegionOptions } from "@/lib/data-query/types";
|
||||
import type { FilterDefinition } from "@/core/plugins/PluginTypes";
|
||||
import { getAllPluginSnapshots, getEngineUrl } from "@/lib/data-query/service";
|
||||
import { getAllPluginSnapshots } from "@/lib/data-query/service";
|
||||
import { getLocalSourceIds } from "@/lib/data-query/localSources";
|
||||
import { readActiveSessions, readGlobeState } from "@/lib/globeStateStore";
|
||||
import { readSessionCatalog } from "@/lib/mcpSessionCatalog";
|
||||
@@ -112,7 +112,7 @@ export interface ListStreamingPluginsResult {
|
||||
* are not blocked waiting for a slow engine.
|
||||
*/
|
||||
async function probeEngineReachable(): Promise<boolean> {
|
||||
const engineUrl = getEngineUrl() + "/manifest";
|
||||
const engineUrl = (process.env.WWV_DATA_ENGINE_URL ?? "http://localhost:5001") + "/manifest";
|
||||
try {
|
||||
const res = await fetch(engineUrl, { signal: AbortSignal.timeout(2000) });
|
||||
return res.ok;
|
||||
|
||||
@@ -64,7 +64,7 @@ describe("resolveEngineUrl", () => {
|
||||
await fetchLocalEngineManifest();
|
||||
|
||||
const url = resolveEngineUrl("plugin-local");
|
||||
expect(url).toMatch(/ws:\/\/localhost:\d+\/stream/);
|
||||
expect(url).toContain("localhost:5000/stream");
|
||||
});
|
||||
|
||||
it("should fall back to plugin's custom streamUrl if not local", () => {
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
// src/core/data/__tests__/engineManifest.test.ts
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { fetchLocalEngineManifest, resetManifestCache } from '../engineManifest';
|
||||
|
||||
const mockFetch = vi.fn();
|
||||
|
||||
describe('fetchLocalEngineManifest', () => {
|
||||
beforeEach(() => {
|
||||
resetManifestCache();
|
||||
mockFetch.mockReset();
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
delete process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL;
|
||||
});
|
||||
|
||||
it('probes port 5000 when NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL is not set', async () => {
|
||||
mockFetch.mockResolvedValue({ ok: false } as Response);
|
||||
await fetchLocalEngineManifest();
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
expect.stringContaining(':5000/manifest'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
|
||||
it('probes the env-configured URL when NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL is set', async () => {
|
||||
process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL = 'http://localhost:5003';
|
||||
mockFetch.mockResolvedValue({ ok: false } as Response);
|
||||
await fetchLocalEngineManifest();
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
expect.stringContaining(':5003/manifest'),
|
||||
expect.any(Object)
|
||||
);
|
||||
});
|
||||
|
||||
it('returns plugin list when engine responds successfully', async () => {
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ plugins: ['flight-tracker', 'ais-tracker'] }),
|
||||
} as Response);
|
||||
const result = await fetchLocalEngineManifest();
|
||||
expect(result).toEqual(['flight-tracker', 'ais-tracker']);
|
||||
});
|
||||
});
|
||||
@@ -1,42 +0,0 @@
|
||||
// src/core/data/__tests__/resolveEngineUrl.test.ts
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('../engineManifest', () => ({
|
||||
localEngineHasPlugin: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/core/plugins/PluginManager', () => ({
|
||||
pluginManager: {
|
||||
getPlugin: vi.fn().mockReturnValue(null),
|
||||
getManifest: vi.fn().mockReturnValue(null),
|
||||
},
|
||||
}));
|
||||
|
||||
import { resolveEngineUrl } from '../resolveEngineUrl';
|
||||
import { localEngineHasPlugin } from '../engineManifest';
|
||||
|
||||
describe('resolveEngineUrl', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(localEngineHasPlugin).mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL;
|
||||
});
|
||||
|
||||
it('uses port 5000 by default when no env var is set', () => {
|
||||
vi.mocked(localEngineHasPlugin).mockReturnValue(true);
|
||||
expect(resolveEngineUrl('test-plugin')).toContain(':5000/stream');
|
||||
});
|
||||
|
||||
it('uses the configured URL when NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL is set', () => {
|
||||
process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL = 'http://localhost:5003';
|
||||
vi.mocked(localEngineHasPlugin).mockReturnValue(true);
|
||||
expect(resolveEngineUrl('test-plugin')).toContain(':5003/stream');
|
||||
});
|
||||
|
||||
it('falls back to cloud URL when local engine does not have the plugin', () => {
|
||||
vi.mocked(localEngineHasPlugin).mockReturnValue(false);
|
||||
expect(resolveEngineUrl('test-plugin')).toContain('worldwideview.dev');
|
||||
});
|
||||
});
|
||||
@@ -8,13 +8,16 @@ let manifestFetched = false;
|
||||
/**
|
||||
* Resolve the base URL of the local data engine.
|
||||
*
|
||||
* Reads NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL (default http://localhost:5000).
|
||||
* Only set this in .env.local for local development — never in production.
|
||||
* Each plugin owns its own streamUrl; this variable is only for routing
|
||||
* locally-developed plugin seeders to a local engine instance.
|
||||
* Always checks localhost:5000 — the port docker-compose.yml binds for
|
||||
* wwv-data-engine. NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL is intentionally
|
||||
* NOT used here: that variable belongs to each plugin's own declared engine
|
||||
* URL (production, third-party, etc.) and must not poison local detection.
|
||||
* Mixing the two caused the production engine to be reported as "local".
|
||||
*/
|
||||
function getLocalEngineBase() {
|
||||
return process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL ?? 'http://localhost:5000';
|
||||
const port = process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_PORT || '5000';
|
||||
if (typeof window === "undefined") return `http://localhost:${port}`;
|
||||
return `${window.location.protocol}//${window.location.hostname}:${port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,19 +20,19 @@ function toWsStreamUrl(url: string): string {
|
||||
const DEFAULT_ENGINE_URL = toWsStreamUrl(RAW_ENGINE_URL);
|
||||
|
||||
function getLocalWsUrl() {
|
||||
const base = process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL ?? 'http://localhost:5000';
|
||||
const ws = base.replace(/^https:\/\//, 'wss://').replace(/^http:\/\//, 'ws://');
|
||||
return ws.replace(/\/+$/, '') + '/stream';
|
||||
const port = process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_PORT || '5000';
|
||||
if (typeof window === "undefined") return `ws://localhost:${port}/stream`;
|
||||
return `${window.location.protocol === "https:" ? "wss:" : "ws:"}//${window.location.hostname}:${port}/stream`;
|
||||
}
|
||||
/**
|
||||
* Resolves the WebSocket engine URL for a given plugin.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Local engine (if running at NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL and has this plugin's seeder)
|
||||
* 1. Local engine (if running at localhost:5000 and has this plugin's seeder)
|
||||
* 2. Plugin's ServerPluginConfig.streamUrl (code-based plugins)
|
||||
* 3. Plugin's PluginManifest.dataSource.streamUrl (manifest-based plugins)
|
||||
* 4. NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL env var
|
||||
* 5. Fallback: wss://dataenginev2.worldwideview.dev/stream (cloud)
|
||||
* 5. Fallback: wss://dataengine.worldwideview.dev/stream (cloud)
|
||||
*/
|
||||
export function resolveEngineUrl(pluginId: string): string {
|
||||
// 1. Local engine (split-routing) - PRIORITY #1
|
||||
|
||||
@@ -70,18 +70,18 @@ export async function injectHostGlobals(): Promise<void> {
|
||||
};
|
||||
|
||||
// REST Engine URL (Fallback)
|
||||
// Note: Local engine interception (via NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL) happens dynamically
|
||||
// inside resolveEngineUrl.ts during plugin routing. These variables act as global fallbacks.
|
||||
// Note: Local Docker-based engine interception (localhost:5000) happens dynamically inside
|
||||
// resolveEngineUrl.ts during plugin routing. These variables act as global fallbacks.
|
||||
const envDataEngine = process.env.NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL;
|
||||
if (envDataEngine) {
|
||||
(globalThis as Record<string, unknown>).__WWV_ENGINE_URL__ = envDataEngine;
|
||||
} else {
|
||||
// ALWAYS default to the cloud engine unless explicitly told otherwise via env var
|
||||
(globalThis as Record<string, unknown>).__WWV_ENGINE_URL__ = 'https://dataenginev2.worldwideview.dev';
|
||||
(globalThis as Record<string, unknown>).__WWV_ENGINE_URL__ = 'https://dataengine.worldwideview.dev';
|
||||
}
|
||||
|
||||
// WebSocket Engine URL
|
||||
const fallbackWs = envDataEngine ? `${envDataEngine.replace(/^http/, "ws")}/stream` : 'wss://dataenginev2.worldwideview.dev/stream';
|
||||
const fallbackWs = envDataEngine ? `${envDataEngine.replace(/^http/, "ws")}/stream` : 'wss://dataengine.worldwideview.dev/stream';
|
||||
(globalThis as Record<string, unknown>).__WWV_WS_ENGINE_URL__ = fallbackWs;
|
||||
|
||||
console.log("[HostGlobals] React and SDK injected for dynamic plugins");
|
||||
|
||||
@@ -123,7 +123,7 @@ describe("DeclarativePlugin", () => {
|
||||
getCurrentTime: () => new Date(),
|
||||
env: {},
|
||||
edition: "local",
|
||||
getEngineUrl: () => "http://test-engine/",
|
||||
getEngineUrl: () => "http://localhost:5001",
|
||||
});
|
||||
|
||||
const entities = await plugin.fetch({ start: new Date(), end: new Date() });
|
||||
|
||||
@@ -12,11 +12,7 @@ import type { FilterValue } from "@/core/plugins/PluginTypes";
|
||||
import { hasLocalSource, resolveLocalSnapshot, getLocalSourceIds } from "./localSources";
|
||||
|
||||
export function getEngineUrl(): string {
|
||||
// WWV_DATA_ENGINE_URL = production (docker-compose, server-side only)
|
||||
// NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL = local dev override
|
||||
return process.env.WWV_DATA_ENGINE_URL
|
||||
?? process.env.NEXT_PUBLIC_WWV_LOCAL_ENGINE_URL
|
||||
?? "http://localhost:5000";
|
||||
return process.env.WWV_DATA_ENGINE_URL ?? "http://localhost:5001";
|
||||
}
|
||||
|
||||
function normalizeEntity(raw: unknown): GeoEntity | null {
|
||||
|
||||
Reference in new issue
Block a user