Compare commits
68
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48a17bf3ee | ||
|
|
2e4c074f77 | ||
|
|
e2108f5049 | ||
|
|
b76067c0bb | ||
|
|
04e614e1ed | ||
|
|
0d14a63246 | ||
|
|
781fe9e790 | ||
|
|
f0c2d81e18 | ||
|
|
f86b7d1e94 | ||
|
|
28704e6fe1 | ||
|
|
01e291a40e | ||
|
|
909e150b3d | ||
|
|
1fe3a77383 | ||
|
|
998f98851e | ||
|
|
9c51cdb8a2 | ||
|
|
d74ae694a4 | ||
|
|
ec4c89fe51 | ||
|
|
44e8ff9b70 | ||
|
|
cc90d3c2a3 | ||
|
|
0e6268bf72 | ||
|
|
764af74d14 | ||
|
|
29269c668a | ||
|
|
c60a0c9d56 | ||
|
|
05b2dce394 | ||
|
|
390518be0f | ||
|
|
e07bc60672 | ||
|
|
d75618f9aa | ||
|
|
aa34e1c49d | ||
|
|
2f618ce2cf | ||
|
|
085f4aa90c | ||
|
|
c1ddb91268 | ||
|
|
b3bc62ad8e | ||
|
|
32a4df4952 | ||
|
|
3ac36446bf | ||
|
|
ac95a64115 | ||
|
|
1d07698df7 | ||
|
|
583622f091 | ||
|
|
bfdbd0313d | ||
|
|
a70f93af0a | ||
|
|
26970e6adc | ||
|
|
63bc49949a | ||
|
|
ce45f2d4b6 | ||
|
|
bec1080cb8 | ||
|
|
ed4826484c | ||
|
|
77e6bf1d55 | ||
|
|
fa6413e953 | ||
|
|
53a9931e94 | ||
|
|
055ead345a | ||
|
|
89c247e913 | ||
|
|
df7df20c46 | ||
|
|
b738072e05 | ||
|
|
414ac3b194 | ||
|
|
f3b48ac302 | ||
|
|
bea39455e6 | ||
|
|
1a984251b6 | ||
|
|
67ca2abdbc | ||
|
|
b10e7d0a7f | ||
|
|
6be56b9966 | ||
|
|
4951a2d047 | ||
|
|
3fa1aeb80c | ||
|
|
c6b27a6079 | ||
|
|
c8c619210f | ||
|
|
01f923b806 | ||
|
|
487cb46f03 | ||
|
|
d05f93e610 | ||
|
|
701b7b3b23 | ||
|
|
84129f1d9e | ||
|
|
60dbc5af66 |
No files matched your search
@@ -1,5 +1,9 @@
|
||||
DATABASE_URL=file:./data/wwv.db
|
||||
|
||||
# Default plugins enabled on boot for demo edition
|
||||
# Comma-separated list of plugin IDs
|
||||
NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS=iranwarlive,satellite,gps-jamming
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
|
||||
@@ -8,4 +12,4 @@ NEXT_PUBLIC_WWV_EDITION=local
|
||||
# [WARNING] NEVER commit real secrets here. Real secrets must ONLY go into .env.local!
|
||||
AUTH_SECRET=changeme_d64c1f35b447da89cf6ed5a0988da1379b21904edb91a87c3f5a63942e3d23a4_changeme
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,10 @@ ACLED_PASSWORD=
|
||||
# Generate any random string, then set the same value in the marketplace instance config.
|
||||
WWV_BRIDGE_TOKEN=
|
||||
|
||||
# Default plugins enabled on boot for demo edition
|
||||
# Comma-separated list of plugin IDs
|
||||
NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS=
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
typecheck:
|
||||
name: Type Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: npx prisma generate
|
||||
- run: pnpm exec tsc --noEmit
|
||||
|
||||
test:
|
||||
name: Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: npx prisma generate
|
||||
- run: pnpm test
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [typecheck, test]
|
||||
env:
|
||||
DATABASE_URL: file:./data/wwv.db
|
||||
AUTH_SECRET: ci-placeholder-secret-not-used-in-tests
|
||||
NEXT_PUBLIC_WWV_EDITION: local
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: npx prisma generate
|
||||
- run: pnpm build
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Claude Code Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, ready_for_review, reopened]
|
||||
# Optional: Only run on specific file changes
|
||||
# paths:
|
||||
# - "src/**/*.ts"
|
||||
# - "src/**/*.tsx"
|
||||
# - "src/**/*.js"
|
||||
# - "src/**/*.jsx"
|
||||
|
||||
jobs:
|
||||
claude-review:
|
||||
# Optional: Filter by PR author
|
||||
# if: |
|
||||
# github.event.pull_request.user.login == 'external-contributor' ||
|
||||
# github.event.pull_request.user.login == 'new-developer' ||
|
||||
# github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code Review
|
||||
id: claude-review
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
|
||||
plugins: 'code-review@claude-code-plugins'
|
||||
prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
||||
# prompt: 'Update the pull request description to include a summary of changes.'
|
||||
|
||||
# Optional: Add claude_args to customize behavior and configuration
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
# claude_args: '--allowed-tools Bash(gh pr *)'
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
name: Docker Build and Publish
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main", "feat/docker-ci" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=sha
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
+10
-3
@@ -22,6 +22,7 @@ node_modules/
|
||||
|
||||
# claude code IDE config
|
||||
/.claude/
|
||||
/.cline/
|
||||
|
||||
# performance
|
||||
/performance/
|
||||
@@ -29,9 +30,8 @@ node_modules/
|
||||
# artifacts
|
||||
/artifacts/
|
||||
|
||||
# scripts
|
||||
/scripts/
|
||||
!/scripts/setup.mjs
|
||||
# scripts (tracked, but temporary scratchpads should go in /local-scripts/)
|
||||
|
||||
|
||||
# local database
|
||||
/data/
|
||||
@@ -80,3 +80,10 @@ packages/*/dist/
|
||||
|
||||
# Developmental scripts & temporary sandbox
|
||||
/local-scripts/
|
||||
/public/plugins-local/
|
||||
|
||||
# git worktrees
|
||||
/.worktrees/
|
||||
|
||||
# proprietary extracted services
|
||||
packages/wwv-data-engine/
|
||||
@@ -101,10 +101,13 @@ Visibility Toggle → DataBusSubscriber subscribes to layer via WsClient
|
||||
→ Store → EntityRenderer → Globe
|
||||
```
|
||||
|
||||
Three plugin architectures exist:
|
||||
1. **Static** — GeoJSON file in `public/data/`, loaded by `StaticDataPlugin`
|
||||
2. **Active Proxied** — Next.js API routes in `src/app/api/` as CORS proxy
|
||||
3. **Microservice** — Standalone Fastify container with SQLite (e.g., `iranwarlive-backend`)
|
||||
Four plugin architectures exist (All-Bundle Model):
|
||||
1. **Data Engine Seeder** — Standalone Fastify container with SQLite microservice or unified seeders (e.g., `iranwarlive-backend`).
|
||||
2. **Dynamic CDN Loaded (Bundle)** — Externally developed plugins dynamically imported at runtime via ES module CDNs (e.g., `unpkg.com` version-pinned URLs).
|
||||
3. **Static Compiled (Bundle)** — Static GeoJSON data wrapped into JS bundles via `wwvStaticCompiler` during build/sync (previously `StaticDataPlugin`).
|
||||
4. **Active Proxied (Bundle)** — Next.js API routes bundled to provide frontend interactions (previously `DeclarativePlugin`).
|
||||
|
||||
All plugins are now dynamically imported at runtime as ES module bundles via `loadPluginFromManifest` utilizing `import(/* webpackIgnore: true */ entry)`. The legacy `StaticDataPlugin` and `DeclarativePlugin` runtimes are fully deprecated.
|
||||
|
||||
Plugin types are re-exported from SDK through `src/core/plugins/PluginTypes.ts` and `PluginManifest.ts` — **source of truth is always `@worldwideview/wwv-plugin-sdk`**.
|
||||
|
||||
@@ -128,6 +131,18 @@ Engine push /stream → DataBusSubscriber WsClient router
|
||||
→ StackManager (co-located entity grouping)
|
||||
```
|
||||
|
||||
### 4.3.1 Split-Routing (Cloud Fallback)
|
||||
|
||||
When a local engine is detected at `localhost:5001`, the frontend fetches its `/manifest` to discover available seeders. Per-plugin routing via `resolveEngineUrl`:
|
||||
- Plugin's seeder is in local manifest → `ws://localhost:5001/stream`
|
||||
- Plugin's seeder is NOT in local manifest → `wss://dataengine.worldwideview.dev/stream`
|
||||
|
||||
This enables core contributors running the public (community) engine locally to see local data for their seeders while proprietary sources (aviation, maritime) continue streaming from the cloud.
|
||||
|
||||
The data engine uses a **public upstream / private fork** model:
|
||||
- `wwv-data-engine` (public) — community seeders, contributors fork and PR here
|
||||
- `wwv-data-engine-internal` (private) — all seeders, deploys to Coolify via `git merge upstream/main`
|
||||
|
||||
### 4.4 Rendering Pipeline
|
||||
|
||||
- **Primitive-based**: Uses `PointPrimitiveCollection`, `BillboardCollection`, `LabelCollection` — NOT Cesium Entity API
|
||||
@@ -197,6 +212,7 @@ Built-in plugins are instantiated in `AppShell.tsx` and registered via `PluginRe
|
||||
> - **Never** reference Cursor IDE rules; we use the open `.agents/` standard.
|
||||
> - **MUST**: You MUST update Semantic Versioning numbering inside the relevant `package.json` file prior to executing any code commits, adhering strictly to the `[/commit]` workflow rules (`feat:` -> Minor, `fix/refactor/perf:` -> Patch).
|
||||
> - **MUST Detail Commit Levels & Bumps**: On description changes or release notes, you must detail the level of commit (Major/Minor/Fix) for *each* individual change. If there are multiple accumulated changes, you MUST EITHER commit them individually and bump the version each time, OR commit them all at once and bump the version multiple times.
|
||||
> - **MUST Explain Complex Concepts Simply**: Whenever providing a complicated technical explanation to the user, you MUST include a simple explanation below it. Use an analogy with reference to the correct terminology, comparing the concept to something from everyday life to ensure the user easily understands it.
|
||||
|
||||
### 5.8 Workspace Hygiene
|
||||
Whenever agents generate temporary debugging scripts, test REST endpoints via `.mjs`, or dump traces/JSON outputs, they **MUST** save these exclusively inside `/local-scripts/`. The root directory is strictly for production configuration files.
|
||||
@@ -213,6 +229,7 @@ Whenever agents generate temporary debugging scripts, test REST endpoints via `.
|
||||
| `NEXT_PUBLIC_BING_MAPS_KEY` | No | Bing Maps imagery |
|
||||
| `NEXT_PUBLIC_GOOGLE_MAPS_API_KEY` | No | Google 3D Tiles |
|
||||
| `NEXT_PUBLIC_WWV_EDITION` | No | `local` / `cloud` / `demo` (default: `local`) |
|
||||
| `NEXT_PUBLIC_WWV_PLUGIN_DATA_ENGINE_URL` | No | Override engine WebSocket URL (default: cloud) |
|
||||
| `OPENSKY_CREDENTIALS` | No | Comma-separated `id:secret` pairs for credential rotation |
|
||||
| `WWV_BRIDGE_TOKEN` | No | Shared secret for marketplace → WWV install bridge |
|
||||
| `WWV_DEMO_ADMIN_SECRET` | No | Demo edition admin password |
|
||||
@@ -279,7 +296,10 @@ Configured in `next.config.ts` `headers()`:
|
||||
| Repo | Purpose |
|
||||
|---|---|
|
||||
| `worldwideview` | Main application (this repo) |
|
||||
| `wwv-data-engine` | Open-source data engine — community seeders (PUBLIC) |
|
||||
| `wwv-data-engine-internal` | Proprietary data engine — all seeders (PRIVATE, deployed to Coolify) |
|
||||
| `worldwideview-marketplace` | Plugin marketplace web app |
|
||||
| `worldwideview-plugins` | Published npm plugin packages |
|
||||
| `worldwideview-web` | Marketing / landing page |
|
||||
|
||||
---
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<div align="center">
|
||||
|
||||
<!-- Generated: 2026-04-19 02:20:00 UTC -->
|
||||
# WorldWideView 🌍
|
||||
|
||||
**The Open-Source, Plugin-Driven Geospatial Intelligence Engine**
|
||||
@@ -16,42 +17,53 @@
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Build a Plugin in 5 Minutes
|
||||
|
||||
WorldWideView’s architecture means **everything is a plugin**. The core engine is completely data-agnostic. Rather than touching complex 3D rendering code, you can build a plugin that simply pipes a REST API into GeoJSON, and the engine handles the rest!
|
||||
|
||||
Ready to add a new intelligence layer (like Earthquakes, Weather, or Satellites)? We have a scaffold generator ready for you:
|
||||
|
||||
```bash
|
||||
# 1. Generate the boilerplate plugin from a template
|
||||
pnpm run scaffold-osm-plugin my-new-data-source
|
||||
|
||||
# 2. Add your data fetching logic in the generated files
|
||||
# 3. Reload the dev server and watch it map on the globe!
|
||||
```
|
||||
|
||||
📖 **[Read the Full Plugin Guide](docs/PLUGIN_GUIDE.md)** for a deep dive into hooking up websockets or static datasets.
|
||||
WorldWideView is a real-time geospatial engine visualizing live global data on an interactive 3D globe. Utilizing a dynamic "All-Bundle" plugin architecture, independent data sources—like live aircraft, satellites, or OSINT news clusters—are ingested and rendered decoupled from the core 3D viewer.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Installation & Setup (Local Development)
|
||||
## 🛠 Prerequisites
|
||||
|
||||
```bash
|
||||
git clone https://github.com/silvertakana/worldwideview.git
|
||||
cd worldwideview
|
||||
pnpm install
|
||||
pnpm run setup # generates .env.local with AUTH_SECRET
|
||||
pnpm run dev:all # starts the app, data engine, and marketplace concurrently
|
||||
```
|
||||
Before starting, ensure you have the required tools installed based on how you want to use WorldWideView:
|
||||
- **For Self-Hosting:** You only need Docker.
|
||||
- **For Local Development/Contributing:** You need Node.js and `pnpm`.
|
||||
|
||||
Visit `http://localhost:3000` to see the live globe.
|
||||
### Installing Docker (For Self-Hosting)
|
||||
- **Windows / Mac:** Download and install [Docker Desktop](https://www.docker.com/products/docker-desktop/). *(Windows users: Ensure WSL 2 integration is enabled in Settings > Resources > WSL Integration).*
|
||||
- **Linux (Ubuntu/Debian):**
|
||||
```bash
|
||||
curl -fsSL https://get.docker.com -o get-docker.sh
|
||||
sudo sh get-docker.sh
|
||||
```
|
||||
|
||||
### Installing pnpm (For Developers)
|
||||
If you want to contribute to the code, you'll need the `pnpm` package manager:
|
||||
- **Windows (PowerShell):** `iwr https://get.pnpm.io/install.ps1 -useb | iex`
|
||||
- **Mac / Linux:** `curl -fsSL https://get.pnpm.io/install.sh | sh -`
|
||||
- **Or via npm:** `npm install -g pnpm`
|
||||
|
||||
---
|
||||
|
||||
## 🐋 Production Deployment (Docker & Coolify)
|
||||
## 🐋 Self-Hosting & Production Deployment (Docker)
|
||||
|
||||
WorldWideView uses a multi-stage Dockerfile designed for standalone output, natively supporting deployment via Coolify or standard Docker Compose. The `docker-compose.yml` spins up the front-end application alongside the high-performance Data Engine and Redis cache.
|
||||
WorldWideView uses a multi-stage Dockerfile designed for standalone output, natively supporting deployment via Coolify or standard Docker Compose.
|
||||
|
||||
### Instant Self-Hosting (Recommended)
|
||||
Get the platform running on your own server instantly using our automated Docker Compose deployment scripts:
|
||||
|
||||
**Mac/Linux:**
|
||||
```bash
|
||||
mkdir worldwideview && cd worldwideview
|
||||
curl -fsSL https://raw.githubusercontent.com/silvertakana/worldwideview/main/setup.sh | bash
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
```powershell
|
||||
mkdir worldwideview; cd worldwideview
|
||||
Invoke-WebRequest -Uri https://raw.githubusercontent.com/silvertakana/worldwideview/main/setup.ps1 -UseBasicParsing | Invoke-Expression
|
||||
```
|
||||
|
||||
### Manual Docker Deployment
|
||||
If you prefer to deploy manually:
|
||||
```bash
|
||||
# Start the core app, Redis cache, and Data Engine microservice in production mode
|
||||
docker compose up -d
|
||||
@@ -65,14 +77,123 @@ When deploying to production, ensure you mount the following volumes to prevent
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Installation & Setup (Local Development)
|
||||
|
||||
Get the source code running locally for contributing or modifying the core platform.
|
||||
|
||||
### Quick Start Scripts
|
||||
**Mac/Linux:**
|
||||
```bash
|
||||
git clone https://github.com/silvertakana/worldwideview.git
|
||||
cd worldwideview
|
||||
bash local-dev.sh
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
```powershell
|
||||
git clone https://github.com/silvertakana/worldwideview.git
|
||||
cd worldwideview
|
||||
.\local-dev.ps1
|
||||
```
|
||||
|
||||
### Manual Setup
|
||||
```bash
|
||||
git clone https://github.com/silvertakana/worldwideview.git
|
||||
cd worldwideview
|
||||
pnpm install
|
||||
pnpm run setup # generates .env.local with AUTH_SECRET
|
||||
pnpm run dev:all # boots the UI, cache layers, and the marketplace engine
|
||||
```
|
||||
Visit `http://localhost:3000` to see the live globe.
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Plugin Development Quick Start
|
||||
|
||||
WorldWideView’s architecture means **everything is a plugin**. The core engine is completely data-agnostic. You can build your own WorldWideView plugins using the officially supported CLI toolchain.
|
||||
|
||||
### 1. Scaffold your plugin
|
||||
The plugin folder is pure code. It contains no Docker boilerplate.
|
||||
```bash
|
||||
npx @worldwideview/create-plugin my-layer
|
||||
cd my-layer
|
||||
npm install
|
||||
```
|
||||
|
||||
### 2. Sideload the plugin into your platform
|
||||
The CLI acts as a bridge, injecting your compiled plugin directly into your running WorldWideView instance.
|
||||
|
||||
**If you are running the platform via Docker:**
|
||||
Ensure your Docker Compose file has a volume mount exposing the plugin directory:
|
||||
`volumes: [ "./my-local-plugins:/app/public/plugins-local" ]`
|
||||
And set the environment variable: `WWV_PLUGIN_DEV=true`
|
||||
|
||||
Then, tell the CLI where that exposed folder is:
|
||||
```bash
|
||||
# Save the path so you only have to type it once
|
||||
npx wwv config set wwv-path /absolute/path/to/my-local-plugins
|
||||
|
||||
# Start the dev server and link the plugin!
|
||||
npm run link
|
||||
```
|
||||
|
||||
**If you cloned the platform repository (`pnpm dev`):**
|
||||
```bash
|
||||
npx wwv config set wwv-path ../worldwideview
|
||||
npm run link
|
||||
```
|
||||
|
||||
The CLI will start Vite in watch mode. Every time you save a file, it compiles in milliseconds and pushes the update directly into your WorldWideView instance. Refresh your browser to see changes!
|
||||
|
||||
### 3. Validate & Publish
|
||||
If you are building a simple, static data layer and don't need local testing:
|
||||
```bash
|
||||
npm run validate
|
||||
npm run build
|
||||
npm publish
|
||||
```
|
||||
|
||||
📖 **[Read the Full Plugin Guide](docs/PLUGIN_GUIDE.md)** for a deep dive into hooking up websockets or static datasets.
|
||||
|
||||
---
|
||||
|
||||
## 🧠 Core Architecture & Philosophies
|
||||
|
||||
- **Modular Intelligence**: Independent plugins supply the data. The core engine remains lightweight and isolated.
|
||||
- **Open-Core Data Engine**: The real-time backend data pipeline operates on a public/private split-routing model. Community seeders (like ISS tracking) live in the open-source `wwv-data-engine`, while proprietary sources seamlessly interleave via our internal cloud endpoints.
|
||||
- **Dynamic, Decentralized Plugins**: The plugin ecosystem uses a decentralized architecture fetching ES module CDN bundles (e.g., from unpkg) at runtime. The core engine is a lightweight shell that loads data providers dynamically via the marketplace.
|
||||
- **High-Performance Rendering**: Engineered for scale using raw Cesium primitives to smoothly handle upwards of 100,000+ objects simultaneously without GPU stalls.
|
||||
- **Cinematic Situational Awareness**: Implements horizon culling, smooth camera tracking, orientation-locked billboarding, and stack spiderification for dense locations.
|
||||
|
||||
---
|
||||
|
||||
## 🏛 Repository Ecosystem
|
||||
|
||||
WorldWideView follows an **Open-Core** philosophy distributed across several independent, specialized repositories:
|
||||
|
||||
1. **[worldwideview](https://github.com/silvertakana/worldwideview)** *(This Repo)*
|
||||
The main Next.js frontend, CesiumJS rendering engine, and core plugin loading framework.
|
||||
2. **[wwv-data-engine](https://github.com/silvertakana/wwv-data-engine)**
|
||||
The open-source community data backend. It polls public APIs (like ISS tracking or USGS earthquakes) and streams them to the frontend via WebSocket.
|
||||
*(Note: The official production deployment uses a proprietary internal fork of this engine that includes additional restricted seeders like aviation and maritime).*
|
||||
3. **[worldwideview-plugins](https://github.com/silvertakana/worldwideview-plugins)**
|
||||
Our first-party maintained plugins (borders, nuclear sites, military bases, etc.) developed directly by the core team.
|
||||
4. **[worldwideview-marketplace](https://github.com/silvertakana/worldwideview-marketplace)**
|
||||
The web application driving the plugin directory and community submissions.
|
||||
5. **[worldwideview-web](https://github.com/silvertakana/worldwideview-web)**
|
||||
The marketing, documentation, and landing site.
|
||||
|
||||
---
|
||||
|
||||
## 🏗 Development & Code Map
|
||||
|
||||
### Key Entry Points
|
||||
- UI Loader: `src/components/layout/AppShell.tsx`
|
||||
- Dynamic Plugin Ingestion: `src/core/plugins/loadPluginFromManifest.ts`
|
||||
- Global WebSocket Event Loop: `src/core/data/DataBus.ts`
|
||||
- Environment Config: `.env.local` alongside `src/core/edition.ts`
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentation Index
|
||||
|
||||
- **[Setup & Installation Guide](docs/SETUP.md)**: Detailed environment and local development setup.
|
||||
@@ -80,6 +201,13 @@ When deploying to production, ensure you mount the following volumes to prevent
|
||||
- **[Contributions Guide](CONTRIBUTING.md)**: How to help out, coding standards, and commit conventions.
|
||||
- **[User Guide](docs/USER_GUIDE.md)**: Application features and navigation tips.
|
||||
|
||||
### Internal Engineering Docs
|
||||
- **[Build System](docs/build-system.md)**: Find compile targets for UI bundles and internal `wwvStaticCompiler` plugin bundling.
|
||||
- **[Testing Overview](docs/testing.md)**: Understanding Vitest strategies and isolated typing execution targets.
|
||||
- **[Development Patterns](docs/development.md)**: View standard hook approaches, Zustand architecture rules, and snippet targets.
|
||||
- **[Deployment Workflows](docs/deployment.md)**: Detailed insights on Next.js standalone execution and persistent database volume arrays.
|
||||
- **[Source File Catalog](docs/files.md)**: High-level mappings of core ingestion controllers to rendering context interfaces.
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing & Community
|
||||
@@ -89,5 +217,3 @@ We are actively looking for contributors to help build new data plugins and opti
|
||||
- Want to build something? Check out our **[Pinned Plugin Wishlist](https://github.com/silvertakana/worldwideview/issues)** for great first issues!
|
||||
|
||||
> **Fair-Use Notice:** This application may contain copyrighted material the use of which has not always been specifically authorized by the copyright owner. Such material is made available for educational purposes, situational awareness, and to advance understanding of global events under "fair use" (Section 107 of the US Copyright Law).
|
||||
|
||||
✅ **vibe-kanban is working**
|
||||
+17
-5
@@ -54,7 +54,19 @@ Each stage is **self-contained and demo-able**. Any stage can be shipped indepen
|
||||
|
||||
---
|
||||
|
||||
## Stage 5: Publisher Workflow
|
||||
## Stage 5: AI & Sharing Features (Immediate User Value)
|
||||
**Demo:** "Chat with the AI to explore the globe, record a narrated briefing, and share the view via a URL."
|
||||
|
||||
- [ ] Unified Data Format schema (WWVDocument) with snapshot radius bounding
|
||||
- [ ] Base64url view link sharing (`#v=...`)
|
||||
- [ ] Shared `ActionExecutor` core for Animation & Copilot
|
||||
- [ ] Animation Engine (TrackScheduler, Player, overlays, TTS voice)
|
||||
- [ ] MCP Server (tools wrapping the action vocabulary)
|
||||
- [ ] Hybrid Spatial Awareness (Pre-computed summaries + MCP spatial query tools)
|
||||
|
||||
---
|
||||
|
||||
## Stage 6: Publisher Workflow
|
||||
**Demo:** "A plugin author submits their plugin. After review, it appears in the catalog."
|
||||
|
||||
- [ ] Marketplace user accounts (Supabase Auth — email/password + GitHub OAuth)
|
||||
@@ -66,7 +78,7 @@ Each stage is **self-contained and demo-able**. Any stage can be shipped indepen
|
||||
|
||||
---
|
||||
|
||||
## Stage 6: Cloud Platform & Accounts
|
||||
## Stage 7: Cloud Platform & Accounts
|
||||
**Demo:** "User registers, gets [user].app.worldwideview.dev, installs plugins from marketplace."
|
||||
|
||||
- [ ] `app.worldwideview.dev` — auth/register/dashboard
|
||||
@@ -81,7 +93,7 @@ Each stage is **self-contained and demo-able**. Any stage can be shipped indepen
|
||||
|
||||
---
|
||||
|
||||
## Stage 7: Built-In Data Services
|
||||
## Stage 8: Built-In Data Services
|
||||
**Demo:** "Aviation and maritime data with full history, served from api.worldwideview.dev."
|
||||
|
||||
- [ ] `api.worldwideview.dev` — aviation/maritime polling + persistence
|
||||
@@ -93,7 +105,7 @@ Each stage is **self-contained and demo-able**. Any stage can be shipped indepen
|
||||
|
||||
---
|
||||
|
||||
## Stage 8: Trust Tiers, Extensions & Polish
|
||||
## Stage 9: Trust Tiers, Extensions & Polish
|
||||
**Demo:** "Install a verified extension plugin that adds cockpit view to aviation."
|
||||
|
||||
- [x] Ed25519-signed plugin registry (`GET /api/registry` on marketplace)
|
||||
@@ -111,7 +123,7 @@ Each stage is **self-contained and demo-able**. Any stage can be shipped indepen
|
||||
|
||||
---
|
||||
|
||||
## Stage 9: Infrastructure & Monetization
|
||||
## Stage 10: Infrastructure & Monetization
|
||||
**Demo:** "Buy Pro, paste license key, unlock snapshot history and higher rate limits."
|
||||
|
||||
- [ ] Coolify on ThinkPad T480s — deploy all services
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"message":"fix(aviation): implement exponential backoff for rate limits","tree":"55c74a09dac7ce178ecf94d5cb48b2502e57e759","parents":["e25ddc0b648b757ea32b7376bab50d8e75d6d330"]}
|
||||
@@ -0,0 +1,54 @@
|
||||
const fs = require("fs");
|
||||
const { execSync } = require("child_process");
|
||||
|
||||
const uuidsToDelete = [
|
||||
"i331gbc1zzytx7via8lntdm6",
|
||||
"pow71xsobdas7etf5r3hy5ug",
|
||||
"l79dyd03lhtcpgi7swkoxm55",
|
||||
"ry6160tpy4gh2xjfwr011no0",
|
||||
"gfe78t8jm53lltentz19ze60",
|
||||
"e1cg7q8khyvpti5rkp351559",
|
||||
"c35cvvqxhdbslk8yc24g04wk",
|
||||
"ijj9pqkavlusy2wmzsbjtc74",
|
||||
"ghe6bzlr6ex6j51lgxzyhd46",
|
||||
"h3v45w2u1essulzeg56i27a3",
|
||||
"oxk05amog4fla0lskmkfx64x",
|
||||
"yxm4mn60lfp9qah0q4yf2hij",
|
||||
"ezjxo4ehs96lv0ji1n2r0auw",
|
||||
"t12x7pgptsqiu5gbvv35gun5",
|
||||
"urq1ll00a3vv9on6pulmpofe",
|
||||
"eb5zgw5wlgvio2gggbiphpam",
|
||||
"ld1h3pody4nxj3b44pbskl53",
|
||||
"x111xfygqxrlbf5wqlx59f6d",
|
||||
"se1weqr4dj7thp4p1l055wfc",
|
||||
"cqses6urxff0n94i0u6lktzt",
|
||||
"nvcir33nyuvzk5niy3s7eg9w",
|
||||
"s63onga2s6v8r3k10lrdwzth",
|
||||
"cf3uedqnjmwv48h7tm91d95r",
|
||||
"o9124dnnieau9wxoefyxvfgu",
|
||||
"kxzobc8qlvigbsk1d35dpiyi",
|
||||
"eh8lzjbsy2c9kwj2emztcmf6",
|
||||
"umfsos2vcmotfk2cfpqdoycc",
|
||||
"yn1mdh0fngiati6r4625qvv9"
|
||||
];
|
||||
|
||||
const token = "1|vPZPgvB8vVi8ZD5ZKS8lrCrwNlPofVv4I8Tjmqmyd01f92d2";
|
||||
const baseUrl = "https://coolify.worldwideview.dev/api/v1";
|
||||
const appUuid = "nmn55t4io3myfubs72worn7k";
|
||||
|
||||
async function run() {
|
||||
for (const id of uuidsToDelete) {
|
||||
console.log("Deleting", id);
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/applications/${appUuid}/envs/${id}`, {
|
||||
method: "DELETE",
|
||||
headers: { "Authorization": `Bearer ${token}` }
|
||||
});
|
||||
console.log(id, res.status);
|
||||
} catch(e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
@@ -13,46 +13,8 @@ services:
|
||||
- "WWV_DEMO_ADMIN_SECRET=${WWV_DEMO_ADMIN_SECRET:-}"
|
||||
- "WWV_BRIDGE_TOKEN=${WWV_BRIDGE_TOKEN:-}"
|
||||
|
||||
wwv-data-engine:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./packages/wwv-data-engine/Dockerfile
|
||||
ports:
|
||||
- "5001:5001"
|
||||
volumes:
|
||||
- engine-data:/app/packages/wwv-data-engine/data
|
||||
restart: always
|
||||
depends_on:
|
||||
- redis
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
- "NODE_ENV=production"
|
||||
- "REDIS_URL=${REDIS_URL:-redis://redis:6379}"
|
||||
networks:
|
||||
- coolify
|
||||
- default
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.data-engine.rule=Host(`dataengine.worldwideview.dev`)"
|
||||
- "traefik.http.routers.data-engine.entrypoints=websecure"
|
||||
- "traefik.http.routers.data-engine.tls=true"
|
||||
- "traefik.http.routers.data-engine.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.services.data-engine.loadbalancer.server.port=5001"
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: always
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
volumes:
|
||||
wwv-data:
|
||||
engine-data:
|
||||
redis-data:
|
||||
|
||||
networks:
|
||||
coolify:
|
||||
|
||||
+19
-56
@@ -1,62 +1,25 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# System Architecture
|
||||
|
||||
WorldWideView is a high-performance, event-driven geospatial intelligence engine. It is designed to ingest live, fast-moving signals—such as aviation ADS-B, maritime AIS, or satellite feeds—and transform them into cinematic, real-time visual layers on a 3D globe.
|
||||
WorldWideView operates on a decoupled Core-and-Plugin architecture. The central platform provides the 3D globe, HUD, timeline, and rendering optimizations, while separate intelligence domains are entirely handled by external plugin packages that are bundled and ingested at runtime.
|
||||
|
||||
## Module Breakdown
|
||||
The underlying data backbone uses a WebSocket Firehose, streaming continuous high-frequency metrics from standalone data engine microservices perfectly isolated from the visualization layer.
|
||||
|
||||
1. **`src/core/plugins`**: The registration and lifecycle management layer. Defines how plugins are booted and destroyed.
|
||||
2. **`src/core/data`**: The "Heartbeat" of the system.
|
||||
- **DataBus**: Decentralized event pipeline for all system actions.
|
||||
- **PollingManager**: Intelligent scheduler for external API calls with backoff logic.
|
||||
- **CacheLayer**: 2-stage persistent caching (In-Memory + IndexedDB).
|
||||
3. **`src/core/globe`**: The Rendering Engine.
|
||||
- **Cesium Integration**: Low-level control over the Cesium Viewer.
|
||||
- **EntityRenderer**: High-performance "Primitive" renderer.
|
||||
4. **`src/plugins`**: Domain-specific logic (Aviation, Maritime, etc.).
|
||||
## Component Map
|
||||
- **UI Platform:** `src/components/layout/AppShell.tsx`
|
||||
- **3D Visualizer:** `src/core/globe/GlobeView.tsx`
|
||||
- **State Store:** `src/core/state/globeSlice.ts` and peers.
|
||||
- **Event Bus:** `src/core/data/DataBus.ts`
|
||||
- **Microservice Connectors:** `packages/wwv-plugin-sdk/src/PluginManifest.ts`
|
||||
|
||||
## Performance: Primitives vs. Entities
|
||||
## Key Files
|
||||
- `src/core/globe/GlobeView.tsx` (lines 60-150): The central CesiumJS ingestion point and scene orchestrator.
|
||||
- `src/core/globe/EntityRenderer.tsx` (lines 45-120): High-performance batched Primitive mapping for geometric entities.
|
||||
- `src/core/plugins/PluginManager.ts` (lines 20-100): Validates and bootstraps compiled ES Modules from CDN/local sources.
|
||||
|
||||
One of our core design decisions is using **Cesium Primitives** instead of the standard high-level **Entity API** for high-count datasets.
|
||||
|
||||
| Feature | Entity API | Primitive API (WWV) |
|
||||
|---|---|---|
|
||||
| **Abstraction** | High (Easy to use) | Low (Direct GPU access) |
|
||||
| **Overhead** | Significant per-entity JS objects | Minimal (Batched rendering) |
|
||||
| **Max Capacity** | ~1,000 entities | **100,000+ points/billboards** |
|
||||
| **WWV Use Case** | Info window content | Live data point visualization |
|
||||
|
||||
**Why?** The Entity API in Cesium is great for rich features but triggers significant CPU overhead when managing thousands of moving objects. By using `PointPrimitiveCollection` and `BillboardCollection`, WorldWideView batches these draw calls, ensuring 60FPS even with dense global data.
|
||||
|
||||
## Data Pipeline (Example: Aviation)
|
||||
|
||||
The following diagram trace the journey of a single data point from a remote sensor to the user's screen:
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant S as Remote API (e.g. OpenSky)
|
||||
participant P as Plugin (AviationPlugin)
|
||||
participant D as DataBus
|
||||
participant R as EntityRenderer
|
||||
participant G as GPU (Cesium)
|
||||
|
||||
Note over S, P: Polling Interval (e.g. 30s)
|
||||
S->>P: raw JSON data
|
||||
rect rgb(200, 230, 255)
|
||||
Note right of P: Mapping to GeoEntity
|
||||
P->>P: parse, filter, sanitize
|
||||
end
|
||||
P->>D: emit("dataUpdated", { entities })
|
||||
D->>R: trigger render cycle
|
||||
rect rgb(255, 230, 200)
|
||||
Note right of R: Batch Update Primitives
|
||||
R->>G: update collection buffer
|
||||
end
|
||||
G-->>G: 60FPS visualization
|
||||
```
|
||||
|
||||
## Design Principles
|
||||
|
||||
- **Single Responsibility (SRP)**: Plugins only handle data mapping; they don't know about the UI or the Cache.
|
||||
- **Dependency Inversion**: The `PluginManager` communicates with plugins through the `WorldPlugin` interface, allowing new plugins to be added without modifying core code.
|
||||
- **Event-Driven Architecture**: The system is reactive. Components subscribe to what they need, reducing prop-drilling and unnecessary re-renders.
|
||||
- **Defensive Programming**: All external API calls in plugins are wrapped in error boundaries and handled by the `PollingManager`'s backoff logic to prevent system-wide failures.
|
||||
## Data Flow
|
||||
The real-time data flow triggers globally via WebSockets:
|
||||
1. `src/lib/WsClient.ts` captures the socket frame from `dataengine.worldwideview.dev`.
|
||||
2. Emits payload through `src/core/data/DataBus.ts` (lines 30-50).
|
||||
3. The `DataBusSubscriber.tsx` hook catches the event, sanitizing and merging the data into `src/core/state/dataSlice.ts`.
|
||||
4. `EntityRenderer` automatically reacts to State changes, triggering low-level Cesium Primitive re-draws against the GPU.
|
||||
@@ -0,0 +1,20 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Build System
|
||||
|
||||
WorldWideView operates using a strict `pnpm` monorepo workspace to manage the core Next.js application alongside its surrounding plugin ecosystem.
|
||||
|
||||
The build relies heavily on Next.js standalone output tracing optimization, allowing minimal-footprint Docker images via Multi-stage resolution.
|
||||
|
||||
## Key Files
|
||||
- `pnpm-workspace.yaml`: Governs the inclusion of `packages/*` elements.
|
||||
- `next.config.ts` (lines 10-45): Implements `output: "standalone"` and strict Content-Security-Policy headers.
|
||||
- `Dockerfile` (lines 1-110): Executes the multi-stage Extractor Pattern (`deps` → `builder` → `runner`).
|
||||
|
||||
## Build Workflows
|
||||
- **Dependency Install:** `pnpm install` must be run from the root directory to respect strict workspace hoisting natively.
|
||||
- **Development Proxy:** `pnpm dev` triggers the frontend server locally with active HMR, while executing pre-build scripts like `copy-cesium.mjs`.
|
||||
- **Plugin Compilation:** Run `pnpm -r build` inside `packages/worldwideview-plugins/` to trigger Vite bundle execution mapped to `dist/frontend.mjs`.
|
||||
|
||||
## Platform Setup
|
||||
- **Cesium Resources:** Required assets (Workers, Assets, Widgets) are structurally bound into the `public/cesium/` folder during the pre-build pipeline initialized by `scripts/copy-cesium.mjs` (lines 10-38).
|
||||
- **SQLite Engine:** `prisma.config.ts` must export native javascript objects (free of dev CLI wrappers) for Next.js to trace correctly in the final standalone Node build.
|
||||
+27
-45
@@ -21,63 +21,45 @@ There are **three plugin formats**:
|
||||
## Prerequisites
|
||||
|
||||
- Node.js ≥ 18
|
||||
- pnpm (`npm install -g pnpm`)
|
||||
- A npm account (for publishing to the npm registry)
|
||||
- The WorldWideView monorepo cloned locally
|
||||
- An NPM account (for publishing)
|
||||
- A running WorldWideView instance (local clone or Docker container)
|
||||
|
||||
---
|
||||
|
||||
## Step 1 — Create Your Plugin Package
|
||||
## Step 1 — Scaffold Your Plugin
|
||||
|
||||
All plugins live under `packages/` in the monorepo.
|
||||
WorldWideView plugins are strictly decoupled from the core application. You do **not** build them inside the WorldWideView monorepo. Instead, use the official CLI tool:
|
||||
|
||||
```bash
|
||||
mkdir packages/wwv-plugin-my-data
|
||||
cd packages/wwv-plugin-my-data
|
||||
npx @worldwideview/create-plugin@latest my-data-layer
|
||||
cd my-data-layer
|
||||
npm install
|
||||
```
|
||||
|
||||
Create `package.json`:
|
||||
This will generate a ready-to-use template including your `package.json`, `plugin.json`, and Vite build configuration.
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@worldwideview/wwv-plugin-my-data",
|
||||
"version": "1.0.0",
|
||||
"description": "WorldWideView plugin — my custom data layer",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": ["src"],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"worldwideview-plugin",
|
||||
"geojson",
|
||||
"cesium",
|
||||
"globe"
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/silvertakana/worldwideview.git",
|
||||
"directory": "packages/wwv-plugin-my-data"
|
||||
},
|
||||
"license": "ISC",
|
||||
"author": "WorldWideView",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "*"
|
||||
},
|
||||
"worldwideview": {
|
||||
"id": "my-data",
|
||||
"icon": "MapPin",
|
||||
"category": "Custom",
|
||||
"format": "bundle"
|
||||
}
|
||||
}
|
||||
```
|
||||
### Linking During Development
|
||||
|
||||
To see your plugin live on the globe while you develop:
|
||||
|
||||
1. Configure your CLI to point to your running WorldWideView instance:
|
||||
```bash
|
||||
npx wwv config set wwv-path C:\path\to\your\worldwideview
|
||||
```
|
||||
2. Link the plugin:
|
||||
```bash
|
||||
npm run link
|
||||
```
|
||||
3. Run the dev watcher to rebuild on save:
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
*Note: If you are running WorldWideView via Docker, ensure `WWV_PLUGIN_DEV=true` is set in your environment variables and mount your plugin's `dist/` folder into the container's `/app/plugins-local/` directory.*
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The `"worldwideview"` object block is strictly required if you intend to publish your plugin to the WorldWideView Marketplace. The marketplace uses this internal manifest to determine how to categorize and display your plugin automatically.
|
||||
> The `worldwideview` object block inside your `package.json` is strictly required for the CLI link command and if you intend to publish your plugin to the WorldWideView Marketplace. The marketplace uses this internal manifest to determine how to categorize and display your plugin automatically.
|
||||
> * `id`: The unique string identifier.
|
||||
> * `icon`: The Lucide React icon name (e.g. `MapPin`, `Satellite`, `Plane`).
|
||||
> * `category`: Must match an existing category (e.g. `Aviation`, `Maritime`, `Natural Disaster`, `Custom`).
|
||||
> * `format`: Either `bundle`, `static`, or `declarative`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The SDK is declared as a **peer dependency**, not a direct dependency. The host application provides it at runtime.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Deployment
|
||||
|
||||
WorldWideView is container-first. The codebase relies entirely on Dockerizing the Next.js `output: "standalone"` artifact and its localized Prisma SQLite core to enable horizontal orchestration.
|
||||
|
||||
Deployments are coordinated directly via Coolify natively compiling the Docker configurations.
|
||||
|
||||
## Package Types
|
||||
- **Next.js Core Bundle:** Built directly via `pnpm build`, extracting into `<root>/.next/standalone`.
|
||||
- **Plugin Microservices:** Hosted alongside the Next.js router natively using Fastify images (defined in `docker-compose.yml`), enabling real-time Python/Node scraping backends exclusively connected via docker network aliases.
|
||||
- **CDN ESM Bundles:** Independent frontend plugins run via Vite rollup configurations (`vite.config.ts`), generating `dist/frontend.mjs` payloads published to the public NPM registry.
|
||||
|
||||
## Platform Deployment
|
||||
- **Dockerfile:** `Dockerfile` explicitly targets `node:22-alpine` instances to restrict vulnerability surface areas while mapping the necessary system volume at `/app/data` to capture the `wwv.db` SQLite instantiation safely out-of-container (lines 80-112).
|
||||
- **Environment Hydration:** Coolify directly passes the active environment headers `.env` values at runtime, utilizing `.env.local` natively for credentials like `AUTH_SECRET` and `DATABASE_URL`.
|
||||
|
||||
## Reference
|
||||
The core production mapping operates cleanly off HTTP port `3000`. Refer to `next.config.ts` (lines 5-15) for strict caching strategies utilized globally across deployments.
|
||||
@@ -0,0 +1,26 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Development
|
||||
|
||||
Development on WorldWideView leverages modern, rigid structural constraints to ensure cross-plugin consistency, extreme browser optimization, and a minimal global footprint.
|
||||
|
||||
New features are designed within isolated workspaces, using SDK structures rather than forcing logic into the core engine.
|
||||
|
||||
## Code Style
|
||||
- **TypeScript Strictness:** Strict types are enforced across all domains (`tsconfig.json` lines 4-20). Any use of `any` inside platform routers is forbidden.
|
||||
- **Styling constraints:** All CSS is strictly limited to Vanilla CSS via CSS Modules (`.module.css`). Tailwind is prohibited to prevent styling collisions natively with dynamically loaded elements from third-party plugins. (`src/app/globals.css`).
|
||||
|
||||
## Common Patterns
|
||||
- **Env Variable Injection:** Plugins read remote contexts generically instantiated via the SDK.
|
||||
```typescript
|
||||
// From wwv-plugin-aviation/src/index.ts:60-70
|
||||
let engineBase = this.context?.env?.DATA_ENGINE_URL || "https://dataengine.worldwideview.dev";
|
||||
```
|
||||
- **Zustand Slices:** The store is heavily splintered based on structural domain, keeping the hook listeners granular so mapping updates don't aggressively trigger `AppShell` root rerenders. (`src/core/state/globeSlice.ts`).
|
||||
|
||||
## Workflows
|
||||
- **Local Database Setup:** Run `pnpm db:reset` to nuke and redeploy the standard Prisma schema across `data/wwv.db`.
|
||||
- **Generating Plugins:** Use the global CLI `npx @worldwideview/create-plugin@latest <name>` to initialize a decoupled plugin outside of the monorepo.
|
||||
- **Linking Plugins:** Use `npm run link` in the plugin directory after setting `npx wwv config set wwv-path <path>`.
|
||||
|
||||
## Reference
|
||||
File size is constrained to a 150-line soft cap. Hooks and computational math helpers must be rigorously offloaded (e.g. `src/core/globe/hooks/useModelRendering.ts`).
|
||||
@@ -0,0 +1,25 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Files Catalog
|
||||
|
||||
WorldWideView maps rigid structural directories to manage its layered microservice architecture, distinguishing tightly between UI layout routing, system memory, 3D visualization, and microservice ingestion mechanisms.
|
||||
|
||||
Every major component is siloed deeply to ensure the monolithic main engine remains perfectly isolated and agnostic to individual data sources.
|
||||
|
||||
## Core Source Files
|
||||
- `src/app/` - The Next.js 16 core routing platform, housing layout constraints, `api/` backends, and full-screen auth portals.
|
||||
- `src/components/` - Segmented React JSX components handling overlay panels, timeline UI, and floating popups.
|
||||
- `src/core/plugins/` - Essential bridge scripts that validate manifested metadata parameters before dynamically allocating JS module environments to run external logic. (`PluginManager.ts`)
|
||||
|
||||
## Platform Implementation
|
||||
- `src/core/globe/GlobeView.tsx` - Initializer mapping global config hooks to the main Cesium `Viewer`.
|
||||
- `src/core/globe/EntityRenderer.tsx` - Extremely hot processing loop extracting Zustand state variables directly to WebGL GPU hardware calls via Cesium Primitives.
|
||||
- `src/core/globe/StackManager.ts` - Spiderifier spatial engine dynamically resolving overlapping coordinate markers geometrically in 3D-space.
|
||||
|
||||
## Build System
|
||||
- `Dockerfile` - Multi-stage container instructions targeting a 110MB production Alpine build footprint.
|
||||
- `next.config.ts` - NextJS 16 overrides forcing standalone rendering extraction and tight CSP headers.
|
||||
- `prisma/schema.prisma` - DB tables strictly configuring Prisma to bind exclusively towards locally-mounted SQLite structures.
|
||||
|
||||
## Configuration
|
||||
- `.agents/` - Complete repository workflow configuration (Antigravity Code). Contains AI `rules/`, `skills/`, and internal architectural `context/` for automatic coordination.
|
||||
- `public/cesium/` - Runtime engine bundles duplicated exclusively during pre-build tracking (copied from `node_modules`).
|
||||
@@ -0,0 +1,21 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Project Overview
|
||||
|
||||
WorldWideView is a real-time geospatial intelligence engine that visualizes live global data on an interactive 3D globe. It leverages a modern frontend stack to render high-density datasets (aircraft, maritime vessels, conflict events, environmental data) synchronously.
|
||||
|
||||
The core value proposition is mapping diverse, decoupled intelligence streams onto a central geographic interface using a dynamically loaded plugin architecture. Data sources operate completely decoupled from the display platform, streaming observations via WebSockets for sub-second visual updates.
|
||||
|
||||
## Key Files
|
||||
- `package.json` (lines 1-80) - Core dependencies and project metadata.
|
||||
- `src/app/layout.tsx` (lines 15-40) - Global application shell and React structure.
|
||||
- `src/core/plugins/PluginManager.ts` (lines 20-56) - Heart of the engine, registers and lifecycle-manages external intelligence views.
|
||||
|
||||
## Technology Stack
|
||||
- **Framework:** Next.js 16 (App Router) — `src/app/`
|
||||
- **3D Engine:** CesiumJS + Resium — `src/core/globe/GlobeView.tsx` (lines 40-90)
|
||||
- **State Management:** Zustand — `src/core/state/store.ts` (slice pattern)
|
||||
- **Database:** Prisma ORM w/ SQLite locally — `prisma/schema.prisma`
|
||||
- **Typing:** TypeScript 5 w/ Strict Mode — `tsconfig.json`
|
||||
|
||||
## Platform Support
|
||||
Built as a highly portable Next.js container, it supports `local`, `cloud`, and `demo` editions via the `NEXT_PUBLIC_WWV_EDITION` flag declared in `.env.local` and evaluated at runtime in `src/core/edition.ts`.
|
||||
@@ -0,0 +1,145 @@
|
||||
# Expand OSM Chips Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Expand the available OSM search chips to include a curated list of ~30 pre-loaded OSINT-relevant tags, and add a live Taginfo API autocomplete search to dynamically find and select any of the millions of other OSM tags.
|
||||
|
||||
**Architecture:** We will replace the current small `COMMON_TAGS` array with a richer, categorized list inside the `OSMSidebar` file. To allow access to "all" tags without crashing the UI, we will add a debounced fetch hook that hits `https://taginfo.openstreetmap.org/api/4/tags/popular` (or the equivalent search endpoint) when the user types in the filter box. These dynamically fetched tags will temporarily populate the chip list.
|
||||
|
||||
**Tech Stack:** React, TypeScript, Taginfo API
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Update Pre-Loaded COMMON_TAGS
|
||||
|
||||
**Files:**
|
||||
- Modify: `c:\dev\worldwideview\packages\wwv-plugin-osm-search\src\components\OSMSidebar.tsx:6-11`
|
||||
|
||||
- [ ] **Step 1: Replace existing COMMON_TAGS array**
|
||||
|
||||
```tsx
|
||||
const COMMON_TAGS = [
|
||||
// Military & Security
|
||||
"military=base", "military=bunker", "amenity=police", "amenity=prison",
|
||||
// Aviation & Maritime
|
||||
"aeroway=aerodrome", "aeroway=helipad", "aeroway=hangar", "man_made=pier", "harbour=yes",
|
||||
// Infrastructure & Utilities
|
||||
"power=plant", "power=substation", "telecom=antenna", "man_made=communications_tower", "man_made=water_tower",
|
||||
// Transport
|
||||
"highway=bus_stop", "railway=station", "amenity=fuel", "amenity=parking",
|
||||
// Medical & Emergency
|
||||
"amenity=hospital", "amenity=fire_station", "amenity=clinic",
|
||||
// General Commercial & Industrial
|
||||
"shop=supermarket", "tourism=hotel", "industrial=factory", "landuse=industrial",
|
||||
// Education & Amenities
|
||||
"amenity=school", "amenity=university", "amenity=cafe", "building=house"
|
||||
];
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/wwv-plugin-osm-search/src/components/OSMSidebar.tsx
|
||||
git commit -m "feat(osm-search): expand quick preset tags with OSINT categories"
|
||||
```
|
||||
|
||||
### Task 2: Implement Taginfo Dynamic Autocomplete
|
||||
|
||||
**Files:**
|
||||
- Modify: `c:\dev\worldwideview\packages\wwv-plugin-osm-search\src\components\OSMSidebar.tsx`
|
||||
|
||||
- [ ] **Step 1: Add state and debounced fetch logic inside OSMSidebar component**
|
||||
|
||||
```tsx
|
||||
const [searchText, setSearchText] = useState("");
|
||||
const [activeTags, setActiveTags] = useState<string[]>([]);
|
||||
const [dynamicTags, setDynamicTags] = useState<string[]>([]);
|
||||
const [isSearchingApi, setIsSearchingApi] = useState(false);
|
||||
|
||||
// Add this useEffect below the useState hooks
|
||||
React.useEffect(() => {
|
||||
if (searchText.length < 3) {
|
||||
setDynamicTags([]);
|
||||
return;
|
||||
}
|
||||
|
||||
const timer = setTimeout(async () => {
|
||||
setIsSearchingApi(true);
|
||||
try {
|
||||
// Fetch tags matching the search term from OSM Taginfo
|
||||
const res = await fetch(`https://taginfo.openstreetmap.org/api/4/search/by_keyword?query=${encodeURIComponent(searchText)}`);
|
||||
const data = await res.json();
|
||||
if (data && data.data) {
|
||||
// Extract exact key=value pairs or just keys, format as "key=value"
|
||||
const tags = data.data
|
||||
.filter((item: any) => item.key && item.value)
|
||||
.map((item: any) => `${item.key}=${item.value}`)
|
||||
.slice(0, 15); // Limit to top 15 results
|
||||
setDynamicTags(tags);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Taginfo fetch failed", err);
|
||||
} finally {
|
||||
setIsSearchingApi(false);
|
||||
}
|
||||
}, 500); // 500ms debounce
|
||||
|
||||
return () => clearTimeout(timer);
|
||||
}, [searchText]);
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update rendering logic to merge static and dynamic tags**
|
||||
|
||||
Look for the `renderedTags` definition and replace it:
|
||||
|
||||
```tsx
|
||||
// Combine filtered common tags with newly fetched dynamic tags, removing duplicates
|
||||
const filteredCommon = searchText
|
||||
? COMMON_TAGS.filter(t => t.toLowerCase().includes(searchText.toLowerCase()))
|
||||
: COMMON_TAGS;
|
||||
|
||||
const renderedTags = Array.from(new Set([...filteredCommon, ...dynamicTags]));
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add loading indicator to the search input area**
|
||||
|
||||
Look for the `<input>` element for `searchText` and wrap it/add a loading indicator:
|
||||
|
||||
```tsx
|
||||
<div style={{ position: "relative" }}>
|
||||
<input
|
||||
style={{
|
||||
width: "100%",
|
||||
padding: "8px",
|
||||
paddingRight: "24px",
|
||||
backgroundColor: "rgba(0,0,0,0.3)",
|
||||
color: "#fff",
|
||||
border: "1px solid var(--border-subtle)",
|
||||
borderRadius: "4px",
|
||||
fontSize: "13px"
|
||||
}}
|
||||
placeholder="Filter or search OSM for tags..."
|
||||
value={searchText}
|
||||
onChange={e => setSearchText(e.target.value)}
|
||||
/>
|
||||
{isSearchingApi && (
|
||||
<span style={{
|
||||
position: "absolute",
|
||||
right: "8px",
|
||||
top: "50%",
|
||||
transform: "translateY(-50%)",
|
||||
fontSize: "10px",
|
||||
color: "var(--text-muted)"
|
||||
}}>
|
||||
⏳
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/wwv-plugin-osm-search/src/components/OSMSidebar.tsx
|
||||
git commit -m "feat(osm-search): add live Taginfo autocomplete for unlisted tags"
|
||||
```
|
||||
@@ -0,0 +1,139 @@
|
||||
# Plugin Versioning Streamlining Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Refactor all WorldWideView plugin classes to derive their `version` property dynamically from their local `package.json` file rather than hardcoding string literals.
|
||||
|
||||
**Architecture:** We will replace the hardcoded `version = "x.y.z"` strings in the core `WorldPlugin` interface implementations across all monorepo packages. We will utilize TypeScript's JSON module resolution feature to `import pkg from "../package.json"` and map `version = pkg.version`. This ensures that `pnpm version patch` is the single source of truth and eliminates the double-bookkeeping that previously made CI fixes tedious.
|
||||
|
||||
**Tech Stack:** TypeScript, Next.js / bundler (JSON module resolution), pnpm workspaces.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Verify TypeScript Configuration
|
||||
|
||||
**Files:**
|
||||
- Modify: N/A (Verification step)
|
||||
|
||||
- [ ] **Step 1: Check `tsconfig.json` for JSON resolution**
|
||||
|
||||
Run: `cat tsconfig.json | grep resolveJsonModule`
|
||||
Expected: `"resolveJsonModule": true`
|
||||
|
||||
- [ ] **Step 2: Check compiler compatibility**
|
||||
|
||||
Run: `pnpm exec tsc --noEmit`
|
||||
Expected: PASS (exit code 0)
|
||||
|
||||
## Task 2: Refactor Group A Plugins (Core Infrastructure)
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/wwv-plugin-aviation/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-maritime/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-military-aviation/src/index.ts`
|
||||
|
||||
- [ ] **Step 1: Update plugin classes**
|
||||
|
||||
For each file above, modify the imports and the class body:
|
||||
|
||||
```typescript
|
||||
import pkg from "../package.json";
|
||||
// ...
|
||||
export class AviationPlugin implements WorldPlugin { // (or extends BasePlugin)
|
||||
// ...
|
||||
version = pkg.version;
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify compilation**
|
||||
|
||||
Run: `pnpm exec tsc --noEmit`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "refactor(plugins): derive version dynamically for core infrastructure plugins [Patch]"
|
||||
```
|
||||
|
||||
## Task 3: Refactor Group B Plugins (Facilities & Infrastructure)
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/wwv-plugin-airports/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-seaports/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-spaceports/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-embassies/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-lighthouses/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-nuclear/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-undersea-cables/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-mineral-mines/src/index.ts`
|
||||
|
||||
- [ ] **Step 1: Update plugin classes**
|
||||
|
||||
For each file above, replace `version = "..."` with `version = pkg.version` and add `import pkg from "../package.json";`.
|
||||
|
||||
- [ ] **Step 2: Verify compilation**
|
||||
|
||||
Run: `pnpm exec tsc --noEmit`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "refactor(plugins): derive version dynamically for facility plugins [Patch]"
|
||||
```
|
||||
|
||||
## Task 4: Refactor Group C Plugins (Hazards & Events)
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/wwv-plugin-earthquakes/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-volcanoes/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-wildfire/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-conflict-events/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-conflict-zones/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-civil-unrest/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-cyber-attacks/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-international-sanctions/src/index.tsx`
|
||||
|
||||
- [ ] **Step 1: Update plugin classes**
|
||||
|
||||
For each file above, replace `version = "..."` with `version = pkg.version` and add `import pkg from "../package.json";`. Note that some files are `.tsx`.
|
||||
|
||||
- [ ] **Step 2: Verify compilation**
|
||||
|
||||
Run: `pnpm exec tsc --noEmit`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "refactor(plugins): derive version dynamically for event and hazard plugins [Patch]"
|
||||
```
|
||||
|
||||
## Task 5: Refactor Group D Plugins (Misc & Utilities)
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/wwv-plugin-satellite/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-surveillance-satellites/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-osm-search/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-iranwarlive/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-daynight/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-camera/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-borders/src/index.ts`
|
||||
- Modify: `packages/wwv-plugin-gps-jamming/src/index.tsx`
|
||||
- Modify: `packages/wwv-plugin-air-defense/src/index.tsx`
|
||||
|
||||
- [ ] **Step 1: Update plugin classes**
|
||||
|
||||
For each file above, replace `version = "..."` with `version = pkg.version` and add `import pkg from "../package.json";`.
|
||||
|
||||
- [ ] **Step 2: Verify compilation and tests**
|
||||
|
||||
Run: `pnpm exec tsc --noEmit`
|
||||
Run: `pnpm test`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "refactor(plugins): derive version dynamically for utility plugins [Patch]"
|
||||
```
|
||||
@@ -0,0 +1,124 @@
|
||||
# Demo Default Plugins Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Enable specific plugins by default on the demo edition via environment variables so the user is immediately impressed.
|
||||
|
||||
**Architecture:** Update `layersSlice.ts` to accept an optional `defaultEnabled` parameter in `initLayer`. In `AppShell.tsx`, look up `NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS` when the edition is `demo`, and pass the flag true to `initLayer` for matching plugins during registration. Document the variable in `.env.example`.
|
||||
|
||||
**Tech Stack:** Next.js, Zustand, TypeScript.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Update layer slice definition
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/core/state/layersSlice.ts`
|
||||
|
||||
- [ ] **Step 1: Write the implementation**
|
||||
|
||||
Update `initLayer` in `src/core/state/layersSlice.ts` to accept a second parameter:
|
||||
|
||||
```typescript
|
||||
// Add defaultEnabled parameter to the interface
|
||||
export interface LayersSlice {
|
||||
// ...
|
||||
initLayer: (pluginId: string, defaultEnabled?: boolean) => void;
|
||||
}
|
||||
|
||||
// ...
|
||||
|
||||
// Use the parameter in the state updater
|
||||
initLayer: (pluginId, defaultEnabled = false) =>
|
||||
set((state) => ({
|
||||
layers: {
|
||||
...state.layers,
|
||||
[pluginId]: state.layers[pluginId] || { enabled: defaultEnabled, entityCount: 0, loading: false },
|
||||
},
|
||||
})),
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add src/core/state/layersSlice.ts
|
||||
npm run commit -- "feat: add defaultEnabled param to initLayer in layers slice"
|
||||
```
|
||||
|
||||
### Task 2: Apply default configuration in AppShell
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/components/layout/AppShell.tsx`
|
||||
|
||||
- [ ] **Step 1: Write the implementation**
|
||||
|
||||
Add `isDemo` import at the top of `src/components/layout/AppShell.tsx`:
|
||||
|
||||
```typescript
|
||||
import { isDemo } from "@/core/edition";
|
||||
```
|
||||
|
||||
In `AppShell.tsx`, around line 71, inside `startPlatform()`, fetch the value from the environment if the instance is demo:
|
||||
|
||||
```typescript
|
||||
// Fetch disabled built-in plugins before registration
|
||||
let disabledIds = new Set<string>();
|
||||
// ... (keep existing try/catch block)
|
||||
|
||||
// Setup demo defaults
|
||||
const demoDefaultPlugins = new Set<string>();
|
||||
if (isDemo) {
|
||||
const envVar = process.env.NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS || "";
|
||||
envVar.split(",").forEach(s => {
|
||||
const clean = s.trim();
|
||||
if (clean) demoDefaultPlugins.add(clean);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Then update the `pluginManager` registration loop to inject the boolean for `initLayer`:
|
||||
|
||||
```typescript
|
||||
for (const plugin of pluginRegistry.getAll()) {
|
||||
await pluginManager.registerPlugin(plugin);
|
||||
initLayer(plugin.id, demoDefaultPlugins.has(plugin.id));
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it passes**
|
||||
|
||||
Run type checks to verify no compilation issues:
|
||||
`npx tsc --noEmit`
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add src/components/layout/AppShell.tsx
|
||||
npm run commit -- "feat: parse and apply default demo plugins via NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS"
|
||||
```
|
||||
|
||||
### Task 3: Document environment definitions
|
||||
|
||||
**Files:**
|
||||
- Modify: `.env.example`
|
||||
- Modify: `.env`
|
||||
|
||||
- [ ] **Step 1: Write the implementation**
|
||||
|
||||
Update `.env.example` and `.env` around the `NEXT_PUBLIC_WWV_EDITION=local` definition:
|
||||
|
||||
```dotenv
|
||||
# Default plugins enabled on boot for demo edition
|
||||
# Comma-separated list of plugin IDs (e.g. aviation,maritime,iranwarlive)
|
||||
NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS=
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add .env.example .env
|
||||
npm run commit -- "docs: add NEXT_PUBLIC_DEMO_DEFAULT_PLUGINS to env templates"
|
||||
```
|
||||
@@ -0,0 +1,177 @@
|
||||
# Decouple Plugin Proxies Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Strip out hardcoded proxy endpoints from the `worldwideview` core repository to enforce a fully decoupled plugin ecosystem, replacing them with direct browser fetching or a universal CORS proxy.
|
||||
|
||||
**Architecture:** Remove preferential API routes (`/api/earthquake`, `/api/undersea-cables`, `/api/camera/*`) from the `worldwideview` core app. Promote the existing camera proxy to a universal `/api/proxy` with strict SSRF defenses. Provide direct browser-fetch code for the `wwv-plugin-earthquakes` where CORS permits, and migrate traffic camera fetchers directly into the `wwv-plugin-camera` package to fetch via the universal proxy.
|
||||
|
||||
**Tech Stack:** Next.js 16 (App Router), TypeScript, pnpm workspace
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Promote Universal Proxy in Core Repository
|
||||
|
||||
**Files:**
|
||||
- Create (Move): `src/app/api/proxy/route.ts`
|
||||
- Delete: `src/app/api/camera/proxy/route.ts`
|
||||
|
||||
- [ ] **Step 1: Relocate the proxy route and update logs**
|
||||
|
||||
Move the existing camera proxy to act as a system-wide universal proxy.
|
||||
|
||||
Run: `mkdir -p src/app/api/proxy`
|
||||
Run: `mv src/app/api/camera/proxy/route.ts src/app/api/proxy/route.ts`
|
||||
|
||||
- [ ] **Step 2: Update the error logging prefix in the new file**
|
||||
|
||||
Modify `src/app/api/proxy/route.ts` near line 97:
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "Update the error logging prefix",
|
||||
"replace_chunks": [
|
||||
{
|
||||
"start_line": 96,
|
||||
"end_line": 98,
|
||||
"target_content": " } catch (error: any) {\n console.error(\"[CameraProxy] Error fetching target URL:\", error);\n return NextResponse.json(",
|
||||
"replacement_content": " } catch (error: any) {\n console.error(\"[UniversalProxy] Error fetching target URL:\", error);\n return NextResponse.json("
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Commit the Universal Proxy**
|
||||
|
||||
```bash
|
||||
git add src/app/api/proxy/route.ts src/app/api/camera/proxy/route.ts
|
||||
git commit -m "refactor: promote camera proxy to universal proxy"
|
||||
```
|
||||
|
||||
### Task 2: Strip Preferential Proxy Routes From Core Application
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/app/api/earthquake/route.ts`
|
||||
- Delete: `src/app/api/undersea-cables/route.ts`
|
||||
- Delete: everything in `src/app/api/camera` codebase (`caltrans`, `extract`, `gdot`, `test`, `tfl`, `traffic`)
|
||||
|
||||
- [ ] **Step 1: Delete all plugin-specific Next.js API routes**
|
||||
|
||||
Run: `rm -rf src/app/api/earthquake`
|
||||
Run: `rm -rf src/app/api/undersea-cables`
|
||||
|
||||
**WAIT**: For `src/app/api/camera`, we must physically copy the fetchers into `worldwideview-plugins` before we delete them.
|
||||
|
||||
Run: `cp -r src/app/api/camera/caltrans ../worldwideview-plugins/packages/wwv-plugin-camera/src/`
|
||||
Run: `cp -r src/app/api/camera/gdot ../worldwideview-plugins/packages/wwv-plugin-camera/src/`
|
||||
Run: `cp -r src/app/api/camera/tfl ../worldwideview-plugins/packages/wwv-plugin-camera/src/`
|
||||
Run: `rm -rf src/app/api/camera`
|
||||
|
||||
- [ ] **Step 2: Commit the deleted API routes**
|
||||
|
||||
```bash
|
||||
git add src/app/api/earthquake src/app/api/undersea-cables src/app/api/camera
|
||||
git commit -m "refactor!: strip preferential plugin APIs from core"
|
||||
```
|
||||
|
||||
### Task 3: Migrate Earthquakes and Undersea Cables Plugins
|
||||
|
||||
**Files:**
|
||||
- Modify: `../worldwideview-plugins/packages/wwv-plugin-earthquakes/src/index.ts`
|
||||
- Modify: `../worldwideview-plugins/packages/wwv-plugin-undersea-cables/src/index.ts`
|
||||
|
||||
- [ ] **Step 1: Update Earthquakes plugin to direct browser fetch (Option A)**
|
||||
|
||||
In `../worldwideview-plugins/packages/wwv-plugin-earthquakes/src/index.ts`, replace the `/api/earthquake` fetch with the USGS direct URL.
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "Replace the API fetch with direct USGS fetch and remove apiBasePath",
|
||||
"replace_chunks": [
|
||||
{
|
||||
"start_line": 38,
|
||||
"end_line": 40,
|
||||
"target_content": " try {\n const res = await globalThis.fetch(`/api/earthquake`);\n if (!res.ok) {",
|
||||
"replacement_content": " try {\n const res = await globalThis.fetch(`https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson`);\n if (!res.ok) {"
|
||||
},
|
||||
{
|
||||
"start_line": 94,
|
||||
"end_line": 96,
|
||||
"target_content": " getServerConfig(): ServerPluginConfig {\n return { apiBasePath: \"/api/earthquake\", pollingIntervalMs: 120000, historyEnabled: false };\n }",
|
||||
"replacement_content": " getServerConfig(): ServerPluginConfig {\n return { pollingIntervalMs: 120000, historyEnabled: false };\n }"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update Undersea Cables plugin to use Universal Proxy (Option B)**
|
||||
|
||||
In `../worldwideview-plugins/packages/wwv-plugin-undersea-cables/src/index.ts`:
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "Proxy route to universal proxy",
|
||||
"replace_chunks": [
|
||||
{
|
||||
"start_line": 35,
|
||||
"end_line": 37,
|
||||
"target_content": " // Proxy route to bypass CORS for Telegeography submarine cable map API\n const url = \"/api/undersea-cables\";\n ",
|
||||
"replacement_content": " // Proxy route to bypass CORS for Telegeography submarine cable map API\n const targetUrl = encodeURIComponent(\"https://www.submarinecablemap.com/api/v3/cable/cable-geo.json\");\n const url = `/api/proxy?url=${targetUrl}`;\n "
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Commit the plugin migrations**
|
||||
|
||||
```bash
|
||||
cd ../worldwideview-plugins
|
||||
git add packages/wwv-plugin-earthquakes/src/index.ts packages/wwv-plugin-undersea-cables/src/index.ts
|
||||
git commit -m "refactor: transition earthquake and undersea cables to decoupled fetching"
|
||||
cd ../worldwideview
|
||||
```
|
||||
|
||||
### Task 4: Refactor Camera Plugin to Fetch Natively
|
||||
|
||||
**Files:**
|
||||
- Modify: `../worldwideview-plugins/packages/wwv-plugin-camera/src/index.ts`
|
||||
- Delete: `caltrans`, `gdot`, `tfl` previously copied components (combine into `trafficFetchers.ts`)
|
||||
|
||||
- [ ] **Step 1: Consolidate the camera fetchers**
|
||||
|
||||
Create `../worldwideview-plugins/packages/wwv-plugin-camera/src/trafficFetchers.ts`.
|
||||
Migrate the code from the `caltransFetcher.ts`, `gdotFetcher.ts`, and `tflFetcher.ts` files that we copied in Task 2.
|
||||
Crucially, when doing `fetch(...)` in these fetchers, if the endpoint doesn't support CORS, wrap it in `/api/proxy?url=${encodeURIComponent(...) }` so the browser can fetch it. Ensure the data models align.
|
||||
|
||||
- [ ] **Step 2: Update `wwv-plugin-camera/src/index.ts` to call the fetchers natively**
|
||||
|
||||
In `../worldwideview-plugins/packages/wwv-plugin-camera/src/index.ts` function `loadTrafficCameras()`:
|
||||
|
||||
```json
|
||||
{
|
||||
"instruction": "Replace API fetch with native client-side parallel fetching",
|
||||
"replace_chunks": [
|
||||
{
|
||||
"start_line": 106,
|
||||
"end_line": 118,
|
||||
"target_content": " private async loadTrafficCameras(): Promise<void> {\n try {\n const res = await fetch(\"/api/camera/traffic\");\n if (!res.ok) throw new Error(`API returned ${res.status}`);\n const data = await res.json();\n if (data.cameras && Array.isArray(data.cameras)) {\n this.sourceBuckets[\"default\"] = data.cameras.map(\n (f: unknown, i: number) => mapGeoJsonFeature(f, i, \"traffic\"),\n );\n }\n } catch (err) {\n console.warn(\"[CameraPlugin] Traffic cameras API failed:\", err);\n }",
|
||||
"replacement_content": " private async loadTrafficCameras(): Promise<void> {\n try {\n const { fetchCaltrans, fetchGdot, fetchTfl } = await import(\"./trafficFetchers\");\n const [caltrans, gdot, tfl] = await Promise.allSettled([\n fetchCaltrans(), fetchGdot(), fetchTfl()\n ]);\n \n const cameras = [];\n if (caltrans.status === \"fulfilled\") cameras.push(...caltrans.value);\n if (gdot.status === \"fulfilled\") cameras.push(...gdot.value);\n if (tfl.status === \"fulfilled\") cameras.push(...tfl.value);\n\n if (cameras.length > 0) {\n this.sourceBuckets[\"default\"] = cameras.map(\n (f: unknown, i: number) => mapGeoJsonFeature(f, i, \"traffic\"),\n );\n }\n } catch (err) {\n console.warn(\"[CameraPlugin] Traffic cameras API failed:\", err);\n }"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Remove the extra node API files from the plugin source tree**
|
||||
|
||||
Run: `rm -rf ../worldwideview-plugins/packages/wwv-plugin-camera/src/caltrans`
|
||||
Run: `rm -rf ../worldwideview-plugins/packages/wwv-plugin-camera/src/gdot`
|
||||
Run: `rm -rf ../worldwideview-plugins/packages/wwv-plugin-camera/src/tfl`
|
||||
|
||||
- [ ] **Step 4: Commit Camera plugin refactoring**
|
||||
|
||||
```bash
|
||||
cd ../worldwideview-plugins
|
||||
git add packages/wwv-plugin-camera/src
|
||||
git commit -m "feat: migrate traffic camera fetchers to client-side plugin bundle"
|
||||
cd ../worldwideview
|
||||
```
|
||||
@@ -0,0 +1,25 @@
|
||||
<!-- Generated: 2026-04-19 15:23:00 UTC -->
|
||||
# Testing
|
||||
|
||||
WorldWideView maintains a robust Vitest testing infrastructure utilizing `jsdom` environments to mock browser globals and React Testing Library utilities for accurate integration flow testing.
|
||||
|
||||
The primary requirement is full test coverage over authentication routes, edge-case platform parsing, and API rate-limit security.
|
||||
|
||||
## Test Types
|
||||
- **Unit Logic:** Verifies math, utilities, and configuration mappings (e.g. `edition.test.ts`).
|
||||
- **Middleware Integration:** Evaluates Request/Response blocks (e.g. `rateLimit.test.ts`).
|
||||
- **Component Mocking:** Exercises the React ecosystem state behaviors (e.g. `DeclarativePlugin.test.ts`).
|
||||
|
||||
## Key Files
|
||||
- `vitest.config.ts` (lines 5-25): Defines the inclusion criteria spanning `src/lib/**`, `src/core/**`, and `src/plugins/**`.
|
||||
- `src/lib/rateLimit.test.ts` (lines 15-60): Demonstrates test encapsulation regarding the Upstash Redis fallbacks.
|
||||
- `src/core/edition.test.ts` (lines 10-30): Confirms valid feature flag evaluations across local/demo builds.
|
||||
|
||||
## Running Tests
|
||||
Run tests natively within the root directory utilizing `pnpm`:
|
||||
```bash
|
||||
# Execute standard test suite runs over all watched directories
|
||||
pnpm test
|
||||
# OR
|
||||
vitest run
|
||||
```
|
||||
@@ -0,0 +1,6 @@
|
||||
const { PrismaClient } = require('./src/generated/prisma');
|
||||
const prisma = new PrismaClient();
|
||||
prisma.installedPlugin.findMany().then(r => console.log(JSON.stringify(r.map(record => {
|
||||
const manifest = JSON.parse(record.config);
|
||||
return { id: manifest.id, entry: manifest.entry, format: manifest.format };
|
||||
}), null, 2))).catch(console.error).finally(() => prisma.$disconnect());
|
||||
@@ -0,0 +1,9 @@
|
||||
import sqlite3
|
||||
import json
|
||||
conn = sqlite3.connect('data/wwv.db')
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('SELECT id, dataSource FROM installed_plugins')
|
||||
rows = cursor.fetchall()
|
||||
for r in rows:
|
||||
if r[1]:
|
||||
print(f"plugin={r[0]}, dataSource={r[1]}")
|
||||
@@ -0,0 +1,37 @@
|
||||
Write-Host "[*] Setting up WorldWideView for Local Development..."
|
||||
|
||||
# Check for pnpm
|
||||
try {
|
||||
$null = Get-Command pnpm -ErrorAction Stop
|
||||
} catch {
|
||||
Write-Host "[Error] pnpm is not installed or not in PATH."
|
||||
Write-Host "Please install it first: https://pnpm.io/installation"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "[*] Installing dependencies..."
|
||||
pnpm install
|
||||
|
||||
Write-Host "[*] Running initial setup (generating secrets)..."
|
||||
pnpm run setup
|
||||
|
||||
# Check for the sibling Data Engine repository
|
||||
if (-not (Test-Path "../wwv-data-engine")) {
|
||||
Write-Host ""
|
||||
Write-Host "=====================================================================" -ForegroundColor Yellow
|
||||
Write-Host "[!] NOTICE: Local Data Engine not found at ../wwv-data-engine" -ForegroundColor Yellow
|
||||
Write-Host ""
|
||||
Write-Host "Frontend-Only Mode: You are developing the frontend UI." -ForegroundColor Green
|
||||
Write-Host "WorldWideView will automatically stream data from the Cloud Engine." -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Write-Host "Full-Stack Mode: If you want to develop backend data seeders, you" -ForegroundColor Cyan
|
||||
Write-Host "must clone the open-source data engine as a sibling directory:" -ForegroundColor Cyan
|
||||
Write-Host " cd ..; git clone https://github.com/silvertakana/wwv-data-engine"
|
||||
Write-Host " cd wwv-data-engine; pnpm install"
|
||||
Write-Host "=====================================================================" -ForegroundColor Yellow
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
Write-Host "[*] Starting local Next.js frontend server..."
|
||||
Write-Host " (To run the data engine backends concurrently, run: pnpm dev:all)"
|
||||
pnpm run dev
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "🌍 Setting up WorldWideView for Local Development..."
|
||||
|
||||
# Check for pnpm
|
||||
if ! command -v pnpm &> /dev/null; then
|
||||
echo "❌ Error: pnpm is required but not installed."
|
||||
echo "Please install it first: https://pnpm.io/installation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📦 Installing dependencies..."
|
||||
pnpm install
|
||||
|
||||
echo "🔐 Running initial setup (generating secrets)..."
|
||||
pnpm run setup
|
||||
|
||||
# Check for the sibling Data Engine repository
|
||||
if [ ! -d "../wwv-data-engine" ]; then
|
||||
echo ""
|
||||
echo "====================================================================="
|
||||
echo "⚠️ NOTICE: Local Data Engine not found at ../wwv-data-engine"
|
||||
echo ""
|
||||
echo "Frontend-Only Mode: You are developing the frontend UI."
|
||||
echo "WorldWideView will automatically stream data from the Cloud Engine."
|
||||
echo ""
|
||||
echo "Full-Stack Mode: If you want to develop backend data seeders, you"
|
||||
echo "must clone the open-source data engine as a sibling directory:"
|
||||
echo " cd .. && git clone https://github.com/silvertakana/wwv-data-engine"
|
||||
echo " cd wwv-data-engine && pnpm install"
|
||||
echo "====================================================================="
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "🚀 Starting local Next.js frontend server..."
|
||||
echo " (To run the data engine backends concurrently, run: pnpm dev:all)"
|
||||
pnpm run dev
|
||||
Whitespace-only changes.
+24
-32
@@ -3,26 +3,7 @@ import type { NextConfig } from "next";
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
serverExternalPackages: ["better-sqlite3", "@prisma/adapter-better-sqlite3", "@prisma/client", "prisma"],
|
||||
transpilePackages: [
|
||||
"@worldwideview/wwv-plugin-aviation",
|
||||
"@worldwideview/wwv-plugin-borders",
|
||||
"@worldwideview/wwv-plugin-camera",
|
||||
"@worldwideview/wwv-plugin-conflict-events",
|
||||
"@worldwideview/wwv-plugin-civil-unrest",
|
||||
"@worldwideview/wwv-plugin-daynight",
|
||||
"@worldwideview/wwv-plugin-undersea-cables",
|
||||
"@worldwideview/wwv-plugin-earthquakes",
|
||||
"@worldwideview/wwv-plugin-iranwarlive",
|
||||
"@worldwideview/wwv-plugin-maritime",
|
||||
"@worldwideview/wwv-plugin-military-aviation",
|
||||
"@worldwideview/wwv-plugin-satellite",
|
||||
"@worldwideview/wwv-plugin-gps-jamming",
|
||||
"@worldwideview/wwv-plugin-surveillance-satellites",
|
||||
"@worldwideview/wwv-plugin-cyber-attacks",
|
||||
"@worldwideview/wwv-plugin-international-sanctions",
|
||||
"@worldwideview/wwv-plugin-sdk",
|
||||
"@worldwideview/wwv-plugin-wildfire",
|
||||
"resium",
|
||||
transpilePackages: ["@worldwideview/wwv-plugin-sdk","resium",
|
||||
"react-player",
|
||||
"satellite.js"
|
||||
],
|
||||
@@ -44,7 +25,7 @@ const nextConfig: NextConfig = {
|
||||
value: [
|
||||
"default-src 'self'",
|
||||
// CesiumJS requires unsafe-eval (worker compilation) and unsafe-inline (styles)
|
||||
"script-src 'self' 'unsafe-eval' 'unsafe-inline' blob: https://cdn.jsdelivr.net https://analytics.worldwideview.dev https://va.vercel-scripts.com https://pagead2.googlesyndication.com https://adservice.google.com https://www.googletagservices.com https://ep2.adtrafficquality.google https://static.cloudflareinsights.com",
|
||||
"script-src 'self' 'unsafe-eval' 'unsafe-inline' blob: https://unpkg.com https://cdn.jsdelivr.net https://analytics.worldwideview.dev https://va.vercel-scripts.com https://pagead2.googlesyndication.com https://adservice.google.com https://www.googletagservices.com https://ep2.adtrafficquality.google https://static.cloudflareinsights.com",
|
||||
"style-src 'self' 'unsafe-inline' fonts.googleapis.com",
|
||||
"font-src 'self' fonts.gstatic.com",
|
||||
// Camera streams load images/MJPEG from arbitrary IPs worldwide — http: https: required
|
||||
@@ -71,16 +52,7 @@ const nextConfig: NextConfig = {
|
||||
},
|
||||
];
|
||||
},
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
source: "/api/external/:source*",
|
||||
// Proxy to the shared data engine locally or over docker network
|
||||
// We use process.env to override the url if running Next.js outside docker
|
||||
destination: `${process.env.WWV_DATA_ENGINE_URL || "http://wwv-data-engine:5001"}/data/:source*`,
|
||||
},
|
||||
];
|
||||
},
|
||||
|
||||
outputFileTracingExcludes: {
|
||||
"*": ["./public/cesium/**"],
|
||||
},
|
||||
@@ -112,4 +84,24 @@ const nextConfig: NextConfig = {
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
import { withSentryConfig } from "@sentry/nextjs";
|
||||
|
||||
export default withSentryConfig(nextConfig, {
|
||||
// For all available options, see:
|
||||
// https://github.com/getsentry/sentry-webpack-plugin#options
|
||||
org: process.env.SENTRY_ORG,
|
||||
project: process.env.SENTRY_PROJECT,
|
||||
sentryUrl: process.env.GLITCHTIP_SERVER_URL,
|
||||
authToken: process.env.SENTRY_AUTH_TOKEN,
|
||||
|
||||
// Automatically tree-shake Sentry logger statements to reduce bundle size
|
||||
disableLogger: true,
|
||||
|
||||
// Hides source maps from generated client bundles
|
||||
sourcemaps: {
|
||||
disable: true,
|
||||
},
|
||||
|
||||
// Upload a larger set of source maps for prettier stack traces (increases build time)
|
||||
widenClientFileUpload: true,
|
||||
});
|
||||
+9
-23
@@ -1,46 +1,29 @@
|
||||
{
|
||||
"name": "worldwideview",
|
||||
"version": "1.7.1",
|
||||
"version": "1.18.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"setup": "node scripts/setup.mjs",
|
||||
"test-worktree": "node scripts/test-worktree.mjs",
|
||||
"clean": "node -e \"const fs=require('fs');fs.rmSync('.next',{recursive:true,force:true})\"",
|
||||
"dev": "next dev --webpack",
|
||||
"dev:all": "npx concurrently \"pnpm dev\" \"pnpm --filter wwv-data-engine dev\"",
|
||||
"dev:all": "concurrently -n \"next,backends\" -c \"magenta,blue\" \"pnpm dev\" \"pnpm dev:backends\"",
|
||||
"dev:backends": "cd ../wwv-data-engine && pnpm dev",
|
||||
"predev": "npx prisma migrate deploy && node scripts/copy-cesium.mjs",
|
||||
"copy-cesium": "node scripts/copy-cesium.mjs",
|
||||
"build": "next build --webpack",
|
||||
"start": "next start",
|
||||
"test": "vitest run",
|
||||
"start:backends": "pnpm --filter wwv-data-engine start",
|
||||
"dev:backends": "pnpm --filter wwv-data-engine dev",
|
||||
"clean:backends": "pnpm --filter wwv-data-engine clean:db",
|
||||
"db:reset": "npx prisma migrate reset --force"
|
||||
},
|
||||
"dependencies": {
|
||||
"@auth/prisma-adapter": "^2.11.1",
|
||||
"@prisma/adapter-better-sqlite3": "^7.5.0",
|
||||
"@prisma/client": "^7.5.0",
|
||||
"@sentry/nextjs": "^10.48.0",
|
||||
"@supabase/supabase-js": "^2.98.0",
|
||||
"@vercel/analytics": "^1.6.1",
|
||||
"@worldwideview/wwv-plugin-aviation": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-borders": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-camera": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-civil-unrest": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-conflict-events": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-cyber-attacks": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-daynight": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-earthquakes": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-gps-jamming": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-international-sanctions": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-iranwarlive": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-maritime": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-military-aviation": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-satellite": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-surveillance-satellites": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-undersea-cables": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-wildfire": "workspace:*",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cesium": "^1.139.0",
|
||||
"cheerio": "^1.2.0",
|
||||
@@ -48,7 +31,7 @@
|
||||
"insecam-api": "^0.0.3",
|
||||
"jose": "^6.2.2",
|
||||
"lucide-react": "^0.576.0",
|
||||
"next": "16.2.1",
|
||||
"next": "16.2.3",
|
||||
"next-auth": "^5.0.0-beta.30",
|
||||
"prisma": "^7.5.0",
|
||||
"react": "19.2.3",
|
||||
@@ -70,9 +53,12 @@
|
||||
"@types/react-dom": "^19",
|
||||
"@types/ws": "^8.18.1",
|
||||
"@vitejs/plugin-react": "^5.1.4",
|
||||
"concurrently": "^9.2.1",
|
||||
"copy-webpack-plugin": "^14.0.0",
|
||||
"jsdom": "^28.1.0",
|
||||
"rollup-plugin-external-globals": "^0.13.0",
|
||||
"typescript": "^5",
|
||||
"vite": "^8.0.8",
|
||||
"vitest": "^4.0.18"
|
||||
},
|
||||
"pnpm": {
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"name": "@worldwideview/create-plugin",
|
||||
"version": "0.1.0",
|
||||
"description": "Scaffold a new WorldWideView plugin",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"create-wwv-plugin": "./dist/index.mjs"
|
||||
},
|
||||
"files": ["dist", "templates"],
|
||||
"scripts": {
|
||||
"build": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"esbuild": "^0.25.0"
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import { scaffold } from "./scaffold.js";
|
||||
|
||||
const name = process.argv[2];
|
||||
if (!name) {
|
||||
console.error("Usage: npx @worldwideview/create-plugin <plugin-name>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
scaffold(name);
|
||||
@@ -1,45 +0,0 @@
|
||||
import { mkdirSync, writeFileSync, readFileSync, readdirSync } from "fs";
|
||||
import { join, resolve, dirname } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function scaffold(name: string): void {
|
||||
const slug = name.toLowerCase().replace(/[^a-z0-9-]/g, "-");
|
||||
const dir = resolve(process.cwd(), slug);
|
||||
|
||||
console.log(`\n🌍 Creating WorldWideView plugin: ${slug}\n`);
|
||||
mkdirSync(join(dir, "src"), { recursive: true });
|
||||
|
||||
const templateDir = join(__dirname, "..", "templates");
|
||||
const files = readdirSync(templateDir);
|
||||
|
||||
for (const file of files) {
|
||||
let content = readFileSync(join(templateDir, file), "utf8");
|
||||
content = content.replace(/\{\{SLUG\}\}/g, slug);
|
||||
content = content.replace(
|
||||
/\{\{NAME\}\}/g,
|
||||
slug
|
||||
.split("-")
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(""),
|
||||
);
|
||||
|
||||
// Strip .tpl extension and route index.ts into src/
|
||||
const outName = file.replace(/\.tpl$/, "");
|
||||
const dest =
|
||||
outName === "index.ts"
|
||||
? join(dir, "src", outName)
|
||||
: join(dir, outName);
|
||||
|
||||
writeFileSync(dest, content);
|
||||
console.log(` ✓ ${outName === "index.ts" ? "src/" + outName : outName}`);
|
||||
}
|
||||
|
||||
console.log(`\n✅ Done! Next steps:`);
|
||||
console.log(` cd ${slug}`);
|
||||
console.log(` npm install`);
|
||||
console.log(` npm run dev # Watch mode`);
|
||||
console.log(` npm run build # Production bundle`);
|
||||
console.log(` npm publish # Publish to npm\n`);
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
/**
|
||||
* {{NAME}} — WorldWideView Plugin
|
||||
*
|
||||
* Build with: npm run build
|
||||
* Publish with: npm publish
|
||||
*/
|
||||
|
||||
import type {
|
||||
WorldPlugin,
|
||||
GeoEntity,
|
||||
TimeRange,
|
||||
PluginContext,
|
||||
LayerConfig,
|
||||
CesiumEntityOptions,
|
||||
PluginCategory,
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
export default class {{NAME}}Plugin implements WorldPlugin {
|
||||
readonly id = "{{SLUG}}";
|
||||
readonly name = "{{NAME}}";
|
||||
readonly description = "A custom WorldWideView plugin";
|
||||
readonly icon = "📍";
|
||||
readonly category: PluginCategory = "custom";
|
||||
readonly version = "1.0.0";
|
||||
|
||||
async initialize(_ctx: PluginContext): Promise<void> {
|
||||
console.log(`[{{NAME}}] Initialized`);
|
||||
}
|
||||
|
||||
destroy(): void {}
|
||||
|
||||
async fetch(_timeRange: TimeRange): Promise<GeoEntity[]> {
|
||||
// Replace with your data source
|
||||
return [];
|
||||
}
|
||||
|
||||
getPollingInterval(): number {
|
||||
return 60_000;
|
||||
}
|
||||
|
||||
getLayerConfig(): LayerConfig {
|
||||
return { color: "#3b82f6", clusterEnabled: true, clusterDistance: 40 };
|
||||
}
|
||||
|
||||
renderEntity(_entity: GeoEntity): CesiumEntityOptions {
|
||||
return { type: "point", color: "#3b82f6", size: 6 };
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "wwv-plugin-{{SLUG}}",
|
||||
"version": "1.0.0",
|
||||
"description": "WorldWideView plugin: {{NAME}}",
|
||||
"type": "module",
|
||||
"module": "dist/index.mjs",
|
||||
"files": ["dist"],
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"dev": "vite build --watch"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "*",
|
||||
"vite": "^6.0.0",
|
||||
"rollup-plugin-external-globals": "^0.12.0",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^19.0.0"
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"id": "{{SLUG}}",
|
||||
"name": "{{NAME}}",
|
||||
"version": "1.0.0",
|
||||
"description": "A custom WorldWideView plugin",
|
||||
"type": "data-layer",
|
||||
"format": "bundle",
|
||||
"trust": "unverified",
|
||||
"capabilities": ["data:own"],
|
||||
"category": "custom",
|
||||
"icon": "📍",
|
||||
"entry": "dist/index.mjs"
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"jsx": "react-jsx",
|
||||
"declaration": false,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import externalGlobals from "rollup-plugin-external-globals";
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
lib: {
|
||||
entry: "src/index.ts",
|
||||
formats: ["es"],
|
||||
fileName: "index",
|
||||
},
|
||||
rollupOptions: {
|
||||
external: ["react", "react-dom", "react/jsx-runtime"],
|
||||
plugins: [
|
||||
externalGlobals({
|
||||
"react": "globalThis.__WWV_HOST__.React",
|
||||
"react-dom": "globalThis.__WWV_HOST__.ReactDOM",
|
||||
"react/jsx-runtime": "globalThis.__WWV_HOST__.jsxRuntime",
|
||||
}),
|
||||
],
|
||||
},
|
||||
minify: true,
|
||||
sourcemap: false,
|
||||
},
|
||||
});
|
||||
@@ -1,65 +0,0 @@
|
||||
# Stage 0: Extract package.json files for layer caching
|
||||
FROM alpine AS extractor
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN find . -type f \! -name 'package.json' \! -name 'pnpm-workspace.yaml' \! -name 'pnpm-lock.yaml' -delete && \
|
||||
find . -type d -empty -delete
|
||||
|
||||
# Build Stage
|
||||
FROM node:22-alpine AS builder
|
||||
|
||||
# better-sqlite3 requires python3 and build tools for native compilation
|
||||
RUN apk add --no-cache python3 make g++
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Enable corepack for pnpm
|
||||
RUN corepack enable
|
||||
|
||||
# Copy only the extracted package.jsons
|
||||
COPY --from=extractor /app ./
|
||||
|
||||
# Install dependencies with cache mount for pnpm store
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --filter @worldwideview/wwv-data-engine...
|
||||
|
||||
# Copy source and build
|
||||
COPY packages/wwv-data-engine ./packages/wwv-data-engine
|
||||
RUN pnpm --filter @worldwideview/wwv-data-engine build
|
||||
|
||||
# Production Stage
|
||||
FROM node:22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Enable corepack for pnpm
|
||||
RUN corepack enable
|
||||
|
||||
# Copy only the extracted package.jsons
|
||||
COPY --from=extractor /app ./
|
||||
|
||||
# Install ONLY production dependencies
|
||||
# Need python/make for building native modules during install
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
|
||||
apk add --no-cache python3 make g++ && \
|
||||
pnpm install --prod --filter @worldwideview/wwv-data-engine... && \
|
||||
apk del python3 make g++
|
||||
|
||||
# Copy built artifacts from builder
|
||||
COPY --from=builder /app/packages/wwv-data-engine/dist ./packages/wwv-data-engine/dist
|
||||
COPY --from=builder /app/packages/wwv-data-engine/seedData ./packages/wwv-data-engine/seedData
|
||||
|
||||
# Copy prisma schema and generate client for production
|
||||
COPY --from=builder /app/packages/wwv-data-engine/prisma ./packages/wwv-data-engine/prisma
|
||||
RUN cd packages/wwv-data-engine && npx prisma generate
|
||||
|
||||
WORKDIR /app/packages/wwv-data-engine
|
||||
|
||||
# Create directory for SQLite DB
|
||||
RUN mkdir -p /app/packages/wwv-data-engine/data
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=5001
|
||||
|
||||
EXPOSE 5001
|
||||
|
||||
CMD ["node", "dist/server.js"]
|
||||
@@ -1,173 +0,0 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import path from 'path';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, '../../.env.local') });
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
async function main() {
|
||||
console.log('[Migration] Starting migration from local engine.db to Supabase');
|
||||
|
||||
const dbPath = process.argv[2] || path.join(process.cwd(), 'data', 'engine.db');
|
||||
console.log(`[Migration] Reading from SQLite database at: ${dbPath}`);
|
||||
|
||||
const db = new Database(dbPath, { readonly: true });
|
||||
|
||||
// 1. Migrate IranwarEvent
|
||||
try {
|
||||
console.log('[Migration] Migrating iranwar_events...');
|
||||
const events = db.prepare('SELECT event_id, payload, timestamp, fetched_at FROM iranwar_events').all();
|
||||
console.log(`[Migration] Found ${events.length} iranwar_events. Iterating...`);
|
||||
let count = 0;
|
||||
for (const evt of events as any[]) {
|
||||
const parsedTs = typeof evt.timestamp === 'string' ? new Date(evt.timestamp).getTime() : (evt.timestamp || Date.now());
|
||||
await prisma.iranwarEvent.upsert({
|
||||
where: { eventId: evt.event_id },
|
||||
create: {
|
||||
eventId: evt.event_id,
|
||||
payload: evt.payload,
|
||||
timestamp: BigInt(parsedTs),
|
||||
fetchedAt: BigInt(evt.fetched_at || Date.now())
|
||||
},
|
||||
update: {
|
||||
payload: evt.payload,
|
||||
timestamp: BigInt(parsedTs),
|
||||
fetchedAt: BigInt(evt.fetched_at || Date.now())
|
||||
}
|
||||
});
|
||||
count++;
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated ${count} iranwar_events.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed iranwar_events: ${e.message}`);
|
||||
}
|
||||
|
||||
// 2. Migrate Earthquake
|
||||
try {
|
||||
console.log('[Migration] Migrating earthquakes...');
|
||||
const quakes = db.prepare('SELECT id, payload, source_ts, fetched_at FROM earthquakes').all();
|
||||
console.log(`[Migration] Found ${quakes.length} earthquakes. Migrating in bulk...`);
|
||||
const qData = quakes.map((q: any) => ({
|
||||
id: String(q.id),
|
||||
payload: q.payload,
|
||||
sourceTs: BigInt(q.source_ts || Date.now()),
|
||||
fetchedAt: BigInt(q.fetched_at || Date.now())
|
||||
}));
|
||||
if (qData.length > 0) {
|
||||
await prisma.earthquake.createMany({ data: qData, skipDuplicates: true });
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated earthquakes.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed earthquakes: ${e.message}`);
|
||||
}
|
||||
|
||||
// 3. Migrate Wildfire
|
||||
try {
|
||||
console.log('[Migration] Migrating wildfires...');
|
||||
const fires = db.prepare('SELECT id, payload, source_ts, fetched_at FROM wildfires').all();
|
||||
console.log(`[Migration] Found ${fires.length} wildfires. Migrating in bulk...`);
|
||||
const fData = fires.map((f: any) => ({
|
||||
id: String(f.id),
|
||||
payload: f.payload,
|
||||
sourceTs: BigInt(f.source_ts || Date.now()),
|
||||
fetchedAt: BigInt(f.fetched_at || Date.now())
|
||||
}));
|
||||
if (fData.length > 0) {
|
||||
await prisma.wildfire.createMany({ data: fData, skipDuplicates: true });
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated wildfires.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed wildfires: ${e.message}`);
|
||||
}
|
||||
|
||||
// 4. Migrate MaritimeHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating maritime_history...');
|
||||
const maritime = db.prepare('SELECT mmsi, ts, lat, lon, hdg, spd, fetched_at FROM maritime_history').all();
|
||||
console.log(`[Migration] Found ${maritime.length} maritime_history records. Migrating in bulk...`);
|
||||
|
||||
// Chunk it in case it's huge
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < maritime.length; i += chunkSize) {
|
||||
const chunk = maritime.slice(i, i + chunkSize).map((x: any) => ({
|
||||
mmsi: String(x.mmsi),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.maritimeHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${maritime.length} maritime_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated maritime_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed maritime_history: ${e.message}`);
|
||||
}
|
||||
|
||||
// 5. Migrate AviationHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating aviation_history...');
|
||||
const aviation = db.prepare('SELECT icao24, ts, lat, lon, alt, hdg, spd, fetched_at FROM aviation_history').all();
|
||||
console.log(`[Migration] Found ${aviation.length} aviation_history records. Migrating in bulk...`);
|
||||
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < aviation.length; i += chunkSize) {
|
||||
const chunk = aviation.slice(i, i + chunkSize).map((x: any) => ({
|
||||
icao24: String(x.icao24),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
alt: x.alt,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.aviationHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${aviation.length} aviation_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated aviation_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed aviation_history: ${e.message}`);
|
||||
}
|
||||
|
||||
// 6. Migrate MilitaryAviationHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating military_aviation_history...');
|
||||
const maviation = db.prepare('SELECT hex, ts, lat, lon, alt, hdg, spd, fetched_at FROM military_aviation_history').all();
|
||||
console.log(`[Migration] Found ${maviation.length} military_aviation_history records. Migrating in bulk...`);
|
||||
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < maviation.length; i += chunkSize) {
|
||||
const chunk = maviation.slice(i, i + chunkSize).map((x: any) => ({
|
||||
hex: String(x.hex),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
alt: x.alt,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.militaryAviationHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${maviation.length} military_aviation_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated military_aviation_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed military_aviation_history: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log('[Migration] Completed migration from local SQLite to Supabase!');
|
||||
}
|
||||
|
||||
main()
|
||||
.catch(e => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
@@ -1,61 +0,0 @@
|
||||
[Migration] Starting migration from local engine.db to Supabase
|
||||
[Migration] Reading from SQLite database at: C:\dev\worldwideview\packages\wwv-data-engine\data\engine.db
|
||||
[Migration] Migrating iranwar_events...
|
||||
[Migration] Found 242 iranwar_events. Iterating...
|
||||
[Migration] Skipped or failed iranwar_events:
|
||||
Invalid `prisma.iranwarEvent.upsert()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:31:39
|
||||
|
||||
28 console.log(`[Migration] Found ${events.length} iranwar_events. Iterating...`);
|
||||
29 let count = 0;
|
||||
30 for (const evt of events as any[]) {
|
||||
→ 31 await prisma.iranwarEvent.upsert({
|
||||
where: {
|
||||
eventId: "day-30-evt-0200z"
|
||||
},
|
||||
create: {
|
||||
eventId: "day-30-evt-0200z",
|
||||
[Migration] Migrating earthquakes...
|
||||
payload: "{\"event_id\":\"day-30-evt-0200z\",\"type\":\"AIR STRIKE\",\"location\":\"Tehran, Iran\",\"timestamp\":\"2026-03-28T02:00:00.000Z\",\"confidence\":\"Confirmed\",\"event_summary\":\"The combined US-Israeli force conducted precision strikes in Tehran targeting two facilities. The Marine Industries Organization ( ), which oversees 's naval weapons systems including missile-equipped fast boats and drone vessels critical to the blockade, sustained direct hits. A second strike targeted the Physics Department of 's University of Science and Technology ( ), a facility assessed by Western intelligence agencies as linked to nuclear and research programs. The described the strikes as targeting 's capacity to sustain offensive naval and ballistic operations. No casualty figures were released for either site.\",\"source_url\":\"https://www.msn.com/en-in/news/world/sea-of-mourners-gather-in-tehran-for-irgc-commander-alireza-tangsiri-s-funeral/ar-AA1ZVr9D\",\"preview_image\":null,\"_osint_meta\":{\"casualties\":0,\"coordinates\":{\"lat\":35.6892523,\"lng\":51.3896004}}}",
|
||||
timestamp: "2026-03-28T02:00:00.000Z",
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
fetchedAt: 1775088300955
|
||||
},
|
||||
update: {
|
||||
payload: "{\"event_id\":\"day-30-evt-0200z\",\"type\":\"AIR STRIKE\",\"location\":\"Tehran, Iran\",\"timestamp\":\"2026-03-28T02:00:00.000Z\",\"confidence\":\"Confirmed\",\"event_summary\":\"The combined US-Israeli force conducted precision strikes in Tehran targeting two facilities. The Marine Industries Organization ( ), which oversees 's naval weapons systems including missile-equipped fast boats and drone vessels critical to the blockade, sustained direct hits. A second strike targeted the Physics Department of 's University of Science and Technology ( ), a facility assessed by Western intelligence agencies as linked to nuclear and research programs. The described the strikes as targeting 's capacity to sustain offensive naval and ballistic operations. No casualty figures were released for either site.\",\"source_url\":\"https://www.msn.com/en-in/news/world/sea-of-mourners-gather-in-tehran-for-irgc-commander-alireza-tangsiri-s-funeral/ar-AA1ZVr9D\",\"preview_image\":null,\"_osint_meta\":{\"casualties\":0,\"coordinates\":{\"lat\":35.6892523,\"lng\":51.3896004}}}",
|
||||
timestamp: "2026-03-28T02:00:00.000Z",
|
||||
fetchedAt: 1775088300955
|
||||
}
|
||||
})
|
||||
|
||||
Argument `timestamp`: Invalid value provided. Expected Int, provided String.
|
||||
[Migration] Found 145 earthquakes. Migrating in bulk...
|
||||
[Migration] Skipped or failed earthquakes:
|
||||
Invalid `prisma.earthquake.createMany()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:64:37
|
||||
|
||||
61 fetchedAt: q.fetched_at
|
||||
62 }));
|
||||
63 if (qData.length > 0) {
|
||||
→ 64 await prisma.earthquake.createMany(
|
||||
Error occurred during query execution:
|
||||
ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(Error { kind: ToSql(2), cause: Some(Error { kind: ConversionError("Unable to fit integer value '1775077787921' into an INT4 (32-bit signed integer)."), original_code: None, original_message: None }) }), transient: false })
|
||||
[Migration] Migrating wildfires...
|
||||
[Migration] Found 64649 wildfires. Migrating in bulk...
|
||||
[Migration] Skipped or failed wildfires:
|
||||
Invalid `prisma.wildfire.createMany()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:83:35
|
||||
|
||||
80 fetchedAt: f.fetched_at
|
||||
81 }));
|
||||
82 if (fData.length > 0) {
|
||||
→ 83 await prisma.wildfire.createMany(
|
||||
Error occurred during query execution:
|
||||
ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(Error { kind: ToSql(2), cause: Some(Error { kind: ConversionError("Unable to fit integer value '1774915260000' into an INT4 (32-bit signed integer)."), original_code: None, original_message: None }) }), transient: false })
|
||||
[Migration] Migrating maritime_history...
|
||||
[Migration] Skipped or failed maritime_history: no such column: heading
|
||||
[Migration] Migrating aviation_history...
|
||||
[Migration] Skipped or failed aviation_history: no such column: heading
|
||||
[Migration] Migrating military_aviation_history...
|
||||
[Migration] Completed migration from local SQLite to Supabase!
|
||||
[Migration] Skipped or failed military_aviation_history: no such column: heading
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-data-engine",
|
||||
"version": "1.2.0",
|
||||
"description": "WWV Data Engine - Dual-tier Shared Backend",
|
||||
"main": "dist/server.js",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"build": "prisma generate && tsc",
|
||||
"start": "node dist/server.js",
|
||||
"prisma:generate": "prisma generate",
|
||||
"prisma:push": "prisma db push"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.2.0",
|
||||
"@prisma/client": "^6.4.1",
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"csv-parse": "^5.6.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"fastify": "^5.0.0",
|
||||
"geoip-lite": "^2.0.1",
|
||||
"ioredis": "^5.4.1",
|
||||
"node-cron": "^3.0.3",
|
||||
"prisma": "^6.4.1",
|
||||
"sax": "^1.6.0",
|
||||
"ws": "^8.18.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.11",
|
||||
"@types/geoip-lite": "^1.4.4",
|
||||
"@types/node": "^22.0.0",
|
||||
"@types/node-cron": "^3.0.11",
|
||||
"@types/sax": "^1.2.7",
|
||||
"@types/ws": "^8.5.12",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.2"
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
let dbUrl = process.env.SUPABASE_DATABASE_URL;
|
||||
|
||||
if (!dbUrl) {
|
||||
try {
|
||||
const envPath = path.resolve(process.cwd(), "../../.env.local");
|
||||
if (fs.existsSync(envPath)) {
|
||||
const envContent = fs.readFileSync(envPath, "utf8");
|
||||
const match = envContent.match(/^SUPABASE_DATABASE_URL=(.*)$/m);
|
||||
if (match) {
|
||||
dbUrl = match[1].trim();
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
datasource: {
|
||||
url: dbUrl,
|
||||
},
|
||||
};
|
||||
@@ -1,84 +0,0 @@
|
||||
// This is your Prisma schema file for the WWV Data Engine
|
||||
// It connects to Supabase to store historical timeline data.
|
||||
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
}
|
||||
|
||||
model IranwarEvent {
|
||||
eventId String @id @map("event_id")
|
||||
payload String // Stored as JSON string to mirror existing SQLite structure
|
||||
timestamp BigInt
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([timestamp])
|
||||
@@map("iranwar_events")
|
||||
}
|
||||
|
||||
model Earthquake {
|
||||
id String @id
|
||||
payload String // Stored as JSON string
|
||||
sourceTs BigInt @map("source_ts")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([sourceTs])
|
||||
@@map("earthquakes")
|
||||
}
|
||||
|
||||
model Wildfire {
|
||||
id String @id
|
||||
payload String // Stored as JSON string
|
||||
sourceTs BigInt @map("source_ts")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([sourceTs])
|
||||
@@map("wildfires")
|
||||
}
|
||||
|
||||
model MaritimeHistory {
|
||||
mmsi String
|
||||
ts BigInt
|
||||
lat Float
|
||||
lon Float
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([mmsi, ts])
|
||||
@@index([ts])
|
||||
@@map("maritime_history")
|
||||
}
|
||||
|
||||
model AviationHistory {
|
||||
icao24 String
|
||||
ts BigInt
|
||||
lat Float?
|
||||
lon Float?
|
||||
alt Float?
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([icao24, ts])
|
||||
@@index([ts])
|
||||
@@map("aviation_history")
|
||||
}
|
||||
|
||||
model MilitaryAviationHistory {
|
||||
hex String
|
||||
ts BigInt
|
||||
lat Float?
|
||||
lon Float?
|
||||
alt Float?
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([hex, ts])
|
||||
@@index([ts])
|
||||
@@map("military_aviation_history")
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
Loaded Prisma config from prisma.config.ts.
|
||||
|
||||
Prisma schema loaded from prisma\schema.prisma.
|
||||
Datasource "db": PostgreSQL database "postgres", schema "public" at "db.oqtfnwsefozpvsoysufm.supabase.co:5432"
|
||||
|
||||
Error: P1001: Can't reach database server at `db.oqtfnwsefozpvsoysufm.supabase.co:5432`
|
||||
|
||||
Please make sure your database server is running at `db.oqtfnwsefozpvsoysufm.supabase.co:5432`.
|
||||
@@ -1,48 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const csvPath = 'C:/dev/worldwideview/artifacts/Events.csv';
|
||||
const lines = fs.readFileSync(csvPath, 'utf-8').trim().split('\n');
|
||||
|
||||
const events = [];
|
||||
const parseCSVLine = (text) => {
|
||||
let ret = [], q = false, cv = '';
|
||||
for (let c of text) {
|
||||
if (c === '\"') { q = !q; }
|
||||
else if (c === ',' && !q) { ret.push(cv); cv = ''; }
|
||||
else { cv += c; }
|
||||
}
|
||||
ret.push(cv);
|
||||
return ret;
|
||||
};
|
||||
|
||||
for(const line of lines) {
|
||||
const parts = parseCSVLine(line);
|
||||
if(parts.length < 10) continue;
|
||||
|
||||
const [time, date, type, location, casualties, sourceUrl, sourceTitle, summary, eventId, day, articleUrl] = parts;
|
||||
|
||||
// time is like '02:00Z', date is like '28-Mar-26'
|
||||
const isoDate = new Date(`${date} ${time.replace('Z', '')} UTC`).toISOString();
|
||||
|
||||
events.push({
|
||||
event_id: eventId,
|
||||
type: type,
|
||||
location: location,
|
||||
timestamp: isoDate,
|
||||
confidence: 'Confirmed',
|
||||
event_summary: summary,
|
||||
source_url: articleUrl,
|
||||
preview_image: null,
|
||||
_osint_meta: {
|
||||
casualties: parseInt(casualties) || 0,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const outDir = 'C:/dev/worldwideview/packages/wwv-data-engine/seedData';
|
||||
if (!fs.existsSync(outDir)) {
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(path.join(outDir, 'iranwar_seed.json'), JSON.stringify(events, null, 2));
|
||||
console.log('Saved ' + events.length + ' events to seed.json');
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,207 +0,0 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
|
||||
// Define the path for the SQLite database. Note Docker volume mounts to /app/packages/wwv-data-engine/data
|
||||
const dbPath = process.env.DB_PATH || path.join(__dirname, '..', 'data', 'engine.db');
|
||||
|
||||
export const db = new Database(dbPath, {
|
||||
// Use verbose logging if needed for debugging
|
||||
// verbose: console.log
|
||||
});
|
||||
|
||||
// Enable Write-Ahead Logging for better concurrent read/write performance
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
db.pragma('busy_timeout = 5000'); // Wait up to 5 seconds if DB is locked
|
||||
|
||||
/**
|
||||
* Initialize all required tables for the different seeders.
|
||||
* This runs synchronously on boot.
|
||||
*/
|
||||
export function initDB() {
|
||||
console.log(`[DB] Initializing SQLite database at ${dbPath}`);
|
||||
|
||||
// IranWarLive table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS iranwar_events (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Earthquakes table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS earthquakes (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Wildfires table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS wildfires (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Maritime AIS history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS maritime_history (
|
||||
mmsi TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (mmsi, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast maritime history lookups by MMSI + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_maritime_history_mmsi_ts ON maritime_history(mmsi, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_maritime_history_ts ON maritime_history(ts);
|
||||
`);
|
||||
|
||||
// Aviation history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS aviation_history (
|
||||
icao24 TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
alt REAL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (icao24, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast aviation history lookups by ICAO24 + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_aviation_history_icao24_ts ON aviation_history(icao24, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_aviation_history_ts ON aviation_history(ts);
|
||||
`);
|
||||
|
||||
// Military Aviation history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS military_aviation_history (
|
||||
hex TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
alt REAL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (hex, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast military aviation history lookups by hex + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_military_aviation_history_hex_ts ON military_aviation_history(hex, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_military_aviation_history_ts ON military_aviation_history(ts);
|
||||
`);
|
||||
|
||||
// GPS Jamming daily interference map
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS gps_jamming (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Conflict Events (ACLED) DB table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS conflict_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Civil Unrest DB table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS civil_unrest (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Cyber Attacks table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS cyber_attacks (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for cyber attacks time-range queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_cyber_attacks_source_ts ON cyber_attacks(source_ts);
|
||||
`);
|
||||
|
||||
// Sanctions table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS sanctions (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
console.log('[DB] All tables initialized successfully.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune history rows older than the retention period.
|
||||
* Runs periodically to prevent unbounded table growth.
|
||||
*/
|
||||
const RETENTION_HOURS = 24;
|
||||
|
||||
export function pruneHistoryTables() {
|
||||
const cutoff = Math.floor(Date.now() / 1000) - RETENTION_HOURS * 3600;
|
||||
|
||||
const tables = [
|
||||
'aviation_history',
|
||||
'military_aviation_history',
|
||||
'maritime_history',
|
||||
];
|
||||
|
||||
for (const table of tables) {
|
||||
const result = db.prepare(`DELETE FROM ${table} WHERE ts < ?`).run(cutoff);
|
||||
if (result.changes > 0) {
|
||||
console.log(`[DB] Pruned ${result.changes} rows from ${table}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import geoip from 'geoip-lite';
|
||||
|
||||
export interface GeoLocation {
|
||||
lat: number;
|
||||
lon: number;
|
||||
country: string;
|
||||
city: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Geolocate an IPv4 address using the local geoip-lite database.
|
||||
* Returns null for private/unresolvable IPs.
|
||||
*/
|
||||
export function geolocateIp(ip: string): GeoLocation | null {
|
||||
const lookup = geoip.lookup(ip);
|
||||
if (!lookup || !lookup.ll) return null;
|
||||
|
||||
return {
|
||||
lat: lookup.ll[0],
|
||||
lon: lookup.ll[1],
|
||||
country: lookup.country || 'Unknown',
|
||||
city: lookup.city || 'Unknown',
|
||||
};
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var prismaDataEngine: PrismaClient | undefined;
|
||||
}
|
||||
|
||||
export const prisma =
|
||||
globalThis.prismaDataEngine ||
|
||||
new PrismaClient();
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') globalThis.prismaDataEngine = prisma;
|
||||
@@ -1,96 +0,0 @@
|
||||
import Redis from 'ioredis';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import zlib from 'zlib';
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env.local') });
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env') });
|
||||
|
||||
// ioredis gracefully handles standard redis:// as well as rediss:// (TLS, used by Upstash)
|
||||
const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
|
||||
|
||||
if (redisUrl.includes('upstash.io') && redisUrl.startsWith('redis://')) {
|
||||
console.error('\n\x1b[31m[CRITICAL CONFIG ERROR]\x1b[0m 🚨 You are connecting to Upstash without TLS.');
|
||||
console.error('Upstash requires the rediss:// protocol. Please update your REDIS_URL in .env.local to start with rediss:// instead of redis://\n');
|
||||
}
|
||||
|
||||
console.log(`[Redis] Connecting to ${redisUrl.replace(/:[^:@]+@/, ':***@')} ...`);
|
||||
|
||||
export const redis = new Redis(redisUrl, {
|
||||
// Common reconnect strategy
|
||||
retryStrategy(times) {
|
||||
const delay = Math.min(times * 50, 2000);
|
||||
return delay;
|
||||
},
|
||||
maxRetriesPerRequest: 3,
|
||||
});
|
||||
|
||||
redis.on('error', (err) => {
|
||||
console.error('[Redis] Connection Error against URL:', redisUrl.replace(/:[^:@]+@/, ':***@'));
|
||||
console.error('[Redis] Error Object:', err);
|
||||
});
|
||||
|
||||
redis.on('ready', () => {
|
||||
console.log('[Redis] Connected and ready.');
|
||||
});
|
||||
|
||||
import { broadcastPluginData } from './websocket';
|
||||
|
||||
const lastSnapshotTimes = new Map<string, number>();
|
||||
const SNAPSHOT_THROTTLE_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
/**
|
||||
* Convenience method to write a JSON payload to Redis with an expiration.
|
||||
* Writes are throttled to save Redis requests, but websockets are always broadcasted.
|
||||
*/
|
||||
export async function setLiveSnapshot(source: string, payload: any, ttlSeconds: number) {
|
||||
try {
|
||||
// 1. ALWAYS broadcast newly updated entities to any active WebSocket subscribers
|
||||
// The data pipeline relies on this for high-frequency HUD updates.
|
||||
broadcastPluginData(source, payload);
|
||||
|
||||
// 2. Throttle Redis snapshots to prevent blowing past 500K max request limits (e.g. Upstash)
|
||||
const now = Date.now();
|
||||
const lastTime = lastSnapshotTimes.get(source) || 0;
|
||||
if (now - lastTime < SNAPSHOT_THROTTLE_MS) {
|
||||
return; // Skip Redis write
|
||||
}
|
||||
lastSnapshotTimes.set(source, now);
|
||||
|
||||
const key = `data:${source}:live`;
|
||||
const jsonStr = JSON.stringify(payload);
|
||||
|
||||
// Compress JSON strings to drastically reduce Upstash payload size
|
||||
const compressed = zlib.gzipSync(Buffer.from(jsonStr, 'utf-8'));
|
||||
await redis.set(key, compressed as any, 'EX', ttlSeconds);
|
||||
|
||||
// Also save a metadata key so healthcheck can verify
|
||||
await redis.set(`meta:${source}:last_run`, Date.now().toString(), 'EX', ttlSeconds * 2);
|
||||
|
||||
console.log(`[Redis] Snapshot saved to Redis for ${source} (${(compressed.length / 1024).toFixed(2)} KB)`);
|
||||
} catch (error) {
|
||||
console.error(`[Redis] Failed to snapshot ${source}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method to read a JSON payload from Redis.
|
||||
*/
|
||||
export async function getLiveSnapshot(source: string) {
|
||||
const key = `data:${source}:live`;
|
||||
try {
|
||||
// We must use getBuffer natively in ioredis to prevent it from coercing raw bytes to a utf8 string
|
||||
const data = await redis.getBuffer(key);
|
||||
if (!data) return null;
|
||||
|
||||
try {
|
||||
const decompressed = zlib.unzipSync(data);
|
||||
return JSON.parse(decompressed.toString('utf-8'));
|
||||
} catch {
|
||||
// Fallback if data was stored as plain string before compression was introduced
|
||||
return JSON.parse(data.toString('utf-8'));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Redis] Failed to get live snapshot ${source}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/aviation', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT icao24, ts, lat, lon, hdg, spd, alt
|
||||
FROM aviation_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.icao24);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.icao24, { ...r, on_ground: r.alt <= 0 });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "15m" -> 900s, "1h" -> 3600s)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('aviation')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(plane) => nowTs - plane.last_updated <= maxAge
|
||||
);
|
||||
|
||||
return {
|
||||
source: 'aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[Aviation Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -1,28 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM civil_unrest WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
fastify.get('/data/civil_unrest', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('civil_unrest');
|
||||
if (liveData) {
|
||||
return { data: liveData };
|
||||
}
|
||||
return { data: [] };
|
||||
});
|
||||
|
||||
fastify.get('/data/civil_unrest/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "civil_unrest",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -1,28 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM conflict_events WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
fastify.get('/data/conflict_events', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('conflict_events');
|
||||
if (liveData) {
|
||||
return { data: liveData };
|
||||
}
|
||||
return { data: [] };
|
||||
});
|
||||
|
||||
fastify.get('/data/conflict_events/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "conflict_events",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -1,45 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare(
|
||||
'SELECT payload FROM cyber_attacks WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC'
|
||||
);
|
||||
|
||||
// Standard live endpoint → returns Redis snapshot
|
||||
fastify.get('/data/cyber_attacks', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('cyber_attacks');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: 'cyber_attacks',
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
};
|
||||
});
|
||||
|
||||
// History endpoint → queries SQLite
|
||||
fastify.get('/data/cyber_attacks/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply
|
||||
.status(400)
|
||||
.send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({
|
||||
start: parseInt(start, 10),
|
||||
end: parseInt(end, 10),
|
||||
}) as { payload: string }[];
|
||||
const historyItems = rows.map((row) => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: 'cyber_attacks',
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length,
|
||||
};
|
||||
});
|
||||
@@ -1,38 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM earthquakes WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/earthquakes', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('earthquakes');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "earthquakes",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/earthquakes/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: "earthquakes",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -1,33 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM gps_jamming WHERE source_ts >= @start AND source_ts <= @end');
|
||||
|
||||
fastify.get('/data/gps_jamming', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('gps_jamming');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "gps_jamming",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
fastify.get('/data/gps_jamming/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "gps_jamming",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -1,13 +0,0 @@
|
||||
import './iranwarlive';
|
||||
import './earthquakes';
|
||||
import './wildfires';
|
||||
import './maritime';
|
||||
import './aviation';
|
||||
import './military-aviation';
|
||||
import './satellite';
|
||||
import './gpsjam';
|
||||
import './conflictEvents';
|
||||
import './civilUnrest';
|
||||
import './cyberAttacks';
|
||||
import './sanctions';
|
||||
import './surveillanceSat';
|
||||
@@ -1,37 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getAllEventsQuery = db.prepare('SELECT payload FROM iranwar_events ORDER BY timestamp DESC');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/iranwarlive', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('iranwarlive');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
|
||||
// Fallback to SQLite if Redis is empty/cleared
|
||||
const rows = getAllEventsQuery.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "iranwarlive",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: history
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/iranwarlive/history', async (request, reply) => {
|
||||
// Iran war live has small enough history that we can return it all.
|
||||
// In a massive set, we would use query params: ?start=ts&end=ts
|
||||
const rows = getAllEventsQuery.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
version: "2.0",
|
||||
source: "iranwarlive",
|
||||
last_updated: new Date().toISOString(),
|
||||
items: history
|
||||
};
|
||||
});
|
||||
@@ -1,96 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/maritime', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT mmsi, ts, lat, lon, hdg, spd
|
||||
FROM maritime_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.mmsi);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.mmsi, r);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'maritime',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "6h" -> seconds)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('maritime')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(ship) => nowTs - ship.last_updated <= maxAge
|
||||
);
|
||||
|
||||
// If lookback was requested, grab history points from SQLite
|
||||
if (lookbackSeconds > 0 && items.length > 0) {
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT mmsi, ts, lat, lon
|
||||
FROM maritime_history
|
||||
WHERE ts >= @startTs
|
||||
`);
|
||||
const allHistory = historyQuery.all({ startTs: nowTs - lookbackSeconds }) as any[];
|
||||
|
||||
const historyMap = new Map<string, any[]>();
|
||||
for (const row of allHistory) {
|
||||
if (!historyMap.has(row.mmsi)) historyMap.set(row.mmsi, []);
|
||||
historyMap.get(row.mmsi)!.push({ lat: row.lat, lon: row.lon, ts: row.ts });
|
||||
}
|
||||
|
||||
for (const ship of items) {
|
||||
const pts = historyMap.get(ship.mmsi);
|
||||
if (pts && pts.length > 0) {
|
||||
// Sort chronologically ascending
|
||||
pts.sort((a, b) => a.ts - b.ts);
|
||||
ship.history = pts;
|
||||
} else {
|
||||
ship.history = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'maritime',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[Maritime Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -1,69 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/military-aviation', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT hex, ts, lat, lon, hdg, spd, alt
|
||||
FROM military_aviation_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.hex);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.hex, { ...r, on_ground: r.alt <= 0 });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'military-aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
ac: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "15m" -> 900s, "1h" -> 3600s)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('military-aviation')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(plane) => nowTs - plane.last_updated <= maxAge
|
||||
);
|
||||
|
||||
return {
|
||||
source: 'military-aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
ac: items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[MilitaryAviation Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -1,15 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
// Sanctions are Redis-only (no SQLite table) — live data only
|
||||
fastify.get('/data/sanctions', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('sanctions');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
id: 'sanctions-live',
|
||||
timestamp: null,
|
||||
items: []
|
||||
};
|
||||
});
|
||||
@@ -1,54 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
import { globalsTLECache, propagateAll, DEFAULT_GROUPS } from '../seeders/satellite';
|
||||
|
||||
fastify.get('/data/satellite', async (request: any, reply) => {
|
||||
const { lookback, time, group } = request.query;
|
||||
|
||||
const targetGroups = group ? [group as string] : DEFAULT_GROUPS;
|
||||
|
||||
// Historical playback mode snapshot using TLE on the fly
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTime = new Date(parseInt(time, 10));
|
||||
|
||||
let items: any[] = [];
|
||||
const seen = new Set<number>();
|
||||
|
||||
for (const g of targetGroups) {
|
||||
const records = globalsTLECache.get(g);
|
||||
if (!records) continue;
|
||||
|
||||
const positions = propagateAll(records, targetTime, g);
|
||||
for (const p of positions) {
|
||||
if (!seen.has(p.noradId)) {
|
||||
items.push(p);
|
||||
seen.add(p.noradId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: "satellite",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
};
|
||||
}
|
||||
|
||||
// 1. Get hot fleet from Redis for current time
|
||||
const fleetObj = await getLiveSnapshot('satellite') || {};
|
||||
let items = Object.values(fleetObj) as any[];
|
||||
|
||||
if (group) {
|
||||
items = items.filter(s => s.group === group);
|
||||
}
|
||||
|
||||
return {
|
||||
source: "satellite",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: null,
|
||||
items,
|
||||
totalCount: items.length
|
||||
};
|
||||
});
|
||||
@@ -1,11 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
// Surveillance Satellites split route
|
||||
fastify.get('/data/surveillance_satellites', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('surveillance_satellites');
|
||||
if (liveData) {
|
||||
return { satellites: Object.values(liveData) };
|
||||
}
|
||||
return { satellites: [] };
|
||||
});
|
||||
@@ -1,38 +0,0 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM wildfires WHERE source_ts >= @start AND source_ts <= @end');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/wildfires', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('wildfires');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "wildfires",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/wildfires/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: "wildfires",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -1,70 +0,0 @@
|
||||
import cron from 'node-cron';
|
||||
import { pruneHistoryTables } from './db';
|
||||
|
||||
// Define the interface for a seeder definition
|
||||
export interface SeederDefinition {
|
||||
name: string;
|
||||
cron?: string; // Optional: Cron expression if it runs on a schedule
|
||||
fn?: () => Promise<void>; // Optional: Function to run on the cron schedule
|
||||
init?: () => void; // Optional: Initialization function (e.g., for websockets)
|
||||
}
|
||||
|
||||
// Registry to hold all registered seeders
|
||||
let registeredSeeders: SeederDefinition[] = [];
|
||||
|
||||
// Registry to track the last run time of each cron seeder
|
||||
export const seederStatus: Record<string, number | null> = {};
|
||||
|
||||
/**
|
||||
* Register a seeder to be scheduled when the engine boots.
|
||||
*/
|
||||
export function registerSeeder(seeder: SeederDefinition) {
|
||||
registeredSeeders.push(seeder);
|
||||
if (seeder.cron) {
|
||||
seederStatus[seeder.name] = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler. Initializes websocket listeners and registers cron jobs.
|
||||
*/
|
||||
export function startScheduler() {
|
||||
console.log('[Scheduler] Starting data engine scheduler...');
|
||||
|
||||
for (const seeder of registeredSeeders) {
|
||||
// 1. Run init handlers (like websocket listeners)
|
||||
if (seeder.init) {
|
||||
console.log(`[Scheduler] Initializing persistent seeder: ${seeder.name}`);
|
||||
seeder.init();
|
||||
}
|
||||
|
||||
// 2. Schedule cron jobs
|
||||
if (seeder.cron && seeder.fn) {
|
||||
console.log(`[Scheduler] Scheduling cron seeder: ${seeder.name} (${seeder.cron})`);
|
||||
|
||||
cron.schedule(seeder.cron, async () => {
|
||||
try {
|
||||
console.log(`[Scheduler] Running seeder: ${seeder.name} ...`);
|
||||
await seeder.fn!();
|
||||
seederStatus[seeder.name] = Date.now();
|
||||
} catch (error: any) {
|
||||
console.error(`[Scheduler] Seeder ${seeder.name} failed:`, error.message);
|
||||
}
|
||||
});
|
||||
|
||||
// Kick off the first run immediately for cron jobs
|
||||
console.log(`[Scheduler] Kickstarting initial run for ${seeder.name}...`);
|
||||
seeder.fn().then(() => {
|
||||
seederStatus[seeder.name] = Date.now();
|
||||
}).catch((err) => {
|
||||
console.error(`[Scheduler] Initial run for ${seeder.name} failed:`, err.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Data retention: prune old history rows every hour
|
||||
cron.schedule('0 * * * *', () => {
|
||||
console.log('[Scheduler] Running data retention pruning...');
|
||||
pruneHistoryTables();
|
||||
});
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
const seedPath = path.join(__dirname, '..', '..', 'data', 'fallback', 'iranwar_seed.json');
|
||||
|
||||
async function delay(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function extractMedia(url: string): Promise<{img: string | null, vid: string | null, realUrl: string | null}> {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5"
|
||||
},
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
const text = await res.text();
|
||||
const $ = cheerio.load(text);
|
||||
|
||||
let realUrl = res.url;
|
||||
|
||||
// Sometimes Bing wraps in a redirect page
|
||||
if ($('noscript').text().includes('refresh')) {
|
||||
const redirectMatch = $('noscript').html()?.match(/URL=([^"']+)/i);
|
||||
if (redirectMatch && redirectMatch[1]) {
|
||||
const redir = redirectMatch[1].replace(/&/g, '&');
|
||||
try {
|
||||
const redirRes = await fetch(redir, { headers: { "User-Agent": "Mozilla/5.0" }, signal: AbortSignal.timeout(10000) });
|
||||
const redirHtml = await redirRes.text();
|
||||
const $2 = cheerio.load(redirHtml);
|
||||
realUrl = redirRes.url;
|
||||
return { ...parseMetas($2, realUrl), realUrl };
|
||||
} catch(e) {}
|
||||
}
|
||||
}
|
||||
|
||||
return { ...parseMetas($, realUrl), realUrl };
|
||||
} catch {
|
||||
return { img: null, vid: null, realUrl: null };
|
||||
}
|
||||
}
|
||||
|
||||
function parseMetas($: cheerio.CheerioAPI, source: string) {
|
||||
const img = $('meta[property="og:image"]').attr('content') || $('meta[name="twitter:image"]').attr('content') || null;
|
||||
const vid = $('meta[property="og:video:url"]').attr('content') || $('meta[property="og:video"]').attr('content') || null;
|
||||
return { img, vid };
|
||||
}
|
||||
|
||||
async function run() {
|
||||
console.log("Loading existing seed...");
|
||||
if (!fs.existsSync(seedPath)) {
|
||||
console.error("Seed not found!");
|
||||
return;
|
||||
}
|
||||
|
||||
const events: any[] = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
||||
let count = 0;
|
||||
|
||||
for (const event of events) {
|
||||
const isSearchable = !["POSTURING", "INTERCEPTION", "AIRSPACE CLOSED"].includes(event.type?.toUpperCase() || "");
|
||||
|
||||
if (isSearchable && event.location && event.type) {
|
||||
console.log(`[Hydrate] Searching Bing News for: ${event.type} ${event.location}`);
|
||||
const query = encodeURIComponent(`${event.type} ${event.location} Iran Israel conflict`);
|
||||
try {
|
||||
const searchRes = await fetch(`https://www.bing.com/news/search?q=${query}&format=rss`, {
|
||||
headers: { "User-Agent": "Mozilla/5.0" },
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (searchRes.ok) {
|
||||
const xml = await searchRes.text();
|
||||
const $ = cheerio.load(xml, { xmlMode: true });
|
||||
const items = $('item');
|
||||
if (items.length > 0) {
|
||||
const articleUrl = $(items[0]).find('link').text();
|
||||
console.log(` -> Found article string: ${articleUrl.substring(0, 50)}...`);
|
||||
const media = await extractMedia(articleUrl);
|
||||
|
||||
event.source_url = media.realUrl || articleUrl;
|
||||
if (media.vid) event.preview_video = media.vid;
|
||||
if (media.img) event.preview_image = media.img;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(` -> Search failed: ${err.message}`);
|
||||
}
|
||||
fs.writeFileSync(seedPath, JSON.stringify(events, null, 2));
|
||||
await delay(500);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Extraction complete. Modified ${count} events.`);
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
@@ -1,55 +0,0 @@
|
||||
export async function withRetry<T>(fn: () => Promise<T>, maxRetries = 3, delayMs = 1000): Promise<T> {
|
||||
let lastErr: Error | undefined;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err: any) {
|
||||
lastErr = err;
|
||||
if (attempt < maxRetries) {
|
||||
const wait = delayMs * Math.pow(2, attempt); // Exponential backoff
|
||||
console.warn(`[Retry] Attempt ${attempt + 1}/${maxRetries} failed: ${err.message}. Waiting ${wait}ms...`);
|
||||
await new Promise(r => setTimeout(r, wait));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
export async function fetchWithTimeout(url: string, options: RequestInit = {}, timeoutMs = 15000): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const id = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} - ${response.statusText}`);
|
||||
}
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates distance in Kilometers between two coordinates.
|
||||
* Useful for filtering events by proximity (e.g., nuclear test sites).
|
||||
*/
|
||||
export function haversineKm(lat1: number, lon1: number, lat2: number, lon2: number): number {
|
||||
const R = 6371; // km
|
||||
const dLat = (lat2 - lat1) * Math.PI / 180;
|
||||
const dLon = (lon2 - lon1) * Math.PI / 180;
|
||||
const a =
|
||||
Math.sin(dLat/2) * Math.sin(dLat/2) +
|
||||
Math.cos(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) *
|
||||
Math.sin(dLon/2) * Math.sin(dLon/2);
|
||||
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
|
||||
return R * c;
|
||||
}
|
||||
|
||||
export const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36';
|
||||
|
||||
export function sleep(ms: number) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
import cron from 'node-cron';
|
||||
import { db } from '../db';
|
||||
import { redis, setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const OPENSKY_TOKEN_URL = "https://auth.opensky-network.org/auth/realms/opensky-network/protocol/openid-connect/token";
|
||||
const OPENSKY_DATA_URL = "https://opensky-network.org/api/states/all";
|
||||
const ROTATION_THRESHOLD = 50;
|
||||
const POLLING_INTERVAL_MS = 5000;
|
||||
const FLUSH_INTERVAL_MS = 5000;
|
||||
|
||||
let messageBuffer: any[] = [];
|
||||
|
||||
interface OpenSkyCredential {
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
accessToken: string | null;
|
||||
tokenExpiry: number;
|
||||
creditsRemaining: number | null;
|
||||
exhausted: boolean;
|
||||
}
|
||||
|
||||
const pool = {
|
||||
_openskyPool: [] as OpenSkyCredential[],
|
||||
_openskyActiveIdx: 0,
|
||||
_lastResetTime: Date.now()
|
||||
};
|
||||
|
||||
function initCredentialPool(): void {
|
||||
if (pool._openskyPool.length > 0) return;
|
||||
|
||||
const creds: OpenSkyCredential[] = [];
|
||||
const raw = process.env.OPENSKY_CREDENTIALS;
|
||||
|
||||
if (raw) {
|
||||
for (const pair of raw.split(",")) {
|
||||
const trimmed = pair.trim();
|
||||
const colonIdx = trimmed.indexOf(":");
|
||||
if (colonIdx === -1) continue;
|
||||
const clientId = trimmed.slice(0, colonIdx);
|
||||
const clientSecret = trimmed.slice(colonIdx + 1);
|
||||
if (clientId && clientSecret) {
|
||||
creds.push({ clientId, clientSecret, accessToken: null, tokenExpiry: 0, creditsRemaining: null, exhausted: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (creds.length === 0) {
|
||||
// Fallback
|
||||
const clientId = process.env.OPENSKY_CLIENTID;
|
||||
const clientSecret = process.env.OPENSKY_CLIENTSECRET;
|
||||
if (clientId && clientSecret) {
|
||||
creds.push({ clientId, clientSecret, accessToken: null, tokenExpiry: 0, creditsRemaining: null, exhausted: false });
|
||||
}
|
||||
}
|
||||
|
||||
pool._openskyPool = creds;
|
||||
pool._openskyActiveIdx = 0;
|
||||
pool._lastResetTime = Date.now();
|
||||
console.log(`[Aviation] Initialised pool with ${creds.length} credential(s).`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets all credentials in the pool to non-exhausted state.
|
||||
*/
|
||||
function resetCredentialPool(): void {
|
||||
console.log('[Aviation] Resetting credential pool exhaustion flags...');
|
||||
for (const cred of pool._openskyPool) {
|
||||
cred.exhausted = false;
|
||||
cred.creditsRemaining = null;
|
||||
}
|
||||
pool._openskyActiveIdx = 0;
|
||||
pool._lastResetTime = Date.now();
|
||||
}
|
||||
|
||||
function getActiveCredential(): OpenSkyCredential | null {
|
||||
const creds = pool._openskyPool;
|
||||
if (!creds || creds.length === 0) return null;
|
||||
|
||||
const current = creds[pool._openskyActiveIdx];
|
||||
if (current && !current.exhausted) return current;
|
||||
|
||||
for (let i = 0; i < creds.length; i++) {
|
||||
if (!creds[i].exhausted) {
|
||||
pool._openskyActiveIdx = i;
|
||||
console.log(`[Aviation] Rotated → now using credential ${i + 1}/${creds.length}: ${creds[i].clientId}`);
|
||||
return creds[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Emergency Reset: If all are exhausted, but it's been > 12 hours since last reset,
|
||||
// try resetting everything once before giving up.
|
||||
const twelveHoursMs = 12 * 60 * 60 * 1000;
|
||||
if (Date.now() - pool._lastResetTime > twelveHoursMs) {
|
||||
console.warn(`[Aviation] Emergency Reset: All credentials exhausted, but last reset was > 12h ago. Attempting pool reset.`);
|
||||
resetCredentialPool();
|
||||
return getActiveCredential();
|
||||
}
|
||||
|
||||
console.warn(`[Aviation] All ${creds.length} credentials exhausted.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
function rotateCredential(): void {
|
||||
const current = pool._openskyPool[pool._openskyActiveIdx];
|
||||
if (current) current.exhausted = true;
|
||||
getActiveCredential();
|
||||
}
|
||||
|
||||
async function fetchTokenForCredential(cred: OpenSkyCredential): Promise<string | null> {
|
||||
const now = Date.now();
|
||||
if (cred.accessToken && now < cred.tokenExpiry) return cred.accessToken;
|
||||
|
||||
try {
|
||||
const res = await fetch(OPENSKY_TOKEN_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({
|
||||
grant_type: "client_credentials",
|
||||
client_id: cred.clientId,
|
||||
client_secret: cred.clientSecret,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) return null;
|
||||
|
||||
const data = await res.json();
|
||||
cred.accessToken = data.access_token;
|
||||
cred.tokenExpiry = now + data.expires_in * 1000 - 30_000;
|
||||
return cred.accessToken;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Poller ──────────────────────────────────────────────────
|
||||
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO aviation_history (icao24, ts, lat, lon, alt, hdg, spd, fetched_at)
|
||||
VALUES (@icao24, @ts, @lat, @lon, @alt, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function pollOpenSky() {
|
||||
const pollStart = Date.now();
|
||||
console.log(`[Aviation] Poll starting...`);
|
||||
const cred = getActiveCredential();
|
||||
let token = null;
|
||||
|
||||
if (cred) {
|
||||
token = await fetchTokenForCredential(cred);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (token) {
|
||||
headers["Authorization"] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 12000);
|
||||
const response = await fetch(OPENSKY_DATA_URL, { headers, signal: controller.signal });
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Handle limits
|
||||
if (response.status === 429) {
|
||||
console.warn('[Aviation] 429 Rate Limit hit.');
|
||||
if (cred) rotateCredential();
|
||||
return;
|
||||
}
|
||||
|
||||
const creditsRemainingStr = response.headers.get("x-ratelimit-remaining");
|
||||
if (creditsRemainingStr && cred) {
|
||||
const rem = parseInt(creditsRemainingStr, 10);
|
||||
cred.creditsRemaining = rem;
|
||||
if (rem <= ROTATION_THRESHOLD) rotateCredential();
|
||||
}
|
||||
|
||||
if (!response.ok) throw new Error(`Status ${response.status}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.states && Array.isArray(data.states)) {
|
||||
const activeStates = data.states.filter((s: any) => s[6] !== null && s[5] !== null);
|
||||
messageBuffer.push(activeStates);
|
||||
console.log(`[Aviation] Poll OK: ${activeStates.length} aircraft in ${Date.now() - pollStart}ms`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`[Aviation] Polling Error (${Date.now() - pollStart}ms): ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function flushBuffer() {
|
||||
if (messageBuffer.length === 0) return;
|
||||
|
||||
// We only take the most recent state array from the buffer queue completely resetting it
|
||||
const latestStates = messageBuffer.pop();
|
||||
messageBuffer = [];
|
||||
|
||||
if (!latestStates || latestStates.length === 0) return;
|
||||
|
||||
const fetchedAt = Math.floor(Date.now() / 1000);
|
||||
const fleetObj: Record<string, any> = Object.create(null);
|
||||
let insertedCount = 0;
|
||||
|
||||
const insertMany = db.transaction((states) => {
|
||||
for (const s of states) {
|
||||
const icao24 = s[0] as string;
|
||||
const callsign = (s[1] as string)?.trim() || null;
|
||||
const ts = s[3] as number || s[4] as number || fetchedAt; // time_position or last_contact
|
||||
const lon = s[5] as number;
|
||||
const lat = s[6] as number;
|
||||
const alt = s[7] as number; // baro_altitude
|
||||
const on_ground = s[8] as boolean;
|
||||
const spd = s[9] as number;
|
||||
const hdg = s[10] as number;
|
||||
|
||||
// 1. Write History to SQLite
|
||||
if (!on_ground) {
|
||||
const result = insertHistory.run({
|
||||
icao24, ts, lat, lon, alt, hdg, spd, fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
// 2. Prepare Live Cache structure
|
||||
// Zero out speed for grounded aircraft so frontend stops extrapolating
|
||||
const stateObj = {
|
||||
icao24,
|
||||
callsign,
|
||||
origin_country: s[2],
|
||||
time_position: s[3],
|
||||
last_contact: s[4],
|
||||
lon,
|
||||
lat,
|
||||
alt,
|
||||
on_ground,
|
||||
spd: on_ground ? 0 : spd,
|
||||
hdg: on_ground ? 0 : hdg,
|
||||
vertical_rate: s[11],
|
||||
sensors: s[12],
|
||||
geo_altitude: s[13],
|
||||
squawk: s[14],
|
||||
spi: s[15],
|
||||
position_source: s[16],
|
||||
last_updated: fetchedAt
|
||||
};
|
||||
|
||||
fleetObj[icao24] = stateObj;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
insertMany(latestStates);
|
||||
// Execute Redis batch via compressed snapshot (10 mins)
|
||||
await setLiveSnapshot('aviation', fleetObj, 10 * 60);
|
||||
} catch (err) {
|
||||
console.error('[Aviation] Buffer flush failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export function startAviationPoller() {
|
||||
initCredentialPool();
|
||||
console.log('[Aviation] Starting background polling...');
|
||||
|
||||
// Daily reset at midnight
|
||||
cron.schedule('0 0 * * *', () => {
|
||||
resetCredentialPool();
|
||||
});
|
||||
|
||||
setInterval(pollOpenSky, POLLING_INTERVAL_MS);
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
|
||||
// Initial fetch
|
||||
pollOpenSky();
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "aviation",
|
||||
init: startAviationPoller
|
||||
});
|
||||
@@ -1,146 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
interface GdeltFeature {
|
||||
type: string;
|
||||
geometry: { type: string, coordinates: number[] };
|
||||
properties: {
|
||||
urlpubtimedate: string;
|
||||
name: string;
|
||||
domain: string;
|
||||
url: string;
|
||||
urltone: number;
|
||||
};
|
||||
}
|
||||
|
||||
const GDELT_URL = 'http://api.gdeltproject.org/api/v1/gkg_geojson?query=protest OR riot OR demonstration OR strike OR clash&maxrows=2500';
|
||||
|
||||
function classifyGdeltEventType(name: string) {
|
||||
const lowerName = name.toLowerCase();
|
||||
if (lowerName.includes('riot')) return 'Riots';
|
||||
if (lowerName.includes('clash')) return 'Riots';
|
||||
if (lowerName.includes('strike')) return 'Strikes';
|
||||
if (lowerName.includes('demonstration')) return 'Demonstrations';
|
||||
return 'Protests';
|
||||
}
|
||||
|
||||
function classifyGdeltSubType(name: string, count: number) {
|
||||
const lowerName = name.toLowerCase();
|
||||
if (count > 50 || lowerName.includes('riot') || lowerName.includes('clash')) return 'Violent demonstration';
|
||||
if (lowerName.includes('strike')) return 'Labor strike';
|
||||
return 'Peaceful protest';
|
||||
}
|
||||
|
||||
const insertUnrest = db.prepare(
|
||||
'INSERT OR REPLACE INTO civil_unrest (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)'
|
||||
);
|
||||
|
||||
export async function seedCivilUnrest() {
|
||||
console.log('[CivilUnrest] Fetching from GDELT API...');
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(GDELT_URL, { headers: { 'User-Agent': 'WWV-Data-Engine' } }, 25000), 3, 5000);
|
||||
if (!res.ok) {
|
||||
console.warn(`[CivilUnrest] Failed to fetch. HTTP ${res.status}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const json = await res.json();
|
||||
const features = json.features as GdeltFeature[];
|
||||
|
||||
if (!features || features.length === 0) {
|
||||
console.log('[CivilUnrest] No events returned from GDELT.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Aggregate by location
|
||||
const locationMap = new Map<string, any>();
|
||||
for (const feature of features) {
|
||||
const name = feature.properties?.name || '';
|
||||
if (!name) continue;
|
||||
|
||||
const coords = feature.geometry?.coordinates;
|
||||
if (!Array.isArray(coords) || coords.length < 2) continue;
|
||||
|
||||
const [lon, lat] = coords;
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon) || lat < -90 || lat > 90 || lon < -180 || lon > 180) continue;
|
||||
|
||||
// Grid snap to ~11km clustering
|
||||
const key = `${lat.toFixed(1)}:${lon.toFixed(1)}`;
|
||||
const existing = locationMap.get(key);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
existing.urls.push(feature.properties.url);
|
||||
if (feature.properties.urltone < existing.worstTone) {
|
||||
existing.worstTone = feature.properties.urltone;
|
||||
}
|
||||
} else {
|
||||
locationMap.set(key, {
|
||||
name,
|
||||
lat,
|
||||
lon,
|
||||
count: 1,
|
||||
worstTone: feature.properties.urltone ?? 0,
|
||||
date: feature.properties.urlpubtimedate,
|
||||
urls: [feature.properties.url]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
const items: any[] = [];
|
||||
|
||||
for (const [, loc] of locationMap) {
|
||||
if (loc.count < 3) continue; // Filter noise
|
||||
|
||||
const country = loc.name.split(',').pop()?.trim() || "Unknown";
|
||||
const eventType = classifyGdeltEventType(loc.name);
|
||||
|
||||
// Create aggregated cluster event
|
||||
const item = {
|
||||
id: `gdelt-${loc.lat.toFixed(2)}-${loc.lon.toFixed(2)}`,
|
||||
lat: loc.lat,
|
||||
lon: loc.lon,
|
||||
type: eventType,
|
||||
subType: classifyGdeltSubType(loc.name, loc.count),
|
||||
actor1: "General Public",
|
||||
actor2: "N/A",
|
||||
fatalities: 0,
|
||||
country,
|
||||
location: loc.name,
|
||||
date: loc.date,
|
||||
source: "GDELT",
|
||||
notes: `${loc.count} clustered reports. Worst Tone: ${loc.worstTone.toFixed(1)}`,
|
||||
reportCount: loc.count
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
insertUnrest.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: new Date(item.date).getTime(),
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`[CivilUnrest] Clustered ${features.length} mentions into ${items.length} confirmed unrest events.`);
|
||||
|
||||
await setLiveSnapshot(
|
||||
'civil_unrest',
|
||||
{
|
||||
source: 'gdelt',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items,
|
||||
totalCount: items.length
|
||||
},
|
||||
86400
|
||||
);
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'civilUnrest',
|
||||
cron: '*/15 * * * *',
|
||||
fn: seedCivilUnrest
|
||||
});
|
||||
@@ -1,140 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import crypto from 'crypto';
|
||||
|
||||
interface ConflictEvent {
|
||||
id: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
type: string;
|
||||
subType: string;
|
||||
actor1: string;
|
||||
actor2: string;
|
||||
fatalities: number;
|
||||
date: string;
|
||||
source: string;
|
||||
notes: string;
|
||||
}
|
||||
|
||||
const CONFLICT_HOTSPOTS = [
|
||||
{ lat: 48.3794, lon: 31.1656, radius: 5 }, // Ukraine
|
||||
{ lat: 31.0461, lon: 34.8516, radius: 2 }, // Israel/Gaza
|
||||
{ lat: 15.5007, lon: 32.5599, radius: 6 }, // Sudan
|
||||
{ lat: 15.3694, lon: 44.1910, radius: 3 }, // Yemen
|
||||
{ lat: 14.4974, lon: 14.4524, radius: 8 }, // Sahel region
|
||||
{ lat: 16.8661, lon: 96.1951, radius: 5 }, // Myanmar
|
||||
{ lat: 5.1521, lon: 46.1996, radius: 4 }, // Somalia
|
||||
];
|
||||
|
||||
const EVENT_TYPES = [
|
||||
'Battles',
|
||||
'Explosions/Remote violence',
|
||||
'Violence against civilians',
|
||||
'Protests',
|
||||
'Riots',
|
||||
'Strategic developments'
|
||||
];
|
||||
|
||||
const WEAPON_TYPES = [
|
||||
'Artillery',
|
||||
'Airstrike',
|
||||
'Drone strike',
|
||||
'Armed clashes',
|
||||
'IED',
|
||||
'Small arms'
|
||||
];
|
||||
|
||||
function generateMockConflictEvents(): ConflictEvent[] {
|
||||
const events: ConflictEvent[] = [];
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
for (const hotspot of CONFLICT_HOTSPOTS) {
|
||||
const numEvents = Math.floor(Math.random() * 10) + 5; // 5-15 events per hotspot
|
||||
|
||||
for (let i = 0; i < numEvents; i++) {
|
||||
const u = Math.random();
|
||||
const v = Math.random();
|
||||
const w = hotspot.radius / 111.0;
|
||||
const t = 2 * Math.PI * v;
|
||||
const x = w * Math.cos(t);
|
||||
const y = w * Math.sin(t);
|
||||
|
||||
const lat = hotspot.lat + y;
|
||||
const lon = hotspot.lon + x;
|
||||
const type = EVENT_TYPES[Math.floor(Math.random() * EVENT_TYPES.length)];
|
||||
const subType = WEAPON_TYPES[Math.floor(Math.random() * WEAPON_TYPES.length)];
|
||||
const fatalities = Math.floor(Math.random() * 10) === 0 ? Math.floor(Math.random() * 50) : Math.floor(Math.random() * 5); // mostly small, some mass
|
||||
|
||||
events.push({
|
||||
id: crypto.randomUUID(),
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
type,
|
||||
subType,
|
||||
actor1: 'Unidentified Armed Group',
|
||||
actor2: 'State Forces',
|
||||
fatalities,
|
||||
date: today,
|
||||
source: 'Mock ACLED Data',
|
||||
notes: 'Reported ' + type.toLowerCase() + ' involving ' + subType.toLowerCase() + '.'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
const insertStmt = db.prepare('INSERT OR REPLACE INTO conflict_events (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function fetchConflictEvents() {
|
||||
console.log('[Seeder: ConflictEvents] Mocking ACLED data generation...');
|
||||
const events = generateMockConflictEvents();
|
||||
|
||||
const now = Date.now();
|
||||
let inserted = 0;
|
||||
|
||||
const insertMany = db.transaction((rows: any[]) => {
|
||||
for (const row of rows) {
|
||||
insertStmt.run(row);
|
||||
inserted++;
|
||||
}
|
||||
});
|
||||
|
||||
const dbRows = events.map(e => ({
|
||||
id: e.id,
|
||||
payload: JSON.stringify(e),
|
||||
source_ts: now,
|
||||
fetched_at: now
|
||||
}));
|
||||
|
||||
insertMany(dbRows);
|
||||
console.log('[Seeder: ConflictEvents] Inserted ' + inserted + ' events into DB.');
|
||||
|
||||
try {
|
||||
const geoEntities = events.map(e => ({
|
||||
id: 'conflict-' + e.id,
|
||||
latitude: e.latitude,
|
||||
longitude: e.longitude,
|
||||
properties: {
|
||||
type: e.type,
|
||||
subType: e.subType,
|
||||
fatalities: e.fatalities,
|
||||
actor1: e.actor1,
|
||||
actor2: e.actor2,
|
||||
date: e.date,
|
||||
notes: e.notes
|
||||
}
|
||||
}));
|
||||
|
||||
await setLiveSnapshot('conflict_events', geoEntities, 3600 * 24); // Cache for 24 hours
|
||||
} catch (err) {
|
||||
console.warn('[Seeder: ConflictEvents] Redis cache failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'conflictEvents',
|
||||
cron: '0 0 * * *', // Run daily at midnight
|
||||
fn: fetchConflictEvents
|
||||
});
|
||||
@@ -1,155 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { geolocateIp } from '../geoip';
|
||||
|
||||
const OTX_BASE = 'https://otx.alienvault.com/api/v1';
|
||||
|
||||
const insertCyberAttack = db.prepare(
|
||||
'INSERT OR REPLACE INTO cyber_attacks (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)'
|
||||
);
|
||||
|
||||
interface OtxIndicator {
|
||||
id: number;
|
||||
indicator: string;
|
||||
type: string;
|
||||
title: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface OtxPulse {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
created: string;
|
||||
modified: string;
|
||||
adversary: string;
|
||||
targeted_countries: string[];
|
||||
attack_ids: { id: string; name: string }[];
|
||||
malware_families: string[];
|
||||
indicators: OtxIndicator[];
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
export async function seedCyberAttacks() {
|
||||
const apiKey = process.env.OTX_API_KEY;
|
||||
if (!apiKey) {
|
||||
console.warn('[CyberAttacks] OTX_API_KEY not set — skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[CyberAttacks] Polling AlienVault OTX...');
|
||||
|
||||
// Fetch pulses modified in the last 48 hours
|
||||
const since = new Date(Date.now() - 48 * 3600 * 1000).toISOString();
|
||||
const url = `${OTX_BASE}/pulses/subscribed?modified_since=${since}&limit=50`;
|
||||
|
||||
const res = await withRetry(async () => {
|
||||
try {
|
||||
return await fetchWithTimeout(url, {
|
||||
headers: { 'X-OTX-API-KEY': apiKey, 'User-Agent': 'WWV-Data-Engine' },
|
||||
});
|
||||
} catch(err) {
|
||||
console.error('[CyberAttacks] Fetch error details:', err.message);
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
const data = await res.json();
|
||||
|
||||
if (!data?.results || !Array.isArray(data.results)) {
|
||||
console.warn('[CyberAttacks] Invalid OTX response');
|
||||
return;
|
||||
}
|
||||
|
||||
const pulses: OtxPulse[] = data.results;
|
||||
const fetchedAt = Date.now();
|
||||
const items: any[] = [];
|
||||
const seenIps = new Set<string>();
|
||||
|
||||
for (const pulse of pulses) {
|
||||
// Extract only IPv4 indicators
|
||||
const ipIndicators = (pulse.indicators || []).filter(
|
||||
(ind) => ind.type === 'IPv4'
|
||||
);
|
||||
|
||||
for (const ind of ipIndicators) {
|
||||
if (seenIps.has(ind.indicator)) continue;
|
||||
seenIps.add(ind.indicator);
|
||||
|
||||
const geo = geolocateIp(ind.indicator);
|
||||
if (!geo) continue; // Skip un-geolocatable IPs
|
||||
|
||||
const threatType = classifyThreat(pulse);
|
||||
const item = {
|
||||
id: `otx-${pulse.id}-${ind.id}`,
|
||||
ip: ind.indicator,
|
||||
lat: geo.lat,
|
||||
lon: geo.lon,
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
threatType,
|
||||
adversary: pulse.adversary || 'Unknown',
|
||||
pulseName: pulse.name,
|
||||
pulseDescription: pulse.description?.slice(0, 300) || '',
|
||||
malwareFamilies: pulse.malware_families || [],
|
||||
tags: pulse.tags?.slice(0, 5) || [],
|
||||
targetedCountries: pulse.targeted_countries || [],
|
||||
pulseId: pulse.id,
|
||||
pulseCreated: pulse.created,
|
||||
pulseModified: pulse.modified,
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
|
||||
// Persist to SQLite
|
||||
insertCyberAttack.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: new Date(pulse.modified).getTime(),
|
||||
fetched_at: fetchedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[CyberAttacks] Processed ${pulses.length} pulses → ${items.length} geolocated indicators.`
|
||||
);
|
||||
|
||||
// Save to Redis
|
||||
await setLiveSnapshot(
|
||||
'cyber_attacks',
|
||||
{
|
||||
source: 'cyber_attacks',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items,
|
||||
totalCount: items.length,
|
||||
},
|
||||
7200 // 2 hour TTL
|
||||
);
|
||||
}
|
||||
|
||||
function classifyThreat(pulse: OtxPulse): string {
|
||||
const tags = (pulse.tags || []).map((t) => t.toLowerCase());
|
||||
const name = pulse.name.toLowerCase();
|
||||
const desc = (pulse.description || '').toLowerCase();
|
||||
const combined = [...tags, name, desc].join(' ');
|
||||
|
||||
if (combined.includes('apt') || combined.includes('advanced persistent'))
|
||||
return 'APT';
|
||||
if (combined.includes('ransomware')) return 'Ransomware';
|
||||
if (combined.includes('botnet')) return 'Botnet';
|
||||
if (combined.includes('phishing')) return 'Phishing';
|
||||
if (combined.includes('ddos')) return 'DDoS';
|
||||
if (combined.includes('malware') || combined.includes('trojan'))
|
||||
return 'Malware';
|
||||
if (combined.includes('c2') || combined.includes('command and control'))
|
||||
return 'C2 Server';
|
||||
return 'Other';
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'cyber_attacks',
|
||||
cron: '0 */2 * * *', // Every 2 hours
|
||||
fn: seedCyberAttacks,
|
||||
});
|
||||
@@ -1,99 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry, haversineKm } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const KNOWN_TEST_SITES = [
|
||||
{ name: 'Punggye-ri (North Korea)', lat: 41.278, lon: 129.088 },
|
||||
{ name: 'Lop Nur (China)', lat: 40.75, lon: 89.6 },
|
||||
{ name: 'Nevada Test Site (USA)', lat: 37.13, lon: -116.04 },
|
||||
{ name: 'Semipalatinsk (Kazakhstan)', lat: 50.4, lon: 77.8 },
|
||||
{ name: 'Novaya Zemlya (Russia)', lat: 73.3, lon: 54.9 },
|
||||
{ name: 'Pokhran (India)', lat: 27.09, lon: 71.75 },
|
||||
{ name: 'Chagai (Pakistan)', lat: 28.79, lon: 64.91 },
|
||||
{ name: 'Moruroa (France)', lat: -21.83, lon: -138.88 },
|
||||
];
|
||||
|
||||
const insertEarthquake = db.prepare('INSERT OR IGNORE INTO earthquakes (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function seedEarthquakes() {
|
||||
console.log('[Earthquakes] Polling USGS...');
|
||||
|
||||
// USGS GeoJSON Feed: 4.5+ magnitude, past 7 days
|
||||
const url = 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_week.geojson';
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(url));
|
||||
const data = await res.json();
|
||||
const fetchedAt = Date.now();
|
||||
|
||||
if (!data?.features || !Array.isArray(data.features)) {
|
||||
console.warn('[Earthquakes] Invalid response from USGS');
|
||||
return;
|
||||
}
|
||||
|
||||
const items = [];
|
||||
let insertedCount = 0;
|
||||
|
||||
for (const feature of data.features) {
|
||||
const { id, properties, geometry } = feature;
|
||||
const [lon, lat, depth] = geometry.coordinates;
|
||||
const sourceTs = properties.time;
|
||||
|
||||
// Detect nuclear test site proximity (< 10km)
|
||||
let nearTestSite = false;
|
||||
let nearestSiteName = null;
|
||||
let minDistance = Infinity;
|
||||
|
||||
for (const site of KNOWN_TEST_SITES) {
|
||||
const dist = haversineKm(lat, lon, site.lat, site.lon);
|
||||
if (dist < minDistance) {
|
||||
minDistance = dist;
|
||||
if (dist < 10) { // Within 10km is highly suspicious
|
||||
nearTestSite = true;
|
||||
nearestSiteName = site.name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const item = {
|
||||
id,
|
||||
place: properties.place,
|
||||
magnitude: properties.mag,
|
||||
depth_km: depth,
|
||||
lat,
|
||||
lon,
|
||||
occurredAt: sourceTs,
|
||||
url: properties.url,
|
||||
nearTestSite,
|
||||
nearestSiteName,
|
||||
distanceToTestSiteKm: minDistance < 50 ? minDistance : undefined
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
|
||||
// Save to SQLite
|
||||
const result = insertEarthquake.run({
|
||||
id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
console.log(`[Earthquakes] Parsed ${items.length} earthquakes. Saved ${insertedCount} new to SQLite.`);
|
||||
|
||||
// Save to Redis Live Cache
|
||||
await setLiveSnapshot('earthquakes', {
|
||||
source: "earthquakes",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
}, 3600); // 1 hour TTL
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "earthquakes",
|
||||
cron: "0 * * * *", // Every hour
|
||||
fn: seedEarthquakes
|
||||
});
|
||||
@@ -1,95 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
const insertGpsJam = db.prepare('INSERT OR REPLACE INTO gps_jamming (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
const HOTSPOTS = [
|
||||
{ name: 'Eastern Europe / Ukraine', lat: 48.0, lon: 37.0, radiusDeg: 5, intensity: 'high' },
|
||||
{ name: 'Baltic Sea Region', lat: 58.0, lon: 20.0, radiusDeg: 4, intensity: 'high' },
|
||||
{ name: 'Middle East', lat: 33.0, lon: 36.0, radiusDeg: 6, intensity: 'high' },
|
||||
{ name: 'Black Sea', lat: 43.0, lon: 34.0, radiusDeg: 3, intensity: 'medium' },
|
||||
{ name: 'Korean Peninsula', lat: 38.0, lon: 127.0, radiusDeg: 2, intensity: 'medium' },
|
||||
];
|
||||
|
||||
function generateMockData() {
|
||||
const items = [];
|
||||
const now = Date.now();
|
||||
|
||||
for (const hotspot of HOTSPOTS) {
|
||||
// Generate 50-150 points per hotspot
|
||||
const count = Math.floor(Math.random() * 100) + 50;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
// Random offset within radius using somewhat normal distribution
|
||||
const u = Math.random() + Math.random() - 1;
|
||||
const v = Math.random() + Math.random() - 1;
|
||||
|
||||
const lat = hotspot.lat + (u * hotspot.radiusDeg);
|
||||
const lon = hotspot.lon + (v * hotspot.radiusDeg);
|
||||
|
||||
// Intensity decreases slightly away from center
|
||||
const distFromCenter = Math.sqrt(u*u + v*v);
|
||||
let pointIntensity = hotspot.intensity;
|
||||
|
||||
if (hotspot.intensity === 'high' && distFromCenter > 0.6) {
|
||||
pointIntensity = 'medium';
|
||||
}
|
||||
if (pointIntensity === 'medium' && distFromCenter > 0.8) {
|
||||
pointIntensity = 'low';
|
||||
}
|
||||
|
||||
items.push({
|
||||
id: randomUUID(),
|
||||
lat,
|
||||
lon,
|
||||
interferenceLevel: pointIntensity,
|
||||
timestamp: now,
|
||||
region: hotspot.name
|
||||
});
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function seedGpsJam() {
|
||||
console.log('[GPS Jamming] Generating daily interference map snapshot...');
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
const sourceTs = fetchedAt; // Represents end of the 24h period for the map
|
||||
const items = generateMockData();
|
||||
|
||||
let insertedCount = 0;
|
||||
|
||||
db.transaction(() => {
|
||||
// Optional: clear old data to keep DB size small if this is just a daily map
|
||||
db.exec('DELETE FROM gps_jamming');
|
||||
|
||||
for (const item of items) {
|
||||
const result = insertGpsJam.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
})();
|
||||
|
||||
console.log(`[GPS Jamming] Generated ${items.length} points. Saved ${insertedCount} to SQLite.`);
|
||||
|
||||
// Save to Redis Live Cache
|
||||
await setLiveSnapshot('gps_jamming', {
|
||||
source: "gpsjam_mock",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
}, 86400); // 24 hours TTL
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "gps_jamming",
|
||||
cron: "0 0 * * *", // Once daily at midnight
|
||||
fn: seedGpsJam
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
import './iranwarlive';
|
||||
import './earthquakes';
|
||||
import './wildfires';
|
||||
import './maritime';
|
||||
import './aviation';
|
||||
import './military-aviation';
|
||||
import './satellite';
|
||||
// --- TEMPORARILY DISABLED FOR DEMO (MOCK DATA) ---
|
||||
import './gpsjam';
|
||||
// import './conflictEvents';
|
||||
import './civilUnrest';
|
||||
import './cyberAttacks';
|
||||
import './sanctions';
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { z } from 'zod';
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
// Zod Schema for input validation to strip unrecognized injection keys
|
||||
const itemSchema = z.object({
|
||||
event_id: z.string().max(255),
|
||||
type: z.string().max(255).nullable().optional(),
|
||||
location: z.string().max(500).nullable().optional(),
|
||||
timestamp: z.string().max(100),
|
||||
confidence: z.string().max(100).nullable().optional(),
|
||||
event_summary: z.string().max(10000).nullable().optional(),
|
||||
source_url: z.string().max(2000).nullable().optional(),
|
||||
preview_image: z.string().url().max(2000).nullable().optional(),
|
||||
_osint_meta: z.any().optional()
|
||||
});
|
||||
|
||||
const insertEvent = db.prepare(`
|
||||
INSERT INTO iranwar_events (event_id, payload, timestamp, fetched_at)
|
||||
VALUES (@event_id, @payload, @timestamp, @fetched_at)
|
||||
ON CONFLICT(event_id) DO UPDATE SET
|
||||
payload=excluded.payload,
|
||||
timestamp=excluded.timestamp
|
||||
`);
|
||||
const getTopEvents = db.prepare('SELECT payload FROM iranwar_events ORDER BY timestamp DESC LIMIT 500');
|
||||
|
||||
let hasHydratedSeed = false;
|
||||
|
||||
export async function seedIranWarLive() {
|
||||
// --- SELF-HEALING HISTORY SEED ---
|
||||
if (!hasHydratedSeed) {
|
||||
console.log('[IranWarLive] Initializing: Hydrating/Upserting active fallback seed...');
|
||||
const seedPath = path.join(__dirname, '..', '..', 'seedData', 'iranwar_seed.json');
|
||||
if (fs.existsSync(seedPath)) {
|
||||
const fallbackData = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
let variables = 0;
|
||||
const upsertMany = db.transaction((events: any[]) => {
|
||||
for (const item of events) {
|
||||
try {
|
||||
const validatedItem = itemSchema.parse(item);
|
||||
const result = insertEvent.run({
|
||||
event_id: validatedItem.event_id,
|
||||
payload: JSON.stringify(validatedItem),
|
||||
timestamp: validatedItem.timestamp,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
} catch(err) { /* ignore validation errors in fallback */ }
|
||||
}
|
||||
});
|
||||
upsertMany(fallbackData);
|
||||
console.log(`[IranWarLive] Boot hydration complete. Merged ${insertedCount} seed events.`);
|
||||
}
|
||||
hasHydratedSeed = true;
|
||||
}
|
||||
|
||||
console.log('[IranWarLive] Polling iranwarlive.com/feed.json...');
|
||||
|
||||
let data: any = null;
|
||||
try {
|
||||
const response = await withRetry(() => fetchWithTimeout('https://iranwarlive.com/feed.json', {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cache-Control": "no-cache",
|
||||
"Accept-Language": "en-US,en;q=0.9"
|
||||
}
|
||||
}));
|
||||
if(!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
data = await response.json();
|
||||
} catch(err: any) {
|
||||
console.warn(`[IranWarLive] Failed to fetch live feed (anti-bot block?): ${err.message}. Using local database cache.`);
|
||||
}
|
||||
|
||||
if (data && data.items && Array.isArray(data.items) && data.items.length > 0) {
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
|
||||
// Find completely new items to hydrate
|
||||
const placeholders = data.items.map(() => '?').join(',');
|
||||
const existingIdsStmt = db.prepare(`SELECT event_id FROM iranwar_events WHERE event_id IN (${placeholders})`);
|
||||
const existingIdsRow = existingIdsStmt.all(...data.items.map((i: any) => i.event_id)) as { event_id: string }[];
|
||||
const existingIds = new Set(existingIdsRow.map(row => row.event_id));
|
||||
|
||||
const newItems = data.items.filter((item: any) => !existingIds.has(item.event_id));
|
||||
|
||||
if (newItems.length > 0) {
|
||||
console.log(`[IranWarLive] Found ${newItems.length} new events. Hydrating og:images...`);
|
||||
for (const item of newItems) {
|
||||
if (item.source_url) {
|
||||
try {
|
||||
const htmlRes = await fetchWithTimeout(item.source_url, { headers: { "User-Agent": "WorldWideView-OSINT/1.0" } }, 5000);
|
||||
const html = await htmlRes.text();
|
||||
const ogMatch = html.match(/<meta[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/i)
|
||||
|| html.match(/<meta[^>]*content=["']([^"']+)["'][^>]*property=["']og:image["']/i);
|
||||
if (ogMatch && ogMatch[1]) {
|
||||
item.preview_image = ogMatch[1];
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(`[IranWarLive] Failed to hydrate image for ${item.event_id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const insertMany = db.transaction((events: any[]) => {
|
||||
for (const item of events) {
|
||||
try {
|
||||
const validatedItem = itemSchema.parse(item);
|
||||
const result = insertEvent.run({
|
||||
event_id: validatedItem.event_id,
|
||||
payload: JSON.stringify(validatedItem),
|
||||
timestamp: validatedItem.timestamp,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
} catch(err: any) {
|
||||
console.warn(`[IranWarLive] Skipped item due to validation error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(newItems);
|
||||
if (insertedCount > 0) {
|
||||
console.log(`[IranWarLive] Added ${insertedCount} new hydrated events to history.`);
|
||||
}
|
||||
} else {
|
||||
console.log('[IranWarLive] No new events found.');
|
||||
}
|
||||
}
|
||||
|
||||
// ALways update Redis hot cache with the latest 500 events
|
||||
const rows = getTopEvents.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
await setLiveSnapshot('iranwarlive', {
|
||||
source: "iranwarlive",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: history
|
||||
}, 3600); // 1 hour TTL in Redis
|
||||
}
|
||||
|
||||
// Register with scheduler
|
||||
registerSeeder({
|
||||
name: "iranwarlive",
|
||||
cron: "*/1 * * * *", // Every minute
|
||||
fn: seedIranWarLive
|
||||
});
|
||||
@@ -1,147 +0,0 @@
|
||||
import WebSocket from 'ws';
|
||||
import { db } from '../db';
|
||||
import { redis, setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const AISSTREAM_URL = 'wss://stream.aisstream.io/v0/stream';
|
||||
const API_KEY = process.env.AISSTREAM_API_KEY;
|
||||
|
||||
// Buffer to batch SQLite inserts
|
||||
let messageBuffer: any[] = [];
|
||||
const activeFleetCache = new Map<string, any>();
|
||||
const FLUSH_INTERVAL_MS = 15000;
|
||||
let isFlushIntervalRunning = false;
|
||||
|
||||
// SQLite insert statement
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO maritime_history (mmsi, ts, lat, lon, hdg, spd, fetched_at)
|
||||
VALUES (@mmsi, @ts, @lat, @lon, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function flushBuffer() {
|
||||
if (messageBuffer.length === 0) return;
|
||||
|
||||
const batch = [...messageBuffer];
|
||||
messageBuffer = [];
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
|
||||
// 1. Bulk insert to SQLite for permanent history
|
||||
const insertMany = db.transaction((msgs) => {
|
||||
for (const msg of msgs) {
|
||||
if (!msg.MetaData?.MMSI || !msg.Message?.PositionReport) continue;
|
||||
|
||||
const mmsi = msg.MetaData.MMSI.toString();
|
||||
const report = msg.Message.PositionReport;
|
||||
const ts = Math.floor(new Date(msg.MetaData.time_utc).getTime() / 1000); // Epoch seconds
|
||||
|
||||
const result = insertHistory.run({
|
||||
mmsi,
|
||||
ts,
|
||||
lat: report.Latitude,
|
||||
lon: report.Longitude,
|
||||
hdg: report.TrueHeading,
|
||||
spd: report.Sog,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
insertMany(batch);
|
||||
if (insertedCount > 0) {
|
||||
// console.log(`[Maritime] Flushed ${insertedCount} new positions to SQLite history`);
|
||||
}
|
||||
|
||||
// 2. Update memory cache and serialize to snapshot
|
||||
const nowSecs = Math.floor(Date.now() / 1000);
|
||||
// Cleanup stale ships (>6 hours)
|
||||
for (const [mmsi, ship] of activeFleetCache.entries()) {
|
||||
if (nowSecs - ship.last_updated > 6 * 3600) {
|
||||
activeFleetCache.delete(mmsi);
|
||||
}
|
||||
}
|
||||
|
||||
for (const msg of batch) {
|
||||
if (!msg.MetaData?.MMSI || !msg.Message?.PositionReport) continue;
|
||||
const mmsi = msg.MetaData.MMSI.toString();
|
||||
const report = msg.Message.PositionReport;
|
||||
const ts = Math.floor(new Date(msg.MetaData.time_utc).getTime() / 1000);
|
||||
|
||||
const shipState = {
|
||||
id: `mmsi-${mmsi}`,
|
||||
mmsi,
|
||||
name: msg.MetaData.ShipName ? msg.MetaData.ShipName.trim() : `Unknown (${mmsi})`,
|
||||
lat: report.Latitude,
|
||||
lon: report.Longitude,
|
||||
hdg: report.TrueHeading,
|
||||
spd: report.Sog,
|
||||
last_updated: ts
|
||||
};
|
||||
|
||||
activeFleetCache.set(mmsi, shipState);
|
||||
}
|
||||
|
||||
await setLiveSnapshot('maritime', Object.fromEntries(activeFleetCache), 6 * 3600);
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Maritime] Buffer flush failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export function startMaritimeWebsocket() {
|
||||
if (!API_KEY) {
|
||||
console.warn('[Maritime] Skipping AIS websocket: AISSTREAM_API_KEY not set.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[Maritime] Connecting to AisStream.io...');
|
||||
const ws = new WebSocket(AISSTREAM_URL);
|
||||
|
||||
ws.on('open', () => {
|
||||
console.log('[Maritime] WebSocket connected. Subscribing to global feed...');
|
||||
// Bounding box for English Channel / Strait of Hormuz / global (up to API tier)
|
||||
// Here we request global or specific high-traffic boxes
|
||||
const subscriptionMessage = {
|
||||
APIKey: API_KEY,
|
||||
BoundingBoxes: [[[-90, -180], [90, 180]]], // Global (requires paid tier for real use)
|
||||
FiltersShipMMSI: [], // All ships
|
||||
FilterMessageTypes: ["PositionReport"]
|
||||
};
|
||||
ws.send(JSON.stringify(subscriptionMessage));
|
||||
});
|
||||
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString());
|
||||
if (msg.MessageType === "PositionReport" || msg.Message?.PositionReport) {
|
||||
messageBuffer.push(msg);
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parse errors
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('error', (err) => {
|
||||
console.error('[Maritime] WebSocket error:', err.message);
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
console.log('[Maritime] WebSocket closed. Reconnecting in 5s...');
|
||||
setTimeout(startMaritimeWebsocket, 5000);
|
||||
});
|
||||
|
||||
// Start background flush loop safely
|
||||
if (!isFlushIntervalRunning) {
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
isFlushIntervalRunning = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Register initialization logic. No cron needed, runs infinitely.
|
||||
registerSeeder({
|
||||
name: "maritime",
|
||||
init: startMaritimeWebsocket
|
||||
});
|
||||
@@ -1,101 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const ADSB_LOL_URL = "https://api.adsb.lol/v2/mil";
|
||||
const POLLING_INTERVAL_MS = 60000;
|
||||
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO military_aviation_history (hex, ts, lat, lon, alt, hdg, spd, fetched_at)
|
||||
VALUES (@hex, @ts, @lat, @lon, @alt, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function pollMilitaryAviation() {
|
||||
try {
|
||||
const response = await fetch(ADSB_LOL_URL, {
|
||||
headers: {
|
||||
"User-Agent": "WorldWideView-DataEngine"
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.warn('[MilitaryAviation] 429 Rate Limit hit.');
|
||||
}
|
||||
throw new Error(`Status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (!data.ac || !Array.isArray(data.ac)) return;
|
||||
|
||||
const fetchedAt = Math.floor(Date.now() / 1000);
|
||||
const fleetObj: Record<string, any> = Object.create(null);
|
||||
let insertedCount = 0;
|
||||
|
||||
const insertMany = db.transaction((aircraft) => {
|
||||
for (const ac of aircraft) {
|
||||
if (ac.lat == null || ac.lon == null) continue;
|
||||
|
||||
const hex = ac.hex;
|
||||
// Some fields might be missing depending on coverage
|
||||
const ts = Math.floor((ac.seen_pos != null ? (Date.now() - ac.seen_pos * 1000) : Date.now()) / 1000); // Approximate timestamp based on seen_pos
|
||||
const lon = ac.lon;
|
||||
const lat = ac.lat;
|
||||
const alt = typeof ac.alt_baro === "number" ? ac.alt_baro : 0;
|
||||
const on_ground = ac.alt_baro === "ground";
|
||||
const spd = ac.gs || 0;
|
||||
const hdg = ac.track || 0;
|
||||
|
||||
// 1. Write History to SQLite
|
||||
if (!on_ground) {
|
||||
const result = insertHistory.run({
|
||||
hex, ts, lat, lon, alt, hdg, spd, fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
// 2. Prepare Live Cache structure
|
||||
fleetObj[hex] = {
|
||||
hex,
|
||||
flight: ac.flight || null,
|
||||
r: ac.r || null,
|
||||
t: ac.t || null,
|
||||
lat,
|
||||
lon,
|
||||
alt_baro: ac.alt_baro,
|
||||
alt_geom: ac.alt_geom,
|
||||
gs: ac.gs,
|
||||
track: ac.track,
|
||||
squawk: ac.squawk,
|
||||
dbFlags: ac.dbFlags,
|
||||
category: ac.category,
|
||||
emergency: ac.emergency,
|
||||
seen: ac.seen,
|
||||
seen_pos: ac.seen_pos,
|
||||
last_updated: fetchedAt
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(data.ac);
|
||||
|
||||
// Execute Redis batch via compressed snapshot (60 mins to match slow moving or intermittent adsb.lol coverage)
|
||||
await setLiveSnapshot('military-aviation', fleetObj, 60 * 60);
|
||||
|
||||
} catch (err: any) {
|
||||
console.error(`[MilitaryAviation] Polling Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function startMilitaryAviationPoller() {
|
||||
console.log('[MilitaryAviation] Starting background polling...');
|
||||
setInterval(pollMilitaryAviation, POLLING_INTERVAL_MS);
|
||||
|
||||
// Initial fetch
|
||||
pollMilitaryAviation();
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "military-aviation",
|
||||
init: startMilitaryAviationPoller
|
||||
});
|
||||
@@ -1,224 +0,0 @@
|
||||
import sax from 'sax';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { fetchWithTimeout, CHROME_UA } from '../seed-utils';
|
||||
|
||||
const OFAC_SOURCES = [
|
||||
{ label: 'SDN', url: 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports/sdn_advanced.xml' },
|
||||
{ label: 'CONSOLIDATED', url: 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports/cons_advanced.xml' },
|
||||
];
|
||||
|
||||
function local(name: string) {
|
||||
const colon = name.indexOf(':');
|
||||
return colon === -1 ? name : name.slice(colon + 1);
|
||||
}
|
||||
|
||||
// ── SAX Parser adapted from WorldMonitor ──
|
||||
async function fetchSource(source: typeof OFAC_SOURCES[0]) {
|
||||
console.log(`[Sanctions] Fetching OFAC ${source.label}...`);
|
||||
const t0 = Date.now();
|
||||
const response = await fetchWithTimeout(source.url, {
|
||||
headers: { 'User-Agent': CHROME_UA }
|
||||
}, 45_000);
|
||||
|
||||
return new Promise<{ entries: any[] }>((resolve, reject) => {
|
||||
const parser = sax.parser(true, { trim: false, normalize: false });
|
||||
|
||||
const areaCodes = new Map();
|
||||
const featureTypes = new Map();
|
||||
const legalBasis = new Map();
|
||||
const locations = new Map();
|
||||
const parties = new Map();
|
||||
const entries: any[] = [];
|
||||
let bytesReceived = 0;
|
||||
|
||||
const stack: string[] = [];
|
||||
let text = '';
|
||||
|
||||
let inAreaCodeValues = false;
|
||||
let inFeatureTypeValues = false;
|
||||
let inLegalBasisValues = false;
|
||||
let inLocations = false;
|
||||
let inDistinctParties = false;
|
||||
let inSanctionsEntries = false;
|
||||
|
||||
let refId = '', refShortRef = '', refDescription = '';
|
||||
let locId = '';
|
||||
let locAreaCodeIds: string[] | null = null;
|
||||
let partyFixedRef = '';
|
||||
let profileId = '', profileSubTypeId = '';
|
||||
let aliases: any[] | null = null;
|
||||
let curAlias: any = null;
|
||||
let inDocumentedName = false;
|
||||
let namePartsBuf: string[] | null = null;
|
||||
let profileFeatures: any[] | null = null;
|
||||
let curFeature: any = null;
|
||||
|
||||
let entryId = '', entryProfileId = '';
|
||||
|
||||
function resolveLocation(locId: string) {
|
||||
if (!locAreaCodeIds) return { codes: [], names: [] };
|
||||
const ids = locAreaCodeIds;
|
||||
const mapped = ids.map((id) => areaCodes.get(id)).filter(Boolean);
|
||||
const pairs = [...new Map(mapped.map((item) => [item.code, item.name])).entries()]
|
||||
.filter(([code]) => code.length > 0);
|
||||
return { codes: pairs.map(([c]) => c), names: pairs.map(([, n]) => n) };
|
||||
}
|
||||
|
||||
function finalizeParty() {
|
||||
const seen = new Map();
|
||||
for (const feat of profileFeatures ?? []) {
|
||||
if (!/location/i.test(featureTypes.get(feat.featureTypeId) || '')) continue;
|
||||
for (const lid of feat.locationIds) {
|
||||
const loc = locations.get(lid);
|
||||
if (!loc) continue;
|
||||
loc.codes.forEach((code: string, i: number) => { if (code && !seen.has(code)) seen.set(code, loc.names[i] ?? ''); });
|
||||
}
|
||||
}
|
||||
parties.set(profileId, { countryCodes: [...seen.keys()] });
|
||||
}
|
||||
|
||||
function finalizeEntry() {
|
||||
const party = parties.get(entryProfileId);
|
||||
entries.push({ countryCodes: party?.countryCodes ?? [] });
|
||||
}
|
||||
|
||||
parser.onopentag = (node: any) => {
|
||||
const name = local(node.name);
|
||||
const attrs = node.attributes as any;
|
||||
stack.push(name);
|
||||
text = '';
|
||||
|
||||
switch (name) {
|
||||
case 'AreaCodeValues': inAreaCodeValues = true; break;
|
||||
case 'FeatureTypeValues': inFeatureTypeValues = true; break;
|
||||
case 'LegalBasisValues': inLegalBasisValues = true; break;
|
||||
case 'Locations': inLocations = true; break;
|
||||
case 'DistinctParties': inDistinctParties = true; break;
|
||||
case 'SanctionsEntries': inSanctionsEntries = true; break;
|
||||
|
||||
case 'AreaCode': if (inAreaCodeValues) { refId = attrs.ID || ''; refDescription = attrs.Description || ''; } break;
|
||||
case 'FeatureType': if (inFeatureTypeValues) refId = attrs.ID || ''; break;
|
||||
case 'Location': if (inLocations) { locId = attrs.ID || ''; locAreaCodeIds = []; } break;
|
||||
case 'LocationAreaCode': if (locAreaCodeIds && attrs.AreaCodeID) locAreaCodeIds.push(attrs.AreaCodeID); break;
|
||||
|
||||
case 'DistinctParty': if (inDistinctParties) { partyFixedRef = attrs.FixedRef || ''; aliases = []; profileFeatures = []; } break;
|
||||
case 'Profile': if (inDistinctParties) { profileId = attrs.ID || partyFixedRef; profileSubTypeId = attrs.PartySubTypeID || ''; } break;
|
||||
case 'Feature': if (inDistinctParties) curFeature = { featureTypeId: attrs.FeatureTypeID || '', locationIds: [] }; break;
|
||||
case 'VersionLocation': if (curFeature && attrs.LocationID) curFeature.locationIds.push(attrs.LocationID); break;
|
||||
|
||||
case 'SanctionsEntry':
|
||||
if (inSanctionsEntries) { entryId = attrs.ID || ''; entryProfileId = attrs.ProfileID || ''; }
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
parser.onclosetag = (rawName: string) => {
|
||||
const name = local(rawName);
|
||||
const t = text.trim();
|
||||
text = '';
|
||||
stack.pop();
|
||||
|
||||
switch (name) {
|
||||
case 'AreaCodeValues': inAreaCodeValues = false; break;
|
||||
case 'FeatureTypeValues': inFeatureTypeValues = false; break;
|
||||
case 'Locations': inLocations = false; break;
|
||||
case 'DistinctParties': inDistinctParties = false; break;
|
||||
case 'SanctionsEntries': inSanctionsEntries = false; break;
|
||||
|
||||
case 'AreaCode': if (inAreaCodeValues && refId) areaCodes.set(refId, { code: t, name: refDescription }); break;
|
||||
case 'FeatureType': if (inFeatureTypeValues && refId) featureTypes.set(refId, t); break;
|
||||
case 'Location':
|
||||
if (locAreaCodeIds !== null) {
|
||||
locations.set(locId, resolveLocation(locId));
|
||||
locId = ''; locAreaCodeIds = null;
|
||||
}
|
||||
break;
|
||||
case 'Feature': if (curFeature) { profileFeatures!.push(curFeature); curFeature = null; } break;
|
||||
case 'Profile':
|
||||
if (inDistinctParties && profileId) finalizeParty();
|
||||
profileId = ''; profileSubTypeId = ''; aliases = []; profileFeatures = [];
|
||||
break;
|
||||
case 'SanctionsEntry': finalizeEntry(); entryId = ''; entryProfileId = ''; break;
|
||||
}
|
||||
};
|
||||
|
||||
parser.ontext = (chunk: string) => { text += chunk; };
|
||||
parser.oncdata = (chunk: string) => { text += chunk; };
|
||||
parser.onerror = (err: any) => { parser.resume(); };
|
||||
parser.onend = () => { resolve({ entries }); };
|
||||
|
||||
if (!response.body) return reject(new Error('No response body'));
|
||||
|
||||
// We treat response.body as an AsyncIterable (Node fetch extension)
|
||||
(async () => {
|
||||
try {
|
||||
for await (const chunk of (response.body as any)) {
|
||||
parser.write(Buffer.from(chunk).toString('utf-8'));
|
||||
}
|
||||
parser.close();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchSanctionsData() {
|
||||
try {
|
||||
console.log('[Sanctions] Fetching OFAC feeds using SAX streaming...');
|
||||
|
||||
let allEntries: any[] = [];
|
||||
for (const source of OFAC_SOURCES) {
|
||||
const result = await fetchSource(source);
|
||||
allEntries = allEntries.concat(result.entries);
|
||||
}
|
||||
|
||||
// Aggregate by iso2 country codes
|
||||
const countryCounts: Record<string, number> = {};
|
||||
for (const entry of allEntries) {
|
||||
for (const code of entry.countryCodes) {
|
||||
if (code && code !== 'XX') {
|
||||
countryCounts[code] = (countryCounts[code] || 0) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert dict to GeoEntities
|
||||
const items = Object.entries(countryCounts).map(([code, count]) => {
|
||||
let level = 'low';
|
||||
if (count > 500) level = 'high';
|
||||
else if (count > 50) level = 'medium';
|
||||
|
||||
return {
|
||||
id: `sanction-${code}`,
|
||||
countryCode: code,
|
||||
count: count,
|
||||
level,
|
||||
// Assign a placeholder coordinate, though frontend rendering will extract
|
||||
// 3D bounds from /borders.geojson using countryCode instead.
|
||||
latitude: 0,
|
||||
longitude: 0,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
});
|
||||
|
||||
const sanctionsObj = {
|
||||
id: 'sanctions-live',
|
||||
timestamp: new Date().toISOString(),
|
||||
items
|
||||
};
|
||||
|
||||
// Cache for live polling (24 hr TTL)
|
||||
await setLiveSnapshot('sanctions', sanctionsObj, 86400);
|
||||
console.log(`[Sanctions] Published ${items.length} sanctioned countries.`);
|
||||
} catch (err) {
|
||||
console.error('[Sanctions] Error:', err);
|
||||
}
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "sanctions",
|
||||
cron: "0 * * * *", // run hourly
|
||||
fn: fetchSanctionsData
|
||||
});
|
||||
@@ -1,168 +0,0 @@
|
||||
// Dynamic import required by cjs/esm interop
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const BASE_URL = "https://celestrak.org/NORAD/elements/gp.php";
|
||||
|
||||
export const DEFAULT_GROUPS = [
|
||||
"stations", // ISS, Tiangong, etc.
|
||||
"visual", // Brightest 100 satellites
|
||||
"weather", // Weather satellites
|
||||
"gps-ops", // GPS constellation
|
||||
"resource", // Earth observation / reconnaissance
|
||||
"military", // Military reconnaissance
|
||||
] as const;
|
||||
|
||||
export interface CelesTrakGP {
|
||||
OBJECT_NAME: string;
|
||||
TLE_LINE1: string;
|
||||
TLE_LINE2: string;
|
||||
NORAD_CAT_ID: number;
|
||||
COUNTRY_CODE?: string;
|
||||
OBJECT_TYPE?: string;
|
||||
PERIOD?: number;
|
||||
}
|
||||
|
||||
// In-memory global TLE cache for computing on-the-fly positions
|
||||
export const globalsTLECache = new Map<string, CelesTrakGP[]>();
|
||||
|
||||
async function fetchTLEGroup(group: string): Promise<CelesTrakGP[]> {
|
||||
const url = `${BASE_URL}?GROUP=${encodeURIComponent(group)}&FORMAT=tle`;
|
||||
const res = await fetch(url, { cache: "no-store", headers: { "User-Agent": "WorldWideView" } });
|
||||
|
||||
if (!res.ok) {
|
||||
console.error(`[SatelliteSeeder] CelesTrak returned ${res.status} for group=${group}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const text = await res.text();
|
||||
const lines = text.split("\n").map(l => l.trim()).filter(l => l.length > 0);
|
||||
const data: CelesTrakGP[] = [];
|
||||
|
||||
for (let i = 0; i < lines.length - 2; i += 3) {
|
||||
data.push({
|
||||
OBJECT_NAME: lines[i],
|
||||
TLE_LINE1: lines[i + 1],
|
||||
TLE_LINE2: lines[i + 2],
|
||||
NORAD_CAT_ID: parseInt(lines[i + 1].substring(2, 7).trim(), 10),
|
||||
} as CelesTrakGP);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
async function refreshAllTLEs() {
|
||||
console.log('[SatelliteSeeder] Refreshing TLEs from Celestrak...');
|
||||
for (const group of DEFAULT_GROUPS) {
|
||||
try {
|
||||
const records = await fetchTLEGroup(group);
|
||||
if (records.length > 0) {
|
||||
globalsTLECache.set(group, records);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`[SatelliteSeeder] Error fetching ${group}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exported standard propagation function
|
||||
export function propagateAll(records: CelesTrakGP[], time: Date, group: string): any[] {
|
||||
const satellite = require('satellite.js');
|
||||
const gmst = satellite.gstime(time);
|
||||
const results: any[] = [];
|
||||
|
||||
for (const rec of records) {
|
||||
try {
|
||||
const satrec = satellite.twoline2satrec(rec.TLE_LINE1, rec.TLE_LINE2);
|
||||
const pv = satellite.propagate(satrec, time);
|
||||
|
||||
if (!pv.position || typeof pv.position === "boolean" || !pv.velocity || typeof pv.velocity === "boolean") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const geo = satellite.eciToGeodetic(pv.position as any, gmst);
|
||||
const lat = satellite.degreesLat(geo.latitude);
|
||||
const lon = satellite.degreesLong(geo.longitude);
|
||||
const alt = geo.height; // km
|
||||
|
||||
if (!isFinite(lat) || !isFinite(lon) || !isFinite(alt)) continue;
|
||||
if (alt < 0 || alt > 100000) continue;
|
||||
|
||||
const vel = pv.velocity as any;
|
||||
const speed = Math.sqrt(vel.x ** 2 + vel.y ** 2 + vel.z ** 2) * 1000;
|
||||
const heading = (Math.atan2(vel.x, vel.z) * (180 / Math.PI) + 360) % 360;
|
||||
|
||||
results.push({
|
||||
noradId: rec.NORAD_CAT_ID,
|
||||
name: rec.OBJECT_NAME,
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
altitude: alt,
|
||||
heading,
|
||||
speed,
|
||||
group,
|
||||
country: rec.COUNTRY_CODE,
|
||||
objectType: rec.OBJECT_TYPE,
|
||||
period: rec.PERIOD,
|
||||
});
|
||||
} catch {
|
||||
// Error in propagation, skip
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function computeAndPublishPositions() {
|
||||
const now = new Date();
|
||||
const civilianObj: Record<string, any> = Object.create(null);
|
||||
const militaryObj: Record<string, any> = Object.create(null);
|
||||
let totalCivilian = 0;
|
||||
let totalMilitary = 0;
|
||||
|
||||
for (const group of DEFAULT_GROUPS) {
|
||||
const records = globalsTLECache.get(group);
|
||||
if (!records) continue;
|
||||
|
||||
const isMilitary = group === "military" || group === "resource";
|
||||
const positions = propagateAll(records, now, group);
|
||||
for (const p of positions) {
|
||||
if (isMilitary) {
|
||||
militaryObj[p.noradId] = p;
|
||||
totalMilitary++;
|
||||
} else {
|
||||
civilianObj[p.noradId] = p;
|
||||
totalCivilian++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalCivilian > 0) {
|
||||
await setLiveSnapshot('satellite', civilianObj, 60 * 60);
|
||||
}
|
||||
if (totalMilitary > 0) {
|
||||
await setLiveSnapshot('surveillance_satellites', militaryObj, 60 * 60);
|
||||
}
|
||||
}
|
||||
|
||||
let syncParams = {
|
||||
tleFetchInterval: 1000 * 60 * 60, // 1 hour
|
||||
publishInterval: 1000 * 15, // 15 seconds
|
||||
tleIntervalId: null as any,
|
||||
publishIntervalId: null as any
|
||||
};
|
||||
|
||||
export function startSatelliteSeeder() {
|
||||
console.log('[SatelliteSeeder] Starting satellite TLE seeder.');
|
||||
|
||||
refreshAllTLEs().then(() => {
|
||||
computeAndPublishPositions();
|
||||
syncParams.publishIntervalId = setInterval(computeAndPublishPositions, syncParams.publishInterval);
|
||||
});
|
||||
|
||||
syncParams.tleIntervalId = setInterval(refreshAllTLEs, syncParams.tleFetchInterval);
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "satellite",
|
||||
init: startSatelliteSeeder
|
||||
});
|
||||
@@ -1,150 +0,0 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
interface FIRMSRecord {
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
bright_ti4: number;
|
||||
scan: number;
|
||||
track: number;
|
||||
acq_date: string;
|
||||
acq_time: string;
|
||||
satellite: string;
|
||||
confidence: string;
|
||||
version: string;
|
||||
bright_ti5: number;
|
||||
frp: number;
|
||||
daynight: string;
|
||||
}
|
||||
|
||||
function parseCSV(csv: string): FIRMSRecord[] {
|
||||
const lines = csv.trim().split("\n");
|
||||
if (lines.length < 2) return [];
|
||||
|
||||
const headers = lines[0].split(",").map((h) => h.trim());
|
||||
const records: FIRMSRecord[] = [];
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = lines[i].split(",");
|
||||
if (values.length < headers.length) continue;
|
||||
|
||||
const record: Record<string, string | number> = {};
|
||||
headers.forEach((header, idx) => {
|
||||
record[header] = values[idx]?.trim() || "";
|
||||
});
|
||||
|
||||
const lat = parseFloat(record["latitude"] as string);
|
||||
const lon = parseFloat(record["longitude"] as string);
|
||||
if (isNaN(lat) || isNaN(lon)) continue;
|
||||
|
||||
records.push({
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
bright_ti4: parseFloat(record["bright_ti4"] as string) || 0,
|
||||
scan: parseFloat(record["scan"] as string) || 0,
|
||||
track: parseFloat(record["track"] as string) || 0,
|
||||
acq_date: (record["acq_date"] as string) || "",
|
||||
acq_time: (record["acq_time"] as string) || "",
|
||||
satellite: (record["satellite"] as string) || "",
|
||||
confidence: (record["confidence"] as string) || "",
|
||||
version: (record["version"] as string) || "",
|
||||
bright_ti5: parseFloat(record["bright_ti5"] as string) || 0,
|
||||
frp: parseFloat(record["frp"] as string) || 0,
|
||||
daynight: (record["daynight"] as string) || "",
|
||||
});
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
const insertWildfire = db.prepare('INSERT OR IGNORE INTO wildfires (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function seedWildfires() {
|
||||
console.log('[Wildfires] Polling NASA FIRMS...');
|
||||
|
||||
// Use open CSV feed
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/data/active_fire/suomi-npp-viirs-c2/csv/SUOMI_VIIRS_C2_Global_24h.csv`;
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(url));
|
||||
const csv = await res.text();
|
||||
const fires = parseCSV(csv);
|
||||
const fetchedAt = Date.now();
|
||||
|
||||
if (fires.length === 0) {
|
||||
console.log('[Wildfires] No fires found or parsing failed.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Multi-Tier Clustering logic (Preserved from original implementation)
|
||||
const tiers = [
|
||||
{ level: 1, size: 2.0 }, // Macro: ~220km
|
||||
{ level: 2, size: 0.5 }, // Meso: ~55km
|
||||
{ level: 3, size: 0.05 }, // Micro: ~5.5km
|
||||
];
|
||||
|
||||
const allClusteredFires: (FIRMSRecord & { tier: number, id: string })[] = [];
|
||||
|
||||
for (const tier of tiers) {
|
||||
const clustered = new Map<string, FIRMSRecord & { tier: number, id: string }>();
|
||||
|
||||
for (const fire of fires) {
|
||||
const gridId = `${Math.floor(fire.latitude / tier.size)}_${Math.floor(fire.longitude / tier.size)}`;
|
||||
const existing = clustered.get(gridId);
|
||||
if (existing) {
|
||||
existing.frp += fire.frp;
|
||||
if (fire.confidence === "high" || (fire.confidence === "nominal" && existing.confidence === "low")) {
|
||||
existing.confidence = fire.confidence;
|
||||
}
|
||||
} else {
|
||||
// Creating a unique ID based on date, time, lat, lon to allow deduplication in SQLite
|
||||
const id = `firm_${fire.acq_date}_${fire.acq_time}_${Math.round(fire.latitude*1000)}_${Math.round(fire.longitude*1000)}_t${tier.level}`;
|
||||
clustered.set(gridId, { ...fire, tier: tier.level, id });
|
||||
}
|
||||
}
|
||||
allClusteredFires.push(...Array.from(clustered.values()));
|
||||
}
|
||||
|
||||
// Save to SQLite History
|
||||
let insertedCount = 0;
|
||||
const insertMany = db.transaction((firesList) => {
|
||||
for (const f of firesList) {
|
||||
// Create a pseudo-timestamp from acq_date and acq_time: "2024-04-01" "1430"
|
||||
const timeStr = f.acq_time.toString().padStart(4, '0');
|
||||
const tsStr = `${f.acq_date}T${timeStr.substring(0,2)}:${timeStr.substring(2,4)}:00Z`;
|
||||
let sourceTs = 0;
|
||||
try {
|
||||
sourceTs = new Date(tsStr).getTime();
|
||||
if (isNaN(sourceTs)) sourceTs = fetchedAt;
|
||||
} catch (e) {
|
||||
sourceTs = fetchedAt;
|
||||
}
|
||||
|
||||
const result = insertWildfire.run({
|
||||
id: f.id,
|
||||
payload: JSON.stringify(f),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(allClusteredFires);
|
||||
console.log(`[Wildfires] Parsed ${fires.length} raw fires -> Clustered ${allClusteredFires.length} points. Saved ${insertedCount} new to SQLite.`);
|
||||
|
||||
// Save full clustered map to Redis
|
||||
await setLiveSnapshot('wildfires', {
|
||||
source: "wildfires",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: allClusteredFires,
|
||||
totalCount: allClusteredFires.length
|
||||
}, 1800); // 30 mins TTL
|
||||
}
|
||||
|
||||
// Register with scheduler
|
||||
registerSeeder({
|
||||
name: "wildfires",
|
||||
cron: "*/15 * * * *", // Every 15 minutes
|
||||
fn: seedWildfires
|
||||
});
|
||||
@@ -1,86 +0,0 @@
|
||||
import dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
|
||||
import Fastify from 'fastify';
|
||||
import { prisma } from './prisma';
|
||||
import { startScheduler } from './scheduler';
|
||||
import { seederStatus } from './scheduler';
|
||||
|
||||
// Boot Fastify
|
||||
export const fastify = Fastify({
|
||||
logger: false // Keep it clean for the console
|
||||
});
|
||||
|
||||
import fastifyWebsocket from '@fastify/websocket';
|
||||
import fastifyRateLimit from '@fastify/rate-limit';
|
||||
import { handleConnection } from './websocket';
|
||||
|
||||
fastify.register(fastifyRateLimit, {
|
||||
max: 100,
|
||||
timeWindow: '1 minute'
|
||||
});
|
||||
|
||||
fastify.register(fastifyWebsocket);
|
||||
|
||||
fastify.register(async function (fastify) {
|
||||
// @ts-ignore - RouteShorthandOptions augmentation missing for websocket in strict mode
|
||||
fastify.get('/stream', { websocket: true }, (connection: any, req) => {
|
||||
handleConnection(connection, req);
|
||||
});
|
||||
});
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '5001', 10);
|
||||
|
||||
fastify.get('/health', async (request, reply) => {
|
||||
return {
|
||||
status: 'ok',
|
||||
engine: 'wwv-data-engine',
|
||||
timestamp: Date.now(),
|
||||
seeders: seederStatus
|
||||
};
|
||||
});
|
||||
|
||||
async function start() {
|
||||
try {
|
||||
// 1. Initialize Prisma Database (Supabase)
|
||||
try {
|
||||
await prisma.$connect();
|
||||
} catch (dbErr) {
|
||||
console.error('[Server] Prisma could not connect. Supabase historical data sync disabled.', dbErr instanceof Error ? dbErr.message : String(dbErr));
|
||||
}
|
||||
|
||||
// 1.5. Initialize Local SQLite (Fallback/Secondary History)
|
||||
const { initDB } = await import('./db.js');
|
||||
initDB();
|
||||
|
||||
// 2. Register Routes
|
||||
await import('./routes/index.js');
|
||||
|
||||
// 3. Start the Fastify API Server
|
||||
await fastify.listen({ port: PORT, host: '0.0.0.0' });
|
||||
console.log(`[Server] WWV Data Engine listening on port ${PORT}`);
|
||||
|
||||
// 4. Import seeder registry (this registers them)
|
||||
await import('./seeders/index.js');
|
||||
|
||||
// 5. Start the Cron Scheduler
|
||||
startScheduler();
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Server] Fatal error during startup:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
async function gracefulShutdown(signal: string) {
|
||||
console.log(`\n[Server] ${signal} received. Shutting down...`);
|
||||
await fastify.close();
|
||||
await prisma.$disconnect();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||||
|
||||
start();
|
||||
@@ -1,61 +0,0 @@
|
||||
import type { WebSocket } from 'ws';
|
||||
import { getLiveSnapshot } from './redis';
|
||||
|
||||
// Track active connections and their subscriptions
|
||||
const connections = new Set<WebSocket>();
|
||||
const subscriptions = new Map<WebSocket, Set<string>>();
|
||||
|
||||
export function handleConnection(connection: WebSocket, request: any) {
|
||||
// Option A (Secure Defaults): In a highly public plugin ecosystem,
|
||||
// checking the token is optional/opt-in via env vars.
|
||||
const requireToken = process.env.REQUIRE_WS_TOKEN === 'true';
|
||||
const providedToken = request.query?.token;
|
||||
|
||||
if (requireToken && providedToken !== process.env.API_SECRET) {
|
||||
connection.send(JSON.stringify({ error: 'Unauthorized: Invalid or missing token' }));
|
||||
connection.close(1008);
|
||||
return;
|
||||
}
|
||||
|
||||
connections.add(connection);
|
||||
subscriptions.set(connection, new Set());
|
||||
|
||||
connection.on('message', async (message: string) => {
|
||||
try {
|
||||
const data = JSON.parse(message);
|
||||
if (data.action === 'subscribe' && data.pluginId) {
|
||||
subscriptions.get(connection)?.add(data.pluginId);
|
||||
|
||||
// Push the most recent cached snapshot to the client immediately upon subscribing
|
||||
// This eliminates the "half a minute wait" for cron-driven seeders to flush
|
||||
const latestSnapshot = await getLiveSnapshot(data.pluginId);
|
||||
if (latestSnapshot && connections.has(connection)) {
|
||||
connection.send(JSON.stringify({
|
||||
type: 'data',
|
||||
pluginId: data.pluginId,
|
||||
payload: latestSnapshot
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (data.action === 'unsubscribe' && data.pluginId) {
|
||||
subscriptions.get(connection)?.delete(data.pluginId);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[WS] Invalid message error/fetch error', e);
|
||||
}
|
||||
});
|
||||
|
||||
connection.on('close', () => {
|
||||
connections.delete(connection);
|
||||
subscriptions.delete(connection);
|
||||
});
|
||||
}
|
||||
|
||||
export function broadcastPluginData(pluginId: string, payload: any) {
|
||||
const message = JSON.stringify({ type: 'data', pluginId, payload });
|
||||
for (const connection of connections) {
|
||||
if (subscriptions.get(connection)?.has(pluginId)) {
|
||||
connection.send(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const db = new Database('./data/engine.db');
|
||||
|
||||
try {
|
||||
const rows = db.prepare('SELECT payload FROM iranwar_events').all();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
JSON.parse(row.payload);
|
||||
} catch(err) {
|
||||
console.log('JSON Parse Failed for:', row.payload);
|
||||
}
|
||||
}
|
||||
console.log('Tested rows correctly');
|
||||
} catch(e) {
|
||||
console.error(e);
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
fetch('http://localhost:5050/data/iranwarlive/history')
|
||||
.then(res => Promise.all([res.status, res.text()]))
|
||||
.then(([status, body]) => console.log('Status:', status, '\nBody:', body))
|
||||
.catch(console.error);
|
||||
@@ -1,39 +0,0 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
|
||||
async function fetchWithTimeout(url, options = {}, timeoutMs = 15000) {
|
||||
const controller = new AbortController();
|
||||
const id = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} - ${response.statusText}`);
|
||||
}
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(id);
|
||||
}
|
||||
}
|
||||
|
||||
async function test() {
|
||||
const OTX_BASE = 'https://otx.alienvault.com/api/v1';
|
||||
const apiKey = process.env.OTX_API_KEY;
|
||||
const since = new Date(Date.now() - 48 * 3600 * 1000).toISOString();
|
||||
const url = `${OTX_BASE}/pulses/subscribed?modified_since=${since}&limit=50`;
|
||||
|
||||
console.log('Testing fetchWithTimeout inside the application context...');
|
||||
console.log('Using API key:', apiKey ? apiKey.substring(0, 5) + '...' + apiKey.substring(apiKey.length - 5) : 'undefined');
|
||||
try {
|
||||
const res = await fetchWithTimeout(url, {
|
||||
headers: { 'X-OTX-API-KEY': apiKey }
|
||||
});
|
||||
console.log('Status:', res.status);
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -1,14 +0,0 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
import { seedCyberAttacks } from './src/seeders/cyberAttacks.ts';
|
||||
|
||||
async function test() {
|
||||
console.log('Testing seedCyberAttacks...');
|
||||
try {
|
||||
await seedCyberAttacks();
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -1,14 +0,0 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
import { seedCyberAttacks } from './src/seeders/cyberAttacks.js';
|
||||
|
||||
async function test() {
|
||||
console.log('Testing seedCyberAttacks...');
|
||||
try {
|
||||
await seedCyberAttacks();
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2022",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
import Redis from 'ioredis';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env.local') });
|
||||
|
||||
const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6379');
|
||||
|
||||
async function checkAviation() {
|
||||
console.log('Checking Aviation Live Snapshot...');
|
||||
const data = await redis.get('live_snapshot_aviation');
|
||||
if (!data) {
|
||||
console.log('No aviation snapshot found in Redis.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const snapshot = JSON.parse(data);
|
||||
const aircraft = Object.values(snapshot);
|
||||
if (aircraft.length === 0) {
|
||||
console.log('Snapshot is empty.');
|
||||
} else {
|
||||
const first = aircraft[0];
|
||||
console.log(`Found ${aircraft.length} aircraft.`);
|
||||
console.log(`Last updated: ${new Date(first.last_updated).toLocaleString()}`);
|
||||
console.log(`Current time: ${new Date().toLocaleString()}`);
|
||||
|
||||
const diffSec = (Date.now() - first.last_updated) / 1000;
|
||||
console.log(`Data age: ${diffSec.toFixed(2)} seconds`);
|
||||
|
||||
if (diffSec < 10) {
|
||||
console.log('✅ Polling verified: Data is fresh (under 10s).');
|
||||
} else {
|
||||
console.log('❌ Polling stall: Data is stale.');
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
checkAviation().catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-lib-aviation",
|
||||
"version": "1.0.0",
|
||||
"description": "Shared abstract base classes for WorldWideView aviation plugins (commercial, military, specialized)",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": [
|
||||
"src"
|
||||
],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"aviation",
|
||||
"aircraft",
|
||||
"base-plugin"
|
||||
],
|
||||
"author": "WorldWideView",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import type {
|
||||
WorldPlugin,
|
||||
GeoEntity,
|
||||
TimeRange,
|
||||
PluginContext,
|
||||
LayerConfig,
|
||||
CesiumEntityOptions,
|
||||
SelectionBehavior,
|
||||
ServerPluginConfig,
|
||||
FilterDefinition
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
/**
|
||||
* Base abstraction for aviation-based plugins.
|
||||
* Inheriting plugins inherit default clustering, models, selection, and rendering logic.
|
||||
*/
|
||||
export abstract class BaseAviationPlugin implements WorldPlugin {
|
||||
abstract id: string;
|
||||
abstract name: string;
|
||||
abstract description: string;
|
||||
abstract icon: any;
|
||||
category = "aviation" as const;
|
||||
abstract version: string;
|
||||
|
||||
protected context: PluginContext | null = null;
|
||||
|
||||
// Configurable rendering defaults
|
||||
protected defaultLayerColor = "#3b82f6";
|
||||
protected defaultTrailColor = "#00fff7";
|
||||
protected iconUrl = "/plane-icon.svg";
|
||||
protected modelUrl = "/airplane/scene.gltf";
|
||||
|
||||
async initialize(ctx: PluginContext): Promise<void> { this.context = ctx; }
|
||||
destroy(): void { this.context = null; }
|
||||
|
||||
// Implementations must define how payload comes via fetch and websocket
|
||||
abstract fetch(timeRange: TimeRange): Promise<GeoEntity[]>;
|
||||
abstract mapWebsocketPayload(payload: any): GeoEntity[];
|
||||
|
||||
// Server backend configurations
|
||||
abstract getServerConfig(): ServerPluginConfig;
|
||||
|
||||
// Legends and Filters
|
||||
abstract getFilterDefinitions(): FilterDefinition[];
|
||||
abstract getLegend(): { label: string; color: string; filterId?: string; filterValue?: string }[];
|
||||
|
||||
getPollingInterval(): number {
|
||||
return 0; // Disabled in favor of WebSocket firehose
|
||||
}
|
||||
|
||||
getLayerConfig(): LayerConfig {
|
||||
return {
|
||||
color: this.defaultLayerColor,
|
||||
clusterEnabled: true,
|
||||
clusterDistance: 40,
|
||||
maxEntities: 5000
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Define the color based on the altitude (meters) specific to the aviation domain
|
||||
*/
|
||||
protected abstract getAltitudeColor(altitudeMeters: number | null): string;
|
||||
|
||||
renderEntity(entity: GeoEntity): CesiumEntityOptions {
|
||||
const alt = entity.properties.altitude_m as number | null;
|
||||
const isAirborne = !entity.properties.on_ground;
|
||||
|
||||
return {
|
||||
type: "model",
|
||||
iconUrl: this.iconUrl,
|
||||
size: isAirborne ? 8 : 5,
|
||||
modelUrl: this.modelUrl,
|
||||
modelScale: 2.56,
|
||||
modelMinPixelSize: 16,
|
||||
modelHeadingOffset: 180,
|
||||
color: this.getAltitudeColor(alt),
|
||||
rotation: entity.heading,
|
||||
labelText: entity.label || undefined,
|
||||
labelFont: "11px JetBrains Mono, monospace"
|
||||
};
|
||||
}
|
||||
|
||||
getSelectionBehavior(entity: GeoEntity): SelectionBehavior | null {
|
||||
if (entity.properties.on_ground) return null;
|
||||
return {
|
||||
showTrail: true,
|
||||
trailDurationSec: 60,
|
||||
trailStepSec: 5,
|
||||
trailColor: this.defaultTrailColor,
|
||||
flyToOffsetMultiplier: 3,
|
||||
flyToBaseDistance: 30000
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"declaration": true,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"jsx": "react-jsx"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-lib-facilities",
|
||||
"version": "1.0.0",
|
||||
"description": "Shared abstract base classes for WorldWideView facility plugins (airports, seaports, nuclear sites)",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": [
|
||||
"src"
|
||||
],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"facility",
|
||||
"infrastructure",
|
||||
"base-plugin"
|
||||
],
|
||||
"author": "WorldWideView",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import {
|
||||
createSvgIconUrl,
|
||||
type WorldPlugin,
|
||||
type GeoEntity,
|
||||
type TimeRange,
|
||||
type PluginContext,
|
||||
type LayerConfig,
|
||||
type CesiumEntityOptions,
|
||||
type ServerPluginConfig,
|
||||
type PluginCategory
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
export abstract class BaseFacilityPlugin implements WorldPlugin {
|
||||
abstract id: string;
|
||||
abstract name: string;
|
||||
abstract description: string;
|
||||
abstract icon: any;
|
||||
abstract category: PluginCategory;
|
||||
abstract version: string;
|
||||
|
||||
protected context: PluginContext | null = null;
|
||||
protected iconUrls: Record<string, string> = {};
|
||||
|
||||
// Configurable rendering defaults
|
||||
protected defaultLayerColor = "#3b82f6";
|
||||
protected clusterDistance = 50;
|
||||
protected maxEntities = 5000;
|
||||
protected iconScale = 1.0;
|
||||
|
||||
async initialize(ctx: PluginContext): Promise<void> { this.context = ctx; }
|
||||
destroy(): void { this.context = null; }
|
||||
|
||||
// Generally facilities are static plugins (data loaded via mapJsonToEntities locally),
|
||||
// but we stub this out just in case
|
||||
async fetch(_timeRange: TimeRange): Promise<GeoEntity[]> { return []; }
|
||||
getPollingInterval(): number { return 0; }
|
||||
|
||||
getLayerConfig(): LayerConfig {
|
||||
return {
|
||||
color: this.defaultLayerColor,
|
||||
clusterEnabled: true,
|
||||
clusterDistance: this.clusterDistance,
|
||||
maxEntities: this.maxEntities
|
||||
};
|
||||
}
|
||||
|
||||
protected getEntityColor(entity: GeoEntity): string {
|
||||
return this.defaultLayerColor;
|
||||
}
|
||||
|
||||
protected getEntityIcon(entity: GeoEntity): any {
|
||||
return this.icon;
|
||||
}
|
||||
|
||||
renderEntity(entity: GeoEntity): CesiumEntityOptions {
|
||||
const color = this.getEntityColor(entity);
|
||||
const iconComponent = this.getEntityIcon(entity);
|
||||
|
||||
const cacheKey = `${iconComponent?.displayName || iconComponent?.name || "default"}-${color}`;
|
||||
|
||||
if (!this.iconUrls[cacheKey]) {
|
||||
this.iconUrls[cacheKey] = createSvgIconUrl(iconComponent, { color });
|
||||
}
|
||||
|
||||
return {
|
||||
type: "billboard",
|
||||
iconUrl: this.iconUrls[cacheKey],
|
||||
color: color,
|
||||
iconScale: this.iconScale
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"declaration": true,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"jsx": "react-jsx"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-lib-incidents",
|
||||
"version": "1.0.0",
|
||||
"description": "Shared abstract base classes for WorldWideView incident plugins (earthquakes, wildfires, conflicts)",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": [
|
||||
"src"
|
||||
],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"incident",
|
||||
"disaster",
|
||||
"base-plugin"
|
||||
],
|
||||
"author": "WorldWideView",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*"
|
||||
}
|
||||
}
|
||||
Loaded 100 of 334 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user