Compare commits
248
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc53db9012 | ||
|
|
d05f93e610 | ||
|
|
701b7b3b23 | ||
|
|
1eff015bbb | ||
|
|
84129f1d9e | ||
|
|
60dbc5af66 | ||
|
|
8abc2ea2ef | ||
|
|
76243e08c0 | ||
|
|
57d6bbbe02 | ||
|
|
e1910011aa | ||
|
|
f81d9315e0 | ||
|
|
48cf43e68d | ||
|
|
f30a539ebf | ||
|
|
57a602eb60 | ||
|
|
a2c8dd2f55 | ||
|
|
f272a9a617 | ||
|
|
8fe71c637d | ||
|
|
aee5154eaf | ||
|
|
ae8c422c2f | ||
|
|
e61928b2a1 | ||
|
|
7749804365 | ||
|
|
c115058008 | ||
|
|
86c97ec191 | ||
|
|
b5b925dd06 | ||
|
|
7488677f7a | ||
|
|
14a8bf5369 | ||
|
|
96e4a1c591 | ||
|
|
6cb6dfd1d6 | ||
|
|
c31871b321 | ||
|
|
245118412f | ||
|
|
a75a4f0f9a | ||
|
|
f8664d8255 | ||
|
|
dad91079b2 | ||
|
|
273112424c | ||
|
|
9cc51f483a | ||
|
|
0adef65616 | ||
|
|
5b2b8a0a8d | ||
|
|
f33e7efb8d | ||
|
|
425dcbbc8e | ||
|
|
484104a2b0 | ||
|
|
7e34ab6da0 | ||
|
|
5b45d79790 | ||
|
|
7684d23398 | ||
|
|
a5610f0c8d | ||
|
|
4d178544e5 | ||
|
|
8a58f4a732 | ||
|
|
026eaae074 | ||
|
|
6515b43711 | ||
|
|
94a0ca4731 | ||
|
|
de60ed3ffe | ||
|
|
f5898b253a | ||
|
|
561a44fe77 | ||
|
|
b431fc526a | ||
|
|
3deefc9607 | ||
|
|
20363d509b | ||
|
|
134c3a5443 | ||
|
|
ab4cb2553a | ||
|
|
3bc3b3461f | ||
|
|
11a75b6458 | ||
|
|
62b8436353 | ||
|
|
875eda7ea2 | ||
|
|
82246daab0 | ||
|
|
7513ff9dfa | ||
|
|
96b032c2d0 | ||
|
|
de4f77d250 | ||
|
|
7c59ba4dc3 | ||
|
|
951e070466 | ||
|
|
adadce06b1 | ||
|
|
2341409bb4 | ||
|
|
7fee8e9fcf | ||
|
|
682c21296c | ||
|
|
aec6f20edc | ||
|
|
c89b55884f | ||
|
|
ded3ae9416 | ||
|
|
0808326967 | ||
|
|
889bd5aade | ||
|
|
a17aafa6f6 | ||
|
|
18da3a492c | ||
|
|
2fd3f84b2f | ||
|
|
68d350572f | ||
|
|
7660e707f7 | ||
|
|
f510c7de34 | ||
|
|
35e9e7d70d | ||
|
|
74f3793313 | ||
|
|
913e25ccac | ||
|
|
2ec151c68b | ||
|
|
cf49eef883 | ||
|
|
b56dacf04f | ||
|
|
22bb77492e | ||
|
|
ca1447f26a | ||
|
|
7f1bf00ef7 | ||
|
|
aeb9ddb96b | ||
|
|
3852e1175d | ||
|
|
1474e28de0 | ||
|
|
8a5ace61b2 | ||
|
|
932713e7a2 | ||
|
|
50228a43c8 | ||
|
|
326c69e92f | ||
|
|
e3225d4dab | ||
|
|
23d1538ea2 | ||
|
|
5f89715ee6 | ||
|
|
22f22ead37 | ||
|
|
5eff117972 | ||
|
|
b92e5b3683 | ||
|
|
b79f0f025d | ||
|
|
f40615ee21 | ||
|
|
222ae0950e | ||
|
|
afe89f8c0e | ||
|
|
867e495353 | ||
|
|
6263f7766b | ||
|
|
d65600786f | ||
|
|
a3a04d84c0 | ||
|
|
f295609ea2 | ||
|
|
80064996e9 | ||
|
|
973bef6386 | ||
|
|
4e2287aa5f | ||
|
|
17b8c7bb62 | ||
|
|
d10c68004a | ||
|
|
10830efca5 | ||
|
|
c3d9b2b536 | ||
|
|
3a04c64e79 | ||
|
|
eea424a826 | ||
|
|
4edcce2468 | ||
|
|
d54d9a480c | ||
|
|
b7d76b5e58 | ||
|
|
e83e45e8b9 | ||
|
|
86b7550c38 | ||
|
|
09b4b44c3e | ||
|
|
c6b26d20bc | ||
|
|
36ee84857b | ||
|
|
25b19ec635 | ||
|
|
614d3af796 | ||
|
|
bc05d93ce8 | ||
|
|
c0789f831c | ||
|
|
3be23a3ebc | ||
|
|
abd40ede1b | ||
|
|
c7d980390d | ||
|
|
aab5e39aa3 | ||
|
|
05bc8856f6 | ||
|
|
562a53e853 | ||
|
|
419744e64b | ||
|
|
b008537694 | ||
|
|
11919731d2 | ||
|
|
2a285312b0 | ||
|
|
027672d885 | ||
|
|
e016342dea | ||
|
|
a56e2b6340 | ||
|
|
b1fa384476 | ||
|
|
efa886ad3e | ||
|
|
3d25b2642a | ||
|
|
ca7125f3ac | ||
|
|
c247e3dde0 | ||
|
|
a25d7ce09b | ||
|
|
1bbd97b382 | ||
|
|
5999ef7d8d | ||
|
|
a764eefcd8 | ||
|
|
f2bd3fecfe | ||
|
|
ee5702f21e | ||
|
|
c4e9dd3951 | ||
|
|
2e928d2d64 | ||
|
|
8ec9465274 | ||
|
|
de40d79662 | ||
|
|
853c173c37 | ||
|
|
fcef4d46fa | ||
|
|
3e8a54a541 | ||
|
|
5c97ceaa64 | ||
|
|
9312c40862 | ||
|
|
52a08dfd7b | ||
|
|
fe1c80e798 | ||
|
|
bce7da4006 | ||
|
|
534ccc03af | ||
|
|
0736b7a051 | ||
|
|
c706d83009 | ||
|
|
6b3f351d38 | ||
|
|
62c333e99a | ||
|
|
e46096dd87 | ||
|
|
48c4045f84 | ||
|
|
c656c86baa | ||
|
|
975c4b1d38 | ||
|
|
38fc546b12 | ||
|
|
66f9d705d7 | ||
|
|
652aa3f94f | ||
|
|
e6995e3a17 | ||
|
|
ba8d031d19 | ||
|
|
4d33d434c0 | ||
|
|
350f70c34f | ||
|
|
0c54d6ca1d | ||
|
|
9a92640469 | ||
|
|
44c05efd95 | ||
|
|
b19ec0af6e | ||
|
|
0a3d7c96ea | ||
|
|
665ad5317c | ||
|
|
5ca9be2070 | ||
|
|
e6dc172871 | ||
|
|
74d61f3e8c | ||
|
|
875e12cb22 | ||
|
|
c773eacceb | ||
|
|
91123ecc1b | ||
|
|
51b7fe01d5 | ||
|
|
f98615e198 | ||
|
|
0d795bf5e8 | ||
|
|
76b7fea27b | ||
|
|
2e93b45927 | ||
|
|
8b7b4204b4 | ||
|
|
5d62ce138a | ||
|
|
706fde7610 | ||
|
|
7db1980e0f | ||
|
|
60a584d03f | ||
|
|
9f9e0fb81a | ||
|
|
665893ef80 | ||
|
|
f5ea9d6432 | ||
|
|
f4c420a665 | ||
|
|
6c6f44cc7e | ||
|
|
c4f32ece18 | ||
|
|
6cfef39e1b | ||
|
|
bccdc6aa26 | ||
|
|
fcdea75bdc | ||
|
|
9748d02820 | ||
|
|
39c7324872 | ||
|
|
53f4a02d07 | ||
|
|
3f6c9bf7c7 | ||
|
|
3df54a8c47 | ||
|
|
1d90209d51 | ||
|
|
f64d5a61ba | ||
|
|
c995716d02 | ||
|
|
899d23e2e5 | ||
|
|
59b0c638c7 | ||
|
|
9e8ea82480 | ||
|
|
3c516ff021 | ||
|
|
e77626d84d | ||
|
|
5f85a1d9e8 | ||
|
|
4845b72cc8 | ||
|
|
3522c7646f | ||
|
|
97188a9c64 | ||
|
|
90b9c029df | ||
|
|
c2722856d8 | ||
|
|
1505b0902b | ||
|
|
83752b96c9 | ||
|
|
9db38cb0de | ||
|
|
522e976739 | ||
|
|
19b521c5ca | ||
|
|
e8e2311422 | ||
|
|
41e2413af4 | ||
|
|
a3e19cecef | ||
|
|
62daafc4a0 | ||
|
|
3f17dd21f8 | ||
|
|
400ac5aa13 | ||
|
|
83d329e407 |
No files matched your search
@@ -0,0 +1,16 @@
|
||||
# Database Migrations & Schema Modification Rules
|
||||
|
||||
## 1. Defensive Migrations
|
||||
When making database schema changes, you must ensure that generated migrations are resilient against environmental discrepancies.
|
||||
|
||||
- **Review Generated SQL**: Always review the `.sql` files generated by `npx prisma migrate dev`.
|
||||
- **Add "IF EXISTS" clauses**: SQLite and Prisma can sometimes produce brittle drop statements (`DROP INDEX "my_index"`). To prevent migrations from crashing if a state desynchronization occurs (e.g., the index was already dropped or never created), manually edit the migration file to use `DROP INDEX IF EXISTS "my_index";` or `DROP TABLE IF EXISTS "table_name";`.
|
||||
- **Data Preservation**: When dropping columns or renaming, SQLite does not cleanly support it and requires table recreation. Carefully review the `INSERT INTO "new_table" SELECT ...` clauses to ensure data is correctly mapped.
|
||||
|
||||
## 2. Managing Failed Migrations
|
||||
If a migration fails in production:
|
||||
- **Never blindly mark as resolved**: Do not run `prisma migrate resolve --applied <migration_name>` unless you have explicitly verified that the database schema precisely matches the expected state.
|
||||
- If you use `resolve --applied`, Prisma will permanently skip that migration in the future, leaving your database structurally out of sync with your Prisma Client.
|
||||
|
||||
## 3. Backups Before DDL
|
||||
Never perform direct DDL operations (ALTER, DROP, CREATE) on a production database (via Prisma or direct SQLite access) without first confirming an automated backup exists, or manually taking a backup of the `wwv.db` file.
|
||||
@@ -1,95 +0,0 @@
|
||||
---
|
||||
trigger: model_decision
|
||||
description: This is an explanation of the purpose of the project
|
||||
---
|
||||
|
||||
WorldWideView is best described as a modular, real‑time geospatial intelligence engine built on top of a CesiumJS 3D globe, designed to ingest live data streams, transform them through a plugin system, and render them as high‑performance visual layers. The repository you have open shows a Next.js application with a generalized plugin architecture and a Cesium viewer at its core .
|
||||
|
||||
🌍 What WorldWideView fundamentally is
|
||||
WorldWideView is a real‑time situational awareness platform. It takes raw, fast‑moving geospatial signals—like aircraft positions, maritime AIS, satellite orbits, or any other sensor feed—and turns them into interactive, cinematic layers on a 3D globe. The system is built to be:
|
||||
|
||||
Modular, so each data source is a plugin.
|
||||
|
||||
High‑performance, using Cesium primitives instead of entities.
|
||||
|
||||
Extensible, allowing new data types without rewriting the core.
|
||||
|
||||
Real‑time, updating the globe smoothly as new data arrives.
|
||||
|
||||
Visual, using icons, arrows, and 3D models to represent moving objects.
|
||||
|
||||
It’s essentially your own open, customizable version of tools like SkyGlass, ADSBexchange 3D, or WorldView (Bilawal Sidhu’s demo), but engineered for long‑term scalability.
|
||||
|
||||
🧩 Core components of the system
|
||||
Plugin system (your recent commits)
|
||||
Your latest commit message—“generalize plugin system, remove aviation special treatment” —shows that the architecture now treats every data source as a plugin. This means:
|
||||
|
||||
Each plugin defines how it fetches, parses, and visualizes data.
|
||||
|
||||
The core engine doesn’t care whether the plugin is aviation, maritime, satellites, or something else.
|
||||
|
||||
You can add new intelligence layers without touching the core.
|
||||
|
||||
This is a major architectural milestone.
|
||||
|
||||
Cesium rendering engine
|
||||
The project initializes a Cesium globe inside a Next.js app. Cesium handles:
|
||||
|
||||
Globe rendering
|
||||
|
||||
Camera control
|
||||
|
||||
Terrain and imagery layers
|
||||
|
||||
Worker‑based geometry processing
|
||||
|
||||
High‑performance primitives for thousands of objects
|
||||
|
||||
Your open tabs (Cesium picking performance, CZML, map type switching) show you’re actively tuning this layer.
|
||||
|
||||
Real‑time data ingestion
|
||||
The system is designed to ingest live feeds such as:
|
||||
|
||||
ADS‑B aircraft positions
|
||||
|
||||
AIS maritime positions
|
||||
|
||||
Satellite TLE propagation
|
||||
|
||||
Custom sensor networks
|
||||
|
||||
Each plugin transforms raw data into Cesium‑ready primitives.
|
||||
|
||||
Visual representation
|
||||
Your recent work on low‑poly aircraft icons and arrow models shows the platform is moving toward:
|
||||
|
||||
Lightweight, directional markers
|
||||
|
||||
Orientation‑aware icons
|
||||
|
||||
Smooth fly‑to tracking
|
||||
|
||||
Cinematic camera behavior
|
||||
|
||||
This is what gives the platform its “WorldView‑like” feel.
|
||||
|
||||
🚀 What makes WorldWideView unique
|
||||
1. It’s modular from the ground up
|
||||
Most geospatial dashboards hard‑code aviation or maritime logic. You’ve removed that. Any data source can become a plugin.
|
||||
|
||||
2. It’s built for performance
|
||||
You’re migrating from Cesium Entities to Primitives, optimizing picking, and reducing GPU stalls—this is what allows thousands of objects to render smoothly.
|
||||
|
||||
3. It’s designed for real‑time intelligence
|
||||
Not just visualization—actual situational awareness. The architecture supports:
|
||||
|
||||
live updates
|
||||
|
||||
event‑driven pipelines
|
||||
|
||||
time‑dynamic objects
|
||||
|
||||
replay capability (future feature)
|
||||
|
||||
4. It’s open and extensible
|
||||
Unlike proprietary tools, your system can be extended by anyone who writes a plugin.
|
||||
@@ -1,12 +0,0 @@
|
||||
---
|
||||
description: cesium
|
||||
---
|
||||
|
||||
https://cesium.com/
|
||||
https://cesium.com/platform/cesiumjs/
|
||||
https://cesium.com/platform/cesiumjs/ion-sdk/
|
||||
https://sandcastle.cesium.com/?id=czml
|
||||
https://cesium.com/learn/cesiumjs-learn/
|
||||
https://cesium.com/learn/cesiumjs-learn/cesiumjs-camera/
|
||||
https://cesium.com/learn/cesiumjs-learn/cesiumjs-geometry-appearances/
|
||||
https://cesium.com/learn/cesiumjs-learn/cesiumjs-flight-tracker/
|
||||
@@ -0,0 +1,25 @@
|
||||
node_modules
|
||||
**/node_modules
|
||||
**/dist
|
||||
**/.next
|
||||
.git
|
||||
|
||||
# Database files at ANY depth
|
||||
/data
|
||||
packages/*/data
|
||||
**/*.db
|
||||
**/*.sqlite
|
||||
**/*.sqlite3
|
||||
**/*-journal
|
||||
**/*.db-wal
|
||||
**/*.db-shm
|
||||
|
||||
# Environment files
|
||||
.env*
|
||||
!*.env.example
|
||||
|
||||
# Non-build directories
|
||||
artifacts/
|
||||
performance/
|
||||
tmp/
|
||||
*.geojson
|
||||
@@ -0,0 +1,11 @@
|
||||
DATABASE_URL=file:./data/wwv.db
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
|
||||
# Auth Secret (REQUIRED for Docker — generate with: openssl rand -hex 32)
|
||||
# Must be stable across container restarts to preserve sessions.
|
||||
# [WARNING] NEVER commit real secrets here. Real secrets must ONLY go into .env.local!
|
||||
AUTH_SECRET=changeme_d64c1f35b447da89cf6ed5a0988da1379b21904edb91a87c3f5a63942e3d23a4_changeme
|
||||
|
||||
|
||||
+46
-4
@@ -1,15 +1,57 @@
|
||||
# Environment Variables Example
|
||||
# ============================================================
|
||||
# .env — committed to git (non-secret defaults)
|
||||
# ============================================================
|
||||
# This file is tracked by git. Only put non-secret values here.
|
||||
# Prisma reads DATABASE_URL from this file.
|
||||
|
||||
# Cesium Ion Access Token (Required for terrain and default imagery)
|
||||
# Get your token at https://ion.cesium.com/
|
||||
# Database (SQLite for local — zero config; PostgreSQL for cloud)
|
||||
DATABASE_URL=file:./data/wwv.db
|
||||
|
||||
# ============================================================
|
||||
# .env.local — NOT committed (secrets & API keys)
|
||||
# ============================================================
|
||||
# Copy everything below into a new file called .env.local
|
||||
# .env.local is gitignored and overrides .env values.
|
||||
|
||||
# Auth Secret (REQUIRED — generate with: openssl rand -hex 32)
|
||||
# Must remain stable across restarts to preserve sessions and tokens.
|
||||
# For Docker: also set this in the host environment or a .env file
|
||||
# at the project root so docker-compose can inject it.
|
||||
AUTH_SECRET=
|
||||
|
||||
# Cesium Ion Access Token (optional — get one free at https://ion.cesium.com/)
|
||||
NEXT_PUBLIC_CESIUM_ION_TOKEN=
|
||||
|
||||
# Bing Maps API Key (Optional, for high-res imagery)
|
||||
# Get your key at https://www.bingmapsportal.com/
|
||||
NEXT_PUBLIC_BING_MAPS_KEY=
|
||||
|
||||
# Supabase Configuration (Required for persistence and historical data)
|
||||
# Supabase Configuration (Optional — cloud-only features)
|
||||
# Found in your Supabase Project Settings > API
|
||||
NEXT_PUBLIC_SUPABASE_URL=
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY=
|
||||
SUPABASE_SERVICE_ROLE_KEY=
|
||||
SUPABASE_DATABASE_URL=
|
||||
|
||||
# OpenSky Network — multi-account credential rotation
|
||||
# Comma-separated clientId:clientSecret pairs.
|
||||
# Example: OPENSKY_CREDENTIALS=acct1-id:acct1-secret,acct2-id:acct2-secret
|
||||
OPENSKY_CREDENTIALS=
|
||||
|
||||
# ACLED API — Civil Unrest & Conflict data (free for non-commercial use)
|
||||
# Register at https://developer.acleddata.com/ to obtain credentials.
|
||||
# Data is updated weekly (Mondays) with global protest, riot, and violence events.
|
||||
ACLED_EMAIL=
|
||||
ACLED_PASSWORD=
|
||||
|
||||
# Marketplace Bridge Token — shared secret for marketplace → WWV install requests
|
||||
# Generate any random string, then set the same value in the marketplace instance config.
|
||||
WWV_BRIDGE_TOKEN=
|
||||
|
||||
# Edition: "local" | "cloud" | "demo"
|
||||
NEXT_PUBLIC_WWV_EDITION=local
|
||||
|
||||
# Demo Admin Secret (demo edition only)
|
||||
# Allows operator plugin management on the demo instance via x-wwv-admin-secret header.
|
||||
# Generate any random string. Only set this on the demo deployment.
|
||||
# WWV_DEMO_ADMIN_SECRET=
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Create a report to help us improve WorldWideView
|
||||
title: '[BUG] '
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots/Logs**
|
||||
If applicable, add screenshots or console output to help explain your problem.
|
||||
|
||||
**Environment (please complete the following information):**
|
||||
- OS: [e.g. Windows, macOS]
|
||||
- Browser [e.g. chrome, safari, firefox]
|
||||
- WorldWideView Edition [e.g. local, demo, cloud]
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
name: Core Feature Request
|
||||
about: Suggest an idea for the core WorldWideView engine
|
||||
title: '[FEATURE] '
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Note:** If this is about adding a new data source (like mapping vessels or plotting weather), please use the **Plugin Proposal** template instead! This template is for core engine features (like camera logic, filtering, or rendering pipeline upgrades).
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
name: Plugin Proposal
|
||||
about: Suggest a new real-time data source plugin
|
||||
title: '[PLUGIN] '
|
||||
labels: plugin-idea, help-wanted
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**What data source do you want to add?**
|
||||
Please provide a brief description of the intelligence data (e.g., Global Earthquakes, Shipping Vessels, Weather Radar, Local Train Networks).
|
||||
|
||||
**Is there an available API?**
|
||||
Please link to any public or authenticated APIs that provide this data.
|
||||
*(e.g., USGS Earthquake API: https://earthquake.usgs.gov/earthquakes/feed/v1.0/geojson.php)*
|
||||
|
||||
**What should the plugin render?**
|
||||
- What kind of marker? (e.g., dots, icons, 3D models)
|
||||
- Does it require historical tracking (paths), or just current position?
|
||||
|
||||
**Would you be interested in building this?**
|
||||
- [ ] Yes, I'd like to try building it!
|
||||
- [ ] No, just putting the idea out there.
|
||||
|
||||
*(Looking to build it yourself? Check out the [Build a Plugin in 5 Minutes](https://github.com/silvertakana/worldwideview#build-a-plugin-in-5-minutes) guide in the README!)*
|
||||
@@ -0,0 +1,44 @@
|
||||
## Summary
|
||||
|
||||
<!-- A short description of what this PR does and why. -->
|
||||
|
||||
## Type of Change
|
||||
|
||||
- [ ] Bug fix
|
||||
- [ ] New feature
|
||||
- [ ] New plugin
|
||||
- [ ] Refactor / code quality
|
||||
- [ ] Documentation
|
||||
- [ ] Performance improvement
|
||||
|
||||
## Related Issue
|
||||
|
||||
Closes #<!-- issue number -->
|
||||
|
||||
## Changes Made
|
||||
|
||||
<!-- List the key changes. Be specific about files and modules affected. -->
|
||||
|
||||
-
|
||||
-
|
||||
|
||||
## Testing
|
||||
|
||||
- [ ] I ran `npm test` and all tests pass
|
||||
- [ ] I manually tested this in the browser against a live data source
|
||||
- [ ] I added or updated tests for my changes
|
||||
|
||||
## Plugin Checklist (if applicable)
|
||||
|
||||
- [ ] Plugin is registered with `PluginRegistry`
|
||||
- [ ] Plugin does not import or depend on core rendering logic directly
|
||||
- [ ] Plugin cleans up Cesium primitives on unmount/disable
|
||||
- [ ] No hardcoded credentials or API keys committed
|
||||
|
||||
## Screenshots / Recordings
|
||||
|
||||
<!-- If this changes the UI or globe rendering, include a screenshot or screen recording. -->
|
||||
|
||||
## Notes for Reviewer
|
||||
|
||||
<!-- Any context that would help the reviewer understand the approach or trade-offs. -->
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Claude Code Review
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, ready_for_review, reopened]
|
||||
# Optional: Only run on specific file changes
|
||||
# paths:
|
||||
# - "src/**/*.ts"
|
||||
# - "src/**/*.tsx"
|
||||
# - "src/**/*.js"
|
||||
# - "src/**/*.jsx"
|
||||
|
||||
jobs:
|
||||
claude-review:
|
||||
# Optional: Filter by PR author
|
||||
# if: |
|
||||
# github.event.pull_request.user.login == 'external-contributor' ||
|
||||
# github.event.pull_request.user.login == 'new-developer' ||
|
||||
# github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code Review
|
||||
id: claude-review
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
|
||||
plugins: 'code-review@claude-code-plugins'
|
||||
prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
||||
# prompt: 'Update the pull request description to include a summary of changes.'
|
||||
|
||||
# Optional: Add claude_args to customize behavior and configuration
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
# claude_args: '--allowed-tools Bash(gh pr:*)'
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Publish Plugin to NPM
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "wwv-plugin-*-v*" # e.g. wwv-plugin-aviation-v1.0.1
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # Required for OIDC trusted publishing (no NPM_TOKEN needed)
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Detect package name from tag
|
||||
id: pkg
|
||||
run: |
|
||||
TAG="${{ github.ref_name }}"
|
||||
# Strips the -vX.Y.Z suffix: "wwv-plugin-aviation-v1.0.1" → "wwv-plugin-aviation"
|
||||
PKG=$(echo "$TAG" | sed 's/-v[0-9].*//')
|
||||
echo "package=$PKG" >> $GITHUB_OUTPUT
|
||||
echo "Publishing: $PKG"
|
||||
|
||||
- name: Build package
|
||||
run: npm run build -w packages/${{ steps.pkg.outputs.package }}
|
||||
|
||||
- name: Publish to NPM (OIDC)
|
||||
run: npm publish packages/${{ steps.pkg.outputs.package }} --access public --provenance
|
||||
+36
-1
@@ -1,7 +1,7 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
node_modules/
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
@@ -17,12 +17,37 @@
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# agents
|
||||
/.agents/
|
||||
|
||||
# claude code IDE config
|
||||
/.claude/
|
||||
|
||||
# performance
|
||||
/performance/
|
||||
|
||||
# artifacts
|
||||
/artifacts/
|
||||
|
||||
# scripts
|
||||
/scripts/
|
||||
!/scripts/setup.mjs
|
||||
|
||||
# local database
|
||||
/data/
|
||||
packages/*/data/
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
|
||||
# temporary files and credentials
|
||||
/tmp/
|
||||
|
||||
# generated prisma client
|
||||
/src/generated/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
@@ -38,6 +63,7 @@ yarn-error.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
@@ -45,3 +71,12 @@ yarn-error.log*
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
||||
# Plugin build artifacts
|
||||
packages/*/dist/
|
||||
|
||||
# Reference Reference Codebases
|
||||
/worldmonitor/
|
||||
|
||||
# Developmental scripts & temporary sandbox
|
||||
/local-scripts/
|
||||
Vendored
+36
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Next.js: debug server-side",
|
||||
"type": "node-terminal",
|
||||
"request": "launch",
|
||||
"command": "pnpm run dev",
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"name": "Next.js: debug client-side",
|
||||
"type": "chrome",
|
||||
"request": "launch",
|
||||
"url": "http://localhost:3000",
|
||||
"webRoot": "${workspaceFolder}",
|
||||
"sourceMapPathOverrides": {
|
||||
"webpack://_N_E/*": "${webRoot}/*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "WWV Data Engine: debug",
|
||||
"type": "node-terminal",
|
||||
"request": "launch",
|
||||
"command": "pnpm --filter wwv-data-engine run dev",
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"name": "Full Stack: debug",
|
||||
"type": "node-terminal",
|
||||
"request": "launch",
|
||||
"command": "pnpm run dev:all",
|
||||
"cwd": "${workspaceFolder}"
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "npm: dev:all",
|
||||
"type": "npm",
|
||||
"script": "dev:all",
|
||||
"problemMatcher": [],
|
||||
"isBackground": true,
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"panel": "new"
|
||||
},
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
# WorldWideView — Agent Rules
|
||||
|
||||
## 1. Project Identity
|
||||
|
||||
WorldWideView is a **real-time geospatial intelligence engine** that visualizes live global data on an interactive 3D globe. Built with **Next.js 16**, **CesiumJS**, **React 19**, and **Zustand**, it renders everything from live aircraft and maritime vessels to conflict events, satellites, and environmental data — all through a modular plugin architecture.
|
||||
|
||||
### Target Inspiration
|
||||
Our primary design, feature-set, and operational layout goal is to mimic the structure and capabilities of `www.worldmonitor.app`.
|
||||
- **Reference Codebase**: [GitHub - koala73/worldmonitor](https://github.com/koala73/worldmonitor)
|
||||
---
|
||||
|
||||
## 2. Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|---|---|
|
||||
| Framework | Next.js 16 (App Router, `output: "standalone"`) |
|
||||
| Language | TypeScript 5, strict mode |
|
||||
| 3D Engine | CesiumJS + Resium (Google Photorealistic 3D Tiles) |
|
||||
| State | Zustand (slice-based: globe, layers, timeline, UI, filters, data, config, favorites, geojson) |
|
||||
| Event Bus | Custom typed `DataBus` (pub/sub singleton) |
|
||||
| Styling | Vanilla CSS — **no Tailwind** |
|
||||
| Database | SQLite via Prisma (local), PostgreSQL (cloud) |
|
||||
| Auth | NextAuth v5 beta (Credentials provider, JWT sessions) |
|
||||
| Package Manager | pnpm (monorepo with `pnpm-workspace.yaml`) |
|
||||
| Testing | Vitest + jsdom + React Testing Library |
|
||||
| Deployment | Docker multi-stage build → Coolify |
|
||||
| Analytics | Vercel Analytics / custom `trackEvent` |
|
||||
|
||||
---
|
||||
|
||||
## 3. Directory Structure
|
||||
|
||||
```
|
||||
worldwideview/
|
||||
├── src/
|
||||
│ ├── app/ # Next.js App Router (pages, API routes, layouts)
|
||||
│ │ ├── api/ # Server-side API routes (auth, aviation, camera, etc.)
|
||||
│ │ ├── login/ # Login page
|
||||
│ │ ├── setup/ # First-time setup page
|
||||
│ │ └── globals.css # Root stylesheet
|
||||
│ ├── components/
|
||||
│ │ ├── common/ # Shared UI: BootOverlay, FloatingWindow, PluginIcon
|
||||
│ │ ├── layout/ # AppShell, Header, SearchBar, DataBusSubscriber
|
||||
│ │ ├── panels/ # LayerPanel, EntityInfoCard, FilterPanel, GraphicsSettings
|
||||
│ │ ├── timeline/ # Timeline component
|
||||
│ │ ├── marketplace/ # Plugin install/unverified dialogs
|
||||
│ │ ├── ui/ # Tooltip, ReloadToast
|
||||
│ │ └── video/ # Floating video manager
|
||||
│ ├── core/
|
||||
│ │ ├── data/ # DataBus, PollingManager, CacheLayer, SmartFetcher
|
||||
│ │ ├── filters/ # filterEngine (applies plugin filters to entities)
|
||||
│ │ ├── globe/ # GlobeView, EntityRenderer, AnimationLoop, StackManager,
|
||||
│ │ │ │ CameraController, InteractionHandler, SelectionHandler,
|
||||
│ │ │ │ ModelManager, ImageryProviderFactory
|
||||
│ │ │ └── hooks/ # useCameraActions, useEntityRendering, useModelRendering, etc.
|
||||
│ │ ├── hooks/ # useBootSequence, useIsMobile, useMarketplaceSync
|
||||
│ │ ├── plugins/ # PluginManager, PluginRegistry, PluginManifest,
|
||||
│ │ │ │ loadPluginFromManifest, validateManifest, InstalledPluginsLoader
|
||||
│ │ │ └── loaders/ # DeclarativePlugin, StaticDataPlugin, mapJsonToEntities
|
||||
│ │ └── state/ # Zustand store + slices (config, data, globe, layers, timeline, ui, etc.)
|
||||
│ ├── lib/ # auth, db, rateLimit, analytics, AIS stream, marketplace APIs
|
||||
│ ├── plugins/ # GeoJSON plugin registrations
|
||||
│ ├── styles/ # HUD animations CSS
|
||||
│ ├── types/ # GeoJSON types, Umami types
|
||||
│ └── generated/ # Prisma generated client (gitignored)
|
||||
├── packages/ # pnpm monorepo workspace packages
|
||||
│ ├── wwv-plugin-sdk/ # Plugin SDK: type definitions, manifest schema
|
||||
│ ├── wwv-plugin-aviation/
|
||||
│ ├── wwv-plugin-maritime/
|
||||
│ ├── wwv-plugin-wildfire/
|
||||
│ ├── wwv-plugin-borders/
|
||||
│ ├── wwv-plugin-camera/
|
||||
│ ├── wwv-plugin-military-aviation/
|
||||
│ ├── wwv-plugin-satellite/
|
||||
│ ├── wwv-plugin-iranwarlive/ # Includes standalone backend/ microservice
|
||||
│ └── wwv-plugin-{airports,embassies,lighthouses,nuclear,seaports,spaceports,volcanoes}/
|
||||
├── prisma/ # schema.prisma, migrations/
|
||||
├── public/ # Static assets, Cesium workers, plugin GeoJSON data
|
||||
├── scripts/ # Build scripts (copy-cesium, scaffold-osm-plugin, setup)
|
||||
├── data/ # SQLite database (gitignored, Docker volume)
|
||||
├── Dockerfile # Multi-stage production build
|
||||
├── docker-compose.yml # Main app + microservice backends
|
||||
└── .agents/ # Agent documentation, rules, skills, workflows
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Architecture Patterns
|
||||
|
||||
### 4.1 Plugin System (Core Abstraction)
|
||||
|
||||
Every data source is a **plugin** implementing the `WorldPlugin` interface from `@worldwideview/wwv-plugin-sdk`. The lifecycle utilizes a real-time WebSocket Firehose pipeline:
|
||||
|
||||
```text
|
||||
PluginRegistry.register() → PluginManager.registerPlugin()
|
||||
→ plugin.initialize(context)
|
||||
|
||||
Visibility Toggle → DataBusSubscriber subscribes to layer via WsClient
|
||||
→ Engine responds with instantaneous websocket snapshots over /stream
|
||||
→ WsClient pipes to DataBus.emit("dataUpdated", WsStreamPayload)
|
||||
→ Store → EntityRenderer → Globe
|
||||
```
|
||||
|
||||
Three plugin architectures exist:
|
||||
1. **Static** — GeoJSON file in `public/data/`, loaded by `StaticDataPlugin`
|
||||
2. **Active Proxied** — Next.js API routes in `src/app/api/` as CORS proxy
|
||||
3. **Microservice** — Standalone Fastify container with SQLite (e.g., `iranwarlive-backend`)
|
||||
|
||||
Plugin types are re-exported from SDK through `src/core/plugins/PluginTypes.ts` and `PluginManifest.ts` — **source of truth is always `@worldwideview/wwv-plugin-sdk`**.
|
||||
|
||||
### 4.2 State Management
|
||||
|
||||
Zustand store with **nine slices**: `globe`, `layers`, `timeline`, `ui`, `filter`, `data`, `config`, `favorites`, `geojson`. Each slice is in its own file under `src/core/state/`.
|
||||
|
||||
- Access via `useStore` hook or `useStore.getState()` outside React
|
||||
- Plugin settings stored in `configSlice.dataConfig.pluginSettings`
|
||||
- Polling intervals stored in `configSlice.dataConfig.pollingIntervals`
|
||||
|
||||
### 4.3 Data Pipeline
|
||||
|
||||
```text
|
||||
Engine push /stream → DataBusSubscriber WsClient router
|
||||
→ WsClient.handleMessage() → DataBus.emit("websocketData")
|
||||
→ DataBusSubscriber → _hydrateSnapshot() → Store.entitiesByPlugin
|
||||
→ GlobeView (memoized visible entities)
|
||||
→ EntityRenderer (billboard/point primitives)
|
||||
→ AnimationLoop (horizon culling, hover/selection)
|
||||
→ StackManager (co-located entity grouping)
|
||||
```
|
||||
|
||||
### 4.4 Rendering Pipeline
|
||||
|
||||
- **Primitive-based**: Uses `PointPrimitiveCollection`, `BillboardCollection`, `LabelCollection` — NOT Cesium Entity API
|
||||
- **Chunked processing**: Large datasets (10k+) rendered via `ChunkedProcessor`
|
||||
- **LOD system**: Model-type entities promoted to 3D models at close range (`useModelRendering`)
|
||||
- **Horizon culling**: Manual dot-product calculation against Earth radius (NOT depth testing)
|
||||
- **Stack/Spiderifier**: `StackManager` groups co-located entities; `stackAnimation` handles expansion
|
||||
|
||||
### 4.5 Edition System
|
||||
|
||||
Three editions controlled by `NEXT_PUBLIC_WWV_EDITION`:
|
||||
- **`local`** — Self-hosted, full features, auth enabled
|
||||
- **`cloud`** — Managed cloud instance, full features
|
||||
- **`demo`** — Public demo, auth disabled, optional admin via `WWV_DEMO_ADMIN_SECRET`
|
||||
|
||||
Feature flags derived from edition in `src/core/edition.ts`.
|
||||
|
||||
---
|
||||
|
||||
## 5. Critical Conventions
|
||||
|
||||
### 5.1 File Size
|
||||
|
||||
**Max 150 lines per file.** If a file grows beyond this, modularize it. Extract helpers, split components, use hooks.
|
||||
|
||||
### 5.2 Import Aliases
|
||||
|
||||
- `@/*` → `./src/*`
|
||||
- `@worldwideview/wwv-plugin-sdk` → `./packages/wwv-plugin-sdk/src`
|
||||
- Each plugin has its own alias in `tsconfig.json`
|
||||
|
||||
### 5.3 CSS Rules
|
||||
|
||||
- **Vanilla CSS only** — no Tailwind, no CSS-in-JS
|
||||
- Global styles in `src/app/globals.css`
|
||||
- Component-scoped styles use CSS Modules (`.module.css`) or co-located `.css` files
|
||||
- HUD animations in `src/styles/hud-animations.css`
|
||||
|
||||
### 5.4 Rendering Entity Rules
|
||||
|
||||
When returning `CesiumEntityOptions` from `renderEntity()`:
|
||||
|
||||
- **Points**: Use `type: "point"` with `color`, `size`, `outlineColor`, `outlineWidth`
|
||||
- **Billboards**: Use `type: "billboard"` with `iconUrl`, `color`, `iconScale`
|
||||
- **NEVER mix**: Do not use `size`/`outlineWidth`/`outlineColor` on billboard entities — causes GPU clipping
|
||||
|
||||
### 5.5 Plugin Registration
|
||||
|
||||
Built-in plugins are instantiated in `AppShell.tsx` and registered via `PluginRegistry` → `PluginManager`. Marketplace-installed plugins are loaded from the database via `InstalledPluginsLoader`.
|
||||
|
||||
### 5.6 Workspace Rules
|
||||
|
||||
- Always run `pnpm install` from project root after creating new packages
|
||||
- Plugin packages go in `packages/wwv-plugin-<name>/`
|
||||
- Microservice backends go in `packages/wwv-plugin-<name>/backend/`
|
||||
- Both globs (`packages/*` and `packages/*/backend`) are in `pnpm-workspace.yaml`
|
||||
- Add new plugins to `transpilePackages` in `next.config.ts` and `paths` in `tsconfig.json`
|
||||
|
||||
### 5.7 AI Meta-Directives: Antigravity Standard (Claude Code)
|
||||
|
||||
> [!NOTE]
|
||||
> This repository is orchestrated via the **Antigravity open standard** using **Claude Code** as the active agent. The entry point for Claude Code is `CLAUDE.md` at the project root.
|
||||
|
||||
> [!WARNING]
|
||||
> - **Always** use standard `.md` file extensions for rules, skills, and workflows.
|
||||
> - **Never** use proprietary `.mdc` extensions.
|
||||
> - **Never** reference Cursor IDE rules; we use the open `.agents/` standard.
|
||||
> - **MUST**: You MUST update Semantic Versioning numbering inside the relevant `package.json` file prior to executing any code commits, adhering strictly to the `[/commit]` workflow rules (`feat:` -> Minor, `fix/refactor/perf:` -> Patch).
|
||||
> - **MUST Detail Commit Levels & Bumps**: On description changes or release notes, you must detail the level of commit (Major/Minor/Fix) for *each* individual change. If there are multiple accumulated changes, you MUST EITHER commit them individually and bump the version each time, OR commit them all at once and bump the version multiple times.
|
||||
|
||||
### 5.8 Workspace Hygiene
|
||||
Whenever agents generate temporary debugging scripts, test REST endpoints via `.mjs`, or dump traces/JSON outputs, they **MUST** save these exclusively inside `/local-scripts/`. The root directory is strictly for production configuration files.
|
||||
|
||||
---
|
||||
|
||||
## 6. Environment Variables
|
||||
|
||||
| Variable | Required | Description |
|
||||
|---|---|---|
|
||||
| `DATABASE_URL` | Yes | SQLite: `file:./data/wwv.db` |
|
||||
| `AUTH_SECRET` | Yes | JWT signing secret (generate with `openssl rand -hex 32`) |
|
||||
| `NEXT_PUBLIC_CESIUM_ION_TOKEN` | No | Cesium Ion access token |
|
||||
| `NEXT_PUBLIC_BING_MAPS_KEY` | No | Bing Maps imagery |
|
||||
| `NEXT_PUBLIC_GOOGLE_MAPS_API_KEY` | No | Google 3D Tiles |
|
||||
| `NEXT_PUBLIC_WWV_EDITION` | No | `local` / `cloud` / `demo` (default: `local`) |
|
||||
| `OPENSKY_CREDENTIALS` | No | Comma-separated `id:secret` pairs for credential rotation |
|
||||
| `WWV_BRIDGE_TOKEN` | No | Shared secret for marketplace → WWV install bridge |
|
||||
| `WWV_DEMO_ADMIN_SECRET` | No | Demo edition admin password |
|
||||
| `IRANWARLIVE_BACKEND_URL` | No | Override for IranWarLive microservice URL |
|
||||
|
||||
Secrets go in `.env.local` (gitignored). Non-secrets go in `.env` (committed).
|
||||
|
||||
---
|
||||
|
||||
## 7. Development Commands
|
||||
|
||||
```bash
|
||||
pnpm install # Install all workspace dependencies
|
||||
pnpm run setup # Generate .env.local with AUTH_SECRET (first-time setup)
|
||||
pnpm dev # Next.js frontend only (auto-runs prisma migrate deploy + copy-cesium)
|
||||
pnpm dev:all # Frontend + wwv-data-engine concurrently (normal development)
|
||||
pnpm dev:backends # Run only the data engine in dev mode
|
||||
pnpm build # Production build
|
||||
pnpm test # Run all Vitest tests (scoped to src/lib, src/core, src/plugins)
|
||||
pnpm db:reset # Reset and re-migrate the frontend SQLite database (destructive)
|
||||
pnpm start:backends # Start all plugin microservice backends in parallel
|
||||
pnpm clean:backends # Wipe all plugin SQLite databases
|
||||
pnpm run scaffold-osm-plugin <name> # Generate a new plugin from scaffold
|
||||
```
|
||||
|
||||
Frontend runs at `http://localhost:3000`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Deployment
|
||||
|
||||
- **Docker**: Multi-stage Dockerfile using the Extractor Pattern (`deps` → `builder` → `runner`). The `node_modules` folders must be explicitly untracked in from `git` across the workspace to prevent corrupted BuildKit contextual cache overlaps during the `COPY . .` stage.
|
||||
- **Standalone output**: `next.config.ts` uses `output: "standalone"`.
|
||||
- **Cesium assets**: Copied to `public/cesium/` via `scripts/copy-cesium.mjs` at build time, excluded from output tracing.
|
||||
- **Prisma Configuration**: `prisma.config.ts` must export a native javascript object instead of dynamically importing CLI wrapper binaries (`prisma/config` or `dotenv`). The standalone Next.js tracer strips CLI devDependencies during the build, which will cause fatal runtime container crashes if imported.
|
||||
- **Microservices**: Separate containers defined in `docker-compose.yml`, proxied via `next.config.ts` rewrites.
|
||||
- **Coolify**: Deployed via Dockerfile builder natively mapping environment variables continuously into the container shell. Persistent storage mount required at `/app/data` for the SQLite root node database.
|
||||
- **Docker volumes**: Mount `/app/data` (frontend SQLite), `/app/packages/wwv-data-engine/data` (data engine SQLite), `/data` (Redis)
|
||||
|
||||
---
|
||||
|
||||
## 9. Testing Strategy
|
||||
|
||||
- **Framework**: Vitest with jsdom environment
|
||||
- **Coverage**: `src/lib/**`, `src/core/**`, `src/plugins/**`
|
||||
- **Run**: `pnpm test` (or `vitest run`)
|
||||
- **Key test files**: `rateLimit.test.ts`, `edition.test.ts`, `demoAdmin.test.ts`, `DeclarativePlugin.test.ts`, `cors.test.ts`, `repository.test.ts`, `marketplaceToken.test.ts`
|
||||
|
||||
---
|
||||
|
||||
## 10. Security Headers
|
||||
|
||||
Configured in `next.config.ts` `headers()`:
|
||||
- **CSP**: Restrictive with exceptions for CesiumJS (`unsafe-eval`, `unsafe-inline`), camera streams (`http: https:`), and analytics
|
||||
- **X-Frame-Options**: DENY
|
||||
- **X-Content-Type-Options**: nosniff
|
||||
- **Referrer-Policy**: strict-origin-when-cross-origin
|
||||
- **Permissions-Policy**: camera/microphone disabled, geolocation self-only
|
||||
|
||||
---
|
||||
|
||||
## 11. Related Repositories
|
||||
|
||||
| Repo | Purpose |
|
||||
|---|---|
|
||||
| `worldwideview` | Main application (this repo) |
|
||||
| `worldwideview-marketplace` | Plugin marketplace web app |
|
||||
| `worldwideview-web` | Marketing / landing page |
|
||||
|
||||
---
|
||||
|
||||
## 12. On-Demand Rules
|
||||
|
||||
Read the relevant rule file when working in that domain:
|
||||
|
||||
| Rule | When to use | Path |
|
||||
|---|---|---|
|
||||
| `monorepo-workflow` | pnpm commands, adding packages, workspace config | `.agents/rules/monorepo-workflow.md` |
|
||||
| `plugin-architecture` | Creating/modifying plugins, lifecycle, registration | `.agents/rules/plugin-architecture.md` |
|
||||
| `cesium-rendering` | Globe rendering, entity types, primitives, LOD, culling | `.agents/rules/cesium-rendering.md` |
|
||||
| `state-management` | Zustand slices, store access, plugin settings | `.agents/rules/state-management.md` |
|
||||
| `database-migrations` | Prisma schema changes, migrations, SQLite/Postgres | `.agents/rules/database-migrations.md` |
|
||||
| `continuous-improvement` | When to create/update rules, skills, or workflows | `.agents/rules/continuous-improvement.md` |
|
||||
| `context-and-memory` | How to orient and maintain project context between sessions | `.agents/rules/context-and-memory.md` |
|
||||
|
||||
---
|
||||
|
||||
## 13. Slash Commands / Workflows
|
||||
|
||||
Invoke by name. Read the skill/workflow file and follow its steps.
|
||||
|
||||
| Command | Description | File |
|
||||
|---|---|---|
|
||||
| `/commit` | **Required before every commit** — bump semver + conventional commit | `.agents/skills/commit/SKILL.md` |
|
||||
| `/remember` | Save a lesson, constraint, or fact into `.agents/` permanent memory | `.agents/skills/remember/SKILL.md` |
|
||||
| `/pr-review` | 6-role comprehensive pull request review | `.agents/skills/pr-review/SKILL.md` |
|
||||
| `/update-context` | Sync `.agents/context/` with current project state | Global skill |
|
||||
| `/local-dev` | Check, start, and troubleshoot local dev environment | `.agents/workflows/local-dev.md` |
|
||||
| `/data-engine-cli` | Use the wwv-data-engine CLI wrapper | `.agents/workflows/data-engine-cli.md` |
|
||||
| `/debugging-coolify` | Troubleshoot deployed apps on Coolify via MCP/SSH | `.agents/workflows/debugging-coolify.md` |
|
||||
| `/five` | Five Whys root cause analysis | `.agents/workflows/five.md` |
|
||||
| `/stitch-to-nextjs` | Generate UI with Stitch MCP, port into Next.js | `.agents/workflows/stitch-to-nextjs.md` |
|
||||
| `/bing-news-hydration` | Hydrate event attributes with Bing RSS news | `.agents/workflows/bing-news-hydration.md` |
|
||||
| `/generate-user-roadmap` | Generate updated user-facing roadmap | `.agents/workflows/generate-user-roadmap.md` |
|
||||
|
||||
---
|
||||
|
||||
## 14. Agent Skills Reference
|
||||
|
||||
Refer to these skill documents for specialized tasks:
|
||||
|
||||
### Project Skills (`.agents/skills/`)
|
||||
|
||||
| Skill | When to Use |
|
||||
|---|---|
|
||||
| `worldwideview-plugin-creation` | **Use when creating any plugin** — strict architectural checklist |
|
||||
| `plugin-creation-master-guide.md` | Decision matrix for choosing plugin architecture |
|
||||
| `osm-static-plugin-creation.md` | Creating static GeoJSON plugins from OpenStreetMap |
|
||||
| `microservice-plugin-creation.md` | Building standalone Fastify microservice backends |
|
||||
| `database-operations.md` | Prisma schema changes, migrations, database queries |
|
||||
| `database-incident-recovery-procedures.md` | Authoritative protocol for safely restoring a broken production database |
|
||||
| `index-documentation.md` | Maintaining project documentation index |
|
||||
| `context7` | Fetch up-to-date library docs via Context7 API |
|
||||
| `cesium-context7` | CesiumJS-specific documentation lookup |
|
||||
|
||||
### Global Skills
|
||||
|
||||
52 skills are available across all projects. See `.agents/global-skills-index.md` for the full list and invocation paths.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and maintainers of **WorldWideView** pledge to make participation in our project and community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, colour, religion, or sexual identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
|
||||
|
||||
---
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behaviour that contributes to a positive environment:
|
||||
|
||||
- Using welcoming and inclusive language
|
||||
- Being respectful of differing viewpoints and experiences
|
||||
- Gracefully accepting constructive criticism
|
||||
- Focusing on what is best for the community
|
||||
- Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behaviour:
|
||||
|
||||
- The use of sexualised language or imagery, and sexual attention or advances of any kind
|
||||
- Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
- Public or private harassment
|
||||
- Publishing others' private information without explicit permission
|
||||
- Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
---
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Maintainers are responsible for clarifying and enforcing our standards of acceptable behaviour and will take appropriate and fair corrective action in response to any behaviour that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
Maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct.
|
||||
|
||||
---
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all project spaces — GitHub issues, pull requests, discussions, and any other official channels.
|
||||
|
||||
---
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by opening a GitHub issue marked **[CoC]** or by contacting the maintainer directly via GitHub.
|
||||
|
||||
All complaints will be reviewed and investigated promptly and fairly. The maintainer is obligated to respect the privacy and security of the reporter.
|
||||
|
||||
---
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
# Contributing to WorldWideView
|
||||
|
||||
Thank you for your interest in contributing to **WorldWideView** — a modular, real-time geospatial intelligence engine!
|
||||
|
||||
This project is licensed under the [Elastic License 2.0](./LICENSE). By submitting a contribution, you agree that your code will be made available under those same terms.
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Getting Started](#getting-started)
|
||||
- [Development Setup](#development-setup)
|
||||
- [Project Structure](#project-structure)
|
||||
- [How to Contribute](#how-to-contribute)
|
||||
- [Plugin Contributions](#plugin-contributions)
|
||||
- [Code Standards](#code-standards)
|
||||
- [Commit & Branch Conventions](#commit--branch-conventions)
|
||||
- [Pull Request Process](#pull-request-process)
|
||||
- [Running Tests](#running-tests)
|
||||
|
||||
---
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Fork** the repository on GitHub.
|
||||
2. **Clone** your fork locally:
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/worldwideview.git
|
||||
cd worldwideview
|
||||
```
|
||||
3. **Install dependencies**:
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
4. **Generate environment file** (auto-creates `.env.local` with `AUTH_SECRET`):
|
||||
```bash
|
||||
npm run setup
|
||||
```
|
||||
5. **Start the dev server**:
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Visit `http://localhost:3000` to confirm everything is running.
|
||||
|
||||
---
|
||||
|
||||
## Development Setup
|
||||
|
||||
| Requirement | Version |
|
||||
|-------------|---------|
|
||||
| Node.js | 18+ |
|
||||
| npm | 9+ |
|
||||
|
||||
See [`docs/SETUP.md`](docs/SETUP.md) for detailed environment setup, including Cesium Ion token configuration.
|
||||
|
||||
---
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/
|
||||
core/ # Core engine: state, data bus, plugin registry
|
||||
plugins/ # Data source plugins (ADS-B, AIS, satellites, etc.)
|
||||
components/ # React UI components
|
||||
hooks/ # Custom React hooks
|
||||
docs/ # Architecture and user guides
|
||||
```
|
||||
|
||||
See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for a deep dive into the rendering pipeline and plugin lifecycle.
|
||||
|
||||
---
|
||||
|
||||
## How to Contribute
|
||||
|
||||
Ways you can help:
|
||||
|
||||
- 🐛 **Bug fixes** — Check the [issue tracker](https://github.com/silvertakana/worldwideview/issues)
|
||||
- 🧩 **New plugins** — Add a new data source layer (see below)
|
||||
- 📖 **Documentation** — Improve guides or add examples
|
||||
- ⚡ **Performance** — Cesium primitive optimizations, GPU stall reduction
|
||||
- 🧪 **Tests** — Expand Vitest coverage
|
||||
|
||||
For large features, **open an issue first** to discuss the design before writing code.
|
||||
|
||||
---
|
||||
|
||||
## Plugin Contributions
|
||||
|
||||
The core extension point of WorldWideView is its **plugin system**. Each plugin is a self-contained data layer that:
|
||||
|
||||
1. Fetches or subscribes to a data source
|
||||
2. Transforms raw data into Cesium-ready primitives
|
||||
3. Registers itself with the `PluginRegistry`
|
||||
|
||||
See [`docs/PLUGIN_GUIDE.md`](docs/PLUGIN_GUIDE.md) for a full walkthrough. New plugins are very welcome — if you have access to a live geospatial data feed, a plugin is the best way to contribute.
|
||||
|
||||
---
|
||||
|
||||
## Code Standards
|
||||
|
||||
This project follows these principles:
|
||||
|
||||
- **Single Responsibility Principle** — each file/module does one thing
|
||||
- **DRY & SOLID** — avoid duplication, favour composition
|
||||
- **Clean Architecture** — UI, logic, and data are separate concerns
|
||||
- **Defensive Programming** — validate inputs, handle edge cases explicitly
|
||||
- **File size** — keep files under 150 lines; split into modules if needed
|
||||
|
||||
Use TypeScript strictly. Avoid `any` unless absolutely unavoidable.
|
||||
|
||||
---
|
||||
|
||||
## Commit & Branch Conventions
|
||||
|
||||
**Branch naming:**
|
||||
```
|
||||
feat/short-description
|
||||
fix/short-description
|
||||
docs/short-description
|
||||
refactor/short-description
|
||||
```
|
||||
|
||||
**Commit messages** follow [Conventional Commits](https://www.conventionalcommits.org/):
|
||||
```
|
||||
feat: add maritime AIS plugin
|
||||
fix: resolve camera snap on globe reset
|
||||
docs: update plugin guide with lifecycle diagram
|
||||
refactor: extract billboard factory from CesiumMap
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Pull Request Process
|
||||
|
||||
1. Ensure your branch is up to date with `main`.
|
||||
2. Run tests and confirm they pass: `npm test`
|
||||
3. Fill out the [PR template](.github/PULL_REQUEST_TEMPLATE.md) completely.
|
||||
4. Request a review — PRs need at least one approval before merging.
|
||||
5. Squash commits on merge if the history is noisy.
|
||||
|
||||
---
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
npm test # Run all tests via Vitest
|
||||
npm run test:ui # Open Vitest UI (interactive)
|
||||
```
|
||||
|
||||
Tests live alongside source files in `__tests__/` directories or as `.test.ts` files.
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
# Stage 0: Extract package.json files for layer caching
|
||||
FROM alpine AS extractor
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN find . -type f \! -name 'package.json' \! -name 'pnpm-workspace.yaml' \! -name 'pnpm-lock.yaml' -delete && \
|
||||
find . -type d -empty -delete
|
||||
|
||||
# Stage 1: Install ALL dependencies (needed for build)
|
||||
FROM node:22-alpine AS deps
|
||||
RUN corepack enable pnpm
|
||||
RUN apk add --no-cache python3 make g++
|
||||
WORKDIR /app
|
||||
# Copy only the extracted package.jsons
|
||||
COPY --from=extractor /app ./
|
||||
# Install dependencies with cache mount for pnpm store
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile
|
||||
|
||||
|
||||
# Stage 3: Build the application
|
||||
FROM deps AS builder
|
||||
# Copy full source code
|
||||
COPY . .
|
||||
RUN DATABASE_URL="file:./data/wwv.db" npx prisma generate
|
||||
|
||||
# Create an empty SQLite database with all tables applied
|
||||
RUN mkdir -p ./data && DATABASE_URL=file:./data/wwv.db npx prisma migrate deploy
|
||||
|
||||
# Next.js inlines NEXT_PUBLIC_* vars at build time — must be declared as ARGs
|
||||
ARG NEXT_PUBLIC_CESIUM_ION_TOKEN
|
||||
ARG NEXT_PUBLIC_GOOGLE_MAPS_API_KEY
|
||||
ARG NEXT_PUBLIC_BING_MAPS_KEY
|
||||
ARG NEXT_PUBLIC_WWV_EDITION
|
||||
ARG NEXT_PUBLIC_WS_ENGINE_URL
|
||||
ARG NEXT_PUBLIC_ADSENSE_CLIENT_ID
|
||||
ARG NEXT_PUBLIC_SUPABASE_URL
|
||||
ARG NEXT_PUBLIC_SUPABASE_ANON_KEY
|
||||
|
||||
# Run Next.js build with Webpack cache mounted
|
||||
RUN --mount=type=cache,target=/app/.next/cache NODE_OPTIONS="--max_old_space_size=3072" pnpm run build
|
||||
RUN node scripts/copy-cesium.mjs
|
||||
|
||||
# Stage 4: Production runner
|
||||
FROM node:22-alpine AS runner
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV HOSTNAME=0.0.0.0
|
||||
ENV PORT=3000
|
||||
ENV DATABASE_URL=file:./data/wwv.db
|
||||
ENV AUTH_TRUST_HOST=true
|
||||
|
||||
# Copy Prisma schema + migrations for runtime DB init
|
||||
COPY --from=builder /app/prisma ./prisma
|
||||
COPY --from=builder /app/prisma.config.ts ./prisma.config.ts
|
||||
|
||||
# Copy Prisma generated client
|
||||
COPY --from=builder /app/src/generated ./src/generated
|
||||
|
||||
# Copy standalone server output
|
||||
COPY --from=builder /app/.next/standalone ./
|
||||
|
||||
# We no longer copy proddeps/node_modules. Next.js standalone output
|
||||
# already traces and copies all the exact node_modules needed for production.
|
||||
|
||||
# Copy static assets that standalone mode does NOT include
|
||||
COPY --from=builder /app/.next/static ./.next/static
|
||||
COPY --from=builder /app/public ./public
|
||||
|
||||
# Entrypoint: migrate DB on first run, then start server
|
||||
COPY docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
RUN chmod +x ./docker-entrypoint.sh
|
||||
|
||||
# Declare /app/data as a persistent volume mount point.
|
||||
VOLUME ["/app/data"]
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
ENTRYPOINT ["./docker-entrypoint.sh"]
|
||||
@@ -1,250 +0,0 @@
|
||||
# World Wide View — Implementation Plan
|
||||
|
||||
A next-generation, open-source geospatial intelligence platform combining **Worldview's cinematic CesiumJS 3D globe** with **Worldmonitor's data-rich ingestion engine**, built around a **modular plugin ecosystem**, and designed for both **community self-hosting** and **commercial SaaS deployment**.
|
||||
|
||||
## User Review Required
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **CesiumJS + Google 3D Tiles (Recommended)**: For a "low cost, fast, and efficient" setup that looks like Bilawal's "Worldview" videos, I recommend **CesiumJS with Google Maps Photorealistic 3D Tiles**.
|
||||
> - **Cost**: Google Provides a **$200/month free credit**, which covers ~33,000 sessions (each session is 3 hours of 3D globe usage). This is essentially **$0 for small-to-medium volume**.
|
||||
> - **Efficiency**: CesiumJS is built natively for 3D Tiles and time-dynamic data (CZML), making it the "fastest" path to build the cinematic features you want.
|
||||
> - **Alternatives**: MapLibre GL JS is "lighter" for standard maps but lacks the high-end 3D terrain/building polish. Deck.gl is better suited for the dashboard overlays (which we can still integrate into Cesium).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Scope for this bootstrap**: This plan covers **Phase 1 — Foundation/MVP** only. This gets you a working 3D globe app with the plugin architecture, 3 live data layers, a timeline, cinematic camera, and a sidebar intelligence panel. Subsequent phases will be planned after Phase 1 ships.
|
||||
|
||||
> [!WARNING]
|
||||
> **API keys required**:
|
||||
> - **Google Maps Tile API Key** (Required for Photorealistic 3D Tiles — Includes $200/mo credit).
|
||||
> - **OpenSky Network** account (free at [opensky-network.org](https://opensky-network.org)).
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### Project Scaffolding
|
||||
|
||||
Summary: Initialize a Next.js (App Router) + TypeScript project with a modular architecture, designed from day one as a plugin ecosystem.
|
||||
|
||||
#### [NEW] [package.json](file:///c:/dev/World%20Wide%20View/package.json)
|
||||
Root `package.json` with Next.js, React 18, TypeScript, CesiumJS, and Resium. Will include a custom `copy-cesium` script to move static assets to `/public/cesium` for Next.js compatibility.
|
||||
|
||||
#### [NEW] [next.config.mjs](file:///c:/dev/World%20Wide%20View/next.config.mjs)
|
||||
Next.js config with path aliases, environment variable handling, and specific Webpack exclusions for CesiumJS Node.js dependencies.
|
||||
|
||||
#### [NEW] [tsconfig.json](file:///c:/dev/World%20Wide%20View/tsconfig.json)
|
||||
TypeScript config with strict mode, path aliases for `@/core`, `@/plugins`, `@/components`.
|
||||
|
||||
#### [NEW] [.env.example](file:///c:/dev/World%20Wide%20View/.env.example)
|
||||
Template with all required/optional environment variables documented.
|
||||
|
||||
---
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
c:\dev\World Wide View\
|
||||
├── public/ # Static assets (including /cesium)
|
||||
├── src/
|
||||
│ ├── app/ # Next.js App Router
|
||||
│ │ ├── page.tsx # Main entry (Client component)
|
||||
│ │ ├── layout.tsx # Root layout
|
||||
│ │ ├── globals.css # Global styles
|
||||
│ │ └── api/ # Next.js Route Handlers (Backend)
|
||||
│ │ ├── aviation/route.ts# OpenSky proxy
|
||||
│ │ ├── maritime/route.ts# AIS proxy
|
||||
│ │ └── wildfire/route.ts# NASA FIRMS proxy
|
||||
│ ├── core/ # Core platform code
|
||||
│ │ ├── globe/ # CesiumJS globe wrapper (Dynamic Import)
|
||||
│ │ │ ├── GlobeView.tsx # Main 3D globe component
|
||||
│ │ │ ├── CameraController.ts # Cinematic camera system
|
||||
│ │ │ ├── LayerRenderer.ts # Renders plugin data on globe
|
||||
│ │ │ └── TimelineSync.ts # Syncs Cesium clock with UI
|
||||
│ │ ├── plugins/ # Plugin system
|
||||
│ │ │ ├── PluginManager.ts # Plugin lifecycle manager
|
||||
│ │ │ ├── PluginTypes.ts # Plugin interface definitions
|
||||
│ │ │ └── PluginRegistry.ts# Registration & discovery
|
||||
│ │ ├── data/ # Data management
|
||||
│ │ │ ├── DataBus.ts # Event bus for data flow
|
||||
│ │ │ ├── CacheLayer.ts # In-memory + IndexedDB caching
|
||||
│ │ │ └── PollingManager.ts# Manages refresh intervals
|
||||
│ │ └── state/ # Global state management
|
||||
│ │ └── store.ts # Zustand store
|
||||
│ ├── plugins/ # Built-in data source plugins
|
||||
│ │ ├── aviation/ # OpenSky / ADS-B
|
||||
│ │ ├── maritime/ # AIS vessel tracking
|
||||
│ │ └── wildfire/ # NASA FIRMS
|
||||
│ ├── components/ # UI components
|
||||
│ │ ├── layout/ # AppShell, Header, Sidebar
|
||||
│ │ ├── panels/ # LayerPanel, IntelPanel, Detail
|
||||
│ │ ├── timeline/ # Timeline scrubber
|
||||
│ │ └── common/ # Loading, Toast, etc.
|
||||
│ └── utils/ # Helpers
|
||||
├── .env.example
|
||||
├── package.json
|
||||
├── tsconfig.json
|
||||
└── next.config.mjs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Core Globe Component
|
||||
|
||||
#### [NEW] [GlobeView.tsx](file:///c:/dev/World%20Wide%20View/src/core/globe/GlobeView.tsx)
|
||||
- CesiumJS `Viewer` inside React using Resium
|
||||
- WGS84 ellipsoid, terrain, imagery layers
|
||||
- Request render mode for performance
|
||||
- Integrates with `TimelineSync` for time-based filtering
|
||||
|
||||
#### [NEW] [CameraController.ts](file:///c:/dev/World%20Wide%20View/src/core/globe/CameraController.ts)
|
||||
- Cinematic fly-to animations with easing (Worldview-style smooth transitions)
|
||||
- 8 regional presets (Global, Americas, Europe, MENA, Asia, Africa, Oceania, LatAm)
|
||||
- Orbit mode for idle/attract state
|
||||
- URL state sync (lat, lon, zoom, heading, pitch)
|
||||
|
||||
#### [NEW] [LayerRenderer.ts](file:///c:/dev/World%20Wide%20View/src/core/globe/LayerRenderer.ts)
|
||||
- Renders plugin data as Cesium entities (points, billboards, polylines, polygons)
|
||||
- Smart clustering via `EntityCluster`
|
||||
- Progressive disclosure (detail at higher zoom)
|
||||
- Color/size coding based on data attributes
|
||||
|
||||
#### [NEW] [TimelineSync.ts](file:///c:/dev/World%20Wide%20View/src/core/globe/TimelineSync.ts)
|
||||
- Bidirectional sync between Cesium clock and UI timeline
|
||||
- Time window filtering (1h, 6h, 24h, 48h, 7d)
|
||||
- Playback controls, speed multiplier
|
||||
- Notifies plugins of current time window
|
||||
|
||||
---
|
||||
|
||||
### Plugin System
|
||||
|
||||
The heart of the modular architecture. Every data source implements a standard interface.
|
||||
|
||||
#### [NEW] [PluginTypes.ts](file:///c:/dev/World%20Wide%20View/src/core/plugins/PluginTypes.ts)
|
||||
```typescript
|
||||
export interface WorldPlugin {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
icon: string;
|
||||
category: 'aviation' | 'maritime' | 'conflict' | 'natural-disaster' |
|
||||
'infrastructure' | 'cyber' | 'economic' | 'custom';
|
||||
version: string;
|
||||
|
||||
initialize(context: PluginContext): Promise<void>;
|
||||
destroy(): void;
|
||||
fetch(timeRange: TimeRange): Promise<GeoEntity[]>;
|
||||
getPollingInterval(): number;
|
||||
getLayerConfig(): LayerConfig;
|
||||
renderEntity(entity: GeoEntity): CesiumEntityOptions;
|
||||
|
||||
// Optional UI hooks
|
||||
getSidebarComponent?(): React.ComponentType;
|
||||
getDetailComponent?(): React.ComponentType<{ entity: GeoEntity }>;
|
||||
}
|
||||
```
|
||||
|
||||
#### [NEW] [PluginManager.ts](file:///c:/dev/World%20Wide%20View/src/core/plugins/PluginManager.ts)
|
||||
- Plugin lifecycle (register, initialize, enable, disable, destroy)
|
||||
- Routes data from plugins → `LayerRenderer`
|
||||
- Manages polling per plugin
|
||||
- Emits events via `DataBus`
|
||||
|
||||
#### [NEW] [PluginRegistry.ts](file:///c:/dev/World%20Wide%20View/src/core/plugins/PluginRegistry.ts)
|
||||
- Static registry of built-in plugins + dynamic registration API
|
||||
|
||||
---
|
||||
|
||||
### Built-in Plugins (3 for MVP)
|
||||
|
||||
#### [NEW] Aviation — [src/plugins/aviation/](file:///c:/dev/World%20Wide%20View/src/plugins/aviation/)
|
||||
- **Source**: OpenSky Network REST API
|
||||
- **Rendering**: Aircraft billboards with heading rotation, altitude coloring
|
||||
- **Polling**: 10s | **Backend proxy**: `/api/aviation`
|
||||
|
||||
#### [NEW] Maritime — [src/plugins/maritime/](file:///c:/dev/World%20Wide%20View/src/plugins/maritime/)
|
||||
- **Source**: Public AIS feeds
|
||||
- **Rendering**: Ship icons, color-coded by vessel type
|
||||
- **Polling**: 60s | **Backend proxy**: `/api/maritime`
|
||||
|
||||
#### [NEW] Wildfire — [src/plugins/wildfire/](file:///c:/dev/World%20Wide%20View/src/plugins/wildfire/)
|
||||
- **Source**: NASA FIRMS (VIIRS thermal hotspots)
|
||||
- **Rendering**: Heat-colored point clusters (FRP intensity)
|
||||
- **Polling**: 5min | **Backend proxy**: `/api/wildfire`
|
||||
|
||||
---
|
||||
|
||||
### UI Components
|
||||
|
||||
Dark, cinematic, premium interface — glassmorphism panels floating over the globe.
|
||||
|
||||
#### [NEW] [AppShell.tsx](file:///c:/dev/World%20Wide%20View/src/components/layout/AppShell.tsx)
|
||||
Full-viewport layout with globe as background, floating glassmorphism header, collapsible sidebars, bottom timeline.
|
||||
|
||||
#### [NEW] [Header.tsx](file:///c:/dev/World%20Wide%20View/src/components/layout/Header.tsx)
|
||||
Branding, Cmd+K search, region presets, time window selector, settings.
|
||||
|
||||
#### [NEW] [LayerPanel.tsx](file:///c:/dev/World%20Wide%20View/src/components/panels/LayerPanel.tsx)
|
||||
Toggleable plugin layers grouped by category with entity counts.
|
||||
|
||||
#### [NEW] [Timeline.tsx](file:///c:/dev/World%20Wide%20View/src/components/timeline/Timeline.tsx)
|
||||
Horizontal time scrubber, playback controls, event density sparkline.
|
||||
|
||||
#### [NEW] [index.css](file:///c:/dev/World%20Wide%20View/src/styles/index.css)
|
||||
Dark-mode-first design system: deep navy/charcoal palette, glassmorphism, Inter + JetBrains Mono typography, smooth micro-animations.
|
||||
|
||||
---
|
||||
|
||||
### Backend API (Next.js Route Handlers)
|
||||
|
||||
#### [NEW] [src/app/api/](file:///c:/dev/World%20Wide%20View/src/app/api/)
|
||||
Next.js Route Handlers utilizing Web Standard Request/Response APIs. Handles rate limiting, server-side data fetching (proxying), and caching (via Next.js `fetch` cache or in-memory). This replaces the need for a separate Express server.
|
||||
|
||||
#### [NEW] Route files
|
||||
- `src/app/api/aviation/route.ts` — OpenSky proxy
|
||||
- `src/app/api/maritime/route.ts` — AIS proxy
|
||||
- `src/app/api/wildfire/route.ts` — NASA FIRMS proxy
|
||||
|
||||
---
|
||||
|
||||
### State Management
|
||||
|
||||
#### [NEW] [store.ts](file:///c:/dev/World%20Wide%20View/src/core/state/store.ts)
|
||||
Zustand store with slices: `globe`, `layers`, `timeline`, `ui`, `data`.
|
||||
|
||||
---
|
||||
|
||||
## Verification Plan
|
||||
|
||||
### Automated Tests
|
||||
|
||||
**Build verification:**
|
||||
```bash
|
||||
cd "c:\dev\World Wide View"
|
||||
npm run build
|
||||
```
|
||||
|
||||
**Type checking:**
|
||||
```bash
|
||||
cd "c:\dev\World Wide View"
|
||||
npx tsc --noEmit
|
||||
```
|
||||
|
||||
### Browser Verification
|
||||
1. Open `http://localhost:5173` → CesiumJS globe renders with terrain, header/sidebar/timeline visible
|
||||
2. Enable aviation layer → aircraft icons appear, clicking shows detail
|
||||
3. Enable wildfire layer → fire hotspots render as colored points
|
||||
4. Scrub timeline → data filters by time window
|
||||
5. Click region presets → smooth cinematic fly-to animations
|
||||
|
||||
### Manual Verification
|
||||
1. Run `npm run dev` — starts the Next.js integrated development server
|
||||
2. Verify 3D globe renders with terrain and satellite imagery (ensuring SSR is properly disabled for the globe component)
|
||||
3. Toggle layers via layer panel
|
||||
4. Drag timeline to filter events
|
||||
5. Test camera preset fly-tos
|
||||
6. Resize browser — panels collapse gracefully
|
||||
|
||||
> [!TIP]
|
||||
> First run requires `VITE_CESIUM_TOKEN` in `.env` (free from [cesium.com/ion](https://cesium.com/ion)). App shows a warning if missing but still renders a basic globe.
|
||||
@@ -1,38 +1,91 @@
|
||||
# WorldWideView
|
||||
<div align="center">
|
||||
|
||||
**WorldWideView** is a modular, real-time geospatial intelligence engine built on top of a CesiumJS 3D globe. It is designed to ingest live data streams, transform them through a generalized plugin system, and render them as high-performance visual layers.
|
||||
# WorldWideView 🌍
|
||||
|
||||
As a real-time situational awareness platform, it turns raw geospatial signals—such as aircraft positions, maritime AIS, or satellite orbits—into interactive, cinematic layers on a high-fidelity 3D globe.
|
||||
**The Open-Source, Plugin-Driven Geospatial Intelligence Engine**
|
||||
|
||||
## Key Philosophies
|
||||
[](LICENSE)
|
||||
[](CONTRIBUTING.md)
|
||||
|
||||
- **Modular Intelligence**: Every data source is a plugin. The core engine is data-agnostic.
|
||||
- **High-Performance Rendering**: Engineered for scale using Cesium primitives to handle 100,000+ objects smoothly.
|
||||
- **Situational Awareness**: Designed for real-time updates and cinematic visualization (orientated icons, smooth tracking).
|
||||
- **Extensible & Open**: Add new intelligence layers without touching the core architecture.
|
||||
*A modular situational awareness platform designed to ingest live data streams and render them as interactive, cinematic layers on a high-fidelity CesiumJS 3D globe.*
|
||||
|
||||
## Quick Start
|
||||
<!-- 📸 Replace with a high-res GIF or screenshot of the platform running with multiple layers enabled -->
|
||||

|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Build a Plugin in 5 Minutes
|
||||
|
||||
WorldWideView’s architecture means **everything is a plugin**. The core engine is completely data-agnostic. Rather than touching complex 3D rendering code, you can build a plugin that simply pipes a REST API into GeoJSON, and the engine handles the rest!
|
||||
|
||||
Ready to add a new intelligence layer (like Earthquakes, Weather, or Satellites)? We have a scaffold generator ready for you:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-repo/worldwideview.git
|
||||
# 1. Generate the boilerplate plugin from a template
|
||||
pnpm run scaffold-osm-plugin my-new-data-source
|
||||
|
||||
# 2. Add your data fetching logic in the generated files
|
||||
# 3. Reload the dev server and watch it map on the globe!
|
||||
```
|
||||
|
||||
📖 **[Read the Full Plugin Guide](docs/PLUGIN_GUIDE.md)** for a deep dive into hooking up websockets or static datasets.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Installation & Setup (Local Development)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/silvertakana/worldwideview.git
|
||||
cd worldwideview
|
||||
npm install
|
||||
npm run dev
|
||||
pnpm install
|
||||
pnpm run setup # generates .env.local with AUTH_SECRET
|
||||
pnpm run dev:all # starts the app, data engine, and marketplace concurrently
|
||||
```
|
||||
|
||||
Visit `http://localhost:3000` to see the live globe.
|
||||
|
||||
## Fair-Use Notice
|
||||
---
|
||||
|
||||
This application may contain copyrighted material the use of which has not always been specifically authorized by the copyright owner. Such material is made available for educational purposes, situational awareness, and to advance understanding of global events. It is believed that this constitutes a "fair use" of any such copyrighted material as provided for in section 107 of the US Copyright Law.
|
||||
## 🐋 Production Deployment (Docker & Coolify)
|
||||
|
||||
## Key Documentation Sections
|
||||
WorldWideView uses a multi-stage Dockerfile designed for standalone output, natively supporting deployment via Coolify or standard Docker Compose. The `docker-compose.yml` spins up the front-end application alongside the high-performance Data Engine and Redis cache.
|
||||
|
||||
```bash
|
||||
# Start the core app, Redis cache, and Data Engine microservice in production mode
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
**Persistent Storage Requirements:**
|
||||
When deploying to production, ensure you mount the following volumes to prevent data loss:
|
||||
- Mount `/app/data` to persist the core SQLite database.
|
||||
- Mount `/app/packages/wwv-data-engine/data` to persist the custom plugin microservices SQLite layer.
|
||||
- Mount `/data` to persist the standalone Redis container.
|
||||
|
||||
---
|
||||
|
||||
## 🧠 Core Architecture & Philosophies
|
||||
|
||||
- **Modular Intelligence**: Independent plugins supply the data. The core engine remains lightweight and isolated.
|
||||
- **High-Performance Rendering**: Engineered for scale using raw Cesium primitives to smoothly handle upwards of 100,000+ objects simultaneously without GPU stalls.
|
||||
- **Cinematic Situational Awareness**: Implements horizon culling, smooth camera tracking, orientation-locked billboarding, and stack spiderification for dense locations.
|
||||
|
||||
---
|
||||
|
||||
## 📚 Documentation Index
|
||||
|
||||
- **[Setup & Installation Guide](docs/SETUP.md)**: Detailed environment and local development setup.
|
||||
- **[Architecture (Engineering Depth)](docs/ARCHITECTURE.md)**: Deep dive into the Cesium rendering pipeline, event data bus, and performance optimizations.
|
||||
- **[Plugin System Guide](docs/PLUGIN_GUIDE.md)**: How to build and register custom data layers.
|
||||
- **[Contributions Guide](CONTRIBUTING.md)**: How to help out, coding standards, and commit conventions.
|
||||
- **[User Guide](docs/USER_GUIDE.md)**: Application features and navigation tips.
|
||||
|
||||
---
|
||||
|
||||
For a full list of resources, see the **[Documentation Index](docs/index.md)**.
|
||||
## 🤝 Contributing & Community
|
||||
|
||||
We are actively looking for contributors to help build new data plugins and optimize the core 3D engine!
|
||||
- Found a bug or have an idea? Search the [Issue Tracker](https://github.com/silvertakana/worldwideview/issues) and submit a report.
|
||||
- Want to build something? Check out our **[Pinned Plugin Wishlist](https://github.com/silvertakana/worldwideview/issues)** for great first issues!
|
||||
|
||||
> **Fair-Use Notice:** This application may contain copyrighted material the use of which has not always been specifically authorized by the copyright owner. Such material is made available for educational purposes, situational awareness, and to advance understanding of global events under "fair use" (Section 107 of the US Copyright Law).
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
# WorldWideView — Implementation Roadmap
|
||||
|
||||
Each stage is **self-contained and demo-able**. Any stage can be shipped independently.
|
||||
|
||||
---
|
||||
|
||||
## Stage 1: Plugin SDK & Manifest Schema ✅
|
||||
**Demo:** "Here's the schema that defines a plugin. Our existing aviation plugin has a manifest file."
|
||||
|
||||
- [x] `plugin.json` manifest files for each existing plugin
|
||||
- [x] Manifest validation logic (`validateManifest()`)
|
||||
- [x] `type` field on `WorldPlugin` interface (`"data-layer" | "extension"`)
|
||||
- [x] `capabilities` field on `WorldPlugin` interface
|
||||
- [x] `@worldwideview/wwv-plugin-sdk` npm package — shared types published to npm
|
||||
|
||||
---
|
||||
|
||||
## Stage 2: Marketplace Browse UI (Static Catalog) ✅
|
||||
**Demo:** "Open the marketplace, see all available plugins with categories and search."
|
||||
|
||||
- [x] `worldwideview-marketplace` repo (Next.js)
|
||||
- [x] Browse page — grid of plugin cards with icons, descriptions, categories
|
||||
- [x] Search + category filter
|
||||
- [x] Plugin detail page (readme, version history, install count)
|
||||
|
||||
---
|
||||
|
||||
## Stage 3: Prisma ORM & Local Database ✅
|
||||
**Demo:** "Run `pnpm run dev` on a fresh clone. SQLite auto-creates. App works with zero config."
|
||||
|
||||
- [x] Prisma (`prisma` + `@prisma/client`)
|
||||
- [x] Schema: `InstalledPlugin`, `Setting`, `User`, `AviationHistory`, `NpmCache`
|
||||
- [x] SQLite default: `DATABASE_URL=file:./data/wwv.db`
|
||||
- [x] `predev` script: `prisma migrate deploy`
|
||||
- [x] DB auto-creates on first run — zero manual setup
|
||||
|
||||
---
|
||||
|
||||
## Stage 4: Bridge API & One-Click Install ✅
|
||||
**Demo:** "Click 'Install' on marketplace → plugin appears in your local WWV instance."
|
||||
|
||||
- [x] `GET /api/marketplace/install-redirect` — session-based redirect install flow
|
||||
- [x] `GET /api/marketplace/grant-token` — issues marketplace JWT without install
|
||||
- [x] `GET /api/marketplace/status` — list installed plugins (session or JWT auth)
|
||||
- [x] `POST /api/marketplace/uninstall` — delete installed plugin record
|
||||
- [x] `GET /api/marketplace/load` — serve valid manifests to client at startup
|
||||
- [x] `GET /api/auth/setup-status` — CORS-enabled connection test endpoint
|
||||
- [x] `src/lib/marketplace/marketplaceToken.ts` — `issueMarketplaceToken()` / `verifyMarketplaceToken()`
|
||||
- [x] `src/lib/marketplace/auth.ts` — `validateMarketplaceAuth()` (session → JWT → legacy token)
|
||||
- [x] `src/lib/marketplace/registryClient.ts` — Ed25519 registry verification + 5-min cache
|
||||
- [x] Marketplace: `InstanceConfig`, `InstallButton`, `useInstalledPlugins`, `InstalledPluginCard`, Manage page
|
||||
- [x] NPM-driven submission: Admin enters package name → marketplace fetches + parses `"worldwideview"` block from `package.json`
|
||||
- [x] `NpmCache` table — decouples marketplace from live NPM registry
|
||||
|
||||
---
|
||||
|
||||
## Stage 5: Publisher Workflow
|
||||
**Demo:** "A plugin author submits their plugin. After review, it appears in the catalog."
|
||||
|
||||
- [ ] Marketplace user accounts (Supabase Auth — email/password + GitHub OAuth)
|
||||
- [ ] Publisher submission form (upload bundle, fill metadata, declare capabilities)
|
||||
- [ ] Admin review queue (approve/reject with notes)
|
||||
- [ ] Version management (semver, changelogs)
|
||||
- [ ] Publisher dashboard (install counts, download analytics)
|
||||
- [ ] Publisher namespaces (`publisher.plugin-name`)
|
||||
|
||||
---
|
||||
|
||||
## Stage 6: Cloud Platform & Accounts
|
||||
**Demo:** "User registers, gets [user].app.worldwideview.dev, installs plugins from marketplace."
|
||||
|
||||
- [ ] `app.worldwideview.dev` — auth/register/dashboard
|
||||
- [ ] Auto-provisioning on signup (tenant DB row → instance live immediately)
|
||||
- [ ] RLS tenant isolation in PostgreSQL
|
||||
- [ ] Cloud edition adapters (`auth.ts`, `storage.ts`, `tenant.ts`)
|
||||
- [ ] Runtime settings store + setup wizard
|
||||
- [ ] License key validation with RSA public key
|
||||
- [ ] Tier gating (free/pro/enterprise feature flags)
|
||||
- [ ] Cloudflare Turnstile + nginx rate limiting for bot prevention
|
||||
- [ ] SSO: marketplace redirects to `app.worldwideview.dev` for login
|
||||
|
||||
---
|
||||
|
||||
## Stage 7: Built-In Data Services
|
||||
**Demo:** "Aviation and maritime data with full history, served from api.worldwideview.dev."
|
||||
|
||||
- [ ] `api.worldwideview.dev` — aviation/maritime polling + persistence
|
||||
- [ ] Redesign aviation as declarative plugin pointing to `api.worldwideview.dev`
|
||||
- [ ] Redesign maritime the same way
|
||||
- [ ] History/timeline with `%t` parameter substitution
|
||||
- [ ] Snapshot capture (pro tier — counts toward storage quota)
|
||||
- [ ] API key proxy with per-tenant-per-plugin rate limiting and response caching
|
||||
|
||||
---
|
||||
|
||||
## Stage 8: Trust Tiers, Extensions & Polish
|
||||
**Demo:** "Install a verified extension plugin that adds cockpit view to aviation."
|
||||
|
||||
- [x] Ed25519-signed plugin registry (`GET /api/registry` on marketplace)
|
||||
- [x] Server-side trust stamping at install time
|
||||
- [x] Trust tier badges (`TrustBadge` component — Official / Verified / Unverified)
|
||||
- [x] Unverified plugin warning dialog (`UnverifiedPluginDialog` + `useMarketplaceSync`)
|
||||
- [ ] Extension plugin type support in `PluginManager`
|
||||
- [ ] Activation events and contribution points
|
||||
- [ ] Capability enforcement at runtime (`PluginPermissionError`)
|
||||
- [ ] Web Worker sandboxing for unverified plugins (`comlink`)
|
||||
- [ ] Reviews & ratings system
|
||||
- [ ] Extension packs (bundles of related plugins)
|
||||
- [ ] Auto-update mechanism (local: manual opt-in; cloud: automatic CDN swap)
|
||||
- [ ] Plausible/Umami self-hosted analytics
|
||||
|
||||
---
|
||||
|
||||
## Stage 9: Infrastructure & Monetization
|
||||
**Demo:** "Buy Pro, paste license key, unlock snapshot history and higher rate limits."
|
||||
|
||||
- [ ] Coolify on ThinkPad T480s — deploy all services
|
||||
- [ ] Uptime Kuma monitoring (all subdomains)
|
||||
- [ ] Stripe Checkout for license purchase
|
||||
- [ ] Storage quota enforcement (cloud only)
|
||||
- [ ] Stripe Connect for marketplace revenue split on paid plugins
|
||||
- [ ] Cloudflare R2 as CDN for plugin bundles (`cdn.worldwideview.dev`)
|
||||
|
||||
---
|
||||
|
||||
## Resolved Architectural Decisions
|
||||
|
||||
| Decision | Resolution |
|
||||
|---|---|
|
||||
| **License** | EL2.0 for WorldWideView core; Open Source for marketplace |
|
||||
| **Shared types** | `@worldwideview/plugin-sdk` npm package (MIT) |
|
||||
| **Hosting** | Coolify (ThinkPad T480s). Landing page on Vercel (static export) |
|
||||
| **Database** | Supabase PostgreSQL (cloud), SQLite (local), Prisma ORM |
|
||||
| **Tenant isolation** | Row-Level Security (RLS) in shared PostgreSQL |
|
||||
| **User settings** | Runtime settings store in DB, not `.env` |
|
||||
| **Bot prevention** | Email verification + Cloudflare Turnstile + rate limiting |
|
||||
| **Plugin execution** | Client-side (browser), not server-side |
|
||||
| **Plugin architecture** | VS Code model — activation events, contribution points, namespaces |
|
||||
| **Server-side compute** | Not supported — third-party backends handle compute |
|
||||
| **Auth** | Auth.js on `app.worldwideview.dev`. `Credentials` (local), `@auth/supabase-adapter` (cloud). Marketplace uses SSO redirect |
|
||||
| **Instance provisioning** | Auto-provision on signup — just a DB row insert |
|
||||
| **API proxy** | `apikey` plugins routed through cached proxy, per-tenant-per-plugin rate limits |
|
||||
| **History** | Third-party `historyUrl` with `%t` (all tiers), WWV snapshot capture (pro only) |
|
||||
| **Built-in data** | Aviation/maritime → declarative plugins → `api.worldwideview.dev` |
|
||||
| **CI/CD** | GitHub Actions → GHCR → Coolify webhook |
|
||||
| **Plugin trust registry** | Ed25519-signed JSON, public key hardcoded in WWV, trust stamped server-side at install |
|
||||
| **Package manager** | pnpm workspaces (migrated from npm) |
|
||||
| **NPM submission flow** | Admin enters package name → marketplace infers manifest from `"worldwideview"` block in `package.json` |
|
||||
| **NPM caching** | `NpmCache` table decouples marketplace from live NPM registry |
|
||||
@@ -1,46 +0,0 @@
|
||||
# WorldWideView — Phase 1 MVP Task Breakdown
|
||||
|
||||
## Planning
|
||||
- [/] Explore existing codebase and understand project state
|
||||
- [/] Write implementation plan for Phase 1 MVP
|
||||
- [ ] Get user approval on implementation plan
|
||||
|
||||
## Phase 1: Project Scaffolding
|
||||
- [ ] Initialize Next.js + TypeScript project with App Router
|
||||
- [ ] Configure CesiumJS integration (Webpack, static assets, SSR bypass)
|
||||
- [ ] Set up design system (CSS custom properties, glassmorphism, typography)
|
||||
- [ ] Configure Zustand state management
|
||||
- [ ] Set up path aliases and project structure
|
||||
|
||||
## Phase 2: Core Globe Component
|
||||
- [ ] Implement `GlobeView.tsx` — CesiumJS viewer with Google 3D Tiles
|
||||
- [ ] Implement `CameraController.ts` — cinematic fly-to, orbit mode, presets
|
||||
- [ ] Implement `LayerRenderer.ts` — entity rendering, clustering, progressive disclosure
|
||||
- [ ] Implement `TimelineSync.ts` — clock sync, time-window filtering, playback
|
||||
|
||||
## Phase 3: Plugin System
|
||||
- [ ] Define `PluginTypes.ts` — `WorldPlugin` interface, `GeoEntity`, `LayerConfig`
|
||||
- [ ] Implement `PluginManager.ts` — lifecycle, polling, data routing
|
||||
- [ ] Implement `PluginRegistry.ts` — static + dynamic registration
|
||||
- [ ] Implement `DataBus.ts` — event bus for cross-plugin communication
|
||||
- [ ] Implement `CacheLayer.ts` — in-memory + IndexedDB caching
|
||||
- [ ] Implement `PollingManager.ts` — per-plugin refresh
|
||||
|
||||
## Phase 4: Built-in Plugins (MVP)
|
||||
- [ ] Aviation plugin (OpenSky Network)
|
||||
- [ ] Maritime plugin (AIS feeds)
|
||||
- [ ] Wildfire plugin (NASA FIRMS)
|
||||
- [ ] Backend API proxy routes for each plugin
|
||||
|
||||
## Phase 5: UI Components
|
||||
- [ ] `AppShell.tsx` — full-viewport layout with floating panels
|
||||
- [ ] `Header.tsx` — branding, search, region presets, time selector
|
||||
- [ ] `LayerPanel.tsx` — toggleable layer list grouped by category
|
||||
- [ ] `Timeline.tsx` — horizontal scrubber with playback controls
|
||||
- [ ] `IntelPanel.tsx` — entity detail and intelligence sidebar
|
||||
- [ ] Global CSS design system (dark mode, glassmorphism, animations)
|
||||
|
||||
## Phase 6: Verification
|
||||
- [ ] Build verification (`npm run build` + type-check)
|
||||
- [ ] Browser verification — globe renders, layers toggle, camera presets work
|
||||
- [ ] Manual verification — time scrubbing, responsive layout, API proxies
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
wwv:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- wwv-data:/app/data
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
- "AUTH_SECRET=${AUTH_SECRET:-default_secret_for_local_dev}"
|
||||
- "WWV_DEMO_ADMIN_SECRET=${WWV_DEMO_ADMIN_SECRET:-}"
|
||||
- "WWV_BRIDGE_TOKEN=${WWV_BRIDGE_TOKEN:-}"
|
||||
|
||||
wwv-data-engine:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./packages/wwv-data-engine/Dockerfile
|
||||
ports:
|
||||
- "5001:5001"
|
||||
volumes:
|
||||
- engine-data:/app/packages/wwv-data-engine/data
|
||||
restart: always
|
||||
depends_on:
|
||||
- redis
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
environment:
|
||||
- "NODE_ENV=production"
|
||||
- "REDIS_URL=${REDIS_URL:-redis://redis:6379}"
|
||||
networks:
|
||||
- coolify
|
||||
- default
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.data-engine.rule=Host(`dataengine.worldwideview.dev`)"
|
||||
- "traefik.http.routers.data-engine.entrypoints=websecure"
|
||||
- "traefik.http.routers.data-engine.tls=true"
|
||||
- "traefik.http.routers.data-engine.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.services.data-engine.loadbalancer.server.port=5001"
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: always
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
volumes:
|
||||
wwv-data:
|
||||
engine-data:
|
||||
redis-data:
|
||||
|
||||
networks:
|
||||
coolify:
|
||||
external: true
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# ─── Docker Entrypoint ───────────────────────────────────────
|
||||
# Ensures the SQLite database exists and is migrated before
|
||||
# starting the application. On first run with a fresh volume
|
||||
# the DB file won't exist yet, so we run prisma migrate deploy.
|
||||
|
||||
set -e
|
||||
|
||||
mkdir -p ./data
|
||||
echo "[entrypoint] Running database migrations..."
|
||||
npx -y prisma migrate deploy
|
||||
echo "[entrypoint] Migrations complete."
|
||||
|
||||
exec node server.js
|
||||
@@ -20,7 +20,7 @@ One of our core design decisions is using **Cesium Primitives** instead of the s
|
||||
|
||||
| Feature | Entity API | Primitive API (WWV) |
|
||||
|---|---|---|
|
||||
| **Abstration** | High (Easy to use) | Low (Direct GPU access) |
|
||||
| **Abstraction** | High (Easy to use) | Low (Direct GPU access) |
|
||||
| **Overhead** | Significant per-entity JS objects | Minimal (Batched rendering) |
|
||||
| **Max Capacity** | ~1,000 entities | **100,000+ points/billboards** |
|
||||
| **WWV Use Case** | Info window content | Live data point visualization |
|
||||
|
||||
+22
-28
@@ -5,46 +5,40 @@ Follow these steps to set up your development environment and get WorldWideView
|
||||
## Requirements
|
||||
|
||||
- **Node.js**: v18.17 or higher (tested on v20+)
|
||||
- **NPM**: v9 or higher
|
||||
- **pnpm**: v8 or higher (`npm install -g pnpm`)
|
||||
- **Browser**: Chrome, Firefox, or Edge (CesiumJS requires WebGL support)
|
||||
|
||||
## Environment Setup
|
||||
|
||||
WorldWideView requires several API keys to function correctly. You can copy the provided `.env.example` file to `.env.local` and fill in the values:
|
||||
Run the setup script to auto-generate `.env.local` with a secure `AUTH_SECRET`:
|
||||
|
||||
```bash
|
||||
cp .env.example .env.local
|
||||
pnpm run setup
|
||||
```
|
||||
|
||||
Or manually create a `.env.local` file in the root directory:
|
||||
This copies `.env.example` → `.env.local` and fills in `AUTH_SECRET` automatically. You can then optionally add API keys for enhanced features (Cesium Ion, Bing Maps, OpenSky, etc.).
|
||||
|
||||
```env
|
||||
# Cesium Ion Access Token (Required for terrain and default imagery)
|
||||
NEXT_PUBLIC_CESIUM_ION_TOKEN=your_token_here
|
||||
|
||||
# Bing Maps API Key (Optional, for high-res imagery)
|
||||
NEXT_PUBLIC_BING_MAPS_KEY=your_key_here
|
||||
|
||||
# Supabase Configuration (Required for persistence and historical data)
|
||||
NEXT_PUBLIC_SUPABASE_URL=your_project_url
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY=your_anon_key
|
||||
SUPABASE_SERVICE_ROLE_KEY=your_service_role_key
|
||||
```
|
||||
> [!NOTE]
|
||||
> `AUTH_SECRET` must remain stable across restarts. The setup script generates it once and writes it to `.env.local`. Never commit `.env.local` to git.
|
||||
|
||||
## Running Locally
|
||||
|
||||
1. **Install Dependencies**:
|
||||
```bash
|
||||
npm install
|
||||
pnpm install
|
||||
```
|
||||
|
||||
2. **Start Dev Server**:
|
||||
2. **Generate environment file** (first time only):
|
||||
```bash
|
||||
npm run dev
|
||||
pnpm run setup
|
||||
```
|
||||
*Note: Our `dev` script automatically runs `predev` to copy Cesium assets to the public directory.*
|
||||
|
||||
3. **Open Browser**:
|
||||
3. **Start Dev Server**:
|
||||
```bash
|
||||
pnpm run dev:all
|
||||
```
|
||||
|
||||
4. **Open Browser**:
|
||||
Navigate to [http://localhost:3000](http://localhost:3000).
|
||||
|
||||
## Folder Structure Explanation
|
||||
@@ -53,9 +47,9 @@ Understanding the project layout:
|
||||
|
||||
```text
|
||||
worldwideview/
|
||||
├── .agents/ # Agentic AI workflows and memory
|
||||
├── docs/ # Project documentation (Markdowns)
|
||||
├── public/ # Static assets (Imagery, GeoJSON, Cesium dependencies)
|
||||
├── docs/ # Project documentation
|
||||
├── packages/ # Plugin packages (pnpm workspaces)
|
||||
├── public/ # Static assets (GeoJSON data files, Cesium dependencies)
|
||||
├── scripts/ # Build and utility scripts (e.g., copy-cesium.mjs)
|
||||
└── src/
|
||||
├── app/ # Next.js App Router (Layouts, Pages, API Routes)
|
||||
@@ -64,8 +58,8 @@ worldwideview/
|
||||
│ ├── data/ # Data management (DataBus, Polling, Cache)
|
||||
│ ├── globe/ # Cesium-specific hooks and view logic
|
||||
│ └── plugins/ # Plugin system base and registry
|
||||
├── lib/ # Utility libraries and server-side polling
|
||||
└── plugins/ # Feature-specific plugins (Aviation, Maritime, etc.)
|
||||
├── lib/ # Utility libraries and server-side logic
|
||||
└── plugins/ # Built-in plugins (Aviation, Maritime, etc.)
|
||||
```
|
||||
|
||||
## Verification
|
||||
@@ -77,5 +71,5 @@ To verify your setup:
|
||||
|
||||
---
|
||||
|
||||
> [!IMPORTANT]
|
||||
> If you encounter issues with hot reloading (HMR), ensure you are running with the `--webpack` flag in the dev script (this is the default in our `package.json`).
|
||||
> [!TIP]
|
||||
> The database is created automatically on first run — no manual setup is required. If you need to reset it, delete `data/wwv.db` and restart the dev server.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 816 KiB |
@@ -0,0 +1,362 @@
|
||||
# Creating a WorldWideView Plugin
|
||||
|
||||
This guide walks you through building, publishing, and listing your own WorldWideView plugin from scratch.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
WorldWideView plugins display geospatial data on a 3D Cesium globe. Every plugin must implement the `WorldPlugin` interface from the **Plugin SDK** (`@worldwideview/wwv-plugin-sdk`).
|
||||
|
||||
There are **three plugin formats**:
|
||||
|
||||
| Format | Use Case | Example |
|
||||
|---|---|---|
|
||||
| **`static`** | GeoJSON file loaded at startup — no API calls at runtime | Military Bases, Nuclear Facilities, Embassies |
|
||||
| **`bundle`** | Full TypeScript class with custom logic, live API data, UI components | Aviation, Maritime, Wildfire |
|
||||
| **`declarative`** | JSON-only manifest that points to an external API — no code needed | *(Future)* |
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js ≥ 18
|
||||
- pnpm (`npm install -g pnpm`)
|
||||
- A npm account (for publishing to the npm registry)
|
||||
- The WorldWideView monorepo cloned locally
|
||||
|
||||
---
|
||||
|
||||
## Step 1 — Create Your Plugin Package
|
||||
|
||||
All plugins live under `packages/` in the monorepo.
|
||||
|
||||
```bash
|
||||
mkdir packages/wwv-plugin-my-data
|
||||
cd packages/wwv-plugin-my-data
|
||||
```
|
||||
|
||||
Create `package.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@worldwideview/wwv-plugin-my-data",
|
||||
"version": "1.0.0",
|
||||
"description": "WorldWideView plugin — my custom data layer",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": ["src"],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"worldwideview-plugin",
|
||||
"geojson",
|
||||
"cesium",
|
||||
"globe"
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/silvertakana/worldwideview.git",
|
||||
"directory": "packages/wwv-plugin-my-data"
|
||||
},
|
||||
"license": "ISC",
|
||||
"author": "WorldWideView",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "*"
|
||||
},
|
||||
"worldwideview": {
|
||||
"id": "my-data",
|
||||
"icon": "MapPin",
|
||||
"category": "Custom",
|
||||
"format": "bundle"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The `"worldwideview"` object block is strictly required if you intend to publish your plugin to the WorldWideView Marketplace. The marketplace uses this internal manifest to determine how to categorize and display your plugin automatically.
|
||||
> * `id`: The unique string identifier.
|
||||
> * `icon`: The Lucide React icon name (e.g. `MapPin`, `Satellite`, `Plane`).
|
||||
> * `category`: Must match an existing category (e.g. `Aviation`, `Maritime`, `Natural Disaster`, `Custom`).
|
||||
> * `format`: Either `bundle`, `static`, or `declarative`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The SDK is declared as a **peer dependency**, not a direct dependency. The host application provides it at runtime.
|
||||
|
||||
---
|
||||
|
||||
## Step 2 — Write Your Plugin Code
|
||||
|
||||
Create `src/index.ts`. Here are examples for the two most common formats:
|
||||
|
||||
### Option A — Static GeoJSON Plugin (simplest)
|
||||
|
||||
For static plugins, the npm package is just metadata. The actual loading is handled by the `StaticDataPlugin` loader at runtime, driven by a manifest + GeoJSON file.
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* @worldwideview/wwv-plugin-my-data
|
||||
*
|
||||
* Static GeoJSON plugin — my custom dataset.
|
||||
*
|
||||
* This package exists for npm registry metadata. The actual plugin logic
|
||||
* is handled by WorldWideView's StaticDataPlugin loader at runtime using
|
||||
* the manifest + GeoJSON data file served from the host application.
|
||||
*/
|
||||
export const PLUGIN_ID = "my-data" as const;
|
||||
```
|
||||
|
||||
For static plugins you also need:
|
||||
1. A **GeoJSON file** placed at `public/data/my_data.geojson` in the main app
|
||||
2. A **manifest entry** in the marketplace (see Step 4)
|
||||
|
||||
### Option B — Bundle Plugin (full control)
|
||||
|
||||
For bundle plugins, you write a class that implements the `WorldPlugin` interface:
|
||||
|
||||
```typescript
|
||||
import type {
|
||||
WorldPlugin,
|
||||
GeoEntity,
|
||||
TimeRange,
|
||||
PluginContext,
|
||||
LayerConfig,
|
||||
CesiumEntityOptions,
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
export class MyDataPlugin implements WorldPlugin {
|
||||
// ─── Required Metadata ──────────────────────────────
|
||||
id = "my-data";
|
||||
name = "My Data Layer";
|
||||
description = "Displays custom points on the globe.";
|
||||
icon = "MapPin"; // Lucide icon name (string)
|
||||
category = "custom" as const;
|
||||
version = "1.0.0";
|
||||
|
||||
// ─── Lifecycle ──────────────────────────────────────
|
||||
async initialize(ctx: PluginContext): Promise<void> {
|
||||
// Called once when the plugin is loaded.
|
||||
// Use ctx.apiBaseUrl, ctx.timeRange, ctx.onDataUpdate, etc.
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
// Called when the plugin is unloaded. Clean up timers, sockets, etc.
|
||||
}
|
||||
|
||||
// ─── Data ───────────────────────────────────────────
|
||||
async fetch(timeRange: TimeRange): Promise<GeoEntity[]> {
|
||||
// Fetch your data from an API and return GeoEntity objects.
|
||||
const res = await fetch("https://api.example.com/points");
|
||||
const data = await res.json();
|
||||
|
||||
return data.map((item: any) => ({
|
||||
id: String(item.id),
|
||||
pluginId: this.id,
|
||||
latitude: item.lat,
|
||||
longitude: item.lon,
|
||||
altitude: 0,
|
||||
timestamp: new Date(),
|
||||
label: item.name,
|
||||
properties: { type: item.type },
|
||||
}));
|
||||
}
|
||||
|
||||
getPollingInterval(): number {
|
||||
return 60_000; // Re-fetch every 60 seconds
|
||||
}
|
||||
|
||||
// ─── Rendering ──────────────────────────────────────
|
||||
getLayerConfig(): LayerConfig {
|
||||
return {
|
||||
color: "#3b82f6",
|
||||
clusterEnabled: true,
|
||||
clusterDistance: 40,
|
||||
maxEntities: 5000,
|
||||
};
|
||||
}
|
||||
|
||||
renderEntity(entity: GeoEntity): CesiumEntityOptions {
|
||||
return {
|
||||
type: "point",
|
||||
color: "#3b82f6",
|
||||
size: 8,
|
||||
};
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 3 — Publish to npm
|
||||
|
||||
```bash
|
||||
cd packages/wwv-plugin-my-data
|
||||
pnpm publish --access public
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> The marketplace pulls metadata (version, description, keywords, readme) directly from the npm registry. You do not need to duplicate this information anywhere else.
|
||||
|
||||
---
|
||||
|
||||
## Step 4 — Submit to the Marketplace
|
||||
|
||||
Once your package is published to npm, submit it for listing in the WorldWideView Marketplace:
|
||||
|
||||
1. Go to [marketplace.worldwideview.dev/submit](https://marketplace.worldwideview.dev/submit) (or open the Admin UI on a local marketplace instance)
|
||||
2. Enter your **npm package name** (e.g. `@worldwideview/wwv-plugin-my-data`)
|
||||
3. The marketplace automatically fetches your metadata from npm and reads the `"worldwideview"` block from your `package.json`
|
||||
4. Your plugin enters a **pending** state for review
|
||||
5. Once approved, it appears in the browse catalog
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The `"worldwideview"` block in your `package.json` is required for submission. Make sure `id`, `icon`, `category`, and `format` are correctly set before publishing.
|
||||
|
||||
### For static format plugins
|
||||
|
||||
If your plugin uses `"format": "static"`, your **GeoJSON file** must be hosted and accessible at a public URL, or included in the `public/data/` folder of the host WorldWideView application. The `dataFile` path in your manifest should point to that URL.
|
||||
|
||||
If you are contributing a plugin for inclusion in the official WorldWideView release, place the GeoJSON at `public/data/<name>.geojson` and open a pull request.
|
||||
|
||||
---
|
||||
|
||||
## Step 5 — Test Locally
|
||||
|
||||
1. Start all WorldWideView services: `pnpm run dev:all` (in the monorepo root)
|
||||
2. (Wait for both the main app and marketplace to start)
|
||||
3. Open the marketplace at `http://localhost:3001/browse`
|
||||
4. Find your plugin and click **Install**
|
||||
5. Verify the data appears on the globe at `http://localhost:3000`
|
||||
|
||||
---
|
||||
|
||||
## SDK Interface Reference
|
||||
|
||||
The `WorldPlugin` interface requires these properties and methods:
|
||||
|
||||
### Required Properties
|
||||
|
||||
| Property | Type | Description |
|
||||
|---|---|---|
|
||||
| `id` | `string` | Unique plugin identifier (e.g., `"my-data"`) |
|
||||
| `name` | `string` | Display name shown in the sidebar |
|
||||
| `description` | `string` | Short description |
|
||||
| `icon` | `string \| ComponentType` | Lucide icon name or React component |
|
||||
| `category` | `PluginCategory` | One of: `aviation`, `maritime`, `conflict`, `natural-disaster`, `infrastructure`, `cyber`, `economic`, `custom` |
|
||||
| `version` | `string` | Semver version string |
|
||||
|
||||
### Required Methods
|
||||
|
||||
| Method | Returns | Description |
|
||||
|---|---|---|
|
||||
| `initialize(ctx)` | `Promise<void>` | Set up the plugin. Receives `PluginContext` with API URLs, time range, callbacks |
|
||||
| `destroy()` | `void` | Clean up resources (timers, sockets, etc.) |
|
||||
| `fetch(timeRange)` | `Promise<GeoEntity[]>` | Return the data to display on the globe |
|
||||
| `getPollingInterval()` | `number` | Milliseconds between `fetch()` calls |
|
||||
| `getLayerConfig()` | `LayerConfig` | Layer appearance (color, clustering, limits) |
|
||||
| `renderEntity(entity)` | `CesiumEntityOptions` | How each entity renders on the globe |
|
||||
|
||||
### Optional Methods
|
||||
|
||||
| Method | Returns | Description |
|
||||
|---|---|---|
|
||||
| `getSelectionBehavior(entity)` | `SelectionBehavior \| null` | Trail / fly-to on click |
|
||||
| `getServerConfig()` | `ServerPluginConfig` | Server-side API routing |
|
||||
| `getFilterDefinitions()` | `FilterDefinition[]` | Declarative filter UI |
|
||||
| `getSidebarComponent()` | `ComponentType` | Custom sidebar panel |
|
||||
| `getDetailComponent()` | `ComponentType<{ entity }>` | Custom detail panel |
|
||||
| `getSettingsComponent()` | `ComponentType<{ pluginId }>` | Custom settings panel |
|
||||
|
||||
### Key Types
|
||||
|
||||
```typescript
|
||||
interface GeoEntity {
|
||||
id: string;
|
||||
pluginId: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
altitude?: number;
|
||||
heading?: number;
|
||||
speed?: number;
|
||||
timestamp: Date;
|
||||
label?: string;
|
||||
properties: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface LayerConfig {
|
||||
color: string;
|
||||
iconUrl?: string;
|
||||
clusterEnabled: boolean;
|
||||
clusterDistance: number;
|
||||
minZoomLevel?: number;
|
||||
maxEntities?: number;
|
||||
}
|
||||
|
||||
interface CesiumEntityOptions {
|
||||
type: "billboard" | "point" | "polyline" | "polygon" | "label" | "model";
|
||||
color?: string;
|
||||
size?: number;
|
||||
iconUrl?: string;
|
||||
rotation?: number;
|
||||
modelUrl?: string;
|
||||
modelScale?: number;
|
||||
// ... more options
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Real Examples
|
||||
|
||||
### Borders Plugin (bundle format)
|
||||
|
||||
A minimal bundle plugin that implements the `WorldPlugin` interface directly:
|
||||
|
||||
- **Source**: `packages/wwv-plugin-borders/src/index.ts`
|
||||
- **npm**: `@worldwideview/wwv-plugin-borders`
|
||||
|
||||
### Military Bases Plugin (static format)
|
||||
|
||||
A static GeoJSON plugin where the npm package is just metadata:
|
||||
|
||||
- **Source**: `packages/wwv-plugin-military-bases/src/index.ts`
|
||||
- **GeoJSON**: `public/data/military_bases.geojson`
|
||||
- **npm**: `@worldwideview/wwv-plugin-military-bases`
|
||||
- **Manifest**: Defined in `worldwideview-marketplace/src/data/pluginManifests.ts`
|
||||
|
||||
---
|
||||
|
||||
## How It All Connects
|
||||
|
||||
```
|
||||
Developer writes plugin
|
||||
│
|
||||
▼
|
||||
Publish to npm ► npm registry (stores version, description, keywords, readme)
|
||||
│
|
||||
▼
|
||||
Submit npm package name to Marketplace
|
||||
│
|
||||
▼
|
||||
Marketplace fetches metadata from npm ← npm registry
|
||||
and reads the "worldwideview" block from package.json
|
||||
│
|
||||
▼
|
||||
Review → Plugin appears in browse catalog
|
||||
│
|
||||
▼
|
||||
User clicks "Install"
|
||||
│
|
||||
▼
|
||||
Marketplace sends manifest to WorldWideView instance
|
||||
│
|
||||
▼
|
||||
Plugin saved to database (Prisma) → loaded on next startup
|
||||
│
|
||||
▼
|
||||
loadPluginFromManifest() routes by format:
|
||||
static → StaticDataPlugin (GeoJSON loader)
|
||||
bundle → dynamic import()
|
||||
│
|
||||
▼
|
||||
Plugin renders on the 3D globe
|
||||
```
|
||||
@@ -0,0 +1,216 @@
|
||||
# Plugin Architecture Migration Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Migrate the WorldWideView plugin architecture from compile-time hardcoded imports to a dynamic, runtime-loaded ecosystem where external developers can build, host, and distribute plugins without touching the core repository.
|
||||
|
||||
**Architecture:** We will introduce a bundler (`tsup`) to the SDK to compile `bundle` plugins into browser-ready ECMAScript Modules (ESM). We will expose shared dependencies (`react`, `cesium`, `lucide-react`) via a global `window.__WWV_SHARED__` object. Finally, we will refactor `AppShell.tsx` to dynamically load all plugins at runtime via `import(/* webpackIgnore: true */)`, removing the hardcoded imports, and we will update `loadPluginFromManifest.ts` to seamlessly handle the global dependency injection.
|
||||
|
||||
**Tech Stack:** Next.js, React, CesiumJS, tsup.
|
||||
|
||||
---
|
||||
|
||||
## User Review Required
|
||||
|
||||
> [!WARNING]
|
||||
> **Breaking Change**: This migration will decouple your built-in plugins from `AppShell.tsx`. They will now act as true "external" plugins that are loaded at runtime. During the transition, if the build steps aren't executed, the plugins will temporarily stop appearing on the globe. We will mitigate this by ensuring the build step runs before `next dev`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Static GeoJSON Files**: Since static `.geojson` files currently live in `public/data`, we will leave them there during phase 1 of this migration to ensure the app doesn't break, but we will point the plugin manifests to load them via absolute or absolute-relative URLs (`/data/...`). In phase 2, we will introduce a symlink script to move them out of `public` entirely.
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Build Tool Choice**: I am proposing `tsup` because it requires zero configuration to output multiple formats, automatically externalizes dependencies, and natively transpiles TypeScript/React. Are you comfortable adding `tsup` to the `wwv-plugin-sdk` package?
|
||||
2. **Local Serving**: To test "built-in" plugins using the dynamic loading method, we need the Next.js app to serve their compiled `plugin.js` bundles. My plan adds a quick copy script to the root `package.json` that copies their `dist` folders into `public/builtin-plugins/` so they can be fetched locally via HTTP. Does this sound good?
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### Task 1: Initialize Global Shared Dependencies
|
||||
|
||||
**Files:**
|
||||
- Modify: `c:\dev\worldwideview\src\components\layout\AppShell.tsx`
|
||||
- Create: `c:\dev\worldwideview\src\core\plugins\PluginGlobals.ts`
|
||||
|
||||
- [ ] **Step 1: Create the PluginGlobals initializer**
|
||||
|
||||
```typescript
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom";
|
||||
import * as ReactDomServer from "react-dom/server";
|
||||
import { Plane, Ship, Camera, Shield, Flame, Map, Swords, Atom, Landmark, Mountain, PlaneTakeoff, Anchor, Lamp, Rocket, SunMoon, Cable, Pickaxe, ShieldAlert, Crosshair, SatelliteDish, Hand, Radar, Scale } from "lucide-react";
|
||||
import * as Cesium from "cesium";
|
||||
import * as WWVPluginSDK from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
export function initPluginGlobals() {
|
||||
if (typeof window === "undefined") return;
|
||||
|
||||
// Create a registry of Lucide icons commonly used by plugins to prevent them from bundling the massive icon library
|
||||
const lucideIcons = {
|
||||
Plane, Ship, Camera, Shield, Flame, Map, Swords, Atom, Landmark, Mountain,
|
||||
PlaneTakeoff, Anchor, Lamp, Rocket, SunMoon, Cable, Pickaxe, ShieldAlert,
|
||||
Crosshair, SatelliteDish, Hand, Radar, Scale
|
||||
};
|
||||
|
||||
(window as any).__WWV_SHARED__ = {
|
||||
"react": React,
|
||||
"react-dom": ReactDOM,
|
||||
"react-dom/server": ReactDomServer,
|
||||
"cesium": Cesium,
|
||||
"@worldwideview/wwv-plugin-sdk": WWVPluginSDK,
|
||||
"lucide-react": lucideIcons
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Inject globals in AppShell before anything loads**
|
||||
|
||||
In `AppShell.tsx`, import and call `initPluginGlobals()` at the top of the file, outside of any React components.
|
||||
|
||||
```tsx
|
||||
import { initPluginGlobals } from "@/core/plugins/PluginGlobals";
|
||||
import dynamic from "next/dynamic";
|
||||
|
||||
// Initialize globals immediately so they exist before any dynamic imports
|
||||
if (typeof window !== "undefined") {
|
||||
initPluginGlobals();
|
||||
}
|
||||
//... rest of the file
|
||||
```
|
||||
|
||||
### Task 2: Configure SDK Build Tool (`tsup`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `c:\dev\worldwideview\packages\wwv-plugin-sdk\package.json`
|
||||
- Create: `c:\dev\worldwideview\packages\wwv-plugin-sdk\bin\wwv-plugin-build.js`
|
||||
|
||||
- [ ] **Step 1: Add tsup dependency**
|
||||
|
||||
Run: `pnpm --filter @worldwideview/wwv-plugin-sdk add -D tsup`
|
||||
|
||||
- [ ] **Step 2: Create the wwv-plugin-build CLI wrapper**
|
||||
|
||||
Add this to `wwv-plugin-sdk/bin/wwv-plugin-build.js`:
|
||||
|
||||
```javascript
|
||||
#!/usr/bin/env node
|
||||
const { build } = require("tsup");
|
||||
const path = require("path");
|
||||
|
||||
async function run() {
|
||||
const cwd = process.cwd();
|
||||
await build({
|
||||
entry: ["src/index.ts"],
|
||||
outDir: "dist",
|
||||
format: ["esm"],
|
||||
target: "es2022",
|
||||
clean: true,
|
||||
dts: false,
|
||||
minify: false,
|
||||
sourcemap: true,
|
||||
// Mark all these as externals so they aren't bundled
|
||||
external: ["react", "react-dom", "cesium", "@worldwideview/wwv-plugin-sdk", "lucide-react"],
|
||||
// Provide a shim that converts bare imports into global lookups
|
||||
esbuildPlugins: [{
|
||||
name: "wwv-globals",
|
||||
setup(build) {
|
||||
build.onResolve({ filter: /^(react|react-dom|cesium|@worldwideview\/wwv-plugin-sdk|lucide-react)$/ }, args => {
|
||||
return { path: args.path, namespace: "wwv-global" };
|
||||
});
|
||||
build.onLoad({ filter: /.*/, namespace: "wwv-global" }, args => {
|
||||
let globalKey = args.path;
|
||||
if (args.path === "lucide-react") return { contents: `module.exports = window.__WWV_SHARED__["lucide-react"];` };
|
||||
return { contents: `module.exports = window.__WWV_SHARED__["${globalKey}"];` };
|
||||
});
|
||||
}
|
||||
}]
|
||||
});
|
||||
}
|
||||
run();
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update SDK package.json to expose the bin**
|
||||
|
||||
```json
|
||||
"bin": {
|
||||
"wwv-plugin-build": "./bin/wwv-plugin-build.js"
|
||||
},
|
||||
```
|
||||
|
||||
### Task 3: Refactor Plugins to use Build Tool and Manifests
|
||||
|
||||
**Files:**
|
||||
- Modify: `wwv-plugin-aviation/package.json`
|
||||
- Modify: `wwv-plugin-aviation/plugin.json` (Create)
|
||||
|
||||
- [ ] **Step 1: Add build script to aviation plugin**
|
||||
|
||||
```json
|
||||
"scripts": {
|
||||
"build": "wwv-plugin-build"
|
||||
},
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Create plugin.json for aviation**
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "aviation",
|
||||
"name": "Aviation",
|
||||
"version": "1.0.7",
|
||||
"type": "data-layer",
|
||||
"format": "bundle",
|
||||
"trust": "built-in",
|
||||
"capabilities": ["data:own", "network:fetch"],
|
||||
"category": "aviation",
|
||||
"icon": "Plane",
|
||||
"entry": "/builtin-plugins/aviation/index.mjs"
|
||||
}
|
||||
```
|
||||
|
||||
*(Repeat for all other built-in plugins)*
|
||||
|
||||
### Task 4: Dynamic Initial Loader
|
||||
|
||||
**Files:**
|
||||
- Modify: `c:\dev\worldwideview\src\components\layout\AppShell.tsx`
|
||||
- Modify: `c:\dev\worldwideview\package.json`
|
||||
|
||||
- [ ] **Step 1: Remove Hardcoded Plugins**
|
||||
|
||||
In `AppShell.tsx`, delete the `builtIns` array and `import` statements at the top.
|
||||
|
||||
- [ ] **Step 2: Fetch and load all plugins via manifest**
|
||||
|
||||
Modify `AppShell.tsx` `startPlatform`:
|
||||
|
||||
```typescript
|
||||
// Fetch the unified list of ALL installed plugins (both built-in and dynamic)
|
||||
const res = await fetch("/api/marketplace/load");
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
for (const manifest of data.manifests) {
|
||||
if (disabledIds.has(manifest.id)) continue;
|
||||
try {
|
||||
await pluginManager.loadFromManifest(manifest);
|
||||
initLayer(manifest.id);
|
||||
} catch (e) {
|
||||
console.error("Failed to load plugin", manifest.id, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add predev copy-script**
|
||||
|
||||
Modify `package.json` to copy compiled plugin `dist` folders to `public/builtin-plugins` so the Next.js server can serve them at the `/builtin-plugins/...` path.
|
||||
|
||||
## Verification Plan
|
||||
|
||||
### Automated Tests
|
||||
- Run `pnpm --filter wwv-plugin-aviation build` to ensure `tsup` generates a valid ES Module with no bundled React.
|
||||
- Verify `dist/index.mjs` contains `window.__WWV_SHARED__["react"]` lookups.
|
||||
|
||||
### Manual Verification
|
||||
- Start the Next.js server.
|
||||
- Open network tab in browser and verify `GET /builtin-plugins/aviation/index.mjs` succeeds.
|
||||
- Verify the Aviation plugin appears on the globe and renders planes successfully, indicating that it successfully retrieved Cesium and React from the global shared context.
|
||||
@@ -0,0 +1,475 @@
|
||||
# Demo Google AdSense Panel Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a Google AdSense vertical ad strip to the right side of the demo instance, with a dismissible banner explaining it's temporary to fund hosting.
|
||||
|
||||
**Architecture:** Three layers: (1) load the AdSense script conditionally in the root layout only when `isDemo` is true, (2) a reusable `AdUnit` client component that calls `adsbygoogle.push({})`, (3) a `DemoAdStrip` shell component that wraps the ad unit + a "why ads?" dismissible message, positioned as a fixed vertical strip on the right edge. CSP headers in `next.config.ts` must be updated to allow the AdSense domains.
|
||||
|
||||
**Tech Stack:** Next.js 16 (App Router, `next/script`), Google AdSense, vanilla CSS, Vitest
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Action | Responsibility |
|
||||
|---|---|---|
|
||||
| `src/app/layout.tsx` | MODIFY | Conditionally load AdSense `<Script>` for demo |
|
||||
| `next.config.ts` | MODIFY | Add AdSense domains to CSP `script-src` and `frame-src` |
|
||||
| `.env` | MODIFY | Add `NEXT_PUBLIC_ADSENSE_CLIENT_ID` placeholder |
|
||||
| `src/components/ads/AdUnit.tsx` | CREATE | Generic reusable AdSense unit component |
|
||||
| `src/components/ads/DemoAdStrip.tsx` | CREATE | Demo-only vertical strip: ad + "why ads?" message |
|
||||
| `src/components/ads/DemoAdStrip.css` | CREATE | Styles for the vertical ad strip |
|
||||
| `src/components/layout/AppShell.tsx` | MODIFY | Import and render `<DemoAdStrip />` |
|
||||
| `src/types/adsense.d.ts` | CREATE | TypeScript ambient declaration for `window.adsbygoogle` |
|
||||
|
||||
---
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Before starting:** You need a Google AdSense publisher ID (`ca-pub-XXXXXXXXXXXXXXXX`) and an ad slot ID. These go into `.env` as `NEXT_PUBLIC_ADSENSE_CLIENT_ID` and are passed to the ad unit component. You can get these from [Google AdSense](https://www.google.com/adsense/). Without them the ads will not render, but the component framework will still work.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: TypeScript Ambient Declaration for AdSense
|
||||
|
||||
**Files:**
|
||||
- Create: `src/types/adsense.d.ts`
|
||||
|
||||
- [ ] **Step 1: Create the ambient type declaration**
|
||||
|
||||
```ts
|
||||
// src/types/adsense.d.ts
|
||||
interface Window {
|
||||
adsbygoogle: Array<Record<string, unknown>>;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add src/types/adsense.d.ts
|
||||
git commit -m "chore: add adsense ambient type declaration"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Environment Variable
|
||||
|
||||
**Files:**
|
||||
- Modify: `.env`
|
||||
|
||||
- [ ] **Step 1: Add the AdSense client ID placeholder**
|
||||
|
||||
Add the following line to the end of `.env`:
|
||||
|
||||
```
|
||||
# Google AdSense — only used on demo edition
|
||||
NEXT_PUBLIC_ADSENSE_CLIENT_ID=
|
||||
```
|
||||
|
||||
The actual value (`ca-pub-XXXXXXXXXXXXXXXX`) should be set in `.env.local` for local testing or in the deployment environment. The ad slot IDs will be hardcoded in the component since there's only one placement.
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add .env
|
||||
git commit -m "chore: add NEXT_PUBLIC_ADSENSE_CLIENT_ID env var"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: CSP Headers Update
|
||||
|
||||
**Files:**
|
||||
- Modify: `next.config.ts:29-52`
|
||||
|
||||
- [ ] **Step 1: Update CSP in next.config.ts**
|
||||
|
||||
In the `headers()` function, modify the `script-src` and `frame-src` directives to allow AdSense domains.
|
||||
|
||||
```ts
|
||||
// Change this line (line 39):
|
||||
"script-src 'self' 'unsafe-eval' 'unsafe-inline' blob: https://analytics.worldwideview.dev https://va.vercel-scripts.com",
|
||||
|
||||
// To this:
|
||||
"script-src 'self' 'unsafe-eval' 'unsafe-inline' blob: https://analytics.worldwideview.dev https://va.vercel-scripts.com https://pagead2.googlesyndication.com https://adservice.google.com https://www.googletagservices.com",
|
||||
```
|
||||
|
||||
```ts
|
||||
// Change this line (line 49):
|
||||
"frame-src 'self' *.youtube.com *.youtube-nocookie.com *.twitch.tv *.vimeo.com *.webcamera.pl *.ivideon.com *.rtsp.me *.bnu.tv",
|
||||
|
||||
// To this:
|
||||
"frame-src 'self' *.youtube.com *.youtube-nocookie.com *.twitch.tv *.vimeo.com *.webcamera.pl *.ivideon.com *.rtsp.me *.bnu.tv https://googleads.g.doubleclick.net https://tpc.googlesyndication.com",
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add next.config.ts
|
||||
git commit -m "feat: add Google AdSense domains to CSP headers"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: AdSense Script in Root Layout
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/app/layout.tsx:24-39`
|
||||
|
||||
- [ ] **Step 1: Conditionally load AdSense script for demo edition**
|
||||
|
||||
Add the AdSense `<Script>` tag inside `<head>`, using the `NEXT_PUBLIC_` env var for the client ID and the `NEXT_PUBLIC_WWV_EDITION` env var to gate loading.
|
||||
|
||||
```tsx
|
||||
// src/app/layout.tsx — updated <head> section
|
||||
<head>
|
||||
{/* Load CesiumJS base styles (optional, but helps with UI widgets if used later) */}
|
||||
<link rel="stylesheet" href="/cesium/Widgets/widgets.css" />
|
||||
</head>
|
||||
<body suppressHydrationWarning>
|
||||
{children}
|
||||
<Analytics />
|
||||
<Script
|
||||
src="https://analytics.worldwideview.dev/script.js"
|
||||
data-website-id="2c8f6c09-2651-4a2a-af99-b8cee1612b9a"
|
||||
strategy="afterInteractive"
|
||||
/>
|
||||
{process.env.NEXT_PUBLIC_WWV_EDITION === "demo" &&
|
||||
process.env.NEXT_PUBLIC_ADSENSE_CLIENT_ID && (
|
||||
<Script
|
||||
src={`https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${process.env.NEXT_PUBLIC_ADSENSE_CLIENT_ID}`}
|
||||
crossOrigin="anonymous"
|
||||
strategy="afterInteractive"
|
||||
/>
|
||||
)}
|
||||
</body>
|
||||
```
|
||||
|
||||
> **Note:** We use the raw env var at the layout level (Server Component) rather than importing `isDemo` which uses `process.env.NEXT_PUBLIC_WWV_EDITION` anyway. This avoids importing the edition module into the root layout.
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add src/app/layout.tsx
|
||||
git commit -m "feat: conditionally load AdSense script on demo edition"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Reusable AdUnit Component
|
||||
|
||||
**Files:**
|
||||
- Create: `src/components/ads/AdUnit.tsx`
|
||||
|
||||
- [ ] **Step 1: Create the generic AdSense unit component**
|
||||
|
||||
```tsx
|
||||
// src/components/ads/AdUnit.tsx
|
||||
"use client";
|
||||
|
||||
import { useEffect, useRef } from "react";
|
||||
|
||||
interface AdUnitProps {
|
||||
adSlot: string;
|
||||
adFormat?: string;
|
||||
style?: React.CSSProperties;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function AdUnit({ adSlot, adFormat = "auto", style, className }: AdUnitProps) {
|
||||
const pushed = useRef(false);
|
||||
const clientId = process.env.NEXT_PUBLIC_ADSENSE_CLIENT_ID;
|
||||
|
||||
useEffect(() => {
|
||||
if (pushed.current || !clientId) return;
|
||||
try {
|
||||
(window.adsbygoogle = window.adsbygoogle || []).push({});
|
||||
pushed.current = true;
|
||||
} catch (err) {
|
||||
console.error("[AdUnit] AdSense push error:", err);
|
||||
}
|
||||
}, [clientId]);
|
||||
|
||||
if (!clientId) return null;
|
||||
|
||||
return (
|
||||
<div className={className} style={style}>
|
||||
<ins
|
||||
className="adsbygoogle"
|
||||
style={{ display: "block" }}
|
||||
data-ad-client={clientId}
|
||||
data-ad-slot={adSlot}
|
||||
data-ad-format={adFormat}
|
||||
data-full-width-responsive="false"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
> **Key detail:** The `pushed` ref prevents double-push on React strict mode / hot reload. `data-full-width-responsive="false"` prevents the ad from expanding beyond the container — critical for a fixed-width vertical strip.
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add src/components/ads/AdUnit.tsx
|
||||
git commit -m "feat: add reusable AdUnit component"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: DemoAdStrip Component + Styles
|
||||
|
||||
**Files:**
|
||||
- Create: `src/components/ads/DemoAdStrip.tsx`
|
||||
- Create: `src/components/ads/DemoAdStrip.css`
|
||||
|
||||
- [ ] **Step 1: Create the CSS for the vertical ad strip**
|
||||
|
||||
```css
|
||||
/* src/components/ads/DemoAdStrip.css */
|
||||
|
||||
.demo-ad-strip {
|
||||
position: fixed;
|
||||
right: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 160px;
|
||||
z-index: 45;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: var(--space-md);
|
||||
padding: var(--space-sm);
|
||||
pointer-events: none;
|
||||
transition: transform var(--duration-slow) var(--ease-smooth),
|
||||
opacity var(--duration-slow) var(--ease-smooth);
|
||||
}
|
||||
|
||||
.demo-ad-strip--hidden {
|
||||
transform: translateX(200px);
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.demo-ad-strip__ad {
|
||||
pointer-events: auto;
|
||||
width: 160px;
|
||||
min-height: 600px;
|
||||
}
|
||||
|
||||
.demo-ad-strip__banner {
|
||||
pointer-events: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: var(--space-xs);
|
||||
padding: var(--space-sm) var(--space-md);
|
||||
background: var(--bg-glass);
|
||||
backdrop-filter: blur(var(--glass-blur));
|
||||
-webkit-backdrop-filter: blur(var(--glass-blur));
|
||||
border: 1px solid var(--border-subtle);
|
||||
border-radius: var(--radius-md);
|
||||
max-width: 160px;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: all var(--duration-fast) var(--ease-smooth);
|
||||
}
|
||||
|
||||
.demo-ad-strip__banner:hover {
|
||||
border-color: rgba(var(--accent-rgb), 0.3);
|
||||
box-shadow: var(--glow-cyan);
|
||||
}
|
||||
|
||||
.demo-ad-strip__banner-label {
|
||||
font-size: 9px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.demo-ad-strip__banner-text {
|
||||
font-size: 11px;
|
||||
line-height: 1.4;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.demo-ad-strip__close {
|
||||
position: absolute;
|
||||
top: var(--space-xs);
|
||||
right: var(--space-xs);
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 50%;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: var(--text-muted);
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
transition: color var(--duration-fast) var(--ease-smooth);
|
||||
}
|
||||
|
||||
.demo-ad-strip__close:hover {
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
/* Hide on mobile — ads shouldn't overlay the full-screen globe */
|
||||
@media (max-width: 768px) {
|
||||
.demo-ad-strip {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Create the DemoAdStrip component**
|
||||
|
||||
```tsx
|
||||
// src/components/ads/DemoAdStrip.tsx
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { isDemo } from "@/core/edition";
|
||||
import { AdUnit } from "./AdUnit";
|
||||
import { X } from "lucide-react";
|
||||
import "./DemoAdStrip.css";
|
||||
|
||||
/** The ad slot ID for the vertical skyscraper unit — set this in AdSense dashboard. */
|
||||
const VERTICAL_AD_SLOT = "REPLACE_WITH_YOUR_AD_SLOT_ID";
|
||||
|
||||
export function DemoAdStrip() {
|
||||
const [showMessage, setShowMessage] = useState(false);
|
||||
const [closed, setClosed] = useState(false);
|
||||
|
||||
if (!isDemo || closed) return null;
|
||||
|
||||
return (
|
||||
<aside className="demo-ad-strip" aria-label="Sponsored content">
|
||||
<AdUnit
|
||||
adSlot={VERTICAL_AD_SLOT}
|
||||
adFormat="vertical"
|
||||
className="demo-ad-strip__ad"
|
||||
/>
|
||||
|
||||
<div
|
||||
className="demo-ad-strip__banner"
|
||||
onClick={() => setShowMessage((prev) => !prev)}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onKeyDown={(e) => { if (e.key === "Enter") setShowMessage((prev) => !prev); }}
|
||||
>
|
||||
<button
|
||||
className="demo-ad-strip__close"
|
||||
onClick={(e) => { e.stopPropagation(); setClosed(true); }}
|
||||
aria-label="Dismiss ad panel"
|
||||
>
|
||||
<X size={12} />
|
||||
</button>
|
||||
<span className="demo-ad-strip__banner-label">Why ads?</span>
|
||||
{showMessage ? (
|
||||
<span className="demo-ad-strip__banner-text">
|
||||
This isn't permanent — it's temporary because I'm a single
|
||||
dev and need to cover hosting costs. Thank you for understanding.
|
||||
</span>
|
||||
) : (
|
||||
<span className="demo-ad-strip__banner-text">
|
||||
Click to learn more
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add src/components/ads/DemoAdStrip.tsx src/components/ads/DemoAdStrip.css
|
||||
git commit -m "feat: add DemoAdStrip with Google AdSense vertical unit"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Integrate into AppShell
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/components/layout/AppShell.tsx:8-9,168`
|
||||
|
||||
- [ ] **Step 1: Add import**
|
||||
|
||||
After the existing `CameraStatsPanel` import (line 8), add:
|
||||
|
||||
```tsx
|
||||
import CameraStatsPanel from "@/components/panels/CameraStatsPanel";
|
||||
import { DemoAdStrip } from "@/components/ads/DemoAdStrip"; // ADD THIS
|
||||
import { Timeline } from "@/components/timeline/Timeline";
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add the JSX**
|
||||
|
||||
After `<EntityInfoCard />` (line 168), add:
|
||||
|
||||
```tsx
|
||||
<EntityInfoCard />
|
||||
<DemoAdStrip />
|
||||
<Timeline />
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add src/components/layout/AppShell.tsx
|
||||
git commit -m "feat: integrate DemoAdStrip into AppShell"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: ads.txt for AdSense Verification
|
||||
|
||||
**Files:**
|
||||
- Create: `public/ads.txt`
|
||||
|
||||
- [ ] **Step 1: Create ads.txt**
|
||||
|
||||
```
|
||||
google.com, ca-pub-XXXXXXXXXXXXXXXX, DIRECT, f08c47fec0942fa0
|
||||
```
|
||||
|
||||
> Replace `ca-pub-XXXXXXXXXXXXXXXX` with your actual publisher ID. This file is required by AdSense for domain verification.
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add public/ads.txt
|
||||
git commit -m "chore: add ads.txt for AdSense domain verification"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Do you have a Google AdSense publisher ID yet?** If not, you'll need to sign up at https://www.google.com/adsense/ and get your `ca-pub-XXXX` ID and create a 160x600 "Wide Skyscraper" ad unit to get the slot ID. The code will render nothing gracefully until these are configured.
|
||||
|
||||
> [!WARNING]
|
||||
> **CSP changes are required for AdSense to work.** The current CSP is restrictive and blocks `pagead2.googlesyndication.com` and `googleads.g.doubleclick.net`. Task 3 adds them. Without this, ads will be silently blocked by the browser.
|
||||
|
||||
---
|
||||
|
||||
## Verification Plan
|
||||
|
||||
### Manual Verification
|
||||
|
||||
1. Set `NEXT_PUBLIC_WWV_EDITION=demo` and `NEXT_PUBLIC_ADSENSE_CLIENT_ID=ca-pub-XXXX` in `.env.local`
|
||||
2. Run `pnpm dev`
|
||||
3. Verify: A 160px vertical strip appears on the right edge of the screen
|
||||
4. Verify: The "Why ads?" banner is clickable and toggles the explanation message
|
||||
5. Verify: Clicking the X dismisses the entire strip
|
||||
6. Verify: On mobile viewport widths (< 768px), the strip is hidden
|
||||
7. Verify: Set `NEXT_PUBLIC_WWV_EDITION=local` → strip does not render at all
|
||||
8. Verify: Browser console shows no CSP violations related to AdSense domains
|
||||
File diff suppressed because it is too large.
Load diff
+336749
File diff suppressed because it is too large.
Load diff
+84
-36
@@ -1,46 +1,94 @@
|
||||
import type { NextConfig } from "next";
|
||||
import path from "path";
|
||||
import webpack from "webpack";
|
||||
import CopyPlugin from "copy-webpack-plugin";
|
||||
|
||||
const cesiumSource = path.resolve(__dirname, "node_modules/cesium/Build/Cesium");
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
serverExternalPackages: ["better-sqlite3", "@prisma/adapter-better-sqlite3", "@prisma/client", "prisma"],
|
||||
transpilePackages: [
|
||||
"@worldwideview/wwv-plugin-aviation",
|
||||
"@worldwideview/wwv-plugin-borders",
|
||||
"@worldwideview/wwv-plugin-camera",
|
||||
"@worldwideview/wwv-plugin-conflict-events",
|
||||
"@worldwideview/wwv-plugin-civil-unrest",
|
||||
"@worldwideview/wwv-plugin-daynight",
|
||||
"@worldwideview/wwv-plugin-undersea-cables",
|
||||
"@worldwideview/wwv-plugin-earthquakes",
|
||||
"@worldwideview/wwv-plugin-iranwarlive",
|
||||
"@worldwideview/wwv-plugin-maritime",
|
||||
"@worldwideview/wwv-plugin-military-aviation",
|
||||
"@worldwideview/wwv-plugin-satellite",
|
||||
"@worldwideview/wwv-plugin-gps-jamming",
|
||||
"@worldwideview/wwv-plugin-surveillance-satellites",
|
||||
"@worldwideview/wwv-plugin-cyber-attacks",
|
||||
"@worldwideview/wwv-plugin-international-sanctions",
|
||||
"@worldwideview/wwv-plugin-sdk",
|
||||
"@worldwideview/wwv-plugin-wildfire",
|
||||
"resium",
|
||||
"react-player",
|
||||
"satellite.js"
|
||||
],
|
||||
experimental: {
|
||||
memoryBasedWorkersCount: true,
|
||||
cpus: 2,
|
||||
optimizePackageImports: ["lucide-react"],
|
||||
},
|
||||
typescript: {
|
||||
ignoreBuildErrors: true,
|
||||
},
|
||||
async headers() {
|
||||
return [
|
||||
{
|
||||
source: "/(.*)",
|
||||
headers: [
|
||||
{
|
||||
key: "Content-Security-Policy",
|
||||
value: [
|
||||
"default-src 'self'",
|
||||
// CesiumJS requires unsafe-eval (worker compilation) and unsafe-inline (styles)
|
||||
"script-src 'self' 'unsafe-eval' 'unsafe-inline' blob: https://cdn.jsdelivr.net https://analytics.worldwideview.dev https://va.vercel-scripts.com https://pagead2.googlesyndication.com https://adservice.google.com https://www.googletagservices.com https://ep2.adtrafficquality.google https://static.cloudflareinsights.com",
|
||||
"style-src 'self' 'unsafe-inline' fonts.googleapis.com",
|
||||
"font-src 'self' fonts.gstatic.com",
|
||||
// Camera streams load images/MJPEG from arbitrary IPs worldwide — http: https: required
|
||||
"img-src 'self' data: blob: http: https:",
|
||||
// Camera HLS streams and external data fetches need arbitrary origins
|
||||
"connect-src 'self' http: https: ws: wss:",
|
||||
// HLS video streams from arbitrary camera sources
|
||||
"media-src 'self' blob: http: https:",
|
||||
// Embeddable video platforms for camera iframes
|
||||
"frame-src 'self' *.youtube.com *.youtube-nocookie.com *.twitch.tv *.vimeo.com *.webcamera.pl *.ivideon.com *.rtsp.me *.bnu.tv https://googleads.g.doubleclick.net https://tpc.googlesyndication.com https://ep2.adtrafficquality.google https://*.google.com",
|
||||
"worker-src 'self' blob:",
|
||||
"frame-ancestors 'none'",
|
||||
].join("; "),
|
||||
},
|
||||
{ key: "X-Frame-Options", value: "DENY" },
|
||||
{ key: "X-Content-Type-Options", value: "nosniff" },
|
||||
{ key: "Referrer-Policy", value: "strict-origin-when-cross-origin" },
|
||||
{ key: "X-DNS-Prefetch-Control", value: "on" },
|
||||
{
|
||||
key: "Permissions-Policy",
|
||||
value: "camera=(), microphone=(), geolocation=(self)",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
},
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
source: "/api/external/:source*",
|
||||
// Proxy to the shared data engine locally or over docker network
|
||||
// We use process.env to override the url if running Next.js outside docker
|
||||
destination: `${process.env.WWV_DATA_ENGINE_URL || "http://wwv-data-engine:5001"}/data/:source*`,
|
||||
},
|
||||
];
|
||||
},
|
||||
outputFileTracingExcludes: {
|
||||
"*": ["./public/cesium/**"],
|
||||
},
|
||||
env: {
|
||||
CESIUM_BASE_URL: "/cesium",
|
||||
},
|
||||
webpack: (config, { isServer }) => {
|
||||
webpack: (config, { isServer, webpack }) => {
|
||||
if (!isServer) {
|
||||
// Copy Cesium static assets to public/cesium (Production only)
|
||||
if (process.env.NODE_ENV === "production" || !isServer) {
|
||||
// We only add it here if it's production or if we're not using predev
|
||||
// To be safe, let's just make it only run if NOT in dev
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
config.plugins?.push(
|
||||
new CopyPlugin({
|
||||
patterns: [
|
||||
{
|
||||
from: path.join(cesiumSource, "Workers"),
|
||||
to: path.resolve(__dirname, "public/cesium/Workers"),
|
||||
},
|
||||
{
|
||||
from: path.join(cesiumSource, "ThirdParty"),
|
||||
to: path.resolve(__dirname, "public/cesium/ThirdParty"),
|
||||
},
|
||||
{
|
||||
from: path.join(cesiumSource, "Assets"),
|
||||
to: path.resolve(__dirname, "public/cesium/Assets"),
|
||||
},
|
||||
{
|
||||
from: path.join(cesiumSource, "Widgets"),
|
||||
to: path.resolve(__dirname, "public/cesium/Widgets"),
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Define CESIUM_BASE_URL for Cesium's worker resolution
|
||||
config.plugins?.push(
|
||||
new webpack.DefinePlugin({
|
||||
|
||||
Generated
-4480
File diff suppressed because it is too large.
Load diff
+60
-7
@@ -1,36 +1,89 @@
|
||||
{
|
||||
"name": "worldwideview",
|
||||
"version": "0.1.0",
|
||||
"version": "1.7.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"setup": "node scripts/setup.mjs",
|
||||
"clean": "node -e \"const fs=require('fs');fs.rmSync('.next',{recursive:true,force:true})\"",
|
||||
"dev": "next dev --webpack",
|
||||
"predev": "node scripts/copy-cesium.mjs",
|
||||
"dev:all": "npx concurrently \"pnpm dev\" \"pnpm --filter wwv-data-engine dev\"",
|
||||
"predev": "npx prisma migrate deploy && node scripts/copy-cesium.mjs",
|
||||
"copy-cesium": "node scripts/copy-cesium.mjs",
|
||||
"build": "next build --webpack",
|
||||
"start": "next start"
|
||||
"start": "next start",
|
||||
"test": "vitest run",
|
||||
"start:backends": "pnpm --filter wwv-data-engine start",
|
||||
"dev:backends": "pnpm --filter wwv-data-engine dev",
|
||||
"clean:backends": "pnpm --filter wwv-data-engine clean:db",
|
||||
"db:reset": "npx prisma migrate reset --force"
|
||||
},
|
||||
"dependencies": {
|
||||
"@auth/prisma-adapter": "^2.11.1",
|
||||
"@prisma/adapter-better-sqlite3": "^7.5.0",
|
||||
"@prisma/client": "^7.5.0",
|
||||
"@supabase/supabase-js": "^2.98.0",
|
||||
"@vercel/analytics": "^1.6.1",
|
||||
"@worldwideview/wwv-plugin-aviation": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-borders": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-camera": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-civil-unrest": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-conflict-events": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-cyber-attacks": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-daynight": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-earthquakes": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-gps-jamming": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-international-sanctions": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-iranwarlive": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-maritime": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-military-aviation": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-satellite": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-surveillance-satellites": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-undersea-cables": "workspace:*",
|
||||
"@worldwideview/wwv-plugin-wildfire": "workspace:*",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cesium": "^1.139.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"hls.js": "^1.6.15",
|
||||
"insecam-api": "^0.0.3",
|
||||
"jsdom": "^28.1.0",
|
||||
"jose": "^6.2.2",
|
||||
"lucide-react": "^0.576.0",
|
||||
"next": "16.1.6",
|
||||
"next": "16.2.1",
|
||||
"next-auth": "^5.0.0-beta.30",
|
||||
"prisma": "^7.5.0",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"react-player": "^3.4.0",
|
||||
"resium": "^1.19.4",
|
||||
"resium": "1.19.4",
|
||||
"satellite.js": "^7.0.0",
|
||||
"tslib": "^2.8.1",
|
||||
"ws": "^8.14.2",
|
||||
"zustand": "^5.0.11"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/dom": "^10.4.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/bcryptjs": "^3.0.0",
|
||||
"@types/cesium": "^1.70.4",
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"@types/ws": "^8.18.1",
|
||||
"@vitejs/plugin-react": "^5.1.4",
|
||||
"copy-webpack-plugin": "^14.0.0",
|
||||
"typescript": "^5"
|
||||
"jsdom": "^28.1.0",
|
||||
"typescript": "^5",
|
||||
"vitest": "^4.0.18"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"@prisma/engines",
|
||||
"better-sqlite3",
|
||||
"esbuild",
|
||||
"prisma",
|
||||
"protobufjs",
|
||||
"sharp",
|
||||
"unrs-resolver"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "@worldwideview/create-plugin",
|
||||
"version": "0.1.0",
|
||||
"description": "Scaffold a new WorldWideView plugin",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"create-wwv-plugin": "./dist/index.mjs"
|
||||
},
|
||||
"files": ["dist", "templates"],
|
||||
"scripts": {
|
||||
"build": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=dist/index.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"esbuild": "^0.25.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env node
|
||||
import { scaffold } from "./scaffold.js";
|
||||
|
||||
const name = process.argv[2];
|
||||
if (!name) {
|
||||
console.error("Usage: npx @worldwideview/create-plugin <plugin-name>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
scaffold(name);
|
||||
@@ -0,0 +1,45 @@
|
||||
import { mkdirSync, writeFileSync, readFileSync, readdirSync } from "fs";
|
||||
import { join, resolve, dirname } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function scaffold(name: string): void {
|
||||
const slug = name.toLowerCase().replace(/[^a-z0-9-]/g, "-");
|
||||
const dir = resolve(process.cwd(), slug);
|
||||
|
||||
console.log(`\n🌍 Creating WorldWideView plugin: ${slug}\n`);
|
||||
mkdirSync(join(dir, "src"), { recursive: true });
|
||||
|
||||
const templateDir = join(__dirname, "..", "templates");
|
||||
const files = readdirSync(templateDir);
|
||||
|
||||
for (const file of files) {
|
||||
let content = readFileSync(join(templateDir, file), "utf8");
|
||||
content = content.replace(/\{\{SLUG\}\}/g, slug);
|
||||
content = content.replace(
|
||||
/\{\{NAME\}\}/g,
|
||||
slug
|
||||
.split("-")
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(""),
|
||||
);
|
||||
|
||||
// Strip .tpl extension and route index.ts into src/
|
||||
const outName = file.replace(/\.tpl$/, "");
|
||||
const dest =
|
||||
outName === "index.ts"
|
||||
? join(dir, "src", outName)
|
||||
: join(dir, outName);
|
||||
|
||||
writeFileSync(dest, content);
|
||||
console.log(` ✓ ${outName === "index.ts" ? "src/" + outName : outName}`);
|
||||
}
|
||||
|
||||
console.log(`\n✅ Done! Next steps:`);
|
||||
console.log(` cd ${slug}`);
|
||||
console.log(` npm install`);
|
||||
console.log(` npm run dev # Watch mode`);
|
||||
console.log(` npm run build # Production bundle`);
|
||||
console.log(` npm publish # Publish to npm\n`);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* {{NAME}} — WorldWideView Plugin
|
||||
*
|
||||
* Build with: npm run build
|
||||
* Publish with: npm publish
|
||||
*/
|
||||
|
||||
import type {
|
||||
WorldPlugin,
|
||||
GeoEntity,
|
||||
TimeRange,
|
||||
PluginContext,
|
||||
LayerConfig,
|
||||
CesiumEntityOptions,
|
||||
PluginCategory,
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
export default class {{NAME}}Plugin implements WorldPlugin {
|
||||
readonly id = "{{SLUG}}";
|
||||
readonly name = "{{NAME}}";
|
||||
readonly description = "A custom WorldWideView plugin";
|
||||
readonly icon = "📍";
|
||||
readonly category: PluginCategory = "custom";
|
||||
readonly version = "1.0.0";
|
||||
|
||||
async initialize(_ctx: PluginContext): Promise<void> {
|
||||
console.log(`[{{NAME}}] Initialized`);
|
||||
}
|
||||
|
||||
destroy(): void {}
|
||||
|
||||
async fetch(_timeRange: TimeRange): Promise<GeoEntity[]> {
|
||||
// Replace with your data source
|
||||
return [];
|
||||
}
|
||||
|
||||
getPollingInterval(): number {
|
||||
return 60_000;
|
||||
}
|
||||
|
||||
getLayerConfig(): LayerConfig {
|
||||
return { color: "#3b82f6", clusterEnabled: true, clusterDistance: 40 };
|
||||
}
|
||||
|
||||
renderEntity(_entity: GeoEntity): CesiumEntityOptions {
|
||||
return { type: "point", color: "#3b82f6", size: 6 };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "wwv-plugin-{{SLUG}}",
|
||||
"version": "1.0.0",
|
||||
"description": "WorldWideView plugin: {{NAME}}",
|
||||
"type": "module",
|
||||
"module": "dist/index.mjs",
|
||||
"files": ["dist"],
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"dev": "vite build --watch"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "*",
|
||||
"vite": "^6.0.0",
|
||||
"rollup-plugin-external-globals": "^0.12.0",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": "^19.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "{{SLUG}}",
|
||||
"name": "{{NAME}}",
|
||||
"version": "1.0.0",
|
||||
"description": "A custom WorldWideView plugin",
|
||||
"type": "data-layer",
|
||||
"format": "bundle",
|
||||
"trust": "unverified",
|
||||
"capabilities": ["data:own"],
|
||||
"category": "custom",
|
||||
"icon": "📍",
|
||||
"entry": "dist/index.mjs"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"jsx": "react-jsx",
|
||||
"declaration": false,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { defineConfig } from "vite";
|
||||
import externalGlobals from "rollup-plugin-external-globals";
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
lib: {
|
||||
entry: "src/index.ts",
|
||||
formats: ["es"],
|
||||
fileName: "index",
|
||||
},
|
||||
rollupOptions: {
|
||||
external: ["react", "react-dom", "react/jsx-runtime"],
|
||||
plugins: [
|
||||
externalGlobals({
|
||||
"react": "globalThis.__WWV_HOST__.React",
|
||||
"react-dom": "globalThis.__WWV_HOST__.ReactDOM",
|
||||
"react/jsx-runtime": "globalThis.__WWV_HOST__.jsxRuntime",
|
||||
}),
|
||||
],
|
||||
},
|
||||
minify: true,
|
||||
sourcemap: false,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
# Stage 0: Extract package.json files for layer caching
|
||||
FROM alpine AS extractor
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN find . -type f \! -name 'package.json' \! -name 'pnpm-workspace.yaml' \! -name 'pnpm-lock.yaml' -delete && \
|
||||
find . -type d -empty -delete
|
||||
|
||||
# Build Stage
|
||||
FROM node:22-alpine AS builder
|
||||
|
||||
# better-sqlite3 requires python3 and build tools for native compilation
|
||||
RUN apk add --no-cache python3 make g++
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Enable corepack for pnpm
|
||||
RUN corepack enable
|
||||
|
||||
# Copy only the extracted package.jsons
|
||||
COPY --from=extractor /app ./
|
||||
|
||||
# Install dependencies with cache mount for pnpm store
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --filter @worldwideview/wwv-data-engine...
|
||||
|
||||
# Copy source and build
|
||||
COPY packages/wwv-data-engine ./packages/wwv-data-engine
|
||||
RUN pnpm --filter @worldwideview/wwv-data-engine build
|
||||
|
||||
# Production Stage
|
||||
FROM node:22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Enable corepack for pnpm
|
||||
RUN corepack enable
|
||||
|
||||
# Copy only the extracted package.jsons
|
||||
COPY --from=extractor /app ./
|
||||
|
||||
# Install ONLY production dependencies
|
||||
# Need python/make for building native modules during install
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
|
||||
apk add --no-cache python3 make g++ && \
|
||||
pnpm install --prod --filter @worldwideview/wwv-data-engine... && \
|
||||
apk del python3 make g++
|
||||
|
||||
# Copy built artifacts from builder
|
||||
COPY --from=builder /app/packages/wwv-data-engine/dist ./packages/wwv-data-engine/dist
|
||||
COPY --from=builder /app/packages/wwv-data-engine/seedData ./packages/wwv-data-engine/seedData
|
||||
|
||||
# Copy prisma schema and generate client for production
|
||||
COPY --from=builder /app/packages/wwv-data-engine/prisma ./packages/wwv-data-engine/prisma
|
||||
RUN cd packages/wwv-data-engine && npx prisma generate
|
||||
|
||||
WORKDIR /app/packages/wwv-data-engine
|
||||
|
||||
# Create directory for SQLite DB
|
||||
RUN mkdir -p /app/packages/wwv-data-engine/data
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=5001
|
||||
|
||||
EXPOSE 5001
|
||||
|
||||
CMD ["node", "dist/server.js"]
|
||||
@@ -0,0 +1,173 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import path from 'path';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, '../../.env.local') });
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
async function main() {
|
||||
console.log('[Migration] Starting migration from local engine.db to Supabase');
|
||||
|
||||
const dbPath = process.argv[2] || path.join(process.cwd(), 'data', 'engine.db');
|
||||
console.log(`[Migration] Reading from SQLite database at: ${dbPath}`);
|
||||
|
||||
const db = new Database(dbPath, { readonly: true });
|
||||
|
||||
// 1. Migrate IranwarEvent
|
||||
try {
|
||||
console.log('[Migration] Migrating iranwar_events...');
|
||||
const events = db.prepare('SELECT event_id, payload, timestamp, fetched_at FROM iranwar_events').all();
|
||||
console.log(`[Migration] Found ${events.length} iranwar_events. Iterating...`);
|
||||
let count = 0;
|
||||
for (const evt of events as any[]) {
|
||||
const parsedTs = typeof evt.timestamp === 'string' ? new Date(evt.timestamp).getTime() : (evt.timestamp || Date.now());
|
||||
await prisma.iranwarEvent.upsert({
|
||||
where: { eventId: evt.event_id },
|
||||
create: {
|
||||
eventId: evt.event_id,
|
||||
payload: evt.payload,
|
||||
timestamp: BigInt(parsedTs),
|
||||
fetchedAt: BigInt(evt.fetched_at || Date.now())
|
||||
},
|
||||
update: {
|
||||
payload: evt.payload,
|
||||
timestamp: BigInt(parsedTs),
|
||||
fetchedAt: BigInt(evt.fetched_at || Date.now())
|
||||
}
|
||||
});
|
||||
count++;
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated ${count} iranwar_events.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed iranwar_events: ${e.message}`);
|
||||
}
|
||||
|
||||
// 2. Migrate Earthquake
|
||||
try {
|
||||
console.log('[Migration] Migrating earthquakes...');
|
||||
const quakes = db.prepare('SELECT id, payload, source_ts, fetched_at FROM earthquakes').all();
|
||||
console.log(`[Migration] Found ${quakes.length} earthquakes. Migrating in bulk...`);
|
||||
const qData = quakes.map((q: any) => ({
|
||||
id: String(q.id),
|
||||
payload: q.payload,
|
||||
sourceTs: BigInt(q.source_ts || Date.now()),
|
||||
fetchedAt: BigInt(q.fetched_at || Date.now())
|
||||
}));
|
||||
if (qData.length > 0) {
|
||||
await prisma.earthquake.createMany({ data: qData, skipDuplicates: true });
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated earthquakes.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed earthquakes: ${e.message}`);
|
||||
}
|
||||
|
||||
// 3. Migrate Wildfire
|
||||
try {
|
||||
console.log('[Migration] Migrating wildfires...');
|
||||
const fires = db.prepare('SELECT id, payload, source_ts, fetched_at FROM wildfires').all();
|
||||
console.log(`[Migration] Found ${fires.length} wildfires. Migrating in bulk...`);
|
||||
const fData = fires.map((f: any) => ({
|
||||
id: String(f.id),
|
||||
payload: f.payload,
|
||||
sourceTs: BigInt(f.source_ts || Date.now()),
|
||||
fetchedAt: BigInt(f.fetched_at || Date.now())
|
||||
}));
|
||||
if (fData.length > 0) {
|
||||
await prisma.wildfire.createMany({ data: fData, skipDuplicates: true });
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated wildfires.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed wildfires: ${e.message}`);
|
||||
}
|
||||
|
||||
// 4. Migrate MaritimeHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating maritime_history...');
|
||||
const maritime = db.prepare('SELECT mmsi, ts, lat, lon, hdg, spd, fetched_at FROM maritime_history').all();
|
||||
console.log(`[Migration] Found ${maritime.length} maritime_history records. Migrating in bulk...`);
|
||||
|
||||
// Chunk it in case it's huge
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < maritime.length; i += chunkSize) {
|
||||
const chunk = maritime.slice(i, i + chunkSize).map((x: any) => ({
|
||||
mmsi: String(x.mmsi),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.maritimeHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${maritime.length} maritime_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated maritime_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed maritime_history: ${e.message}`);
|
||||
}
|
||||
|
||||
// 5. Migrate AviationHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating aviation_history...');
|
||||
const aviation = db.prepare('SELECT icao24, ts, lat, lon, alt, hdg, spd, fetched_at FROM aviation_history').all();
|
||||
console.log(`[Migration] Found ${aviation.length} aviation_history records. Migrating in bulk...`);
|
||||
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < aviation.length; i += chunkSize) {
|
||||
const chunk = aviation.slice(i, i + chunkSize).map((x: any) => ({
|
||||
icao24: String(x.icao24),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
alt: x.alt,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.aviationHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${aviation.length} aviation_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated aviation_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed aviation_history: ${e.message}`);
|
||||
}
|
||||
|
||||
// 6. Migrate MilitaryAviationHistory
|
||||
try {
|
||||
console.log('[Migration] Migrating military_aviation_history...');
|
||||
const maviation = db.prepare('SELECT hex, ts, lat, lon, alt, hdg, spd, fetched_at FROM military_aviation_history').all();
|
||||
console.log(`[Migration] Found ${maviation.length} military_aviation_history records. Migrating in bulk...`);
|
||||
|
||||
const chunkSize = 5000;
|
||||
for (let i = 0; i < maviation.length; i += chunkSize) {
|
||||
const chunk = maviation.slice(i, i + chunkSize).map((x: any) => ({
|
||||
hex: String(x.hex),
|
||||
ts: BigInt(x.ts),
|
||||
lat: x.lat,
|
||||
lon: x.lon,
|
||||
alt: x.alt,
|
||||
heading: x.hdg,
|
||||
speed: x.spd,
|
||||
fetchedAt: BigInt(x.fetched_at || Date.now())
|
||||
}));
|
||||
await prisma.militaryAviationHistory.createMany({ data: chunk as any, skipDuplicates: true });
|
||||
console.log(`[Migration] Migrated ${i + chunk.length}/${maviation.length} military_aviation_history...`);
|
||||
}
|
||||
console.log(`[Migration] Successfully migrated military_aviation_history.`);
|
||||
} catch (e: any) {
|
||||
console.error(`[Migration] Skipped or failed military_aviation_history: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log('[Migration] Completed migration from local SQLite to Supabase!');
|
||||
}
|
||||
|
||||
main()
|
||||
.catch(e => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
[Migration] Starting migration from local engine.db to Supabase
|
||||
[Migration] Reading from SQLite database at: C:\dev\worldwideview\packages\wwv-data-engine\data\engine.db
|
||||
[Migration] Migrating iranwar_events...
|
||||
[Migration] Found 242 iranwar_events. Iterating...
|
||||
[Migration] Skipped or failed iranwar_events:
|
||||
Invalid `prisma.iranwarEvent.upsert()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:31:39
|
||||
|
||||
28 console.log(`[Migration] Found ${events.length} iranwar_events. Iterating...`);
|
||||
29 let count = 0;
|
||||
30 for (const evt of events as any[]) {
|
||||
→ 31 await prisma.iranwarEvent.upsert({
|
||||
where: {
|
||||
eventId: "day-30-evt-0200z"
|
||||
},
|
||||
create: {
|
||||
eventId: "day-30-evt-0200z",
|
||||
[Migration] Migrating earthquakes...
|
||||
payload: "{\"event_id\":\"day-30-evt-0200z\",\"type\":\"AIR STRIKE\",\"location\":\"Tehran, Iran\",\"timestamp\":\"2026-03-28T02:00:00.000Z\",\"confidence\":\"Confirmed\",\"event_summary\":\"The combined US-Israeli force conducted precision strikes in Tehran targeting two facilities. The Marine Industries Organization ( ), which oversees 's naval weapons systems including missile-equipped fast boats and drone vessels critical to the blockade, sustained direct hits. A second strike targeted the Physics Department of 's University of Science and Technology ( ), a facility assessed by Western intelligence agencies as linked to nuclear and research programs. The described the strikes as targeting 's capacity to sustain offensive naval and ballistic operations. No casualty figures were released for either site.\",\"source_url\":\"https://www.msn.com/en-in/news/world/sea-of-mourners-gather-in-tehran-for-irgc-commander-alireza-tangsiri-s-funeral/ar-AA1ZVr9D\",\"preview_image\":null,\"_osint_meta\":{\"casualties\":0,\"coordinates\":{\"lat\":35.6892523,\"lng\":51.3896004}}}",
|
||||
timestamp: "2026-03-28T02:00:00.000Z",
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
fetchedAt: 1775088300955
|
||||
},
|
||||
update: {
|
||||
payload: "{\"event_id\":\"day-30-evt-0200z\",\"type\":\"AIR STRIKE\",\"location\":\"Tehran, Iran\",\"timestamp\":\"2026-03-28T02:00:00.000Z\",\"confidence\":\"Confirmed\",\"event_summary\":\"The combined US-Israeli force conducted precision strikes in Tehran targeting two facilities. The Marine Industries Organization ( ), which oversees 's naval weapons systems including missile-equipped fast boats and drone vessels critical to the blockade, sustained direct hits. A second strike targeted the Physics Department of 's University of Science and Technology ( ), a facility assessed by Western intelligence agencies as linked to nuclear and research programs. The described the strikes as targeting 's capacity to sustain offensive naval and ballistic operations. No casualty figures were released for either site.\",\"source_url\":\"https://www.msn.com/en-in/news/world/sea-of-mourners-gather-in-tehran-for-irgc-commander-alireza-tangsiri-s-funeral/ar-AA1ZVr9D\",\"preview_image\":null,\"_osint_meta\":{\"casualties\":0,\"coordinates\":{\"lat\":35.6892523,\"lng\":51.3896004}}}",
|
||||
timestamp: "2026-03-28T02:00:00.000Z",
|
||||
fetchedAt: 1775088300955
|
||||
}
|
||||
})
|
||||
|
||||
Argument `timestamp`: Invalid value provided. Expected Int, provided String.
|
||||
[Migration] Found 145 earthquakes. Migrating in bulk...
|
||||
[Migration] Skipped or failed earthquakes:
|
||||
Invalid `prisma.earthquake.createMany()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:64:37
|
||||
|
||||
61 fetchedAt: q.fetched_at
|
||||
62 }));
|
||||
63 if (qData.length > 0) {
|
||||
→ 64 await prisma.earthquake.createMany(
|
||||
Error occurred during query execution:
|
||||
ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(Error { kind: ToSql(2), cause: Some(Error { kind: ConversionError("Unable to fit integer value '1775077787921' into an INT4 (32-bit signed integer)."), original_code: None, original_message: None }) }), transient: false })
|
||||
[Migration] Migrating wildfires...
|
||||
[Migration] Found 64649 wildfires. Migrating in bulk...
|
||||
[Migration] Skipped or failed wildfires:
|
||||
Invalid `prisma.wildfire.createMany()` invocation in
|
||||
C:\dev\worldwideview\packages\wwv-data-engine\migrate-sqlite-to-supabase.ts:83:35
|
||||
|
||||
80 fetchedAt: f.fetched_at
|
||||
81 }));
|
||||
82 if (fData.length > 0) {
|
||||
→ 83 await prisma.wildfire.createMany(
|
||||
Error occurred during query execution:
|
||||
ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(Error { kind: ToSql(2), cause: Some(Error { kind: ConversionError("Unable to fit integer value '1774915260000' into an INT4 (32-bit signed integer)."), original_code: None, original_message: None }) }), transient: false })
|
||||
[Migration] Migrating maritime_history...
|
||||
[Migration] Skipped or failed maritime_history: no such column: heading
|
||||
[Migration] Migrating aviation_history...
|
||||
[Migration] Skipped or failed aviation_history: no such column: heading
|
||||
[Migration] Migrating military_aviation_history...
|
||||
[Migration] Completed migration from local SQLite to Supabase!
|
||||
[Migration] Skipped or failed military_aviation_history: no such column: heading
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-data-engine",
|
||||
"version": "1.2.1",
|
||||
"description": "WWV Data Engine - Dual-tier Shared Backend",
|
||||
"main": "dist/server.js",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"build": "prisma generate && tsc",
|
||||
"start": "node dist/server.js",
|
||||
"prisma:generate": "prisma generate",
|
||||
"prisma:push": "prisma db push"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.2.0",
|
||||
"@prisma/client": "^6.4.1",
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"csv-parse": "^5.6.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"fastify": "^5.0.0",
|
||||
"geoip-lite": "^2.0.1",
|
||||
"ioredis": "^5.4.1",
|
||||
"node-cron": "^3.0.3",
|
||||
"prisma": "^6.4.1",
|
||||
"sax": "^1.6.0",
|
||||
"ws": "^8.18.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.11",
|
||||
"@types/geoip-lite": "^1.4.4",
|
||||
"@types/node": "^22.0.0",
|
||||
"@types/node-cron": "^3.0.11",
|
||||
"@types/sax": "^1.2.7",
|
||||
"@types/ws": "^8.5.12",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// This is your Prisma schema file for the WWV Data Engine
|
||||
// It connects to Supabase to store historical timeline data.
|
||||
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("SUPABASE_DATABASE_URL")
|
||||
}
|
||||
|
||||
model IranwarEvent {
|
||||
eventId String @id @map("event_id")
|
||||
payload String // Stored as JSON string to mirror existing SQLite structure
|
||||
timestamp BigInt
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([timestamp])
|
||||
@@map("iranwar_events")
|
||||
}
|
||||
|
||||
model Earthquake {
|
||||
id String @id
|
||||
payload String // Stored as JSON string
|
||||
sourceTs BigInt @map("source_ts")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([sourceTs])
|
||||
@@map("earthquakes")
|
||||
}
|
||||
|
||||
model Wildfire {
|
||||
id String @id
|
||||
payload String // Stored as JSON string
|
||||
sourceTs BigInt @map("source_ts")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@index([sourceTs])
|
||||
@@map("wildfires")
|
||||
}
|
||||
|
||||
model MaritimeHistory {
|
||||
mmsi String
|
||||
ts BigInt
|
||||
lat Float
|
||||
lon Float
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([mmsi, ts])
|
||||
@@index([ts])
|
||||
@@map("maritime_history")
|
||||
}
|
||||
|
||||
model AviationHistory {
|
||||
icao24 String
|
||||
ts BigInt
|
||||
lat Float?
|
||||
lon Float?
|
||||
alt Float?
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([icao24, ts])
|
||||
@@index([ts])
|
||||
@@map("aviation_history")
|
||||
}
|
||||
|
||||
model MilitaryAviationHistory {
|
||||
hex String
|
||||
ts BigInt
|
||||
lat Float?
|
||||
lon Float?
|
||||
alt Float?
|
||||
heading Float? @map("hdg")
|
||||
speed Float? @map("spd")
|
||||
fetchedAt BigInt @map("fetched_at")
|
||||
|
||||
@@id([hex, ts])
|
||||
@@index([ts])
|
||||
@@map("military_aviation_history")
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
Loaded Prisma config from prisma.config.ts.
|
||||
|
||||
Prisma schema loaded from prisma\schema.prisma.
|
||||
Datasource "db": PostgreSQL database "postgres", schema "public" at "db.oqtfnwsefozpvsoysufm.supabase.co:5432"
|
||||
|
||||
Error: P1001: Can't reach database server at `db.oqtfnwsefozpvsoysufm.supabase.co:5432`
|
||||
|
||||
Please make sure your database server is running at `db.oqtfnwsefozpvsoysufm.supabase.co:5432`.
|
||||
@@ -0,0 +1,48 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const csvPath = 'C:/dev/worldwideview/artifacts/Events.csv';
|
||||
const lines = fs.readFileSync(csvPath, 'utf-8').trim().split('\n');
|
||||
|
||||
const events = [];
|
||||
const parseCSVLine = (text) => {
|
||||
let ret = [], q = false, cv = '';
|
||||
for (let c of text) {
|
||||
if (c === '\"') { q = !q; }
|
||||
else if (c === ',' && !q) { ret.push(cv); cv = ''; }
|
||||
else { cv += c; }
|
||||
}
|
||||
ret.push(cv);
|
||||
return ret;
|
||||
};
|
||||
|
||||
for(const line of lines) {
|
||||
const parts = parseCSVLine(line);
|
||||
if(parts.length < 10) continue;
|
||||
|
||||
const [time, date, type, location, casualties, sourceUrl, sourceTitle, summary, eventId, day, articleUrl] = parts;
|
||||
|
||||
// time is like '02:00Z', date is like '28-Mar-26'
|
||||
const isoDate = new Date(`${date} ${time.replace('Z', '')} UTC`).toISOString();
|
||||
|
||||
events.push({
|
||||
event_id: eventId,
|
||||
type: type,
|
||||
location: location,
|
||||
timestamp: isoDate,
|
||||
confidence: 'Confirmed',
|
||||
event_summary: summary,
|
||||
source_url: articleUrl,
|
||||
preview_image: null,
|
||||
_osint_meta: {
|
||||
casualties: parseInt(casualties) || 0,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const outDir = 'C:/dev/worldwideview/packages/wwv-data-engine/seedData';
|
||||
if (!fs.existsSync(outDir)) {
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(path.join(outDir, 'iranwar_seed.json'), JSON.stringify(events, null, 2));
|
||||
console.log('Saved ' + events.length + ' events to seed.json');
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,207 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
|
||||
// Define the path for the SQLite database. Note Docker volume mounts to /app/packages/wwv-data-engine/data
|
||||
const dbPath = process.env.DB_PATH || path.join(__dirname, '..', 'data', 'engine.db');
|
||||
|
||||
export const db = new Database(dbPath, {
|
||||
// Use verbose logging if needed for debugging
|
||||
// verbose: console.log
|
||||
});
|
||||
|
||||
// Enable Write-Ahead Logging for better concurrent read/write performance
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
db.pragma('busy_timeout = 5000'); // Wait up to 5 seconds if DB is locked
|
||||
|
||||
/**
|
||||
* Initialize all required tables for the different seeders.
|
||||
* This runs synchronously on boot.
|
||||
*/
|
||||
export function initDB() {
|
||||
console.log(`[DB] Initializing SQLite database at ${dbPath}`);
|
||||
|
||||
// IranWarLive table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS iranwar_events (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Earthquakes table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS earthquakes (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Wildfires table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS wildfires (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Maritime AIS history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS maritime_history (
|
||||
mmsi TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (mmsi, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast maritime history lookups by MMSI + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_maritime_history_mmsi_ts ON maritime_history(mmsi, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_maritime_history_ts ON maritime_history(ts);
|
||||
`);
|
||||
|
||||
// Aviation history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS aviation_history (
|
||||
icao24 TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
alt REAL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (icao24, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast aviation history lookups by ICAO24 + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_aviation_history_icao24_ts ON aviation_history(icao24, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_aviation_history_ts ON aviation_history(ts);
|
||||
`);
|
||||
|
||||
// Military Aviation history table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS military_aviation_history (
|
||||
hex TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
alt REAL,
|
||||
hdg REAL,
|
||||
spd REAL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (hex, ts)
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for fast military aviation history lookups by hex + time range
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_military_aviation_history_hex_ts ON military_aviation_history(hex, ts);
|
||||
`);
|
||||
|
||||
// Index for time-range-only queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_military_aviation_history_ts ON military_aviation_history(ts);
|
||||
`);
|
||||
|
||||
// GPS Jamming daily interference map
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS gps_jamming (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Conflict Events (ACLED) DB table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS conflict_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Civil Unrest DB table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS civil_unrest (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Cyber Attacks table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS cyber_attacks (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
// Index for cyber attacks time-range queries (playback mode)
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_cyber_attacks_source_ts ON cyber_attacks(source_ts);
|
||||
`);
|
||||
|
||||
// Sanctions table
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS sanctions (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload JSON NOT NULL,
|
||||
source_ts INTEGER NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
console.log('[DB] All tables initialized successfully.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune history rows older than the retention period.
|
||||
* Runs periodically to prevent unbounded table growth.
|
||||
*/
|
||||
const RETENTION_HOURS = 24;
|
||||
|
||||
export function pruneHistoryTables() {
|
||||
const cutoff = Math.floor(Date.now() / 1000) - RETENTION_HOURS * 3600;
|
||||
|
||||
const tables = [
|
||||
'aviation_history',
|
||||
'military_aviation_history',
|
||||
'maritime_history',
|
||||
];
|
||||
|
||||
for (const table of tables) {
|
||||
const result = db.prepare(`DELETE FROM ${table} WHERE ts < ?`).run(cutoff);
|
||||
if (result.changes > 0) {
|
||||
console.log(`[DB] Pruned ${result.changes} rows from ${table}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import geoip from 'geoip-lite';
|
||||
|
||||
export interface GeoLocation {
|
||||
lat: number;
|
||||
lon: number;
|
||||
country: string;
|
||||
city: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Geolocate an IPv4 address using the local geoip-lite database.
|
||||
* Returns null for private/unresolvable IPs.
|
||||
*/
|
||||
export function geolocateIp(ip: string): GeoLocation | null {
|
||||
const lookup = geoip.lookup(ip);
|
||||
if (!lookup || !lookup.ll) return null;
|
||||
|
||||
return {
|
||||
lat: lookup.ll[0],
|
||||
lon: lookup.ll[1],
|
||||
country: lookup.country || 'Unknown',
|
||||
city: lookup.city || 'Unknown',
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var prismaDataEngine: PrismaClient | undefined;
|
||||
}
|
||||
|
||||
export const prisma =
|
||||
globalThis.prismaDataEngine ||
|
||||
new PrismaClient();
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') globalThis.prismaDataEngine = prisma;
|
||||
@@ -0,0 +1,96 @@
|
||||
import Redis from 'ioredis';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import zlib from 'zlib';
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env.local') });
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env') });
|
||||
|
||||
// ioredis gracefully handles standard redis:// as well as rediss:// (TLS, used by Upstash)
|
||||
const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
|
||||
|
||||
if (redisUrl.includes('upstash.io') && redisUrl.startsWith('redis://')) {
|
||||
console.error('\n\x1b[31m[CRITICAL CONFIG ERROR]\x1b[0m 🚨 You are connecting to Upstash without TLS.');
|
||||
console.error('Upstash requires the rediss:// protocol. Please update your REDIS_URL in .env.local to start with rediss:// instead of redis://\n');
|
||||
}
|
||||
|
||||
console.log(`[Redis] Connecting to ${redisUrl.replace(/:[^:@]+@/, ':***@')} ...`);
|
||||
|
||||
export const redis = new Redis(redisUrl, {
|
||||
// Common reconnect strategy
|
||||
retryStrategy(times) {
|
||||
const delay = Math.min(times * 50, 2000);
|
||||
return delay;
|
||||
},
|
||||
maxRetriesPerRequest: 3,
|
||||
});
|
||||
|
||||
redis.on('error', (err) => {
|
||||
console.error('[Redis] Connection Error against URL:', redisUrl.replace(/:[^:@]+@/, ':***@'));
|
||||
console.error('[Redis] Error Object:', err);
|
||||
});
|
||||
|
||||
redis.on('ready', () => {
|
||||
console.log('[Redis] Connected and ready.');
|
||||
});
|
||||
|
||||
import { broadcastPluginData } from './websocket';
|
||||
|
||||
const lastSnapshotTimes = new Map<string, number>();
|
||||
const SNAPSHOT_THROTTLE_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
/**
|
||||
* Convenience method to write a JSON payload to Redis with an expiration.
|
||||
* Writes are throttled to save Redis requests, but websockets are always broadcasted.
|
||||
*/
|
||||
export async function setLiveSnapshot(source: string, payload: any, ttlSeconds: number) {
|
||||
try {
|
||||
// 1. ALWAYS broadcast newly updated entities to any active WebSocket subscribers
|
||||
// The data pipeline relies on this for high-frequency HUD updates.
|
||||
broadcastPluginData(source, payload);
|
||||
|
||||
// 2. Throttle Redis snapshots to prevent blowing past 500K max request limits (e.g. Upstash)
|
||||
const now = Date.now();
|
||||
const lastTime = lastSnapshotTimes.get(source) || 0;
|
||||
if (now - lastTime < SNAPSHOT_THROTTLE_MS) {
|
||||
return; // Skip Redis write
|
||||
}
|
||||
lastSnapshotTimes.set(source, now);
|
||||
|
||||
const key = `data:${source}:live`;
|
||||
const jsonStr = JSON.stringify(payload);
|
||||
|
||||
// Compress JSON strings to drastically reduce Upstash payload size
|
||||
const compressed = zlib.gzipSync(Buffer.from(jsonStr, 'utf-8'));
|
||||
await redis.set(key, compressed as any, 'EX', ttlSeconds);
|
||||
|
||||
// Also save a metadata key so healthcheck can verify
|
||||
await redis.set(`meta:${source}:last_run`, Date.now().toString(), 'EX', ttlSeconds * 2);
|
||||
|
||||
console.log(`[Redis] Snapshot saved to Redis for ${source} (${(compressed.length / 1024).toFixed(2)} KB)`);
|
||||
} catch (error) {
|
||||
console.error(`[Redis] Failed to snapshot ${source}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method to read a JSON payload from Redis.
|
||||
*/
|
||||
export async function getLiveSnapshot(source: string) {
|
||||
const key = `data:${source}:live`;
|
||||
try {
|
||||
// We must use getBuffer natively in ioredis to prevent it from coercing raw bytes to a utf8 string
|
||||
const data = await redis.getBuffer(key);
|
||||
if (!data) return null;
|
||||
|
||||
try {
|
||||
const decompressed = zlib.unzipSync(data);
|
||||
return JSON.parse(decompressed.toString('utf-8'));
|
||||
} catch {
|
||||
// Fallback if data was stored as plain string before compression was introduced
|
||||
return JSON.parse(data.toString('utf-8'));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Redis] Failed to get live snapshot ${source}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/aviation', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT icao24, ts, lat, lon, hdg, spd, alt
|
||||
FROM aviation_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.icao24);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.icao24, { ...r, on_ground: r.alt <= 0 });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "15m" -> 900s, "1h" -> 3600s)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('aviation')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(plane) => nowTs - plane.last_updated <= maxAge
|
||||
);
|
||||
|
||||
return {
|
||||
source: 'aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[Aviation Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM civil_unrest WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
fastify.get('/data/civil_unrest', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('civil_unrest');
|
||||
if (liveData) {
|
||||
return { data: liveData };
|
||||
}
|
||||
return { data: [] };
|
||||
});
|
||||
|
||||
fastify.get('/data/civil_unrest/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "civil_unrest",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM conflict_events WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
fastify.get('/data/conflict_events', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('conflict_events');
|
||||
if (liveData) {
|
||||
return { data: liveData };
|
||||
}
|
||||
return { data: [] };
|
||||
});
|
||||
|
||||
fastify.get('/data/conflict_events/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "conflict_events",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare(
|
||||
'SELECT payload FROM cyber_attacks WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC'
|
||||
);
|
||||
|
||||
// Standard live endpoint → returns Redis snapshot
|
||||
fastify.get('/data/cyber_attacks', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('cyber_attacks');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: 'cyber_attacks',
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
};
|
||||
});
|
||||
|
||||
// History endpoint → queries SQLite
|
||||
fastify.get('/data/cyber_attacks/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply
|
||||
.status(400)
|
||||
.send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({
|
||||
start: parseInt(start, 10),
|
||||
end: parseInt(end, 10),
|
||||
}) as { payload: string }[];
|
||||
const historyItems = rows.map((row) => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: 'cyber_attacks',
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length,
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM earthquakes WHERE source_ts >= @start AND source_ts <= @end ORDER BY source_ts DESC');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/earthquakes', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('earthquakes');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "earthquakes",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/earthquakes/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: "earthquakes",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM gps_jamming WHERE source_ts >= @start AND source_ts <= @end');
|
||||
|
||||
fastify.get('/data/gps_jamming', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('gps_jamming');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "gps_jamming",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
fastify.get('/data/gps_jamming/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "gps_jamming",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import './iranwarlive';
|
||||
import './earthquakes';
|
||||
import './wildfires';
|
||||
import './maritime';
|
||||
import './aviation';
|
||||
import './military-aviation';
|
||||
import './satellite';
|
||||
import './gpsjam';
|
||||
import './conflictEvents';
|
||||
import './civilUnrest';
|
||||
import './cyberAttacks';
|
||||
import './sanctions';
|
||||
import './surveillanceSat';
|
||||
@@ -0,0 +1,37 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getAllEventsQuery = db.prepare('SELECT payload FROM iranwar_events ORDER BY timestamp DESC');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/iranwarlive', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('iranwarlive');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
|
||||
// Fallback to SQLite if Redis is empty/cleared
|
||||
const rows = getAllEventsQuery.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
return {
|
||||
source: "iranwarlive",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: history
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/iranwarlive/history', async (request, reply) => {
|
||||
// Iran war live has small enough history that we can return it all.
|
||||
// In a massive set, we would use query params: ?start=ts&end=ts
|
||||
const rows = getAllEventsQuery.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
version: "2.0",
|
||||
source: "iranwarlive",
|
||||
last_updated: new Date().toISOString(),
|
||||
items: history
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/maritime', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT mmsi, ts, lat, lon, hdg, spd
|
||||
FROM maritime_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.mmsi);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.mmsi, r);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'maritime',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "6h" -> seconds)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('maritime')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(ship) => nowTs - ship.last_updated <= maxAge
|
||||
);
|
||||
|
||||
// If lookback was requested, grab history points from SQLite
|
||||
if (lookbackSeconds > 0 && items.length > 0) {
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT mmsi, ts, lat, lon
|
||||
FROM maritime_history
|
||||
WHERE ts >= @startTs
|
||||
`);
|
||||
const allHistory = historyQuery.all({ startTs: nowTs - lookbackSeconds }) as any[];
|
||||
|
||||
const historyMap = new Map<string, any[]>();
|
||||
for (const row of allHistory) {
|
||||
if (!historyMap.has(row.mmsi)) historyMap.set(row.mmsi, []);
|
||||
historyMap.get(row.mmsi)!.push({ lat: row.lat, lon: row.lon, ts: row.ts });
|
||||
}
|
||||
|
||||
for (const ship of items) {
|
||||
const pts = historyMap.get(ship.mmsi);
|
||||
if (pts && pts.length > 0) {
|
||||
// Sort chronologically ascending
|
||||
pts.sort((a, b) => a.ts - b.ts);
|
||||
ship.history = pts;
|
||||
} else {
|
||||
ship.history = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'maritime',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[Maritime Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
fastify.get('/data/military-aviation', async (request: any, reply) => {
|
||||
try {
|
||||
const { lookback, time } = request.query;
|
||||
|
||||
// Historical playback mode snapshot
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTs = Math.floor(parseInt(time, 10) / 1000);
|
||||
const historyQuery = db.prepare(`
|
||||
SELECT hex, ts, lat, lon, hdg, spd, alt
|
||||
FROM military_aviation_history
|
||||
WHERE ts BETWEEN @start AND @end
|
||||
`);
|
||||
const historyRows = historyQuery.all({
|
||||
start: targetTs - 60,
|
||||
end: targetTs + 60,
|
||||
}) as any[];
|
||||
|
||||
const closest = new Map();
|
||||
for (const r of historyRows) {
|
||||
const prev = closest.get(r.hex);
|
||||
if (!prev || Math.abs(r.ts - targetTs) < Math.abs(prev.ts - targetTs)) {
|
||||
closest.set(r.hex, { ...r, on_ground: r.alt <= 0 });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'military-aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
ac: Array.from(closest.values()),
|
||||
totalCount: closest.size,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse lookback (e.g., "15m" -> 900s, "1h" -> 3600s)
|
||||
let lookbackSeconds = 0;
|
||||
if (lookback && typeof lookback === 'string') {
|
||||
const match = lookback.match(/^(\d+)([hm])$/);
|
||||
if (match) {
|
||||
const val = parseInt(match[1], 10);
|
||||
lookbackSeconds = match[2] === 'h' ? val * 3600 : val * 60;
|
||||
}
|
||||
}
|
||||
|
||||
// Get live fleet from Redis and filter by freshness
|
||||
const fleetObj = (await getLiveSnapshot('military-aviation')) || {};
|
||||
const activeFleet = Object.values(fleetObj) as any[];
|
||||
const nowTs = Math.floor(Date.now() / 1000);
|
||||
const maxAge = lookbackSeconds > 0 ? lookbackSeconds : 3600;
|
||||
const items = activeFleet.filter(
|
||||
(plane) => nowTs - plane.last_updated <= maxAge
|
||||
);
|
||||
|
||||
return {
|
||||
source: 'military-aviation',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: lookbackSeconds || null,
|
||||
ac: items,
|
||||
totalCount: items.length,
|
||||
};
|
||||
} catch (err: any) {
|
||||
console.error('[MilitaryAviation Route] ERROR:', err?.message || err);
|
||||
reply.status(500).send({ error: 'Internal error', message: err?.message });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
// Sanctions are Redis-only (no SQLite table) — live data only
|
||||
fastify.get('/data/sanctions', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('sanctions');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
id: 'sanctions-live',
|
||||
timestamp: null,
|
||||
items: []
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
import { globalsTLECache, propagateAll, DEFAULT_GROUPS } from '../seeders/satellite';
|
||||
|
||||
fastify.get('/data/satellite', async (request: any, reply) => {
|
||||
const { lookback, time, group } = request.query;
|
||||
|
||||
const targetGroups = group ? [group as string] : DEFAULT_GROUPS;
|
||||
|
||||
// Historical playback mode snapshot using TLE on the fly
|
||||
if (time && typeof time === 'string') {
|
||||
const targetTime = new Date(parseInt(time, 10));
|
||||
|
||||
let items: any[] = [];
|
||||
const seen = new Set<number>();
|
||||
|
||||
for (const g of targetGroups) {
|
||||
const records = globalsTLECache.get(g);
|
||||
if (!records) continue;
|
||||
|
||||
const positions = propagateAll(records, targetTime, g);
|
||||
for (const p of positions) {
|
||||
if (!seen.has(p.noradId)) {
|
||||
items.push(p);
|
||||
seen.add(p.noradId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: "satellite",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: 0,
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
};
|
||||
}
|
||||
|
||||
// 1. Get hot fleet from Redis for current time
|
||||
const fleetObj = await getLiveSnapshot('satellite') || {};
|
||||
let items = Object.values(fleetObj) as any[];
|
||||
|
||||
if (group) {
|
||||
items = items.filter(s => s.group === group);
|
||||
}
|
||||
|
||||
return {
|
||||
source: "satellite",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
lookbackSeconds: null,
|
||||
items,
|
||||
totalCount: items.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import { fastify } from '../server';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
// Surveillance Satellites split route
|
||||
fastify.get('/data/surveillance_satellites', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('surveillance_satellites');
|
||||
if (liveData) {
|
||||
return { satellites: Object.values(liveData) };
|
||||
}
|
||||
return { satellites: [] };
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { fastify } from '../server';
|
||||
import { db } from '../db';
|
||||
import { getLiveSnapshot } from '../redis';
|
||||
|
||||
const getHistoryQuery = db.prepare('SELECT payload FROM wildfires WHERE source_ts >= @start AND source_ts <= @end');
|
||||
|
||||
// Standard live endpoint -> returns Redis snapshot
|
||||
fastify.get('/data/wildfires', async (request, reply) => {
|
||||
const liveData = await getLiveSnapshot('wildfires');
|
||||
if (liveData) {
|
||||
return liveData;
|
||||
}
|
||||
return {
|
||||
source: "wildfires",
|
||||
fetchedAt: null,
|
||||
items: [],
|
||||
totalCount: 0
|
||||
};
|
||||
});
|
||||
|
||||
// Full history endpoint -> queries SQLite
|
||||
fastify.get('/data/wildfires/history', async (request: any, reply) => {
|
||||
const { start, end } = request.query;
|
||||
|
||||
if (!start || !end) {
|
||||
return reply.status(400).send({ error: 'Missing start or end query params (Unix timestamps in ms)' });
|
||||
}
|
||||
|
||||
const rows = getHistoryQuery.all({ start: parseInt(start, 10), end: parseInt(end, 10) }) as { payload: string }[];
|
||||
const historyItems = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
return {
|
||||
source: "wildfires",
|
||||
timeRange: { start: parseInt(start, 10), end: parseInt(end, 10) },
|
||||
items: historyItems,
|
||||
totalCount: historyItems.length
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import cron from 'node-cron';
|
||||
import { pruneHistoryTables } from './db';
|
||||
|
||||
// Define the interface for a seeder definition
|
||||
export interface SeederDefinition {
|
||||
name: string;
|
||||
cron?: string; // Optional: Cron expression if it runs on a schedule
|
||||
fn?: () => Promise<void>; // Optional: Function to run on the cron schedule
|
||||
init?: () => void; // Optional: Initialization function (e.g., for websockets)
|
||||
}
|
||||
|
||||
// Registry to hold all registered seeders
|
||||
let registeredSeeders: SeederDefinition[] = [];
|
||||
|
||||
// Registry to track the last run time of each cron seeder
|
||||
export const seederStatus: Record<string, number | null> = {};
|
||||
|
||||
/**
|
||||
* Register a seeder to be scheduled when the engine boots.
|
||||
*/
|
||||
export function registerSeeder(seeder: SeederDefinition) {
|
||||
registeredSeeders.push(seeder);
|
||||
if (seeder.cron) {
|
||||
seederStatus[seeder.name] = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler. Initializes websocket listeners and registers cron jobs.
|
||||
*/
|
||||
export function startScheduler() {
|
||||
console.log('[Scheduler] Starting data engine scheduler...');
|
||||
|
||||
for (const seeder of registeredSeeders) {
|
||||
// 1. Run init handlers (like websocket listeners)
|
||||
if (seeder.init) {
|
||||
console.log(`[Scheduler] Initializing persistent seeder: ${seeder.name}`);
|
||||
seeder.init();
|
||||
}
|
||||
|
||||
// 2. Schedule cron jobs
|
||||
if (seeder.cron && seeder.fn) {
|
||||
console.log(`[Scheduler] Scheduling cron seeder: ${seeder.name} (${seeder.cron})`);
|
||||
|
||||
cron.schedule(seeder.cron, async () => {
|
||||
try {
|
||||
console.log(`[Scheduler] Running seeder: ${seeder.name} ...`);
|
||||
await seeder.fn!();
|
||||
seederStatus[seeder.name] = Date.now();
|
||||
} catch (error: any) {
|
||||
console.error(`[Scheduler] Seeder ${seeder.name} failed:`, error.message);
|
||||
}
|
||||
});
|
||||
|
||||
// Kick off the first run immediately for cron jobs
|
||||
console.log(`[Scheduler] Kickstarting initial run for ${seeder.name}...`);
|
||||
seeder.fn().then(() => {
|
||||
seederStatus[seeder.name] = Date.now();
|
||||
}).catch((err) => {
|
||||
console.error(`[Scheduler] Initial run for ${seeder.name} failed:`, err.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Data retention: prune old history rows every hour
|
||||
cron.schedule('0 * * * *', () => {
|
||||
console.log('[Scheduler] Running data retention pruning...');
|
||||
pruneHistoryTables();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
const seedPath = path.join(__dirname, '..', '..', 'data', 'fallback', 'iranwar_seed.json');
|
||||
|
||||
async function delay(ms: number) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function extractMedia(url: string): Promise<{img: string | null, vid: string | null, realUrl: string | null}> {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5"
|
||||
},
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
|
||||
const text = await res.text();
|
||||
const $ = cheerio.load(text);
|
||||
|
||||
let realUrl = res.url;
|
||||
|
||||
// Sometimes Bing wraps in a redirect page
|
||||
if ($('noscript').text().includes('refresh')) {
|
||||
const redirectMatch = $('noscript').html()?.match(/URL=([^"']+)/i);
|
||||
if (redirectMatch && redirectMatch[1]) {
|
||||
const redir = redirectMatch[1].replace(/&/g, '&');
|
||||
try {
|
||||
const redirRes = await fetch(redir, { headers: { "User-Agent": "Mozilla/5.0" }, signal: AbortSignal.timeout(10000) });
|
||||
const redirHtml = await redirRes.text();
|
||||
const $2 = cheerio.load(redirHtml);
|
||||
realUrl = redirRes.url;
|
||||
return { ...parseMetas($2, realUrl), realUrl };
|
||||
} catch(e) {}
|
||||
}
|
||||
}
|
||||
|
||||
return { ...parseMetas($, realUrl), realUrl };
|
||||
} catch {
|
||||
return { img: null, vid: null, realUrl: null };
|
||||
}
|
||||
}
|
||||
|
||||
function parseMetas($: cheerio.CheerioAPI, source: string) {
|
||||
const img = $('meta[property="og:image"]').attr('content') || $('meta[name="twitter:image"]').attr('content') || null;
|
||||
const vid = $('meta[property="og:video:url"]').attr('content') || $('meta[property="og:video"]').attr('content') || null;
|
||||
return { img, vid };
|
||||
}
|
||||
|
||||
async function run() {
|
||||
console.log("Loading existing seed...");
|
||||
if (!fs.existsSync(seedPath)) {
|
||||
console.error("Seed not found!");
|
||||
return;
|
||||
}
|
||||
|
||||
const events: any[] = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
||||
let count = 0;
|
||||
|
||||
for (const event of events) {
|
||||
const isSearchable = !["POSTURING", "INTERCEPTION", "AIRSPACE CLOSED"].includes(event.type?.toUpperCase() || "");
|
||||
|
||||
if (isSearchable && event.location && event.type) {
|
||||
console.log(`[Hydrate] Searching Bing News for: ${event.type} ${event.location}`);
|
||||
const query = encodeURIComponent(`${event.type} ${event.location} Iran Israel conflict`);
|
||||
try {
|
||||
const searchRes = await fetch(`https://www.bing.com/news/search?q=${query}&format=rss`, {
|
||||
headers: { "User-Agent": "Mozilla/5.0" },
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (searchRes.ok) {
|
||||
const xml = await searchRes.text();
|
||||
const $ = cheerio.load(xml, { xmlMode: true });
|
||||
const items = $('item');
|
||||
if (items.length > 0) {
|
||||
const articleUrl = $(items[0]).find('link').text();
|
||||
console.log(` -> Found article string: ${articleUrl.substring(0, 50)}...`);
|
||||
const media = await extractMedia(articleUrl);
|
||||
|
||||
event.source_url = media.realUrl || articleUrl;
|
||||
if (media.vid) event.preview_video = media.vid;
|
||||
if (media.img) event.preview_image = media.img;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(` -> Search failed: ${err.message}`);
|
||||
}
|
||||
fs.writeFileSync(seedPath, JSON.stringify(events, null, 2));
|
||||
await delay(500);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Extraction complete. Modified ${count} events.`);
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
@@ -0,0 +1,55 @@
|
||||
export async function withRetry<T>(fn: () => Promise<T>, maxRetries = 3, delayMs = 1000): Promise<T> {
|
||||
let lastErr: Error | undefined;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err: any) {
|
||||
lastErr = err;
|
||||
if (attempt < maxRetries) {
|
||||
const wait = delayMs * Math.pow(2, attempt); // Exponential backoff
|
||||
console.warn(`[Retry] Attempt ${attempt + 1}/${maxRetries} failed: ${err.message}. Waiting ${wait}ms...`);
|
||||
await new Promise(r => setTimeout(r, wait));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
export async function fetchWithTimeout(url: string, options: RequestInit = {}, timeoutMs = 15000): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const id = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} - ${response.statusText}`);
|
||||
}
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates distance in Kilometers between two coordinates.
|
||||
* Useful for filtering events by proximity (e.g., nuclear test sites).
|
||||
*/
|
||||
export function haversineKm(lat1: number, lon1: number, lat2: number, lon2: number): number {
|
||||
const R = 6371; // km
|
||||
const dLat = (lat2 - lat1) * Math.PI / 180;
|
||||
const dLon = (lon2 - lon1) * Math.PI / 180;
|
||||
const a =
|
||||
Math.sin(dLat/2) * Math.sin(dLat/2) +
|
||||
Math.cos(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) *
|
||||
Math.sin(dLon/2) * Math.sin(dLon/2);
|
||||
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
|
||||
return R * c;
|
||||
}
|
||||
|
||||
export const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36';
|
||||
|
||||
export function sleep(ms: number) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
import cron from 'node-cron';
|
||||
import { db } from '../db';
|
||||
import { redis, setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const OPENSKY_TOKEN_URL = "https://auth.opensky-network.org/auth/realms/opensky-network/protocol/openid-connect/token";
|
||||
const OPENSKY_DATA_URL = "https://opensky-network.org/api/states/all";
|
||||
const ROTATION_THRESHOLD = 50;
|
||||
const POLLING_INTERVAL_MS = 5000;
|
||||
const FLUSH_INTERVAL_MS = 5000;
|
||||
|
||||
let messageBuffer: any[] = [];
|
||||
|
||||
interface OpenSkyCredential {
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
accessToken: string | null;
|
||||
tokenExpiry: number;
|
||||
creditsRemaining: number | null;
|
||||
exhausted: boolean;
|
||||
}
|
||||
|
||||
const pool = {
|
||||
_openskyPool: [] as OpenSkyCredential[],
|
||||
_openskyActiveIdx: 0,
|
||||
_lastResetTime: Date.now()
|
||||
};
|
||||
|
||||
function initCredentialPool(): void {
|
||||
if (pool._openskyPool.length > 0) return;
|
||||
|
||||
const creds: OpenSkyCredential[] = [];
|
||||
const raw = process.env.OPENSKY_CREDENTIALS;
|
||||
|
||||
if (raw) {
|
||||
for (const pair of raw.split(",")) {
|
||||
const trimmed = pair.trim();
|
||||
const colonIdx = trimmed.indexOf(":");
|
||||
if (colonIdx === -1) continue;
|
||||
const clientId = trimmed.slice(0, colonIdx);
|
||||
const clientSecret = trimmed.slice(colonIdx + 1);
|
||||
if (clientId && clientSecret) {
|
||||
creds.push({ clientId, clientSecret, accessToken: null, tokenExpiry: 0, creditsRemaining: null, exhausted: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (creds.length === 0) {
|
||||
// Fallback
|
||||
const clientId = process.env.OPENSKY_CLIENTID;
|
||||
const clientSecret = process.env.OPENSKY_CLIENTSECRET;
|
||||
if (clientId && clientSecret) {
|
||||
creds.push({ clientId, clientSecret, accessToken: null, tokenExpiry: 0, creditsRemaining: null, exhausted: false });
|
||||
}
|
||||
}
|
||||
|
||||
pool._openskyPool = creds;
|
||||
pool._openskyActiveIdx = 0;
|
||||
pool._lastResetTime = Date.now();
|
||||
console.log(`[Aviation] Initialised pool with ${creds.length} credential(s).`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets all credentials in the pool to non-exhausted state.
|
||||
*/
|
||||
function resetCredentialPool(): void {
|
||||
console.log('[Aviation] Resetting credential pool exhaustion flags...');
|
||||
for (const cred of pool._openskyPool) {
|
||||
cred.exhausted = false;
|
||||
cred.creditsRemaining = null;
|
||||
}
|
||||
pool._openskyActiveIdx = 0;
|
||||
pool._lastResetTime = Date.now();
|
||||
}
|
||||
|
||||
function getActiveCredential(): OpenSkyCredential | null {
|
||||
const creds = pool._openskyPool;
|
||||
if (!creds || creds.length === 0) return null;
|
||||
|
||||
const current = creds[pool._openskyActiveIdx];
|
||||
if (current && !current.exhausted) return current;
|
||||
|
||||
for (let i = 0; i < creds.length; i++) {
|
||||
if (!creds[i].exhausted) {
|
||||
pool._openskyActiveIdx = i;
|
||||
console.log(`[Aviation] Rotated → now using credential ${i + 1}/${creds.length}: ${creds[i].clientId}`);
|
||||
return creds[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Emergency Reset: If all are exhausted, but it's been > 12 hours since last reset,
|
||||
// try resetting everything once before giving up.
|
||||
const twelveHoursMs = 12 * 60 * 60 * 1000;
|
||||
if (Date.now() - pool._lastResetTime > twelveHoursMs) {
|
||||
console.warn(`[Aviation] Emergency Reset: All credentials exhausted, but last reset was > 12h ago. Attempting pool reset.`);
|
||||
resetCredentialPool();
|
||||
return getActiveCredential();
|
||||
}
|
||||
|
||||
console.warn(`[Aviation] All ${creds.length} credentials exhausted.`);
|
||||
return null;
|
||||
}
|
||||
|
||||
function rotateCredential(): void {
|
||||
const current = pool._openskyPool[pool._openskyActiveIdx];
|
||||
if (current) current.exhausted = true;
|
||||
getActiveCredential();
|
||||
}
|
||||
|
||||
async function fetchTokenForCredential(cred: OpenSkyCredential): Promise<string | null> {
|
||||
const now = Date.now();
|
||||
if (cred.accessToken && now < cred.tokenExpiry) return cred.accessToken;
|
||||
|
||||
try {
|
||||
const res = await fetch(OPENSKY_TOKEN_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({
|
||||
grant_type: "client_credentials",
|
||||
client_id: cred.clientId,
|
||||
client_secret: cred.clientSecret,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) return null;
|
||||
|
||||
const data = await res.json();
|
||||
cred.accessToken = data.access_token;
|
||||
cred.tokenExpiry = now + data.expires_in * 1000 - 30_000;
|
||||
return cred.accessToken;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Poller ──────────────────────────────────────────────────
|
||||
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO aviation_history (icao24, ts, lat, lon, alt, hdg, spd, fetched_at)
|
||||
VALUES (@icao24, @ts, @lat, @lon, @alt, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function pollOpenSky() {
|
||||
const pollStart = Date.now();
|
||||
console.log(`[Aviation] Poll starting...`);
|
||||
const cred = getActiveCredential();
|
||||
let token = null;
|
||||
|
||||
if (cred) {
|
||||
token = await fetchTokenForCredential(cred);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (token) {
|
||||
headers["Authorization"] = `Bearer ${token}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 12000);
|
||||
const response = await fetch(OPENSKY_DATA_URL, { headers, signal: controller.signal });
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Handle limits
|
||||
if (response.status === 429) {
|
||||
console.warn('[Aviation] 429 Rate Limit hit.');
|
||||
if (cred) rotateCredential();
|
||||
return;
|
||||
}
|
||||
|
||||
const creditsRemainingStr = response.headers.get("x-ratelimit-remaining");
|
||||
if (creditsRemainingStr && cred) {
|
||||
const rem = parseInt(creditsRemainingStr, 10);
|
||||
cred.creditsRemaining = rem;
|
||||
if (rem <= ROTATION_THRESHOLD) rotateCredential();
|
||||
}
|
||||
|
||||
if (!response.ok) throw new Error(`Status ${response.status}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.states && Array.isArray(data.states)) {
|
||||
const activeStates = data.states.filter((s: any) => s[6] !== null && s[5] !== null);
|
||||
messageBuffer.push(activeStates);
|
||||
console.log(`[Aviation] Poll OK: ${activeStates.length} aircraft in ${Date.now() - pollStart}ms`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`[Aviation] Polling Error (${Date.now() - pollStart}ms): ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function flushBuffer() {
|
||||
if (messageBuffer.length === 0) return;
|
||||
|
||||
// We only take the most recent state array from the buffer queue completely resetting it
|
||||
const latestStates = messageBuffer.pop();
|
||||
messageBuffer = [];
|
||||
|
||||
if (!latestStates || latestStates.length === 0) return;
|
||||
|
||||
const fetchedAt = Math.floor(Date.now() / 1000);
|
||||
const fleetObj: Record<string, any> = Object.create(null);
|
||||
let insertedCount = 0;
|
||||
|
||||
const insertMany = db.transaction((states) => {
|
||||
for (const s of states) {
|
||||
const icao24 = s[0] as string;
|
||||
const callsign = (s[1] as string)?.trim() || null;
|
||||
const ts = s[3] as number || s[4] as number || fetchedAt; // time_position or last_contact
|
||||
const lon = s[5] as number;
|
||||
const lat = s[6] as number;
|
||||
const alt = s[7] as number; // baro_altitude
|
||||
const on_ground = s[8] as boolean;
|
||||
const spd = s[9] as number;
|
||||
const hdg = s[10] as number;
|
||||
|
||||
// 1. Write History to SQLite
|
||||
if (!on_ground) {
|
||||
const result = insertHistory.run({
|
||||
icao24, ts, lat, lon, alt, hdg, spd, fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
// 2. Prepare Live Cache structure
|
||||
// Zero out speed for grounded aircraft so frontend stops extrapolating
|
||||
const stateObj = {
|
||||
icao24,
|
||||
callsign,
|
||||
origin_country: s[2],
|
||||
time_position: s[3],
|
||||
last_contact: s[4],
|
||||
lon,
|
||||
lat,
|
||||
alt,
|
||||
on_ground,
|
||||
spd: on_ground ? 0 : spd,
|
||||
hdg: on_ground ? 0 : hdg,
|
||||
vertical_rate: s[11],
|
||||
sensors: s[12],
|
||||
geo_altitude: s[13],
|
||||
squawk: s[14],
|
||||
spi: s[15],
|
||||
position_source: s[16],
|
||||
last_updated: fetchedAt
|
||||
};
|
||||
|
||||
fleetObj[icao24] = stateObj;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
insertMany(latestStates);
|
||||
// Execute Redis batch via compressed snapshot (10 mins)
|
||||
await setLiveSnapshot('aviation', fleetObj, 10 * 60);
|
||||
} catch (err) {
|
||||
console.error('[Aviation] Buffer flush failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export function startAviationPoller() {
|
||||
initCredentialPool();
|
||||
console.log('[Aviation] Starting background polling...');
|
||||
|
||||
// Daily reset at midnight
|
||||
cron.schedule('0 0 * * *', () => {
|
||||
resetCredentialPool();
|
||||
});
|
||||
|
||||
setInterval(pollOpenSky, POLLING_INTERVAL_MS);
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
|
||||
// Initial fetch
|
||||
pollOpenSky();
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "aviation",
|
||||
init: startAviationPoller
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
interface GdeltFeature {
|
||||
type: string;
|
||||
geometry: { type: string, coordinates: number[] };
|
||||
properties: {
|
||||
urlpubtimedate: string;
|
||||
name: string;
|
||||
domain: string;
|
||||
url: string;
|
||||
urltone: number;
|
||||
};
|
||||
}
|
||||
|
||||
const GDELT_URL = 'http://api.gdeltproject.org/api/v1/gkg_geojson?query=protest OR riot OR demonstration OR strike OR clash&maxrows=2500';
|
||||
|
||||
function classifyGdeltEventType(name: string) {
|
||||
const lowerName = name.toLowerCase();
|
||||
if (lowerName.includes('riot')) return 'Riots';
|
||||
if (lowerName.includes('clash')) return 'Riots';
|
||||
if (lowerName.includes('strike')) return 'Strikes';
|
||||
if (lowerName.includes('demonstration')) return 'Demonstrations';
|
||||
return 'Protests';
|
||||
}
|
||||
|
||||
function classifyGdeltSubType(name: string, count: number) {
|
||||
const lowerName = name.toLowerCase();
|
||||
if (count > 50 || lowerName.includes('riot') || lowerName.includes('clash')) return 'Violent demonstration';
|
||||
if (lowerName.includes('strike')) return 'Labor strike';
|
||||
return 'Peaceful protest';
|
||||
}
|
||||
|
||||
const insertUnrest = db.prepare(
|
||||
'INSERT OR REPLACE INTO civil_unrest (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)'
|
||||
);
|
||||
|
||||
export async function seedCivilUnrest() {
|
||||
console.log('[CivilUnrest] Fetching from GDELT API...');
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(GDELT_URL, { headers: { 'User-Agent': 'WWV-Data-Engine' } }, 25000), 3, 5000);
|
||||
if (!res.ok) {
|
||||
console.warn(`[CivilUnrest] Failed to fetch. HTTP ${res.status}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const json = await res.json();
|
||||
const features = json.features as GdeltFeature[];
|
||||
|
||||
if (!features || features.length === 0) {
|
||||
console.log('[CivilUnrest] No events returned from GDELT.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Aggregate by location
|
||||
const locationMap = new Map<string, any>();
|
||||
for (const feature of features) {
|
||||
const name = feature.properties?.name || '';
|
||||
if (!name) continue;
|
||||
|
||||
const coords = feature.geometry?.coordinates;
|
||||
if (!Array.isArray(coords) || coords.length < 2) continue;
|
||||
|
||||
const [lon, lat] = coords;
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon) || lat < -90 || lat > 90 || lon < -180 || lon > 180) continue;
|
||||
|
||||
// Grid snap to ~11km clustering
|
||||
const key = `${lat.toFixed(1)}:${lon.toFixed(1)}`;
|
||||
const existing = locationMap.get(key);
|
||||
if (existing) {
|
||||
existing.count++;
|
||||
existing.urls.push(feature.properties.url);
|
||||
if (feature.properties.urltone < existing.worstTone) {
|
||||
existing.worstTone = feature.properties.urltone;
|
||||
}
|
||||
} else {
|
||||
locationMap.set(key, {
|
||||
name,
|
||||
lat,
|
||||
lon,
|
||||
count: 1,
|
||||
worstTone: feature.properties.urltone ?? 0,
|
||||
date: feature.properties.urlpubtimedate,
|
||||
urls: [feature.properties.url]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
const items: any[] = [];
|
||||
|
||||
for (const [, loc] of locationMap) {
|
||||
if (loc.count < 3) continue; // Filter noise
|
||||
|
||||
const country = loc.name.split(',').pop()?.trim() || "Unknown";
|
||||
const eventType = classifyGdeltEventType(loc.name);
|
||||
|
||||
// Create aggregated cluster event
|
||||
const item = {
|
||||
id: `gdelt-${loc.lat.toFixed(2)}-${loc.lon.toFixed(2)}`,
|
||||
lat: loc.lat,
|
||||
lon: loc.lon,
|
||||
type: eventType,
|
||||
subType: classifyGdeltSubType(loc.name, loc.count),
|
||||
actor1: "General Public",
|
||||
actor2: "N/A",
|
||||
fatalities: 0,
|
||||
country,
|
||||
location: loc.name,
|
||||
date: loc.date,
|
||||
source: "GDELT",
|
||||
notes: `${loc.count} clustered reports. Worst Tone: ${loc.worstTone.toFixed(1)}`,
|
||||
reportCount: loc.count
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
insertUnrest.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: new Date(item.date).getTime(),
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`[CivilUnrest] Clustered ${features.length} mentions into ${items.length} confirmed unrest events.`);
|
||||
|
||||
await setLiveSnapshot(
|
||||
'civil_unrest',
|
||||
{
|
||||
source: 'gdelt',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items,
|
||||
totalCount: items.length
|
||||
},
|
||||
86400
|
||||
);
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'civilUnrest',
|
||||
cron: '*/15 * * * *',
|
||||
fn: seedCivilUnrest
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import crypto from 'crypto';
|
||||
|
||||
interface ConflictEvent {
|
||||
id: string;
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
type: string;
|
||||
subType: string;
|
||||
actor1: string;
|
||||
actor2: string;
|
||||
fatalities: number;
|
||||
date: string;
|
||||
source: string;
|
||||
notes: string;
|
||||
}
|
||||
|
||||
const CONFLICT_HOTSPOTS = [
|
||||
{ lat: 48.3794, lon: 31.1656, radius: 5 }, // Ukraine
|
||||
{ lat: 31.0461, lon: 34.8516, radius: 2 }, // Israel/Gaza
|
||||
{ lat: 15.5007, lon: 32.5599, radius: 6 }, // Sudan
|
||||
{ lat: 15.3694, lon: 44.1910, radius: 3 }, // Yemen
|
||||
{ lat: 14.4974, lon: 14.4524, radius: 8 }, // Sahel region
|
||||
{ lat: 16.8661, lon: 96.1951, radius: 5 }, // Myanmar
|
||||
{ lat: 5.1521, lon: 46.1996, radius: 4 }, // Somalia
|
||||
];
|
||||
|
||||
const EVENT_TYPES = [
|
||||
'Battles',
|
||||
'Explosions/Remote violence',
|
||||
'Violence against civilians',
|
||||
'Protests',
|
||||
'Riots',
|
||||
'Strategic developments'
|
||||
];
|
||||
|
||||
const WEAPON_TYPES = [
|
||||
'Artillery',
|
||||
'Airstrike',
|
||||
'Drone strike',
|
||||
'Armed clashes',
|
||||
'IED',
|
||||
'Small arms'
|
||||
];
|
||||
|
||||
function generateMockConflictEvents(): ConflictEvent[] {
|
||||
const events: ConflictEvent[] = [];
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
for (const hotspot of CONFLICT_HOTSPOTS) {
|
||||
const numEvents = Math.floor(Math.random() * 10) + 5; // 5-15 events per hotspot
|
||||
|
||||
for (let i = 0; i < numEvents; i++) {
|
||||
const u = Math.random();
|
||||
const v = Math.random();
|
||||
const w = hotspot.radius / 111.0;
|
||||
const t = 2 * Math.PI * v;
|
||||
const x = w * Math.cos(t);
|
||||
const y = w * Math.sin(t);
|
||||
|
||||
const lat = hotspot.lat + y;
|
||||
const lon = hotspot.lon + x;
|
||||
const type = EVENT_TYPES[Math.floor(Math.random() * EVENT_TYPES.length)];
|
||||
const subType = WEAPON_TYPES[Math.floor(Math.random() * WEAPON_TYPES.length)];
|
||||
const fatalities = Math.floor(Math.random() * 10) === 0 ? Math.floor(Math.random() * 50) : Math.floor(Math.random() * 5); // mostly small, some mass
|
||||
|
||||
events.push({
|
||||
id: crypto.randomUUID(),
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
type,
|
||||
subType,
|
||||
actor1: 'Unidentified Armed Group',
|
||||
actor2: 'State Forces',
|
||||
fatalities,
|
||||
date: today,
|
||||
source: 'Mock ACLED Data',
|
||||
notes: 'Reported ' + type.toLowerCase() + ' involving ' + subType.toLowerCase() + '.'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
const insertStmt = db.prepare('INSERT OR REPLACE INTO conflict_events (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function fetchConflictEvents() {
|
||||
console.log('[Seeder: ConflictEvents] Mocking ACLED data generation...');
|
||||
const events = generateMockConflictEvents();
|
||||
|
||||
const now = Date.now();
|
||||
let inserted = 0;
|
||||
|
||||
const insertMany = db.transaction((rows: any[]) => {
|
||||
for (const row of rows) {
|
||||
insertStmt.run(row);
|
||||
inserted++;
|
||||
}
|
||||
});
|
||||
|
||||
const dbRows = events.map(e => ({
|
||||
id: e.id,
|
||||
payload: JSON.stringify(e),
|
||||
source_ts: now,
|
||||
fetched_at: now
|
||||
}));
|
||||
|
||||
insertMany(dbRows);
|
||||
console.log('[Seeder: ConflictEvents] Inserted ' + inserted + ' events into DB.');
|
||||
|
||||
try {
|
||||
const geoEntities = events.map(e => ({
|
||||
id: 'conflict-' + e.id,
|
||||
latitude: e.latitude,
|
||||
longitude: e.longitude,
|
||||
properties: {
|
||||
type: e.type,
|
||||
subType: e.subType,
|
||||
fatalities: e.fatalities,
|
||||
actor1: e.actor1,
|
||||
actor2: e.actor2,
|
||||
date: e.date,
|
||||
notes: e.notes
|
||||
}
|
||||
}));
|
||||
|
||||
await setLiveSnapshot('conflict_events', geoEntities, 3600 * 24); // Cache for 24 hours
|
||||
} catch (err) {
|
||||
console.warn('[Seeder: ConflictEvents] Redis cache failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'conflictEvents',
|
||||
cron: '0 0 * * *', // Run daily at midnight
|
||||
fn: fetchConflictEvents
|
||||
});
|
||||
@@ -0,0 +1,156 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { geolocateIp } from '../geoip';
|
||||
|
||||
const OTX_BASE = 'https://otx.alienvault.com/api/v1';
|
||||
|
||||
const insertCyberAttack = db.prepare(
|
||||
'INSERT OR REPLACE INTO cyber_attacks (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)'
|
||||
);
|
||||
|
||||
interface OtxIndicator {
|
||||
id: number;
|
||||
indicator: string;
|
||||
type: string;
|
||||
title: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface OtxPulse {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
created: string;
|
||||
modified: string;
|
||||
adversary: string;
|
||||
targeted_countries: string[];
|
||||
attack_ids: { id: string; name: string }[];
|
||||
malware_families: string[];
|
||||
indicators: OtxIndicator[];
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
export async function seedCyberAttacks() {
|
||||
const apiKey = process.env.OTX_API_KEY;
|
||||
if (!apiKey) {
|
||||
console.warn('[CyberAttacks] OTX_API_KEY not set — skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[CyberAttacks] Polling AlienVault OTX...');
|
||||
|
||||
// Fetch pulses modified in the last 48 hours
|
||||
const since = new Date(Date.now() - 48 * 3600 * 1000).toISOString();
|
||||
const url = `${OTX_BASE}/pulses/subscribed?modified_since=${since}&limit=50`;
|
||||
|
||||
const res = await withRetry(async () => {
|
||||
try {
|
||||
return await fetchWithTimeout(url, {
|
||||
headers: { 'X-OTX-API-KEY': apiKey, 'User-Agent': 'WWV-Data-Engine' },
|
||||
});
|
||||
} catch(err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error('[CyberAttacks] Fetch error details:', msg);
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
const data = await res.json();
|
||||
|
||||
if (!data?.results || !Array.isArray(data.results)) {
|
||||
console.warn('[CyberAttacks] Invalid OTX response');
|
||||
return;
|
||||
}
|
||||
|
||||
const pulses: OtxPulse[] = data.results;
|
||||
const fetchedAt = Date.now();
|
||||
const items: any[] = [];
|
||||
const seenIps = new Set<string>();
|
||||
|
||||
for (const pulse of pulses) {
|
||||
// Extract only IPv4 indicators
|
||||
const ipIndicators = (pulse.indicators || []).filter(
|
||||
(ind) => ind.type === 'IPv4'
|
||||
);
|
||||
|
||||
for (const ind of ipIndicators) {
|
||||
if (seenIps.has(ind.indicator)) continue;
|
||||
seenIps.add(ind.indicator);
|
||||
|
||||
const geo = geolocateIp(ind.indicator);
|
||||
if (!geo) continue; // Skip un-geolocatable IPs
|
||||
|
||||
const threatType = classifyThreat(pulse);
|
||||
const item = {
|
||||
id: `otx-${pulse.id}-${ind.id}`,
|
||||
ip: ind.indicator,
|
||||
lat: geo.lat,
|
||||
lon: geo.lon,
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
threatType,
|
||||
adversary: pulse.adversary || 'Unknown',
|
||||
pulseName: pulse.name,
|
||||
pulseDescription: pulse.description?.slice(0, 300) || '',
|
||||
malwareFamilies: pulse.malware_families || [],
|
||||
tags: pulse.tags?.slice(0, 5) || [],
|
||||
targetedCountries: pulse.targeted_countries || [],
|
||||
pulseId: pulse.id,
|
||||
pulseCreated: pulse.created,
|
||||
pulseModified: pulse.modified,
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
|
||||
// Persist to SQLite
|
||||
insertCyberAttack.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: new Date(pulse.modified).getTime(),
|
||||
fetched_at: fetchedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[CyberAttacks] Processed ${pulses.length} pulses → ${items.length} geolocated indicators.`
|
||||
);
|
||||
|
||||
// Save to Redis
|
||||
await setLiveSnapshot(
|
||||
'cyber_attacks',
|
||||
{
|
||||
source: 'cyber_attacks',
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items,
|
||||
totalCount: items.length,
|
||||
},
|
||||
7200 // 2 hour TTL
|
||||
);
|
||||
}
|
||||
|
||||
function classifyThreat(pulse: OtxPulse): string {
|
||||
const tags = (pulse.tags || []).map((t) => t.toLowerCase());
|
||||
const name = pulse.name.toLowerCase();
|
||||
const desc = (pulse.description || '').toLowerCase();
|
||||
const combined = [...tags, name, desc].join(' ');
|
||||
|
||||
if (combined.includes('apt') || combined.includes('advanced persistent'))
|
||||
return 'APT';
|
||||
if (combined.includes('ransomware')) return 'Ransomware';
|
||||
if (combined.includes('botnet')) return 'Botnet';
|
||||
if (combined.includes('phishing')) return 'Phishing';
|
||||
if (combined.includes('ddos')) return 'DDoS';
|
||||
if (combined.includes('malware') || combined.includes('trojan'))
|
||||
return 'Malware';
|
||||
if (combined.includes('c2') || combined.includes('command and control'))
|
||||
return 'C2 Server';
|
||||
return 'Other';
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: 'cyber_attacks',
|
||||
cron: '0 */2 * * *', // Every 2 hours
|
||||
fn: seedCyberAttacks,
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry, haversineKm } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const KNOWN_TEST_SITES = [
|
||||
{ name: 'Punggye-ri (North Korea)', lat: 41.278, lon: 129.088 },
|
||||
{ name: 'Lop Nur (China)', lat: 40.75, lon: 89.6 },
|
||||
{ name: 'Nevada Test Site (USA)', lat: 37.13, lon: -116.04 },
|
||||
{ name: 'Semipalatinsk (Kazakhstan)', lat: 50.4, lon: 77.8 },
|
||||
{ name: 'Novaya Zemlya (Russia)', lat: 73.3, lon: 54.9 },
|
||||
{ name: 'Pokhran (India)', lat: 27.09, lon: 71.75 },
|
||||
{ name: 'Chagai (Pakistan)', lat: 28.79, lon: 64.91 },
|
||||
{ name: 'Moruroa (France)', lat: -21.83, lon: -138.88 },
|
||||
];
|
||||
|
||||
const insertEarthquake = db.prepare('INSERT OR IGNORE INTO earthquakes (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function seedEarthquakes() {
|
||||
console.log('[Earthquakes] Polling USGS...');
|
||||
|
||||
// USGS GeoJSON Feed: 4.5+ magnitude, past 7 days
|
||||
const url = 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_week.geojson';
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(url));
|
||||
const data = await res.json();
|
||||
const fetchedAt = Date.now();
|
||||
|
||||
if (!data?.features || !Array.isArray(data.features)) {
|
||||
console.warn('[Earthquakes] Invalid response from USGS');
|
||||
return;
|
||||
}
|
||||
|
||||
const items = [];
|
||||
let insertedCount = 0;
|
||||
|
||||
for (const feature of data.features) {
|
||||
const { id, properties, geometry } = feature;
|
||||
const [lon, lat, depth] = geometry.coordinates;
|
||||
const sourceTs = properties.time;
|
||||
|
||||
// Detect nuclear test site proximity (< 10km)
|
||||
let nearTestSite = false;
|
||||
let nearestSiteName = null;
|
||||
let minDistance = Infinity;
|
||||
|
||||
for (const site of KNOWN_TEST_SITES) {
|
||||
const dist = haversineKm(lat, lon, site.lat, site.lon);
|
||||
if (dist < minDistance) {
|
||||
minDistance = dist;
|
||||
if (dist < 10) { // Within 10km is highly suspicious
|
||||
nearTestSite = true;
|
||||
nearestSiteName = site.name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const item = {
|
||||
id,
|
||||
place: properties.place,
|
||||
magnitude: properties.mag,
|
||||
depth_km: depth,
|
||||
lat,
|
||||
lon,
|
||||
occurredAt: sourceTs,
|
||||
url: properties.url,
|
||||
nearTestSite,
|
||||
nearestSiteName,
|
||||
distanceToTestSiteKm: minDistance < 50 ? minDistance : undefined
|
||||
};
|
||||
|
||||
items.push(item);
|
||||
|
||||
// Save to SQLite
|
||||
const result = insertEarthquake.run({
|
||||
id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
console.log(`[Earthquakes] Parsed ${items.length} earthquakes. Saved ${insertedCount} new to SQLite.`);
|
||||
|
||||
// Save to Redis Live Cache
|
||||
await setLiveSnapshot('earthquakes', {
|
||||
source: "earthquakes",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
}, 3600); // 1 hour TTL
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "earthquakes",
|
||||
cron: "0 * * * *", // Every hour
|
||||
fn: seedEarthquakes
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
const insertGpsJam = db.prepare('INSERT OR REPLACE INTO gps_jamming (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
const HOTSPOTS = [
|
||||
{ name: 'Eastern Europe / Ukraine', lat: 48.0, lon: 37.0, radiusDeg: 5, intensity: 'high' },
|
||||
{ name: 'Baltic Sea Region', lat: 58.0, lon: 20.0, radiusDeg: 4, intensity: 'high' },
|
||||
{ name: 'Middle East', lat: 33.0, lon: 36.0, radiusDeg: 6, intensity: 'high' },
|
||||
{ name: 'Black Sea', lat: 43.0, lon: 34.0, radiusDeg: 3, intensity: 'medium' },
|
||||
{ name: 'Korean Peninsula', lat: 38.0, lon: 127.0, radiusDeg: 2, intensity: 'medium' },
|
||||
];
|
||||
|
||||
function generateMockData() {
|
||||
const items = [];
|
||||
const now = Date.now();
|
||||
|
||||
for (const hotspot of HOTSPOTS) {
|
||||
// Generate 50-150 points per hotspot
|
||||
const count = Math.floor(Math.random() * 100) + 50;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
// Random offset within radius using somewhat normal distribution
|
||||
const u = Math.random() + Math.random() - 1;
|
||||
const v = Math.random() + Math.random() - 1;
|
||||
|
||||
const lat = hotspot.lat + (u * hotspot.radiusDeg);
|
||||
const lon = hotspot.lon + (v * hotspot.radiusDeg);
|
||||
|
||||
// Intensity decreases slightly away from center
|
||||
const distFromCenter = Math.sqrt(u*u + v*v);
|
||||
let pointIntensity = hotspot.intensity;
|
||||
|
||||
if (hotspot.intensity === 'high' && distFromCenter > 0.6) {
|
||||
pointIntensity = 'medium';
|
||||
}
|
||||
if (pointIntensity === 'medium' && distFromCenter > 0.8) {
|
||||
pointIntensity = 'low';
|
||||
}
|
||||
|
||||
items.push({
|
||||
id: randomUUID(),
|
||||
lat,
|
||||
lon,
|
||||
interferenceLevel: pointIntensity,
|
||||
timestamp: now,
|
||||
region: hotspot.name
|
||||
});
|
||||
}
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function seedGpsJam() {
|
||||
console.log('[GPS Jamming] Generating daily interference map snapshot...');
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
const sourceTs = fetchedAt; // Represents end of the 24h period for the map
|
||||
const items = generateMockData();
|
||||
|
||||
let insertedCount = 0;
|
||||
|
||||
db.transaction(() => {
|
||||
// Optional: clear old data to keep DB size small if this is just a daily map
|
||||
db.exec('DELETE FROM gps_jamming');
|
||||
|
||||
for (const item of items) {
|
||||
const result = insertGpsJam.run({
|
||||
id: item.id,
|
||||
payload: JSON.stringify(item),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
})();
|
||||
|
||||
console.log(`[GPS Jamming] Generated ${items.length} points. Saved ${insertedCount} to SQLite.`);
|
||||
|
||||
// Save to Redis Live Cache
|
||||
await setLiveSnapshot('gps_jamming', {
|
||||
source: "gpsjam_mock",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: items,
|
||||
totalCount: items.length
|
||||
}, 86400); // 24 hours TTL
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "gps_jamming",
|
||||
cron: "0 0 * * *", // Once daily at midnight
|
||||
fn: seedGpsJam
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
import './iranwarlive';
|
||||
import './earthquakes';
|
||||
import './wildfires';
|
||||
import './maritime';
|
||||
import './aviation';
|
||||
import './military-aviation';
|
||||
import './satellite';
|
||||
// --- TEMPORARILY DISABLED FOR DEMO (MOCK DATA) ---
|
||||
import './gpsjam';
|
||||
// import './conflictEvents';
|
||||
import './civilUnrest';
|
||||
import './cyberAttacks';
|
||||
import './sanctions';
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { z } from 'zod';
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
// Zod Schema for input validation to strip unrecognized injection keys
|
||||
const itemSchema = z.object({
|
||||
event_id: z.string().max(255),
|
||||
type: z.string().max(255).nullable().optional(),
|
||||
location: z.string().max(500).nullable().optional(),
|
||||
timestamp: z.string().max(100),
|
||||
confidence: z.string().max(100).nullable().optional(),
|
||||
event_summary: z.string().max(10000).nullable().optional(),
|
||||
source_url: z.string().max(2000).nullable().optional(),
|
||||
preview_image: z.string().url().max(2000).nullable().optional(),
|
||||
_osint_meta: z.any().optional()
|
||||
});
|
||||
|
||||
const insertEvent = db.prepare(`
|
||||
INSERT INTO iranwar_events (event_id, payload, timestamp, fetched_at)
|
||||
VALUES (@event_id, @payload, @timestamp, @fetched_at)
|
||||
ON CONFLICT(event_id) DO UPDATE SET
|
||||
payload=excluded.payload,
|
||||
timestamp=excluded.timestamp
|
||||
`);
|
||||
const getTopEvents = db.prepare('SELECT payload FROM iranwar_events ORDER BY timestamp DESC LIMIT 500');
|
||||
|
||||
let hasHydratedSeed = false;
|
||||
|
||||
export async function seedIranWarLive() {
|
||||
// --- SELF-HEALING HISTORY SEED ---
|
||||
if (!hasHydratedSeed) {
|
||||
console.log('[IranWarLive] Initializing: Hydrating/Upserting active fallback seed...');
|
||||
const seedPath = path.join(__dirname, '..', '..', 'seedData', 'iranwar_seed.json');
|
||||
if (fs.existsSync(seedPath)) {
|
||||
const fallbackData = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
let variables = 0;
|
||||
const upsertMany = db.transaction((events: any[]) => {
|
||||
for (const item of events) {
|
||||
try {
|
||||
const validatedItem = itemSchema.parse(item);
|
||||
const result = insertEvent.run({
|
||||
event_id: validatedItem.event_id,
|
||||
payload: JSON.stringify(validatedItem),
|
||||
timestamp: validatedItem.timestamp,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
} catch(err) { /* ignore validation errors in fallback */ }
|
||||
}
|
||||
});
|
||||
upsertMany(fallbackData);
|
||||
console.log(`[IranWarLive] Boot hydration complete. Merged ${insertedCount} seed events.`);
|
||||
}
|
||||
hasHydratedSeed = true;
|
||||
}
|
||||
|
||||
console.log('[IranWarLive] Polling iranwarlive.com/feed.json...');
|
||||
|
||||
let data: any = null;
|
||||
try {
|
||||
const response = await withRetry(() => fetchWithTimeout('https://iranwarlive.com/feed.json', {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cache-Control": "no-cache",
|
||||
"Accept-Language": "en-US,en;q=0.9"
|
||||
}
|
||||
}));
|
||||
if(!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
data = await response.json();
|
||||
} catch(err: any) {
|
||||
console.warn(`[IranWarLive] Failed to fetch live feed (anti-bot block?): ${err.message}. Using local database cache.`);
|
||||
}
|
||||
|
||||
if (data && data.items && Array.isArray(data.items) && data.items.length > 0) {
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
|
||||
// Find completely new items to hydrate
|
||||
const placeholders = data.items.map(() => '?').join(',');
|
||||
const existingIdsStmt = db.prepare(`SELECT event_id FROM iranwar_events WHERE event_id IN (${placeholders})`);
|
||||
const existingIdsRow = existingIdsStmt.all(...data.items.map((i: any) => i.event_id)) as { event_id: string }[];
|
||||
const existingIds = new Set(existingIdsRow.map(row => row.event_id));
|
||||
|
||||
const newItems = data.items.filter((item: any) => !existingIds.has(item.event_id));
|
||||
|
||||
if (newItems.length > 0) {
|
||||
console.log(`[IranWarLive] Found ${newItems.length} new events. Hydrating og:images...`);
|
||||
for (const item of newItems) {
|
||||
if (item.source_url) {
|
||||
try {
|
||||
const htmlRes = await fetchWithTimeout(item.source_url, { headers: { "User-Agent": "WorldWideView-OSINT/1.0" } }, 5000);
|
||||
const html = await htmlRes.text();
|
||||
const ogMatch = html.match(/<meta[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/i)
|
||||
|| html.match(/<meta[^>]*content=["']([^"']+)["'][^>]*property=["']og:image["']/i);
|
||||
if (ogMatch && ogMatch[1]) {
|
||||
item.preview_image = ogMatch[1];
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(`[IranWarLive] Failed to hydrate image for ${item.event_id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const insertMany = db.transaction((events: any[]) => {
|
||||
for (const item of events) {
|
||||
try {
|
||||
const validatedItem = itemSchema.parse(item);
|
||||
const result = insertEvent.run({
|
||||
event_id: validatedItem.event_id,
|
||||
payload: JSON.stringify(validatedItem),
|
||||
timestamp: validatedItem.timestamp,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
} catch(err: any) {
|
||||
console.warn(`[IranWarLive] Skipped item due to validation error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(newItems);
|
||||
if (insertedCount > 0) {
|
||||
console.log(`[IranWarLive] Added ${insertedCount} new hydrated events to history.`);
|
||||
}
|
||||
} else {
|
||||
console.log('[IranWarLive] No new events found.');
|
||||
}
|
||||
}
|
||||
|
||||
// ALways update Redis hot cache with the latest 500 events
|
||||
const rows = getTopEvents.all() as { payload: string }[];
|
||||
const history = rows.map(row => JSON.parse(row.payload));
|
||||
|
||||
await setLiveSnapshot('iranwarlive', {
|
||||
source: "iranwarlive",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: history
|
||||
}, 3600); // 1 hour TTL in Redis
|
||||
}
|
||||
|
||||
// Register with scheduler
|
||||
registerSeeder({
|
||||
name: "iranwarlive",
|
||||
cron: "*/1 * * * *", // Every minute
|
||||
fn: seedIranWarLive
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
import WebSocket from 'ws';
|
||||
import { db } from '../db';
|
||||
import { redis, setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const AISSTREAM_URL = 'wss://stream.aisstream.io/v0/stream';
|
||||
const API_KEY = process.env.AISSTREAM_API_KEY;
|
||||
|
||||
// Buffer to batch SQLite inserts
|
||||
let messageBuffer: any[] = [];
|
||||
const activeFleetCache = new Map<string, any>();
|
||||
const FLUSH_INTERVAL_MS = 15000;
|
||||
let isFlushIntervalRunning = false;
|
||||
|
||||
// SQLite insert statement
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO maritime_history (mmsi, ts, lat, lon, hdg, spd, fetched_at)
|
||||
VALUES (@mmsi, @ts, @lat, @lon, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function flushBuffer() {
|
||||
if (messageBuffer.length === 0) return;
|
||||
|
||||
const batch = [...messageBuffer];
|
||||
messageBuffer = [];
|
||||
|
||||
const fetchedAt = Date.now();
|
||||
let insertedCount = 0;
|
||||
|
||||
// 1. Bulk insert to SQLite for permanent history
|
||||
const insertMany = db.transaction((msgs) => {
|
||||
for (const msg of msgs) {
|
||||
if (!msg.MetaData?.MMSI || !msg.Message?.PositionReport) continue;
|
||||
|
||||
const mmsi = msg.MetaData.MMSI.toString();
|
||||
const report = msg.Message.PositionReport;
|
||||
const ts = Math.floor(new Date(msg.MetaData.time_utc).getTime() / 1000); // Epoch seconds
|
||||
|
||||
const result = insertHistory.run({
|
||||
mmsi,
|
||||
ts,
|
||||
lat: report.Latitude,
|
||||
lon: report.Longitude,
|
||||
hdg: report.TrueHeading,
|
||||
spd: report.Sog,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
insertMany(batch);
|
||||
if (insertedCount > 0) {
|
||||
// console.log(`[Maritime] Flushed ${insertedCount} new positions to SQLite history`);
|
||||
}
|
||||
|
||||
// 2. Update memory cache and serialize to snapshot
|
||||
const nowSecs = Math.floor(Date.now() / 1000);
|
||||
// Cleanup stale ships (>6 hours)
|
||||
for (const [mmsi, ship] of activeFleetCache.entries()) {
|
||||
if (nowSecs - ship.last_updated > 6 * 3600) {
|
||||
activeFleetCache.delete(mmsi);
|
||||
}
|
||||
}
|
||||
|
||||
for (const msg of batch) {
|
||||
if (!msg.MetaData?.MMSI || !msg.Message?.PositionReport) continue;
|
||||
const mmsi = msg.MetaData.MMSI.toString();
|
||||
const report = msg.Message.PositionReport;
|
||||
const ts = Math.floor(new Date(msg.MetaData.time_utc).getTime() / 1000);
|
||||
|
||||
const shipState = {
|
||||
id: `mmsi-${mmsi}`,
|
||||
mmsi,
|
||||
name: msg.MetaData.ShipName ? msg.MetaData.ShipName.trim() : `Unknown (${mmsi})`,
|
||||
lat: report.Latitude,
|
||||
lon: report.Longitude,
|
||||
hdg: report.TrueHeading,
|
||||
spd: report.Sog,
|
||||
last_updated: ts
|
||||
};
|
||||
|
||||
activeFleetCache.set(mmsi, shipState);
|
||||
}
|
||||
|
||||
await setLiveSnapshot('maritime', Object.fromEntries(activeFleetCache), 6 * 3600);
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Maritime] Buffer flush failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export function startMaritimeWebsocket() {
|
||||
if (!API_KEY) {
|
||||
console.warn('[Maritime] Skipping AIS websocket: AISSTREAM_API_KEY not set.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[Maritime] Connecting to AisStream.io...');
|
||||
const ws = new WebSocket(AISSTREAM_URL);
|
||||
|
||||
ws.on('open', () => {
|
||||
console.log('[Maritime] WebSocket connected. Subscribing to global feed...');
|
||||
// Bounding box for English Channel / Strait of Hormuz / global (up to API tier)
|
||||
// Here we request global or specific high-traffic boxes
|
||||
const subscriptionMessage = {
|
||||
APIKey: API_KEY,
|
||||
BoundingBoxes: [[[-90, -180], [90, 180]]], // Global (requires paid tier for real use)
|
||||
FiltersShipMMSI: [], // All ships
|
||||
FilterMessageTypes: ["PositionReport"]
|
||||
};
|
||||
ws.send(JSON.stringify(subscriptionMessage));
|
||||
});
|
||||
|
||||
ws.on('message', (data) => {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString());
|
||||
if (msg.MessageType === "PositionReport" || msg.Message?.PositionReport) {
|
||||
messageBuffer.push(msg);
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parse errors
|
||||
}
|
||||
});
|
||||
|
||||
ws.on('error', (err) => {
|
||||
console.error('[Maritime] WebSocket error:', err.message);
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
console.log('[Maritime] WebSocket closed. Reconnecting in 5s...');
|
||||
setTimeout(startMaritimeWebsocket, 5000);
|
||||
});
|
||||
|
||||
// Start background flush loop safely
|
||||
if (!isFlushIntervalRunning) {
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
isFlushIntervalRunning = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Register initialization logic. No cron needed, runs infinitely.
|
||||
registerSeeder({
|
||||
name: "maritime",
|
||||
init: startMaritimeWebsocket
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const ADSB_LOL_URL = "https://api.adsb.lol/v2/mil";
|
||||
const POLLING_INTERVAL_MS = 60000;
|
||||
|
||||
const insertHistory = db.prepare(`
|
||||
INSERT OR IGNORE INTO military_aviation_history (hex, ts, lat, lon, alt, hdg, spd, fetched_at)
|
||||
VALUES (@hex, @ts, @lat, @lon, @alt, @hdg, @spd, @fetched_at)
|
||||
`);
|
||||
|
||||
async function pollMilitaryAviation() {
|
||||
try {
|
||||
const response = await fetch(ADSB_LOL_URL, {
|
||||
headers: {
|
||||
"User-Agent": "WorldWideView-DataEngine"
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
console.warn('[MilitaryAviation] 429 Rate Limit hit.');
|
||||
}
|
||||
throw new Error(`Status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (!data.ac || !Array.isArray(data.ac)) return;
|
||||
|
||||
const fetchedAt = Math.floor(Date.now() / 1000);
|
||||
const fleetObj: Record<string, any> = Object.create(null);
|
||||
let insertedCount = 0;
|
||||
|
||||
const insertMany = db.transaction((aircraft) => {
|
||||
for (const ac of aircraft) {
|
||||
if (ac.lat == null || ac.lon == null) continue;
|
||||
|
||||
const hex = ac.hex;
|
||||
// Some fields might be missing depending on coverage
|
||||
const ts = Math.floor((ac.seen_pos != null ? (Date.now() - ac.seen_pos * 1000) : Date.now()) / 1000); // Approximate timestamp based on seen_pos
|
||||
const lon = ac.lon;
|
||||
const lat = ac.lat;
|
||||
const alt = typeof ac.alt_baro === "number" ? ac.alt_baro : 0;
|
||||
const on_ground = ac.alt_baro === "ground";
|
||||
const spd = ac.gs || 0;
|
||||
const hdg = ac.track || 0;
|
||||
|
||||
// 1. Write History to SQLite
|
||||
if (!on_ground) {
|
||||
const result = insertHistory.run({
|
||||
hex, ts, lat, lon, alt, hdg, spd, fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
|
||||
// 2. Prepare Live Cache structure
|
||||
fleetObj[hex] = {
|
||||
hex,
|
||||
flight: ac.flight || null,
|
||||
r: ac.r || null,
|
||||
t: ac.t || null,
|
||||
lat,
|
||||
lon,
|
||||
alt_baro: ac.alt_baro,
|
||||
alt_geom: ac.alt_geom,
|
||||
gs: ac.gs,
|
||||
track: ac.track,
|
||||
squawk: ac.squawk,
|
||||
dbFlags: ac.dbFlags,
|
||||
category: ac.category,
|
||||
emergency: ac.emergency,
|
||||
seen: ac.seen,
|
||||
seen_pos: ac.seen_pos,
|
||||
last_updated: fetchedAt
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(data.ac);
|
||||
|
||||
// Execute Redis batch via compressed snapshot (60 mins to match slow moving or intermittent adsb.lol coverage)
|
||||
await setLiveSnapshot('military-aviation', fleetObj, 60 * 60);
|
||||
|
||||
} catch (err: any) {
|
||||
console.error(`[MilitaryAviation] Polling Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function startMilitaryAviationPoller() {
|
||||
console.log('[MilitaryAviation] Starting background polling...');
|
||||
setInterval(pollMilitaryAviation, POLLING_INTERVAL_MS);
|
||||
|
||||
// Initial fetch
|
||||
pollMilitaryAviation();
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "military-aviation",
|
||||
init: startMilitaryAviationPoller
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import sax from 'sax';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
import { fetchWithTimeout, CHROME_UA } from '../seed-utils';
|
||||
|
||||
const OFAC_SOURCES = [
|
||||
{ label: 'SDN', url: 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports/sdn_advanced.xml' },
|
||||
{ label: 'CONSOLIDATED', url: 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports/cons_advanced.xml' },
|
||||
];
|
||||
|
||||
function local(name: string) {
|
||||
const colon = name.indexOf(':');
|
||||
return colon === -1 ? name : name.slice(colon + 1);
|
||||
}
|
||||
|
||||
// ── SAX Parser adapted from WorldMonitor ──
|
||||
async function fetchSource(source: typeof OFAC_SOURCES[0]) {
|
||||
console.log(`[Sanctions] Fetching OFAC ${source.label}...`);
|
||||
const t0 = Date.now();
|
||||
const response = await fetchWithTimeout(source.url, {
|
||||
headers: { 'User-Agent': CHROME_UA }
|
||||
}, 45_000);
|
||||
|
||||
return new Promise<{ entries: any[] }>((resolve, reject) => {
|
||||
const parser = sax.parser(true, { trim: false, normalize: false });
|
||||
|
||||
const areaCodes = new Map();
|
||||
const featureTypes = new Map();
|
||||
const legalBasis = new Map();
|
||||
const locations = new Map();
|
||||
const parties = new Map();
|
||||
const entries: any[] = [];
|
||||
let bytesReceived = 0;
|
||||
|
||||
const stack: string[] = [];
|
||||
let text = '';
|
||||
|
||||
let inAreaCodeValues = false;
|
||||
let inFeatureTypeValues = false;
|
||||
let inLegalBasisValues = false;
|
||||
let inLocations = false;
|
||||
let inDistinctParties = false;
|
||||
let inSanctionsEntries = false;
|
||||
|
||||
let refId = '', refShortRef = '', refDescription = '';
|
||||
let locId = '';
|
||||
let locAreaCodeIds: string[] | null = null;
|
||||
let partyFixedRef = '';
|
||||
let profileId = '', profileSubTypeId = '';
|
||||
let aliases: any[] | null = null;
|
||||
let curAlias: any = null;
|
||||
let inDocumentedName = false;
|
||||
let namePartsBuf: string[] | null = null;
|
||||
let profileFeatures: any[] | null = null;
|
||||
let curFeature: any = null;
|
||||
|
||||
let entryId = '', entryProfileId = '';
|
||||
|
||||
function resolveLocation(locId: string) {
|
||||
if (!locAreaCodeIds) return { codes: [], names: [] };
|
||||
const ids = locAreaCodeIds;
|
||||
const mapped = ids.map((id) => areaCodes.get(id)).filter(Boolean);
|
||||
const pairs = [...new Map(mapped.map((item) => [item.code, item.name])).entries()]
|
||||
.filter(([code]) => code.length > 0);
|
||||
return { codes: pairs.map(([c]) => c), names: pairs.map(([, n]) => n) };
|
||||
}
|
||||
|
||||
function finalizeParty() {
|
||||
const seen = new Map();
|
||||
for (const feat of profileFeatures ?? []) {
|
||||
if (!/location/i.test(featureTypes.get(feat.featureTypeId) || '')) continue;
|
||||
for (const lid of feat.locationIds) {
|
||||
const loc = locations.get(lid);
|
||||
if (!loc) continue;
|
||||
loc.codes.forEach((code: string, i: number) => { if (code && !seen.has(code)) seen.set(code, loc.names[i] ?? ''); });
|
||||
}
|
||||
}
|
||||
parties.set(profileId, { countryCodes: [...seen.keys()] });
|
||||
}
|
||||
|
||||
function finalizeEntry() {
|
||||
const party = parties.get(entryProfileId);
|
||||
entries.push({ countryCodes: party?.countryCodes ?? [] });
|
||||
}
|
||||
|
||||
parser.onopentag = (node: any) => {
|
||||
const name = local(node.name);
|
||||
const attrs = node.attributes as any;
|
||||
stack.push(name);
|
||||
text = '';
|
||||
|
||||
switch (name) {
|
||||
case 'AreaCodeValues': inAreaCodeValues = true; break;
|
||||
case 'FeatureTypeValues': inFeatureTypeValues = true; break;
|
||||
case 'LegalBasisValues': inLegalBasisValues = true; break;
|
||||
case 'Locations': inLocations = true; break;
|
||||
case 'DistinctParties': inDistinctParties = true; break;
|
||||
case 'SanctionsEntries': inSanctionsEntries = true; break;
|
||||
|
||||
case 'AreaCode': if (inAreaCodeValues) { refId = attrs.ID || ''; refDescription = attrs.Description || ''; } break;
|
||||
case 'FeatureType': if (inFeatureTypeValues) refId = attrs.ID || ''; break;
|
||||
case 'Location': if (inLocations) { locId = attrs.ID || ''; locAreaCodeIds = []; } break;
|
||||
case 'LocationAreaCode': if (locAreaCodeIds && attrs.AreaCodeID) locAreaCodeIds.push(attrs.AreaCodeID); break;
|
||||
|
||||
case 'DistinctParty': if (inDistinctParties) { partyFixedRef = attrs.FixedRef || ''; aliases = []; profileFeatures = []; } break;
|
||||
case 'Profile': if (inDistinctParties) { profileId = attrs.ID || partyFixedRef; profileSubTypeId = attrs.PartySubTypeID || ''; } break;
|
||||
case 'Feature': if (inDistinctParties) curFeature = { featureTypeId: attrs.FeatureTypeID || '', locationIds: [] }; break;
|
||||
case 'VersionLocation': if (curFeature && attrs.LocationID) curFeature.locationIds.push(attrs.LocationID); break;
|
||||
|
||||
case 'SanctionsEntry':
|
||||
if (inSanctionsEntries) { entryId = attrs.ID || ''; entryProfileId = attrs.ProfileID || ''; }
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
parser.onclosetag = (rawName: string) => {
|
||||
const name = local(rawName);
|
||||
const t = text.trim();
|
||||
text = '';
|
||||
stack.pop();
|
||||
|
||||
switch (name) {
|
||||
case 'AreaCodeValues': inAreaCodeValues = false; break;
|
||||
case 'FeatureTypeValues': inFeatureTypeValues = false; break;
|
||||
case 'Locations': inLocations = false; break;
|
||||
case 'DistinctParties': inDistinctParties = false; break;
|
||||
case 'SanctionsEntries': inSanctionsEntries = false; break;
|
||||
|
||||
case 'AreaCode': if (inAreaCodeValues && refId) areaCodes.set(refId, { code: t, name: refDescription }); break;
|
||||
case 'FeatureType': if (inFeatureTypeValues && refId) featureTypes.set(refId, t); break;
|
||||
case 'Location':
|
||||
if (locAreaCodeIds !== null) {
|
||||
locations.set(locId, resolveLocation(locId));
|
||||
locId = ''; locAreaCodeIds = null;
|
||||
}
|
||||
break;
|
||||
case 'Feature': if (curFeature) { profileFeatures!.push(curFeature); curFeature = null; } break;
|
||||
case 'Profile':
|
||||
if (inDistinctParties && profileId) finalizeParty();
|
||||
profileId = ''; profileSubTypeId = ''; aliases = []; profileFeatures = [];
|
||||
break;
|
||||
case 'SanctionsEntry': finalizeEntry(); entryId = ''; entryProfileId = ''; break;
|
||||
}
|
||||
};
|
||||
|
||||
parser.ontext = (chunk: string) => { text += chunk; };
|
||||
parser.oncdata = (chunk: string) => { text += chunk; };
|
||||
parser.onerror = (err: any) => { parser.resume(); };
|
||||
parser.onend = () => { resolve({ entries }); };
|
||||
|
||||
if (!response.body) return reject(new Error('No response body'));
|
||||
|
||||
// We treat response.body as an AsyncIterable (Node fetch extension)
|
||||
(async () => {
|
||||
try {
|
||||
for await (const chunk of (response.body as any)) {
|
||||
parser.write(Buffer.from(chunk).toString('utf-8'));
|
||||
}
|
||||
parser.close();
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchSanctionsData() {
|
||||
try {
|
||||
console.log('[Sanctions] Fetching OFAC feeds using SAX streaming...');
|
||||
|
||||
let allEntries: any[] = [];
|
||||
for (const source of OFAC_SOURCES) {
|
||||
const result = await fetchSource(source);
|
||||
allEntries = allEntries.concat(result.entries);
|
||||
}
|
||||
|
||||
// Aggregate by iso2 country codes
|
||||
const countryCounts: Record<string, number> = {};
|
||||
for (const entry of allEntries) {
|
||||
for (const code of entry.countryCodes) {
|
||||
if (code && code !== 'XX') {
|
||||
countryCounts[code] = (countryCounts[code] || 0) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert dict to GeoEntities
|
||||
const items = Object.entries(countryCounts).map(([code, count]) => {
|
||||
let level = 'low';
|
||||
if (count > 500) level = 'high';
|
||||
else if (count > 50) level = 'medium';
|
||||
|
||||
return {
|
||||
id: `sanction-${code}`,
|
||||
countryCode: code,
|
||||
count: count,
|
||||
level,
|
||||
// Assign a placeholder coordinate, though frontend rendering will extract
|
||||
// 3D bounds from /borders.geojson using countryCode instead.
|
||||
latitude: 0,
|
||||
longitude: 0,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
});
|
||||
|
||||
const sanctionsObj = {
|
||||
id: 'sanctions-live',
|
||||
timestamp: new Date().toISOString(),
|
||||
items
|
||||
};
|
||||
|
||||
// Cache for live polling (24 hr TTL)
|
||||
await setLiveSnapshot('sanctions', sanctionsObj, 86400);
|
||||
console.log(`[Sanctions] Published ${items.length} sanctioned countries.`);
|
||||
} catch (err) {
|
||||
console.error('[Sanctions] Error:', err);
|
||||
}
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "sanctions",
|
||||
cron: "0 * * * *", // run hourly
|
||||
fn: fetchSanctionsData
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
// Dynamic import required by cjs/esm interop
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
const BASE_URL = "https://celestrak.org/NORAD/elements/gp.php";
|
||||
|
||||
export const DEFAULT_GROUPS = [
|
||||
"stations", // ISS, Tiangong, etc.
|
||||
"visual", // Brightest 100 satellites
|
||||
"weather", // Weather satellites
|
||||
"gps-ops", // GPS constellation
|
||||
"resource", // Earth observation / reconnaissance
|
||||
"military", // Military reconnaissance
|
||||
] as const;
|
||||
|
||||
export interface CelesTrakGP {
|
||||
OBJECT_NAME: string;
|
||||
TLE_LINE1: string;
|
||||
TLE_LINE2: string;
|
||||
NORAD_CAT_ID: number;
|
||||
COUNTRY_CODE?: string;
|
||||
OBJECT_TYPE?: string;
|
||||
PERIOD?: number;
|
||||
}
|
||||
|
||||
// In-memory global TLE cache for computing on-the-fly positions
|
||||
export const globalsTLECache = new Map<string, CelesTrakGP[]>();
|
||||
|
||||
async function fetchTLEGroup(group: string): Promise<CelesTrakGP[]> {
|
||||
const url = `${BASE_URL}?GROUP=${encodeURIComponent(group)}&FORMAT=tle`;
|
||||
const res = await fetch(url, { cache: "no-store", headers: { "User-Agent": "WorldWideView" } });
|
||||
|
||||
if (!res.ok) {
|
||||
console.error(`[SatelliteSeeder] CelesTrak returned ${res.status} for group=${group}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const text = await res.text();
|
||||
const lines = text.split("\n").map(l => l.trim()).filter(l => l.length > 0);
|
||||
const data: CelesTrakGP[] = [];
|
||||
|
||||
for (let i = 0; i < lines.length - 2; i += 3) {
|
||||
data.push({
|
||||
OBJECT_NAME: lines[i],
|
||||
TLE_LINE1: lines[i + 1],
|
||||
TLE_LINE2: lines[i + 2],
|
||||
NORAD_CAT_ID: parseInt(lines[i + 1].substring(2, 7).trim(), 10),
|
||||
} as CelesTrakGP);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
async function refreshAllTLEs() {
|
||||
console.log('[SatelliteSeeder] Refreshing TLEs from Celestrak...');
|
||||
for (const group of DEFAULT_GROUPS) {
|
||||
try {
|
||||
const records = await fetchTLEGroup(group);
|
||||
if (records.length > 0) {
|
||||
globalsTLECache.set(group, records);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`[SatelliteSeeder] Error fetching ${group}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exported standard propagation function
|
||||
export function propagateAll(records: CelesTrakGP[], time: Date, group: string): any[] {
|
||||
const satellite = require('satellite.js');
|
||||
const gmst = satellite.gstime(time);
|
||||
const results: any[] = [];
|
||||
|
||||
for (const rec of records) {
|
||||
try {
|
||||
const satrec = satellite.twoline2satrec(rec.TLE_LINE1, rec.TLE_LINE2);
|
||||
const pv = satellite.propagate(satrec, time);
|
||||
|
||||
if (!pv.position || typeof pv.position === "boolean" || !pv.velocity || typeof pv.velocity === "boolean") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const geo = satellite.eciToGeodetic(pv.position as any, gmst);
|
||||
const lat = satellite.degreesLat(geo.latitude);
|
||||
const lon = satellite.degreesLong(geo.longitude);
|
||||
const alt = geo.height; // km
|
||||
|
||||
if (!isFinite(lat) || !isFinite(lon) || !isFinite(alt)) continue;
|
||||
if (alt < 0 || alt > 100000) continue;
|
||||
|
||||
const vel = pv.velocity as any;
|
||||
const speed = Math.sqrt(vel.x ** 2 + vel.y ** 2 + vel.z ** 2) * 1000;
|
||||
const heading = (Math.atan2(vel.x, vel.z) * (180 / Math.PI) + 360) % 360;
|
||||
|
||||
results.push({
|
||||
noradId: rec.NORAD_CAT_ID,
|
||||
name: rec.OBJECT_NAME,
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
altitude: alt,
|
||||
heading,
|
||||
speed,
|
||||
group,
|
||||
country: rec.COUNTRY_CODE,
|
||||
objectType: rec.OBJECT_TYPE,
|
||||
period: rec.PERIOD,
|
||||
});
|
||||
} catch {
|
||||
// Error in propagation, skip
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function computeAndPublishPositions() {
|
||||
const now = new Date();
|
||||
const civilianObj: Record<string, any> = Object.create(null);
|
||||
const militaryObj: Record<string, any> = Object.create(null);
|
||||
let totalCivilian = 0;
|
||||
let totalMilitary = 0;
|
||||
|
||||
for (const group of DEFAULT_GROUPS) {
|
||||
const records = globalsTLECache.get(group);
|
||||
if (!records) continue;
|
||||
|
||||
const isMilitary = group === "military" || group === "resource";
|
||||
const positions = propagateAll(records, now, group);
|
||||
for (const p of positions) {
|
||||
if (isMilitary) {
|
||||
militaryObj[p.noradId] = p;
|
||||
totalMilitary++;
|
||||
} else {
|
||||
civilianObj[p.noradId] = p;
|
||||
totalCivilian++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalCivilian > 0) {
|
||||
await setLiveSnapshot('satellite', civilianObj, 60 * 60);
|
||||
}
|
||||
if (totalMilitary > 0) {
|
||||
await setLiveSnapshot('surveillance_satellites', militaryObj, 60 * 60);
|
||||
}
|
||||
}
|
||||
|
||||
let syncParams = {
|
||||
tleFetchInterval: 1000 * 60 * 60, // 1 hour
|
||||
publishInterval: 1000 * 15, // 15 seconds
|
||||
tleIntervalId: null as any,
|
||||
publishIntervalId: null as any
|
||||
};
|
||||
|
||||
export function startSatelliteSeeder() {
|
||||
console.log('[SatelliteSeeder] Starting satellite TLE seeder.');
|
||||
|
||||
refreshAllTLEs().then(() => {
|
||||
computeAndPublishPositions();
|
||||
syncParams.publishIntervalId = setInterval(computeAndPublishPositions, syncParams.publishInterval);
|
||||
});
|
||||
|
||||
syncParams.tleIntervalId = setInterval(refreshAllTLEs, syncParams.tleFetchInterval);
|
||||
}
|
||||
|
||||
registerSeeder({
|
||||
name: "satellite",
|
||||
init: startSatelliteSeeder
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import { db } from '../db';
|
||||
import { setLiveSnapshot } from '../redis';
|
||||
import { fetchWithTimeout, withRetry } from '../seed-utils';
|
||||
import { registerSeeder } from '../scheduler';
|
||||
|
||||
interface FIRMSRecord {
|
||||
latitude: number;
|
||||
longitude: number;
|
||||
bright_ti4: number;
|
||||
scan: number;
|
||||
track: number;
|
||||
acq_date: string;
|
||||
acq_time: string;
|
||||
satellite: string;
|
||||
confidence: string;
|
||||
version: string;
|
||||
bright_ti5: number;
|
||||
frp: number;
|
||||
daynight: string;
|
||||
}
|
||||
|
||||
function parseCSV(csv: string): FIRMSRecord[] {
|
||||
const lines = csv.trim().split("\n");
|
||||
if (lines.length < 2) return [];
|
||||
|
||||
const headers = lines[0].split(",").map((h) => h.trim());
|
||||
const records: FIRMSRecord[] = [];
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = lines[i].split(",");
|
||||
if (values.length < headers.length) continue;
|
||||
|
||||
const record: Record<string, string | number> = {};
|
||||
headers.forEach((header, idx) => {
|
||||
record[header] = values[idx]?.trim() || "";
|
||||
});
|
||||
|
||||
const lat = parseFloat(record["latitude"] as string);
|
||||
const lon = parseFloat(record["longitude"] as string);
|
||||
if (isNaN(lat) || isNaN(lon)) continue;
|
||||
|
||||
records.push({
|
||||
latitude: lat,
|
||||
longitude: lon,
|
||||
bright_ti4: parseFloat(record["bright_ti4"] as string) || 0,
|
||||
scan: parseFloat(record["scan"] as string) || 0,
|
||||
track: parseFloat(record["track"] as string) || 0,
|
||||
acq_date: (record["acq_date"] as string) || "",
|
||||
acq_time: (record["acq_time"] as string) || "",
|
||||
satellite: (record["satellite"] as string) || "",
|
||||
confidence: (record["confidence"] as string) || "",
|
||||
version: (record["version"] as string) || "",
|
||||
bright_ti5: parseFloat(record["bright_ti5"] as string) || 0,
|
||||
frp: parseFloat(record["frp"] as string) || 0,
|
||||
daynight: (record["daynight"] as string) || "",
|
||||
});
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
const insertWildfire = db.prepare('INSERT OR IGNORE INTO wildfires (id, payload, source_ts, fetched_at) VALUES (@id, @payload, @source_ts, @fetched_at)');
|
||||
|
||||
export async function seedWildfires() {
|
||||
console.log('[Wildfires] Polling NASA FIRMS...');
|
||||
|
||||
// Use open CSV feed
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/data/active_fire/suomi-npp-viirs-c2/csv/SUOMI_VIIRS_C2_Global_24h.csv`;
|
||||
|
||||
const res = await withRetry(() => fetchWithTimeout(url));
|
||||
const csv = await res.text();
|
||||
const fires = parseCSV(csv);
|
||||
const fetchedAt = Date.now();
|
||||
|
||||
if (fires.length === 0) {
|
||||
console.log('[Wildfires] No fires found or parsing failed.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Multi-Tier Clustering logic (Preserved from original implementation)
|
||||
const tiers = [
|
||||
{ level: 1, size: 2.0 }, // Macro: ~220km
|
||||
{ level: 2, size: 0.5 }, // Meso: ~55km
|
||||
{ level: 3, size: 0.05 }, // Micro: ~5.5km
|
||||
];
|
||||
|
||||
const allClusteredFires: (FIRMSRecord & { tier: number, id: string })[] = [];
|
||||
|
||||
for (const tier of tiers) {
|
||||
const clustered = new Map<string, FIRMSRecord & { tier: number, id: string }>();
|
||||
|
||||
for (const fire of fires) {
|
||||
const gridId = `${Math.floor(fire.latitude / tier.size)}_${Math.floor(fire.longitude / tier.size)}`;
|
||||
const existing = clustered.get(gridId);
|
||||
if (existing) {
|
||||
existing.frp += fire.frp;
|
||||
if (fire.confidence === "high" || (fire.confidence === "nominal" && existing.confidence === "low")) {
|
||||
existing.confidence = fire.confidence;
|
||||
}
|
||||
} else {
|
||||
// Creating a unique ID based on date, time, lat, lon to allow deduplication in SQLite
|
||||
const id = `firm_${fire.acq_date}_${fire.acq_time}_${Math.round(fire.latitude*1000)}_${Math.round(fire.longitude*1000)}_t${tier.level}`;
|
||||
clustered.set(gridId, { ...fire, tier: tier.level, id });
|
||||
}
|
||||
}
|
||||
allClusteredFires.push(...Array.from(clustered.values()));
|
||||
}
|
||||
|
||||
// Save to SQLite History
|
||||
let insertedCount = 0;
|
||||
const insertMany = db.transaction((firesList) => {
|
||||
for (const f of firesList) {
|
||||
// Create a pseudo-timestamp from acq_date and acq_time: "2024-04-01" "1430"
|
||||
const timeStr = f.acq_time.toString().padStart(4, '0');
|
||||
const tsStr = `${f.acq_date}T${timeStr.substring(0,2)}:${timeStr.substring(2,4)}:00Z`;
|
||||
let sourceTs = 0;
|
||||
try {
|
||||
sourceTs = new Date(tsStr).getTime();
|
||||
if (isNaN(sourceTs)) sourceTs = fetchedAt;
|
||||
} catch (e) {
|
||||
sourceTs = fetchedAt;
|
||||
}
|
||||
|
||||
const result = insertWildfire.run({
|
||||
id: f.id,
|
||||
payload: JSON.stringify(f),
|
||||
source_ts: sourceTs,
|
||||
fetched_at: fetchedAt
|
||||
});
|
||||
if (result.changes > 0) insertedCount++;
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(allClusteredFires);
|
||||
console.log(`[Wildfires] Parsed ${fires.length} raw fires -> Clustered ${allClusteredFires.length} points. Saved ${insertedCount} new to SQLite.`);
|
||||
|
||||
// Save full clustered map to Redis
|
||||
await setLiveSnapshot('wildfires', {
|
||||
source: "wildfires",
|
||||
fetchedAt: new Date().toISOString(),
|
||||
items: allClusteredFires,
|
||||
totalCount: allClusteredFires.length
|
||||
}, 1800); // 30 mins TTL
|
||||
}
|
||||
|
||||
// Register with scheduler
|
||||
registerSeeder({
|
||||
name: "wildfires",
|
||||
cron: "*/15 * * * *", // Every 15 minutes
|
||||
fn: seedWildfires
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
|
||||
import Fastify from 'fastify';
|
||||
import { prisma } from './prisma';
|
||||
import { startScheduler } from './scheduler';
|
||||
import { seederStatus } from './scheduler';
|
||||
|
||||
// Boot Fastify
|
||||
export const fastify = Fastify({
|
||||
logger: false // Keep it clean for the console
|
||||
});
|
||||
|
||||
import fastifyWebsocket from '@fastify/websocket';
|
||||
import fastifyRateLimit from '@fastify/rate-limit';
|
||||
import { handleConnection } from './websocket';
|
||||
|
||||
fastify.register(fastifyRateLimit, {
|
||||
max: 100,
|
||||
timeWindow: '1 minute'
|
||||
});
|
||||
|
||||
fastify.register(fastifyWebsocket);
|
||||
|
||||
fastify.register(async function (fastify) {
|
||||
// @ts-ignore - RouteShorthandOptions augmentation missing for websocket in strict mode
|
||||
fastify.get('/stream', { websocket: true }, (connection: any, req) => {
|
||||
handleConnection(connection, req);
|
||||
});
|
||||
});
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '5001', 10);
|
||||
|
||||
fastify.get('/health', async (request, reply) => {
|
||||
return {
|
||||
status: 'ok',
|
||||
engine: 'wwv-data-engine',
|
||||
timestamp: Date.now(),
|
||||
seeders: seederStatus
|
||||
};
|
||||
});
|
||||
|
||||
async function start() {
|
||||
try {
|
||||
// 1. Initialize Prisma Database (Supabase)
|
||||
try {
|
||||
await prisma.$connect();
|
||||
} catch (dbErr) {
|
||||
console.error('[Server] Prisma could not connect. Supabase historical data sync disabled.', dbErr instanceof Error ? dbErr.message : String(dbErr));
|
||||
}
|
||||
|
||||
// 1.5. Initialize Local SQLite (Fallback/Secondary History)
|
||||
const { initDB } = await import('./db.js');
|
||||
initDB();
|
||||
|
||||
// 2. Register Routes
|
||||
await import('./routes/index.js');
|
||||
|
||||
// 3. Start the Fastify API Server
|
||||
await fastify.listen({ port: PORT, host: '0.0.0.0' });
|
||||
console.log(`[Server] WWV Data Engine listening on port ${PORT}`);
|
||||
|
||||
// 4. Import seeder registry (this registers them)
|
||||
await import('./seeders/index.js');
|
||||
|
||||
// 5. Start the Cron Scheduler
|
||||
startScheduler();
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Server] Fatal error during startup:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
async function gracefulShutdown(signal: string) {
|
||||
console.log(`\n[Server] ${signal} received. Shutting down...`);
|
||||
await fastify.close();
|
||||
await prisma.$disconnect();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
||||
|
||||
start();
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { WebSocket } from 'ws';
|
||||
import { getLiveSnapshot } from './redis';
|
||||
|
||||
// Track active connections and their subscriptions
|
||||
const connections = new Set<WebSocket>();
|
||||
const subscriptions = new Map<WebSocket, Set<string>>();
|
||||
|
||||
export function handleConnection(connection: WebSocket, request: any) {
|
||||
// Option A (Secure Defaults): In a highly public plugin ecosystem,
|
||||
// checking the token is optional/opt-in via env vars.
|
||||
const requireToken = process.env.REQUIRE_WS_TOKEN === 'true';
|
||||
const providedToken = request.query?.token;
|
||||
|
||||
if (requireToken && providedToken !== process.env.API_SECRET) {
|
||||
connection.send(JSON.stringify({ error: 'Unauthorized: Invalid or missing token' }));
|
||||
connection.close(1008);
|
||||
return;
|
||||
}
|
||||
|
||||
connections.add(connection);
|
||||
subscriptions.set(connection, new Set());
|
||||
|
||||
connection.on('message', async (message: string) => {
|
||||
try {
|
||||
const data = JSON.parse(message);
|
||||
if (data.action === 'subscribe' && data.pluginId) {
|
||||
subscriptions.get(connection)?.add(data.pluginId);
|
||||
|
||||
// Push the most recent cached snapshot to the client immediately upon subscribing
|
||||
// This eliminates the "half a minute wait" for cron-driven seeders to flush
|
||||
const latestSnapshot = await getLiveSnapshot(data.pluginId);
|
||||
if (latestSnapshot && connections.has(connection)) {
|
||||
connection.send(JSON.stringify({
|
||||
type: 'data',
|
||||
pluginId: data.pluginId,
|
||||
payload: latestSnapshot
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (data.action === 'unsubscribe' && data.pluginId) {
|
||||
subscriptions.get(connection)?.delete(data.pluginId);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[WS] Invalid message error/fetch error', e);
|
||||
}
|
||||
});
|
||||
|
||||
connection.on('close', () => {
|
||||
connections.delete(connection);
|
||||
subscriptions.delete(connection);
|
||||
});
|
||||
}
|
||||
|
||||
export function broadcastPluginData(pluginId: string, payload: any) {
|
||||
const message = JSON.stringify({ type: 'data', pluginId, payload });
|
||||
for (const connection of connections) {
|
||||
if (subscriptions.get(connection)?.has(pluginId)) {
|
||||
connection.send(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const db = new Database('./data/engine.db');
|
||||
|
||||
try {
|
||||
const rows = db.prepare('SELECT payload FROM iranwar_events').all();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
JSON.parse(row.payload);
|
||||
} catch(err) {
|
||||
console.log('JSON Parse Failed for:', row.payload);
|
||||
}
|
||||
}
|
||||
console.log('Tested rows correctly');
|
||||
} catch(e) {
|
||||
console.error(e);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
fetch('http://localhost:5050/data/iranwarlive/history')
|
||||
.then(res => Promise.all([res.status, res.text()]))
|
||||
.then(([status, body]) => console.log('Status:', status, '\nBody:', body))
|
||||
.catch(console.error);
|
||||
@@ -0,0 +1,39 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
|
||||
async function fetchWithTimeout(url, options = {}, timeoutMs = 15000) {
|
||||
const controller = new AbortController();
|
||||
const id = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} - ${response.statusText}`);
|
||||
}
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(id);
|
||||
}
|
||||
}
|
||||
|
||||
async function test() {
|
||||
const OTX_BASE = 'https://otx.alienvault.com/api/v1';
|
||||
const apiKey = process.env.OTX_API_KEY;
|
||||
const since = new Date(Date.now() - 48 * 3600 * 1000).toISOString();
|
||||
const url = `${OTX_BASE}/pulses/subscribed?modified_since=${since}&limit=50`;
|
||||
|
||||
console.log('Testing fetchWithTimeout inside the application context...');
|
||||
console.log('Using API key:', apiKey ? apiKey.substring(0, 5) + '...' + apiKey.substring(apiKey.length - 5) : 'undefined');
|
||||
try {
|
||||
const res = await fetchWithTimeout(url, {
|
||||
headers: { 'X-OTX-API-KEY': apiKey }
|
||||
});
|
||||
console.log('Status:', res.status);
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -0,0 +1,14 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
import { seedCyberAttacks } from './src/seeders/cyberAttacks.ts';
|
||||
|
||||
async function test() {
|
||||
console.log('Testing seedCyberAttacks...');
|
||||
try {
|
||||
await seedCyberAttacks();
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -0,0 +1,14 @@
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config({ path: '../../.env.local' });
|
||||
import { seedCyberAttacks } from './src/seeders/cyberAttacks.js';
|
||||
|
||||
async function test() {
|
||||
console.log('Testing seedCyberAttacks...');
|
||||
try {
|
||||
await seedCyberAttacks();
|
||||
} catch(e) {
|
||||
console.error('Error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
test();
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2022",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import Redis from 'ioredis';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '../../.env.local') });
|
||||
|
||||
const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6379');
|
||||
|
||||
async function checkAviation() {
|
||||
console.log('Checking Aviation Live Snapshot...');
|
||||
const data = await redis.get('live_snapshot_aviation');
|
||||
if (!data) {
|
||||
console.log('No aviation snapshot found in Redis.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const snapshot = JSON.parse(data);
|
||||
const aircraft = Object.values(snapshot);
|
||||
if (aircraft.length === 0) {
|
||||
console.log('Snapshot is empty.');
|
||||
} else {
|
||||
const first = aircraft[0];
|
||||
console.log(`Found ${aircraft.length} aircraft.`);
|
||||
console.log(`Last updated: ${new Date(first.last_updated).toLocaleString()}`);
|
||||
console.log(`Current time: ${new Date().toLocaleString()}`);
|
||||
|
||||
const diffSec = (Date.now() - first.last_updated) / 1000;
|
||||
console.log(`Data age: ${diffSec.toFixed(2)} seconds`);
|
||||
|
||||
if (diffSec < 10) {
|
||||
console.log('✅ Polling verified: Data is fresh (under 10s).');
|
||||
} else {
|
||||
console.log('❌ Polling stall: Data is stale.');
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
checkAviation().catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@worldwideview/wwv-lib-aviation",
|
||||
"version": "1.0.0",
|
||||
"description": "Shared abstract base classes for WorldWideView aviation plugins (commercial, military, specialized)",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"files": [
|
||||
"src"
|
||||
],
|
||||
"keywords": [
|
||||
"worldwideview",
|
||||
"aviation",
|
||||
"aircraft",
|
||||
"base-plugin"
|
||||
],
|
||||
"author": "WorldWideView",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"@worldwideview/wwv-plugin-sdk": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import type {
|
||||
WorldPlugin,
|
||||
GeoEntity,
|
||||
TimeRange,
|
||||
PluginContext,
|
||||
LayerConfig,
|
||||
CesiumEntityOptions,
|
||||
SelectionBehavior,
|
||||
ServerPluginConfig,
|
||||
FilterDefinition
|
||||
} from "@worldwideview/wwv-plugin-sdk";
|
||||
|
||||
/**
|
||||
* Base abstraction for aviation-based plugins.
|
||||
* Inheriting plugins inherit default clustering, models, selection, and rendering logic.
|
||||
*/
|
||||
export abstract class BaseAviationPlugin implements WorldPlugin {
|
||||
abstract id: string;
|
||||
abstract name: string;
|
||||
abstract description: string;
|
||||
abstract icon: any;
|
||||
category = "aviation" as const;
|
||||
abstract version: string;
|
||||
|
||||
protected context: PluginContext | null = null;
|
||||
|
||||
// Configurable rendering defaults
|
||||
protected defaultLayerColor = "#3b82f6";
|
||||
protected defaultTrailColor = "#00fff7";
|
||||
protected iconUrl = "/plane-icon.svg";
|
||||
protected modelUrl = "/airplane/scene.gltf";
|
||||
|
||||
async initialize(ctx: PluginContext): Promise<void> { this.context = ctx; }
|
||||
destroy(): void { this.context = null; }
|
||||
|
||||
// Implementations must define how payload comes via fetch and websocket
|
||||
abstract fetch(timeRange: TimeRange): Promise<GeoEntity[]>;
|
||||
abstract mapWebsocketPayload(payload: any): GeoEntity[];
|
||||
|
||||
// Server backend configurations
|
||||
abstract getServerConfig(): ServerPluginConfig;
|
||||
|
||||
// Legends and Filters
|
||||
abstract getFilterDefinitions(): FilterDefinition[];
|
||||
abstract getLegend(): { label: string; color: string; filterId?: string; filterValue?: string }[];
|
||||
|
||||
getPollingInterval(): number {
|
||||
return 0; // Disabled in favor of WebSocket firehose
|
||||
}
|
||||
|
||||
getLayerConfig(): LayerConfig {
|
||||
return {
|
||||
color: this.defaultLayerColor,
|
||||
clusterEnabled: true,
|
||||
clusterDistance: 40,
|
||||
maxEntities: 5000
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Define the color based on the altitude (meters) specific to the aviation domain
|
||||
*/
|
||||
protected abstract getAltitudeColor(altitudeMeters: number | null): string;
|
||||
|
||||
renderEntity(entity: GeoEntity): CesiumEntityOptions {
|
||||
const alt = entity.properties.altitude_m as number | null;
|
||||
const isAirborne = !entity.properties.on_ground;
|
||||
|
||||
return {
|
||||
type: "model",
|
||||
iconUrl: this.iconUrl,
|
||||
size: isAirborne ? 8 : 5,
|
||||
modelUrl: this.modelUrl,
|
||||
modelScale: 2.56,
|
||||
modelMinPixelSize: 16,
|
||||
modelHeadingOffset: 180,
|
||||
color: this.getAltitudeColor(alt),
|
||||
rotation: entity.heading,
|
||||
labelText: entity.label || undefined,
|
||||
labelFont: "11px JetBrains Mono, monospace"
|
||||
};
|
||||
}
|
||||
|
||||
getSelectionBehavior(entity: GeoEntity): SelectionBehavior | null {
|
||||
if (entity.properties.on_ground) return null;
|
||||
return {
|
||||
showTrail: true,
|
||||
trailDurationSec: 60,
|
||||
trailStepSec: 5,
|
||||
trailColor: this.defaultTrailColor,
|
||||
flyToOffsetMultiplier: 3,
|
||||
flyToBaseDistance: 30000
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"declaration": true,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"jsx": "react-jsx"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Loaded 100 of 645 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user