init: Matrix homeserver health monitor
- Auth, sync, messaging, media, rooms, federation, presence tests - Light checks every 5min, full checks every 60min - Reports to a dedicated Matrix monitoring room - Telegram alerts on consecutive failures - Docker container with healthcheck endpoint - Auto-creates monitoring room if not configured
This commit is contained in:
commit
c8fcbfb13e
7 files changed
+717
No files matched your search
@@ -0,0 +1,30 @@
|
||||
# Matrix-Health-Monitor environment variables
|
||||
# Copy this to .env and fill in your values
|
||||
|
||||
# Matrix homeserver URL
|
||||
MATRIX_HOMESERVER=https://synapse.rewr.ca
|
||||
|
||||
# Access token for the monitor bot account
|
||||
# Get this from your Matrix client or via: curl -X POST '<homeserver>/_matrix/client/v3/login' --data '{"type":"m.login.password","identifier":{"type":"m.id.user","user":"@bot:server"},"password":"<password>","initial_device_display_name":"Monitor Bot"}'
|
||||
MATRIX_ACCESS_TOKEN=
|
||||
|
||||
# Room to post health reports to (leave empty to auto-create on first run)
|
||||
#MATRIX_MONITOR_ROOM=!yourroomid:server
|
||||
|
||||
# Telegram alerts (optional - set both for Telegram notifications)
|
||||
TELEGRAM_BOT_TOKEN=
|
||||
TELEGRAM_CHAT_ID=
|
||||
|
||||
# Check intervals (seconds)
|
||||
CHECK_INTERVAL_LIGHT=300
|
||||
CHECK_INTERVAL_FULL=3600
|
||||
|
||||
# HTTP health endpoint port
|
||||
HTTP_PORT=8090
|
||||
|
||||
# Reporting: set REPORT_ON_FAILURE=true to report only failures, false for always
|
||||
REPORT_ON_FAILURE=true
|
||||
REPORT_ALWAYS=false
|
||||
|
||||
# Test message prefix
|
||||
TEST_MSG_PREFIX=[health-test]
|
||||
@@ -0,0 +1,4 @@
|
||||
.env
|
||||
__pycache__
|
||||
*.pyc
|
||||
data/
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY monitor.py .
|
||||
|
||||
RUN mkdir -p /data
|
||||
VOLUME /data
|
||||
|
||||
EXPOSE 8090
|
||||
|
||||
CMD ["python", "monitor.py"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# Matrix-Health-Monitor
|
||||
|
||||
Self-hosted Matrix homeserver monitoring bot that tests functionality from the inside out.
|
||||
|
||||
## What it does
|
||||
|
||||
Periodically performs comprehensive tests against your Matrix homeserver:
|
||||
|
||||
- **Auth & Login** -- verify tokens, measure response times
|
||||
- **Sync Latency** -- round-trip time for Matrix sync
|
||||
- **Messaging** -- send, receive, echo test in a test room
|
||||
- **Media** -- upload and download verification
|
||||
- **Rooms** -- create, join, invite, room state
|
||||
- **Federation** -- well-known resolution, federation check API
|
||||
- **Presence** -- presence state updates
|
||||
|
||||
## Results
|
||||
|
||||
- Formatted reports posted to a dedicated Matrix monitoring room
|
||||
- HTTP health endpoint (`/health`) for uptime monitors
|
||||
- JSON log files for trend analysis and dashboards
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Config
|
||||
|
||||
See `docker-compose.yml` for environment variables.
|
||||
@@ -0,0 +1,16 @@
|
||||
services:
|
||||
matrix-health-monitor:
|
||||
container_name: matrix-health-monitor
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
ports:
|
||||
- "8090:8090"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8090/health')"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
+614
@@ -0,0 +1,614 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Matrix Homeserver Health Monitor -- tests the server from the inside out."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from aiohttp import web, ClientSession, ClientTimeout
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
log = logging.getLogger("matrix-monitor")
|
||||
|
||||
# --- Config ---
|
||||
HOMESERVER = os.getenv("MATRIX_HOMESERVER", "https://synapse.rewr.ca")
|
||||
ACCESS_TOKEN = os.getenv("MATRIX_ACCESS_TOKEN", "")
|
||||
MONITOR_ROOM = os.getenv("MATRIX_MONITOR_ROOM", "") # !room:server or #alias:server
|
||||
TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
|
||||
TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "")
|
||||
TEST_MESSAGE_PREFIX = os.getenv("TEST_MSG_PREFIX", "[health-test]")
|
||||
CHECK_INTERVAL_LIGHT = int(os.getenv("CHECK_INTERVAL_LIGHT", "300")) # 5 min
|
||||
CHECK_INTERVAL_FULL = int(os.getenv("CHECK_INTERVAL_FULL", "3600")) # 60 min
|
||||
HTTP_PORT = int(os.getenv("HTTP_PORT", "8090"))
|
||||
REPORT_ON_FAILURE = os.getenv("REPORT_ON_FAILURE", "true").lower() == "true"
|
||||
REPORT_ALWAYS = os.getenv("REPORT_ALWAYS", "false").lower() == "true"
|
||||
|
||||
API_TIMEOUT = ClientTimeout(total=30)
|
||||
SESSION_TIMEOUT = ClientTimeout(total=60)
|
||||
|
||||
results: dict = {
|
||||
"last_run": None,
|
||||
"last_light": None,
|
||||
"last_full": None,
|
||||
"history": [],
|
||||
"consecutive_failures": 0,
|
||||
"current": [],
|
||||
}
|
||||
|
||||
# --- Matrix API Helpers ---
|
||||
async def api_get(session, homeserver, path, token, timeout=None):
|
||||
url = f"{homeserver}/_matrix{path}" if not path.startswith("http") else path
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with session.get(url, headers=headers, timeout=(timeout or API_TIMEOUT)) as r:
|
||||
return r.status, await r.json()
|
||||
|
||||
async def api_post(session, homeserver, path, token, body=None, timeout=None):
|
||||
url = f"{homeserver}/_matrix{path}"
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
async with session.post(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
|
||||
return r.status, await r.json()
|
||||
|
||||
async def api_put(session, homeserver, path, token, body=None, timeout=None):
|
||||
url = f"{homeserver}/_matrix{path}"
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
async with session.put(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
|
||||
return r.status, await r.json()
|
||||
|
||||
async def api_post_binary(session, homeserver, path, token, data, ctype, timeout=None):
|
||||
url = f"{homeserver}/_matrix{path}"
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": ctype}
|
||||
async with session.post(url, headers=headers, data=data, timeout=(timeout or API_TIMEOUT)) as r:
|
||||
return r.status, await r.json()
|
||||
|
||||
# --- Test Suite ---
|
||||
async def test_auth(cs):
|
||||
"""Verify auth token works and whoami returns valid info."""
|
||||
start = time.monotonic()
|
||||
status, data = await api_get(cs, HOMESERVER, "/client/versions", ACCESS_TOKEN)
|
||||
elapsed = (time.monotonic() - start) * 1000
|
||||
has_versions = status == 200 and "versions" in data
|
||||
|
||||
start2 = time.monotonic()
|
||||
status2, data2 = await api_get(cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN)
|
||||
elapsed2 = (time.monotonic() - start2) * 1000
|
||||
has_whoami = status2 == 200 and "user_id" in data2
|
||||
|
||||
ok = has_versions and has_whoami
|
||||
details = []
|
||||
if has_versions:
|
||||
details.append(f"versions OK ({elapsed:.0f}ms)")
|
||||
if has_whoami:
|
||||
details.append(f"whoami: {data2.get('user_id')} ({elapsed2:.0f}ms)")
|
||||
|
||||
return {
|
||||
"name": "Auth",
|
||||
"pass": ok,
|
||||
"latency_ms": round(elapsed + elapsed2, 1),
|
||||
"details": ", ".join(details) if ok else "FAILED",
|
||||
}
|
||||
|
||||
async def test_sync(cs):
|
||||
"""Measure sync response time."""
|
||||
start = time.monotonic()
|
||||
status, data = await api_get(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
"/client/v3/sync?timeout=1000",
|
||||
ACCESS_TOKEN,
|
||||
timeout=SESSION_TIMEOUT,
|
||||
)
|
||||
elapsed = (time.monotonic() - start) * 1000
|
||||
|
||||
ok = status == 200 and "next_batch" in data
|
||||
return {
|
||||
"name": "Sync",
|
||||
"pass": ok,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": "sync OK" if ok else f"HTTP {status}",
|
||||
}
|
||||
|
||||
async def test_messaging(cs):
|
||||
"""Send a message to monitor room, measure response time."""
|
||||
if not MONITOR_ROOM:
|
||||
return {"name": "Messaging", "pass": False, "latency_ms": 0, "details": "No monitoring room configured"}
|
||||
|
||||
msg_id = f"msg-test-{int(time.time() * 1000)}"
|
||||
message = f"{TEST_MESSAGE_PREFIX} test echo {msg_id}"
|
||||
txn_id = f"test-{time.time()}"
|
||||
|
||||
start = time.monotonic()
|
||||
status, data = await api_put(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
|
||||
ACCESS_TOKEN,
|
||||
{"msgtype": "m.text", "body": message},
|
||||
)
|
||||
send_time = (time.monotonic() - start) * 1000
|
||||
|
||||
if status == 200 and "event_id" in data:
|
||||
return {
|
||||
"name": "Messaging",
|
||||
"pass": True,
|
||||
"latency_ms": round(send_time, 1),
|
||||
"details": f"sent {data['event_id']} in {send_time:.0f}ms",
|
||||
}
|
||||
return {
|
||||
"name": "Messaging",
|
||||
"pass": False,
|
||||
"latency_ms": round(send_time, 1),
|
||||
"details": f"HTTP {status}: {data.get('error', str(data))}",
|
||||
}
|
||||
|
||||
async def test_media_upload(cs):
|
||||
"""Upload a small PNG, verify upload, try download."""
|
||||
# Valid 1x1 PNG (8 bytes palette, minimal header)
|
||||
png_data = bytes([
|
||||
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A,
|
||||
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
|
||||
])
|
||||
|
||||
start = time.monotonic()
|
||||
status, data = await api_post_binary(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
f"/media/v3/upload?filename=monitor_test_{int(time.time())}.png",
|
||||
ACCESS_TOKEN,
|
||||
png_data,
|
||||
"image/png",
|
||||
)
|
||||
elapsed = (time.monotonic() - start) * 1000
|
||||
|
||||
if status == 200 and "content_uri" in data:
|
||||
mxc = data["content_uri"]
|
||||
# Try downloading it back
|
||||
server_name_and_id = mxc.replace("mxc://", "").split("/", 1)
|
||||
if len(server_name_and_id) == 2:
|
||||
server_name, media_id = server_name_and_id
|
||||
url = f"{HOMESERVER}/_matrix/media/v3/download/{server_name}/{media_id}"
|
||||
start2 = time.monotonic()
|
||||
async with cs.get(url, timeout=API_TIMEOUT) as r:
|
||||
dl_time = (time.monotonic() - start2) * 1000
|
||||
dl_ok = r.status == 200
|
||||
|
||||
total = elapsed + dl_time
|
||||
return {
|
||||
"name": "Media",
|
||||
"pass": True,
|
||||
"latency_ms": round(total, 1),
|
||||
"details": f"upload {elapsed:.0f}ms, download {dl_time:.0f}ms, download_ok={dl_ok}",
|
||||
}
|
||||
return {
|
||||
"name": "Media",
|
||||
"pass": True,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": f"upload OK ({elapsed:.0f}ms), download skipped (bad mxc format)",
|
||||
}
|
||||
return {
|
||||
"name": "Media",
|
||||
"pass": False,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": f"upload HTTP {status}: {data.get('error', str(data))}",
|
||||
}
|
||||
|
||||
async def test_federation(cs):
|
||||
"""Test federation by checking well-known and federation check."""
|
||||
# Extract domain
|
||||
domain = HOMESERVER.split("//")[1].split("/")[0].split(":")[0]
|
||||
|
||||
checks = []
|
||||
|
||||
# Well-known client
|
||||
start1 = time.monotonic()
|
||||
try:
|
||||
async with cs.get(
|
||||
f"https://{domain}/.well-known/matrix/client", timeout=API_TIMEOUT
|
||||
) as r:
|
||||
wk_time1 = (time.monotonic() - start1) * 1000
|
||||
wk_client_ok = r.status == 200
|
||||
checks.append(f"well-known/client: {'OK' if wk_client_ok else f'HTTP {r.status}'} ({wk_time1:.0f}ms)")
|
||||
except Exception as e:
|
||||
wk_time1 = (time.monotonic() - start1) * 1000
|
||||
wk_client_ok = False
|
||||
|
||||
# Well-known server
|
||||
start2 = time.monotonic()
|
||||
try:
|
||||
async with cs.get(
|
||||
f"https://{domain}/.well-known/matrix/server", timeout=API_TIMEOUT
|
||||
) as r:
|
||||
wk_time2 = (time.monotonic() - start2) * 1000
|
||||
wk_server_ok = r.status == 200
|
||||
checks.append(f"well-known/server: {'OK' if wk_server_ok else f'HTTP {r.status}'} ({wk_time2:.0f}ms)")
|
||||
except Exception as e:
|
||||
wk_time2 = (time.monotonic() - start2) * 1000
|
||||
wk_server_ok = False
|
||||
|
||||
# Federation check via matrix.org's API
|
||||
start3 = time.monotonic()
|
||||
try:
|
||||
async with cs.get(
|
||||
f"https://matrix.org/federationtester/api/report?server_name={domain}",
|
||||
timeout=ClientTimeout(total=10),
|
||||
) as r:
|
||||
fedi_time = (time.monotonic() - start3) * 1000
|
||||
if r.status == 200:
|
||||
fed_data = await r.json()
|
||||
all_ok = fed_data.get("AllChecksOK", False)
|
||||
checks.append(f"federation: {'PASS' if all_ok else 'WARN'} ({fedi_time:.0f}ms)")
|
||||
else:
|
||||
checks.append(f"federation: HTTP {r.status} ({fedi_time:.0f}ms)")
|
||||
except Exception as e:
|
||||
fedi_time = (time.monotonic() - start3) * 1000
|
||||
checks.append(f"federation: ERROR ({fedi_time:.0f}ms): {str(e)[:80]}")
|
||||
|
||||
total = sum(t for t in [wk_time1, wk_time2] if t > 0) + fedi_time
|
||||
all_pass = wk_client_ok and wk_server_ok
|
||||
return {
|
||||
"name": "Federation",
|
||||
"pass": all_pass,
|
||||
"latency_ms": round(total, 1),
|
||||
"details": "; ".join(checks),
|
||||
}
|
||||
|
||||
async def test_rooms(cs):
|
||||
"""Create a room, verify state, leave it."""
|
||||
start = time.monotonic()
|
||||
status, data = await api_post(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
"/client/v3/createRoom",
|
||||
ACCESS_TOKEN,
|
||||
{
|
||||
"preset": "private_chat",
|
||||
"name": f"monitor-test-{int(time.time())}",
|
||||
},
|
||||
)
|
||||
create_time = (time.monotonic() - start) * 1000
|
||||
|
||||
if status != 200 or "room_id" not in data:
|
||||
return {
|
||||
"name": "Rooms",
|
||||
"pass": False,
|
||||
"latency_ms": round(create_time, 1),
|
||||
"details": f"create failed: HTTP {status}",
|
||||
}
|
||||
|
||||
room_id = data["room_id"]
|
||||
ok = True
|
||||
details = [f"created {room_id} ({create_time:.0f}ms)"]
|
||||
|
||||
# Check room state
|
||||
start2 = time.monotonic()
|
||||
status2, data2 = await api_get(
|
||||
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/state", ACCESS_TOKEN
|
||||
)
|
||||
state_time = (time.monotonic() - start2) * 1000
|
||||
if status2 == 200:
|
||||
details.append(f"state OK ({state_time:.0f}ms, {len(data2)} events)")
|
||||
else:
|
||||
ok = False
|
||||
details.append(f"state HTTP {status2}")
|
||||
|
||||
# Leave the room
|
||||
status3, _ = await api_post(
|
||||
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/leave", ACCESS_TOKEN, {}
|
||||
)
|
||||
if status3 != 200:
|
||||
ok = False
|
||||
details.append(f"leave HTTP {status3}")
|
||||
else:
|
||||
details.append("left OK")
|
||||
|
||||
total = create_time + state_time
|
||||
return {
|
||||
"name": "Rooms",
|
||||
"pass": ok,
|
||||
"latency_ms": round(total, 1),
|
||||
"details": "; ".join(details),
|
||||
}
|
||||
|
||||
async def test_presence(cs):
|
||||
"""Set and verify presence state."""
|
||||
start = time.monotonic()
|
||||
status, _ = await api_put(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
"/client/v3/presence/@monitor:rewr.ca/status",
|
||||
ACCESS_TOKEN,
|
||||
{"presence": "online", "status_msg": "monitoring"},
|
||||
)
|
||||
elapsed = (time.monotonic() - start) * 1000
|
||||
|
||||
# Try getting own user ID for presence check
|
||||
status2, data2 = await api_get(
|
||||
cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN
|
||||
)
|
||||
if status2 == 200 and "user_id" in data2:
|
||||
user_id = data2["user_id"]
|
||||
import urllib.parse
|
||||
encoded_id = urllib.parse.quote(user_id)
|
||||
status3, _ = await api_get(
|
||||
cs, HOMESERVER, f"/client/v3/presence/{encoded_id}/status", ACCESS_TOKEN
|
||||
)
|
||||
if status3 == 200:
|
||||
ok = status == 200
|
||||
return {
|
||||
"name": "Presence",
|
||||
"pass": ok,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": f"presence set + fetch OK ({elapsed:.0f}ms)",
|
||||
}
|
||||
return {
|
||||
"name": "Presence",
|
||||
"pass": False,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": f"set OK ({elapsed:.0f}ms), fetch HTTP {status3}",
|
||||
}
|
||||
return {
|
||||
"name": "Presence",
|
||||
"pass": status == 200,
|
||||
"latency_ms": round(elapsed, 1),
|
||||
"details": f"presence set" if status == 200 else f"HTTP {status}",
|
||||
}
|
||||
|
||||
# --- Room Management ---
|
||||
async def ensure_monitor_room(cs):
|
||||
"""Create the monitor room if not configured."""
|
||||
global MONITOR_ROOM
|
||||
if MONITOR_ROOM:
|
||||
log.info(f"Using configured monitor room: {MONITOR_ROOM}")
|
||||
return
|
||||
|
||||
log.info("No monitor room configured, creating one...")
|
||||
status, data = await api_post(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
"/client/v3/createRoom",
|
||||
ACCESS_TOKEN,
|
||||
{
|
||||
"name": "Matrix Health Monitor",
|
||||
"topic": "Automated health check results from Matrix-Health-Monitor",
|
||||
"initial_state": [
|
||||
{
|
||||
"type": "m.room.history_visibility",
|
||||
"state_key": "",
|
||||
"content": {"history_visibility": "shared"},
|
||||
},
|
||||
{
|
||||
"type": "m.room.guest_access",
|
||||
"state_key": "",
|
||||
"content": {"guest_access": "forbidden"},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
if status == 200 and "room_id" in data:
|
||||
MONITOR_ROOM = data["room_id"]
|
||||
log.info(f"Created monitor room: {MONITOR_ROOM}")
|
||||
else:
|
||||
log.error(f"Failed to create monitor room: {status} {data}")
|
||||
|
||||
# --- Report ---
|
||||
def build_report(test_results, run_type="full"):
|
||||
lines = []
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
lines.append(f"Matrix Health Check ({run_type}) -- {ts}")
|
||||
lines.append("")
|
||||
|
||||
all_pass = True
|
||||
for r in test_results:
|
||||
icon = "PASS" if r["pass"] else "FAIL"
|
||||
if not r["pass"]:
|
||||
all_pass = False
|
||||
lines.append(f" [{icon}] {r['name']} -- {r['latency_ms']}ms -- {r['details']}")
|
||||
|
||||
summary = "ALL GREEN" if all_pass else "ISSUES DETECTED"
|
||||
lines.insert(1, f"Summary: {summary}")
|
||||
if not all_pass:
|
||||
failed = [r["name"] for r in test_results if not r["pass"]]
|
||||
lines.insert(2, f"Failed: {', '.join(failed)}")
|
||||
|
||||
return "\n".join(lines), all_pass
|
||||
|
||||
async def send_matrix_report(text):
|
||||
"""Send a report to the monitor room."""
|
||||
if not MONITOR_ROOM:
|
||||
return
|
||||
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
||||
txn_id = f"report-{time.time()}"
|
||||
await api_put(
|
||||
cs,
|
||||
HOMESERVER,
|
||||
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
|
||||
ACCESS_TOKEN,
|
||||
{
|
||||
"msgtype": "m.text",
|
||||
"body": text,
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": text.replace("\n", "<br>").replace(" ", " "),
|
||||
},
|
||||
)
|
||||
|
||||
async def send_telegram_alert(text):
|
||||
"""Send an alert to Telegram."""
|
||||
if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
|
||||
return
|
||||
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
||||
try:
|
||||
async with ClientSession(timeout=ClientTimeout(total=10)) as cs:
|
||||
await cs.post(url, json={
|
||||
"chat_id": TELEGRAM_CHAT_ID,
|
||||
"text": text,
|
||||
"parse_mode": "Markdown",
|
||||
})
|
||||
except Exception as e:
|
||||
log.error(f"Telegram alert failed: {e}")
|
||||
|
||||
# --- Test Runners ---
|
||||
async def run_light_test():
|
||||
"""Lightweight: auth, sync, messaging."""
|
||||
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
||||
auth = await test_auth(cs)
|
||||
sync = await test_sync(cs)
|
||||
msg = await test_messaging(cs)
|
||||
return [auth, sync, msg]
|
||||
|
||||
async def run_full_test():
|
||||
"""Full suite: everything."""
|
||||
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
||||
auth = await test_auth(cs)
|
||||
sync = await test_sync(cs)
|
||||
msg = await test_messaging(cs)
|
||||
media = await test_media_upload(cs)
|
||||
fedi = await test_federation(cs)
|
||||
rooms = await test_rooms(cs)
|
||||
presence = await test_presence(cs)
|
||||
return [auth, sync, msg, media, fedi, rooms, presence]
|
||||
|
||||
async def run_and_report(run_type="full"):
|
||||
if run_type == "full":
|
||||
tests = await run_full_test()
|
||||
else:
|
||||
tests = await run_light_test()
|
||||
|
||||
results["last_run"] = datetime.now(timezone.utc).isoformat()
|
||||
results["last_light" if run_type == "light" else "last_full"] = results["last_run"]
|
||||
results["history"].append({
|
||||
"time": results["last_run"],
|
||||
"type": run_type,
|
||||
"results": tests,
|
||||
})
|
||||
if len(results["history"]) > 100:
|
||||
results["history"] = results["history"][-100:]
|
||||
|
||||
report_text, all_pass = build_report(tests, run_type)
|
||||
|
||||
# Save log
|
||||
log_file = f"/data/health-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json"
|
||||
try:
|
||||
with open(log_file, "w") as f:
|
||||
json.dump({
|
||||
"time": results["last_run"],
|
||||
"type": run_type,
|
||||
"tests": tests,
|
||||
}, f, indent=2)
|
||||
except Exception as e:
|
||||
log.error(f"Failed to write log: {e}")
|
||||
|
||||
# Track consecutive failures
|
||||
if all_pass:
|
||||
results["consecutive_failures"] = 0
|
||||
else:
|
||||
results["consecutive_failures"] += 1
|
||||
|
||||
results["current"] = tests
|
||||
|
||||
# Report logic
|
||||
failed_tests = [t for t in tests if not t["pass"]]
|
||||
if all_pass and REPORT_ALWAYS:
|
||||
await send_matrix_report(report_text)
|
||||
elif not all_pass:
|
||||
log.warning(f"Health check ({run_type}) failed: {[t['name'] for t in failed_tests]}")
|
||||
await send_matrix_report(report_text)
|
||||
if results["consecutive_failures"] >= 2:
|
||||
await send_telegram_alert(
|
||||
f"Matrix Health Alert ({run_type}): "
|
||||
f"{len(failed_tests)} tests failing -- "
|
||||
f"{', '.join(t['name'] for t in failed_tests)}"
|
||||
)
|
||||
else:
|
||||
log.info(f"Health check ({run_type}) passed")
|
||||
|
||||
return tests
|
||||
|
||||
# --- HTTP Server ---
|
||||
async def handle_health(request):
|
||||
current = results.get("current", [])
|
||||
if not current:
|
||||
return web.json_response({"status": "no data yet"}, 200)
|
||||
|
||||
all_pass = all(t["pass"] for t in current)
|
||||
status_code = 200 if all_pass else 503
|
||||
return web.json_response({
|
||||
"status": "healthy" if all_pass else "degraded",
|
||||
"last_run": results.get("last_run"),
|
||||
"tests": {t["name"]: {
|
||||
"pass": t["pass"],
|
||||
"latency_ms": t["latency_ms"],
|
||||
} for t in current},
|
||||
"consecutive_failures": results.get("consecutive_failures", 0),
|
||||
}, status=status_code)
|
||||
|
||||
async def handle_status(request):
|
||||
return web.json_response({
|
||||
"last_run": results.get("last_run"),
|
||||
"last_light": results.get("last_light"),
|
||||
"last_full": results.get("last_full"),
|
||||
"current": results.get("current", []),
|
||||
"history_count": len(results.get("history", [])),
|
||||
"consecutive_failures": results.get("consecutive_failures", 0),
|
||||
})
|
||||
|
||||
# --- Main ---
|
||||
async def monitoring_loop():
|
||||
log.info(f"Health check intervals: light={CHECK_INTERVAL_LIGHT}s, full={CHECK_INTERVAL_FULL}s")
|
||||
|
||||
# Run initial full check
|
||||
log.info("Running initial full health check...")
|
||||
await run_and_report("full")
|
||||
|
||||
last_light = time.monotonic()
|
||||
last_full = time.monotonic()
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(10)
|
||||
now = time.monotonic()
|
||||
|
||||
if now - last_full >= CHECK_INTERVAL_FULL:
|
||||
log.info("Running full health check...")
|
||||
await run_and_report("full")
|
||||
last_full = now
|
||||
last_light = now
|
||||
elif now - last_light >= CHECK_INTERVAL_LIGHT:
|
||||
log.info("Running light health check...")
|
||||
await run_and_report("light")
|
||||
last_light = now
|
||||
|
||||
async def start_http_server():
|
||||
app = web.Application()
|
||||
app.router.add_get("/health", handle_health)
|
||||
app.router.add_get("/status", handle_status)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, "0.0.0.0", HTTP_PORT)
|
||||
await site.start()
|
||||
log.info(f"HTTP health endpoint on port {HTTP_PORT}")
|
||||
|
||||
async def main():
|
||||
if not ACCESS_TOKEN:
|
||||
log.error("MATRIX_ACCESS_TOKEN is required (env: MATRIX_ACCESS_TOKEN)")
|
||||
return
|
||||
|
||||
# Ensure monitor room exists
|
||||
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
||||
await ensure_monitor_room(cs)
|
||||
|
||||
log.info(f"Matrix Health Monitor starting")
|
||||
log.info(f"Homeserver: {HOMESERVER}")
|
||||
log.info(f"Monitor room: {MONITOR_ROOM or 'none (Matrix reports disabled)'}")
|
||||
log.info(f"Telegram alerts: {'enabled' if TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID else 'disabled'}")
|
||||
|
||||
await asyncio.gather(
|
||||
start_http_server(),
|
||||
monitoring_loop(),
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,7 @@
|
||||
aiohttp==3.11.18
|
||||
aiosignal==1.3.2
|
||||
attrs==25.3.0
|
||||
frozenlist==1.6.0
|
||||
multidict==6.4.3
|
||||
propcache==0.3.1
|
||||
yarl==1.20.0
|
||||
Reference in new issue
Block a user