init: Matrix homeserver health monitor

- Auth, sync, messaging, media, rooms, federation, presence tests
- Light checks every 5min, full checks every 60min
- Reports to a dedicated Matrix monitoring room
- Telegram alerts on consecutive failures
- Docker container with healthcheck endpoint
- Auto-creates monitoring room if not configured
This commit is contained in:
Arvex committed 2026-04-05 09:58:28 -06:00
commit c8fcbfb13e
7 files changed
+717

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
# Matrix-Health-Monitor environment variables
# Copy this to .env and fill in your values
# Matrix homeserver URL
MATRIX_HOMESERVER=https://synapse.rewr.ca
# Access token for the monitor bot account
# Get this from your Matrix client or via: curl -X POST '<homeserver>/_matrix/client/v3/login' --data '{"type":"m.login.password","identifier":{"type":"m.id.user","user":"@bot:server"},"password":"<password>","initial_device_display_name":"Monitor Bot"}'
MATRIX_ACCESS_TOKEN=
# Room to post health reports to (leave empty to auto-create on first run)
#MATRIX_MONITOR_ROOM=!yourroomid:server
# Telegram alerts (optional - set both for Telegram notifications)
TELEGRAM_BOT_TOKEN=
TELEGRAM_CHAT_ID=
# Check intervals (seconds)
CHECK_INTERVAL_LIGHT=300
CHECK_INTERVAL_FULL=3600
# HTTP health endpoint port
HTTP_PORT=8090
# Reporting: set REPORT_ON_FAILURE=true to report only failures, false for always
REPORT_ON_FAILURE=true
REPORT_ALWAYS=false
# Test message prefix
TEST_MSG_PREFIX=[health-test]
+4
View File
@@ -0,0 +1,4 @@
.env
__pycache__
*.pyc
data/
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY monitor.py .
RUN mkdir -p /data
VOLUME /data
EXPOSE 8090
CMD ["python", "monitor.py"]
+31
View File
@@ -0,0 +1,31 @@
# Matrix-Health-Monitor
Self-hosted Matrix homeserver monitoring bot that tests functionality from the inside out.
## What it does
Periodically performs comprehensive tests against your Matrix homeserver:
- **Auth & Login** -- verify tokens, measure response times
- **Sync Latency** -- round-trip time for Matrix sync
- **Messaging** -- send, receive, echo test in a test room
- **Media** -- upload and download verification
- **Rooms** -- create, join, invite, room state
- **Federation** -- well-known resolution, federation check API
- **Presence** -- presence state updates
## Results
- Formatted reports posted to a dedicated Matrix monitoring room
- HTTP health endpoint (`/health`) for uptime monitors
- JSON log files for trend analysis and dashboards
## Deploy
```bash
docker compose up -d
```
## Config
See `docker-compose.yml` for environment variables.
+16
View File
@@ -0,0 +1,16 @@
services:
matrix-health-monitor:
container_name: matrix-health-monitor
build: .
restart: unless-stopped
env_file: .env
ports:
- "8090:8090"
volumes:
- ./data:/data
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8090/health')"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
+614
View File
@@ -0,0 +1,614 @@
#!/usr/bin/env python3
"""Matrix Homeserver Health Monitor -- tests the server from the inside out."""
import asyncio
import json
import logging
import os
import time
from datetime import datetime, timezone
from aiohttp import web, ClientSession, ClientTimeout
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
log = logging.getLogger("matrix-monitor")
# --- Config ---
HOMESERVER = os.getenv("MATRIX_HOMESERVER", "https://synapse.rewr.ca")
ACCESS_TOKEN = os.getenv("MATRIX_ACCESS_TOKEN", "")
MONITOR_ROOM = os.getenv("MATRIX_MONITOR_ROOM", "") # !room:server or #alias:server
TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "")
TEST_MESSAGE_PREFIX = os.getenv("TEST_MSG_PREFIX", "[health-test]")
CHECK_INTERVAL_LIGHT = int(os.getenv("CHECK_INTERVAL_LIGHT", "300")) # 5 min
CHECK_INTERVAL_FULL = int(os.getenv("CHECK_INTERVAL_FULL", "3600")) # 60 min
HTTP_PORT = int(os.getenv("HTTP_PORT", "8090"))
REPORT_ON_FAILURE = os.getenv("REPORT_ON_FAILURE", "true").lower() == "true"
REPORT_ALWAYS = os.getenv("REPORT_ALWAYS", "false").lower() == "true"
API_TIMEOUT = ClientTimeout(total=30)
SESSION_TIMEOUT = ClientTimeout(total=60)
results: dict = {
"last_run": None,
"last_light": None,
"last_full": None,
"history": [],
"consecutive_failures": 0,
"current": [],
}
# --- Matrix API Helpers ---
async def api_get(session, homeserver, path, token, timeout=None):
url = f"{homeserver}/_matrix{path}" if not path.startswith("http") else path
headers = {"Authorization": f"Bearer {token}"}
async with session.get(url, headers=headers, timeout=(timeout or API_TIMEOUT)) as r:
return r.status, await r.json()
async def api_post(session, homeserver, path, token, body=None, timeout=None):
url = f"{homeserver}/_matrix{path}"
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
async with session.post(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
return r.status, await r.json()
async def api_put(session, homeserver, path, token, body=None, timeout=None):
url = f"{homeserver}/_matrix{path}"
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
async with session.put(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
return r.status, await r.json()
async def api_post_binary(session, homeserver, path, token, data, ctype, timeout=None):
url = f"{homeserver}/_matrix{path}"
headers = {"Authorization": f"Bearer {token}", "Content-Type": ctype}
async with session.post(url, headers=headers, data=data, timeout=(timeout or API_TIMEOUT)) as r:
return r.status, await r.json()
# --- Test Suite ---
async def test_auth(cs):
"""Verify auth token works and whoami returns valid info."""
start = time.monotonic()
status, data = await api_get(cs, HOMESERVER, "/client/versions", ACCESS_TOKEN)
elapsed = (time.monotonic() - start) * 1000
has_versions = status == 200 and "versions" in data
start2 = time.monotonic()
status2, data2 = await api_get(cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN)
elapsed2 = (time.monotonic() - start2) * 1000
has_whoami = status2 == 200 and "user_id" in data2
ok = has_versions and has_whoami
details = []
if has_versions:
details.append(f"versions OK ({elapsed:.0f}ms)")
if has_whoami:
details.append(f"whoami: {data2.get('user_id')} ({elapsed2:.0f}ms)")
return {
"name": "Auth",
"pass": ok,
"latency_ms": round(elapsed + elapsed2, 1),
"details": ", ".join(details) if ok else "FAILED",
}
async def test_sync(cs):
"""Measure sync response time."""
start = time.monotonic()
status, data = await api_get(
cs,
HOMESERVER,
"/client/v3/sync?timeout=1000",
ACCESS_TOKEN,
timeout=SESSION_TIMEOUT,
)
elapsed = (time.monotonic() - start) * 1000
ok = status == 200 and "next_batch" in data
return {
"name": "Sync",
"pass": ok,
"latency_ms": round(elapsed, 1),
"details": "sync OK" if ok else f"HTTP {status}",
}
async def test_messaging(cs):
"""Send a message to monitor room, measure response time."""
if not MONITOR_ROOM:
return {"name": "Messaging", "pass": False, "latency_ms": 0, "details": "No monitoring room configured"}
msg_id = f"msg-test-{int(time.time() * 1000)}"
message = f"{TEST_MESSAGE_PREFIX} test echo {msg_id}"
txn_id = f"test-{time.time()}"
start = time.monotonic()
status, data = await api_put(
cs,
HOMESERVER,
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
ACCESS_TOKEN,
{"msgtype": "m.text", "body": message},
)
send_time = (time.monotonic() - start) * 1000
if status == 200 and "event_id" in data:
return {
"name": "Messaging",
"pass": True,
"latency_ms": round(send_time, 1),
"details": f"sent {data['event_id']} in {send_time:.0f}ms",
}
return {
"name": "Messaging",
"pass": False,
"latency_ms": round(send_time, 1),
"details": f"HTTP {status}: {data.get('error', str(data))}",
}
async def test_media_upload(cs):
"""Upload a small PNG, verify upload, try download."""
# Valid 1x1 PNG (8 bytes palette, minimal header)
png_data = bytes([
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A,
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
])
start = time.monotonic()
status, data = await api_post_binary(
cs,
HOMESERVER,
f"/media/v3/upload?filename=monitor_test_{int(time.time())}.png",
ACCESS_TOKEN,
png_data,
"image/png",
)
elapsed = (time.monotonic() - start) * 1000
if status == 200 and "content_uri" in data:
mxc = data["content_uri"]
# Try downloading it back
server_name_and_id = mxc.replace("mxc://", "").split("/", 1)
if len(server_name_and_id) == 2:
server_name, media_id = server_name_and_id
url = f"{HOMESERVER}/_matrix/media/v3/download/{server_name}/{media_id}"
start2 = time.monotonic()
async with cs.get(url, timeout=API_TIMEOUT) as r:
dl_time = (time.monotonic() - start2) * 1000
dl_ok = r.status == 200
total = elapsed + dl_time
return {
"name": "Media",
"pass": True,
"latency_ms": round(total, 1),
"details": f"upload {elapsed:.0f}ms, download {dl_time:.0f}ms, download_ok={dl_ok}",
}
return {
"name": "Media",
"pass": True,
"latency_ms": round(elapsed, 1),
"details": f"upload OK ({elapsed:.0f}ms), download skipped (bad mxc format)",
}
return {
"name": "Media",
"pass": False,
"latency_ms": round(elapsed, 1),
"details": f"upload HTTP {status}: {data.get('error', str(data))}",
}
async def test_federation(cs):
"""Test federation by checking well-known and federation check."""
# Extract domain
domain = HOMESERVER.split("//")[1].split("/")[0].split(":")[0]
checks = []
# Well-known client
start1 = time.monotonic()
try:
async with cs.get(
f"https://{domain}/.well-known/matrix/client", timeout=API_TIMEOUT
) as r:
wk_time1 = (time.monotonic() - start1) * 1000
wk_client_ok = r.status == 200
checks.append(f"well-known/client: {'OK' if wk_client_ok else f'HTTP {r.status}'} ({wk_time1:.0f}ms)")
except Exception as e:
wk_time1 = (time.monotonic() - start1) * 1000
wk_client_ok = False
# Well-known server
start2 = time.monotonic()
try:
async with cs.get(
f"https://{domain}/.well-known/matrix/server", timeout=API_TIMEOUT
) as r:
wk_time2 = (time.monotonic() - start2) * 1000
wk_server_ok = r.status == 200
checks.append(f"well-known/server: {'OK' if wk_server_ok else f'HTTP {r.status}'} ({wk_time2:.0f}ms)")
except Exception as e:
wk_time2 = (time.monotonic() - start2) * 1000
wk_server_ok = False
# Federation check via matrix.org's API
start3 = time.monotonic()
try:
async with cs.get(
f"https://matrix.org/federationtester/api/report?server_name={domain}",
timeout=ClientTimeout(total=10),
) as r:
fedi_time = (time.monotonic() - start3) * 1000
if r.status == 200:
fed_data = await r.json()
all_ok = fed_data.get("AllChecksOK", False)
checks.append(f"federation: {'PASS' if all_ok else 'WARN'} ({fedi_time:.0f}ms)")
else:
checks.append(f"federation: HTTP {r.status} ({fedi_time:.0f}ms)")
except Exception as e:
fedi_time = (time.monotonic() - start3) * 1000
checks.append(f"federation: ERROR ({fedi_time:.0f}ms): {str(e)[:80]}")
total = sum(t for t in [wk_time1, wk_time2] if t > 0) + fedi_time
all_pass = wk_client_ok and wk_server_ok
return {
"name": "Federation",
"pass": all_pass,
"latency_ms": round(total, 1),
"details": "; ".join(checks),
}
async def test_rooms(cs):
"""Create a room, verify state, leave it."""
start = time.monotonic()
status, data = await api_post(
cs,
HOMESERVER,
"/client/v3/createRoom",
ACCESS_TOKEN,
{
"preset": "private_chat",
"name": f"monitor-test-{int(time.time())}",
},
)
create_time = (time.monotonic() - start) * 1000
if status != 200 or "room_id" not in data:
return {
"name": "Rooms",
"pass": False,
"latency_ms": round(create_time, 1),
"details": f"create failed: HTTP {status}",
}
room_id = data["room_id"]
ok = True
details = [f"created {room_id} ({create_time:.0f}ms)"]
# Check room state
start2 = time.monotonic()
status2, data2 = await api_get(
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/state", ACCESS_TOKEN
)
state_time = (time.monotonic() - start2) * 1000
if status2 == 200:
details.append(f"state OK ({state_time:.0f}ms, {len(data2)} events)")
else:
ok = False
details.append(f"state HTTP {status2}")
# Leave the room
status3, _ = await api_post(
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/leave", ACCESS_TOKEN, {}
)
if status3 != 200:
ok = False
details.append(f"leave HTTP {status3}")
else:
details.append("left OK")
total = create_time + state_time
return {
"name": "Rooms",
"pass": ok,
"latency_ms": round(total, 1),
"details": "; ".join(details),
}
async def test_presence(cs):
"""Set and verify presence state."""
start = time.monotonic()
status, _ = await api_put(
cs,
HOMESERVER,
"/client/v3/presence/@monitor:rewr.ca/status",
ACCESS_TOKEN,
{"presence": "online", "status_msg": "monitoring"},
)
elapsed = (time.monotonic() - start) * 1000
# Try getting own user ID for presence check
status2, data2 = await api_get(
cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN
)
if status2 == 200 and "user_id" in data2:
user_id = data2["user_id"]
import urllib.parse
encoded_id = urllib.parse.quote(user_id)
status3, _ = await api_get(
cs, HOMESERVER, f"/client/v3/presence/{encoded_id}/status", ACCESS_TOKEN
)
if status3 == 200:
ok = status == 200
return {
"name": "Presence",
"pass": ok,
"latency_ms": round(elapsed, 1),
"details": f"presence set + fetch OK ({elapsed:.0f}ms)",
}
return {
"name": "Presence",
"pass": False,
"latency_ms": round(elapsed, 1),
"details": f"set OK ({elapsed:.0f}ms), fetch HTTP {status3}",
}
return {
"name": "Presence",
"pass": status == 200,
"latency_ms": round(elapsed, 1),
"details": f"presence set" if status == 200 else f"HTTP {status}",
}
# --- Room Management ---
async def ensure_monitor_room(cs):
"""Create the monitor room if not configured."""
global MONITOR_ROOM
if MONITOR_ROOM:
log.info(f"Using configured monitor room: {MONITOR_ROOM}")
return
log.info("No monitor room configured, creating one...")
status, data = await api_post(
cs,
HOMESERVER,
"/client/v3/createRoom",
ACCESS_TOKEN,
{
"name": "Matrix Health Monitor",
"topic": "Automated health check results from Matrix-Health-Monitor",
"initial_state": [
{
"type": "m.room.history_visibility",
"state_key": "",
"content": {"history_visibility": "shared"},
},
{
"type": "m.room.guest_access",
"state_key": "",
"content": {"guest_access": "forbidden"},
},
],
},
)
if status == 200 and "room_id" in data:
MONITOR_ROOM = data["room_id"]
log.info(f"Created monitor room: {MONITOR_ROOM}")
else:
log.error(f"Failed to create monitor room: {status} {data}")
# --- Report ---
def build_report(test_results, run_type="full"):
lines = []
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
lines.append(f"Matrix Health Check ({run_type}) -- {ts}")
lines.append("")
all_pass = True
for r in test_results:
icon = "PASS" if r["pass"] else "FAIL"
if not r["pass"]:
all_pass = False
lines.append(f" [{icon}] {r['name']} -- {r['latency_ms']}ms -- {r['details']}")
summary = "ALL GREEN" if all_pass else "ISSUES DETECTED"
lines.insert(1, f"Summary: {summary}")
if not all_pass:
failed = [r["name"] for r in test_results if not r["pass"]]
lines.insert(2, f"Failed: {', '.join(failed)}")
return "\n".join(lines), all_pass
async def send_matrix_report(text):
"""Send a report to the monitor room."""
if not MONITOR_ROOM:
return
async with ClientSession(timeout=API_TIMEOUT) as cs:
txn_id = f"report-{time.time()}"
await api_put(
cs,
HOMESERVER,
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
ACCESS_TOKEN,
{
"msgtype": "m.text",
"body": text,
"format": "org.matrix.custom.html",
"formatted_body": text.replace("\n", "<br>").replace(" ", "&nbsp;&nbsp;"),
},
)
async def send_telegram_alert(text):
"""Send an alert to Telegram."""
if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
return
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
try:
async with ClientSession(timeout=ClientTimeout(total=10)) as cs:
await cs.post(url, json={
"chat_id": TELEGRAM_CHAT_ID,
"text": text,
"parse_mode": "Markdown",
})
except Exception as e:
log.error(f"Telegram alert failed: {e}")
# --- Test Runners ---
async def run_light_test():
"""Lightweight: auth, sync, messaging."""
async with ClientSession(timeout=API_TIMEOUT) as cs:
auth = await test_auth(cs)
sync = await test_sync(cs)
msg = await test_messaging(cs)
return [auth, sync, msg]
async def run_full_test():
"""Full suite: everything."""
async with ClientSession(timeout=API_TIMEOUT) as cs:
auth = await test_auth(cs)
sync = await test_sync(cs)
msg = await test_messaging(cs)
media = await test_media_upload(cs)
fedi = await test_federation(cs)
rooms = await test_rooms(cs)
presence = await test_presence(cs)
return [auth, sync, msg, media, fedi, rooms, presence]
async def run_and_report(run_type="full"):
if run_type == "full":
tests = await run_full_test()
else:
tests = await run_light_test()
results["last_run"] = datetime.now(timezone.utc).isoformat()
results["last_light" if run_type == "light" else "last_full"] = results["last_run"]
results["history"].append({
"time": results["last_run"],
"type": run_type,
"results": tests,
})
if len(results["history"]) > 100:
results["history"] = results["history"][-100:]
report_text, all_pass = build_report(tests, run_type)
# Save log
log_file = f"/data/health-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json"
try:
with open(log_file, "w") as f:
json.dump({
"time": results["last_run"],
"type": run_type,
"tests": tests,
}, f, indent=2)
except Exception as e:
log.error(f"Failed to write log: {e}")
# Track consecutive failures
if all_pass:
results["consecutive_failures"] = 0
else:
results["consecutive_failures"] += 1
results["current"] = tests
# Report logic
failed_tests = [t for t in tests if not t["pass"]]
if all_pass and REPORT_ALWAYS:
await send_matrix_report(report_text)
elif not all_pass:
log.warning(f"Health check ({run_type}) failed: {[t['name'] for t in failed_tests]}")
await send_matrix_report(report_text)
if results["consecutive_failures"] >= 2:
await send_telegram_alert(
f"Matrix Health Alert ({run_type}): "
f"{len(failed_tests)} tests failing -- "
f"{', '.join(t['name'] for t in failed_tests)}"
)
else:
log.info(f"Health check ({run_type}) passed")
return tests
# --- HTTP Server ---
async def handle_health(request):
current = results.get("current", [])
if not current:
return web.json_response({"status": "no data yet"}, 200)
all_pass = all(t["pass"] for t in current)
status_code = 200 if all_pass else 503
return web.json_response({
"status": "healthy" if all_pass else "degraded",
"last_run": results.get("last_run"),
"tests": {t["name"]: {
"pass": t["pass"],
"latency_ms": t["latency_ms"],
} for t in current},
"consecutive_failures": results.get("consecutive_failures", 0),
}, status=status_code)
async def handle_status(request):
return web.json_response({
"last_run": results.get("last_run"),
"last_light": results.get("last_light"),
"last_full": results.get("last_full"),
"current": results.get("current", []),
"history_count": len(results.get("history", [])),
"consecutive_failures": results.get("consecutive_failures", 0),
})
# --- Main ---
async def monitoring_loop():
log.info(f"Health check intervals: light={CHECK_INTERVAL_LIGHT}s, full={CHECK_INTERVAL_FULL}s")
# Run initial full check
log.info("Running initial full health check...")
await run_and_report("full")
last_light = time.monotonic()
last_full = time.monotonic()
while True:
await asyncio.sleep(10)
now = time.monotonic()
if now - last_full >= CHECK_INTERVAL_FULL:
log.info("Running full health check...")
await run_and_report("full")
last_full = now
last_light = now
elif now - last_light >= CHECK_INTERVAL_LIGHT:
log.info("Running light health check...")
await run_and_report("light")
last_light = now
async def start_http_server():
app = web.Application()
app.router.add_get("/health", handle_health)
app.router.add_get("/status", handle_status)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "0.0.0.0", HTTP_PORT)
await site.start()
log.info(f"HTTP health endpoint on port {HTTP_PORT}")
async def main():
if not ACCESS_TOKEN:
log.error("MATRIX_ACCESS_TOKEN is required (env: MATRIX_ACCESS_TOKEN)")
return
# Ensure monitor room exists
async with ClientSession(timeout=API_TIMEOUT) as cs:
await ensure_monitor_room(cs)
log.info(f"Matrix Health Monitor starting")
log.info(f"Homeserver: {HOMESERVER}")
log.info(f"Monitor room: {MONITOR_ROOM or 'none (Matrix reports disabled)'}")
log.info(f"Telegram alerts: {'enabled' if TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID else 'disabled'}")
await asyncio.gather(
start_http_server(),
monitoring_loop(),
)
if __name__ == "__main__":
asyncio.run(main())
+7
View File
@@ -0,0 +1,7 @@
aiohttp==3.11.18
aiosignal==1.3.2
attrs==25.3.0
frozenlist==1.6.0
multidict==6.4.3
propcache==0.3.1
yarl==1.20.0