- Auth, sync, messaging, media, rooms, federation, presence tests - Light checks every 5min, full checks every 60min - Reports to a dedicated Matrix monitoring room - Telegram alerts on consecutive failures - Docker container with healthcheck endpoint - Auto-creates monitoring room if not configured
615 lines
21 KiB
Python
615 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Matrix Homeserver Health Monitor -- tests the server from the inside out."""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from aiohttp import web, ClientSession, ClientTimeout
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
log = logging.getLogger("matrix-monitor")
|
|
|
|
# --- Config ---
|
|
HOMESERVER = os.getenv("MATRIX_HOMESERVER", "https://synapse.rewr.ca")
|
|
ACCESS_TOKEN = os.getenv("MATRIX_ACCESS_TOKEN", "")
|
|
MONITOR_ROOM = os.getenv("MATRIX_MONITOR_ROOM", "") # !room:server or #alias:server
|
|
TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
|
|
TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "")
|
|
TEST_MESSAGE_PREFIX = os.getenv("TEST_MSG_PREFIX", "[health-test]")
|
|
CHECK_INTERVAL_LIGHT = int(os.getenv("CHECK_INTERVAL_LIGHT", "300")) # 5 min
|
|
CHECK_INTERVAL_FULL = int(os.getenv("CHECK_INTERVAL_FULL", "3600")) # 60 min
|
|
HTTP_PORT = int(os.getenv("HTTP_PORT", "8090"))
|
|
REPORT_ON_FAILURE = os.getenv("REPORT_ON_FAILURE", "true").lower() == "true"
|
|
REPORT_ALWAYS = os.getenv("REPORT_ALWAYS", "false").lower() == "true"
|
|
|
|
API_TIMEOUT = ClientTimeout(total=30)
|
|
SESSION_TIMEOUT = ClientTimeout(total=60)
|
|
|
|
results: dict = {
|
|
"last_run": None,
|
|
"last_light": None,
|
|
"last_full": None,
|
|
"history": [],
|
|
"consecutive_failures": 0,
|
|
"current": [],
|
|
}
|
|
|
|
# --- Matrix API Helpers ---
|
|
async def api_get(session, homeserver, path, token, timeout=None):
|
|
url = f"{homeserver}/_matrix{path}" if not path.startswith("http") else path
|
|
headers = {"Authorization": f"Bearer {token}"}
|
|
async with session.get(url, headers=headers, timeout=(timeout or API_TIMEOUT)) as r:
|
|
return r.status, await r.json()
|
|
|
|
async def api_post(session, homeserver, path, token, body=None, timeout=None):
|
|
url = f"{homeserver}/_matrix{path}"
|
|
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
|
async with session.post(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
|
|
return r.status, await r.json()
|
|
|
|
async def api_put(session, homeserver, path, token, body=None, timeout=None):
|
|
url = f"{homeserver}/_matrix{path}"
|
|
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
|
async with session.put(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r:
|
|
return r.status, await r.json()
|
|
|
|
async def api_post_binary(session, homeserver, path, token, data, ctype, timeout=None):
|
|
url = f"{homeserver}/_matrix{path}"
|
|
headers = {"Authorization": f"Bearer {token}", "Content-Type": ctype}
|
|
async with session.post(url, headers=headers, data=data, timeout=(timeout or API_TIMEOUT)) as r:
|
|
return r.status, await r.json()
|
|
|
|
# --- Test Suite ---
|
|
async def test_auth(cs):
|
|
"""Verify auth token works and whoami returns valid info."""
|
|
start = time.monotonic()
|
|
status, data = await api_get(cs, HOMESERVER, "/client/versions", ACCESS_TOKEN)
|
|
elapsed = (time.monotonic() - start) * 1000
|
|
has_versions = status == 200 and "versions" in data
|
|
|
|
start2 = time.monotonic()
|
|
status2, data2 = await api_get(cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN)
|
|
elapsed2 = (time.monotonic() - start2) * 1000
|
|
has_whoami = status2 == 200 and "user_id" in data2
|
|
|
|
ok = has_versions and has_whoami
|
|
details = []
|
|
if has_versions:
|
|
details.append(f"versions OK ({elapsed:.0f}ms)")
|
|
if has_whoami:
|
|
details.append(f"whoami: {data2.get('user_id')} ({elapsed2:.0f}ms)")
|
|
|
|
return {
|
|
"name": "Auth",
|
|
"pass": ok,
|
|
"latency_ms": round(elapsed + elapsed2, 1),
|
|
"details": ", ".join(details) if ok else "FAILED",
|
|
}
|
|
|
|
async def test_sync(cs):
|
|
"""Measure sync response time."""
|
|
start = time.monotonic()
|
|
status, data = await api_get(
|
|
cs,
|
|
HOMESERVER,
|
|
"/client/v3/sync?timeout=1000",
|
|
ACCESS_TOKEN,
|
|
timeout=SESSION_TIMEOUT,
|
|
)
|
|
elapsed = (time.monotonic() - start) * 1000
|
|
|
|
ok = status == 200 and "next_batch" in data
|
|
return {
|
|
"name": "Sync",
|
|
"pass": ok,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": "sync OK" if ok else f"HTTP {status}",
|
|
}
|
|
|
|
async def test_messaging(cs):
|
|
"""Send a message to monitor room, measure response time."""
|
|
if not MONITOR_ROOM:
|
|
return {"name": "Messaging", "pass": False, "latency_ms": 0, "details": "No monitoring room configured"}
|
|
|
|
msg_id = f"msg-test-{int(time.time() * 1000)}"
|
|
message = f"{TEST_MESSAGE_PREFIX} test echo {msg_id}"
|
|
txn_id = f"test-{time.time()}"
|
|
|
|
start = time.monotonic()
|
|
status, data = await api_put(
|
|
cs,
|
|
HOMESERVER,
|
|
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
|
|
ACCESS_TOKEN,
|
|
{"msgtype": "m.text", "body": message},
|
|
)
|
|
send_time = (time.monotonic() - start) * 1000
|
|
|
|
if status == 200 and "event_id" in data:
|
|
return {
|
|
"name": "Messaging",
|
|
"pass": True,
|
|
"latency_ms": round(send_time, 1),
|
|
"details": f"sent {data['event_id']} in {send_time:.0f}ms",
|
|
}
|
|
return {
|
|
"name": "Messaging",
|
|
"pass": False,
|
|
"latency_ms": round(send_time, 1),
|
|
"details": f"HTTP {status}: {data.get('error', str(data))}",
|
|
}
|
|
|
|
async def test_media_upload(cs):
|
|
"""Upload a small PNG, verify upload, try download."""
|
|
# Valid 1x1 PNG (8 bytes palette, minimal header)
|
|
png_data = bytes([
|
|
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A,
|
|
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
|
|
])
|
|
|
|
start = time.monotonic()
|
|
status, data = await api_post_binary(
|
|
cs,
|
|
HOMESERVER,
|
|
f"/media/v3/upload?filename=monitor_test_{int(time.time())}.png",
|
|
ACCESS_TOKEN,
|
|
png_data,
|
|
"image/png",
|
|
)
|
|
elapsed = (time.monotonic() - start) * 1000
|
|
|
|
if status == 200 and "content_uri" in data:
|
|
mxc = data["content_uri"]
|
|
# Try downloading it back
|
|
server_name_and_id = mxc.replace("mxc://", "").split("/", 1)
|
|
if len(server_name_and_id) == 2:
|
|
server_name, media_id = server_name_and_id
|
|
url = f"{HOMESERVER}/_matrix/media/v3/download/{server_name}/{media_id}"
|
|
start2 = time.monotonic()
|
|
async with cs.get(url, timeout=API_TIMEOUT) as r:
|
|
dl_time = (time.monotonic() - start2) * 1000
|
|
dl_ok = r.status == 200
|
|
|
|
total = elapsed + dl_time
|
|
return {
|
|
"name": "Media",
|
|
"pass": True,
|
|
"latency_ms": round(total, 1),
|
|
"details": f"upload {elapsed:.0f}ms, download {dl_time:.0f}ms, download_ok={dl_ok}",
|
|
}
|
|
return {
|
|
"name": "Media",
|
|
"pass": True,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": f"upload OK ({elapsed:.0f}ms), download skipped (bad mxc format)",
|
|
}
|
|
return {
|
|
"name": "Media",
|
|
"pass": False,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": f"upload HTTP {status}: {data.get('error', str(data))}",
|
|
}
|
|
|
|
async def test_federation(cs):
|
|
"""Test federation by checking well-known and federation check."""
|
|
# Extract domain
|
|
domain = HOMESERVER.split("//")[1].split("/")[0].split(":")[0]
|
|
|
|
checks = []
|
|
|
|
# Well-known client
|
|
start1 = time.monotonic()
|
|
try:
|
|
async with cs.get(
|
|
f"https://{domain}/.well-known/matrix/client", timeout=API_TIMEOUT
|
|
) as r:
|
|
wk_time1 = (time.monotonic() - start1) * 1000
|
|
wk_client_ok = r.status == 200
|
|
checks.append(f"well-known/client: {'OK' if wk_client_ok else f'HTTP {r.status}'} ({wk_time1:.0f}ms)")
|
|
except Exception as e:
|
|
wk_time1 = (time.monotonic() - start1) * 1000
|
|
wk_client_ok = False
|
|
|
|
# Well-known server
|
|
start2 = time.monotonic()
|
|
try:
|
|
async with cs.get(
|
|
f"https://{domain}/.well-known/matrix/server", timeout=API_TIMEOUT
|
|
) as r:
|
|
wk_time2 = (time.monotonic() - start2) * 1000
|
|
wk_server_ok = r.status == 200
|
|
checks.append(f"well-known/server: {'OK' if wk_server_ok else f'HTTP {r.status}'} ({wk_time2:.0f}ms)")
|
|
except Exception as e:
|
|
wk_time2 = (time.monotonic() - start2) * 1000
|
|
wk_server_ok = False
|
|
|
|
# Federation check via matrix.org's API
|
|
start3 = time.monotonic()
|
|
try:
|
|
async with cs.get(
|
|
f"https://matrix.org/federationtester/api/report?server_name={domain}",
|
|
timeout=ClientTimeout(total=10),
|
|
) as r:
|
|
fedi_time = (time.monotonic() - start3) * 1000
|
|
if r.status == 200:
|
|
fed_data = await r.json()
|
|
all_ok = fed_data.get("AllChecksOK", False)
|
|
checks.append(f"federation: {'PASS' if all_ok else 'WARN'} ({fedi_time:.0f}ms)")
|
|
else:
|
|
checks.append(f"federation: HTTP {r.status} ({fedi_time:.0f}ms)")
|
|
except Exception as e:
|
|
fedi_time = (time.monotonic() - start3) * 1000
|
|
checks.append(f"federation: ERROR ({fedi_time:.0f}ms): {str(e)[:80]}")
|
|
|
|
total = sum(t for t in [wk_time1, wk_time2] if t > 0) + fedi_time
|
|
all_pass = wk_client_ok and wk_server_ok
|
|
return {
|
|
"name": "Federation",
|
|
"pass": all_pass,
|
|
"latency_ms": round(total, 1),
|
|
"details": "; ".join(checks),
|
|
}
|
|
|
|
async def test_rooms(cs):
|
|
"""Create a room, verify state, leave it."""
|
|
start = time.monotonic()
|
|
status, data = await api_post(
|
|
cs,
|
|
HOMESERVER,
|
|
"/client/v3/createRoom",
|
|
ACCESS_TOKEN,
|
|
{
|
|
"preset": "private_chat",
|
|
"name": f"monitor-test-{int(time.time())}",
|
|
},
|
|
)
|
|
create_time = (time.monotonic() - start) * 1000
|
|
|
|
if status != 200 or "room_id" not in data:
|
|
return {
|
|
"name": "Rooms",
|
|
"pass": False,
|
|
"latency_ms": round(create_time, 1),
|
|
"details": f"create failed: HTTP {status}",
|
|
}
|
|
|
|
room_id = data["room_id"]
|
|
ok = True
|
|
details = [f"created {room_id} ({create_time:.0f}ms)"]
|
|
|
|
# Check room state
|
|
start2 = time.monotonic()
|
|
status2, data2 = await api_get(
|
|
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/state", ACCESS_TOKEN
|
|
)
|
|
state_time = (time.monotonic() - start2) * 1000
|
|
if status2 == 200:
|
|
details.append(f"state OK ({state_time:.0f}ms, {len(data2)} events)")
|
|
else:
|
|
ok = False
|
|
details.append(f"state HTTP {status2}")
|
|
|
|
# Leave the room
|
|
status3, _ = await api_post(
|
|
cs, HOMESERVER, f"/client/v3/rooms/{room_id}/leave", ACCESS_TOKEN, {}
|
|
)
|
|
if status3 != 200:
|
|
ok = False
|
|
details.append(f"leave HTTP {status3}")
|
|
else:
|
|
details.append("left OK")
|
|
|
|
total = create_time + state_time
|
|
return {
|
|
"name": "Rooms",
|
|
"pass": ok,
|
|
"latency_ms": round(total, 1),
|
|
"details": "; ".join(details),
|
|
}
|
|
|
|
async def test_presence(cs):
|
|
"""Set and verify presence state."""
|
|
start = time.monotonic()
|
|
status, _ = await api_put(
|
|
cs,
|
|
HOMESERVER,
|
|
"/client/v3/presence/@monitor:rewr.ca/status",
|
|
ACCESS_TOKEN,
|
|
{"presence": "online", "status_msg": "monitoring"},
|
|
)
|
|
elapsed = (time.monotonic() - start) * 1000
|
|
|
|
# Try getting own user ID for presence check
|
|
status2, data2 = await api_get(
|
|
cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN
|
|
)
|
|
if status2 == 200 and "user_id" in data2:
|
|
user_id = data2["user_id"]
|
|
import urllib.parse
|
|
encoded_id = urllib.parse.quote(user_id)
|
|
status3, _ = await api_get(
|
|
cs, HOMESERVER, f"/client/v3/presence/{encoded_id}/status", ACCESS_TOKEN
|
|
)
|
|
if status3 == 200:
|
|
ok = status == 200
|
|
return {
|
|
"name": "Presence",
|
|
"pass": ok,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": f"presence set + fetch OK ({elapsed:.0f}ms)",
|
|
}
|
|
return {
|
|
"name": "Presence",
|
|
"pass": False,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": f"set OK ({elapsed:.0f}ms), fetch HTTP {status3}",
|
|
}
|
|
return {
|
|
"name": "Presence",
|
|
"pass": status == 200,
|
|
"latency_ms": round(elapsed, 1),
|
|
"details": f"presence set" if status == 200 else f"HTTP {status}",
|
|
}
|
|
|
|
# --- Room Management ---
|
|
async def ensure_monitor_room(cs):
|
|
"""Create the monitor room if not configured."""
|
|
global MONITOR_ROOM
|
|
if MONITOR_ROOM:
|
|
log.info(f"Using configured monitor room: {MONITOR_ROOM}")
|
|
return
|
|
|
|
log.info("No monitor room configured, creating one...")
|
|
status, data = await api_post(
|
|
cs,
|
|
HOMESERVER,
|
|
"/client/v3/createRoom",
|
|
ACCESS_TOKEN,
|
|
{
|
|
"name": "Matrix Health Monitor",
|
|
"topic": "Automated health check results from Matrix-Health-Monitor",
|
|
"initial_state": [
|
|
{
|
|
"type": "m.room.history_visibility",
|
|
"state_key": "",
|
|
"content": {"history_visibility": "shared"},
|
|
},
|
|
{
|
|
"type": "m.room.guest_access",
|
|
"state_key": "",
|
|
"content": {"guest_access": "forbidden"},
|
|
},
|
|
],
|
|
},
|
|
)
|
|
if status == 200 and "room_id" in data:
|
|
MONITOR_ROOM = data["room_id"]
|
|
log.info(f"Created monitor room: {MONITOR_ROOM}")
|
|
else:
|
|
log.error(f"Failed to create monitor room: {status} {data}")
|
|
|
|
# --- Report ---
|
|
def build_report(test_results, run_type="full"):
|
|
lines = []
|
|
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
lines.append(f"Matrix Health Check ({run_type}) -- {ts}")
|
|
lines.append("")
|
|
|
|
all_pass = True
|
|
for r in test_results:
|
|
icon = "PASS" if r["pass"] else "FAIL"
|
|
if not r["pass"]:
|
|
all_pass = False
|
|
lines.append(f" [{icon}] {r['name']} -- {r['latency_ms']}ms -- {r['details']}")
|
|
|
|
summary = "ALL GREEN" if all_pass else "ISSUES DETECTED"
|
|
lines.insert(1, f"Summary: {summary}")
|
|
if not all_pass:
|
|
failed = [r["name"] for r in test_results if not r["pass"]]
|
|
lines.insert(2, f"Failed: {', '.join(failed)}")
|
|
|
|
return "\n".join(lines), all_pass
|
|
|
|
async def send_matrix_report(text):
|
|
"""Send a report to the monitor room."""
|
|
if not MONITOR_ROOM:
|
|
return
|
|
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
|
txn_id = f"report-{time.time()}"
|
|
await api_put(
|
|
cs,
|
|
HOMESERVER,
|
|
f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}",
|
|
ACCESS_TOKEN,
|
|
{
|
|
"msgtype": "m.text",
|
|
"body": text,
|
|
"format": "org.matrix.custom.html",
|
|
"formatted_body": text.replace("\n", "<br>").replace(" ", " "),
|
|
},
|
|
)
|
|
|
|
async def send_telegram_alert(text):
|
|
"""Send an alert to Telegram."""
|
|
if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
|
|
return
|
|
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
|
try:
|
|
async with ClientSession(timeout=ClientTimeout(total=10)) as cs:
|
|
await cs.post(url, json={
|
|
"chat_id": TELEGRAM_CHAT_ID,
|
|
"text": text,
|
|
"parse_mode": "Markdown",
|
|
})
|
|
except Exception as e:
|
|
log.error(f"Telegram alert failed: {e}")
|
|
|
|
# --- Test Runners ---
|
|
async def run_light_test():
|
|
"""Lightweight: auth, sync, messaging."""
|
|
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
|
auth = await test_auth(cs)
|
|
sync = await test_sync(cs)
|
|
msg = await test_messaging(cs)
|
|
return [auth, sync, msg]
|
|
|
|
async def run_full_test():
|
|
"""Full suite: everything."""
|
|
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
|
auth = await test_auth(cs)
|
|
sync = await test_sync(cs)
|
|
msg = await test_messaging(cs)
|
|
media = await test_media_upload(cs)
|
|
fedi = await test_federation(cs)
|
|
rooms = await test_rooms(cs)
|
|
presence = await test_presence(cs)
|
|
return [auth, sync, msg, media, fedi, rooms, presence]
|
|
|
|
async def run_and_report(run_type="full"):
|
|
if run_type == "full":
|
|
tests = await run_full_test()
|
|
else:
|
|
tests = await run_light_test()
|
|
|
|
results["last_run"] = datetime.now(timezone.utc).isoformat()
|
|
results["last_light" if run_type == "light" else "last_full"] = results["last_run"]
|
|
results["history"].append({
|
|
"time": results["last_run"],
|
|
"type": run_type,
|
|
"results": tests,
|
|
})
|
|
if len(results["history"]) > 100:
|
|
results["history"] = results["history"][-100:]
|
|
|
|
report_text, all_pass = build_report(tests, run_type)
|
|
|
|
# Save log
|
|
log_file = f"/data/health-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json"
|
|
try:
|
|
with open(log_file, "w") as f:
|
|
json.dump({
|
|
"time": results["last_run"],
|
|
"type": run_type,
|
|
"tests": tests,
|
|
}, f, indent=2)
|
|
except Exception as e:
|
|
log.error(f"Failed to write log: {e}")
|
|
|
|
# Track consecutive failures
|
|
if all_pass:
|
|
results["consecutive_failures"] = 0
|
|
else:
|
|
results["consecutive_failures"] += 1
|
|
|
|
results["current"] = tests
|
|
|
|
# Report logic
|
|
failed_tests = [t for t in tests if not t["pass"]]
|
|
if all_pass and REPORT_ALWAYS:
|
|
await send_matrix_report(report_text)
|
|
elif not all_pass:
|
|
log.warning(f"Health check ({run_type}) failed: {[t['name'] for t in failed_tests]}")
|
|
await send_matrix_report(report_text)
|
|
if results["consecutive_failures"] >= 2:
|
|
await send_telegram_alert(
|
|
f"Matrix Health Alert ({run_type}): "
|
|
f"{len(failed_tests)} tests failing -- "
|
|
f"{', '.join(t['name'] for t in failed_tests)}"
|
|
)
|
|
else:
|
|
log.info(f"Health check ({run_type}) passed")
|
|
|
|
return tests
|
|
|
|
# --- HTTP Server ---
|
|
async def handle_health(request):
|
|
current = results.get("current", [])
|
|
if not current:
|
|
return web.json_response({"status": "no data yet"}, 200)
|
|
|
|
all_pass = all(t["pass"] for t in current)
|
|
status_code = 200 if all_pass else 503
|
|
return web.json_response({
|
|
"status": "healthy" if all_pass else "degraded",
|
|
"last_run": results.get("last_run"),
|
|
"tests": {t["name"]: {
|
|
"pass": t["pass"],
|
|
"latency_ms": t["latency_ms"],
|
|
} for t in current},
|
|
"consecutive_failures": results.get("consecutive_failures", 0),
|
|
}, status=status_code)
|
|
|
|
async def handle_status(request):
|
|
return web.json_response({
|
|
"last_run": results.get("last_run"),
|
|
"last_light": results.get("last_light"),
|
|
"last_full": results.get("last_full"),
|
|
"current": results.get("current", []),
|
|
"history_count": len(results.get("history", [])),
|
|
"consecutive_failures": results.get("consecutive_failures", 0),
|
|
})
|
|
|
|
# --- Main ---
|
|
async def monitoring_loop():
|
|
log.info(f"Health check intervals: light={CHECK_INTERVAL_LIGHT}s, full={CHECK_INTERVAL_FULL}s")
|
|
|
|
# Run initial full check
|
|
log.info("Running initial full health check...")
|
|
await run_and_report("full")
|
|
|
|
last_light = time.monotonic()
|
|
last_full = time.monotonic()
|
|
|
|
while True:
|
|
await asyncio.sleep(10)
|
|
now = time.monotonic()
|
|
|
|
if now - last_full >= CHECK_INTERVAL_FULL:
|
|
log.info("Running full health check...")
|
|
await run_and_report("full")
|
|
last_full = now
|
|
last_light = now
|
|
elif now - last_light >= CHECK_INTERVAL_LIGHT:
|
|
log.info("Running light health check...")
|
|
await run_and_report("light")
|
|
last_light = now
|
|
|
|
async def start_http_server():
|
|
app = web.Application()
|
|
app.router.add_get("/health", handle_health)
|
|
app.router.add_get("/status", handle_status)
|
|
runner = web.AppRunner(app)
|
|
await runner.setup()
|
|
site = web.TCPSite(runner, "0.0.0.0", HTTP_PORT)
|
|
await site.start()
|
|
log.info(f"HTTP health endpoint on port {HTTP_PORT}")
|
|
|
|
async def main():
|
|
if not ACCESS_TOKEN:
|
|
log.error("MATRIX_ACCESS_TOKEN is required (env: MATRIX_ACCESS_TOKEN)")
|
|
return
|
|
|
|
# Ensure monitor room exists
|
|
async with ClientSession(timeout=API_TIMEOUT) as cs:
|
|
await ensure_monitor_room(cs)
|
|
|
|
log.info(f"Matrix Health Monitor starting")
|
|
log.info(f"Homeserver: {HOMESERVER}")
|
|
log.info(f"Monitor room: {MONITOR_ROOM or 'none (Matrix reports disabled)'}")
|
|
log.info(f"Telegram alerts: {'enabled' if TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID else 'disabled'}")
|
|
|
|
await asyncio.gather(
|
|
start_http_server(),
|
|
monitoring_loop(),
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|