26 Commits

Author SHA1 Message Date
Maxim
14d57120cd Fix traffic map zoom and add paginated history with local city GeoIP 2026-10-15 20:26:14 +03:00
Maxim
c3b2a10d2f Add light traffic theme, interactive map and metric detail dialogs 2026-10-15 20:11:51 +03:00
Maxim
72bebdc2eb Support Alpine OpenRC agents and discover Docker NPMplus and CrowdSec 2026-10-14 20:37:34 +03:00
Maxim
6a940772f0 Add network activity map overlay and configurable monitoring filters 2026-10-09 22:59:32 +03:00
Maxim
f198cc47d9 Automatically install missing conntrack for enabled network collector 2026-10-09 22:36:46 +03:00
Maxim
49de0b1f49 Add opt-in conntrack network activity and CrowdSec firewall counters 2026-10-09 22:32:41 +03:00
Maxim
ea890c01b1 Redesign traffic dashboard to fit viewport with responsive panels 2026-10-09 19:55:46 +03:00
Maxim
f7b347212e Add opt-in NPM and CrowdSec traffic map with selectable agent source 2026-10-09 19:39:13 +03:00
Maxim
72869dcf5f Separate network faults from software drops and fix CPU temperature history 2026-10-07 22:02:08 +03:00
Maxim
1296d052a5 Run guest updates outside agent sandbox 2026-08-21 22:09:35 +03:00
Maxim
f764195ba5 Clarify process CPU and improve Docker actions 2026-08-21 21:58:00 +03:00
Maxim
f6498a2fc1 Add backup maintenance and guest update controls 2026-08-21 21:15:52 +03:00
Maxim
714c29c449 Fix NVMe self-tests and mobile layout 2026-08-21 19:17:30 +03:00
Maxim
21b4f528e0 Add agent operations history and manual links 2026-08-07 22:24:44 +03:00
Maxim
c50b4cf986 Improve agent problem visibility 2026-08-07 21:36:38 +03:00
Maxim
338d53cb2f Add guest root cause diagnostics 2026-08-07 21:23:05 +03:00
Maxim
a2525b53f0 Classify and enrich guest services 2026-08-06 23:24:43 +03:00
Maxim
331f0ee39a Add detailed guest agent diagnostics 2026-08-06 21:12:42 +03:00
Maxim
12aba383f1 Add secure automatic agent updates 2026-08-06 21:01:54 +03:00
Maxim
c618eb018c Expand guest agent system monitoring 2026-08-06 20:48:27 +03:00
Maxim
f1ff15e344 Add read-only guest monitoring agents 2026-08-06 20:24:13 +03:00
Maxim
5401b5b0f2 Add maintenance windows for planned work 2026-08-06 19:35:33 +03:00
Maxim
c01b4b6630 Fix service monitor connection leak 2026-08-05 20:04:29 +03:00
Maxim
a14001b373 Treat historical SMART failures as resolved 2026-08-03 21:17:15 +03:00
Maxim
05450bc949 Improve system health sorting and dashboard spacing 2026-08-03 21:09:30 +03:00
Maxim
8241fba6f4 Add IO attribution anomaly detection and alert correlation 2026-08-03 18:02:13 +03:00
64 changed files with 7805 additions and 218 deletions

343
README.md
View File

@@ -32,7 +32,7 @@ ssh root@192.168.0.10
```bash ```bash
apt-get update apt-get update
apt-get install -y curl ca-certificates tar smartmontools iproute2 iputils-ping apt-get install -y curl ca-certificates tar smartmontools iproute2 iputils-ping ethtool
cd /tmp cd /tmp
curl -fL \ curl -fL \
@@ -123,6 +123,11 @@ ss -lntp | grep 9105
- изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков; - изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков;
- realtime read/write IOPS, MB/s, latency, utilization и глубину очереди каждого диска; - realtime read/write IOPS, MB/s, latency, utilization и глубину очереди каждого диска;
- недельную историю IOPS/utilization и алерты при устойчивом I/O-давлении; - недельную историю IOPS/utilization и алерты при устойчивом I/O-давлении;
- вероятный источник I/O: VM/LXC, процесс хоста или активная backup/snapshot/migration-задача;
- отдельные пороги HDD, SATA SSD и NVMe с исключениями по имени или серийному номеру;
- недельные baseline-аномалии IOPS, latency, температуры, RAM, ZFS и длительности backup;
- корреляцию связанных событий в одну причинно-следственную цепочку;
- время первого появления и последнего наблюдения каждого уведомления;
- автоматический короткий SMART self-test раз в неделю для дисков без свежего теста; - автоматический короткий SMART self-test раз в неделю для дисков без свежего теста;
- datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS; - datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS;
- доступность шлюза, DNS и интернета, packet loss, latency и внешний IP; - доступность шлюза, DNS и интернета, packet loss, latency и внешний IP;
@@ -317,6 +322,78 @@ apcaccess status
- `GET /api/events` — поток метрик SSE с интервалом две секунды. - `GET /api/events` — поток метрик SSE с интервалом две секунды.
- `GET/POST/PUT/DELETE /api/services` — управление проверками сервисов. - `GET/POST/PUT/DELETE /api/services` — управление проверками сервисов.
- `GET /api/services/history` — семидневная история доступности и отклика. - `GET /api/services/history` — семидневная история доступности и отклика.
- `GET/PUT/DELETE /api/settings/maintenance` — состояние, запуск и досрочное завершение плановых работ.
- `GET/DELETE /api/agents` — подключённые гостевые агенты.
- `POST /api/agents/enrollment-token` — одноразовый токен регистрации на 15 минут.
## Maintenance-режим
Во вкладке `Настройки → Maintenance` можно на заданное время отключить все
алерты либо выбрать конкретные сервисы и VM/LXC. Подавленные события не
отправляются по почте и не отображаются как активные, но сохраняются в истории
с причиной плановых работ. После окончания окна обычный мониторинг включается
автоматически.
## Агенты VM и LXC
Read-only агент показывает состояние Linux внутри VM/LXC: CPU, load average,
память, корневой раздел, uptime, ОС, ядро, IP-адреса, systemd-службы, доступные
APT-обновления и необходимость перезагрузки. Для Docker отображаются контейнеры,
health-check, число перезапусков, CPU и память. Агент сам подключается к Dashboard
каждые 10 секунд; открывать входящий порт внутри гостевой системы не нужно.
В подробностях агента systemd-службы разделяются на системные, установленные
дополнительно и локальные unit-файлы. Для каждой службы показываются пакет и путь
unit-файла, время запуска, число перезапусков, последний exit code, память и
накопленное CPU-время, когда соответствующий accounting доступен в systemd.
Расширенная диагностика агента включает последние сообщения проблемных служб,
`Requires`/`After`, PID, команду запуска и слушающие порты; выявляет частые
рестарты, OOM-kill и процессы в D-state. Отдельно отображаются top процессов по
CPU и RAM с накопленным I/O, swap, PSI pressure, все постоянные файловые системы,
их заполнение и использование inode.
Панель «Диагностика homelab» и карта сопоставляют известные IP и порты с VM/LXC,
агентом, systemd-службой или Docker-контейнером и настроенной HTTP/TCP-проверкой.
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
помечаются как предположение.
Для сложных reverse proxy и Docker-сетей в управлении сервисами можно сохранить
ручную связь `сервис → агент/VMID → systemd/Docker/процесс → порт`. Она имеет
приоритет над автоматическим определением и используется для группировки
связанных уведомлений по первопричине.
Dashboard хранит семь дней истории CPU, RAM, swap, PSI и файловых систем каждого
агента. Из подробностей агента доступны только заранее разрешённые действия:
перезапуск известной systemd-службы или Docker-контейнера, `reset-failed` и
обновление списка пакетов. Агент не принимает произвольные shell-команды; каждое
действие требует подтверждения в браузере и сохраняется в журнале управления.
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
минут и после регистрации заменяется индивидуальным секретом агента.
После однократной установки агент обновляется автоматически вслед за Dashboard.
Центральный сервер отдаёт агенту собственный бинарник и SHA-256; агент проверяет
контрольную сумму, атомарно заменяет файл и перезапускается через systemd. Если
агент был удалён из Dashboard, повторная команда с новым токеном автоматически
выполнит новую регистрацию.
В первой версии агент не выполняет команды и не перезапускает службы или
контейнеры. Для чтения Docker ему требуется доступ к `/var/run/docker.sock`,
поэтому служба устанавливается от root. Сам Dashboard пока не имеет
авторизации: используйте локальную сеть или HTTPS через доверенный reverse
proxy и не публикуйте агентские API напрямую в интернет.
Полностью удалить агент из VM/LXC:
```bash
curl -fsSL https://git.myown.center/maxim/ProxmoxDash/raw/branch/main/scripts/uninstall-agent.sh | sh
```
Оформление, пороги, почта и обновления Dashboard находятся в обычных настройках.
Рабочие инструменты — сервисы, агенты, плановые работы и UPS-сценарий — вынесены
в отдельный раздел `Управление` в шапке Dashboard.
## Сборка ## Сборка
@@ -324,3 +401,267 @@ apcaccess status
go test ./... go test ./...
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o proxmox-cpu-dashboard . GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o proxmox-cpu-dashboard .
``` ```
## Сетевые ошибки и температура CPU (v0.19.3)
RX/TX drops отображаются отдельно как информационные счётчики. Их рост сам по
себе не вызывает аварийный алерт: в том числе LLDP на Linux bridge. Настройки
Keenetic, LLDP и group_fwd_mask менять не требуется. Для VM учитываются tap,
для LXC — veth; fwbr/fwpr/fwln остаются в группе интерфейсов хоста.
Новые rx_errors/tx_errors, CRC/frame/FIFO/missed, carrier/link-down вызывают
критический алерт. Sysfs позволяет отделить rx_missed_errors от RX drops,
которые в /proc/net/dev объединены. Для Atlantic дополнительно читаются
InErrors, InDroppedDma, AllocFails, SkbAllocFails и Restarts через ethtool -S.
Установите `ethtool` на существующем хосте для этой дополнительной диагностики.
При отсутствии утилиты остаются стандартные счётчики sysfs. Счётчики ошибок
могут пересекаться: их сумма — прирост счётчиков, а не число уникальных пакетов.
После перезапуска и сброса счётчиков старые значения не создают новые алерты.
Карточка CPU и новые точки истории показывают максимум доступных датчиков CPU,
а не среднее Tctl/Tdie/CCD или ядер. На AMD Tctl может включать управляющее
смещение; подписи и значения всех датчиков доступны в подробностях CPU.
История снимается раз в минуту, а на диапазоне 7 дней усредняется по 10 минутам,
поэтому короткие пики между измерениями не гарантированно попадут на график.
Старые точки до обновления остаются средними: исходные значения датчиков в них
не сохранены. Масштаб графика расширяется для значений выше 100 °C.
Релиз Linux amd64 собирается с номером версии и без CGO:
```bash
./scripts/build-release.sh v0.19.3
```
Скрипт запускает тесты и создаёт в dist бинарник, SHA-256 и установочный архив,
совместимые со встроенным обновлением из Gitea Releases.
## Карта трафика NPM + CrowdSec (v0.20.0)
С v0.20.1 страница помещается в высоту окна: карта и лента в центре,
счётчики сверху, график и рейтинги снизу. На телефоне доступны вкладки
«Карта / События / Статистика». Последние 100 событий листаются по страницам;
нажатие на строку открывает полные подробности.
Откройте **Настройки → Карта трафика**, включите страницу и выберите
агент VM/LXC, где установлены Nginx Proxy Manager и CrowdSec. Например,
`nginxproxymanager · VMID 102`. Необязательно выберите страну сервера для
конечной точки линий на карте. Сохраните настройки: в шапке появится ссылка
«Карта трафика», страница также доступна по `/traffic`.
Функция **по умолчанию выключена**. Только выбранный агент получает команду
включить сбор. При выключении или смене источника старый агент останавливает
сбор после следующего успешного обычного отчёта; сервер сразу прекращает
приём его событий. Выключенная страница и её API недоступны. Это глобальная
настройка экземпляра Dashboard. Карта не создаёт общих алертов или писем.
Агент автоматически обновляется вместе с Dashboard; для карты требуется
v0.20.0 или новее. Он читает стандартные NPM access-логи из
`/opt/npm/data/logs/*_access.log`, включая несжатые файлы ротации. Другой каталог
задаётся переменной окружения службы `DASHBOARD_NPM_LOG_DIR`; значение `off`
полностью запрещает сбор на конкретном агенте. Порт CrowdSec наружу открывать
и новые API-ключи создавать не нужно: используется локальная команда
`cscli alerts list --since 24h --limit 50 -o json` с существующими учётными
данными машины. Права на логи и запуск cscli нужны службе агента.
- Голубой: все записанные HTTP-запросы, включая обычные и локальные обращения.
- Оранжевый: реальные (не simulated) срабатывания сценариев CrowdSec.
- Розовый: решения CrowdSec, например ban или captcha. Это **не** счётчик
фактически отброшенных пакетов и не список действующих сейчас банов.
- HTTP 401/403/502 сами по себе не считаются атакой или блокировкой.
Карта показывает страны источников, живую ленту последних 100 событий,
счётчики за последние 24 часа, уникальные IP HTTP-клиентов, график по минутам
и средний req/s за последние 60 секунд. Есть фильтры типа события, домена и
внешних IP, пауза и полный экран. Локальные адреса видны в ленте без страны.
Все счётчики следуют выбранным фильтрам. Уникальный IP не равен человеку.
Первое включение начинает чтение с конца логов, без импорта старых HTTP-запросов.
Обычно обновление занимает несколько секунд; Nginx записывает запрос после
его завершения. Это не карта открытых TCP-соединений. Позиции чтения и очередь
до 500 событий сохраняются в `traffic-state.json` рядом с конфигом агента;
повторы после обрыва связи устраняются на сервере. При переполнении очереди
чтение ждёт отправки. Если за время длительного простоя логи уже сжаты или
удалены, восстановить эти строки нельзя. Размер очереди чтения и ошибки
парсинга отображаются на самой странице, без глобальных уведомлений.
CrowdSec опрашивается каждые 15 секунд, до 50 последних alerts за 24 часа;
при достижении лимита страница предупреждает о возможной неполноте истории.
Связанный домен берётся из метаданных события, только если он однозначен.
Новый ban не рисуется как новый запрос. Для наблюдения реально отброшенных
firewall пакетов нужен отдельный источник: в этой версии его нет.
В SQLite хранится до 24 часов и 200 000 событий; старые строки удаляются при
приёме новых данных. Данные каждого выбранного агента фильтруются отдельно.
Query string и fragment URL отбрасываются на агенте и повторно на сервере;
заголовки, cookie и тела запросов не передаются. IP, домен и путь сохраняются.
GeoIP работает локально: DB-IP Country Lite за октябрь 2026 встроена в бинарник,
исходящие запросы с IP посетителей не выполняются. Координаты на карте — условные
точки стран Natural Earth, а не точное местоположение посетителя. Дата базы
видна внизу карты. Обновление базы требует нового релиза; автоматического
скачивания нет. Лицензии и происхождение данных: `assets/NOTICE.md`.
Локальный предпросмотр с явно обозначенными искусственными данными:
```bash
DASHBOARD_TRAFFIC_PREVIEW=1 go test -run '^TestTrafficPreview$' -timeout 30m
# http://127.0.0.1:19105/traffic
```
## Сетевая активность TCP/UDP/ICMP (v0.21.0)
В **Настройки → Карта трафика** включите дополнительный сбор TCP/UDP/ICMP.
Он выполняется только выбранным агентом v0.21.0+; на странице карты появится
кнопка **TCP / UDP**, открывающая `/network`. Основной выключатель карты
отключает оба сборщика. Сетевой режим по умолчанию выключен и не создаёт
общих алертов/писем.
На Debian/Ubuntu агент **сам проверяет и устанавливает conntrack**, когда
включён сетевой режим. Выполняются `apt-get update` и установка только
`conntrack` с зависимостями (`--no-install-recommends --no-remove`), без
обновления всей системы. Установка выполняется от root через отдельную
systemd-службу, как существующие операции обновления агента, и не требует
отключать его read-only sandbox. Начатая пакетная операция завершается даже
при выключении сбора. Ход/ошибка видны в диагностике; повтор после ошибки —
не чаще раза в 30 минут работы процесса. После перезапуска агента проверка
выполняется заново. Если conntrack уже установлен, apt-get не запускается.
На других системах, без root или systemd/apt-get, выводится инструкция для
ручной установки. Для счётчиков CrowdSec требуется доступный `nft`; он
обычно уже установлен вместе с nftables-bouncer и автоматически не ставится.
Агенту нужны root/CAP_NET_ADMIN в сетевом namespace хоста. Установщик агента
запускает службу от root. Чтение: `/proc/net/nf_conntrack`, если доступен,
иначе `conntrack -L -f ipv4/ipv6 -o extended`. Ничего устанавливать на хост
самого Dashboard не требуется. Firewall, NAT и sysctl сборщик не меняет.
Страница показывает состояния TCP, UDP/ICMP-потоки, обе пары адресов до/после
NAT (в том числе Docker), локальную GeoIP, топ инициаторов и портов, фильтры,
поиск и детали потока. Список листается по страницам; переход на следующую
фиксирует снимок, «К свежим» возвращает live. Для телефона — вкладки.
Это **снимки таблицы примерно каждые 5 секунд**, не полный журнал соединений:
короткие потоки между снимками могут быть пропущены. До 5 000 записей на
снимок, 8 MiB вывода команды и таймаут 4 секунды; ограничения/ошибки видны в
диагностике. Сохраняется только последний снимок каждого агента. Инициатор
не обязательно внешний клиент: NPM → upstream — отдельный поток. Порт не
доказывает протокол приложения; UDP ASSURED не означает TCP handshake.
Байты/пакеты доступны только при включённом учёте conntrack для конкретных
потоков. Это накопленные значения за жизнь потока, а не трафик за 24 часа.
При отсутствии учёта интерфейс показывает «нет учёта». При необходимости
администратор может включить `net.netfilter.nf_conntrack_acct=1`; изменение
применяется к новым потокам и автоматически не выполняется.
Firewall читается раз в 30 секунд командой `nft -j list ruleset`. Поддержаны
анонимные счётчики непосредственно перед финальным `drop/reject` в таблице
или цепочке с `crowdsec` в имени. Остальные счётчики не выдаются за блокировки.
Это сумма пакетов/байтов выбранных правил с момента их создания, независимо
от фильтров потоков; при пересоздании правил она может уменьшиться.
Для iptables/ipset-only, именованных счётчиков и правил без счётчиков выводится
«недоступно», а не ноль. Отброшенный пакет до подтверждения conntrack может
отсутствовать в таблице потоков. Содержимое пакетов не перехватывается.
## Сетевой слой карты и фильтр мониторинга (v0.22.0)
При включённом сборе TCP/UDP/ICMP переключатель **Сеть на карте** показывает
текущие потоки отдельным слоем: TCP — зелёный, UDP — розовый, ICMP/ICMPv6 —
жёлтый. Импульсы означают новый снимок (~5 секунд), не отдельные пакеты или
новые подключения. Входящие и исходящие потоки имеют разное направление.
Локальные адреса и потоки без страны учитываются отдельно; при отсутствии
свежего снимка 25 секунд сетевой слой гаснет. HTTP-статистика за 24 часа с
текущими потоками не складывается. Фильтры домена и HTTP/CrowdSec относятся
только к HTTP-слою; фильтры внешних IP и мониторинга действуют на оба слоя.
В **Настройки → Карта трафика → Правила мониторинга** задайте по одному правилу
на строку: `IP | домен | префикс пути`. IP точный, включая IPv6; домен и путь
необязательны. Пример: `192.168.0.10` или `192.168.0.10 | example.org | /health`.
Для сетевых потоков применяются только правила без домена/пути, к IP инициатора.
При первом обновлении добавляется правило `192.168.0.10`; его можно изменить
или полностью удалить. Сам фильтр по умолчанию выключен.
**Скрыть мониторинг** на карте и `/network` исключает совпадения из отображения
и соответствующей статистики. Настройка переключателя сохраняется в браузере.
HTTP-проверки остаются в истории и возвращаются при выключении фильтра;
новые правила сразу применяются и к старым записям. Срабатывания и решения
CrowdSec, а также общие счётчики firewall, никогда не скрываются этим фильтром.
Сетевые потоки по-прежнему хранятся только последним снимком, без истории.
## Alpine LXC и Docker NPM/NPM+ (v0.23.0)
Установщик агента поддерживает Linux **x86_64/amd64** с systemd или OpenRC
(Alpine LXC). Для Alpine команда из настроек сначала устанавливает curl и
CA-сертификаты, если curl отсутствует. Бинарник статический, без glibc;
скачивание проверяется по SHA-256. OpenRC-служба включается в default runlevel
и использует supervise-daemon: перезапускает агент после ошибки и обновления.
Команды: `rc-service proxmox-dashboard-agent status/restart` (выберите одно
действие). Журнал: `/var/log/proxmox-dashboard-agent.log`; его ротацию настройте
средствами ОС. Удаление — штатным `scripts/uninstall-agent.sh`.
Обновление пакетов ОС и управление systemd-службами из интерфейса остаются
функциями Debian/Ubuntu/systemd; этот релиз добавляет установку и работу агента
на OpenRC, а не эмуляцию systemd. Включённый сетевой сбор на Alpine ставит
`conntrack-tools` через `apk add --no-cache`. Ограничения CAP_NET_ADMIN и
сетевого namespace LXC сохраняются; никаких изменений конфигурации Proxmox,
privileged/nesting, firewall или sysctl агент не делает.
После выбора агента Docker-хоста и включения карты трафика агент автоматически:
- проверяет стандартные каталоги NPM и NPM+;
- находит запущенные контейнеры с nginx-proxy-manager/npmplus в имени образа
и определяет каталоги логов по mounts `/data`, включая именованные volumes
и вложенные bind mounts;
- читает обычный формат NPM и формат NPM+ `access.log`, включая ротацию;
- находит единственный контейнер образа `crowdsecurity/crowdsec` и выполняет
только `cscli alerts list` через локальный Docker socket. Docker CLI, новый
bouncer key и опубликованный LAPI не нужны. Если контейнера нет, используется
прежний локальный `cscli`.
Поиск повторяется раз в минуту, поэтому пересоздание контейнеров подхватывается
автоматически. При нескольких CrowdSec задайте точный контейнер вручную:
`DASHBOARD_CROWDSEC_CONTAINER=имя` в `/etc/proxmox-dashboard-agent.env` и
перезапустите агент. Каталог логов можно переопределить переменной
`DASHBOARD_NPM_LOG_DIR=/путь/к/логам`; `off` выключает сбор трафика этого агента.
Используется локальный `/var/run/docker.sock`; rootless/удалённые Docker engines
и логи только в stdout этим обнаружением не поддерживаются.
NPM+ должен действительно писать access-логи: включите `LOGROTATE=true` в его
Compose и пересоздайте контейнер, если логирование выключено. Агент показывает
подсказку при отсутствии логов, но не меняет Compose приложений. Это настройка
[NPM+](https://github.com/ZoeyVid/NPMplus/blob/develop/README.md), а не Dashboard.
Stream-логи NPM+ не считаются HTTP-запросами; сетевые потоки собираются conntrack.
Как и раньше, первый запуск чтения начинается с конца существующих логов.
## Интерактивная карта (v0.24.0)
Карта трафика использует ту же настройку `proxmox-theme`, что и основной
Dashboard. Кнопка солнца/луны меняет тему; выбор сохраняется в браузере.
Светлая тема распространяется на карту, график, карточки и модальные окна.
Масштаб карты — от 100% до 600%: кнопки +/−, колесо мыши или жест двумя
пальцами. Увеличенную карту можно перетаскивать; ↺ возвращает весь мир.
При фокусе на карте доступны клавиши +/−, 0 и стрелки. Наведение подсвечивает
контур страны и показывает суточные счётчики, а нажатие открывает подробности.
Четыре карточки статистики открывают окна с итогом, объяснением метрики,
разбивкой по странам и примерами событий. Счётчики стран считаются сервером
по всей доступной истории за 24 часа с текущими фильтрами. Начиная с v0.25.0
история загружается целиком по страницам, без ограничения последними 100 событиями. Открытое
окно фиксирует данные на момент открытия; временная отметка показана внутри.
Сетевые потоки остаются отдельным текущим снимком, не суточной статистикой.
## Карта и полная история (v0.25.0)
Карта использует векторные границы стран: при зуме 100–600% они перерисовываются,
а маркеры и подписи сохраняют размер. Это обзорная карта стран, не уличная карта.
Карточки и окна стран загружают всю сохранённую историю за последние 24 часа
по 50 записей; фильтры учитываются сервером. В карточке IP показаны все уникальные
HTTP IP с количеством запросов. Переходы между страницами фиксируют снимок истории.
Для городов откройте настройки карты и нажмите «Установить базу городов».
Dashboard скачает DB-IP City Lite 2026-10 (~58 МБ сжатая, ~121 МБ на диске),
проверит контрольную сумму и сохранит `dbip-city.mmdb` рядом с `DASHBOARD_DB_PATH`.
IP посетителей не отправляются во внешние сервисы. Город и регион отображаются
в истории и списке IP; местоположение приблизительное и может относиться к VPN
или провайдеру. Для части IP город неизвестен. База устанавливается отдельно от
релиза, автоматически каждый месяц не обновляется.

View File

@@ -57,7 +57,7 @@ func (c *activityCollector) collect(guests GuestsMetrics) ProxmoxActivity {
value := c.value value := c.value
c.mu.RUnlock() c.mu.RUnlock()
c.mu.Lock() c.mu.Lock()
if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 10*time.Minute { if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 15*time.Second {
c.refreshing = true c.refreshing = true
go c.refreshWithGuests(guests) go c.refreshWithGuests(guests)
} }

1076
agent.go Normal file

File diff suppressed because it is too large Load Diff

346
agent_actions.go Normal file
View File

@@ -0,0 +1,346 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
type AgentCommand struct {
ID int64 `json:"id"`
Action string `json:"action"`
Target string `json:"target,omitempty"`
}
type AgentActionResult struct {
ID int64 `json:"id"`
Success bool `json:"success"`
Message string `json:"message"`
}
type AgentAction struct {
ID int64 `json:"id"`
AgentID string `json:"agentId"`
AgentName string `json:"agentName"`
Action string `json:"action"`
Target string `json:"target"`
RequestedAt int64 `json:"requestedAt"`
StartedAt *int64 `json:"startedAt,omitempty"`
CompletedAt *int64 `json:"completedAt,omitempty"`
Status string `json:"status"`
Result string `json:"result"`
}
var safeUnitName = regexp.MustCompile(`^[A-Za-z0-9_.@:-]+\.service$`)
var safeContainerName = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, error) {
action, target = strings.TrimSpace(action), strings.TrimSpace(target)
var raw, name string
if err := s.db.QueryRow(`SELECT report_json,name FROM agents WHERE id=?`, agentID).Scan(&raw, &name); err != nil {
return AgentAction{}, fmt.Errorf("агент не найден")
}
var report AgentReport
_ = json.Unmarshal([]byte(raw), &report)
valid := false
switch action {
case "systemd-restart", "systemd-reset-failed":
if safeUnitName.MatchString(target) {
for _, service := range report.Services {
if service.Name == target {
valid = true
break
}
}
}
case "docker-restart", "docker-check", "docker-update", "docker-remove":
if safeContainerName.MatchString(target) {
for _, container := range report.Containers {
if (container.ID == target || container.Name == target) && (action != "docker-remove" || container.State != "running") {
valid = true
break
}
}
}
case "updates-refresh", "system-update", "system-reboot":
valid = target == ""
}
if !valid {
return AgentAction{}, fmt.Errorf("действие или цель не входят в разрешённый список")
}
now := time.Now().Unix()
result, err := s.db.Exec(`INSERT INTO agent_actions(agent_id,action,target,requested_at,status) VALUES(?,?,?,?, 'queued')`, agentID, action, target, now)
if err != nil {
return AgentAction{}, err
}
id, _ := result.LastInsertId()
return AgentAction{ID: id, AgentID: agentID, AgentName: name, Action: action, Target: target, RequestedAt: now, Status: "queued"}, nil
}
func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
// Обновление пакетов и Docker-образов может быть долгим. Не выдаём одну
// команду повторно, пока предыдущий запуск ещё имеет шанс завершиться.
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-45*time.Minute).Unix())
rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
if err != nil {
return nil, err
}
var commands []AgentCommand
for rows.Next() {
var c AgentCommand
if err = rows.Scan(&c.ID, &c.Action, &c.Target); err != nil {
rows.Close()
return nil, err
}
commands = append(commands, c)
}
rows.Close()
now := time.Now().Unix()
for _, c := range commands {
if _, err = tx.Exec(`UPDATE agent_actions SET status='running',started_at=? WHERE id=?`, now, c.ID); err != nil {
return nil, err
}
}
return commands, tx.Commit()
}
func (s *Store) CompleteAgentActions(agentID string, results []AgentActionResult) {
now := time.Now().Unix()
for _, result := range results {
status := "failed"
if result.Success {
status = "completed"
}
message := result.Message
if len(message) > 4000 {
message = message[:4000]
}
_, _ = s.db.Exec(`UPDATE agent_actions SET status=?,result=?,completed_at=? WHERE id=? AND agent_id=? AND status='running'`, status, message, now, result.ID, agentID)
}
}
func (s *Store) AgentActions(limit int) ([]AgentAction, error) {
if limit < 1 || limit > 200 {
limit = 100
}
rows, err := s.db.Query(`SELECT a.id,a.agent_id,g.name,a.action,a.target,a.requested_at,a.started_at,a.completed_at,a.status,a.result FROM agent_actions a JOIN agents g ON g.id=a.agent_id ORDER BY a.id DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []AgentAction
for rows.Next() {
var v AgentAction
if err = rows.Scan(&v.ID, &v.AgentID, &v.AgentName, &v.Action, &v.Target, &v.RequestedAt, &v.StartedAt, &v.CompletedAt, &v.Status, &v.Result); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func executeAgentCommand(command AgentCommand) AgentActionResult {
result := AgentActionResult{ID: command.ID}
var output string
switch command.Action {
case "systemd-restart":
output = runAgentCombinedCommand(30*time.Second, "systemctl", "restart", command.Target)
case "systemd-reset-failed":
output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
case "docker-restart":
output = restartAgentContainer(command.Target)
case "docker-update":
output = updateAgentContainer(command.Target, true)
case "docker-check":
output = updateAgentContainer(command.Target, false)
case "docker-remove":
output = removeAgentContainer(command.Target)
case "system-update":
output = updateAgentSystem()
case "system-reboot":
output = scheduleAgentReboot()
case "updates-refresh":
guestInventory.Lock()
guestInventory.value.CollectedAt = time.Time{}
guestInventory.Unlock()
result.Success = true
result.Message = "Проверка обновлений поставлена в очередь"
return result
default:
result.Message = "Команда не разрешена"
return result
}
result.Success = !strings.HasPrefix(output, "ERROR: ")
result.Message = strings.TrimSpace(output)
if result.Message == "" && result.Success {
result.Message = "Выполнено успешно"
}
return result
}
func removeAgentContainer(target string) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
request, _ := http.NewRequest(http.MethodDelete, "http://docker/containers/"+target+"?v=false&force=false", nil)
response, err := (&http.Client{Transport: transport, Timeout: 20 * time.Second}).Do(request)
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
if response.StatusCode != http.StatusNoContent {
message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))
return fmt.Sprintf("ERROR: Docker вернул %s: %s", response.Status, strings.TrimSpace(string(message)))
}
return "Остановленный контейнер удалён; volumes сохранены"
}
func updateAgentSystem() string {
aptGet, err := exec.LookPath("apt-get")
if err != nil {
return "ERROR: apt-get не найден; обновление поддерживается для Debian/Ubuntu"
}
if output := runPrivilegedTransient(5*time.Minute, "update-lists", aptGet, "update"); strings.HasPrefix(output, "ERROR: ") {
return explainSystemUpdateError(output)
}
message := runPrivilegedTransient(30*time.Minute, "upgrade", aptGet, "-y", "upgrade")
if strings.HasPrefix(message, "ERROR: ") {
return explainSystemUpdateError(message)
}
guestInventory.Lock()
guestInventory.value.CollectedAt = time.Time{}
guestInventory.Unlock()
return "Обновления системы установлены; список пакетов будет обновлён в следующем отчёте агента"
}
func explainSystemUpdateError(message string) string {
if strings.Contains(strings.ToLower(message), "read-only file system") {
return "ERROR: файловая система агента доступна только для чтения. Обновите Dashboard и агент до v0.19.2 или новее, затем повторите операцию."
}
if len(message) > 1800 {
message = message[len(message)-1800:]
}
return message
}
func runPrivilegedTransient(timeout time.Duration, suffix, name string, args ...string) string {
systemdRun, err := exec.LookPath("systemd-run")
if err != nil {
return "ERROR: systemd-run не найден; невозможно безопасно выйти из read-only sandbox агента"
}
unit := fmt.Sprintf("proxmox-dashboard-%s-%d", suffix, time.Now().UnixNano())
commandArgs := []string{"--quiet", "--wait", "--collect", "--pipe", "--unit=" + unit, "--setenv=DEBIAN_FRONTEND=noninteractive", name}
commandArgs = append(commandArgs, args...)
return runAgentCombinedCommand(timeout, systemdRun, commandArgs...)
}
func scheduleAgentReboot() string {
systemdRun, err := exec.LookPath("systemd-run")
if err != nil {
return "ERROR: systemd-run не найден"
}
systemctl, err := exec.LookPath("systemctl")
if err != nil {
return "ERROR: systemctl не найден"
}
unit := fmt.Sprintf("proxmox-dashboard-reboot-%d", time.Now().UnixNano())
output := runAgentCombinedCommand(15*time.Second, systemdRun, "--quiet", "--collect", "--unit="+unit, "--on-active=20s", systemctl, "reboot")
if strings.HasPrefix(output, "ERROR: ") {
return output
}
return "Перезагрузка запланирована через 20 секунд; кратковременный offline агента ожидаем"
}
func updateAgentContainer(target string, recreate bool) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
response, err := client.Get("http://docker/containers/" + target + "/json")
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
var detail struct {
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"Config"`
}
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
return "ERROR: не удалось получить Compose-конфигурацию контейнера"
}
labels := detail.Config.Labels
service := labels["com.docker.compose.service"]
workdir := labels["com.docker.compose.project.working_dir"]
files := strings.Split(labels["com.docker.compose.project.config_files"], ",")
if service == "" || workdir == "" || len(files) == 0 || !filepath.IsAbs(workdir) || !filepath.IsAbs(files[0]) {
return "ERROR: безопасное обновление доступно только для Docker Compose"
}
if _, err = exec.LookPath("docker"); err != nil {
return "ERROR: Docker CLI не найден"
}
commands := [][]string{{"compose", "-f", files[0], "pull", service}}
if recreate {
commands = append(commands, []string{"compose", "-f", files[0], "up", "-d", "--no-deps", service})
}
for _, args := range commands {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
command := exec.CommandContext(ctx, "docker", args...)
command.Dir = workdir
output, commandErr := command.CombinedOutput()
cancel()
if commandErr != nil {
return "ERROR: " + commandErr.Error() + ": " + strings.TrimSpace(string(output))
}
}
if recreate {
return "Образ загружен, Compose-контейнер пересоздан"
}
return "Проверка завершена: образ загружен; если digest изменился, карточка покажет доступное обновление"
}
func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
output, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
message := strings.TrimSpace(string(output))
if err != nil {
return "ERROR: " + err.Error() + ": " + message
}
return message
}
func restartAgentContainer(target string) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 35 * time.Second}
request, _ := http.NewRequest(http.MethodPost, "http://docker/containers/"+target+"/restart?t=10", bytes.NewReader(nil))
response, err := client.Do(request)
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
if response.StatusCode >= 300 {
return "ERROR: Docker вернул " + response.Status + ": " + strings.TrimSpace(string(body))
}
return "Контейнер перезапущен"
}

77
agent_actions_test.go Normal file
View File

@@ -0,0 +1,77 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea", State: "running"}, {ID: "old123", Name: "old", State: "exited"}}})
_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
if err != nil {
t.Fatal(err)
}
if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
t.Fatal("unsafe unit accepted")
}
if _, err = store.QueueAgentAction("a", "docker-update", "gitea"); err != nil {
t.Fatalf("known Docker container rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "system-update", ""); err != nil {
t.Fatalf("system update rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "docker-remove", "gitea"); err == nil {
t.Fatal("running/unknown-state container accepted for removal")
}
action, err := store.QueueAgentAction("a", "docker-remove", "old")
if err != nil {
t.Fatalf("stopped container rejected for removal: %v", err)
}
commands, err := store.DispatchAgentActions("a")
if err != nil || len(commands) != 3 || commands[2].ID != action.ID {
t.Fatalf("commands=%#v err=%v", commands, err)
}
store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
items, err := store.AgentActions(10)
if err != nil || len(items) != 3 || items[0].Status != "completed" {
t.Fatalf("items=%#v err=%v", items, err)
}
if _, err = store.QueueAgentAction("a", "system-reboot", ""); err != nil {
t.Fatalf("system reboot rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "system-reboot", "now"); err == nil {
t.Fatal("system reboot with unexpected target accepted")
}
}
func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "a", Name: "app", VMID: 105, Online: false}}, ServiceLinks: []ServiceLink{{ServiceID: 7, AgentID: "a", VMID: 105}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{ID: 7, Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Kind: "local", Up: false}}}}}}
alerts := []Alert{{ID: "agent-offline-a-105"}, {ID: "service-7-local"}, {ID: "other"}}
grouped := groupLinkedAlerts(metrics, alerts)
if len(grouped) != 2 {
t.Fatalf("grouped=%#v", grouped)
}
found := false
for _, alert := range grouped {
if alert.ID == "group-agent-offline-a-105" {
found = true
}
}
if !found {
t.Fatalf("group alert missing: %#v", grouped)
}
}
func TestExplainReadOnlySystemUpdateError(t *testing.T) {
message := explainSystemUpdateError("ERROR: apt failed: Read-only file system")
if !strings.Contains(message, "v0.19.2") {
t.Fatalf("read-only failure is not actionable: %s", message)
}
}

336
agent_diagnostics.go Normal file
View File

@@ -0,0 +1,336 @@
package main
import (
"bufio"
"encoding/json"
"net"
"os"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
)
type AgentLog struct {
Time int64 `json:"time"`
Message string `json:"message"`
}
type AgentProcess struct {
PID int `json:"pid"`
Name string `json:"name"`
State string `json:"state"`
CPUPercent float64 `json:"cpuPercent"`
CPUCores float64 `json:"cpuCores"`
HostPercent float64 `json:"hostPercent"`
MemoryBytes uint64 `json:"memoryBytes"`
ReadBytes uint64 `json:"readBytes"`
WriteBytes uint64 `json:"writeBytes"`
Elapsed int64 `json:"elapsedSeconds"`
Command string `json:"command"`
}
type AgentFilesystem struct {
Device string `json:"device"`
Mountpoint string `json:"mountpoint"`
Type string `json:"type"`
TotalBytes uint64 `json:"totalBytes"`
UsedBytes uint64 `json:"usedBytes"`
UsedPercent float64 `json:"usedPercent"`
InodesTotal uint64 `json:"inodesTotal"`
InodesUsed uint64 `json:"inodesUsed"`
InodesPercent float64 `json:"inodesPercent"`
}
type AgentListenPort struct {
Protocol string `json:"protocol"`
Address string `json:"address"`
Port int `json:"port"`
PID int `json:"pid,omitempty"`
Process string `json:"process,omitempty"`
}
type AgentPressureValue struct {
Avg10 float64 `json:"avg10"`
Avg60 float64 `json:"avg60"`
Avg300 float64 `json:"avg300"`
}
type AgentPressure struct {
CPU AgentPressureValue `json:"cpu"`
Memory AgentPressureValue `json:"memory"`
IO AgentPressureValue `json:"io"`
}
var agentOOMCache struct {
sync.Mutex
value []AgentLog
at time.Time
}
func compactSystemdExec(value string) string {
if pathAt := strings.Index(value, "path="); pathAt >= 0 {
value = value[pathAt+5:]
}
if end := strings.IndexAny(value, " ;}"); end > 0 {
value = value[:end]
}
if len(value) > 240 {
value = value[:240]
}
return strings.TrimSpace(value)
}
func limitedFields(value string, limit int) []string {
fields := strings.Fields(value)
if len(fields) > limit {
fields = fields[:limit]
}
return fields
}
func appendUniqueInt(items []int, value int) []int {
for _, item := range items {
if item == value {
return items
}
}
return append(items, value)
}
func collectAgentJournal(unit string, limit int, since string) []AgentLog {
args := []string{"--no-pager", "-o", "json", "--reverse", "-n", strconv.Itoa(limit), "--since", since}
if unit != "" {
args = append(args, "-u", unit)
}
output := runAgentCommand(6*time.Second, "journalctl", args...)
result := []AgentLog{}
for _, line := range strings.Split(output, "\n") {
var entry struct {
Message any `json:"MESSAGE"`
Timestamp string `json:"__REALTIME_TIMESTAMP"`
}
if json.Unmarshal([]byte(line), &entry) != nil {
continue
}
message, ok := entry.Message.(string)
if !ok || strings.TrimSpace(message) == "" {
continue
}
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
}
return result
}
func cachedAgentOOMKills() []AgentLog {
agentOOMCache.Lock()
defer agentOOMCache.Unlock()
if time.Since(agentOOMCache.at) < time.Minute {
return agentOOMCache.value
}
output := runAgentCommand(6*time.Second, "journalctl", "-k", "--since", "24 hours ago", "--reverse", "--no-pager", "-o", "json", "-g", "Out of memory|Killed process")
result := []AgentLog{}
for _, line := range strings.Split(output, "\n") {
var entry struct {
Message any `json:"MESSAGE"`
Timestamp string `json:"__REALTIME_TIMESTAMP"`
}
if json.Unmarshal([]byte(line), &entry) != nil {
continue
}
message, ok := entry.Message.(string)
if !ok {
continue
}
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
if len(result) >= 20 {
break
}
}
agentOOMCache.value, agentOOMCache.at = result, time.Now()
return result
}
func collectAgentProcesses() []AgentProcess {
result := []AgentProcess{}
seen := map[int]bool{}
for _, sortBy := range []string{"-pcpu", "-rss"} {
output := runAgentCommand(5*time.Second, "ps", "-eo", "pid=,comm=,state=,pcpu=,rss=,etimes=,args=", "--sort="+sortBy)
added := 0
for _, line := range strings.Split(output, "\n") {
fields := strings.Fields(line)
if len(fields) < 7 {
continue
}
pid, err := strconv.Atoi(fields[0])
if err != nil || seen[pid] {
continue
}
seen[pid] = true
cpu, _ := strconv.ParseFloat(fields[3], 64)
rss, _ := strconv.ParseUint(fields[4], 10, 64)
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
cpuCount := runtime.NumCPU()
if cpuCount < 1 {
cpuCount = 1
}
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, CPUCores: cpu / 100, HostPercent: cpu / float64(cpuCount), MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
result = append(result, item)
added++
if added >= 12 || len(result) >= 24 {
break
}
}
}
return result
}
func readAgentProcessIO(pid int) (uint64, uint64) {
data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "io"))
if err != nil {
return 0, 0
}
values := map[string]uint64{}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) != 2 {
continue
}
values[strings.TrimSuffix(fields[0], ":")], _ = strconv.ParseUint(fields[1], 10, 64)
}
return values["read_bytes"], values["write_bytes"]
}
func collectAgentFilesystems() []AgentFilesystem {
file, err := os.Open("/proc/self/mounts")
if err != nil {
return nil
}
defer file.Close()
excluded := map[string]bool{"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "cgroup": true, "cgroup2": true, "overlay": true, "squashfs": true, "nsfs": true, "tracefs": true, "debugfs": true, "securityfs": true, "pstore": true, "mqueue": true, "hugetlbfs": true, "fusectl": true, "configfs": true, "autofs": true, "rpc_pipefs": true}
seen := map[string]bool{}
result := []AgentFilesystem{}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 3 || excluded[fields[2]] || seen[fields[1]] {
continue
}
mountpoint := strings.ReplaceAll(fields[1], `\040`, " ")
var stat syscall.Statfs_t
if syscall.Statfs(mountpoint, &stat) != nil || stat.Blocks == 0 {
continue
}
seen[mountpoint] = true
total := stat.Blocks * uint64(stat.Bsize)
free := stat.Bavail * uint64(stat.Bsize)
used := total - free
item := AgentFilesystem{Device: fields[0], Mountpoint: mountpoint, Type: fields[2], TotalBytes: total, UsedBytes: used, UsedPercent: float64(used) / float64(total) * 100, InodesTotal: stat.Files}
if stat.Files > stat.Ffree {
item.InodesUsed = stat.Files - stat.Ffree
item.InodesPercent = float64(item.InodesUsed) / float64(stat.Files) * 100
}
result = append(result, item)
if len(result) >= 30 {
break
}
}
sort.Slice(result, func(i, j int) bool { return result[i].Mountpoint < result[j].Mountpoint })
return result
}
var ssPIDPattern = regexp.MustCompile(`pid=([0-9]+)`)
var ssNamePattern = regexp.MustCompile(`users:\(\(\"([^\"]+)\"`)
func collectAgentListenPorts() []AgentListenPort {
output := runAgentCommand(5*time.Second, "ss", "-H", "-lntup")
result := []AgentListenPort{}
seen := map[string]bool{}
for _, line := range strings.Split(output, "\n") {
fields := strings.Fields(line)
if len(fields) < 5 {
continue
}
protocol := strings.ToLower(fields[0])
local := ""
for _, field := range fields[1:] {
if strings.Contains(field, ":") && !strings.Contains(field, "users:") {
local = field
break
}
}
if local == "" {
continue
}
address, portText, err := net.SplitHostPort(local)
if err != nil {
idx := strings.LastIndex(local, ":")
if idx < 0 {
continue
}
address, portText = local[:idx], local[idx+1:]
}
port, _ := strconv.Atoi(portText)
if port <= 0 {
continue
}
pid := 0
if match := ssPIDPattern.FindStringSubmatch(line); len(match) > 1 {
pid, _ = strconv.Atoi(match[1])
}
process := ""
if match := ssNamePattern.FindStringSubmatch(line); len(match) > 1 {
process = match[1]
}
key := protocol + "|" + address + "|" + portText + "|" + strconv.Itoa(pid)
if seen[key] {
continue
}
seen[key] = true
result = append(result, AgentListenPort{Protocol: protocol, Address: address, Port: port, PID: pid, Process: process})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Port == result[j].Port {
return result[i].Protocol < result[j].Protocol
}
return result[i].Port < result[j].Port
})
return result
}
func collectAgentPressure() AgentPressure {
return AgentPressure{CPU: readPressure("/proc/pressure/cpu"), Memory: readPressure("/proc/pressure/memory"), IO: readPressure("/proc/pressure/io")}
}
func readPressure(path string) AgentPressureValue {
data, err := os.ReadFile(path)
if err != nil {
return AgentPressureValue{}
}
line := strings.SplitN(string(data), "\n", 2)[0]
value := AgentPressureValue{}
for _, field := range strings.Fields(line) {
key, raw, ok := strings.Cut(field, "=")
if !ok {
continue
}
number, _ := strconv.ParseFloat(raw, 64)
switch key {
case "avg10":
value.Avg10 = number
case "avg60":
value.Avg60 = number
case "avg300":
value.Avg300 = number
}
}
return value
}

82
agent_installer_test.go Normal file
View File

@@ -0,0 +1,82 @@
package main
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// Run the real installer in a temporary tree with package/service/download
// commands stubbed. This must never touch the host's init system or packages.
func TestAgentInstallerOpenRC(t *testing.T) {
for _, init := range []string{"openrc", "systemd"} {
t.Run(init, func(t *testing.T) {
root := t.TempDir()
bin := filepath.Join(root, "bin")
os.MkdirAll(bin, 0755)
write := func(name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\nset -eu\n"+body), 0755); err != nil {
t.Fatal(err)
}
}
write("id", "echo 0\n")
write("uname", "echo x86_64\n")
hash := strings.Repeat("a", 64)
write("curl", `case "$1" in -fsSL) ;; *) exit 4 ;; esac
if [ "$#" = 2 ]; then printf '%s' '{"browser_download_url":"https://example.test/proxmox-dashboard-linux-amd64"}'; exit; fi
case "$2" in *.sha256) printf '%s\n' '`+hash+` proxmox-dashboard-linux-amd64' > "$4" ;; *) printf '#!/bin/sh\nprintf "%%s\\n" "$@" > "$CAPTURE"\n' > "$4" ;; esac
`)
write("sha256sum", "printf '%s\\n' '"+hash+" binary'\n")
for _, name := range []string{"systemctl", "rc-service", "rc-update", "supervise-daemon"} {
write(name, `printf '%s %s\n' "$(basename "$0")" "$*" >> "$CALLS"`+"\n")
}
raw, err := os.ReadFile("scripts/install-agent.sh")
if err != nil {
t.Fatal(err)
}
// Redirect every persistent absolute path, including those in generated scripts.
s := string(raw)
for _, prefix := range []string{"/etc/", "/var/lib/", "/usr/local/", "/run/", "/var/log/"} {
s = strings.ReplaceAll(s, prefix, root+prefix)
}
for _, dir := range []string{"etc/systemd/system", "etc/init.d", "var/log"} {
os.MkdirAll(filepath.Join(root, dir), 0755)
}
if init == "systemd" {
os.MkdirAll(filepath.Join(root, "run/systemd/system"), 0755)
}
script := filepath.Join(root, "install.sh")
os.WriteFile(script, []byte(s), 0700)
env := append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "DASHBOARD_URL=https://dashboard.test", "ENROLL_TOKEN=token", "AGENT_NAME=Alpine $(touch should-not-exist)", "VMID=102", "CALLS="+root+"/calls", "CAPTURE="+root+"/args")
cmd := exec.Command("sh", script)
cmd.Env = env
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("%v: %s", err, out)
}
calls, _ := os.ReadFile(root + "/calls")
if init == "openrc" {
if strings.Contains(string(calls), "systemctl") || !strings.Contains(string(calls), "rc-update add proxmox-dashboard-agent default") || !strings.Contains(string(calls), "rc-service proxmox-dashboard-agent restart") {
t.Fatalf("%s", calls)
}
cmd = exec.Command("sh", root+"/usr/local/sbin/proxmox-dashboard-agent-run")
cmd.Env = env
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("launcher: %v %s", err, out)
}
args, _ := os.ReadFile(root + "/args")
if !strings.Contains(string(args), "Alpine $(touch should-not-exist)\n") {
t.Fatalf("name was not passed literally: %s", args)
}
} else if !strings.Contains(string(calls), "systemctl restart proxmox-dashboard-agent.service") {
t.Fatalf("%s", calls)
}
info, _ := os.Stat(root + "/etc/proxmox-dashboard-agent.env")
if info.Mode().Perm() != 0600 {
t.Fatalf("credentials permissions: %v", info.Mode())
}
})
}
}

271
agent_network.go Normal file
View File

@@ -0,0 +1,271 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"time"
)
// Bounded command output and deadline keep large conntrack tables from exhausting the agent.
type networkBuffer struct {
bytes.Buffer
limit int
}
func (b *networkBuffer) Write(p []byte) (int, error) {
if len(p) > b.limit-b.Len() {
return 0, errors.New("output limit exceeded")
}
return b.Buffer.Write(p)
}
func networkCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, name, args...)
out := &networkBuffer{limit: 8 << 20}
stderr := &networkBuffer{limit: 4096}
cmd.Stdout = out
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("%s: %v %s", name, err, trafficText(stderr.String(), 240))
}
return out.Bytes(), nil
}
func parseNetworkFlow(line string) (NetworkFlow, bool) {
var f NetworkFlow
fields := strings.Fields(line)
protoIndex := -1
for i, v := range fields {
if v == "tcp" || v == "udp" || v == "icmp" || v == "icmpv6" {
f.Protocol = v
protoIndex = i
break
}
}
if protoIndex < 0 {
return f, false
}
values := map[string][]string{}
for _, v := range fields[protoIndex+1:] {
if k, val, ok := strings.Cut(v, "="); ok {
values[k] = append(values[k], val)
}
}
get := func(k string, i int) string {
if len(values[k]) > i {
return values[k][i]
}
return ""
}
integer := func(k string, i int) int { n, _ := strconv.Atoi(get(k, i)); return n }
number := func(k string, i int) uint64 { n, _ := strconv.ParseUint(get(k, i), 10, 64); return n }
f.Source = get("src", 0)
f.Destination = get("dst", 0)
f.ReplySource = get("src", 1)
f.ReplyDestination = get("dst", 1)
if net.ParseIP(f.Source) == nil || net.ParseIP(f.Destination) == nil || net.ParseIP(f.ReplySource) == nil || net.ParseIP(f.ReplyDestination) == nil {
return f, false
}
f.SourcePort = integer("sport", 0)
f.DestinationPort = integer("dport", 0)
f.ReplySourcePort = integer("sport", 1)
f.ReplyDestinationPort = integer("dport", 1)
f.State = "TRACKED"
if f.Protocol == "tcp" && len(fields) > protoIndex+3 && !strings.Contains(fields[protoIndex+3], "=") {
f.State = fields[protoIndex+3]
}
if strings.Contains(line, "[UNREPLIED]") {
f.State = "UNREPLIED"
if f.Protocol == "tcp" && len(fields) > protoIndex+3 {
f.State = fields[protoIndex+3]
}
}
if f.Protocol == "udp" && strings.Contains(line, "[ASSURED]") {
f.State = "ASSURED"
}
f.Accounting = len(values["bytes"]) == 2 && len(values["packets"]) == 2
f.Bytes = number("bytes", 0)
f.ReplyBytes = number("bytes", 1)
f.Packets = number("packets", 0)
f.ReplyPackets = number("packets", 1)
f.Zone = get("zone", 0)
return f, true
}
func appendNetworkFlows(snapshot *NetworkSnapshot, raw []byte) {
for _, line := range strings.Split(string(raw), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
f, ok := parseNetworkFlow(line)
if !ok {
if strings.Contains(line, "src=") {
snapshot.ParseErrors++
}
continue
}
if len(snapshot.Flows) >= 5000 {
snapshot.Limited = true
continue
}
snapshot.Flows = append(snapshot.Flows, f)
}
}
// Only counters immediately before a terminal drop/reject in a CrowdSec table/chain
// are reported as blocked packets. General chain and set counters are not summed.
func parseFirewallCounters(raw []byte) ([]FirewallCounter, error) {
var doc struct {
NFTables []struct {
Rule *struct {
Family string `json:"family"`
Table string `json:"table"`
Chain string `json:"chain"`
Handle int `json:"handle"`
Expr []map[string]json.RawMessage `json:"expr"`
} `json:"rule"`
} `json:"nftables"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, err
}
result := []FirewallCounter{}
for _, item := range doc.NFTables {
r := item.Rule
if r == nil || !strings.Contains(strings.ToLower(r.Table+" "+r.Chain), "crowdsec") {
continue
}
if len(r.Expr) < 2 {
continue
}
last := r.Expr[len(r.Expr)-1]
_, drop := last["drop"]
_, reject := last["reject"]
if !drop && !reject {
continue
}
counter, ok := r.Expr[len(r.Expr)-2]["counter"]
if !ok {
continue
}
var c struct {
Packets uint64 `json:"packets"`
Bytes uint64 `json:"bytes"`
}
if json.Unmarshal(counter, &c) != nil || len(counter) == 0 || counter[0] != '{' {
continue
}
result = append(result, FirewallCounter{Name: fmt.Sprintf("%s/%s/%s #%d", r.Family, r.Table, r.Chain, r.Handle), Packets: c.Packets, Bytes: c.Bytes})
if len(result) == 128 {
break
}
}
return result, nil
}
func collectNetwork(ctx context.Context) NetworkSnapshot {
s := NetworkSnapshot{At: time.Now().Unix(), Flows: []NetworkFlow{}}
if runtime.GOOS != "linux" {
s.Error = "Сбор conntrack поддерживается только на Linux"
return s
}
// Proc contains both address families when available. Modern kernels often expose only netlink.
for _, path := range []string{"/proc/net/nf_conntrack", "/proc/net/ip_conntrack"} {
f, err := os.Open(path)
if err != nil {
continue
}
raw, err := io.ReadAll(io.LimitReader(f, (8<<20)+1))
f.Close()
if err != nil {
s.Error = err.Error()
return s
}
if len(raw) > 8<<20 {
s.Error = "Таблица conntrack превышает лимит чтения 8 MiB"
return s
}
appendNetworkFlows(&s, raw)
return s
}
for _, family := range []string{"ipv4", "ipv6"} {
raw, err := networkCommand(ctx, "conntrack", "-L", "-f", family, "-o", "extended")
if err != nil {
s.Error += "Conntrack " + family + ": нужен пакет conntrack и права root/CAP_NET_ADMIN. " + err.Error() + "; "
continue
}
appendNetworkFlows(&s, raw)
}
return s
}
func startAgentNetwork(config agentConfig) func() {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
client := &http.Client{Timeout: 10 * time.Second}
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
var firewall []FirewallCounter
var firewallAt int64
var firewallErr string
lastFirewall := time.Time{}
for {
if ctx.Err() != nil {
return
}
snapshot := NetworkSnapshot{At: time.Now().Unix(), Flows: []NetworkFlow{}}
if message := ensureNetworkDependency(); message != "" {
snapshot.Error = message
} else {
snapshot = collectNetwork(ctx)
}
if time.Since(lastFirewall) >= 30*time.Second {
lastFirewall = time.Now()
raw, err := networkCommand(ctx, "nft", "-j", "list", "ruleset")
firewall = nil
firewallAt = 0
firewallErr = ""
if err == nil {
firewall, err = parseFirewallCounters(raw)
}
if err != nil {
firewallErr = "Счётчики nftables недоступны: " + err.Error()
} else if len(firewall) == 0 {
firewallErr = "В правилах CrowdSec нет поддерживаемых счётчиков перед drop/reject. Это не означает отсутствие блокировок."
} else {
firewallAt = time.Now().Unix()
}
}
snapshot.Firewall = firewall
snapshot.FirewallAt = firewallAt
snapshot.FirewallError = firewallErr
data, _ := json.Marshal(snapshot)
req, err := http.NewRequestWithContext(ctx, "POST", config.Server+"/api/agent/network", bytes.NewReader(data))
if err != nil {
return
}
req.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret)
req.Header.Set("Content-Type", "application/json")
if res, err := client.Do(req); err == nil {
io.Copy(io.Discard, io.LimitReader(res.Body, 4096))
res.Body.Close()
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
return func() { cancel(); <-done }
}

99
agent_network_setup.go Normal file
View File

@@ -0,0 +1,99 @@
package main
import (
"fmt"
"os"
"os/exec"
"runtime"
"strings"
"sync"
"time"
)
// Package work runs in a transient systemd unit, as do existing agent update
// actions. It must not inherit ProtectSystem=strict from the reporting agent.
// A started package transaction is allowed to finish if collection is disabled.
type networkDependencyState struct {
sync.Mutex
running bool
nextAttempt time.Time
message string
}
var networkDependency networkDependencyState
func (d *networkDependencyState) ensure(now time.Time, present bool, install func() error) string {
d.Lock()
defer d.Unlock()
if present {
return ""
}
if d.running {
return "Автоустановка conntrack… Сбор начнётся после установки пакета."
}
if now.Before(d.nextAttempt) {
return d.message + " Повторная попытка не раньше " + d.nextAttempt.Format("15:04:05") + " (время агента)."
}
d.running = true
d.nextAttempt = now.Add(30 * time.Minute)
go func() {
err := install()
d.Lock()
defer d.Unlock()
d.running = false
if err != nil {
d.message = "Не удалось установить conntrack: " + trafficText(err.Error(), 400)
} else {
d.message = "Установка завершена; ожидается появление conntrack в PATH"
}
}()
return "Автоустановка conntrack… Сбор начнётся после установки пакета."
}
func ensureNetworkDependency() string {
if runtime.GOOS != "linux" {
return "Сбор conntrack поддерживается только на Linux"
}
_, err := exec.LookPath("conntrack")
return networkDependency.ensure(time.Now(), err == nil, installNetworkDependency)
}
func installNetworkDependency() error {
if os.Geteuid() != 0 {
return fmt.Errorf("для автоустановки нужны права root")
}
if apk, err := exec.LookPath("apk"); err == nil {
out := runAgentCombinedCommand(10*time.Minute, apk, "add", "--no-cache", "conntrack-tools")
if strings.HasPrefix(out, "ERROR: ") {
return fmt.Errorf("%s", out)
}
if _, err := exec.LookPath("conntrack"); err != nil {
return fmt.Errorf("apk завершился, но conntrack не найден")
}
return nil
}
apt, err := exec.LookPath("apt-get")
if err != nil {
return fmt.Errorf("автоустановка поддерживается на Debian/Ubuntu с apt-get; установите conntrack вручную")
}
run := func(suffix string, args ...string) error {
out := runPrivilegedTransient(10*time.Minute, suffix, apt, args...)
if strings.HasPrefix(out, "ERROR: ") {
return fmt.Errorf("%s", explainSystemUpdateError(out))
}
return nil
}
return installConntrackPackage(run, func() bool { _, err := exec.LookPath("conntrack"); return err == nil })
}
func installConntrackPackage(run func(string, ...string) error, present func() bool) error {
// No upgrade, autoremove, firewall changes, or runtime sysctl changes.
options := []string{"-o", "DPkg::Lock::Timeout=30", "-o", "Acquire::Retries=1", "-o", "Acquire::http::Timeout=20", "-o", "Acquire::https::Timeout=20"}
if err := run("conntrack-lists", append(append([]string{}, options...), "update")...); err != nil {
return err
}
if err := run("conntrack-install", append(append([]string{}, options...), "-y", "--no-install-recommends", "--no-remove", "install", "conntrack")...); err != nil {
return err
}
if !present() {
return fmt.Errorf("apt-get завершился, но conntrack не найден в PATH")
}
return nil
}

152
agent_test.go Normal file
View File

@@ -0,0 +1,152 @@
package main
import (
"os"
"testing"
"time"
)
func TestAgentEnrollmentTokenIsSingleUse(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
token, err := store.CreateAgentEnrollmentToken()
if err != nil {
t.Fatal(err)
}
credentials, err := store.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "docker-01", Hostname: "vm105", VMID: 105})
if err != nil {
t.Fatal(err)
}
if credentials.ID == "" || credentials.Secret == "" {
t.Fatal("credentials were not generated")
}
if _, err = store.EnrollAgent(agentEnrollmentRequest{Token: token, Hostname: "duplicate"}); err == nil {
t.Fatal("token was accepted twice")
}
}
func TestCurrentAgentUpdateManifestMatchesExecutable(t *testing.T) {
manifest, path, err := currentAgentUpdate()
if err != nil {
t.Fatal(err)
}
if manifest.URL != "/api/agent/binary" || len(manifest.SHA256) != 64 {
t.Fatalf("unexpected manifest: %#v", manifest)
}
if _, err = os.Stat(path); err != nil {
t.Fatal(err)
}
}
func TestAgentOnlyInstallsNewerVersion(t *testing.T) {
if !newerAgentVersion("v0.14.0", "v0.13.0") {
t.Fatal("new version was not detected")
}
if newerAgentVersion("v0.12.0", "v0.13.0") {
t.Fatal("downgrade was allowed")
}
if newerAgentVersion("v0.13.0", "v0.13.0") {
t.Fatal("same version was offered")
}
}
func TestShortDockerDigest(t *testing.T) {
if got := shortDockerDigest("sha256:1234567890abcdef"); got != "1234567890ab" {
t.Fatalf("unexpected short digest: %q", got)
}
}
func TestParseAgentIssues(t *testing.T) {
issues := parseAgentIssues(`{"MESSAGE":"disk error","_SYSTEMD_UNIT":"worker.service","__REALTIME_TIMESTAMP":"1722960000000000"}` + "\n" + `{"MESSAGE":"network error","__REALTIME_TIMESTAMP":"1722960001000000"}`)
if len(issues) != 2 || issues[0].Unit != "worker.service" || issues[1].Unit != "system" || issues[0].Time != 1722960000 {
t.Fatalf("unexpected issues: %#v", issues)
}
}
func TestClassifyAgentService(t *testing.T) {
cases := []struct{ path, pkg, priority, want string }{{"/lib/systemd/system/cron.service", "cron", "important", "system"}, {"/lib/systemd/system/jellyfin.service", "jellyfin", "optional", "added"}, {"/etc/systemd/system/my.service", "", "", "local"}, {"/run/systemd/system/transient.service", "", "", "runtime"}}
for _, tc := range cases {
if got := classifyAgentService(tc.path, tc.pkg, tc.priority); got != tc.want {
t.Errorf("%s: got %s want %s", tc.path, got, tc.want)
}
}
}
func TestAgentServiceDiagnosticsHelpers(t *testing.T) {
if got := compactSystemdExec(`{ path=/usr/bin/example ; argv[]=/usr/bin/example --serve ; }`); got != "/usr/bin/example" {
t.Fatalf("unexpected executable: %q", got)
}
fields := limitedFields("a b c d", 2)
if len(fields) != 2 || fields[0] != "a" || fields[1] != "b" {
t.Fatalf("unexpected limited fields: %#v", fields)
}
ports := appendUniqueInt([]int{80}, 80)
ports = appendUniqueInt(ports, 443)
if len(ports) != 2 || ports[1] != 443 {
t.Fatalf("unexpected ports: %#v", ports)
}
}
func TestAgentReportAuthenticationAndOfflineState(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
token, _ := store.CreateAgentEnrollmentToken()
credentials, _ := store.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "docker", Hostname: "vm", VMID: 101})
report := AgentReport{Hostname: "vm", Version: "test", DockerAvailable: true, Containers: []AgentContainer{{ID: "abc", Name: "web", State: "running"}}}
if err = store.SaveAgentReport(credentials.ID, "wrong", "127.0.0.1", report); err == nil {
t.Fatal("invalid secret accepted")
}
if err = store.SaveAgentReport(credentials.ID, credentials.Secret, "127.0.0.1", report); err != nil {
t.Fatal(err)
}
agents, err := store.Agents()
if err != nil {
t.Fatal(err)
}
if len(agents) != 1 || !agents[0].Online || len(agents[0].Report.Containers) != 1 {
t.Fatalf("unexpected agents: %#v", agents)
}
_, _ = store.db.Exec(`UPDATE agents SET last_seen=? WHERE id=?`, time.Now().Add(-time.Minute).Unix(), credentials.ID)
agents, _ = store.Agents()
if agents[0].Online {
t.Fatal("stale agent is online")
}
}
func TestAgentProblemsBecomeMaintenanceAwareAlerts(t *testing.T) {
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "node", Name: "app", VMID: 105, Online: true, Report: AgentReport{
Services: []AgentService{{Name: "postgres.service", State: "failed", SubState: "failed"}},
Containers: []AgentContainer{{ID: "dead", Name: "web", State: "exited", Status: "Exited (1)"}}, DockerAvailable: true,
}}}}
alerts := evaluateAlerts(metrics, defaultThresholds())
window := MaintenanceWindow{Active: true, VMIDs: []int{105}}
active, planned := splitMaintenanceAlerts(alerts, window)
if len(planned) != 2 || len(active) != 0 {
t.Fatalf("active=%d planned=%d alerts=%#v", len(active), len(planned), alerts)
}
}
func TestAgentDiagnosticsBecomeAlerts(t *testing.T) {
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "node", Name: "app", VMID: 105, Online: true, Report: AgentReport{
Services: []AgentService{{Name: "web.service", State: "active", Flapping: true, Restarts: 8, StartedAt: time.Now().Unix()}},
Filesystems: []AgentFilesystem{{Mountpoint: "/data", TotalBytes: 1000, UsedBytes: 950, UsedPercent: 95}},
}}}}
alerts := evaluateAlerts(metrics, defaultThresholds())
wanted := map[string]bool{"agent-flapping-node-web.service-105": false, "agent-filesystem-node-/data-105": false}
for _, alert := range alerts {
if _, ok := wanted[alert.ID]; ok {
wanted[alert.ID] = true
}
}
for id, found := range wanted {
if !found {
t.Fatalf("missing alert %s in %#v", id, alerts)
}
}
}

420
agent_traffic.go Normal file
View File

@@ -0,0 +1,420 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"syscall"
"time"
)
var npmTrafficPattern = regexp.MustCompile(`^\[([^]]+)\] .*? (\d{3}) - ([A-Z]+) (?:https?|HTTP) (\S+) "([^"]*)" \[Client ([^]]+)\]`)
func parseNPMTraffic(line string) (TrafficEvent, bool) {
m := npmTrafficPattern.FindStringSubmatch(line)
if m == nil {
return parseNPMPlusTraffic(line)
}
ip := net.ParseIP(m[6])
if ip == nil {
return TrafficEvent{}, false
}
ts, err := time.Parse("02/Jan/2006:15:04:05 -0700", m[1])
if err != nil {
return TrafficEvent{}, false
}
status, _ := strconv.Atoi(m[2])
return TrafficEvent{Time: ts.Unix(), Kind: "request", IP: ip.String(), Domain: trafficText(m[4], 253), Method: m[3], Path: trafficPath(m[5]), Status: status}, true
}
type trafficCursor struct {
Offset int64 `json:"offset"`
Generation int64 `json:"generation"`
Seen int64 `json:"seen"`
}
type trafficAgentState struct {
BacklogBytes int64 `json:"backlogBytes"`
Initialized bool `json:"initialized"`
StartedAt int64 `json:"startedAt"`
Cursors map[string]trafficCursor `json:"cursors"`
Pending []TrafficEvent `json:"pending"`
ParseErrors uint64 `json:"parseErrors"`
CrowdSeen map[string]int64 `json:"crowdSeen"`
}
func (state *trafficAgentState) readLogs(dirs ...string) (int, error) {
state.BacklogBytes = 0
var paths []string
seenPaths := map[string]bool{}
for _, dir := range dirs {
for _, pattern := range []string{"*_access.log*", "access.log*"} {
matches, err := filepath.Glob(filepath.Join(dir, pattern))
if err != nil {
return 0, err
}
for _, path := range matches {
if !seenPaths[path] {
paths = append(paths, path)
seenPaths[path] = true
}
}
}
}
sort.Strings(paths)
files := 0
budget := int64(1 << 20)
now := time.Now().Unix()
var firstErr error
for _, path := range paths {
if strings.HasSuffix(path, ".gz") {
continue
}
f, err := os.Open(path)
if err != nil {
firstErr = err
continue
}
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() {
f.Close()
continue
}
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
f.Close()
continue
}
active := strings.HasSuffix(path, "_access.log") || filepath.Base(path) == "access.log"
if active {
files++
}
key := fmt.Sprintf("%d:%d", st.Dev, st.Ino)
cursor, known := state.Cursors[key]
if !known {
if !state.Initialized || !active {
cursor.Offset = info.Size()
}
cursor.Generation = now
}
cursor.Seen = now
if info.Size() < cursor.Offset {
cursor.Offset = 0
cursor.Generation++
}
if len(state.Pending) < 400 && budget > 0 {
_, err = f.Seek(cursor.Offset, io.SeekStart)
if err == nil {
reader := bufio.NewReaderSize(io.LimitReader(f, budget), 64<<10)
for len(state.Pending) < 400 && budget > 0 {
line, readErr := reader.ReadString('\n')
// Keep an unfinished line for the next poll. Bound malformed huge lines.
if readErr != nil {
if len(line) >= 64<<10 {
cursor.Offset += int64(len(line))
state.ParseErrors++
}
break
}
offset := cursor.Offset
cursor.Offset += int64(len(line))
budget -= int64(len(line))
if len(line) > 64<<10 {
state.ParseErrors++
continue
}
e, ok := parseNPMTraffic(line)
if !ok {
state.ParseErrors++
continue
}
if e.Time < now-86400 {
continue
}
e.ID = fmt.Sprintf("npm:%s:%d:%d", key, cursor.Generation, offset)
state.Pending = append(state.Pending, e)
}
} else {
firstErr = err
}
}
if info.Size() > cursor.Offset {
state.BacklogBytes += info.Size() - cursor.Offset
}
state.Cursors[key] = cursor
f.Close()
}
for key, cursor := range state.Cursors {
if cursor.Seen < now-7*86400 {
delete(state.Cursors, key)
}
}
if files > 0 {
state.Initialized = true
}
if firstErr != nil {
return files, errors.New("Не удалось прочитать часть access-логов NPM")
}
if files == 0 {
return files, errors.New("Access-логи NPM не найдены")
}
return files, nil
}
type crowdTrafficAlert struct {
ID int64 `json:"id"`
UUID string `json:"uuid"`
CreatedAt string `json:"created_at"`
Scenario string `json:"scenario"`
Simulated bool `json:"simulated"`
Source struct {
IP string `json:"ip"`
} `json:"source"`
Events []struct {
Meta []struct {
Key string `json:"key"`
Value string `json:"value"`
} `json:"meta"`
} `json:"events"`
Decisions []struct {
ID int64 `json:"id"`
Type string `json:"type"`
Value string `json:"value"`
Simulated bool `json:"simulated"`
} `json:"decisions"`
}
func parseCrowdTraffic(data []byte) ([]TrafficEvent, error) {
var alerts []crowdTrafficAlert
if err := json.Unmarshal(data, &alerts); err != nil {
return nil, err
}
out := []TrafficEvent{}
for _, a := range alerts {
if a.Simulated || net.ParseIP(a.Source.IP) == nil {
continue
}
ts, err := time.Parse(time.RFC3339Nano, a.CreatedAt)
if err != nil {
continue
}
id := a.UUID
if id == "" {
id = fmt.Sprintf("%d:%d", a.ID, ts.Unix())
}
// An alert may span multiple domains. Do not invent a one-to-one link.
domains := map[string]bool{}
for _, event := range a.Events {
for _, m := range event.Meta {
if m.Key == "target_fqdn" {
domains[m.Value] = true
}
}
}
domain := ""
if len(domains) == 1 {
for d := range domains {
domain = d
}
}
out = append(out, TrafficEvent{ID: "cs-alert:" + id, Time: ts.Unix(), Kind: "alert", IP: a.Source.IP, Domain: domain, Reason: a.Scenario})
for _, d := range a.Decisions {
if !d.Simulated && net.ParseIP(d.Value) != nil {
out = append(out, TrafficEvent{ID: fmt.Sprintf("cs-decision:%s:%d", id, d.ID), Time: ts.Unix(), Kind: "decision", IP: d.Value, Domain: domain, Reason: d.Type + ": " + a.Scenario})
}
}
}
return out, nil
}
// cscli uses the existing local machine credentials. No new bouncer key or
// public LAPI port is needed, and only the read-only alerts list is executed.
func collectCrowdTraffic(ctx context.Context, container ...string) ([]TrafficEvent, bool, error) {
ctx, cancel := context.WithTimeout(ctx, 6*time.Second)
defer cancel()
var data []byte
if len(container) > 0 && container[0] != "" {
var err error
data, err = dockerCrowdOutput(ctx, container[0])
if err != nil {
return nil, false, errors.New("CrowdSec в Docker недоступен; проверьте контейнер и Docker socket")
}
} else {
cmd := exec.CommandContext(ctx, "cscli", "alerts", "list", "--since", "24h", "--limit", "50", "-o", "json")
var output limitedTrafficOutput
cmd.Stdout = &output
if err := cmd.Run(); err != nil {
return nil, false, errors.New("cscli недоступен на хосте; контейнер CrowdSec не обнаружен")
}
data = output.Bytes()
}
events, err := parseCrowdTraffic(data)
if err != nil {
return nil, false, errors.New("Не удалось разобрать JSON CrowdSec")
}
var raw []json.RawMessage
json.Unmarshal(data, &raw)
return events, len(raw) >= 50, nil
}
type limitedTrafficOutput struct{ bytes.Buffer }
func (b *limitedTrafficOutput) Write(p []byte) (int, error) {
if b.Len()+len(p) > 4<<20 {
return 0, errors.New("CrowdSec response too large")
}
return b.Buffer.Write(p)
}
func saveTrafficAgentState(path string, state trafficAgentState) error {
data, err := json.Marshal(state)
if err != nil {
return err
}
f, err := os.CreateTemp(filepath.Dir(path), ".traffic-")
if err != nil {
return err
}
name := f.Name()
defer os.Remove(name)
if err = f.Chmod(0600); err == nil {
_, err = f.Write(data)
}
if err == nil {
err = f.Sync()
}
closeErr := f.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
return os.Rename(name, path)
}
func runAgentTraffic(ctx context.Context, config agentConfig, stateDir string) {
dir := os.Getenv("DASHBOARD_NPM_LOG_DIR")
if dir == "off" {
return
}
statePath := filepath.Join(stateDir, "traffic-state.json")
state := trafficAgentState{StartedAt: time.Now().Unix(), Cursors: map[string]trafficCursor{}, CrowdSeen: map[string]int64{}}
if data, err := os.ReadFile(statePath); err == nil {
if json.Unmarshal(data, &state) != nil {
fmt.Fprintln(os.Stderr, "traffic: invalid checkpoint; collector stopped")
return
}
} else if !os.IsNotExist(err) {
fmt.Fprintln(os.Stderr, "traffic: checkpoint unreadable")
return
}
if state.Cursors == nil {
state.Cursors = map[string]trafficCursor{}
}
if state.CrowdSeen == nil {
state.CrowdSeen = map[string]int64{}
}
client := &http.Client{Timeout: 10 * time.Second}
lastCrowd := time.Time{}
lastDiscovery := time.Time{}
discovery := trafficDiscovery{}
crowdAt := int64(0)
crowdError := ""
crowdLimited := false
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
default:
}
if time.Since(lastDiscovery) >= time.Minute {
discovery = discoverTraffic(ctx)
lastDiscovery = time.Now()
}
files, logErr := state.readLogs(discovery.Dirs...)
if logErr != nil {
logErr = fmt.Errorf("%s; проверьте access-логи и LOGROTATE=true у NPM+", logErr)
}
if len(state.Pending) < 400 && time.Since(lastCrowd) >= 15*time.Second {
events, limited, err := collectCrowdTraffic(ctx, discovery.CrowdContainer)
if discovery.Error != "" {
if _, localErr := exec.LookPath("cscli"); localErr != nil {
err = errors.New(discovery.Error)
}
}
crowdLimited = limited
lastCrowd = time.Now()
if err != nil {
crowdError = err.Error()
} else {
crowdError = ""
crowdAt = time.Now().Unix()
for _, e := range events {
if _, seen := state.CrowdSeen[e.ID]; !seen && len(state.Pending) < 500 {
state.Pending = append(state.Pending, e)
state.CrowdSeen[e.ID] = e.Time
}
}
for id, ts := range state.CrowdSeen {
if ts < time.Now().Unix()-86400 {
delete(state.CrowdSeen, id)
}
}
}
}
// Persist the pending batch AND offsets before sending. A retry/restart may
// resend events; the server's (agent,event_id) unique key makes it idempotent.
if err := saveTrafficAgentState(statePath, state); err != nil {
fmt.Fprintln(os.Stderr, "traffic: checkpoint write failed; collector stopped")
return
}
batch := TrafficBatch{BacklogBytes: state.BacklogBytes, CrowdSecLimited: crowdLimited, Events: state.Pending, Files: files, ParseErrors: state.ParseErrors, CrowdSecAt: crowdAt, CrowdSecError: crowdError, StartedAt: state.StartedAt}
if logErr != nil {
batch.LogError = logErr.Error()
}
data, _ := json.Marshal(batch)
req, err := http.NewRequestWithContext(ctx, "POST", config.Server+"/api/agent/traffic", bytes.NewReader(data))
if err != nil {
return
}
req.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret)
req.Header.Set("Content-Type", "application/json")
if res, err := client.Do(req); err == nil {
io.Copy(io.Discard, io.LimitReader(res.Body, 4096))
res.Body.Close()
if res.StatusCode == 200 {
state.Pending = nil
if saveTrafficAgentState(statePath, state) != nil {
return
}
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func startAgentTraffic(config agentConfig, stateDir string) func() {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { defer close(done); runAgentTraffic(ctx, config, stateDir) }()
return func() { cancel(); <-done }
}

209
agent_traffic_discovery.go Normal file
View File

@@ -0,0 +1,209 @@
package main
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
type trafficDockerContainer struct {
ID string `json:"Id"`
Image string
Names []string
Mounts []struct{ Source, Destination string }
}
type trafficDiscovery struct {
Dirs []string
CrowdContainer string
Error string
}
func trafficDockerClient() *http.Client {
return &http.Client{Timeout: 8 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}}
}
func discoverTraffic(ctx context.Context) trafficDiscovery {
d := trafficDiscovery{Dirs: []string{"/opt/npm/data/logs", "/opt/npmplus/nginx/logs", "/opt/npmplus/nginx"}}
client := trafficDockerClient()
defer client.CloseIdleConnections()
req, _ := http.NewRequestWithContext(ctx, "GET", "http://docker/containers/json", nil)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
var containers []trafficDockerContainer
if res.StatusCode == 200 && json.NewDecoder(io.LimitReader(res.Body, 2<<20)).Decode(&containers) == nil {
d = trafficSources(containers, d.Dirs)
} else {
d.Error = "Не удалось прочитать список Docker-контейнеров"
}
} else if _, e := os.Stat("/var/run/docker.sock"); e == nil {
d.Error = "Docker socket недоступен агенту"
}
if dir := os.Getenv("DASHBOARD_NPM_LOG_DIR"); dir != "" {
d.Dirs = []string{dir}
}
if container := os.Getenv("DASHBOARD_CROWDSEC_CONTAINER"); container != "" {
d.CrowdContainer = container
d.Error = ""
}
return d
}
func trafficSources(containers []trafficDockerContainer, defaults []string) trafficDiscovery {
d := trafficDiscovery{Dirs: append([]string{}, defaults...)}
var crowd []string
for _, c := range containers {
img := strings.ToLower(c.Image)
if strings.Contains(img, "crowdsecurity/crowdsec:") || strings.HasSuffix(img, "crowdsecurity/crowdsec") || strings.Contains(img, "crowdsecurity/crowdsec@") {
crowd = append(crowd, c.ID)
}
if !strings.Contains(img, "nginx-proxy-manager") && !strings.Contains(img, "npmplus") {
continue
}
for _, target := range []string{"/data/logs", "/data/nginx/logs", "/data/nginx"} {
// Use the most specific mount, including a nested bind or named volume.
best, source := "", ""
for _, m := range c.Mounts {
dst := filepath.Clean(m.Destination)
if filepath.IsAbs(m.Source) && (target == dst || strings.HasPrefix(target, dst+"/")) && len(dst) > len(best) {
best = dst
source = filepath.Join(m.Source, strings.TrimPrefix(target, dst))
}
}
if source != "" {
d.Dirs = append(d.Dirs, source)
}
}
}
if len(crowd) == 1 {
d.CrowdContainer = crowd[0]
} else if len(crowd) > 1 {
d.Error = "Найдено несколько CrowdSec: задайте DASHBOARD_CROWDSEC_CONTAINER"
}
unique := map[string]bool{}
dirs := []string{}
for _, dir := range d.Dirs {
dir = filepath.Clean(dir)
if !unique[dir] {
unique[dir] = true
dirs = append(dirs, dir)
}
}
d.Dirs = dirs
return d
}
// Docker exec uses a fixed read-only command; no shell, credential extraction or published LAPI.
func dockerCrowdOutput(ctx context.Context, container string) ([]byte, error) {
client := trafficDockerClient()
defer client.CloseIdleConnections()
return dockerCrowdWithClient(ctx, container, client)
}
func dockerCrowdWithClient(ctx context.Context, container string, client *http.Client) ([]byte, error) {
if !safeContainerName.MatchString(container) || strings.HasPrefix(container, "-") {
return nil, fmt.Errorf("некорректный контейнер CrowdSec")
}
call := func(path string, payload any) (*http.Response, error) {
raw, _ := json.Marshal(payload)
req, _ := http.NewRequestWithContext(ctx, "POST", "http://docker"+path, bytes.NewReader(raw))
req.Header.Set("Content-Type", "application/json")
return client.Do(req)
}
res, err := call("/containers/"+container+"/exec", map[string]any{"AttachStdout": true, "AttachStderr": true, "Tty": false, "Cmd": []string{"cscli", "alerts", "list", "--since", "24h", "--limit", "50", "-o", "json"}})
if err != nil {
return nil, err
}
var created struct{ ID string }
err = json.NewDecoder(io.LimitReader(res.Body, 4096)).Decode(&created)
res.Body.Close()
if err != nil || res.StatusCode != 201 || !safeContainerName.MatchString(created.ID) {
return nil, fmt.Errorf("Docker не создал cscli exec")
}
res, err = call("/exec/"+created.ID+"/start", map[string]any{"Detach": false, "Tty": false})
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return nil, fmt.Errorf("Docker cscli exec: %s", res.Status)
}
output, err := readDockerTrafficOutput(res.Body)
if err != nil {
return nil, err
}
req, _ := http.NewRequestWithContext(ctx, "GET", "http://docker/exec/"+created.ID+"/json", nil)
check, err := client.Do(req)
if err != nil {
return nil, err
}
defer check.Body.Close()
var status struct {
Running bool
ExitCode int
}
if check.StatusCode != 200 || json.NewDecoder(io.LimitReader(check.Body, 4096)).Decode(&status) != nil || status.Running || status.ExitCode != 0 {
return nil, fmt.Errorf("cscli внутри Docker завершился с ошибкой")
}
return output, nil
}
func readDockerTrafficOutput(r io.Reader) ([]byte, error) {
var out bytes.Buffer
total := uint32(0)
for {
var header [8]byte
_, err := io.ReadFull(r, header[:])
if err == io.EOF {
return out.Bytes(), nil
}
if err != nil {
return nil, err
}
n := binary.BigEndian.Uint32(header[4:])
if n > 4<<20 || total+n > 4<<20 {
return nil, fmt.Errorf("Docker output too large")
}
total += n
if header[0] != 1 && header[0] != 2 {
return nil, fmt.Errorf("invalid Docker stream")
}
if header[0] == 1 {
_, err = io.CopyN(&out, r, int64(n))
} else {
_, err = io.CopyN(io.Discard, r, int64(n))
}
if err != nil {
return nil, err
}
}
}
var npmPlusTrafficPattern = regexp.MustCompile(`^\[([^]]+)\] (\S+) (\S+) [0-9.]+ "([A-Z]+) ([^" ]+) HTTP/[0-9.]+" (\d{3}) `)
func parseNPMPlusTraffic(line string) (TrafficEvent, bool) {
m := npmPlusTrafficPattern.FindStringSubmatch(line)
if m == nil {
return TrafficEvent{}, false
}
ip := net.ParseIP(m[3])
ts, err := time.Parse("02/Jan/2006:15:04:05 -0700", m[1])
if ip == nil || err != nil {
return TrafficEvent{}, false
}
host := m[2]
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
status, _ := strconv.Atoi(m[6])
return TrafficEvent{Time: ts.Unix(), Kind: "request", IP: ip.String(), Domain: trafficText(host, 253), Method: m[4], Path: trafficPath(m[5]), Status: status}, true
}

View File

@@ -0,0 +1,134 @@
package main
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestNPMPlusTraffic(t *testing.T) {
for _, ip := range []string{"8.8.8.8", "2001:4860:4860::8888"} {
line := `[14/Oct/2026:16:24:17 +0000] example.test:8443 ` + ip + ` 0.001 "GET /health?token=secret HTTP/3.0" 200 123 "-" "test" "-" "text/html" TLSv1.3 - - -`
e, ok := parseNPMTraffic(line)
if !ok || e.IP != ip || e.Domain != "example.test" || e.Path != "/health" || e.Status != 200 {
t.Fatalf("%+v %v", e, ok)
}
}
if _, ok := parseNPMTraffic(`[14/Oct/2026:16:24:17 +0000] 443 8.8.8.8 UDP - 200 10 20 0.001`); ok {
t.Fatal("stream counted as HTTP")
}
}
func TestDockerTrafficDiscovery(t *testing.T) {
var containers []trafficDockerContainer
json.Unmarshal([]byte(`[{"Id":"aaa","Image":"ghcr.io/zoeyvid/npmplus:latest","Mounts":[{"Source":"/var/lib/docker/volumes/npm/_data","Destination":"/data"},{"Source":"/srv/logs","Destination":"/data/nginx/logs"}]},{"Id":"bbb","Image":"crowdsecurity/crowdsec:latest"}]`), &containers)
d := trafficSources(containers, nil)
if d.CrowdContainer != "bbb" || d.Error != "" || strings.Join(d.Dirs, ",") != "/var/lib/docker/volumes/npm/_data/logs,/srv/logs,/var/lib/docker/volumes/npm/_data/nginx" {
t.Fatalf("%+v", d)
}
containers = append(containers, containers[1])
d = trafficSources(containers, nil)
if d.CrowdContainer != "" || d.Error == "" {
t.Fatal("ambiguous CrowdSec silently chosen")
}
}
func TestDockerTrafficDemux(t *testing.T) {
var b bytes.Buffer
for _, frame := range []struct {
stream byte
text string
}{{2, "diagnostic"}, {1, "["}, {1, "]"}} {
h := []byte{frame.stream, 0, 0, 0, 0, 0, 0, 0}
binary.BigEndian.PutUint32(h[4:], uint32(len(frame.text)))
b.Write(h)
b.WriteString(frame.text)
}
out, err := readDockerTrafficOutput(&b)
if err != nil || string(out) != "[]" {
t.Fatalf("%q %v", out, err)
}
if _, err := readDockerTrafficOutput(bytes.NewReader([]byte{1, 0, 0})); err == nil {
t.Fatal("truncation accepted")
}
if _, err := readDockerTrafficOutput(bytes.NewReader([]byte{1, 0, 0, 0, 255, 255, 255, 255})); err == nil {
t.Fatal("oversized frame accepted")
}
}
func TestTrafficMultipleDirectoriesAndNPMPlusRotation(t *testing.T) {
a, b := t.TempDir(), t.TempDir()
p := filepath.Join(b, "access.log")
os.WriteFile(filepath.Join(a, "proxy-host-1_access.log"), nil, 0600)
os.WriteFile(p, nil, 0600)
s := trafficAgentState{Cursors: map[string]trafficCursor{}}
if n, err := s.readLogs(a, b, a); err != nil || n != 2 {
t.Fatalf("files %d: %v", n, err)
}
line := "[" + time.Now().UTC().Format("02/Jan/2006:15:04:05 -0700") + `] example.test 8.8.8.8 0.1 "GET / HTTP/2.0" 200 123 "-"` + "\n"
os.WriteFile(p, []byte(line), 0600)
s.readLogs(a, b)
if len(s.Pending) != 1 {
t.Fatal("NPM+ event missing")
}
s.Pending = nil
os.Rename(p, p+".1")
os.WriteFile(p, []byte(line), 0600)
s.readLogs(a, b)
if len(s.Pending) != 1 {
t.Fatalf("rotation duplicated/lost events: %d", len(s.Pending))
}
}
type trafficRoundTrip func(*http.Request) (*http.Response, error)
func (f trafficRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestDockerCrowdExec(t *testing.T) {
for _, exitCode := range []int{0, 1} {
calls := 0
client := &http.Client{Transport: trafficRoundTrip(func(r *http.Request) (*http.Response, error) {
calls++
body := ""
status := 200
switch calls {
case 1:
if r.Method != "POST" || r.URL.Path != "/containers/crowdsec/exec" {
t.Fatalf("unexpected %s %s", r.Method, r.URL)
}
var payload struct {
Cmd []string
Tty bool
}
json.NewDecoder(r.Body).Decode(&payload)
if strings.Join(payload.Cmd, " ") != "cscli alerts list --since 24h --limit 50 -o json" || payload.Tty {
t.Fatalf("unexpected command %+v", payload)
}
status = 201
body = `{"Id":"exec123"}`
case 2:
if r.URL.Path != "/exec/exec123/start" {
t.Fatal(r.URL)
}
body = string([]byte{1, 0, 0, 0, 0, 0, 0, 2}) + "[]"
case 3:
if r.Method != "GET" || r.URL.Path != "/exec/exec123/json" {
t.Fatal(r.URL)
}
body = fmt.Sprintf(`{"Running":false,"ExitCode":%d}`, exitCode)
default:
t.Fatal("extra Docker request")
}
return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
})}
out, err := dockerCrowdWithClient(context.Background(), "crowdsec", client)
if calls != 3 || (exitCode == 0 && (err != nil || string(out) != "[]")) || (exitCode != 0 && err == nil) {
t.Fatalf("calls=%d output=%s err=%v", calls, out, err)
}
}
}

View File

@@ -6,16 +6,18 @@ import (
) )
type AlertEvent struct { type AlertEvent struct {
EventID int64 `json:"eventId"` EventID int64 `json:"eventId"`
ID string `json:"id"` ID string `json:"id"`
Severity string `json:"severity"` Severity string `json:"severity"`
Source string `json:"source"` Source string `json:"source"`
Title string `json:"title"` Title string `json:"title"`
Message string `json:"message"` Message string `json:"message"`
FirstSeen int64 `json:"firstSeen"` FirstSeen int64 `json:"firstSeen"`
LastSeen int64 `json:"lastSeen"` LastSeen int64 `json:"lastSeen"`
ResolvedAt *int64 `json:"resolvedAt,omitempty"` ResolvedAt *int64 `json:"resolvedAt,omitempty"`
AcknowledgedAt *int64 `json:"acknowledgedAt,omitempty"` AcknowledgedAt *int64 `json:"acknowledgedAt,omitempty"`
Planned bool `json:"planned"`
MaintenanceReason string `json:"maintenanceReason,omitempty"`
} }
type RecoveryEvent struct { type RecoveryEvent struct {
@@ -24,13 +26,17 @@ type RecoveryEvent struct {
} }
func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) { func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
return s.SyncAlertsWithMaintenance(alerts, nil, "")
}
func (s *Store) SyncAlertsWithMaintenance(alerts, plannedAlerts []Alert, maintenanceReason string) ([]Alert, error) {
tx, err := s.db.Begin() tx, err := s.db.Begin()
if err != nil { if err != nil {
return alerts, err return alerts, err
} }
defer tx.Rollback() defer tx.Rollback()
now := time.Now().Unix() now := time.Now().Unix()
rows, err := tx.Query(`SELECT id,alert_key FROM alert_events WHERE resolved_at IS NULL`) rows, err := tx.Query(`SELECT id,alert_key,planned FROM alert_events WHERE resolved_at IS NULL`)
if err != nil { if err != nil {
return alerts, err return alerts, err
} }
@@ -38,28 +44,43 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
for rows.Next() { for rows.Next() {
var id int64 var id int64
var key string var key string
if err := rows.Scan(&id, &key); err != nil { var planned bool
if err := rows.Scan(&id, &key, &planned); err != nil {
rows.Close() rows.Close()
return alerts, err return alerts, err
} }
active[key] = id active[alertEventKey(key, planned)] = id
} }
rows.Close() rows.Close()
seen := map[string]bool{} seen := map[string]bool{}
for _, alert := range alerts { syncGroup := func(group []Alert, planned bool) error {
seen[alert.ID] = true for _, alert := range group {
if id, ok := active[alert.ID]; ok { key := alertEventKey(alert.ID, planned)
_, err = tx.Exec(`UPDATE alert_events SET severity=?,source=?,title=?,message=?,last_seen=? WHERE id=?`, alert.Severity, alert.Source, alert.Title, alert.Message, now, id) seen[key] = true
} else { reason := ""
result, insertErr := tx.Exec(`INSERT INTO alert_events(alert_key,severity,source,title,message,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)`, alert.ID, alert.Severity, alert.Source, alert.Title, alert.Message, now, now) if planned {
if insertErr != nil { reason = maintenanceReason
return alerts, insertErr }
if id, ok := active[key]; ok {
_, err = tx.Exec(`UPDATE alert_events SET severity=?,source=?,title=?,message=?,last_seen=?,maintenance_reason=? WHERE id=?`, alert.Severity, alert.Source, alert.Title, alert.Message, now, reason, id)
} else {
result, insertErr := tx.Exec(`INSERT INTO alert_events(alert_key,severity,source,title,message,first_seen,last_seen,planned,maintenance_reason) VALUES(?,?,?,?,?,?,?,?,?)`, alert.ID, alert.Severity, alert.Source, alert.Title, alert.Message, now, now, planned, reason)
if insertErr != nil {
return insertErr
}
active[key], _ = result.LastInsertId()
}
if err != nil {
return err
} }
active[alert.ID], _ = result.LastInsertId()
}
if err != nil {
return alerts, err
} }
return nil
}
if err = syncGroup(alerts, false); err != nil {
return alerts, err
}
if err = syncGroup(plannedAlerts, true); err != nil {
return alerts, err
} }
for key, id := range active { for key, id := range active {
if !seen[key] { if !seen[key] {
@@ -74,7 +95,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
if err = tx.Commit(); err != nil { if err = tx.Commit(); err != nil {
return alerts, err return alerts, err
} }
rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`) rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND planned=0 ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`)
if err != nil { if err != nil {
return alerts, err return alerts, err
} }
@@ -82,7 +103,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
var visible []Alert var visible []Alert
for rows.Next() { for rows.Next() {
var a Alert var a Alert
if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message); err != nil { if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message, &a.FirstSeen, &a.LastSeen); err != nil {
return alerts, err return alerts, err
} }
visible = append(visible, a) visible = append(visible, a)
@@ -90,16 +111,35 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
return visible, rows.Err() return visible, rows.Err()
} }
func alertEventKey(id string, planned bool) string {
if planned {
return id + "\x00planned"
}
return id + "\x00active"
}
func (s *Store) AcknowledgeAlert(eventID int64) error { func (s *Store) AcknowledgeAlert(eventID int64) error {
_, err := s.db.Exec(`UPDATE alert_events SET acknowledged_at=? WHERE id=? AND acknowledged_at IS NULL`, time.Now().Unix(), eventID) _, err := s.db.Exec(`UPDATE alert_events SET acknowledged_at=? WHERE id=? AND acknowledged_at IS NULL`, time.Now().Unix(), eventID)
return err return err
} }
func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) { func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
return s.AlertHistoryBetween(limit, 0, 0)
}
func (s *Store) AlertHistoryBetween(limit int, from, to int64) ([]AlertEvent, error) {
if limit < 1 || limit > 500 { if limit < 1 || limit > 500 {
limit = 200 limit = 200
} }
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at FROM alert_events ORDER BY first_seen DESC LIMIT ?`, limit) query := `SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events`
args := []any{}
if from > 0 && to > from {
query += ` WHERE first_seen>=? AND first_seen<?`
args = append(args, from, to)
}
query += ` ORDER BY first_seen DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.Query(query, args...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -108,7 +148,7 @@ func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
for rows.Next() { for rows.Next() {
var e AlertEvent var e AlertEvent
var resolved, acked sql.NullInt64 var resolved, acked sql.NullInt64
if err = rows.Scan(&e.EventID, &e.ID, &e.Severity, &e.Source, &e.Title, &e.Message, &e.FirstSeen, &e.LastSeen, &resolved, &acked); err != nil { if err = rows.Scan(&e.EventID, &e.ID, &e.Severity, &e.Source, &e.Title, &e.Message, &e.FirstSeen, &e.LastSeen, &resolved, &acked, &e.Planned, &e.MaintenanceReason); err != nil {
return nil, err return nil, err
} }
if resolved.Valid { if resolved.Valid {
@@ -123,7 +163,7 @@ func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
} }
func (s *Store) PendingEmailAlerts() ([]Alert, error) { func (s *Store) PendingEmailAlerts() ([]Alert, error) {
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND email_sent_at IS NULL ORDER BY first_seen`) rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND email_sent_at IS NULL AND planned=0 ORDER BY first_seen`)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -155,7 +195,7 @@ func (s *Store) MarkAlertsEmailed(alerts []Alert) error {
} }
func (s *Store) PendingImmediateAlerts() ([]Alert, error) { func (s *Store) PendingImmediateAlerts() ([]Alert, error) {
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND email_sent_at IS NULL AND (severity='critical' OR alert_key='ups-battery') ORDER BY first_seen`) rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND email_sent_at IS NULL AND planned=0 AND (severity='critical' OR alert_key='ups-battery') ORDER BY first_seen`)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -171,7 +211,7 @@ func (s *Store) PendingImmediateAlerts() ([]Alert, error) {
return out, rows.Err() return out, rows.Err()
} }
func (s *Store) PendingRecoveries() ([]RecoveryEvent, error) { func (s *Store) PendingRecoveries() ([]RecoveryEvent, error) {
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NOT NULL AND email_sent_at IS NOT NULL AND recovery_sent_at IS NULL ORDER BY resolved_at`) rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NOT NULL AND email_sent_at IS NOT NULL AND recovery_sent_at IS NULL AND planned=0 ORDER BY resolved_at`)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@@ -10,7 +10,7 @@ func TestAlertLifecycle(t *testing.T) {
defer store.Close() defer store.Close()
alert := Alert{ID: "disk", Severity: "warning", Source: "Диски", Title: "SMART", Message: "Проверьте диск"} alert := Alert{ID: "disk", Severity: "warning", Source: "Диски", Title: "SMART", Message: "Проверьте диск"}
visible, err := store.SyncAlerts([]Alert{alert}) visible, err := store.SyncAlerts([]Alert{alert})
if err != nil || len(visible) != 1 || visible[0].EventID == 0 { if err != nil || len(visible) != 1 || visible[0].EventID == 0 || visible[0].FirstSeen == 0 || visible[0].LastSeen == 0 {
t.Fatalf("unexpected new alert: %+v %v", visible, err) t.Fatalf("unexpected new alert: %+v %v", visible, err)
} }
firstID := visible[0].EventID firstID := visible[0].EventID

156
alerts.go
View File

@@ -8,12 +8,14 @@ import (
) )
type Alert struct { type Alert struct {
EventID int64 `json:"eventId,omitempty"` EventID int64 `json:"eventId,omitempty"`
ID string `json:"id"` ID string `json:"id"`
Severity string `json:"severity"` Severity string `json:"severity"`
Source string `json:"source"` Source string `json:"source"`
Title string `json:"title"` Title string `json:"title"`
Message string `json:"message"` Message string `json:"message"`
FirstSeen int64 `json:"firstSeen,omitempty"`
LastSeen int64 `json:"lastSeen,omitempty"`
} }
func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert { func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert {
@@ -61,10 +63,21 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent)) add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent))
} }
for _, disk := range metrics.Disks { for _, disk := range metrics.Disks {
diskExcluded := false
for _, value := range strings.Split(thresholds.DiskIOExcluded, ",") {
value = strings.TrimSpace(value)
if value != "" && (strings.EqualFold(value, disk.Name) || strings.EqualFold(value, disk.Serial)) {
diskExcluded = true
break
}
}
if disk.SMARTStatus == "Ошибка" { if disk.SMARTStatus == "Ошибка" {
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска.")) add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
} else if disk.SMARTStatus == "Требует внимания" { } else if disk.SMARTStatus == "Требует внимания" {
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(disk.AttentionReasons, "Проверьте SMART-показатели диска.")) reasons := withoutReason(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался")
if len(reasons) > 0 {
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(reasons, "Проверьте SMART-показатели диска."))
}
} }
if disk.Temperature != nil { if disk.Temperature != nil {
limit := thresholds.DiskTempWarning limit := thresholds.DiskTempWarning
@@ -78,12 +91,16 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
if len(disk.CounterChanges) > 0 { if len(disk.CounterChanges) > 0 {
add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · ")) add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · "))
} }
if disk.IOPressureSeconds >= 60 { if !diskExcluded && disk.IOPressureSeconds >= thresholds.DiskIODuration {
severity := "warning" severity := "warning"
if disk.Utilization >= 98 || disk.LatencyMs >= 100 { if disk.Utilization >= 98 || disk.LatencyMs >= 100 {
severity = "critical" severity = "critical"
} }
add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("Нагрузка держится %.0f сек.: %.0f IOPS, %.1f MB/s, latency %.1f ms, util %.1f%%, очередь %.1f.", disk.IOPressureSeconds, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.Utilization, disk.AverageQueue)) cause := disk.IOCause
if cause == "" {
cause = "источник пока не определён"
}
add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("%s занят на %.1f%%; вероятная причина: %s (уверенность %d%%). %.0f IOPS, %.1f MB/s, latency %.1f ms, очередь %.1f.", disk.Name, disk.Utilization, cause, disk.IOConfidence, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.AverageQueue))
} }
if disk.SelfTestNeverRun { if disk.SelfTestNeverRun {
add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.") add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.")
@@ -183,6 +200,50 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("guests-stopped", "info", "VM/LXC", "Есть остановленные гости", fmt.Sprintf("Остановлено VM/LXC: %d. Это уведомление информационное.", stopped)) add("guests-stopped", "info", "VM/LXC", "Есть остановленные гости", fmt.Sprintf("Остановлено VM/LXC: %d. Это уведомление информационное.", stopped))
} }
} }
for _, agent := range metrics.Agents {
vmSuffix := fmt.Sprintf("-%d", agent.VMID)
if !agent.Online {
add("agent-offline-"+agent.ID+vmSuffix, "critical", "Агенты", agent.Name+": агент не отвечает", fmt.Sprintf("Последний heartbeat: %s назад. VMID %d.", time.Since(time.Unix(agent.LastSeen, 0)).Round(time.Second), agent.VMID))
continue
}
if agent.Report.DockerAvailable {
for _, container := range agent.Report.Containers {
if container.State != "running" {
add("agent-container-stopped-"+agent.ID+"-"+container.ID+vmSuffix, "warning", "Docker", agent.Name+": контейнер "+container.Name+" остановлен", fmt.Sprintf("%s · VMID %d.", container.Status, agent.VMID))
} else if container.Health == "unhealthy" {
add("agent-container-health-"+agent.ID+"-"+container.ID+vmSuffix, "critical", "Docker", agent.Name+": контейнер "+container.Name+" unhealthy", fmt.Sprintf("Docker health-check не проходит · VMID %d.", agent.VMID))
}
}
}
for _, service := range agent.Report.Services {
if service.State == "failed" || service.SubState == "failed" {
add("agent-systemd-"+agent.ID+"-"+service.Name+vmSuffix, "critical", "Агенты", agent.Name+": упала служба "+service.Name, fmt.Sprintf("%s · VMID %d.", firstNonEmpty(service.Description, service.SubState), agent.VMID))
} else if service.Flapping {
add("agent-flapping-"+agent.ID+"-"+service.Name+vmSuffix, "warning", "Агенты", agent.Name+": служба постоянно перезапускается", fmt.Sprintf("%s: %d рестартов, последний запуск %s · VMID %d.", service.Name, service.Restarts, time.Unix(service.StartedAt, 0).Format("15:04:05"), agent.VMID))
}
}
for _, filesystem := range agent.Report.Filesystems {
if filesystem.UsedPercent >= 90 {
add("agent-filesystem-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчивается место "+filesystem.Mountpoint, fmt.Sprintf("Заполнено %.1f%%, свободно %.1f GB · VMID %d.", filesystem.UsedPercent, float64(filesystem.TotalBytes-filesystem.UsedBytes)/1073741824, agent.VMID))
}
if filesystem.InodesPercent >= 90 {
add("agent-inodes-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчиваются inode "+filesystem.Mountpoint, fmt.Sprintf("Использовано %.1f%% inode · VMID %d.", filesystem.InodesPercent, agent.VMID))
}
}
if len(agent.Report.OOMKills) > 0 {
latest := agent.Report.OOMKills[0]
add("agent-oom-"+agent.ID+vmSuffix, "critical", "Агенты", agent.Name+": ядро завершало процессы из-за нехватки памяти", fmt.Sprintf("%s · %s · VMID %d.", time.Unix(latest.Time, 0).Format("02.01 15:04"), latest.Message, agent.VMID))
}
if agent.Report.RebootRequired {
add("agent-reboot-"+agent.ID+vmSuffix, "info", "Агенты", agent.Name+": требуется перезагрузка", fmt.Sprintf("Обновления системы запросили перезагрузку · VMID %d.", agent.VMID))
}
if agent.Report.MemoryTotal > 0 && float64(agent.Report.MemoryUsed)/float64(agent.Report.MemoryTotal)*100 >= 95 {
add("agent-memory-"+agent.ID+vmSuffix, "warning", "Агенты", agent.Name+": почти закончилась память", fmt.Sprintf("Используется %.1f%% RAM · VMID %d.", float64(agent.Report.MemoryUsed)/float64(agent.Report.MemoryTotal)*100, agent.VMID))
}
if agent.Report.RootTotal > 0 && float64(agent.Report.RootUsed)/float64(agent.Report.RootTotal)*100 >= 90 {
add("agent-storage-"+agent.ID+vmSuffix, "critical", "Агенты", agent.Name+": мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%% · VMID %d.", float64(agent.Report.RootUsed)/float64(agent.Report.RootTotal)*100, agent.VMID))
}
}
for _, service := range metrics.Services.Services { for _, service := range metrics.Services.Services {
for _, endpoint := range service.Endpoints { for _, endpoint := range service.Endpoints {
if !endpoint.Up { if !endpoint.Up {
@@ -202,10 +263,13 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
} }
var networkErrors uint64 var networkErrors uint64
for _, iface := range metrics.Network.Interfaces { for _, iface := range metrics.Network.Interfaces {
networkErrors += iface.ReceiveErrors + iface.TransmitErrors + iface.ReceiveDrops + iface.TransmitDrops networkErrors += iface.NewErrors + iface.NewFaults
if iface.LinkDown {
add("network-link-"+iface.Name, "critical", "Сеть", iface.Name+": нет линка", "Используемый интерфейс потерял carrier или перешёл в down.")
}
} }
if networkErrors > 0 { if networkErrors > 0 {
add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors)) add("network-errors", "critical", "Сеть", "Появились новые сетевые ошибки", fmt.Sprintf("Прирост счётчиков ошибок: %d (счётчики могут пересекаться). Software RX/TX drops показаны отдельно и не создают этот алерт.", networkErrors))
} }
if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp { if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp {
add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён.")) add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён."))
@@ -225,20 +289,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
for _, name := range metrics.Network.Health.DisappearedInterfaces { for _, name := range metrics.Network.Health.DisappearedInterfaces {
add("network-interface-missing-"+name, "warning", "Сеть", "Исчез сетевой интерфейс", name+" больше не присутствует в /proc/net/dev.") add("network-interface-missing-"+name, "warning", "Сеть", "Исчез сетевой интерфейс", name+" больше не присутствует в /proc/net/dev.")
} }
for _, iface := range metrics.Network.Interfaces {
if iface.Kind == "Physical" && !iface.Carrier && (iface.Master != "" || iface.ReceiveBytes+iface.TransmitBytes > 0) {
add("network-link-"+iface.Name, "warning", "Сеть", iface.Name+": нет carrier", "Физический линк не поднят.")
}
if (iface.Kind == "Bridge" || iface.Kind == "Bond" || iface.Kind == "VLAN") && iface.State == "down" {
add("network-state-"+iface.Name, "warning", "Сеть", iface.Name+": интерфейс выключен", iface.Kind+" находится в состоянии down.")
}
}
if !metrics.Monitor.LastSuccessfulCollection.IsZero() && time.Since(metrics.Monitor.LastSuccessfulCollection) > 30*time.Second { if !metrics.Monitor.LastSuccessfulCollection.IsZero() && time.Since(metrics.Monitor.LastSuccessfulCollection) > 30*time.Second {
add("monitor-stale", "critical", "Dashboard", "Данные мониторинга устарели", "Последний успешный сбор был "+time.Since(metrics.Monitor.LastSuccessfulCollection).Round(time.Second).String()+" назад.") add("monitor-stale", "critical", "Dashboard", "Данные мониторинга устарели", "Последний успешный сбор был "+time.Since(metrics.Monitor.LastSuccessfulCollection).Round(time.Second).String()+" назад.")
} }
if metrics.Monitor.SlowCollector {
add("monitor-slow", "warning", "Dashboard", "Сбор метрик выполняется слишком долго", fmt.Sprintf("Последний цикл занял %d ms.", metrics.Monitor.CollectionDurationMs))
}
if metrics.Monitor.HistoryWriteError != "" { if metrics.Monitor.HistoryWriteError != "" {
add("monitor-history", "critical", "Dashboard", "Ошибка записи истории", metrics.Monitor.HistoryWriteError) add("monitor-history", "critical", "Dashboard", "Ошибка записи истории", metrics.Monitor.HistoryWriteError)
} }
@@ -266,6 +320,9 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status)) add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status))
} }
} }
for _, trend := range metrics.Trends {
add(trend.ID, trend.Severity, trend.Source, trend.Title, trend.Message)
}
priority := map[string]int{"critical": 0, "warning": 1, "info": 2} priority := map[string]int{"critical": 0, "warning": 1, "info": 2}
sort.SliceStable(alerts, func(i, j int) bool { sort.SliceStable(alerts, func(i, j int) bool {
@@ -277,9 +334,64 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
return alerts return alerts
} }
func correlateAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
pressureAlerts := map[string]bool{}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") {
pressureAlerts[strings.TrimPrefix(a.ID, "disk-io-pressure-")] = true
}
}
var pressure []DiskMetrics
for _, d := range metrics.Disks {
if pressureAlerts[d.Name] && d.IOCause != "" {
pressure = append(pressure, d)
}
}
if len(pressure) == 0 {
return alerts
}
var down []string
for _, service := range metrics.Services.Services {
for _, endpoint := range service.Endpoints {
if !endpoint.Up {
down = append(down, service.Name)
break
}
}
}
d := pressure[0]
chain := d.IOCause + " → " + d.Name + fmt.Sprintf(" util %.1f%% → latency %.1f ms", d.Utilization, d.LatencyMs)
if len(down) > 0 {
chain += " → недоступны/медленны сервисы: " + strings.Join(down, ", ")
}
severity := "warning"
if len(down) > 0 || d.Utilization >= 98 {
severity = "critical"
}
correlated := Alert{ID: "correlation-io-" + d.Name, Severity: severity, Source: "Корреляция", Title: "Обнаружена связанная цепочка I/O-событий", Message: chain + fmt.Sprintf(". Уверенность причины %d%%.", d.IOConfidence)}
filtered := []Alert{correlated}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") || strings.HasPrefix(a.ID, "service-") {
continue
}
filtered = append(filtered, a)
}
return filtered
}
func joinReasons(reasons []string, fallback string) string { func joinReasons(reasons []string, fallback string) string {
if len(reasons) == 0 { if len(reasons) == 0 {
return fallback return fallback
} }
return strings.Join(reasons, " · ") return strings.Join(reasons, " · ")
} }
func withoutReason(reasons []string, excluded string) []string {
filtered := make([]string, 0, len(reasons))
for _, reason := range reasons {
if reason != excluded {
filtered = append(filtered, reason)
}
}
return filtered
}

View File

@@ -16,3 +16,26 @@ func TestEvaluateAlerts(t *testing.T) {
t.Fatalf("неверная сортировка уведомлений: %+v", alerts) t.Fatalf("неверная сортировка уведомлений: %+v", alerts)
} }
} }
func TestSlowCollectionDoesNotCreateAlert(t *testing.T) {
metrics := dashboardMetrics{Monitor: MonitorHealth{SlowCollector: true, CollectionDurationMs: 12308}}
for _, alert := range evaluateAlerts(metrics, defaultThresholds()) {
if alert.ID == "monitor-slow" {
t.Fatalf("медленный цикл не должен создавать уведомление: %+v", alert)
}
}
}
func TestNeverRunSelfTestCreatesOneAlert(t *testing.T) {
metrics := dashboardMetrics{Disks: []DiskMetrics{{
Name: "nvme0n1",
Model: "Test NVMe",
SMARTStatus: "Требует внимания",
SelfTestNeverRun: true,
AttentionReasons: []string{"SMART self-test ещё ни разу не запускался"},
}}}
alerts := evaluateAlerts(metrics, defaultThresholds())
if len(alerts) != 1 || alerts[0].ID != "disk-selftest-nvme0n1" {
t.Fatalf("ожидалось одно точное уведомление о self-test, получено: %+v", alerts)
}
}

20
assets/NOTICE.md Normal file
View File

@@ -0,0 +1,20 @@
# Map data
`world.json` is derived from Natural Earth 1:110m admin-0 countries (coordinates
rounded to two decimal places; only country codes, labels and boundaries kept).
Natural Earth data is public domain: https://www.naturalearthdata.com/about/terms-of-use/
Source: https://github.com/nvkelso/natural-earth-vector/blob/master/geojson/ne_110m_admin_0_countries.geojson
`dbip-country-lite-2026-10.mmdb.gz`: IP Geolocation by DB-IP, October 2026.
Source: https://download.db-ip.com/free/dbip-country-lite-2026-10.mmdb.gz
License: Creative Commons Attribution 4.0 International (CC BY 4.0).
https://creativecommons.org/licenses/by/4.0/
https://db-ip.com/db/download/ip-to-country-lite
The database is unmodified. The traffic page displays the required DB-IP link.
Countries are approximate; plotted positions are country labels, not exact IP locations.
Optional DB-IP City Lite October 2026 is downloaded separately through Settings.
Source: https://download.db-ip.com/free/dbip-city-lite-2026-10.mmdb.gz
License: CC BY 4.0, https://creativecommons.org/licenses/by/4.0/
https://db-ip.com/db/download/ip-to-city-lite
The database is unmodified; city and region are approximate.

Binary file not shown.

40
assets/network.html Normal file
View File

@@ -0,0 +1,40 @@
<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Сетевая активность · Proxmox Dashboard</title><style>
:root{color-scheme:dark;--bg:#090e15;--panel:#111b26;--line:#293846;--muted:#92a8ba;--lime:#c0ee86;--blue:#70dcec}*{box-sizing:border-box}html,body{margin:0;height:100%;overflow:hidden}body{background:radial-gradient(ellipse at 30% 30%,#142738,transparent 65%),var(--bg);color:#e9f0f6;font:13px system-ui,sans-serif;padding:22px;display:grid;grid-template-rows:54px 32px 82px 36px minmax(0,1fr) 22px;gap:12px}header{display:flex;align-items:center;justify-content:space-between;gap:12px}h1{font-size:26px;letter-spacing:-1px;margin:0}small,.muted{color:var(--muted)}.eyebrow{font-size:9px;letter-spacing:2px;color:var(--lime)}nav{display:flex;gap:15px;align-items:center}a{color:var(--blue);text-decoration:none}button,input,select{font:inherit;color:inherit;background:#182430;border:1px solid var(--line);padding:7px 10px;border-radius:8px;min-width:0}button{cursor:pointer}button:disabled{opacity:.35;cursor:default}button:hover{border-color:var(--blue)}:focus-visible{outline:2px solid var(--blue);outline-offset:2px}.status{display:flex;align-items:center;gap:10px;color:var(--muted);min-width:0}.status span{overflow:hidden;white-space:nowrap;text-overflow:ellipsis}.badge{font-size:10px;color:var(--lime);border:1px solid #496036;padding:5px 8px;border-radius:20px;white-space:nowrap}.badge.warn{color:#ffbd80;border-color:#875d36}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:12px}.metric,.panel{border:1px solid var(--line);background:#101a24d9;border-radius:14px;min-width:0;min-height:0;overflow:hidden}.metric{padding:12px 16px}.metric small{font-size:11px}.metric b{display:block;font-size:28px;line-height:1.3;letter-spacing:-1px}.filters{display:flex;gap:8px}.filters input{flex:1}.filters label{display:flex;align-items:center;gap:6px;font-size:11px;color:var(--muted)}.filters label input{accent-color:var(--lime);flex:none;width:13px;height:13px;margin:0;grid-column:auto;grid-row:auto}main{display:grid;grid-template-columns:minmax(0,1fr) 290px;gap:12px;min-height:0}.flows{display:grid;grid-template-rows:40px 27px minmax(0,1fr) 38px;padding:0 14px}.heading{display:flex;align-items:center;justify-content:space-between;gap:8px}h2{font-size:12px;font-weight:600;margin:0}.table-head,.flow{display:grid;grid-template-columns:minmax(0,1.25fr) minmax(0,1.25fr) 80px 108px 95px;gap:12px;align-items:center}.table-head{color:var(--muted);font-size:10px}.flow{height:55px;border-top:1px solid #21303d;cursor:pointer;font:12px ui-monospace,monospace}.flow:hover{background:#20344566}.flow>div{min-width:0}.flow strong{font-weight:500;color:var(--blue);display:block;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.flow small{font:10px system-ui;display:block;margin-top:4px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.state{font-size:10px}.state.ok{color:var(--lime)}.state.pending{color:#ffc085}#rows{min-height:0;overflow:hidden}.pager{display:flex;align-items:center;gap:12px;color:var(--muted);font-size:11px;border-top:1px solid var(--line)}.pager button{padding:3px 8px}.pager #resume{margin-left:auto;color:var(--lime)}aside{display:grid;grid-template-rows:1fr 1fr 1fr;gap:12px;min-height:0}aside .panel{padding:12px 14px}aside h2{margin-bottom:9px}.rank{display:flex;justify-content:space-between;gap:10px;font-size:11px;line-height:22px}.rank span{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.rank b{font-weight:500;color:var(--blue)}.block-value{font-size:24px;color:#c4a9ff;margin:8px 0}.help{font-size:10px;color:var(--muted);line-height:1.5}.empty{color:var(--muted);padding:20px 0;line-height:1.7;font-size:12px}.footer{display:flex;justify-content:space-between;font-size:10px;color:var(--muted);gap:10px}.link-button{background:none;border:0;padding:0;color:var(--blue);font-size:11px}dialog{width:min(650px,calc(100vw - 24px));max-height:85dvh;overflow:auto;border:1px solid #455969;border-radius:18px;background:#141f2b;color:#e9f0f6;padding:22px}dialog::backdrop{background:#03080ccc;backdrop-filter:blur(5px)}dialog .heading{margin-bottom:16px}dialog h2{font-size:19px}.detail{padding:10px 0;border-top:1px solid var(--line);overflow-wrap:anywhere}.detail small{display:block;margin-bottom:4px}dialog p{line-height:1.7}#tabs{display:none}.filters #port{width:110px}
@media(max-width:1100px){main{grid-template-columns:minmax(0,1fr) 220px}.table-head,.flow{grid-template-columns:1fr 1fr 55px 83px;gap:8px}.flow .volume,.table-head .volume{display:none}.flow{font-size:11px}nav{gap:9px}body{padding:14px}aside .panel{padding:9px}aside .rank{line-height:19px}}
@media(max-height:700px) and (min-width:761px){body{padding:12px;gap:8px;grid-template-rows:40px 26px 64px 32px minmax(0,1fr) 18px}.metric{padding:8px 12px}.metric b{font-size:22px}.flow{height:46px}aside{grid-template-rows:1fr 1fr 1fr;gap:6px}aside .rank{line-height:17px}.block-value{font-size:20px;margin:3px 0}aside h2{margin-bottom:4px}.help{font-size:9px}}
@media(max-width:760px){body{padding:10px;gap:8px;grid-template-rows:45px 30px 104px auto 28px minmax(0,1fr) 18px}h1{font-size:18px;white-space:nowrap}.eyebrow{font-size:7px}nav a{font-size:0}nav a:before{content:"HTTP ↗";font-size:11px}nav .dashboard{display:none}.metrics{grid-template-columns:1fr 1fr;gap:6px}.metric{padding:6px 10px}.metric small{font-size:9px}.metric b{font-size:22px}.filters{display:grid;grid-template-columns:1fr 100px;gap:5px}.filters label{grid-row:3;white-space:nowrap;font-size:10px}#netMonitoringControl{grid-column:1}#netExternalControl{grid-column:2}.filters #search{grid-column:1/-1;grid-row:2}.filters select{font-size:11px;padding:6px}.filters #port{width:100%}#tabs{display:flex;gap:6px}#tabs button{flex:1;padding:4px;font-size:11px}#tabs button.active{border-color:var(--blue)}main{display:block}.flows{height:100%;padding:0 10px;grid-template-rows:34px 24px minmax(0,1fr) 36px}aside{display:none;height:100%;grid-template-columns:1fr 1fr;grid-template-rows:minmax(0,1fr) 105px}aside .firewall{grid-column:1/-1}body[data-tab=stats] .flows{display:none}body[data-tab=stats] aside{display:grid}.flow,.table-head{grid-template-columns:1fr 1fr 60px}.flow .protocol,.table-head .protocol{display:none}.flow{height:59px;font-size:10px}.state{font-size:9px}.flow small{font-size:9px}.status{font-size:10px}.status button{font-size:10px}.footer{font-size:8px}.footer .link-button{font-size:9px}aside .rank{line-height:19px}.block-value{display:inline-block;font-size:20px;margin:0 10px 0 0}.help{font-size:9px}.firewall h2{margin-bottom:4px}}
</style></head><body data-tab="flows">
<header><div><div class="eyebrow">PROXMOX / NETWORK OBSERVATORY</div><h1>Сетевая активность<span style="color:var(--lime)">.</span></h1></div><nav><a href="/traffic">HTTP / CrowdSec ↗</a><a class="dashboard" href="/">Dashboard</a><button id="pause">Пауза</button></nav></header>
<div class="status"><b id="live" class="badge warn">ОЖИДАНИЕ</b><span id="source">Подключение к агенту…</span><button class="link-button" id="statusDetails">Диагностика ⓘ</button></div>
<section class="metrics"><div class="metric"><small>TCP · ESTABLISHED</small><b id="established">—</b></div><div class="metric"><small>TCP · SYN_SENT / SYN_RECV</small><b id="pending">—</b></div><div class="metric"><small>UDP · отслеживаемые потоки</small><b id="udp">—</b></div><div class="metric"><small>ICMP / ICMPv6 · записи</small><b id="icmp">—</b></div></section>
<div class="filters"><select id="protocol" aria-label="Протокол"><option value="">Все протоколы</option><option value="tcp">TCP</option><option value="udp">UDP</option><option value="icmp">ICMP / ICMPv6</option></select><select id="port" aria-label="Порт назначения"><option value="">Все порты</option></select><input id="search" placeholder="Найти IP или состояние…" aria-label="Поиск IP или состояния"><label id="netMonitoringControl"><input id="hideMonitoring" type="checkbox">Скрыть мониторинг</label><label id="netExternalControl"><input id="external" type="checkbox">Внешние IP</label></div>
<nav id="tabs"><button class="active" data-tab="flows">Потоки</button><button data-tab="stats">Статистика</button></nav>
<main><section class="panel flows"><div class="heading"><h2>Текущая таблица conntrack</h2><small id="count">—</small></div><div class="table-head"><span>Инициатор</span><span>Назначение / NAT</span><span class="protocol">Протокол</span><span>Состояние</span><span class="volume">Байты → / ←</span></div><div id="rows"><div class="empty">Ожидание первого снимка…</div></div><div class="pager"><button id="prev" aria-label="Предыдущие потоки">←</button><span id="page">—</span><button id="next" aria-label="Следующие потоки">→</button><button id="resume">К свежим ↗</button></div></section>
<aside><section class="panel"><h2>Топ инициаторов · потоки</h2><div id="topIPs"></div></section><section class="panel"><h2>Топ портов назначения</h2><div id="topPorts"></div></section><section class="panel firewall"><div class="heading"><h2>CrowdSec · firewall</h2><button class="link-button" id="firewallDetails">ⓘ</button></div><div id="blocked" class="block-value">—</div><div class="help" id="firewallNote">Ожидание счётчиков</div></section></aside></main>
<footer class="footer"><span>Снимки ~5 с · короткие потоки могут быть пропущены</span><button class="link-button" id="about">Как читать данные ⓘ</button><a href="/#traffic-settings">Настройки ↗</a></footer>
<dialog id="dialog"><div class="heading"><h2 id="title"></h2><button id="close" aria-label="Закрыть">×</button></div><div id="details"></div></dialog>
<script>
const el=id=>document.getElementById(id),safe=v=>String(v??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c])),num=v=>Number(v||0).toLocaleString('ru-RU'),bytes=n=>{n=Number(n||0);return n>=1073741824?(n/1073741824).toFixed(1)+' GiB':n>=1048576?(n/1048576).toFixed(1)+' MiB':n>=1024?(n/1024).toFixed(1)+' KiB':n+' B'},endpoint=(ip,p)=>p?(ip.includes(':')?'['+ip+']':ip)+':'+p:ip;
const names=new Intl.DisplayNames(['ru'],{type:'region'}),country=c=>c==='LAN'?'Локальная сеть':c&&/^[A-Z]{2}$/.test(c)?names.of(c):'Без геолокации';
try{el('hideMonitoring').checked=localStorage.getItem('proxmox-hide-monitoring')==='1'}catch{}
let snapshot=null,frozen=null,page=0,visible=[],paused=false,busy=false,networkError='';
const show=(title,html)=>{el('title').textContent=title;el('details').innerHTML=html;el('dialog').showModal()},fields=rows=>rows.map(([k,v])=>'<div class="detail"><small>'+safe(k)+'</small>'+safe(v)+'</div>').join('');
function filterFlows(){const p=el('protocol').value,port=el('port').value,q=el('search').value.trim().toLowerCase();return ((frozen||snapshot)?.flows||[]).filter(f=>(!p||(p==='icmp'?f.protocol.startsWith('icmp'):f.protocol===p))&&(!port||String(f.destinationPort)===port)&&(!el('external').checked||f.external)&&(!q||[f.source,f.destination,f.replySource,f.replyDestination,f.state].join(' ').toLowerCase().includes(q)))}
function rank(id,items){const counts=new Map();items.forEach(v=>counts.set(v,(counts.get(v)||0)+1));const h=el(id).parentElement.clientHeight;const limit=Math.max(1,Math.min(5,Math.floor((h-44)/22)));el(id).innerHTML=[...counts].sort((a,b)=>b[1]-a[1]||a[0].localeCompare(b[0])).slice(0,limit).map(([k,v])=>'<div class="rank"><span title="'+safe(k)+'">'+safe(k)+'</span><b>'+num(v)+'</b></div>').join('')||'<div class="help">Нет потоков</div>'}
function render(){if(!snapshot)return;const data=frozen||snapshot,items=filterFlows();el('established').textContent=num(items.filter(f=>f.protocol==='tcp'&&f.state==='ESTABLISHED').length);el('pending').textContent=num(items.filter(f=>f.protocol==='tcp'&&['SYN_SENT','SYN_RECV'].includes(f.state)).length);el('udp').textContent=num(items.filter(f=>f.protocol==='udp').length);el('icmp').textContent=num(items.filter(f=>f.protocol.startsWith('icmp')).length);
if(!data.receivedAt||(data.error&&!(data.flows||[]).length)){['established','pending','udp','icmp'].forEach(id=>el(id).textContent='—')}
const fresh=snapshot.receivedAt&&Date.now()/1000-snapshot.receivedAt<25;el('live').textContent=networkError?'НЕТ СВЯЗИ':paused?'ПАУЗА':frozen?'СНИМОК':!fresh?'НЕТ ДАННЫХ':snapshot.error||snapshot.limited||snapshot.parseErrors?'НЕПОЛНЫЕ ДАННЫЕ':'● LIVE';el('live').classList.toggle('warn',!!networkError||!fresh||paused||!!snapshot.error||snapshot.limited||!!snapshot.parseErrors);el('source').textContent=(snapshot.agentName||'Агент')+' · '+(snapshot.at?new Date(data.at*1000).toLocaleTimeString('ru-RU'):'ожидание')+(snapshot.error?' · '+snapshot.error:snapshot.limited?' · лимит 5 000 потоков':'');
const rowHeight=innerWidth<=760?59:innerHeight<=700?46:55,count=Math.max(1,Math.floor(el('rows').clientHeight/rowHeight)),pages=Math.max(1,Math.ceil(items.length/count));page=Math.min(page,pages-1);visible=items.slice(page*count,(page+1)*count);el('count').textContent=num(items.length)+' / '+num(data.flows?.length)+(data.monitoringHidden?' · скрыто '+num(data.monitoringHidden):'')+' · '+(frozen?'снимок':'live');
el('rows').innerHTML=visible.map((f,i)=>{const nat=f.destination!==f.replySource||f.destinationPort!==f.replySourcePort;return '<article class="flow" role="button" tabindex="0" data-flow="'+i+'" aria-label="Поток '+safe(endpoint(f.source,f.sourcePort))+'"><div><strong>'+safe(endpoint(f.source,f.sourcePort))+'</strong><small>'+safe(country(f.country))+(f.monitoring?' · мониторинг':'')+'</small></div><div><strong>'+safe(endpoint(f.destination,f.destinationPort))+'</strong><small>'+safe(nat?'NAT → '+endpoint(f.replySource,f.replySourcePort):country(f.destinationCountry))+'</small></div><div class="protocol">'+safe(f.protocol.toUpperCase())+'</div><div class="state '+(f.state==='ESTABLISHED'?'ok':f.state.includes('SYN')?'pending':'')+'">'+safe(f.state)+'<small>'+safe(f.protocol.toUpperCase())+'</small></div><div class="volume">'+(f.accounting?bytes(f.bytes)+'<small>'+bytes(f.replyBytes)+'</small>':'<small>нет учёта</small>')+'</div></article>'}).join('')||'<div class="empty">'+safe(snapshot.error||'Нет потоков по выбранным фильтрам')+'</div>';
el('page').textContent=(page+1)+' / '+pages;el('prev').disabled=page===0;el('next').disabled=page>=pages-1;rank('topIPs',items.map(f=>f.source));rank('topPorts',items.filter(f=>f.destinationPort).map(f=>f.protocol.toUpperCase()+' / '+f.destinationPort));
const counters=snapshot.firewall||[];el('blocked').textContent=counters.length?num(counters.reduce((a,c)=>a+c.packets,0))+' пак.':'—';el('firewallNote').textContent=counters.length?'С начала жизни счётчиков · '+bytes(counters.reduce((a,c)=>a+c.bytes,0))+' · '+new Date(snapshot.firewallAt*1000).toLocaleTimeString('ru-RU'):snapshot.firewallError||'Счётчики ещё не получены';
}
function openFlow(target){const row=target.closest('[data-flow]');if(!row)return;const f=visible[+row.dataset.flow];if(!f)return;show('Поток '+f.protocol.toUpperCase(),fields([['Инициатор',endpoint(f.source,f.sourcePort)+' · '+country(f.country)],['Исходное назначение',endpoint(f.destination,f.destinationPort)],['Обратное направление (NAT)',endpoint(f.replySource,f.replySourcePort)+' → '+endpoint(f.replyDestination,f.replyDestinationPort)],['Мониторинг',f.monitoring?'Да — правило по IP инициатора':'Нет'],['Состояние / зона',f.state+' / '+(f.zone||'0')],['Инициатор → назначение',f.accounting?bytes(f.bytes)+' · '+num(f.packets)+' пакетов':'Учёт пакетов/байтов не включён для этого потока'],['Ответное направление',f.accounting?bytes(f.replyBytes)+' · '+num(f.replyPackets)+' пакетов':'Нет данных'],['Время снимка',new Date((frozen||snapshot).at*1000).toLocaleString('ru-RU')]]))}
el('rows').onclick=e=>openFlow(e.target);el('rows').onkeydown=e=>{if(e.key==='Enter'||e.key===' '){e.preventDefault();openFlow(e.target)}};el('close').onclick=()=>el('dialog').close();
el('statusDetails').onclick=()=>show('Диагностика агента',fields([['Сбор',snapshot?.error||'Ошибок conntrack не сообщено'],['Связь',networkError||'Последний ответ получен'],['Получен сервером',snapshot?.receivedAt?new Date(snapshot.receivedAt*1000).toLocaleString('ru-RU'):'Ещё не получен'],['Ограничения',snapshot?.limited?'Достигнут лимит 5 000 записей; статистика частичная':'До 5 000 записей в снимке'],['Нераспознанные строки',num(snapshot?.parseErrors)],['Firewall',snapshot?.firewallError||'Поддерживаемые счётчики получены']])+ '<p>На Debian/Ubuntu агент автоматически устанавливает отсутствующий <code>conntrack</code> через apt-get и systemd. Службе нужны root; для чтения таблицы — root/CAP_NET_ADMIN. Ошибка установки повторяется не чаще раза в 30 минут. Для firewall нужен <code>nft</code>. При выключенном учёте conntrack байты недоступны; sysctl и правила автоматически не изменяются.</p>');
el('firewallDetails').onclick=()=>show('Счётчики блокировок',fields((snapshot?.firewall||[]).map(c=>[c.name,num(c.packets)+' пакетов · '+bytes(c.bytes)]))+'<p>'+safe(snapshot?.firewallError||'Накопительные счётчики правил CrowdSec непосредственно перед drop/reject. Сбрасываются при пересоздании правил. Не зависят от фильтров списка потоков; не показывают количество людей или HTTP-запросов.')+'</p>');
el('about').onclick=()=>show('Как читать сетевые данные','<p>Это текущие отслеживаемые потоки TCP, UDP, ICMP и ICMPv6, а не журнал всех пакетов. Снимки делаются примерно раз в 5 секунд: короткие потоки могут быть пропущены. TIME_WAIT и другие состояния закрытия остаются в таблице до истечения таймера.</p><p>Инициатор и назначение — исходное направление conntrack. Обратная пара адресов показывает трансляцию NAT, включая Docker. Соединение NPM с upstream — отдельный поток; оно не считается ещё одним посетителем.</p><p>UDP ASSURED — отметка conntrack, не TCP-соединение. Порт 443 сам по себе не доказывает HTTPS или QUIC. SYN_SENT/SYN_RECV не означают атаку.</p><p>Байты и пакеты накоплены за жизнь конкретного потока, если учёт был включён при его создании. Длительность соединения и точный поток новых подключений по этим снимкам не вычисляются.</p><p>Фильтры влияют на карточки и топы. Firewall показывает отдельные накопительные счётчики без фильтров. Приватные и локальные IP не геолокируются. «Скрыть мониторинг» применяет правила по IP инициатора; домен/путь относятся только к HTTP. Геолокация: <a href="https://db-ip.com">DB-IP</a>, CC BY 4.0.</p>');
function reset(){frozen=null;page=0;render()};['protocol','port','external'].forEach(id=>el(id).onchange=reset);el('search').oninput=reset;el('hideMonitoring').onchange=()=>{try{localStorage.setItem('proxmox-hide-monitoring',el('hideMonitoring').checked?'1':'0')}catch{}frozen=null;page=0;poll(true)};el('next').onclick=()=>{if(!frozen)frozen=snapshot;page++;render()};el('prev').onclick=()=>{page--;render()};el('resume').onclick=reset;el('pause').onclick=()=>{paused=!paused;el('pause').textContent=paused?'Продолжить':'Пауза';render();if(!paused)poll()};document.querySelectorAll('[data-tab]').forEach(b=>{if(b.tagName==='BUTTON')b.onclick=()=>{document.body.dataset.tab=b.dataset.tab;document.querySelectorAll('#tabs button').forEach(t=>t.classList.toggle('active',t===b));render()}});window.addEventListener('resize',render);
async function poll(force=false){if((paused&&!force)||busy||document.hidden)return;busy=true;try{const response=await fetch('/api/network?hideMonitoring='+(el('hideMonitoring').checked?'1':'0'));if(response.status===404){location.replace('/traffic');return}if(!response.ok)throw Error('HTTP '+response.status);snapshot=await response.json();networkError='';const current=el('port').value,ports=[...new Set((snapshot.flows||[]).map(f=>f.destinationPort).filter(Boolean))];if(current&&!ports.includes(+current))ports.push(+current);ports.sort((a,b)=>a-b);el('port').innerHTML='<option value="">Все порты</option>'+ports.map(p=>'<option value="'+p+'">'+p+'</option>').join('');el('port').value=current;render()}catch(e){networkError='Не удалось обновить: '+e.message;el('source').textContent=networkError;el('live').textContent='НЕТ СВЯЗИ';el('live').classList.add('warn')}finally{busy=false}}
poll();setInterval(poll,5000);document.addEventListener('visibilitychange',()=>{if(!document.hidden)poll()});
</script></body></html>

131
assets/traffic.html Normal file

File diff suppressed because one or more lines are too long

1
assets/world.json Normal file

File diff suppressed because one or more lines are too long

38
cpu.go
View File

@@ -3,6 +3,7 @@ package main
import ( import (
"bufio" "bufio"
"fmt" "fmt"
"math"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@@ -33,14 +34,16 @@ type CPUStatic struct {
type CPUMetrics struct { type CPUMetrics struct {
CPUStatic CPUStatic
UsagePercent float64 `json:"usagePercent"` UsagePercent float64 `json:"usagePercent"`
FrequencyMHz float64 `json:"frequencyMhz"` FrequencyMHz float64 `json:"frequencyMhz"`
Load1 float64 `json:"load1"` Load1 float64 `json:"load1"`
Load5 float64 `json:"load5"` Load5 float64 `json:"load5"`
Load15 float64 `json:"load15"` Load15 float64 `json:"load15"`
Uptime uint64 `json:"uptimeSeconds"` Uptime uint64 `json:"uptimeSeconds"`
Temperatures []Temperature `json:"temperatures"` TemperatureCelsius *float64 `json:"temperatureCelsius"`
CollectedAt time.Time `json:"collectedAt"` Temperatures []Temperature `json:"temperatures"`
TopProcesses []AgentProcess `json:"topProcesses"`
CollectedAt time.Time `json:"collectedAt"`
} }
type Temperature struct { type Temperature struct {
@@ -89,10 +92,11 @@ func (c *cpuCollector) collect() (CPUMetrics, error) {
load1, load5, load15, _ := readLoadAverage() load1, load5, load15, _ := readLoadAverage()
uptime, _ := readUptime() uptime, _ := readUptime()
temperatures := readTemperatures()
return CPUMetrics{ return CPUMetrics{
CPUStatic: c.static, UsagePercent: usage, FrequencyMHz: frequency, CPUStatic: c.static, UsagePercent: usage, FrequencyMHz: frequency,
Load1: load1, Load5: load5, Load15: load15, Uptime: uptime, Load1: load1, Load5: load5, Load15: load15, Uptime: uptime,
Temperatures: readTemperatures(), CollectedAt: time.Now(), Temperatures: temperatures, TemperatureCelsius: cpuTemperature(temperatures), TopProcesses: collectAgentProcesses(), CollectedAt: time.Now(),
}, nil }, nil
} }
@@ -284,3 +288,19 @@ func isCPUTemperatureChip(name string) bool {
return false return false
} }
} }
// cpuTemperature is the hottest CPU sensor, shared by live and historical views.
// Do not average Tctl/Tdie with cooler cores or CCDs.
func cpuTemperature(temperatures []Temperature) *float64 {
var hottest *float64
for _, t := range temperatures {
if math.IsNaN(t.Celsius) || math.IsInf(t.Celsius, 0) || t.Celsius <= 0 {
continue
}
if hottest == nil || t.Celsius > *hottest {
value := t.Celsius
hottest = &value
}
}
return hottest
}

24
cpu_temperature_test.go Normal file
View File

@@ -0,0 +1,24 @@
package main
import (
"math"
"testing"
)
func TestCPUTemperatureSharedWithHistory(t *testing.T) {
temperatures := []Temperature{{Label: "k10temp · Tctl", Celsius: 62}, {Label: "k10temp · Tccd1", Celsius: 38}, {Label: "k10temp · Tccd2", Celsius: 41}}
live := cpuTemperature(temperatures)
point := historyPointFromMetrics(dashboardMetrics{CPU: CPUMetrics{Temperatures: temperatures}})
if live == nil || *live != 62 || point.CPUTemperature != *live {
t.Fatalf("live/history disagree: %v, %+v", live, point)
}
}
func TestCPUTemperatureInvalidSensors(t *testing.T) {
if cpuTemperature([]Temperature{{Celsius: math.NaN()}, {Celsius: math.Inf(1)}, {Celsius: 0}}) != nil {
t.Fatal("invalid temperature selected")
}
if cpuTemperature(nil) != nil {
t.Fatal("missing sensor should stay missing")
}
}

150
disks.go
View File

@@ -13,42 +13,48 @@ import (
) )
type DiskMetrics struct { type DiskMetrics struct {
Name string `json:"name"` Name string `json:"name"`
Path string `json:"path"` Path string `json:"path"`
Model string `json:"model"` Model string `json:"model"`
Serial string `json:"serial"` Serial string `json:"serial"`
Type string `json:"type"` Type string `json:"type"`
Protocol string `json:"protocol"` Protocol string `json:"protocol"`
SizeBytes uint64 `json:"sizeBytes"` SizeBytes uint64 `json:"sizeBytes"`
UsedBytes *uint64 `json:"usedBytes"` UsedBytes *uint64 `json:"usedBytes"`
UsagePercent *float64 `json:"usagePercent"` UsagePercent *float64 `json:"usagePercent"`
SMARTStatus string `json:"smartStatus"` SMARTStatus string `json:"smartStatus"`
SMARTAvailable bool `json:"smartAvailable"` SMARTAvailable bool `json:"smartAvailable"`
AttentionReasons []string `json:"attentionReasons"` AttentionReasons []string `json:"attentionReasons"`
Temperature *float64 `json:"temperatureCelsius"` SMARTHistory []string `json:"smartHistory"`
PowerOnHours *uint64 `json:"powerOnHours"` Temperature *float64 `json:"temperatureCelsius"`
WearUsedPercent *float64 `json:"wearUsedPercent"` PowerOnHours *uint64 `json:"powerOnHours"`
Reallocated uint64 `json:"reallocated"` WearUsedPercent *float64 `json:"wearUsedPercent"`
Pending uint64 `json:"pending"` Reallocated uint64 `json:"reallocated"`
Uncorrectable uint64 `json:"uncorrectable"` Pending uint64 `json:"pending"`
CRCErrors uint64 `json:"crcErrors"` Uncorrectable uint64 `json:"uncorrectable"`
MediaErrors uint64 `json:"mediaErrors"` CRCErrors uint64 `json:"crcErrors"`
CounterChanges []string `json:"counterChanges"` MediaErrors uint64 `json:"mediaErrors"`
LastSelfTest string `json:"lastSelfTest"` SMARTErrorCount uint64 `json:"smartErrorCount"`
LastSelfTestAt string `json:"lastSelfTestAt"` CounterChanges []string `json:"counterChanges"`
SelfTestNeverRun bool `json:"selfTestNeverRun"` LastSelfTest string `json:"lastSelfTest"`
ShortTestStarted bool `json:"shortTestStarted"` LastSelfTestAt string `json:"lastSelfTestAt"`
ReadIOPS float64 `json:"readIops"` SelfTestNeverRun bool `json:"selfTestNeverRun"`
WriteIOPS float64 `json:"writeIops"` ShortTestStarted bool `json:"shortTestStarted"`
ReadBytesPerSec float64 `json:"readBytesPerSec"` ReadIOPS float64 `json:"readIops"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"` WriteIOPS float64 `json:"writeIops"`
ReadLatencyMs float64 `json:"readLatencyMs"` ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteLatencyMs float64 `json:"writeLatencyMs"` WriteBytesPerSec float64 `json:"writeBytesPerSec"`
LatencyMs float64 `json:"latencyMs"` ReadLatencyMs float64 `json:"readLatencyMs"`
Utilization float64 `json:"utilizationPercent"` WriteLatencyMs float64 `json:"writeLatencyMs"`
QueueDepth uint64 `json:"queueDepth"` LatencyMs float64 `json:"latencyMs"`
AverageQueue float64 `json:"averageQueue"` Utilization float64 `json:"utilizationPercent"`
IOPressureSeconds float64 `json:"ioPressureSeconds"` QueueDepth uint64 `json:"queueDepth"`
AverageQueue float64 `json:"averageQueue"`
IOPressureSeconds float64 `json:"ioPressureSeconds"`
IOCause string `json:"ioCause"`
IOCauseDetail string `json:"ioCauseDetail"`
IOConfidence int `json:"ioConfidence"`
TopConsumers []IOConsumer `json:"topConsumers"`
} }
type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 } type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 }
@@ -65,11 +71,11 @@ type diskCollector struct {
ioPressureSince map[string]time.Time ioPressureSince map[string]time.Time
} }
func (c *diskCollector) collect() []DiskMetrics { func (c *diskCollector) collect(thresholds AlertThresholds) []DiskMetrics {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
if time.Since(c.updatedAt) < time.Minute && c.cached != nil { if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
c.applyDiskIO(c.cached) c.applyDiskIO(c.cached, thresholds)
return c.cached return c.cached
} }
current := readPhysicalDisks() current := readPhysicalDisks()
@@ -104,12 +110,12 @@ func (c *diskCollector) collect() []DiskMetrics {
} }
} }
c.cached = current c.cached = current
c.applyDiskIO(c.cached) c.applyDiskIO(c.cached, thresholds)
c.updatedAt = time.Now() c.updatedAt = time.Now()
return c.cached return c.cached
} }
func (c *diskCollector) applyDiskIO(disks []DiskMetrics) { func (c *diskCollector) applyDiskIO(disks []DiskMetrics, thresholds AlertThresholds) {
now := time.Now() now := time.Now()
snapshots := readDiskIOStats("/proc/diskstats") snapshots := readDiskIOStats("/proc/diskstats")
if c.ioPrevious == nil { if c.ioPrevious == nil {
@@ -144,11 +150,13 @@ func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
} }
disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100) disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100)
disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000) disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000)
latencyLimit := 20.0 utilLimit, latencyLimit, queueLimit := thresholds.DiskSSDUtil, thresholds.DiskSSDLatency, thresholds.DiskSSDQueue
if disk.Type == "HDD" { if disk.Type == "HDD" {
latencyLimit = 50 utilLimit, latencyLimit, queueLimit = thresholds.DiskHDDUtil, thresholds.DiskHDDLatency, thresholds.DiskHDDQueue
} else if strings.Contains(disk.Type, "NVMe") {
utilLimit, latencyLimit, queueLimit = thresholds.DiskNVMeUtil, thresholds.DiskNVMeLatency, thresholds.DiskNVMeQueue
} }
pressured := disk.Utilization >= 90 || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= 4 pressured := disk.Utilization >= utilLimit || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= queueLimit
if pressured { if pressured {
if c.ioPressureSince[disk.Name].IsZero() { if c.ioPressureSince[disk.Name].IsZero() {
c.ioPressureSince[disk.Name] = now c.ioPressureSince[disk.Name] = now
@@ -197,7 +205,7 @@ func diskCounterChanges(old, current DiskMetrics) []string {
checks := []struct { checks := []struct {
label string label string
old, now uint64 old, now uint64
}{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}} }{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}, {"Ошибки SMART-журнала выросли", old.SMARTErrorCount, current.SMARTErrorCount}}
var changes []string var changes []string
for _, check := range checks { for _, check := range checks {
if check.now > check.old { if check.now > check.old {
@@ -383,6 +391,17 @@ type smartctlJSON struct {
} `json:"table"` } `json:"table"`
} `json:"standard"` } `json:"standard"`
} `json:"ata_smart_self_test_log"` } `json:"ata_smart_self_test_log"`
NVMeSelfTestLog struct {
Table []struct {
SelfTestCode struct {
String string `json:"string"`
} `json:"self_test_code"`
SelfTestResult struct {
String string `json:"string"`
} `json:"self_test_result"`
PowerOnHours uint64 `json:"power_on_hours"`
} `json:"table"`
} `json:"nvme_self_test_log"`
NVMeHealth *struct { NVMeHealth *struct {
Temperature float64 `json:"temperature"` Temperature float64 `json:"temperature"`
PercentageUsed float64 `json:"percentage_used"` PercentageUsed float64 `json:"percentage_used"`
@@ -402,6 +421,10 @@ func applySMART(disk *DiskMetrics) {
if len(output) == 0 { if len(output) == 0 {
return return
} }
applySMARTJSON(disk, output)
}
func applySMARTJSON(disk *DiskMetrics, output []byte) {
var smart smartctlJSON var smart smartctlJSON
if json.Unmarshal(output, &smart) != nil { if json.Unmarshal(output, &smart) != nil {
return return
@@ -443,12 +466,20 @@ func applySMART(disk *DiskMetrics) {
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10)) disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
} }
} }
if len(smart.ATASelfTestLog.Standard.Table) > 0 { if len(smart.NVMeSelfTestLog.Table) > 0 {
last := smart.NVMeSelfTestLog.Table[0]
description := strings.TrimSpace(last.SelfTestCode.String)
result := firstNonEmpty(strings.TrimSpace(last.SelfTestResult.String), "Результат неизвестен")
if description != "" {
disk.LastSelfTest = description + " · " + result
} else {
disk.LastSelfTest = result
}
setLastSelfTestAt(disk, last.PowerOnHours)
} else if len(smart.ATASelfTestLog.Standard.Table) > 0 {
last := smart.ATASelfTestLog.Standard.Table[0] last := smart.ATASelfTestLog.Standard.Table[0]
disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен") disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours { setLastSelfTestAt(disk, last.LifetimeHours)
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339)
}
} else { } else {
disk.SelfTestNeverRun = true disk.SelfTestNeverRun = true
disk.LastSelfTest = "Никогда не запускался" disk.LastSelfTest = "Никогда не запускался"
@@ -461,6 +492,12 @@ func applySMART(disk *DiskMetrics) {
} }
} }
func setLastSelfTestAt(disk *DiskMetrics, testPowerOnHours uint64) {
if disk.PowerOnHours != nil && *disk.PowerOnHours >= testPowerOnHours {
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-testPowerOnHours) * time.Hour).Format(time.RFC3339)
}
}
func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) { func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
wearNames := map[string]bool{ wearNames := map[string]bool{
"Percent_Lifetime_Remain": true, "SSD_Life_Left": true, "Percent_Lifetime_Remain": true, "SSD_Life_Left": true,
@@ -482,16 +519,23 @@ func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
wear := float64(100 - attribute.Value) wear := float64(100 - attribute.Value)
disk.WearUsedPercent = &wear disk.WearUsedPercent = &wear
} }
if attribute.WhenFailed != "" && attribute.WhenFailed != "-" { if strings.EqualFold(attribute.WhenFailed, "FAILING_NOW") {
reason := attribute.Name + " вышел за порог" reason := attribute.Name + " вышел за порог"
if attribute.Raw.String != "" { if attribute.Raw.String != "" {
reason += " (" + attribute.Raw.String + ")" reason += " (" + attribute.Raw.String + ")"
} }
disk.AttentionReasons = append(disk.AttentionReasons, reason) disk.AttentionReasons = append(disk.AttentionReasons, reason)
} else if attribute.WhenFailed != "" && attribute.WhenFailed != "-" {
reason := attribute.Name + " ранее выходил за порог"
if attribute.Raw.String != "" {
reason += " (" + attribute.Raw.String + ")"
}
disk.SMARTHistory = append(disk.SMARTHistory, reason)
} }
} }
if smart.ATAErrorLog.Summary.Count > 0 { if smart.ATAErrorLog.Summary.Count > 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибок в SMART-журнале: "+strconv.Itoa(smart.ATAErrorLog.Summary.Count)) disk.SMARTErrorCount = uint64(smart.ATAErrorLog.Summary.Count)
disk.SMARTHistory = append(disk.SMARTHistory, "Записей в SMART-журнале: "+strconv.Itoa(smart.ATAErrorLog.Summary.Count))
} }
} }
@@ -505,12 +549,12 @@ func applySMARTExitStatus(disk *DiskMetrics, status int) {
disk.AttentionReasons = append(disk.AttentionReasons, "Критический SMART-атрибут достиг порога") disk.AttentionReasons = append(disk.AttentionReasons, "Критический SMART-атрибут достиг порога")
} }
if status&32 != 0 { if status&32 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "SMART-атрибут ранее находился ниже порога") disk.SMARTHistory = append(disk.SMARTHistory, "SMART-атрибут ранее находился ниже порога")
} }
if status&64 != 0 { if status&64 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "В журнале SMART обнаружены ошибки") disk.SMARTHistory = append(disk.SMARTHistory, "В журнале SMART есть старые записи об ошибках")
} }
if status&128 != 0 { if status&128 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "В журнале самотестирования есть ошибки") disk.SMARTHistory = append(disk.SMARTHistory, "В журнале самотестирования есть старые ошибки")
} }
} }

View File

@@ -4,9 +4,46 @@ import (
"encoding/json" "encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"time"
) )
func TestApplySMARTJSONReadsNVMeSelfTestHistory(t *testing.T) {
disk := DiskMetrics{Path: "/dev/nvme0n1"}
output := []byte(`{
"model_name":"ADATA LEGEND 900",
"smart_status":{"passed":true},
"power_on_time":{"hours":960},
"nvme_self_test_log":{
"current_self_test_operation":{"value":0,"string":"No self-test in progress"},
"table":[{
"self_test_code":{"value":1,"string":"Short"},
"self_test_result":{"value":0,"string":"Completed without error"},
"power_on_hours":959
}]
}
}`)
before := time.Now().Add(-2 * time.Hour)
applySMARTJSON(&disk, output)
after := time.Now()
if disk.SelfTestNeverRun {
t.Fatal("завершённый NVMe self-test ошибочно отмечен как никогда не запускавшийся")
}
if disk.LastSelfTest != "Short · Completed without error" {
t.Fatalf("неожиданный результат self-test: %q", disk.LastSelfTest)
}
parsed, err := time.Parse(time.RFC3339, disk.LastSelfTestAt)
if err != nil || parsed.Before(before) || parsed.After(after) {
t.Fatalf("неверно вычислено время self-test: %q, err=%v", disk.LastSelfTestAt, err)
}
if len(disk.AttentionReasons) != 0 || disk.SMARTStatus != "Исправен" {
t.Fatalf("исправный NVMe требует внимания: %+v", disk)
}
}
func TestIsPhysicalDiskName(t *testing.T) { func TestIsPhysicalDiskName(t *testing.T) {
tests := map[string]bool{ tests := map[string]bool{
"sda": true, "sdb": true, "nvme0n1": true, "sda": true, "sdb": true, "nvme0n1": true,
@@ -68,3 +105,37 @@ func TestApplyATAWearAndReason(t *testing.T) {
t.Fatalf("причина предупреждения не добавлена: %+v", disk) t.Fatalf("причина предупреждения не добавлена: %+v", disk)
} }
} }
func TestHistoricalATAFailureDoesNotRequireAttention(t *testing.T) {
var smart smartctlJSON
attribute := struct {
Name string `json:"name"`
Value int `json:"value"`
Threshold int `json:"thresh"`
WhenFailed string `json:"when_failed"`
Raw struct {
String string `json:"string"`
Value int64 `json:"value"`
} `json:"raw"`
}{Name: "Airflow_Temperature_Cel", WhenFailed: "In_the_past"}
attribute.Raw.String = "46 (Min/Max 26/47 #15)"
smart.ATAAttributes.Table = append(smart.ATAAttributes.Table, attribute)
disk := DiskMetrics{}
applyATAAttributes(&disk, smart)
applySMARTExitStatus(&disk, 32|64|128)
if len(disk.AttentionReasons) != 0 {
t.Fatalf("историческая ошибка стала активной: %+v", disk.AttentionReasons)
}
if len(disk.SMARTHistory) != 4 {
t.Fatalf("история SMART потеряна: %+v", disk.SMARTHistory)
}
}
func TestSMARTErrorLogGrowthRequiresAttention(t *testing.T) {
old := DiskMetrics{SMARTErrorCount: 2}
current := DiskMetrics{SMARTErrorCount: 3}
changes := diskCounterChanges(old, current)
if len(changes) != 1 || !strings.Contains(changes[0], "2 → 3") {
t.Fatalf("рост журнала не обнаружен: %+v", changes)
}
}

5
go.mod
View File

@@ -2,7 +2,10 @@ module proxmox-host-info
go 1.25.0 go 1.25.0
require modernc.org/sqlite v1.54.0 require (
github.com/oschwald/maxminddb-golang v1.13.1
modernc.org/sqlite v1.54.0
)
require ( require (
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect

10
go.sum
View File

@@ -1,3 +1,5 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
@@ -10,8 +12,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE=
github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
@@ -21,6 +29,8 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=

View File

@@ -47,6 +47,8 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
iops := d.ReadIOPS + d.WriteIOPS iops := d.ReadIOPS + d.WriteIOPS
util := d.Utilization util := d.Utilization
s.AddEntityHistory("disk-io", d.Name+" · "+d.Model, ts, &iops, &util) s.AddEntityHistory("disk-io", d.Name+" · "+d.Model, ts, &iops, &util)
latency := d.LatencyMs
s.AddEntityHistory("disk-latency", d.Name+" · "+d.Model, ts, &latency, nil)
if d.Temperature == nil { if d.Temperature == nil {
continue continue
} }
@@ -56,6 +58,36 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
} }
s.AddEntityHistory("disk", d.Name+" · "+d.Model, ts, d.Temperature, wear) s.AddEntityHistory("disk", d.Name+" · "+d.Model, ts, d.Temperature, wear)
} }
for _, task := range metrics.Activity.Tasks {
if task.Type == "vzdump" && task.Duration > 0 && task.EndTime >= time.Now().Add(-15*time.Minute).Unix() {
duration := float64(task.Duration)
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
}
}
for _, agent := range metrics.Agents {
if !agent.Online {
continue
}
name := agent.ID + " · " + agent.Name
cpu := agent.Report.CPUPercent
memory := float64(0)
if agent.Report.MemoryTotal > 0 {
memory = float64(agent.Report.MemoryUsed) / float64(agent.Report.MemoryTotal) * 100
}
swap := float64(0)
if agent.Report.SwapTotal > 0 {
swap = float64(agent.Report.SwapUsed) / float64(agent.Report.SwapTotal) * 100
}
s.AddEntityHistory("agent-resource", name, ts, &cpu, &memory)
s.AddEntityHistory("agent-swap", name, ts, &swap, nil)
pressure := agent.Report.Pressure.IO.Avg10
s.AddEntityHistory("agent-pressure", name, ts, &pressure, nil)
for _, filesystem := range agent.Report.Filesystems {
used := filesystem.UsedPercent
inodes := filesystem.InodesPercent
s.AddEntityHistory("agent-filesystem", name+" · "+filesystem.Mountpoint, ts, &used, &inodes)
}
}
s.PruneEntityHistory() s.PruneEntityHistory()
} }
@@ -67,12 +99,10 @@ func historyPointFromMetrics(metrics dashboardMetrics) HistoryPoint {
NetworkTransmit: metrics.Network.TransmitPerSec, UPSCharge: metrics.UPS.ChargePercent, NetworkTransmit: metrics.Network.TransmitPerSec, UPSCharge: metrics.UPS.ChargePercent,
UPSLoad: metrics.UPS.LoadPercent, UPSLoad: metrics.UPS.LoadPercent,
} }
if len(metrics.CPU.Temperatures) > 0 { if temperature := cpuTemperature(metrics.CPU.Temperatures); temperature != nil {
for _, temperature := range metrics.CPU.Temperatures { point.CPUTemperature = *temperature
point.CPUTemperature += temperature.Celsius
}
point.CPUTemperature /= float64(len(metrics.CPU.Temperatures))
} }
var zfsSize, zfsUsed uint64 var zfsSize, zfsUsed uint64
for _, pool := range metrics.ZFS.Pools { for _, pool := range metrics.ZFS.Pools {
zfsSize += pool.SizeBytes zfsSize += pool.SizeBytes

View File

@@ -8,7 +8,7 @@ func TestHistoryPointFromMetrics(t *testing.T) {
Guests: GuestsMetrics{Guests: []GuestMetrics{{Status: "running", CPU: .5, MaxCPU: 4, MemoryBytes: 50, MaxMemory: 100}}}, Guests: GuestsMetrics{Guests: []GuestMetrics{{Status: "running", CPU: .5, MaxCPU: 4, MemoryBytes: 50, MaxMemory: 100}}},
} }
point := historyPointFromMetrics(metrics) point := historyPointFromMetrics(metrics)
if point.CPUTemperature != 50 || point.GuestsCPU != 50 || point.GuestsMemory != 50 { if point.CPUTemperature != 60 || point.GuestsCPU != 50 || point.GuestsMemory != 50 {
t.Fatalf("неожиданная историческая точка: %+v", point) t.Fatalf("неожиданная историческая точка: %+v", point)
} }
} }

178
io_attribution.go Normal file
View File

@@ -0,0 +1,178 @@
package main
import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type IOConsumer struct {
Kind string `json:"kind"`
Name string `json:"name"`
VMID int `json:"vmid,omitempty"`
ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
SharePercent float64 `json:"sharePercent"`
}
type processIOSnapshot struct {
read, write uint64
name, cmdline, cgroup string
}
type ioAttributionCollector struct {
mu sync.Mutex
previous map[int]processIOSnapshot
updatedAt time.Time
cached []IOConsumer
}
var qemuIDPattern = regexp.MustCompile(`(?:^|\s)-id\s+(\d+)`)
var lxcIDPattern = regexp.MustCompile(`(?:lxc(?:\.payload)?[./-]|pve-container@)(\d+)`)
func (c *ioAttributionCollector) collect(guests GuestsMetrics) []IOConsumer {
c.mu.Lock()
defer c.mu.Unlock()
now := time.Now()
current := readProcessIO()
elapsed := now.Sub(c.updatedAt).Seconds()
if c.previous == nil || elapsed <= 0 {
c.previous = current
c.updatedAt = now
return nil
}
names := map[int]string{}
for _, g := range guests.Guests {
names[g.VMID] = g.Name
}
grouped := map[string]*IOConsumer{}
var total float64
for pid, snapshot := range current {
old, ok := c.previous[pid]
if !ok || snapshot.read < old.read || snapshot.write < old.write {
continue
}
read := float64(snapshot.read-old.read) / elapsed
write := float64(snapshot.write-old.write) / elapsed
if read+write < 1024 {
continue
}
kind, name, vmid := classifyIOProcess(snapshot, names)
key := kind + ":" + strconv.Itoa(vmid) + ":" + name
consumer := grouped[key]
if consumer == nil {
consumer = &IOConsumer{Kind: kind, Name: name, VMID: vmid}
grouped[key] = consumer
}
consumer.ReadBytesPerSec += read
consumer.WriteBytesPerSec += write
total += read + write
}
var result []IOConsumer
for _, v := range grouped {
if total > 0 {
v.SharePercent = (v.ReadBytesPerSec + v.WriteBytesPerSec) / total * 100
}
result = append(result, *v)
}
sort.Slice(result, func(i, j int) bool {
return result[i].ReadBytesPerSec+result[i].WriteBytesPerSec > result[j].ReadBytesPerSec+result[j].WriteBytesPerSec
})
if len(result) > 8 {
result = result[:8]
}
c.previous = current
c.updatedAt = now
c.cached = result
return result
}
func classifyIOProcess(p processIOSnapshot, names map[int]string) (kind, name string, vmid int) {
if match := qemuIDPattern.FindStringSubmatch(p.cmdline); len(match) > 1 {
vmid, _ = strconv.Atoi(match[1])
return "VM", firstNonEmpty(names[vmid], "VM "+match[1]), vmid
}
if match := lxcIDPattern.FindStringSubmatch(p.cgroup); len(match) > 1 {
vmid, _ = strconv.Atoi(match[1])
return "LXC", firstNonEmpty(names[vmid], "LXC "+match[1]), vmid
}
return "Процесс", firstNonEmpty(p.name, "PID"), 0
}
func readProcessIO() map[int]processIOSnapshot {
result := map[int]processIOSnapshot{}
entries, _ := filepath.Glob("/proc/[0-9]*/io")
for _, path := range entries {
pid, _ := strconv.Atoi(filepath.Base(filepath.Dir(path)))
data, err := os.ReadFile(path)
if err != nil {
continue
}
var item processIOSnapshot
for _, line := range strings.Split(string(data), "\n") {
key, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
n, _ := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
if key == "read_bytes" {
item.read = n
} else if key == "write_bytes" {
item.write = n
}
}
comm, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "comm"))
cmd, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cmdline"))
cg, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cgroup"))
item.name = strings.TrimSpace(string(comm))
item.cmdline = strings.ReplaceAll(string(cmd), "\x00", " ")
item.cgroup = string(cg)
result[pid] = item
}
return result
}
func attributeDiskIO(metrics *dashboardMetrics, consumers []IOConsumer) {
active := activeIOOperation(metrics.Activity.Tasks)
for i := range metrics.Disks {
disk := &metrics.Disks[i]
disk.TopConsumers = append([]IOConsumer(nil), consumers...)
if active != "" {
disk.IOCause = active
disk.IOCauseDetail = "Активная задача Proxmox совпадает по времени с дисковой нагрузкой"
disk.IOConfidence = 75
} else if len(consumers) > 0 {
top := consumers[0]
disk.IOCause = top.Kind + " · " + top.Name
disk.IOCauseDetail = "Крупнейший потребитель host I/O: " + strconv.FormatFloat(top.SharePercent, 'f', 1, 64) + "%"
disk.IOConfidence = 55
}
}
}
func activeIOOperation(tasks []ProxmoxTask) string {
for _, task := range tasks {
if task.EndTime != 0 && task.Status != "" {
continue
}
label := ""
switch task.Type {
case "vzdump":
label = "Backup"
case "qmigrate":
label = "Миграция"
case "qmsnapshot", "vzsnapshot":
label = "Snapshot"
}
if label != "" {
if task.ID != "" {
label += " VM/LXC " + task.ID
}
return label
}
}
return ""
}

33
io_attribution_test.go Normal file
View File

@@ -0,0 +1,33 @@
package main
import (
"strings"
"testing"
)
func TestClassifyQEMUAndLXCIO(t *testing.T) {
kind, name, vmid := classifyIOProcess(processIOSnapshot{name: "qemu", cmdline: "qemu-system-x86_64 -id 105 -name guest=test"}, map[int]string{105: "Gitea"})
if kind != "VM" || name != "Gitea" || vmid != 105 {
t.Fatalf("unexpected qemu attribution: %s %s %d", kind, name, vmid)
}
kind, name, vmid = classifyIOProcess(processIOSnapshot{name: "worker", cgroup: "0::/lxc.payload.220/ns"}, map[int]string{220: "db"})
if kind != "LXC" || name != "db" || vmid != 220 {
t.Fatalf("unexpected lxc attribution: %s %s %d", kind, name, vmid)
}
}
func TestActiveIOOperation(t *testing.T) {
value := activeIOOperation([]ProxmoxTask{{Type: "vzdump", ID: "105", Status: "", EndTime: 0}})
if !strings.Contains(value, "Backup") || !strings.Contains(value, "105") {
t.Fatalf("unexpected operation: %s", value)
}
}
func TestCorrelateAlerts(t *testing.T) {
metrics := dashboardMetrics{Disks: []DiskMetrics{{Name: "sda", IOCause: "Backup VM/LXC 105", IOConfidence: 90, Utilization: 98, LatencyMs: 80}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Up: false}}}}}}
alerts := []Alert{{ID: "disk-io-pressure-sda", Severity: "critical"}, {ID: "service-1-public", Severity: "critical"}, {ID: "other", Severity: "warning"}}
result := correlateAlerts(metrics, alerts)
if len(result) != 2 || result[0].Source != "Корреляция" || !strings.Contains(result[0].Message, "Gitea") {
t.Fatalf("unexpected correlation: %+v", result)
}
}

12
main.go
View File

@@ -20,11 +20,23 @@ var version = "dev"
func main() { func main() {
showVersion := flag.Bool("version", false, "show version") showVersion := flag.Bool("version", false, "show version")
agentMode := flag.Bool("agent", false, "run as a read-only guest agent")
agentServer := flag.String("agent-server", "", "dashboard URL used by the guest agent")
agentEnrollToken := flag.String("agent-enroll-token", "", "one-time enrollment token")
agentName := flag.String("agent-name", "", "agent display name")
agentVMID := flag.Int("agent-vmid", 0, "Proxmox VM/LXC ID")
agentConfig := flag.String("agent-config", "/var/lib/proxmox-dashboard-agent/agent.json", "agent configuration path")
flag.Parse() flag.Parse()
if *showVersion { if *showVersion {
fmt.Println(version) fmt.Println(version)
return return
} }
if *agentMode {
if err := runGuestAgent(agentOptions{Server: *agentServer, EnrollmentToken: *agentEnrollToken, Name: *agentName, VMID: *agentVMID, ConfigPath: *agentConfig}); err != nil {
log.Fatalf("Ошибка агента: %v", err)
}
return
}
address := os.Getenv("DASHBOARD_ADDR") address := os.Getenv("DASHBOARD_ADDR")
if address == "" { if address == "" {

170
maintenance_window.go Normal file
View File

@@ -0,0 +1,170 @@
package main
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
)
type MaintenanceWindow struct {
Enabled bool `json:"enabled"`
Active bool `json:"active"`
StartedAt int64 `json:"startedAt"`
EndsAt int64 `json:"endsAt"`
Reason string `json:"reason"`
AllAlerts bool `json:"allAlerts"`
ServiceIDs []int64 `json:"serviceIds"`
VMIDs []int `json:"vmids"`
Automatic bool `json:"automatic"`
Source string `json:"source,omitempty"`
}
func automaticBackupMaintenance(activity ProxmoxActivity, links []ServiceLink, now time.Time) MaintenanceWindow {
vmids := []int{}
backupRunning := false
for _, task := range activity.Tasks {
if task.Type != "vzdump" || task.EndTime > 0 || (task.Status != "" && !strings.EqualFold(task.Status, "running")) {
continue
}
backupRunning = true
if vmid := backupTaskVMID(task); vmid > 0 {
vmids = append(vmids, vmid)
}
}
vmids = uniqueInt(vmids)
if !backupRunning {
return MaintenanceWindow{}
}
selected := map[int]bool{}
for _, vmid := range vmids {
selected[vmid] = true
}
serviceIDs := []int64{}
for _, link := range links {
if selected[link.VMID] {
serviceIDs = append(serviceIDs, link.ServiceID)
}
}
return MaintenanceWindow{Enabled: true, Active: true, StartedAt: now.Unix(), EndsAt: now.Add(2 * time.Minute).Unix(), Reason: "Автоматически: выполняется backup VM/LXC", VMIDs: vmids, ServiceIDs: uniqueInt64(serviceIDs), AllAlerts: len(vmids) == 0, Automatic: true, Source: "backup"}
}
func backupTaskVMID(task ProxmoxTask) int {
if vmid, err := strconv.Atoi(strings.TrimSpace(task.ID)); err == nil && vmid > 0 {
return vmid
}
parts := strings.Split(task.UPID, ":")
for i, part := range parts {
if part == "vzdump" && i+1 < len(parts) {
vmid, _ := strconv.Atoi(parts[i+1])
return vmid
}
}
return 0
}
func (s *Store) MaintenanceWindow() MaintenanceWindow {
var value string
var window MaintenanceWindow
if s.db.QueryRow(`SELECT value FROM settings WHERE key='maintenance_window'`).Scan(&value) == nil {
_ = json.Unmarshal([]byte(value), &window)
}
now := time.Now().Unix()
window.Active = window.Enabled && window.StartedAt <= now && window.EndsAt > now
if window.Enabled && window.EndsAt <= now {
window.Enabled = false
data, _ := json.Marshal(window)
_, _ = s.db.Exec(`INSERT INTO settings(key,value) VALUES('maintenance_window',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
}
return window
}
func (s *Store) SaveMaintenanceWindow(window MaintenanceWindow) (MaintenanceWindow, error) {
now := time.Now()
window.Reason = strings.TrimSpace(window.Reason)
if window.Reason == "" {
return window, fmt.Errorf("укажите причину плановых работ")
}
if window.EndsAt <= now.Unix() || window.EndsAt > now.Add(30*24*time.Hour).Unix() {
return window, fmt.Errorf("окончание должно быть в будущем, максимум через 30 дней")
}
if !window.AllAlerts && len(window.ServiceIDs) == 0 && len(window.VMIDs) == 0 {
return window, fmt.Errorf("выберите сервис, VM/LXC или все алерты")
}
window.Enabled = true
window.Active = true
window.StartedAt = now.Unix()
window.ServiceIDs = uniqueInt64(window.ServiceIDs)
window.VMIDs = uniqueInt(window.VMIDs)
data, _ := json.Marshal(window)
_, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('maintenance_window',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
return window, err
}
func (s *Store) StopMaintenanceWindow() (MaintenanceWindow, error) {
window := s.MaintenanceWindow()
window.Enabled = false
window.Active = false
window.EndsAt = time.Now().Unix()
data, _ := json.Marshal(window)
_, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('maintenance_window',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
return window, err
}
func splitMaintenanceAlerts(alerts []Alert, window MaintenanceWindow) (active, planned []Alert) {
if !window.Active {
return alerts, nil
}
for _, alert := range alerts {
if maintenanceMatchesAlert(window, alert) {
planned = append(planned, alert)
} else {
active = append(active, alert)
}
}
return active, planned
}
func maintenanceMatchesAlert(window MaintenanceWindow, alert Alert) bool {
if window.AllAlerts {
return true
}
for _, id := range window.ServiceIDs {
value := strconv.FormatInt(id, 10)
if strings.HasPrefix(alert.ID, "service-"+value+"-") || strings.HasPrefix(alert.ID, "service-tls-"+value+"-") {
return true
}
}
for _, vmid := range window.VMIDs {
value := strconv.Itoa(vmid)
if strings.HasSuffix(alert.ID, "-"+value) || strings.Contains(alert.Message, "VMID "+value) {
return true
}
}
return false
}
func uniqueInt64(values []int64) []int64 {
seen := map[int64]bool{}
result := make([]int64, 0, len(values))
for _, value := range values {
if value > 0 && !seen[value] {
seen[value] = true
result = append(result, value)
}
}
return result
}
func uniqueInt(values []int) []int {
seen := map[int]bool{}
result := make([]int, 0, len(values))
for _, value := range values {
if value > 0 && !seen[value] {
seen[value] = true
result = append(result, value)
}
}
return result
}

View File

@@ -0,0 +1,75 @@
package main
import (
"testing"
"time"
)
func TestMaintenanceFiltersSelectedTargets(t *testing.T) {
window := MaintenanceWindow{Active: true, ServiceIDs: []int64{7}, VMIDs: []int{105}}
alerts := []Alert{
{ID: "service-7-local"},
{ID: "service-tls-8-public"},
{ID: "backup-old-105"},
{ID: "task-failed-x", Message: "VMID 105, status error"},
{ID: "cpu-usage"},
}
active, planned := splitMaintenanceAlerts(alerts, window)
if len(active) != 2 || len(planned) != 3 {
t.Fatalf("неверное разделение: active=%+v planned=%+v", active, planned)
}
}
func TestMaintenanceExpiresAutomatically(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
window, err := store.SaveMaintenanceWindow(MaintenanceWindow{EndsAt: time.Now().Add(time.Hour).Unix(), Reason: "Обновление", AllAlerts: true})
if err != nil || !window.Active {
t.Fatalf("режим не включён: %+v %v", window, err)
}
if _, err = store.db.Exec(`UPDATE settings SET value=? WHERE key='maintenance_window'`, `{"enabled":true,"startedAt":1,"endsAt":2,"reason":"test","allAlerts":true}`); err != nil {
t.Fatal(err)
}
if expired := store.MaintenanceWindow(); expired.Active || expired.Enabled {
t.Fatalf("просроченный режим остался активным: %+v", expired)
}
}
func TestRunningBackupCreatesScopedAutomaticMaintenance(t *testing.T) {
now := time.Now()
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", ID: "105"}, {Type: "vzdump", ID: "106", EndTime: now.Unix()}}}, []ServiceLink{{ServiceID: 7, VMID: 105}, {ServiceID: 8, VMID: 999}}, now)
if !window.Active || !window.Automatic || window.Source != "backup" || len(window.VMIDs) != 1 || window.VMIDs[0] != 105 || len(window.ServiceIDs) != 1 || window.ServiceIDs[0] != 7 {
t.Fatalf("неверное автоматическое окно: %+v", window)
}
}
func TestRunningBackupWithoutVMIDSuppressesAllAlerts(t *testing.T) {
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", UPID: "unknown"}}}, nil, time.Now())
if !window.Active || !window.AllAlerts {
t.Fatalf("backup с неизвестной целью должен безопасно подавить алерты: %+v", window)
}
}
func TestPlannedAlertOnlyAppearsInHistory(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
alert := Alert{ID: "service-7-local", Severity: "critical", Source: "Сервисы", Title: "Недоступен"}
visible, err := store.SyncAlertsWithMaintenance(nil, []Alert{alert}, "Обновление")
if err != nil || len(visible) != 0 {
t.Fatalf("плановый алерт стал активным: %+v %v", visible, err)
}
history, err := store.AlertHistory(10)
if err != nil || len(history) != 1 || !history[0].Planned || history[0].MaintenanceReason != "Обновление" {
t.Fatalf("плановое событие не сохранено: %+v %v", history, err)
}
pending, err := store.PendingEmailAlerts()
if err != nil || len(pending) != 0 {
t.Fatalf("плановое событие попало в почту: %+v %v", pending, err)
}
}

View File

@@ -3,6 +3,7 @@ package main
import ( import (
"bufio" "bufio"
"context" "context"
"fmt"
"io" "io"
"net" "net"
"net/http" "net/http"
@@ -36,27 +37,37 @@ type NetworkHealth struct {
ExternalIPChanged bool `json:"externalIpChanged"` ExternalIPChanged bool `json:"externalIpChanged"`
DisappearedInterfaces []string `json:"disappearedInterfaces"` DisappearedInterfaces []string `json:"disappearedInterfaces"`
CheckedAt time.Time `json:"checkedAt"` CheckedAt time.Time `json:"checkedAt"`
Summary string `json:"summary"`
LikelyCause string `json:"likelyCause"`
RecommendedAction string `json:"recommendedAction"`
} }
type NetworkInterface struct { type NetworkInterface struct {
Name string `json:"name"` Name string `json:"name"`
ReceiveBytes uint64 `json:"receiveBytes"` ReceiveBytes uint64 `json:"receiveBytes"`
TransmitBytes uint64 `json:"transmitBytes"` TransmitBytes uint64 `json:"transmitBytes"`
ReceiveErrors uint64 `json:"receiveErrors"` ReceiveErrors uint64 `json:"receiveErrors"`
TransmitErrors uint64 `json:"transmitErrors"` TransmitErrors uint64 `json:"transmitErrors"`
ReceiveDrops uint64 `json:"receiveDrops"` ReceiveDrops uint64 `json:"receiveDrops"`
TransmitDrops uint64 `json:"transmitDrops"` TransmitDrops uint64 `json:"transmitDrops"`
ReceivePerSec float64 `json:"receivePerSec"` NewErrors uint64 `json:"newErrors"`
TransmitPerSec float64 `json:"transmitPerSec"` NewReceiveDrops uint64 `json:"newReceiveDrops"`
VMID int `json:"vmid,omitempty"` NewTransmitDrops uint64 `json:"newTransmitDrops"`
GuestName string `json:"guestName,omitempty"` FaultCounters map[string]uint64 `json:"faultCounters,omitempty"`
GuestType string `json:"guestType,omitempty"` NewFaults uint64 `json:"newFaults"`
Kind string `json:"kind"` LinkDown bool `json:"linkDown"`
State string `json:"state"` NewDrops uint64 `json:"newDrops"`
Carrier bool `json:"carrier"` ReceivePerSec float64 `json:"receivePerSec"`
SpeedMbps int `json:"speedMbps,omitempty"` TransmitPerSec float64 `json:"transmitPerSec"`
Duplex string `json:"duplex,omitempty"` VMID int `json:"vmid,omitempty"`
Master string `json:"master,omitempty"` GuestName string `json:"guestName,omitempty"`
GuestType string `json:"guestType,omitempty"`
Kind string `json:"kind"`
State string `json:"state"`
Carrier bool `json:"carrier"`
SpeedMbps int `json:"speedMbps,omitempty"`
Duplex string `json:"duplex,omitempty"`
Master string `json:"master,omitempty"`
} }
type networkSnapshot struct { type networkSnapshot struct {
@@ -77,13 +88,14 @@ type networkCollector struct {
} }
func newNetworkCollector(path string) (*networkCollector, error) { func newNetworkCollector(path string) (*networkCollector, error) {
snapshot, _, _, err := readNetworkSnapshot(path) snapshot, _, details, err := readNetworkSnapshot(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
_, _, details, _ := readNetworkSnapshot(path)
previous := map[string]NetworkInterface{} previous := map[string]NetworkInterface{}
for _, v := range details { for _, v := range details {
decorateInterface(&v)
collectInterfaceFaults(&v)
previous[v.Name] = v previous[v.Name] = v
} }
known := map[string]bool{} known := map[string]bool{}
@@ -110,10 +122,11 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
for i := range details { for i := range details {
v := &details[i] v := &details[i]
decorateInterface(v) decorateInterface(v)
if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes { collectInterfaceFaults(v)
v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed if old, ok := c.previousInterfaces[v.Name]; ok {
v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed updateInterfaceDeltas(v, old, elapsed)
} }
v.LinkDown = interfaceLinkDown(*v)
} }
c.previousInterfaces = map[string]NetworkInterface{} c.previousInterfaces = map[string]NetworkInterface{}
for _, v := range details { for _, v := range details {
@@ -138,12 +151,41 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
} }
} }
c.knownInterfaces = currentNames c.knownInterfaces = currentNames
explainNetworkHealth(&c.health, details)
return NetworkMetrics{ return NetworkMetrics{
Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received, Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health, TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
}, nil }, nil
} }
func explainNetworkHealth(health *NetworkHealth, interfaces []NetworkInterface) {
health.Summary, health.LikelyCause, health.RecommendedAction = "Сеть работает штатно", "Проблем не обнаружено", "Действий не требуется"
switch {
case health.Gateway == "":
health.Summary, health.LikelyCause, health.RecommendedAction = "Не найден маршрут в локальную сеть", "На хосте отсутствует default gateway", "Проверьте адрес шлюза и конфигурацию bridge/VLAN"
case !health.GatewayUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Хост не видит локальный шлюз", "Проблема между Proxmox и роутером: линк, bridge, bond или VLAN", "Проверьте кабель, carrier и настройки vmbr/bond/VLAN"
case health.PacketLoss > 0:
health.Summary = fmt.Sprintf("Нестабильная локальная сеть: потеряно %.1f%% пакетов", health.PacketLoss)
health.LikelyCause, health.RecommendedAction = "Перегрузка или ошибки линка между Proxmox и шлюзом", "Проверьте аппаратные ошибки, кабель, порт коммутатора и duplex"
case !health.DNSUp && health.InternetUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Интернет доступен, но DNS не отвечает", "Проблема DNS-сервера или /etc/resolv.conf", "Проверьте настроенные DNS и попробуйте разрешить домен вручную"
case health.DNSUp && !health.InternetUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Локальная сеть и DNS работают, но внешняя проверка недоступна", "Проблема выше роутера, блокировка HTTPS или недоступность провайдера", "Проверьте маршрут наружу, firewall и доступ к внешним адресам"
}
for _, v := range interfaces {
if v.LinkDown {
health.Summary, health.LikelyCause, health.RecommendedAction = v.Name+": линк не поднят", "Нет carrier либо интерфейс административно выключен", "Проверьте кабель, порт коммутатора и состояние интерфейса"
return
}
if v.NewErrors+v.NewFaults > 0 {
health.Summary = fmt.Sprintf("%s: за последний интервал появилось %d изменений счётчиков сетевых ошибок", v.Name, v.NewErrors+v.NewFaults)
health.LikelyCause, health.RecommendedAction = "Счётчики интерфейса сейчас растут из-за перегрузки, кабеля, MTU или duplex", "Проверьте скорость и duplex, кабель, порт коммутатора и MTU на обоих концах"
return
}
}
}
func decorateInterface(v *NetworkInterface) { func decorateInterface(v *NetworkInterface) {
base := filepath.Join("/sys/class/net", v.Name) base := filepath.Join("/sys/class/net", v.Name)
v.State = readTrimmed(filepath.Join(base, "operstate")) v.State = readTrimmed(filepath.Join(base, "operstate"))
@@ -154,6 +196,10 @@ func decorateInterface(v *NetworkInterface) {
v.Master = filepath.Base(target) v.Master = filepath.Base(target)
} }
switch { switch {
case strings.HasPrefix(v.Name, "fwbr"):
v.Kind = "Firewall bridge"
case strings.HasPrefix(v.Name, "fwpr") || strings.HasPrefix(v.Name, "fwln"):
v.Kind = "Firewall link"
case strings.HasPrefix(v.Name, "vmbr"): case strings.HasPrefix(v.Name, "vmbr"):
v.Kind = "Bridge" v.Kind = "Bridge"
case strings.HasPrefix(v.Name, "bond"): case strings.HasPrefix(v.Name, "bond"):
@@ -232,7 +278,7 @@ func readNetworkSnapshot(path string) (networkSnapshot, []string, []NetworkInter
continue continue
} }
fields := strings.Fields(values) fields := strings.Fields(values)
if len(fields) < 9 { if len(fields) < 16 {
continue continue
} }
rx, errRX := strconv.ParseUint(fields[0], 10, 64) rx, errRX := strconv.ParseUint(fields[0], 10, 64)

103
network_faults.go Normal file
View File

@@ -0,0 +1,103 @@
package main
import (
"context"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
func counterDelta(current, previous uint64) uint64 {
if current < previous {
return 0
}
return current - previous
}
func updateInterfaceDeltas(v *NetworkInterface, old NetworkInterface, elapsed float64) {
if elapsed <= 0 {
return
}
v.ReceivePerSec = float64(counterDelta(v.ReceiveBytes, old.ReceiveBytes)) / elapsed
v.TransmitPerSec = float64(counterDelta(v.TransmitBytes, old.TransmitBytes)) / elapsed
v.NewErrors = counterDelta(v.ReceiveErrors, old.ReceiveErrors) + counterDelta(v.TransmitErrors, old.TransmitErrors)
v.NewReceiveDrops = counterDelta(v.ReceiveDrops, old.ReceiveDrops)
v.NewTransmitDrops = counterDelta(v.TransmitDrops, old.TransmitDrops)
v.NewDrops = v.NewReceiveDrops + v.NewTransmitDrops
for name, value := range v.FaultCounters {
if previous, ok := old.FaultCounters[name]; ok {
v.NewFaults += counterDelta(value, previous)
}
}
// A transition matters even if the interface has not carried traffic yet.
v.LinkDown = old.Carrier && !v.Carrier
}
func interfaceLinkDown(v NetworkInterface) bool {
if v.Kind == "Firewall bridge" || v.Kind == "Firewall link" {
return false
}
if v.LinkDown {
return true
}
used := v.Master != "" || v.ReceiveBytes+v.TransmitBytes > 0
if v.Kind == "Physical" {
return used && (!v.Carrier || v.State == "down")
}
return used && v.State == "down"
}
func collectInterfaceFaults(v *NetworkInterface) {
base := filepath.Join("/sys/class/net", v.Name)
// /proc/net/dev folds missed packets into RX drop. Sysfs keeps software
// drops separate from rx_missed_errors, which must still trigger an alert.
if n, err := strconv.ParseUint(readTrimmed(filepath.Join(base, "statistics/rx_dropped")), 10, 64); err == nil {
v.ReceiveDrops = n
}
v.FaultCounters = map[string]uint64{}
for _, name := range []string{"rx_crc_errors", "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", "tx_fifo_errors", "tx_carrier_errors", "carrier_down_count"} {
path := filepath.Join(base, "statistics", name)
if name == "carrier_down_count" {
path = filepath.Join(base, name)
}
if n, err := strconv.ParseUint(readTrimmed(path), 10, 64); err == nil {
v.FaultCounters[name] = n
}
}
// Only Atlantic exposes the driver-specific names below. No configuration
// changes, and a missing ethtool is not itself a network failure.
driver, err := filepath.EvalSymlinks(filepath.Join(base, "device/driver"))
if err != nil || filepath.Base(driver) != "atlantic" {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if out, err := exec.CommandContext(ctx, "ethtool", "-S", v.Name).Output(); err == nil {
for k, n := range parseAtlanticFaults(string(out)) {
v.FaultCounters["atlantic:"+k] = n
}
}
}
func parseAtlanticFaults(output string) map[string]uint64 {
result := map[string]uint64{}
for _, line := range strings.Split(output, "\n") {
key, value, ok := strings.Cut(strings.TrimSpace(line), ":")
if !ok {
continue
}
fields := strings.Fields(key)
if len(fields) == 0 {
continue
}
switch fields[len(fields)-1] {
case "InErrors", "InDroppedDma", "AllocFails", "SkbAllocFails", "Restarts":
if n, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64); err == nil {
result[key] = n
}
}
}
return result
}

67
network_faults_test.go Normal file
View File

@@ -0,0 +1,67 @@
package main
import (
"strings"
"testing"
)
func TestNetworkDropsDoNotAlert(t *testing.T) {
for _, name := range []string{"enp6s0", "vmbr0", "fwbr100i0", "tap100i0"} {
v := NetworkInterface{Name: name, Carrier: true, State: "up", NewDrops: 15, NewReceiveDrops: 15}
metrics := dashboardMetrics{Network: NetworkMetrics{Interfaces: []NetworkInterface{v}}}
for _, a := range evaluateAlerts(metrics, AlertThresholds{}) {
if a.ID == "network-errors" {
t.Fatalf("%s: drops triggered alert", name)
}
}
v.NewErrors = 1
metrics.Network.Interfaces[0] = v
found := false
for _, a := range evaluateAlerts(metrics, AlertThresholds{}) {
if a.ID == "network-errors" {
found = a.Severity == "critical"
}
}
if !found {
t.Fatalf("%s: errors did not trigger critical alert", name)
}
}
}
func TestNetworkHardwareFaultAndLinkAlerts(t *testing.T) {
for _, v := range []NetworkInterface{{Name: "eth0", NewFaults: 1}, {Name: "eth0", LinkDown: true}} {
found := false
for _, a := range evaluateAlerts(dashboardMetrics{Network: NetworkMetrics{Interfaces: []NetworkInterface{v}}}, AlertThresholds{}) {
if strings.HasPrefix(a.ID, "network-") && a.Severity == "critical" {
found = true
}
}
if !found {
t.Fatalf("no critical alert for %+v", v)
}
}
}
func TestNetworkCounterResetsAndFaultBaselines(t *testing.T) {
old := NetworkInterface{ReceiveBytes: 100, ReceiveErrors: 20, TransmitErrors: 4, ReceiveDrops: 8, FaultCounters: map[string]uint64{"rx_missed_errors": 10, "atlantic:InErrors": 9}, Carrier: true}
next := NetworkInterface{ReceiveBytes: 1, ReceiveErrors: 1, TransmitErrors: 6, ReceiveDrops: 10, FaultCounters: map[string]uint64{"rx_missed_errors": 12, "atlantic:InErrors": 1, "new": 100}, Carrier: false}
updateInterfaceDeltas(&next, old, 2)
if next.NewErrors != 2 || next.NewFaults != 2 || next.NewReceiveDrops != 2 || next.ReceivePerSec != 0 || !next.LinkDown {
t.Fatalf("bad deltas: %+v", next)
}
}
func TestAtlanticFaults(t *testing.T) {
got := parseAtlanticFaults("NIC statistics:\n InErrors: 2\n InDroppedDma: 3\n Queue[0] AllocFails: 4\n Queue[1] SkbAllocFails: 5\n Queue[0] Restarts: 6\n Queue[0] InErrors: 7\n InPackets: 100\n Queue[0] XdpDrop: 50\n InErrorsBad: nope")
if len(got) != 6 || got["InDroppedDma"] != 3 || got["Queue[0] Restarts"] != 6 {
t.Fatalf("bad counters: %v", got)
}
}
func TestFirewallNotAttributedToGuest(t *testing.T) {
n := NetworkMetrics{Interfaces: []NetworkInterface{{Name: "fwbr100i0", ReceiveDrops: 100}, {Name: "tap100i0"}}}
decorateNetworkInterfaces(&n, GuestsMetrics{})
if n.Interfaces[0].VMID != 0 || n.Interfaces[1].VMID != 100 {
t.Fatalf("bad VM attribution: %+v", n)
}
}

162
network_flows.go Normal file
View File

@@ -0,0 +1,162 @@
package main
import (
_ "embed"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"time"
)
//go:embed assets/network.html
var networkHTML string
// Original and reply tuples describe one conntrack entry, not two connections.
type NetworkFlow struct {
Monitoring bool `json:"monitoring"`
External bool `json:"external"`
Protocol string `json:"protocol"`
Source string `json:"source"`
Destination string `json:"destination"`
SourcePort int `json:"sourcePort"`
DestinationPort int `json:"destinationPort"`
ReplySource string `json:"replySource"`
ReplyDestination string `json:"replyDestination"`
ReplySourcePort int `json:"replySourcePort"`
ReplyDestinationPort int `json:"replyDestinationPort"`
State string `json:"state"`
Country string `json:"country"`
DestinationCountry string `json:"destinationCountry"`
Packets uint64 `json:"packets"`
ReplyPackets uint64 `json:"replyPackets"`
Bytes uint64 `json:"bytes"`
ReplyBytes uint64 `json:"replyBytes"`
Accounting bool `json:"accounting"`
Zone string `json:"zone,omitempty"`
}
type FirewallCounter struct {
Name string `json:"name"`
Packets uint64 `json:"packets"`
Bytes uint64 `json:"bytes"`
}
type NetworkSnapshot struct {
MonitoringHidden int `json:"monitoringHidden"`
At int64 `json:"at"`
ReceivedAt int64 `json:"receivedAt"`
Flows []NetworkFlow `json:"flows"`
Error string `json:"error,omitempty"`
Limited bool `json:"limited"`
ParseErrors int `json:"parseErrors"`
Firewall []FirewallCounter `json:"firewall"`
FirewallAt int64 `json:"firewallAt"`
FirewallError string `json:"firewallError,omitempty"`
AgentName string `json:"agentName"`
}
func networkExternalIP(value string) bool {
ip := net.ParseIP(value)
if ip == nil || !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
return false
}
if v4 := ip.To4(); v4 != nil && v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
return false
}
return true
}
func (s *Store) SaveNetwork(agent string, snapshot NetworkSnapshot) error {
if len(snapshot.Flows) > 5000 || len(snapshot.Firewall) > 128 {
return errors.New("network snapshot too large")
}
if snapshot.At < time.Now().Unix()-120 || snapshot.At > time.Now().Unix()+60 {
return errors.New("invalid snapshot time")
}
for i := range snapshot.Flows {
f := &snapshot.Flows[i]
if net.ParseIP(f.Source) == nil || net.ParseIP(f.Destination) == nil || net.ParseIP(f.ReplySource) == nil || net.ParseIP(f.ReplyDestination) == nil {
return errors.New("invalid network address")
}
for _, p := range []int{f.SourcePort, f.DestinationPort, f.ReplySourcePort, f.ReplyDestinationPort} {
if p < 0 || p > 65535 {
return errors.New("invalid network port")
}
}
if f.Protocol != "tcp" && f.Protocol != "udp" && f.Protocol != "icmp" && f.Protocol != "icmpv6" {
return errors.New("invalid protocol")
}
f.State = trafficText(f.State, 32)
f.Zone = trafficText(f.Zone, 16)
f.External = networkExternalIP(f.Source) || networkExternalIP(f.Destination)
f.Country = lookupTrafficCountry(f.Source)
f.DestinationCountry = lookupTrafficCountry(f.Destination)
}
snapshot.Error = trafficText(snapshot.Error, 600)
snapshot.FirewallError = trafficText(snapshot.FirewallError, 600)
for i := range snapshot.Firewall {
snapshot.Firewall[i].Name = trafficText(snapshot.Firewall[i].Name, 200)
}
snapshot.MonitoringHidden = 0
snapshot.ReceivedAt = time.Now().Unix()
snapshot.AgentName = ""
raw, err := json.Marshal(snapshot)
if err != nil {
return err
}
_, err = s.db.Exec(`INSERT INTO network_sources(agent,snapshot) VALUES(?,?) ON CONFLICT(agent) DO UPDATE SET snapshot=excluded.snapshot`, agent, string(raw))
return err
}
func networkRoutes(mux *http.ServeMux, s *Store) {
enabled := func() bool { c := s.TrafficSettings(); return c.Enabled && c.NetworkEnabled }
mux.HandleFunc("/network", func(w http.ResponseWriter, r *http.Request) {
if !enabled() {
http.Error(w, "Включите TCP/UDP в настройках карты трафика", 404)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(networkHTML))
})
mux.HandleFunc("/api/network", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(405)
return
}
if !enabled() {
http.Error(w, "Сбор сети отключён", 404)
return
}
settings := s.TrafficSettings()
snapshot, err := s.ReadNetworkSnapshot(settings.AgentID, settings.MonitoringRules, r.URL.Query().Get("hideMonitoring") == "1")
if err != nil {
writeJSON(w, nil, err)
return
}
writeJSON(w, snapshot, nil)
})
mux.HandleFunc("/api/agent/network", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
w.WriteHeader(405)
return
}
id, secret, ok := agentCredentials(r)
if !ok || !s.AuthenticateAgent(id, secret) {
http.Error(w, "Unauthorized", 401)
return
}
c := s.TrafficSettings()
if !c.Enabled || !c.NetworkEnabled || c.AgentID != id {
http.Error(w, "Источник отключён", 403)
return
}
var snapshot NetworkSnapshot
if json.NewDecoder(io.LimitReader(r.Body, 4<<20)).Decode(&snapshot) != nil {
http.Error(w, "Invalid snapshot", 400)
return
}
if err := s.SaveNetwork(id, snapshot); err != nil {
http.Error(w, err.Error(), 400)
return
}
writeJSON(w, map[string]bool{"accepted": true}, nil)
})
}

189
network_flows_test.go Normal file
View File

@@ -0,0 +1,189 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
const networkNATLine = "ipv4 2 tcp 6 431999 ESTABLISHED src=8.8.8.8 dst=192.168.0.12 sport=51000 dport=443 packets=12 bytes=1500 src=172.18.0.2 dst=8.8.8.8 sport=443 dport=51000 packets=16 bytes=2500 [ASSURED] mark=0 zone=2 use=1"
func TestNetworkConntrackTuples(t *testing.T) {
f, ok := parseNetworkFlow(networkNATLine)
if !ok || f.Source != "8.8.8.8" || f.Destination != "192.168.0.12" || f.ReplySource != "172.18.0.2" || f.SourcePort != 51000 || f.ReplySourcePort != 443 || f.State != "ESTABLISHED" || !f.Accounting || f.Bytes != 1500 || f.ReplyBytes != 2500 || f.Zone != "2" {
t.Fatalf("NAT tuple corrupted: %+v", f)
}
udp, ok := parseNetworkFlow("ipv6 10 udp 17 29 src=2001:4860:4860::8888 dst=fd00::2 sport=51000 dport=53 [UNREPLIED] src=fd00::2 dst=2001:4860:4860::8888 sport=53 dport=51000 mark=0 use=1")
if !ok || udp.State != "UNREPLIED" || udp.Accounting || udp.Protocol != "udp" {
t.Fatalf("UDP: %+v", udp)
}
icmp, ok := parseNetworkFlow("icmp 1 29 src=8.8.8.8 dst=192.168.0.12 type=8 code=0 id=3 src=192.168.0.12 dst=8.8.8.8 type=0 code=0 id=3 use=1")
if !ok || icmp.SourcePort != 0 || icmp.State != "TRACKED" {
t.Fatalf("ICMP: %+v", icmp)
}
if _, ok := parseNetworkFlow("tcp 6 20 SYN_SENT src=not-an-ip dst=192.168.0.12"); ok {
t.Fatal("invalid accepted")
}
}
func TestNetworkSnapshotLimit(t *testing.T) {
var s NetworkSnapshot
appendNetworkFlows(&s, []byte(strings.Repeat(networkNATLine+"\n", 5001)))
if len(s.Flows) != 5000 || !s.Limited {
t.Fatalf("limit not enforced %d", len(s.Flows))
}
b := networkBuffer{limit: 2}
if _, err := b.Write([]byte("123")); err == nil || b.Len() != 0 {
t.Fatal("output unbounded")
}
}
func TestFirewallOnlyConfirmedDropCounters(t *testing.T) {
raw := `{"nftables":[{"rule":{"family":"ip","table":"crowdsec","chain":"crowdsec-chain","handle":3,"expr":[{"match":{}},{"counter":{"packets":10,"bytes":1000}},{"drop":null}]}},{"rule":{"table":"crowdsec","expr":[{"counter":{"packets":99}},{"accept":null}]}},{"rule":{"table":"other","expr":[{"counter":{"packets":99}},{"drop":null}]}},{"rule":{"table":"crowdsec","expr":[{"counter":{"packets":99}},{"match":{}},{"drop":null}]}},{"rule":{"table":"crowdsec","expr":[{"counter":"named-counter"},{"drop":null}]}}]}`
counters, err := parseFirewallCounters([]byte(raw))
if err != nil || len(counters) != 1 || counters[0].Packets != 10 || counters[0].Bytes != 1000 {
t.Fatalf("invented blocks: %+v %v", counters, err)
}
if _, err := parseFirewallCounters([]byte("invalid")); err == nil {
t.Fatal("invalid nft JSON accepted")
}
}
func TestNetworkRouteAuthorizationAndSourceSelection(t *testing.T) {
s, a := trafficFixture(t)
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID})
mux := http.NewServeMux()
trafficRoutes(mux, s)
flow, _ := parseNetworkFlow(networkNATLine)
snapshot := NetworkSnapshot{At: time.Now().Unix(), Flows: []NetworkFlow{flow}}
raw, _ := json.Marshal(snapshot)
post := func(token string) int {
r := httptest.NewRequest("POST", "/api/agent/network", bytes.NewReader(raw))
r.Header.Set("Authorization", "Bearer "+token)
w := httptest.NewRecorder()
mux.ServeHTTP(w, r)
return w.Code
}
if got := post(a.ID + "." + a.Secret); got != 403 {
t.Fatalf("disabled accepts data: %d", got)
}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, NetworkEnabled: true, AgentID: a.ID})
if got := post("wrong.token"); got != 401 {
t.Fatalf("unauthenticated: %d", got)
}
if got := post(a.ID + "." + a.Secret); got != 200 {
t.Fatalf("valid: %d", got)
}
w := httptest.NewRecorder()
mux.ServeHTTP(w, httptest.NewRequest("GET", "/api/network", nil))
var got NetworkSnapshot
json.Unmarshal(w.Body.Bytes(), &got)
if w.Code != 200 || len(got.Flows) != 1 || got.Flows[0].Country == "" || got.ReceivedAt == 0 {
t.Fatalf("bad snapshot: %s", w.Body.String())
}
token, _ := s.CreateAgentEnrollmentToken()
other, _ := s.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "other"})
s.SaveTrafficSettings(TrafficSettings{Enabled: true, NetworkEnabled: true, AgentID: other.ID})
if code := post(a.ID + "." + a.Secret); code != 403 {
t.Fatalf("old source accepted: %d", code)
}
w = httptest.NewRecorder()
mux.ServeHTTP(w, httptest.NewRequest("GET", "/api/network", nil))
json.Unmarshal(w.Body.Bytes(), &got)
if len(got.Flows) != 0 {
t.Fatal("old source data leaked")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: false, NetworkEnabled: true, AgentID: a.ID})
for _, path := range []string{"/network", "/api/network"} {
w = httptest.NewRecorder()
mux.ServeHTTP(w, httptest.NewRequest("GET", path, nil))
if w.Code != 404 {
t.Fatalf("disabled page: %s", path)
}
}
}
func TestNetworkRejectBadSnapshots(t *testing.T) {
s, a := trafficFixture(t)
if err := s.SaveNetwork(a.ID, NetworkSnapshot{At: time.Now().Unix() - 300}); err == nil {
t.Fatal("stale timestamp")
}
f, _ := parseNetworkFlow(networkNATLine)
f.SourcePort = 99999
if err := s.SaveNetwork(a.ID, NetworkSnapshot{At: time.Now().Unix(), Flows: []NetworkFlow{f}}); err == nil {
t.Fatal("invalid port")
}
}
func TestNetworkCommandCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := networkCommand(ctx, "conntrack", "-L"); err == nil {
t.Fatal("cancelled command ran")
}
}
func TestNetworkExternalAddresses(t *testing.T) {
for _, ip := range []string{"192.168.1.1", "127.0.0.1", "100.64.0.1", "224.0.0.1", "ff02::1", "fe80::1", "fd00::1", "::"} {
if networkExternalIP(ip) {
t.Fatalf("not external: %s", ip)
}
}
for _, ip := range []string{"8.8.8.8", "2001:4860:4860::8888"} {
if !networkExternalIP(ip) {
t.Fatalf("external: %s", ip)
}
}
}
func TestConntrackInstallerOnlyInstallsRequestedPackage(t *testing.T) {
var calls [][]string
err := installConntrackPackage(func(name string, args ...string) error {
calls = append(calls, append([]string{name}, args...))
return nil
}, func() bool { return true })
if err != nil || len(calls) != 2 {
t.Fatalf("installer: %v %v", calls, err)
}
if calls[0][len(calls[0])-1] != "update" || strings.Join(calls[1][len(calls[1])-5:], " ") != "-y --no-install-recommends --no-remove install conntrack" {
t.Fatalf("unexpected apt operation: %v", calls)
}
calls = nil
err = installConntrackPackage(func(name string, args ...string) error { calls = append(calls, args); return fmt.Errorf("apt locked") }, func() bool { return true })
if err == nil || len(calls) != 1 {
t.Fatal("continued after failed apt update")
}
}
func TestConntrackDependencySingleInstallAndBackoff(t *testing.T) {
var d networkDependencyState
now := time.Now()
started := make(chan struct{})
finish := make(chan struct{})
installer := func() error { close(started); <-finish; return fmt.Errorf("offline") }
if got := d.ensure(now, true, installer); got != "" {
t.Fatal("installed tool should be skipped")
}
d.ensure(now, false, installer)
<-started
// A second call must not start a second installer (which would panic closing started).
d.ensure(now, false, installer)
close(finish)
deadline := time.Now().Add(time.Second)
for {
d.Lock()
running := d.running
d.Unlock()
if !running {
break
}
if time.Now().After(deadline) {
t.Fatal("installer stuck")
}
time.Sleep(time.Millisecond)
}
got := d.ensure(now.Add(time.Minute), false, installer)
if !strings.Contains(got, "offline") || !strings.Contains(got, "Повторная") {
t.Fatalf("backoff missing: %s", got)
}
}

View File

@@ -3,6 +3,7 @@ package main
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
) )
@@ -33,8 +34,28 @@ func TestReadNetworkSnapshot(t *testing.T) {
} }
} }
func TestExplainNetworkHealthDistinguishesDNSFailure(t *testing.T) {
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: false}
explainNetworkHealth(&health, nil)
if !strings.Contains(health.Summary, "DNS") || health.RecommendedAction == "" {
t.Fatalf("непонятная диагностика: %+v", health)
}
}
func TestExplainNetworkHealthIgnoresOldInterfaceCounters(t *testing.T) {
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: true}
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 42}})
if health.Summary != "Сеть работает штатно" {
t.Fatalf("старый накопленный счётчик вызвал текущую тревогу: %+v", health)
}
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 43, NewDrops: 1}})
if health.Summary != "Сеть работает штатно" {
t.Fatalf("software drops вызвали тревогу: %+v", health)
}
}
func TestInterfaceVMID(t *testing.T) { func TestInterfaceVMID(t *testing.T) {
for name, want := range map[string]int{"tap100i0": 100, "veth108i0": 108, "fwbr113i0": 113, "vmbr0": 0, "enp6s0": 0} { for name, want := range map[string]int{"tap100i0": 100, "veth108i0": 108, "fwbr113i0": 0, "vmbr0": 0, "enp6s0": 0} {
if got := interfaceVMID(name); got != want { if got := interfaceVMID(name); got != want {
t.Fatalf("%s: got %d want %d", name, got, want) t.Fatalf("%s: got %d want %d", name, got, want)
} }

17
scripts/build-release.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
version=${1:?Usage: build-release.sh vX.Y.Z}
cd "$(dirname "$0")/.."
go test ./...
mkdir -p dist
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.version=$version" -o dist/proxmox-dashboard-linux-amd64 .
(cd dist && shasum -a 256 proxmox-dashboard-linux-amd64 > proxmox-dashboard-linux-amd64.sha256)
stage=$(mktemp -d)
trap 'rm -rf "$stage"' EXIT INT TERM
package="$stage/proxmox-dashboard-install"
mkdir -p "$package/dist" "$package/scripts"
cp dist/proxmox-dashboard-linux-amd64 dist/proxmox-dashboard-linux-amd64.sha256 "$package/dist/"
cp scripts/install.sh scripts/install-agent.sh scripts/uninstall-agent.sh scripts/proxmox-dashboard-update "$package/scripts/"
cp assets/NOTICE.md "$package/MAP-DATA-NOTICE.md"
cp ./*.service ./*.timer ./*.path README.md "$package/"
tar -czf dist/proxmox-dashboard-install-amd64.tar.gz -C "$stage" proxmox-dashboard-install

120
scripts/install-agent.sh Executable file
View File

@@ -0,0 +1,120 @@
#!/bin/sh
set -eu
if [ "$(id -u)" -ne 0 ]; then
echo "Запустите команду установки от root." >&2
exit 1
fi
if [ -z "${DASHBOARD_URL:-}" ] || [ -z "${ENROLL_TOKEN:-}" ]; then
echo "Не заданы DASHBOARD_URL и ENROLL_TOKEN. Создайте команду в настройках Dashboard." >&2
exit 1
fi
case "$(uname -m)" in x86_64|amd64) ;; *) echo "Поддерживается Linux amd64 (x86_64)." >&2; exit 1 ;; esac
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
agent_init=systemd
elif command -v rc-service >/dev/null 2>&1 && command -v supervise-daemon >/dev/null 2>&1; then
agent_init=openrc
else
echo "Нужен работающий systemd или OpenRC с supervise-daemon." >&2; exit 1
fi
if ! command -v curl >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then apk add --no-cache curl ca-certificates; else echo "Установите curl и ca-certificates." >&2; exit 1; fi
fi
GITEA_URL=${GITEA_URL:-https://git.myown.center}
GITEA_REPOSITORY=${GITEA_REPOSITORY:-maxim/ProxmoxDash}
VMID=${VMID:-0}
AGENT_NAME=${AGENT_NAME:-$(hostname)}
case "$DASHBOARD_URL$ENROLL_TOKEN$AGENT_NAME$VMID" in
*'
'*) echo "Параметры установки не должны содержать переносы строк." >&2; exit 1 ;;
esac
release_json=$(curl -fsSL "$GITEA_URL/api/v1/repos/$GITEA_REPOSITORY/releases/latest")
download_url=$(printf '%s' "$release_json" | sed -n 's/.*"browser_download_url":"\([^"]*\/proxmox-dashboard-linux-amd64\)".*/\1/p' | head -n 1)
if [ -z "$download_url" ]; then
echo "В последнем релизе не найден Linux-бинарник." >&2
exit 1
fi
install -d -m 0700 /var/lib/proxmox-dashboard-agent
temporary_binary=$(mktemp /tmp/proxmox-dashboard-agent.XXXXXX)
trap 'rm -f "$temporary_binary" "$temporary_binary.sha256"' EXIT
curl -fsSL "$download_url" -o "$temporary_binary"
curl -fsSL "$download_url.sha256" -o "$temporary_binary.sha256"
expected_hash=$(awk 'NR==1 {print $1}' "$temporary_binary.sha256")
actual_hash=$(sha256sum "$temporary_binary" | awk '{print $1}')
if [ "${#expected_hash}" -ne 64 ] || [ "$expected_hash" != "$actual_hash" ]; then echo "SHA-256 загрузки не совпал." >&2; exit 1; fi
install -m 0755 "$temporary_binary" /var/lib/proxmox-dashboard-agent/proxmox-dashboard-agent
umask 077
{
printf 'DASHBOARD_URL=%s\n' "$DASHBOARD_URL"
printf 'ENROLL_TOKEN=%s\n' "$ENROLL_TOKEN"
printf 'AGENT_NAME=%s\n' "$AGENT_NAME"
printf 'VMID=%s\n' "$VMID"
} > /etc/proxmox-dashboard-agent.env
if [ "$agent_init" = systemd ]; then
cat > /etc/systemd/system/proxmox-dashboard-agent.service <<'UNIT'
[Unit]
Description=Proxmox Dashboard read-only guest agent
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=simple
EnvironmentFile=/etc/proxmox-dashboard-agent.env
ExecStart=/var/lib/proxmox-dashboard-agent/proxmox-dashboard-agent --agent --agent-server ${DASHBOARD_URL} --agent-enroll-token ${ENROLL_TOKEN} --agent-name ${AGENT_NAME} --agent-vmid ${VMID}
Restart=always
RestartSec=5
NoNewPrivileges=true
ProtectHome=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/lib/proxmox-dashboard-agent
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable proxmox-dashboard-agent.service
systemctl restart proxmox-dashboard-agent.service
else
mkdir -p /usr/local/sbin /etc/init.d
cat > /usr/local/sbin/proxmox-dashboard-agent-run <<'RUN'
#!/bin/sh
set -eu
# Parse literal key=value records, never evaluate configuration as shell code.
while IFS='=' read -r key value; do
case "$key" in DASHBOARD_URL|ENROLL_TOKEN|AGENT_NAME|VMID|DASHBOARD_NPM_LOG_DIR|DASHBOARD_CROWDSEC_CONTAINER) export "$key=$value" ;; esac
done < /etc/proxmox-dashboard-agent.env
exec /var/lib/proxmox-dashboard-agent/proxmox-dashboard-agent --agent --agent-server "$DASHBOARD_URL" --agent-enroll-token "$ENROLL_TOKEN" --agent-name "$AGENT_NAME" --agent-vmid "$VMID"
RUN
chmod 0755 /usr/local/sbin/proxmox-dashboard-agent-run
cat > /etc/init.d/proxmox-dashboard-agent <<'OPENRC'
#!/sbin/openrc-run
description="Proxmox Dashboard guest agent"
command="/usr/local/sbin/proxmox-dashboard-agent-run"
supervisor="supervise-daemon"
respawn_delay=5
respawn_max=0
output_log="/var/log/proxmox-dashboard-agent.log"
error_log="/var/log/proxmox-dashboard-agent.log"
depend() {
need net
after docker
}
start_pre() {
checkpath --file --mode 0600 /var/log/proxmox-dashboard-agent.log
}
OPENRC
chmod 0755 /etc/init.d/proxmox-dashboard-agent
rc-update add proxmox-dashboard-agent default
rc-service proxmox-dashboard-agent restart
fi
rm -f /usr/local/bin/proxmox-dashboard-agent
echo "Агент установлен. Через несколько секунд он появится в Dashboard."

21
scripts/uninstall-agent.sh Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/sh
set -eu
if [ "$(id -u)" -ne 0 ]; then
echo "Запустите удаление от root." >&2
exit 1
fi
if command -v rc-service >/dev/null 2>&1 && [ -f /etc/init.d/proxmox-dashboard-agent ]; then
rc-service proxmox-dashboard-agent stop || true
rc-update del proxmox-dashboard-agent default || true
fi
systemctl disable --now proxmox-dashboard-agent.service 2>/dev/null || true
rm -f /etc/init.d/proxmox-dashboard-agent /usr/local/sbin/proxmox-dashboard-agent-run
rm -f /etc/systemd/system/proxmox-dashboard-agent.service
rm -f /etc/proxmox-dashboard-agent.env
rm -f /usr/local/bin/proxmox-dashboard-agent
rm -rf /var/lib/proxmox-dashboard-agent
systemctl daemon-reload 2>/dev/null || true
systemctl reset-failed proxmox-dashboard-agent.service 2>/dev/null || true
echo "Агент и его локальные учётные данные удалены."

110
service_links.go Normal file
View File

@@ -0,0 +1,110 @@
package main
import (
"fmt"
"sort"
"strings"
)
type ServiceLink struct {
ServiceID int64 `json:"serviceId"`
AgentID string `json:"agentId"`
VMID int `json:"vmid"`
ComponentType string `json:"componentType"`
ComponentName string `json:"componentName"`
Port int `json:"port"`
}
func (s *Store) ServiceLinks() ([]ServiceLink, error) {
rows, err := s.db.Query(`SELECT service_id,agent_id,vmid,component_type,component_name,port FROM service_links ORDER BY service_id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ServiceLink
for rows.Next() {
var v ServiceLink
if err = rows.Scan(&v.ServiceID, &v.AgentID, &v.VMID, &v.ComponentType, &v.ComponentName, &v.Port); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) SaveServiceLink(v ServiceLink) (ServiceLink, error) {
v.AgentID = strings.TrimSpace(v.AgentID)
v.ComponentType = strings.TrimSpace(v.ComponentType)
v.ComponentName = strings.TrimSpace(v.ComponentName)
if v.ServiceID <= 0 {
return v, fmt.Errorf("выберите сервис")
}
if v.AgentID == "" && v.VMID <= 0 {
return v, fmt.Errorf("выберите агент или VMID")
}
if v.ComponentType != "" && v.ComponentType != "systemd" && v.ComponentType != "docker" && v.ComponentType != "process" {
return v, fmt.Errorf("неизвестный тип компонента")
}
if v.Port < 0 || v.Port > 65535 {
return v, fmt.Errorf("некорректный порт")
}
_, err := s.db.Exec(`INSERT INTO service_links(service_id,agent_id,vmid,component_type,component_name,port) VALUES(?,?,?,?,?,?) ON CONFLICT(service_id) DO UPDATE SET agent_id=excluded.agent_id,vmid=excluded.vmid,component_type=excluded.component_type,component_name=excluded.component_name,port=excluded.port`, v.ServiceID, v.AgentID, v.VMID, v.ComponentType, v.ComponentName, v.Port)
return v, err
}
func (s *Store) DeleteServiceLink(serviceID int64) error {
_, err := s.db.Exec(`DELETE FROM service_links WHERE service_id=?`, serviceID)
return err
}
func groupLinkedAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
if len(metrics.ServiceLinks) == 0 {
return alerts
}
agentOffline := map[string]ManagedAgent{}
for _, agent := range metrics.Agents {
if !agent.Online {
agentOffline[agent.ID] = agent
}
}
affected := map[string][]string{}
remove := map[string]bool{}
for _, link := range metrics.ServiceLinks {
agent, offline := agentOffline[link.AgentID]
if !offline {
continue
}
for _, service := range metrics.Services.Services {
if service.ID != link.ServiceID {
continue
}
for _, endpoint := range service.Endpoints {
if !endpoint.Up {
key := fmt.Sprintf("service-%d-%s", service.ID, endpoint.Kind)
remove[key] = true
affected[agent.ID] = append(affected[agent.ID], service.Name)
}
}
}
}
if len(affected) == 0 {
return alerts
}
out := make([]Alert, 0, len(alerts))
for _, alert := range alerts {
skip := remove[alert.ID]
for agentID := range affected {
if alert.ID == "agent-offline-"+agentID+fmt.Sprintf("-%d", agentOffline[agentID].VMID) {
skip = true
}
}
if !skip {
out = append(out, alert)
}
}
for agentID, names := range affected {
sort.Strings(names)
agent := agentOffline[agentID]
out = append(out, Alert{ID: "group-agent-offline-" + agentID + fmt.Sprintf("-%d", agent.VMID), Severity: "critical", Source: "Корреляция", Title: agent.Name + ": агент недоступен, связанные сервисы не отвечают", Message: fmt.Sprintf("VMID %d. Затронуто сервисов: %d — %s. Вероятная первопричина: недоступна гостевая система или агент.", agent.VMID, len(names), strings.Join(names, ", "))})
}
return out
}

View File

@@ -74,6 +74,20 @@ type ServiceHistoryPoint struct {
LatencyMS float64 `json:"latencyMs"` LatencyMS float64 `json:"latencyMs"`
} }
var serviceHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 32,
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 30 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12},
}
var serviceHTTPClient = &http.Client{Transport: serviceHTTPTransport}
func (s *Store) Services() ([]MonitoredService, error) { func (s *Store) Services() ([]MonitoredService, error) {
rows, err := s.db.Query(`SELECT id,name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled FROM services ORDER BY id`) rows, err := s.db.Query(`SELECT id,name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled FROM services ORDER BY id`)
if err != nil { if err != nil {
@@ -296,14 +310,12 @@ func checkServiceEndpoint(v MonitoredService, kind, label, target string) Servic
st.Error = strings.TrimSpace(string(out)) st.Error = strings.TrimSpace(string(out))
} }
default: default:
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}}
client := &http.Client{Transport: tr, Timeout: time.Duration(v.TimeoutSeconds) * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil { if err != nil {
st.Error = err.Error() st.Error = err.Error()
break break
} }
resp, err := client.Do(req) resp, err := serviceHTTPClient.Do(req)
if err != nil { if err != nil {
st.Error = err.Error() st.Error = err.Error()
} else { } else {

View File

@@ -1,6 +1,9 @@
package main package main
import "testing" import (
"testing"
"time"
)
func TestServiceCRUD(t *testing.T) { func TestServiceCRUD(t *testing.T) {
store, err := openStore(":memory:") store, err := openStore(":memory:")
@@ -38,3 +41,15 @@ func TestValidateServiceChecksBothHTTPAddresses(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
func TestHTTPServiceTransportHasBoundedIdlePool(t *testing.T) {
if serviceHTTPClient.Transport != serviceHTTPTransport {
t.Fatal("проверки сервисов должны использовать общий HTTP transport")
}
if serviceHTTPTransport.IdleConnTimeout < time.Second {
t.Fatal("idle-соединения должны автоматически закрываться")
}
if serviceHTTPTransport.MaxIdleConnsPerHost < 1 || serviceHTTPTransport.MaxIdleConnsPerHost > 4 {
t.Fatalf("небезопасный размер пула на хост: %d", serviceHTTPTransport.MaxIdleConnsPerHost)
}
}

View File

@@ -31,6 +31,17 @@ type AlertThresholds struct {
NVMeTempWarning float64 `json:"nvmeTempWarning"` NVMeTempWarning float64 `json:"nvmeTempWarning"`
UPSChargeCritical float64 `json:"upsChargeCritical"` UPSChargeCritical float64 `json:"upsChargeCritical"`
BackupMaxAgeHours float64 `json:"backupMaxAgeHours"` BackupMaxAgeHours float64 `json:"backupMaxAgeHours"`
DiskHDDUtil float64 `json:"diskHddUtil"`
DiskHDDLatency float64 `json:"diskHddLatency"`
DiskHDDQueue float64 `json:"diskHddQueue"`
DiskSSDUtil float64 `json:"diskSsdUtil"`
DiskSSDLatency float64 `json:"diskSsdLatency"`
DiskSSDQueue float64 `json:"diskSsdQueue"`
DiskNVMeUtil float64 `json:"diskNvmeUtil"`
DiskNVMeLatency float64 `json:"diskNvmeLatency"`
DiskNVMeQueue float64 `json:"diskNvmeQueue"`
DiskIODuration float64 `json:"diskIoDuration"`
DiskIOExcluded string `json:"diskIoExcluded"`
} }
func defaultThresholds() AlertThresholds { func defaultThresholds() AlertThresholds {
@@ -40,6 +51,7 @@ func defaultThresholds() AlertThresholds {
StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90, StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90,
DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30, DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30,
BackupMaxAgeHours: 48, BackupMaxAgeHours: 48,
DiskHDDUtil: 90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60,
} }
} }
@@ -94,6 +106,7 @@ func openStore(path string) (*Store, error) {
} }
db.SetMaxOpenConns(1) db.SetMaxOpenConns(1)
statements := []string{ statements := []string{
trafficSchema,
`PRAGMA journal_mode=WAL`, `PRAGMA journal_mode=WAL`,
`PRAGMA busy_timeout=5000`, `PRAGMA busy_timeout=5000`,
`CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)`, `CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
@@ -126,6 +139,11 @@ func openStore(path string) (*Store, error) {
`CREATE INDEX IF NOT EXISTS service_checks_service_ts_idx ON service_checks(service_id,ts)`, `CREATE INDEX IF NOT EXISTS service_checks_service_ts_idx ON service_checks(service_id,ts)`,
`CREATE TABLE IF NOT EXISTS entity_history (kind TEXT NOT NULL,name TEXT NOT NULL,ts INTEGER NOT NULL,value1 REAL,value2 REAL,PRIMARY KEY(kind,name,ts))`, `CREATE TABLE IF NOT EXISTS entity_history (kind TEXT NOT NULL,name TEXT NOT NULL,ts INTEGER NOT NULL,value1 REAL,value2 REAL,PRIMARY KEY(kind,name,ts))`,
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`, `CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
`CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`,
`CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`,
`CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`,
} }
for _, statement := range statements { for _, statement := range statements {
if _, err := db.Exec(statement); err != nil { if _, err := db.Exec(statement); err != nil {
@@ -135,6 +153,9 @@ func openStore(path string) (*Store, error) {
} }
// Older databases get this column without requiring a separate migration tool. // Older databases get this column without requiring a separate migration tool.
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`) _, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`)
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN planned INTEGER NOT NULL DEFAULT 0`)
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN maintenance_reason TEXT NOT NULL DEFAULT ''`)
_, _ = db.Exec(`DELETE FROM alert_events WHERE alert_key='monitor-slow'`)
_, _ = db.Exec(`ALTER TABLE service_checks ADD COLUMN endpoint TEXT NOT NULL DEFAULT 'public'`) _, _ = db.Exec(`ALTER TABLE service_checks ADD COLUMN endpoint TEXT NOT NULL DEFAULT 'public'`)
_, _ = db.Exec(`PRAGMA foreign_keys=ON`) _, _ = db.Exec(`PRAGMA foreign_keys=ON`)
return &Store{db: db, path: path}, nil return &Store{db: db, path: path}, nil
@@ -218,7 +239,7 @@ func (s *Store) SaveThresholds(value AlertThresholds) error {
} }
func defaultEmailSettings() EmailSettings { func defaultEmailSettings() EmailSettings {
return EmailSettings{Port: 587, IntervalMinutes: 60, Severities: []string{"critical", "warning"}, Sources: []string{"CPU", "Память", "Storage", "Диски", "ZFS", "UPS", "VM/LXC", "Proxmox", "Dashboard", "Сервисы", "Система", "Сеть"}} return EmailSettings{Port: 587, IntervalMinutes: 60, Severities: []string{"critical", "warning"}, Sources: []string{"CPU", "Память", "Storage", "Диски", "ZFS", "UPS", "VM/LXC", "Proxmox", "Dashboard", "Сервисы", "Система", "Сеть", "Агенты", "Docker"}}
} }
func (s *Store) EmailSettings(includePassword bool) EmailSettings { func (s *Store) EmailSettings(includePassword bool) EmailSettings {
@@ -285,6 +306,19 @@ func validateThresholds(t AlertThresholds) error {
if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 { if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 {
return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов") return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов")
} }
for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} {
if value < 1 || value > 100 {
return fmt.Errorf("пороги utilization должны быть от 1 до 100")
}
}
for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} {
if value < 0 {
return fmt.Errorf("пороги latency и очереди не могут быть отрицательными")
}
}
if t.DiskIODuration < 5 || t.DiskIODuration > 3600 {
return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд")
}
return nil return nil
} }

355
traffic.go Normal file
View File

@@ -0,0 +1,355 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"time"
)
type TrafficEvent struct {
City string `json:"city,omitempty"`
Region string `json:"region,omitempty"`
Monitoring bool `json:"monitoring"`
ID string `json:"id"`
Sequence int64 `json:"sequence,omitempty"`
Time int64 `json:"time"`
Kind string `json:"kind"`
IP string `json:"ip"`
Domain string `json:"domain,omitempty"`
Method string `json:"method,omitempty"`
Path string `json:"path,omitempty"`
Status int `json:"status,omitempty"`
Reason string `json:"reason,omitempty"`
Country string `json:"country,omitempty"`
Agent string `json:"agent,omitempty"`
}
type TrafficBatch struct {
Events []TrafficEvent `json:"events"`
Files int `json:"files"`
BacklogBytes int64 `json:"backlogBytes"`
CrowdSecLimited bool `json:"crowdsecLimited"`
ParseErrors uint64 `json:"parseErrors"`
LogError string `json:"logError,omitempty"`
CrowdSecError string `json:"crowdsecError,omitempty"`
CrowdSecAt int64 `json:"crowdsecAt"`
StartedAt int64 `json:"startedAt"`
}
type TrafficSource struct {
Agent string `json:"agent"`
Name string `json:"name"`
LastSeen int64 `json:"lastSeen"`
Status TrafficBatch `json:"status"`
}
type TrafficCount struct {
Name string `json:"name"`
Count int64 `json:"count"`
}
type TrafficCountryDetail struct {
Country string `json:"country"`
Requests int64 `json:"requests"`
UniqueIPs int64 `json:"uniqueIps"`
Alerts int64 `json:"alerts"`
Decisions int64 `json:"decisions"`
LastSeen int64 `json:"lastSeen"`
}
type TrafficView struct {
MaxSequence int64 `json:"maxSequence"`
CountryDetails []TrafficCountryDetail `json:"countryDetails"`
Network *NetworkMapView `json:"network,omitempty"`
Events []TrafficEvent `json:"events"`
Sources []TrafficSource `json:"sources"`
Countries []TrafficCount `json:"countries"`
Domains []TrafficCount `json:"domains"`
IPs []TrafficCount `json:"ips"`
Requests int64 `json:"requests"`
UniqueIPs int64 `json:"uniqueIps"`
Alerts int64 `json:"alerts"`
Decisions int64 `json:"decisions"`
QPS float64 `json:"qps"`
Series []TrafficCount `json:"series"`
GeoReady bool `json:"geoReady"`
GeoDate string `json:"geoDate"`
Now int64 `json:"now"`
Settings TrafficSettings `json:"settings"`
Limited bool `json:"limited"`
}
const trafficSchema = `CREATE TABLE IF NOT EXISTS traffic_events (
seq INTEGER PRIMARY KEY AUTOINCREMENT, agent TEXT NOT NULL, event_id TEXT NOT NULL,
ts INTEGER NOT NULL, kind TEXT NOT NULL, ip TEXT NOT NULL, domain TEXT NOT NULL,
method TEXT NOT NULL, path TEXT NOT NULL, status INTEGER NOT NULL, reason TEXT NOT NULL,
country TEXT NOT NULL, UNIQUE(agent,event_id));
CREATE INDEX IF NOT EXISTS traffic_ts ON traffic_events(ts);
CREATE INDEX IF NOT EXISTS traffic_filter ON traffic_events(domain,kind,ts);
CREATE TABLE IF NOT EXISTS network_sources(agent TEXT PRIMARY KEY, snapshot TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS traffic_sources(agent TEXT PRIMARY KEY, last_seen INTEGER NOT NULL, status_json TEXT NOT NULL);`
func (s *Store) SaveTraffic(agent string, batch TrafficBatch) error {
if len(batch.Events) > 500 {
return errors.New("too many traffic events")
}
now := time.Now().Unix()
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
stmt, err := tx.Prepare(`INSERT OR IGNORE INTO traffic_events(agent,event_id,ts,kind,ip,domain,method,path,status,reason,country) VALUES(?,?,?,?,?,?,?,?,?,?,?)`)
if err != nil {
return err
}
defer stmt.Close()
for _, e := range batch.Events {
if e.Kind != "request" && e.Kind != "alert" && e.Kind != "decision" {
return errors.New("invalid traffic kind")
}
if net.ParseIP(e.IP) == nil || len(e.ID) == 0 || len(e.ID) > 160 {
return errors.New("invalid traffic event")
}
if e.Time < now-86400 || e.Time > now+300 {
continue
}
e.IP = net.ParseIP(e.IP).String()
country := lookupTrafficCountry(e.IP)
_, err = stmt.Exec(agent, e.ID, e.Time, e.Kind, e.IP, trafficText(e.Domain, 253), trafficText(e.Method, 16), trafficPath(e.Path), e.Status, trafficText(e.Reason, 256), country)
if err != nil {
return err
}
}
batch.Events = nil
raw, err := json.Marshal(batch)
if err != nil {
return err
}
if _, err = tx.Exec(`INSERT INTO traffic_sources(agent,last_seen,status_json) VALUES(?,?,?) ON CONFLICT(agent) DO UPDATE SET last_seen=excluded.last_seen,status_json=excluded.status_json`, agent, now, string(raw)); err != nil {
return err
}
// Bound both age and total rows, including a high-traffic installation.
if _, err = tx.Exec(`DELETE FROM traffic_events WHERE ts<? OR seq <= COALESCE((SELECT seq FROM traffic_events ORDER BY seq DESC LIMIT 1 OFFSET 200000),0)`, now-86400); err != nil {
return err
}
return tx.Commit()
}
func trafficText(s string, max int) string {
s = strings.Map(func(r rune) rune {
if r < 32 || r == 127 {
return -1
}
return r
}, s)
r := []rune(s)
if len(r) > max {
return string(r[:max])
}
return s
}
func trafficPath(s string) string {
s = strings.SplitN(s, "?", 2)[0]
s = strings.SplitN(s, "#", 2)[0]
return trafficText(s, 256)
}
func (s *Store) TrafficView(domain, kind string, external bool, hideMonitoring ...bool) (TrafficView, error) {
now := time.Now().Unix()
v := TrafficView{Events: []TrafficEvent{}, Sources: []TrafficSource{}, Countries: []TrafficCount{}, Domains: []TrafficCount{}, IPs: []TrafficCount{}, Series: []TrafficCount{}, Now: now, GeoReady: trafficGeoReady(), GeoDate: trafficGeoDate}
v.Settings = s.TrafficSettings()
hide := len(hideMonitoring) > 0 && hideMonitoring[0]
if v.Settings.NetworkEnabled {
snapshot, err := s.ReadNetworkSnapshot(v.Settings.AgentID, v.Settings.MonitoringRules, hide)
if err != nil {
snapshot.Error = "Не удалось прочитать сетевой снимок"
}
network := makeNetworkMap(snapshot, external)
v.Network = &network
}
if err := s.db.QueryRow(`SELECT COALESCE(MAX(seq),0) FROM traffic_events WHERE agent=?`, v.Settings.AgentID).Scan(&v.MaxSequence); err != nil {
return v, err
}
where := ` WHERE ts>=? AND agent=? AND ts<=? AND seq<=?`
args := []any{now - 86400, v.Settings.AgentID, now, v.MaxSequence}
if hide {
clause, params := monitoringSQL(v.Settings.MonitoringRules)
where += clause
args = append(args, params...)
}
if external {
where += ` AND country!='LAN'`
}
if domain != "" {
where += ` AND domain=?`
args = append(args, domain)
}
if kind != "" {
where += ` AND kind=?`
args = append(args, kind)
}
err := s.db.QueryRow(`SELECT COALESCE(SUM(kind='request'),0),COUNT(DISTINCT CASE WHEN kind='request' THEN ip END),COALESCE(SUM(kind='alert'),0),COALESCE(SUM(kind='decision'),0),COALESCE(SUM(kind='request' AND ts>=?),0)/60.0 FROM traffic_events`+where, append([]any{now - 60}, args...)...).Scan(&v.Requests, &v.UniqueIPs, &v.Alerts, &v.Decisions, &v.QPS)
if err != nil {
return v, err
}
rows, err := s.db.Query(`SELECT seq,event_id,ts,kind,ip,domain,method,path,status,reason,country,agent FROM traffic_events`+where+` ORDER BY ts DESC,seq DESC LIMIT 100`, args...)
if err != nil {
return v, err
}
for rows.Next() {
var e TrafficEvent
if err = rows.Scan(&e.Sequence, &e.ID, &e.Time, &e.Kind, &e.IP, &e.Domain, &e.Method, &e.Path, &e.Status, &e.Reason, &e.Country, &e.Agent); err != nil {
rows.Close()
return v, err
}
e.Monitoring = isMonitoringRequest(e, v.Settings.MonitoringRules)
e.City, e.Region = lookupTrafficCity(e.IP)
v.Events = append(v.Events, e)
}
err = rows.Err()
rows.Close()
if err != nil {
return v, err
}
for _, q := range []struct {
column string
out *[]TrafficCount
}{{"country", &v.Countries}, {"domain", &v.Domains}, {"ip", &v.IPs}} {
r, e := s.db.Query(`SELECT `+q.column+`,COUNT(*) FROM traffic_events`+where+` GROUP BY `+q.column+` ORDER BY COUNT(*) DESC LIMIT 50`, args...)
if e != nil {
return v, e
}
for r.Next() {
var c TrafficCount
if e = r.Scan(&c.Name, &c.Count); e != nil {
r.Close()
return v, e
}
*q.out = append(*q.out, c)
}
e = r.Err()
r.Close()
if e != nil {
return v, e
}
}
detailRows, detailErr := s.db.Query(`SELECT country,SUM(kind='request'),COUNT(DISTINCT CASE WHEN kind='request' THEN ip END),SUM(kind='alert'),SUM(kind='decision'),MAX(ts) FROM traffic_events`+where+` GROUP BY country`, args...)
if detailErr != nil {
return v, detailErr
}
v.CountryDetails = []TrafficCountryDetail{}
for detailRows.Next() {
var d TrafficCountryDetail
if err = detailRows.Scan(&d.Country, &d.Requests, &d.UniqueIPs, &d.Alerts, &d.Decisions, &d.LastSeen); err != nil {
detailRows.Close()
return v, err
}
v.CountryDetails = append(v.CountryDetails, d)
}
detailErr = detailRows.Err()
detailRows.Close()
if detailErr != nil {
return v, detailErr
}
r, err := s.db.Query(`SELECT CAST((ts/60)*60 AS TEXT),COUNT(*) FROM traffic_events`+where+` AND kind='request' AND ts>=? GROUP BY ts/60 ORDER BY ts/60`, append(args, now-3600)...)
if err != nil {
return v, err
}
for r.Next() {
var c TrafficCount
if err = r.Scan(&c.Name, &c.Count); err != nil {
r.Close()
return v, err
}
v.Series = append(v.Series, c)
}
err = r.Err()
r.Close()
if err != nil {
return v, err
}
r, err = s.db.Query(`SELECT t.agent,COALESCE(a.name,t.agent),t.last_seen,t.status_json FROM traffic_sources t LEFT JOIN agents a ON a.id=t.agent WHERE t.agent=? ORDER BY t.last_seen DESC`, v.Settings.AgentID)
if err != nil {
return v, err
}
for r.Next() {
var src TrafficSource
var raw string
if err = r.Scan(&src.Agent, &src.Name, &src.LastSeen, &raw); err != nil {
r.Close()
return v, err
}
json.Unmarshal([]byte(raw), &src.Status)
v.Sources = append(v.Sources, src)
}
err = r.Err()
r.Close()
var count int64
s.db.QueryRow(`SELECT COUNT(*) FROM traffic_events`).Scan(&count)
v.Limited = count >= 200000
return v, err
}
func trafficRoutes(mux *http.ServeMux, s *Store) {
trafficSettingsRoute(mux, s)
s.trafficHistoryRoute(mux)
trafficCityRoutes(mux)
networkRoutes(mux, s)
mux.HandleFunc("/traffic", func(w http.ResponseWriter, r *http.Request) {
if !s.TrafficSettings().Enabled {
http.Error(w, "Карта трафика отключена. Включите её в настройках Dashboard.", 404)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(trafficHTML))
})
mux.HandleFunc("/api/traffic", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(405)
return
}
if !s.TrafficSettings().Enabled {
http.Error(w, "Карта отключена", 404)
return
}
view, err := s.TrafficView(r.URL.Query().Get("domain"), r.URL.Query().Get("kind"), r.URL.Query().Get("external") == "1", r.URL.Query().Get("hideMonitoring") == "1")
writeJSON(w, view, err)
})
mux.HandleFunc("/api/traffic/world", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=86400")
w.Write(trafficWorld)
})
mux.HandleFunc("/api/agent/traffic", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
w.WriteHeader(405)
return
}
id, secret, ok := agentCredentials(r)
if !ok || !s.AuthenticateAgent(id, secret) {
http.Error(w, "Unauthorized", 401)
return
}
settings := s.TrafficSettings()
if !settings.Enabled || settings.AgentID != id {
http.Error(w, "Источник отключён", 403)
return
}
var batch TrafficBatch
if json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&batch) != nil {
http.Error(w, "Invalid traffic batch", 400)
return
}
if err := s.SaveTraffic(id, batch); err != nil {
http.Error(w, fmt.Sprint(err), 500)
return
}
writeJSON(w, map[string]bool{"accepted": true}, nil)
})
}

166
traffic_city.go Normal file
View File

@@ -0,0 +1,166 @@
package main
import (
"compress/gzip"
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/oschwald/maxminddb-golang"
)
const cityDownloadURL = "https://download.db-ip.com/free/dbip-city-lite-2026-10.mmdb.gz"
const cityDatabaseSHA1 = "df17bd24390108ed1d24dcb7718dc60f6b9abd4f"
var trafficCity struct {
sync.Mutex
once sync.Once
db *maxminddb.Reader
running bool
message string
}
func trafficCityPath() string {
db := os.Getenv("DASHBOARD_DB_PATH")
if db == "" {
db = defaultDatabasePath
}
return filepath.Join(filepath.Dir(db), "dbip-city.mmdb")
}
func trafficCityReady() bool {
trafficCity.once.Do(func() {
trafficCity.Lock()
defer trafficCity.Unlock()
trafficCity.db, _ = maxminddb.Open(trafficCityPath())
if trafficCity.db != nil && !strings.Contains(strings.ToLower(trafficCity.db.Metadata.DatabaseType), "city") {
trafficCity.db.Close()
trafficCity.db = nil
}
})
trafficCity.Lock()
defer trafficCity.Unlock()
return trafficCity.db != nil
}
func lookupTrafficCity(text string) (string, string) {
ip := net.ParseIP(text)
if ip == nil || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || !ip.IsGlobalUnicast() || !trafficCityReady() {
return "", ""
}
trafficCity.Lock()
defer trafficCity.Unlock()
var r struct {
City struct {
Names map[string]string `maxminddb:"names"`
} `maxminddb:"city"`
Subdivisions []struct {
Names map[string]string `maxminddb:"names"`
} `maxminddb:"subdivisions"`
}
if trafficCity.db.Lookup(ip, &r) != nil {
return "", ""
}
region := ""
if len(r.Subdivisions) > 0 {
region = cityName(r.Subdivisions[0].Names)
}
return cityName(r.City.Names), region
}
func trafficCityRoutes(mux *http.ServeMux) {
mux.HandleFunc("/api/settings/traffic/city-database", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" && r.Method != "POST" {
w.WriteHeader(405)
return
}
trafficCityReady()
trafficCity.Lock()
if r.Method == "POST" && !trafficCity.running && trafficCity.db == nil {
trafficCity.running = true
trafficCity.message = "Загрузка базы городов (около 58 МБ)…"
go func() {
err := installTrafficCity()
trafficCity.Lock()
defer trafficCity.Unlock()
trafficCity.running = false
if err != nil {
trafficCity.message = "Не удалось установить базу: " + err.Error()
} else {
trafficCity.message = "База городов DB-IP City Lite 2026-10 установлена"
}
}()
}
status := map[string]any{"ready": trafficCity.db != nil, "running": trafficCity.running, "message": trafficCity.message, "date": "2026-10"}
trafficCity.Unlock()
writeJSON(w, status, nil)
})
}
func installTrafficCity() error { return installTrafficCityFrom(cityDownloadURL, cityDatabaseSHA1) }
func installTrafficCityFrom(source, expectedHash string) error {
client := &http.Client{Timeout: 15 * time.Minute}
response, err := client.Get(source)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != 200 {
return fmt.Errorf("HTTP %d", response.StatusCode)
}
reader, err := gzip.NewReader(io.LimitReader(response.Body, 70<<20))
if err != nil {
return err
}
defer reader.Close()
path := trafficCityPath()
if err = os.MkdirAll(filepath.Dir(path), 0750); err != nil {
return err
}
file, err := os.CreateTemp(filepath.Dir(path), ".city-*.mmdb")
if err != nil {
return err
}
defer os.Remove(file.Name())
defer file.Close()
hash := sha1.New()
n, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(reader, (160<<20)+1))
if err != nil {
return err
}
if n > 160<<20 {
return fmt.Errorf("база превышает лимит размера")
}
if hex.EncodeToString(hash.Sum(nil)) != expectedHash {
return fmt.Errorf("контрольная сумма базы не совпала")
}
if err = file.Sync(); err != nil {
return err
}
if err = file.Close(); err != nil {
return err
}
db, err := maxminddb.Open(file.Name())
if err != nil {
return err
}
if !strings.Contains(strings.ToLower(db.Metadata.DatabaseType), "city") {
db.Close()
return fmt.Errorf("неверный тип GeoIP-базы")
}
if err = os.Rename(file.Name(), path); err != nil {
db.Close()
return err
}
trafficCity.Lock()
defer trafficCity.Unlock()
if trafficCity.db != nil {
trafficCity.db.Close()
}
trafficCity.db = db
return nil
}

54
traffic_city_test.go Normal file
View File

@@ -0,0 +1,54 @@
package main
import (
"compress/gzip"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
func TestCityInstallRejectsBadDownload(t *testing.T) {
dir := t.TempDir()
t.Setenv("DASHBOARD_DB_PATH", filepath.Join(dir, "dashboard.db"))
existing := []byte("existing database")
if err := os.WriteFile(trafficCityPath(), existing, 0600); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
g := gzip.NewWriter(w)
g.Write([]byte("corrupt database"))
g.Close()
}))
defer server.Close()
if err := installTrafficCityFrom(server.URL, cityDatabaseSHA1); err == nil {
t.Fatal("bad checksum accepted")
}
got, err := os.ReadFile(trafficCityPath())
if err != nil || string(got) != string(existing) {
t.Fatal("existing database replaced on failure")
}
files, _ := filepath.Glob(filepath.Join(dir, ".city-*"))
if len(files) != 0 {
t.Fatal("partial file left behind")
}
}
func TestCityDatabaseDownloaded(t *testing.T) {
source := os.Getenv("DASHBOARD_CITY_TEST_GZIP")
if source == "" {
t.Skip("optional official database integration test")
}
t.Setenv("DASHBOARD_DB_PATH", filepath.Join(t.TempDir(), "dashboard.db"))
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, source) }))
defer server.Close()
trafficCityReady()
if err := installTrafficCityFrom(server.URL, cityDatabaseSHA1); err != nil {
t.Fatal(err)
}
city, region := lookupTrafficCity("20.194.96.114")
t.Logf("city=%q region=%q", city, region)
if city == "" {
t.Fatal("public IP city not decoded")
}
}

44
traffic_country_test.go Normal file
View File

@@ -0,0 +1,44 @@
package main
import (
"fmt"
"testing"
"time"
)
func TestTrafficCountryDetailsUseHistoryAndFilters(t *testing.T) {
s, a := trafficFixture(t)
if _, err := s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, MonitoringRules: []MonitoringRule{{IP: "8.8.8.8"}}}); err != nil {
t.Fatal(err)
}
events := []TrafficEvent{}
for i := 0; i < 120; i++ {
events = append(events, TrafficEvent{ID: fmt.Sprint("http-", i), Time: time.Now().Unix(), Kind: "request", IP: "8.8.8.8", Domain: "example.test", Path: "/", Status: 200})
}
events = append(events, TrafficEvent{ID: "alert", Time: time.Now().Unix(), Kind: "alert", IP: "8.8.8.8", Domain: "example.test", Reason: "test"})
if err := s.SaveTraffic(a.ID, TrafficBatch{Events: events}); err != nil {
t.Fatal(err)
}
v, err := s.TrafficView("", "", false)
if err != nil {
t.Fatal(err)
}
if len(v.Events) != 100 || len(v.CountryDetails) != 1 {
t.Fatalf("events=%d countries=%+v", len(v.Events), v.CountryDetails)
}
d := v.CountryDetails[0]
if d.Requests != 120 || d.UniqueIPs != 1 || d.Alerts != 1 || d.LastSeen == 0 {
t.Fatalf("%+v", d)
}
v, err = s.TrafficView("example.test", "", false, true)
if err != nil {
t.Fatal(err)
}
if len(v.CountryDetails) != 1 || v.CountryDetails[0].Requests != 0 || v.CountryDetails[0].Alerts != 1 {
t.Fatalf("filter: %+v", v.CountryDetails)
}
v, err = s.TrafficView("other.test", "", false)
if err != nil || len(v.CountryDetails) != 0 {
t.Fatalf("domain: %+v %v", v.CountryDetails, err)
}
}

65
traffic_geo.go Normal file
View File

@@ -0,0 +1,65 @@
package main
import (
"bytes"
"compress/gzip"
_ "embed"
"io"
"net"
"sync"
"github.com/oschwald/maxminddb-golang"
)
//go:embed assets/world.json
var trafficWorld []byte
//go:embed assets/dbip-country-lite-2026-10.mmdb.gz
var trafficGeoCompressed []byte
const trafficGeoDate = "2026-10"
var trafficGeo struct {
sync.Once
db *maxminddb.Reader
}
func trafficGeoReady() bool {
trafficGeo.Do(func() {
r, err := gzip.NewReader(bytes.NewReader(trafficGeoCompressed))
if err != nil {
return
}
defer r.Close()
data, err := io.ReadAll(io.LimitReader(r, 32<<20))
if err != nil {
return
}
trafficGeo.db, _ = maxminddb.FromBytes(data)
})
return trafficGeo.db != nil
}
func lookupTrafficCountry(ipText string) string {
ip := net.ParseIP(ipText)
if ip == nil {
return ""
}
if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
return "LAN"
}
if !ip.IsGlobalUnicast() {
return ""
}
if !trafficGeoReady() {
return ""
}
var record struct {
Country struct {
Code string `maxminddb:"iso_code"`
} `maxminddb:"country"`
}
if trafficGeo.db.Lookup(ip, &record) != nil {
return ""
}
return record.Country.Code
}

144
traffic_history.go Normal file
View File

@@ -0,0 +1,144 @@
package main
import (
"net/http"
"strconv"
"strings"
"time"
)
type TrafficIPDetail struct {
IP string `json:"ip"`
Country string `json:"country"`
Requests int64 `json:"requests"`
LastSeen int64 `json:"lastSeen"`
City string `json:"city,omitempty"`
Region string `json:"region,omitempty"`
}
type TrafficHistory struct {
Events []TrafficEvent `json:"events"`
IPs []TrafficIPDetail `json:"ips"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
Snapshot int64 `json:"snapshot"`
MaxSequence int64 `json:"maxSequence"`
CityReady bool `json:"cityReady"`
}
func (s *Store) trafficHistoryRoute(mux *http.ServeMux) {
mux.HandleFunc("/api/traffic/history", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(405)
return
}
if !s.TrafficSettings().Enabled {
http.Error(w, "Карта отключена", 404)
return
}
q := r.URL.Query()
page, _ := strconv.Atoi(q.Get("page"))
snapshot, _ := strconv.ParseInt(q.Get("snapshot"), 10, 64)
maxSeq, _ := strconv.ParseInt(q.Get("maxSequence"), 10, 64)
if q.Get("maxSequence") == "0" {
maxSeq = -1
}
if page < 0 || page > 10000 {
http.Error(w, "Invalid page", 400)
return
}
kind := q.Get("kind")
metricType := q.Get("type")
if metricType != "" && metricType != "request" && metricType != "alert" && metricType != "decision" {
http.Error(w, "Invalid type", 400)
return
}
if kind != "" && kind != "request" && kind != "alert" && kind != "decision" {
http.Error(w, "Invalid kind", 400)
return
}
result, err := s.TrafficHistory(q.Get("domain"), kind, q.Get("country"), q.Get("ip"), q.Get("type"), q.Get("external") == "1", q.Get("hideMonitoring") == "1", q.Get("group") == "ip", page, snapshot, maxSeq)
writeJSON(w, result, err)
})
}
func (s *Store) TrafficHistory(domain, kind, country, ip, metricType string, external, hide, group bool, page int, snapshot, maxSeq int64) (TrafficHistory, error) {
now := time.Now().Unix()
if snapshot <= 0 || snapshot > now {
snapshot = now
}
if snapshot < now-86400 {
snapshot = now
}
settings := s.TrafficSettings()
v := TrafficHistory{Events: []TrafficEvent{}, IPs: []TrafficIPDetail{}, Page: page, PageSize: 50, Snapshot: snapshot, CityReady: trafficCityReady()}
if maxSeq == 0 {
if err := s.db.QueryRow(`SELECT COALESCE(MAX(seq),0) FROM traffic_events WHERE agent=?`, settings.AgentID).Scan(&maxSeq); err != nil {
return v, err
}
}
v.MaxSequence = maxSeq
where := ` WHERE agent=? AND ts>=? AND ts<=? AND seq<=?`
args := []any{settings.AgentID, snapshot - 86400, snapshot, maxSeq}
if hide {
clause, values := monitoringSQL(settings.MonitoringRules)
where += clause
args = append(args, values...)
}
if external {
where += ` AND country!='LAN'`
}
for _, filter := range []struct{ column, value string }{{"domain", domain}, {"kind", kind}, {"kind", metricType}, {"country", country}, {"ip", ip}} {
if filter.value != "" {
where += " AND " + filter.column + "=?"
args = append(args, filter.value)
}
}
count := "COUNT(*)"
if group {
where += ` AND kind='request'`
count = "COUNT(DISTINCT ip)"
}
if err := s.db.QueryRow("SELECT "+count+" FROM traffic_events"+where, args...).Scan(&v.Total); err != nil {
return v, err
}
paged := append(append([]any{}, args...), v.PageSize, page*v.PageSize)
if group {
rows, err := s.db.Query(`SELECT ip,MIN(country),COUNT(*),MAX(ts) FROM traffic_events`+where+` GROUP BY ip ORDER BY COUNT(*) DESC,ip LIMIT ? OFFSET ?`, paged...)
if err != nil {
return v, err
}
defer rows.Close()
for rows.Next() {
var item TrafficIPDetail
if err = rows.Scan(&item.IP, &item.Country, &item.Requests, &item.LastSeen); err != nil {
return v, err
}
item.City, item.Region = lookupTrafficCity(item.IP)
v.IPs = append(v.IPs, item)
}
return v, rows.Err()
}
rows, err := s.db.Query(`SELECT seq,event_id,ts,kind,ip,domain,method,path,status,reason,country,agent FROM traffic_events`+where+` ORDER BY ts DESC,seq DESC LIMIT ? OFFSET ?`, paged...)
if err != nil {
return v, err
}
defer rows.Close()
for rows.Next() {
var e TrafficEvent
if err = rows.Scan(&e.Sequence, &e.ID, &e.Time, &e.Kind, &e.IP, &e.Domain, &e.Method, &e.Path, &e.Status, &e.Reason, &e.Country, &e.Agent); err != nil {
return v, err
}
e.Monitoring = isMonitoringRequest(e, settings.MonitoringRules)
e.City, e.Region = lookupTrafficCity(e.IP)
v.Events = append(v.Events, e)
}
return v, rows.Err()
}
func cityName(names map[string]string) string {
for _, lang := range []string{"ru", "en"} {
if s := strings.TrimSpace(names[lang]); s != "" {
return s
}
}
return ""
}

109
traffic_history_test.go Normal file
View File

@@ -0,0 +1,109 @@
package main
import (
"fmt"
"testing"
"time"
)
func TestTrafficHistoryFullPaginationAndSnapshot(t *testing.T) {
s, a := trafficFixture(t)
if _, err := s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID}); err != nil {
t.Fatal(err)
}
now := time.Now().Unix()
batch := TrafficBatch{}
for i := 0; i < 137; i++ {
batch.Events = append(batch.Events, TrafficEvent{ID: fmt.Sprint(i), Time: now - 60, Kind: "request", IP: "8.8.8.8", Domain: "example.test"})
}
batch.Events = append(batch.Events, TrafficEvent{ID: "old-alert", Time: now - 3600, Kind: "alert", IP: "1.1.1.1", Domain: "other.test"})
if err := s.SaveTraffic(a.ID, batch); err != nil {
t.Fatal(err)
}
first, err := s.TrafficHistory("", "request", "", "", "", false, false, false, 0, now, 0)
if err != nil || first.Total != 137 || len(first.Events) != 50 {
t.Fatalf("first: %+v %v", first, err)
}
if err := s.SaveTraffic(a.ID, TrafficBatch{Events: []TrafficEvent{{ID: "new", Time: now - 60, Kind: "request", IP: "8.8.8.8"}}}); err != nil {
t.Fatal(err)
}
seen := map[int64]bool{}
for page := 0; page < 3; page++ {
h, err := s.TrafficHistory("", "request", "", "", "", false, false, false, page, first.Snapshot, first.MaxSequence)
if err != nil || h.Total != 137 {
t.Fatalf("page %d: %v total %d", page, err, h.Total)
}
for _, e := range h.Events {
if seen[e.Sequence] {
t.Fatal("duplicate page event")
}
seen[e.Sequence] = true
}
}
if len(seen) != 137 {
t.Fatalf("missing history: %d", len(seen))
}
h, err := s.TrafficHistory("other.test", "alert", "", "", "", false, false, false, 0, now, 0)
if err != nil || h.Total != 1 || h.Events[0].ID != "old-alert" {
t.Fatalf("old alert missing: %+v %v", h, err)
}
h, err = s.TrafficHistory("", "alert", "", "", "request", false, false, false, 0, now, 0)
if err != nil || h.Total != 0 {
t.Fatal("metric must respect global kind filter")
}
h, err = s.TrafficHistory("example.test", "", "", "", "", false, false, true, 0, now, 0)
if err != nil || h.Total != 1 || len(h.IPs) != 1 || h.IPs[0].Requests != 137 {
t.Fatalf("IP summary %+v %v", h, err)
}
h, err = s.TrafficHistory("", "", "ZZ", "", "", false, false, false, 0, now, 0)
if err != nil || h.Total != 0 {
t.Fatal("country filter ignored")
}
h, err = s.TrafficHistory("", "", "", "1.1.1.1", "", false, false, false, 0, now, 0)
if err != nil || h.Total != 1 {
t.Fatal("IP filter ignored")
}
}
func TestCityNamesAndLocalAddresses(t *testing.T) {
if cityName(map[string]string{"ru": "Москва", "en": "Moscow"}) != "Москва" || cityName(map[string]string{"en": "Amsterdam"}) != "Amsterdam" {
t.Fatal("city localization")
}
for _, ip := range []string{"192.168.0.10", "127.0.0.1", "::1", "fe80::1", "invalid"} {
c, r := lookupTrafficCity(ip)
if c != "" || r != "" {
t.Fatal("local IP geolocated")
}
}
}
func TestTrafficHistoryMonitoringAndAgentIsolation(t *testing.T) {
s, a := trafficFixture(t)
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, MonitoringRules: []MonitoringRule{{IP: "192.168.0.10"}}})
now := time.Now().Unix()
batch := TrafficBatch{Events: []TrafficEvent{{ID: "check", Time: now, Kind: "request", IP: "192.168.0.10"}, {ID: "alert", Time: now, Kind: "alert", IP: "192.168.0.10"}, {ID: "public", Time: now, Kind: "request", IP: "8.8.8.8"}}}
if err := s.SaveTraffic(a.ID, batch); err != nil {
t.Fatal(err)
}
token, _ := s.CreateAgentEnrollmentToken()
other, err := s.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "other", VMID: 103})
if err != nil {
t.Fatal(err)
}
if err = s.SaveTraffic(other.ID, batch); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
external, hide bool
want int64
}{{false, false, 3}, {false, true, 2}, {true, false, 1}, {true, true, 1}} {
h, err := s.TrafficHistory("", "", "", "", "", tc.external, tc.hide, false, 0, now, 0)
if err != nil || h.Total != tc.want {
t.Fatalf("filters %+v total=%d err=%v", tc, h.Total, err)
}
for _, e := range h.Events {
if e.Agent != a.ID {
t.Fatal("foreign agent leaked")
}
}
}
}

76
traffic_monitoring.go Normal file
View File

@@ -0,0 +1,76 @@
package main
import (
"fmt"
"net"
"strings"
)
// Domain/path restrictions describe HTTP probes only. An IP-only rule also
// matches conntrack initiators; a destination match never hides other clients.
type MonitoringRule struct {
IP string `json:"ip"`
Domain string `json:"domain,omitempty"`
PathPrefix string `json:"pathPrefix,omitempty"`
}
func validateMonitoringRules(rules []MonitoringRule) ([]MonitoringRule, error) {
if len(rules) > 32 {
return nil, fmt.Errorf("Не более 32 правил мониторинга")
}
result := make([]MonitoringRule, 0, len(rules))
for i, r := range rules {
ip := net.ParseIP(strings.TrimSpace(r.IP))
if ip == nil {
return nil, fmt.Errorf("Правило %d: нужен точный IPv4/IPv6-адрес", i+1)
}
r.IP = ip.String()
r.Domain = strings.ToLower(strings.TrimSpace(r.Domain))
r.PathPrefix = strings.TrimSpace(r.PathPrefix)
if len(r.Domain) > 253 || strings.ContainsAny(r.Domain, " /|\t\r\n") || len(r.PathPrefix) > 256 || strings.ContainsAny(r.PathPrefix, "|\r\n") || (r.PathPrefix != "" && !strings.HasPrefix(r.PathPrefix, "/")) {
return nil, fmt.Errorf("Правило %d: проверьте домен и префикс пути (начинается с /)", i+1)
}
result = append(result, r)
}
return result, nil
}
func isMonitoringRequest(e TrafficEvent, rules []MonitoringRule) bool {
if e.Kind != "request" {
return false
}
for _, r := range rules {
if e.IP == r.IP && (r.Domain == "" || strings.EqualFold(e.Domain, r.Domain)) && strings.HasPrefix(e.Path, r.PathPrefix) {
return true
}
}
return false
}
func isMonitoringFlow(f NetworkFlow, rules []MonitoringRule) bool {
for _, r := range rules {
if net.ParseIP(f.Source).Equal(net.ParseIP(r.IP)) && r.Domain == "" && r.PathPrefix == "" {
return true
}
}
return false
}
func monitoringSQL(rules []MonitoringRule) (string, []any) {
var parts []string
var args []any
for _, r := range rules {
p := "(ip=?"
args = append(args, r.IP)
if r.Domain != "" {
p += " AND lower(domain)=?"
args = append(args, r.Domain)
}
if r.PathPrefix != "" {
p += " AND substr(path,1,length(?))=?"
args = append(args, r.PathPrefix, r.PathPrefix)
}
parts = append(parts, p+")")
}
if len(parts) == 0 {
return "", nil
}
return " AND NOT (kind='request' AND (" + strings.Join(parts, " OR ") + "))", args
}

140
traffic_monitoring_test.go Normal file
View File

@@ -0,0 +1,140 @@
package main
import (
"encoding/json"
"testing"
"time"
)
func TestMonitoringRulesValidation(t *testing.T) {
rules, err := validateMonitoringRules([]MonitoringRule{{IP: "2001:db8:0:0::1", Domain: "EXAMPLE.org", PathPrefix: "/health"}})
if err != nil || rules[0].IP != "2001:db8::1" || rules[0].Domain != "example.org" {
t.Fatalf("normalization: %+v %v", rules, err)
}
for _, r := range []MonitoringRule{{IP: "192.168.0.0/24"}, {IP: "8.8.8.8", Domain: "https://example.org"}, {IP: "8.8.8.8", PathPrefix: "health"}} {
if _, err := validateMonitoringRules([]MonitoringRule{r}); err == nil {
t.Fatalf("bad rule accepted: %+v", r)
}
}
if _, err := validateMonitoringRules(make([]MonitoringRule, 33)); err == nil {
t.Fatal("unbounded rules")
}
}
func TestMonitoringFiltersKeepHistoryAndSecurityEvents(t *testing.T) {
s, a := trafficFixture(t)
rules := []MonitoringRule{{IP: "192.168.0.10"}, {IP: "8.8.8.8", Domain: "example.org", PathPrefix: "/health"}}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, MonitoringRules: rules})
now := time.Now().Unix()
batch := TrafficBatch{Events: []TrafficEvent{
{ID: "probe", Time: now, Kind: "request", IP: "192.168.0.10", Domain: "other.org", Path: "/"},
{ID: "health", Time: now, Kind: "request", IP: "8.8.8.8", Domain: "EXAMPLE.org", Path: "/health/live"},
{ID: "visitor", Time: now, Kind: "request", IP: "8.8.8.8", Domain: "example.org", Path: "/login"},
{ID: "other-domain", Time: now, Kind: "request", IP: "8.8.8.8", Domain: "other.org", Path: "/health"},
{ID: "alert", Time: now, Kind: "alert", IP: "192.168.0.10"},
{ID: "ban", Time: now, Kind: "decision", IP: "192.168.0.10"},
}}
if err := s.SaveTraffic(a.ID, batch); err != nil {
t.Fatal(err)
}
all, err := s.TrafficView("", "", false)
if err != nil || all.Requests != 4 {
t.Fatalf("all: %+v %v", all, err)
}
tagged := 0
for _, e := range all.Events {
if e.Monitoring {
tagged++
}
}
if tagged != 2 {
t.Fatalf("tags %d", tagged)
}
filtered, err := s.TrafficView("", "", false, true)
if err != nil || filtered.Requests != 2 || filtered.Alerts != 1 || filtered.Decisions != 1 || len(filtered.Events) != 4 {
t.Fatalf("filter: %+v %v", filtered, err)
}
if len(filtered.Series) != 1 || filtered.Series[0].Count != 2 {
t.Fatalf("graph leaks probes: %+v", filtered.Series)
}
var count int
s.db.QueryRow("SELECT COUNT(*) FROM traffic_events").Scan(&count)
if count != 6 {
t.Fatal("history deleted")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, MonitoringRules: []MonitoringRule{}})
filtered, err = s.TrafficView("", "", false, true)
if err != nil || filtered.Requests != 4 {
t.Fatal("cleared rules not applied retroactively")
}
}
func TestMonitoringPrefixIsLiteralAndDefaultCanBeCleared(t *testing.T) {
s, a := trafficFixture(t)
if len(s.TrafficSettings().MonitoringRules) != 1 {
t.Fatal("missing suggested rule")
}
rules := []MonitoringRule{{IP: "8.8.8.8", PathPrefix: "/health_%"}}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, MonitoringRules: rules})
s.SaveTraffic(a.ID, TrafficBatch{Events: []TrafficEvent{{ID: "x", Time: time.Now().Unix(), Kind: "request", IP: "8.8.8.8", Path: "/health_ok"}}})
v, err := s.TrafficView("", "", false, true)
if err != nil || v.Requests != 1 {
t.Fatal("prefix used as wildcard")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID})
if len(s.TrafficSettings().MonitoringRules) != 0 {
t.Fatal("cannot clear default")
}
}
func TestNetworkMapAndMonitoringSourceOnly(t *testing.T) {
s, a := trafficFixture(t)
rules := []MonitoringRule{{IP: "192.168.0.10"}, {IP: "8.8.8.8", Domain: "example.org"}}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, NetworkEnabled: true, AgentID: a.ID, MonitoringRules: rules})
incoming, _ := parseNetworkFlow(networkNATLine)
probe := incoming
probe.Source = "192.168.0.10"
outgoing := incoming
outgoing.Source = "192.168.0.12"
outgoing.Destination = "1.1.1.1"
outgoing.Protocol = "udp"
local := incoming
local.Source = "192.168.0.15"
local.Destination = "192.168.0.10"
local.Protocol = "icmpv6"
s.SaveNetwork(a.ID, NetworkSnapshot{At: time.Now().Unix(), Flows: []NetworkFlow{incoming, probe, outgoing, local}})
snapshot, err := s.ReadNetworkSnapshot(a.ID, rules, true)
if err != nil || len(snapshot.Flows) != 3 || snapshot.MonitoringHidden != 1 {
t.Fatalf("source-only: %+v %v", snapshot, err)
}
mapped := makeNetworkMap(snapshot, false)
if mapped.TCP != 1 || mapped.UDP != 1 || mapped.ICMP != 1 || mapped.Unmapped != 1 || len(mapped.Points) != 2 {
t.Fatalf("map: %+v", mapped)
}
for _, p := range mapped.Points {
if p.Protocol == "udp" && p.Direction != "out" {
t.Fatal("outbound direction lost")
}
if p.Protocol == "tcp" && p.Direction != "in" {
t.Fatal("inbound direction lost")
}
}
mapped = makeNetworkMap(snapshot, true)
if mapped.ICMP != 0 {
t.Fatal("LAN flow leaked")
}
view, err := s.TrafficView("example.org", "alert", false, true)
if err != nil || view.Network == nil || view.Network.TCP != 1 {
t.Fatal("HTTP filters incorrectly hide network layer")
}
encoded, _ := json.Marshal(view)
if len(encoded) == 0 {
t.Fatal("map not serializable")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID})
view, _ = s.TrafficView("", "", false)
if view.Network != nil {
t.Fatal("disabled network leaked")
}
v6 := NetworkFlow{Source: "2001:db8:0:0::1"}
if !isMonitoringFlow(v6, []MonitoringRule{{IP: "2001:db8::1"}}) {
t.Fatal("IPv6 representation mismatch")
}
}

107
traffic_network_map.go Normal file
View File

@@ -0,0 +1,107 @@
package main
import (
"database/sql"
"encoding/json"
"fmt"
"sort"
)
type NetworkMapPoint struct {
Country string `json:"country"`
Protocol string `json:"protocol"`
Direction string `json:"direction"`
Count int `json:"count"`
}
type NetworkMapView struct {
At int64 `json:"at"`
ReceivedAt int64 `json:"receivedAt"`
TCP int `json:"tcp"`
UDP int `json:"udp"`
ICMP int `json:"icmp"`
Unmapped int `json:"unmapped"`
MonitoringHidden int `json:"monitoringHidden"`
Limited bool `json:"limited"`
Error string `json:"error,omitempty"`
Points []NetworkMapPoint `json:"points"`
}
func (s *Store) ReadNetworkSnapshot(agent string, rules []MonitoringRule, hide bool) (NetworkSnapshot, error) {
var raw, name string
err := s.db.QueryRow(`SELECT n.snapshot,a.name FROM network_sources n JOIN agents a ON a.id=n.agent WHERE n.agent=?`, agent).Scan(&raw, &name)
if err == sql.ErrNoRows {
return NetworkSnapshot{Flows: []NetworkFlow{}, Error: "Ожидание агента v0.21.0 или новее"}, nil
}
if err != nil {
return NetworkSnapshot{}, err
}
var snapshot NetworkSnapshot
if err = json.Unmarshal([]byte(raw), &snapshot); err != nil {
return snapshot, err
}
snapshot.AgentName = name
flows := make([]NetworkFlow, 0, len(snapshot.Flows))
for _, f := range snapshot.Flows {
f.Monitoring = isMonitoringFlow(f, rules)
if hide && f.Monitoring {
snapshot.MonitoringHidden++
continue
}
flows = append(flows, f)
}
snapshot.Flows = flows
return snapshot, nil
}
func makeNetworkMap(snapshot NetworkSnapshot, external bool) NetworkMapView {
v := NetworkMapView{At: snapshot.At, ReceivedAt: snapshot.ReceivedAt, Limited: snapshot.Limited, Error: snapshot.Error, MonitoringHidden: snapshot.MonitoringHidden, Points: []NetworkMapPoint{}}
if snapshot.ParseErrors > 0 {
v.Error += fmt.Sprintf(" Нераспознанных строк: %d", snapshot.ParseErrors)
}
points := map[string]*NetworkMapPoint{}
for _, f := range snapshot.Flows {
if external && !f.External {
continue
}
protocol := f.Protocol
if protocol == "icmpv6" {
protocol = "icmp"
}
switch protocol {
case "tcp":
v.TCP++
case "udp":
v.UDP++
case "icmp":
v.ICMP++
}
country, direction := f.Country, "in"
if !networkExternalIP(f.Source) {
if networkExternalIP(f.Destination) {
country, direction = f.DestinationCountry, "out"
} else {
v.Unmapped++
continue
}
}
if country == "" || country == "LAN" {
v.Unmapped++
continue
}
key := country + "/" + protocol + "/" + direction
if points[key] == nil {
points[key] = &NetworkMapPoint{Country: country, Protocol: protocol, Direction: direction}
}
points[key].Count++
}
for _, p := range points {
v.Points = append(v.Points, *p)
}
sort.Slice(v.Points, func(i, j int) bool {
a, b := v.Points[i], v.Points[j]
if a.Count != b.Count {
return a.Count > b.Count
}
return a.Country+a.Protocol+a.Direction < b.Country+b.Protocol+b.Direction
})
return v
}

92
traffic_preview_test.go Normal file
View File

@@ -0,0 +1,92 @@
package main
import (
"fmt"
"net/http"
"os"
"strings"
"testing"
"time"
)
// Opt-in local visual fixture; never included in production binaries.
func TestTrafficPreview(t *testing.T) {
if os.Getenv("DASHBOARD_TRAFFIC_PREVIEW") != "1" {
t.Skip("manual visual preview")
}
s, a := trafficFixture(t)
s.SaveTrafficSettings(TrafficSettings{Enabled: true, NetworkEnabled: true, MonitoringRules: []MonitoringRule{{IP: "192.168.0.10"}}, AgentID: a.ID, ServerCountry: "RU"})
mux := http.NewServeMux()
trafficRoutes(mux, s)
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(dashboardHTML))
})
mux.HandleFunc("/api/agents", func(w http.ResponseWriter, r *http.Request) { items, _ := s.Agents(); writeJSON(w, items, nil) })
go func() {
i := 0
for {
i++
events := []TrafficEvent{}
for j, ip := range []string{"8.8.8.8", "1.1.1.1", "20.194.96.114", "192.168.0.10", "77.83.39.94", "51.158.145.102", "123.56.127.23"} {
kind := "request"
if j == 4 {
kind = "alert"
}
if j == 5 {
kind = "decision"
}
events = append(events, TrafficEvent{ID: fmt.Sprintf("preview-%d-%d", i, j), Time: time.Now().Unix(), Kind: kind, IP: ip, Domain: "demo.example", Method: "GET", Path: "/demo", Status: 200, Reason: "ДЕМО · crowdsecurity/http-probing"})
}
flows := []NetworkFlow{}
for k := 0; k < 32; k++ {
f, _ := parseNetworkFlow(networkNATLine)
f.SourcePort = 51000 + k
f.ReplyDestinationPort = f.SourcePort
f.Source = []string{"8.8.8.8", "1.1.1.1", "20.194.96.114", "192.168.0.10", "51.158.145.102", "2001:4860:4860::8888"}[k%6]
f.ReplyDestination = f.Source
if k%3 == 0 {
f.Protocol = "udp"
f.State = "ASSURED"
f.DestinationPort = 53
f.ReplySourcePort = 53
}
if k%5 == 0 && f.Protocol == "tcp" {
f.State = "SYN_SENT"
}
if k%7 == 0 {
f.Protocol = "icmp"
f.State = "TRACKED"
f.SourcePort = 0
f.DestinationPort = 0
f.ReplySourcePort = 0
f.ReplyDestinationPort = 0
}
if k == 11 {
f.Source = "192.168.0.12"
f.Destination = "1.1.1.1"
f.ReplySource = "1.1.1.1"
f.ReplyDestination = "192.168.0.12"
}
flows = append(flows, f)
}
s.SaveNetwork(a.ID, NetworkSnapshot{At: time.Now().Unix(), Flows: flows, FirewallAt: time.Now().Unix(), Firewall: []FirewallCounter{{Name: "DEMO ip/crowdsec/drop #3", Packets: 12345, Bytes: 654321}}})
s.SaveTraffic(a.ID, TrafficBatch{Events: events, Files: 22, StartedAt: time.Now().Add(-time.Hour).Unix(), CrowdSecAt: time.Now().Unix()})
time.Sleep(3 * time.Second)
}
}()
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/network" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(strings.Replace(networkHTML, "PROXMOX / NETWORK OBSERVATORY", "ДЕМО / ИСКУССТВЕННЫЕ ДАННЫЕ", 1)))
return
}
if r.URL.Path == "/traffic" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(strings.Replace(trafficHTML, "PROXMOX / LIVE OBSERVATORY", "ДЕМО / ИСКУССТВЕННЫЕ ДАННЫЕ", 1)))
return
}
mux.ServeHTTP(w, r)
})
t.Fatal(http.ListenAndServe("127.0.0.1:19105", handler))
}

80
traffic_settings.go Normal file
View File

@@ -0,0 +1,80 @@
package main
import (
"encoding/json"
"errors"
"io"
"net/http"
"strings"
)
type TrafficSettings struct {
MonitoringRules []MonitoringRule `json:"monitoringRules"`
NetworkEnabled bool `json:"networkEnabled"`
Enabled bool `json:"enabled"`
AgentID string `json:"agentId"`
ServerCountry string `json:"serverCountry"`
}
func (s *Store) TrafficSettings() TrafficSettings {
var c TrafficSettings
s.loadJSONSetting("traffic", &c)
if c.MonitoringRules == nil {
c.MonitoringRules = []MonitoringRule{{IP: "192.168.0.10"}}
}
return c
}
func (s *Store) SaveTrafficSettings(c TrafficSettings) (TrafficSettings, error) {
rules, err := validateMonitoringRules(c.MonitoringRules)
if err != nil {
return c, err
}
c.MonitoringRules = rules
c.AgentID = strings.TrimSpace(c.AgentID)
c.ServerCountry = strings.ToUpper(strings.TrimSpace(c.ServerCountry))
if c.Enabled {
var count int
if s.db.QueryRow(`SELECT COUNT(*) FROM agents WHERE id=?`, c.AgentID).Scan(&count) != nil || count != 1 {
return c, errors.New("Выберите зарегистрированный агент")
}
}
if c.ServerCountry != "" {
var countries []struct {
Code string `json:"code"`
}
json.Unmarshal(trafficWorld, &countries)
found := false
for _, country := range countries {
if country.Code == c.ServerCountry {
found = true
break
}
}
if !found {
return c, errors.New("Неизвестная страна сервера")
}
}
return c, s.saveJSONSetting("traffic", c)
}
func trafficSettingsRoute(mux *http.ServeMux, s *Store) {
mux.HandleFunc("/api/settings/traffic", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
writeJSON(w, s.TrafficSettings(), nil)
case "PUT":
var c TrafficSettings
if json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&c) != nil {
http.Error(w, "Некорректные настройки", 400)
return
}
value, err := s.SaveTrafficSettings(c)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
writeJSON(w, value, nil)
default:
w.WriteHeader(405)
}
})
}

217
traffic_test.go Normal file
View File

@@ -0,0 +1,217 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func trafficFixture(t *testing.T) (*Store, agentConfig) {
t.Helper()
s, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.Close() })
token, err := s.CreateAgentEnrollmentToken()
if err != nil {
t.Fatal(err)
}
a, err := s.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "NPM test", VMID: 102})
if err != nil {
t.Fatal(err)
}
return s, a
}
func npmTestLine(ip, path string) string {
return fmt.Sprintf("[%s] - - 301 - GET https example.test \"%s\" [Client %s] [Length 166] [Gzip -] [Sent-to 192.168.0.9] \"-\" \"-\"\n", time.Now().UTC().Format("02/Jan/2006:15:04:05 -0700"), path, ip)
}
func TestNPMTrafficFormats(t *testing.T) {
for _, status := range []string{"- 200 200", "- - 301", "- 502 502", "- - 401"} {
line := strings.Replace(npmTestLine("2001:4860:4860::8888", "/login?token=secret#fragment"), "- - 301", status, 1)
e, ok := parseNPMTraffic(line)
if !ok || e.Kind != "request" || e.Path != "/login" || e.Domain != "example.test" || e.IP != "2001:4860:4860::8888" {
t.Fatalf("bad parse: %+v %v", e, ok)
}
}
if _, ok := parseNPMTraffic("broken"); ok {
t.Fatal("accepted invalid log")
}
}
func TestCrowdTrafficDoesNotInventBlockedRequests(t *testing.T) {
raw := `[{"id":12,"uuid":"alert-uuid","created_at":"2026-10-09T16:16:43Z","source":{"ip":"20.194.96.114"},"scenario":"crowdsecurity/http-admin-interface-probing","events":[{"meta":[{"key":"target_fqdn","value":"example.test"}]}],"decisions":[{"id":42,"value":"20.194.96.114","type":"ban","simulated":false},{"id":43,"value":"1.1.1.1","type":"ban","simulated":true}]}]`
events, err := parseCrowdTraffic([]byte(raw))
if err != nil || len(events) != 2 {
t.Fatalf("%+v %v", events, err)
}
if events[0].Kind != "alert" || events[1].Kind != "decision" || events[1].Domain != "example.test" {
t.Fatalf("bad categories %+v", events)
}
simulated := strings.Replace(raw, `"id":12`, `"simulated":true,"id":12`, 1)
events, err = parseCrowdTraffic([]byte(simulated))
if err != nil || len(events) != 0 {
t.Fatal("simulation counted as live")
}
}
func TestTrafficLogCheckpointRotationAndPartialLines(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "proxy-host-1_access.log")
os.WriteFile(path, []byte(npmTestLine("1.1.1.1", "/old")), 0600)
state := trafficAgentState{Cursors: map[string]trafficCursor{}}
if _, err := state.readLogs(dir); err != nil {
t.Fatal(err)
}
if len(state.Pending) != 0 {
t.Fatal("imported old logs on first activation")
}
appendFile := func(path, s string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
t.Fatal(err)
}
f.WriteString(s)
f.Close()
}
line := npmTestLine("8.8.8.8", "/new")
appendFile(path, line[:len(line)/2])
state.readLogs(dir)
if len(state.Pending) != 0 {
t.Fatal("partial line emitted")
}
appendFile(path, line[len(line)/2:])
state.readLogs(dir)
if len(state.Pending) != 1 {
t.Fatal("complete line missing")
}
id := state.Pending[0].ID
state.Pending = nil
checkpoint := filepath.Join(dir, "state.json")
if err := saveTrafficAgentState(checkpoint, state); err != nil {
t.Fatal(err)
}
raw, _ := os.ReadFile(checkpoint)
var restored trafficAgentState
json.Unmarshal(raw, &restored)
os.Rename(path, path+".1")
appendFile(path+".1", npmTestLine("9.9.9.9", "/rotated"))
os.WriteFile(path, []byte(npmTestLine("1.0.0.1", "/fresh")), 0600)
restored.readLogs(dir)
if len(restored.Pending) != 2 {
t.Fatalf("rotation lost events %+v", restored)
}
for _, e := range restored.Pending {
if e.ID == id {
t.Fatal("duplicate id")
}
}
restored.Pending = nil
restored.readLogs(dir)
if len(restored.Pending) != 0 {
t.Fatal("re-read same lines")
}
}
func TestTrafficStoreDedupFiltersAndSanitization(t *testing.T) {
s, a := trafficFixture(t)
if s.TrafficSettings().Enabled {
t.Fatal("enabled by default")
}
if _, err := s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID}); err != nil {
t.Fatal(err)
}
now := time.Now().Unix()
batch := TrafficBatch{Files: 22, Events: []TrafficEvent{{ID: "1", Time: now, Kind: "request", IP: "1.1.1.1", Domain: "example.test", Path: "/login?secret=x", Status: 502}, {ID: "2", Time: now, Kind: "request", IP: "192.168.0.10", Domain: "example.test", Status: 200}, {ID: "3", Time: now, Kind: "alert", IP: "1.1.1.1", Reason: "probe"}, {ID: "4", Time: now, Kind: "decision", IP: "1.1.1.1", Reason: "ban"}}}
for i := 0; i < 2; i++ {
if err := s.SaveTraffic(a.ID, batch); err != nil {
t.Fatal(err)
}
}
v, err := s.TrafficView("", "", false)
if err != nil {
t.Fatal(err)
}
if v.Requests != 2 || v.UniqueIPs != 2 || v.Alerts != 1 || v.Decisions != 1 || len(v.Events) != 4 {
t.Fatalf("wrong counts: %+v", v)
}
for _, e := range v.Events {
if strings.Contains(e.Path, "secret") {
t.Fatal("secret saved")
}
}
v, err = s.TrafficView("example.test", "request", true)
if err != nil || v.Requests != 1 {
t.Fatalf("bad filter %+v %v", v, err)
}
batch.Events = []TrafficEvent{{ID: "invalid", Kind: "blocked", IP: "1.1.1.1", Time: now}}
if s.SaveTraffic(a.ID, batch) == nil {
t.Fatal("accepted invented block event")
}
}
func TestTrafficAPIOptInAndSourceAuthorization(t *testing.T) {
s, a := trafficFixture(t)
mux := http.NewServeMux()
trafficRoutes(mux, s)
get := func(path string) int {
r := httptest.NewRecorder()
mux.ServeHTTP(r, httptest.NewRequest("GET", path, nil))
return r.Code
}
if get("/traffic") != 404 || get("/api/traffic") != 404 {
t.Fatal("disabled page exposed")
}
send := func(secret string) int {
r := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/api/agent/traffic", strings.NewReader(`{"events":[]}`))
req.Header.Set("Authorization", "Bearer "+a.ID+"."+secret)
mux.ServeHTTP(r, req)
return r.Code
}
if send("wrong") != 401 || send(a.Secret) != 403 {
t.Fatal("disabled/unauthorized ingestion accepted")
}
if _, err := s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: "missing"}); err == nil {
t.Fatal("unknown agent accepted")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: a.ID, ServerCountry: "RU"})
if send(a.Secret) != 200 || get("/traffic") != 200 {
t.Fatal("selected agent not working")
}
token, _ := s.CreateAgentEnrollmentToken()
other, _ := s.EnrollAgent(agentEnrollmentRequest{Token: token, Name: "Other"})
s.SaveTrafficSettings(TrafficSettings{Enabled: true, AgentID: other.ID})
if send(a.Secret) != 403 {
t.Fatal("old source still accepted")
}
s.SaveTrafficSettings(TrafficSettings{Enabled: false, AgentID: other.ID})
if get("/api/traffic") != 404 {
t.Fatal("disable failed")
}
}
func TestTrafficGeoDatabase(t *testing.T) {
if !trafficGeoReady() {
t.Fatal("embedded database invalid")
}
if lookupTrafficCountry("192.168.0.10") != "LAN" {
t.Fatal("LAN mapped to country")
}
if c := lookupTrafficCountry("8.8.8.8"); len(c) != 2 {
t.Fatalf("bad public IP geo: %q", c)
}
}
func TestTrafficCheckpointKeepsPendingUntilAcknowledged(t *testing.T) {
path := filepath.Join(t.TempDir(), "state.json")
s := trafficAgentState{Pending: []TrafficEvent{{ID: "retry-me", IP: "1.1.1.1"}}, Cursors: map[string]trafficCursor{"file": {Offset: 100}}}
if err := saveTrafficAgentState(path, s); err != nil {
t.Fatal(err)
}
raw, _ := os.ReadFile(path)
if !bytes.Contains(raw, []byte("retry-me")) {
t.Fatal("pending batch lost")
}
}

6
traffic_web.go Normal file
View File

@@ -0,0 +1,6 @@
package main
import _ "embed"
//go:embed assets/traffic.html
var trafficHTML string

107
trends.go Normal file
View File

@@ -0,0 +1,107 @@
package main
import (
"database/sql"
"fmt"
"sync"
"time"
)
type entityAverage struct {
Value1, Value2 float64
Count int
}
type TrendAnomaly struct {
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
}
type trendCollector struct {
mu sync.Mutex
updatedAt time.Time
cached []TrendAnomaly
}
func (s *Store) entityAverages(kind string) map[string]entityAverage {
return s.entityAveragesSince(kind, 7*24*time.Hour)
}
func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage {
rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix())
if err != nil {
return nil
}
defer rows.Close()
result := map[string]entityAverage{}
for rows.Next() {
var name string
var v1, v2 sql.NullFloat64
var count int
if rows.Scan(&name, &v1, &v2, &count) == nil {
result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count}
}
}
return result
}
func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.updatedAt) < time.Minute {
return c.cached
}
var out []TrendAnomaly
iops := store.entityAverages("disk-io")
latency := store.entityAverages("disk-latency")
latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour)
temps := store.entityAverages("disk")
for _, d := range metrics.Disks {
name := d.Name + " · " + d.Model
currentIOPS := d.ReadIOPS + d.WriteIOPS
if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 {
out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)})
}
if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 {
out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)})
}
if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 {
out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)})
}
if d.Temperature != nil {
if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 {
out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)})
}
}
}
guestAvg := store.entityAverages("guest")
for _, g := range metrics.Guests.Guests {
if g.MaxMemory == 0 {
continue
}
name := fmt.Sprintf("%d · %s", g.VMID, g.Name)
current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100
if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 {
out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)})
}
}
zfsAverage := store.entityAverages("zfs")
for _, pool := range metrics.ZFS.Pools {
if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 {
out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)})
}
}
backupAverage := store.entityAverages("backup-duration")
for _, task := range metrics.Activity.Tasks {
if task.Type != "vzdump" || task.Duration <= 0 {
continue
}
name := "VMID " + task.ID
if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 {
out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)})
}
}
c.cached = out
c.updatedAt = time.Now()
return out
}

452
web.go

File diff suppressed because one or more lines are too long