5 Commits

Author SHA1 Message Date
Maxim
1296d052a5 Run guest updates outside agent sandbox 2026-08-21 22:09:35 +03:00
Maxim
f764195ba5 Clarify process CPU and improve Docker actions 2026-08-21 21:58:00 +03:00
Maxim
f6498a2fc1 Add backup maintenance and guest update controls 2026-08-21 21:15:52 +03:00
Maxim
714c29c449 Fix NVMe self-tests and mobile layout 2026-08-21 19:17:30 +03:00
Maxim
21b4f528e0 Add agent operations history and manual links 2026-08-07 22:24:44 +03:00
20 changed files with 1062 additions and 77 deletions

View File

@@ -358,6 +358,17 @@ CPU и RAM с накопленным I/O, swap, PSI pressure, все посто
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
помечаются как предположение.
Для сложных reverse proxy и Docker-сетей в управлении сервисами можно сохранить
ручную связь `сервис → агент/VMID → systemd/Docker/процесс → порт`. Она имеет
приоритет над автоматическим определением и используется для группировки
связанных уведомлений по первопричине.
Dashboard хранит семь дней истории CPU, RAM, swap, PSI и файловых систем каждого
агента. Из подробностей агента доступны только заранее разрешённые действия:
перезапуск известной systemd-службы или Docker-контейнера, `reset-failed` и
обновление списка пакетов. Агент не принимает произвольные shell-команды; каждое
действие требует подтверждения в браузере и сохраняется в журнале управления.
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
минут и после регистрации заменяется индивидуальным секретом агента.

175
agent.go
View File

@@ -13,6 +13,7 @@ import (
"io"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -40,16 +41,22 @@ type agentConfig struct {
}
type AgentContainer struct {
ID string `json:"id"`
Name string `json:"name"`
Image string `json:"image"`
State string `json:"state"`
Status string `json:"status"`
Health string `json:"health,omitempty"`
Restarts int64 `json:"restarts"`
CPUPercent float64 `json:"cpuPercent"`
MemoryUsage uint64 `json:"memoryUsage"`
Ports []string `json:"ports,omitempty"`
ID string `json:"id"`
Name string `json:"name"`
Image string `json:"image"`
State string `json:"state"`
Status string `json:"status"`
Health string `json:"health,omitempty"`
Restarts int64 `json:"restarts"`
CPUPercent float64 `json:"cpuPercent"`
MemoryUsage uint64 `json:"memoryUsage"`
Ports []string `json:"ports,omitempty"`
Compose bool `json:"compose"`
UpdateAvailable bool `json:"updateAvailable"`
CurrentVersion string `json:"currentVersion,omitempty"`
CurrentDigest string `json:"currentDigest,omitempty"`
AvailableVersion string `json:"availableVersion,omitempty"`
AvailableDigest string `json:"availableDigest,omitempty"`
}
type AgentInterface struct {
@@ -86,37 +93,39 @@ type AgentIssue struct {
}
type AgentReport struct {
Hostname string `json:"hostname"`
MachineID string `json:"machineId"`
Version string `json:"version"`
OS string `json:"os"`
Arch string `json:"arch"`
OSName string `json:"osName"`
Kernel string `json:"kernel"`
UptimeSeconds int64 `json:"uptimeSeconds"`
Load1 float64 `json:"load1"`
CPUPercent float64 `json:"cpuPercent"`
MemoryTotal uint64 `json:"memoryTotal"`
MemoryUsed uint64 `json:"memoryUsed"`
SwapTotal uint64 `json:"swapTotal"`
SwapUsed uint64 `json:"swapUsed"`
RootTotal uint64 `json:"rootTotal"`
RootUsed uint64 `json:"rootUsed"`
DockerAvailable bool `json:"dockerAvailable"`
DockerError string `json:"dockerError,omitempty"`
Containers []AgentContainer `json:"containers"`
Interfaces []AgentInterface `json:"interfaces"`
Services []AgentService `json:"services"`
Issues []AgentIssue `json:"issues"`
Processes []AgentProcess `json:"processes"`
Filesystems []AgentFilesystem `json:"filesystems"`
ListenPorts []AgentListenPort `json:"listenPorts"`
Pressure AgentPressure `json:"pressure"`
OOMKills []AgentLog `json:"oomKills"`
UpdatesAvailable int `json:"updatesAvailable"`
RebootRequired bool `json:"rebootRequired"`
InventoryAt int64 `json:"inventoryAt"`
CollectedAt int64 `json:"collectedAt"`
Hostname string `json:"hostname"`
MachineID string `json:"machineId"`
Version string `json:"version"`
OS string `json:"os"`
Arch string `json:"arch"`
OSName string `json:"osName"`
Kernel string `json:"kernel"`
UptimeSeconds int64 `json:"uptimeSeconds"`
Load1 float64 `json:"load1"`
CPUPercent float64 `json:"cpuPercent"`
CPUCount int `json:"cpuCount"`
MemoryTotal uint64 `json:"memoryTotal"`
MemoryUsed uint64 `json:"memoryUsed"`
SwapTotal uint64 `json:"swapTotal"`
SwapUsed uint64 `json:"swapUsed"`
RootTotal uint64 `json:"rootTotal"`
RootUsed uint64 `json:"rootUsed"`
DockerAvailable bool `json:"dockerAvailable"`
DockerError string `json:"dockerError,omitempty"`
Containers []AgentContainer `json:"containers"`
Interfaces []AgentInterface `json:"interfaces"`
Services []AgentService `json:"services"`
Issues []AgentIssue `json:"issues"`
Processes []AgentProcess `json:"processes"`
Filesystems []AgentFilesystem `json:"filesystems"`
ListenPorts []AgentListenPort `json:"listenPorts"`
Pressure AgentPressure `json:"pressure"`
OOMKills []AgentLog `json:"oomKills"`
UpdatesAvailable int `json:"updatesAvailable"`
RebootRequired bool `json:"rebootRequired"`
InventoryAt int64 `json:"inventoryAt"`
CollectedAt int64 `json:"collectedAt"`
ActionResults []AgentActionResult `json:"actionResults,omitempty"`
}
type agentInventory struct {
@@ -137,6 +146,7 @@ type AgentUpdateManifest struct {
type agentReportResponse struct {
Accepted bool `json:"accepted"`
Update *AgentUpdateManifest `json:"update,omitempty"`
Commands []AgentCommand `json:"commands,omitempty"`
}
var guestInventory struct {
@@ -150,6 +160,11 @@ var guestIssues struct {
collectedAt time.Time
}
var guestActionResults struct {
sync.Mutex
value []AgentActionResult
}
var agentBinaryInfo struct {
sync.Once
manifest AgentUpdateManifest
@@ -258,6 +273,8 @@ func (s *Store) SaveAgentReport(id, secret, remote string, report AgentReport) e
if !s.AuthenticateAgent(id, secret) {
return errors.New("агент не авторизован")
}
s.CompleteAgentActions(id, report.ActionResults)
report.ActionResults = nil
data, _ := json.Marshal(report)
_, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id)
return err
@@ -325,6 +342,9 @@ func runGuestAgent(opts agentOptions) error {
client := &http.Client{Timeout: 12 * time.Second}
for {
report := collectAgentReport()
guestActionResults.Lock()
report.ActionResults = append([]AgentActionResult(nil), guestActionResults.value...)
guestActionResults.Unlock()
data, _ := json.Marshal(report)
req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data))
req.Header.Set("Content-Type", "application/json")
@@ -351,7 +371,23 @@ func runGuestAgent(opts agentOptions) error {
}
if response.StatusCode == http.StatusOK {
var result agentReportResponse
if json.Unmarshal(responseData, &result) == nil && result.Update != nil && newerAgentVersion(result.Update.Version, version) {
if json.Unmarshal(responseData, &result) == nil {
if len(report.ActionResults) > 0 {
guestActionResults.Lock()
guestActionResults.value = nil
guestActionResults.Unlock()
}
if len(result.Commands) > 0 {
executed := make([]AgentActionResult, 0, len(result.Commands))
for _, command := range result.Commands {
executed = append(executed, executeAgentCommand(command))
}
guestActionResults.Lock()
guestActionResults.value = append(guestActionResults.value, executed...)
guestActionResults.Unlock()
}
}
if result.Update != nil && newerAgentVersion(result.Update.Version, version) {
if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil {
requestErr = fmt.Errorf("автообновление: %v", updateErr)
} else {
@@ -363,7 +399,9 @@ func runGuestAgent(opts agentOptions) error {
if requestErr != nil {
fmt.Fprintf(os.Stderr, "agent: %v\n", requestErr)
}
time.Sleep(10 * time.Second)
// Короткий интервал нужен для интерактивных команд из панели. Тяжёлый
// системный инвентарь отдельно кэшируется и не запускается каждые 3 секунды.
time.Sleep(3 * time.Second)
}
}
@@ -499,7 +537,7 @@ func saveAgentConfig(path string, c agentConfig) error {
func collectAgentReport() AgentReport {
hostname, _ := os.Hostname()
machineID, _ := os.ReadFile("/etc/machine-id")
report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CollectedAt: time.Now().Unix()}
report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CPUCount: runtime.NumCPU(), CollectedAt: time.Now().Unix()}
if data, err := os.ReadFile("/proc/uptime"); err == nil {
fields := strings.Fields(string(data))
if len(fields) > 0 {
@@ -885,14 +923,23 @@ func collectDockerContainers() ([]AgentContainer, string) {
inspect, inspectErr := client.Get("http://docker/containers/" + v.ID + "/json")
if inspectErr == nil {
var detail struct {
Image string
RestartCount int64
State struct{ Health *struct{ Status string } }
Config struct {
Labels map[string]string
}
}
if json.NewDecoder(inspect.Body).Decode(&detail) == nil {
item.Restarts = detail.RestartCount
if detail.State.Health != nil {
item.Health = detail.State.Health.Status
}
item.Compose = detail.Config.Labels["com.docker.compose.service"] != ""
item.CurrentVersion, item.CurrentDigest, _ = inspectDockerImage(client, detail.Image, v.Image)
var availableID string
item.AvailableVersion, item.AvailableDigest, availableID = inspectDockerImage(client, v.Image, v.Image)
item.UpdateAvailable = detail.Image != "" && availableID != "" && detail.Image != availableID
}
inspect.Body.Close()
}
@@ -943,6 +990,46 @@ func collectDockerContainers() ([]AgentContainer, string) {
}
return result, ""
}
func inspectDockerImage(client *http.Client, reference, fallback string) (string, string, string) {
if reference == "" {
return "", "", ""
}
response, err := client.Get("http://docker/images/" + url.PathEscape(reference) + "/json")
if err != nil {
return "", "", ""
}
defer response.Body.Close()
var detail struct {
ID string
RepoTags []string
RepoDigests []string
Config struct{ Labels map[string]string }
}
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
return "", "", ""
}
version := firstNonEmpty(detail.Config.Labels["org.opencontainers.image.version"], detail.Config.Labels["version"], firstString(detail.RepoTags), fallback, shortDockerDigest(detail.ID))
digest := ""
if len(detail.RepoDigests) > 0 {
digest = detail.RepoDigests[0]
if _, value, found := strings.Cut(digest, "@"); found {
digest = value
}
}
if digest == "" {
digest = detail.ID
}
return version, shortDockerDigest(digest), detail.ID
}
func shortDockerDigest(value string) string {
value = strings.TrimPrefix(value, "sha256:")
if len(value) > 12 {
value = value[:12]
}
return value
}
func firstString(v []string) string {
if len(v) > 0 {
return v[0]

346
agent_actions.go Normal file
View File

@@ -0,0 +1,346 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
)
type AgentCommand struct {
ID int64 `json:"id"`
Action string `json:"action"`
Target string `json:"target,omitempty"`
}
type AgentActionResult struct {
ID int64 `json:"id"`
Success bool `json:"success"`
Message string `json:"message"`
}
type AgentAction struct {
ID int64 `json:"id"`
AgentID string `json:"agentId"`
AgentName string `json:"agentName"`
Action string `json:"action"`
Target string `json:"target"`
RequestedAt int64 `json:"requestedAt"`
StartedAt *int64 `json:"startedAt,omitempty"`
CompletedAt *int64 `json:"completedAt,omitempty"`
Status string `json:"status"`
Result string `json:"result"`
}
var safeUnitName = regexp.MustCompile(`^[A-Za-z0-9_.@:-]+\.service$`)
var safeContainerName = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, error) {
action, target = strings.TrimSpace(action), strings.TrimSpace(target)
var raw, name string
if err := s.db.QueryRow(`SELECT report_json,name FROM agents WHERE id=?`, agentID).Scan(&raw, &name); err != nil {
return AgentAction{}, fmt.Errorf("агент не найден")
}
var report AgentReport
_ = json.Unmarshal([]byte(raw), &report)
valid := false
switch action {
case "systemd-restart", "systemd-reset-failed":
if safeUnitName.MatchString(target) {
for _, service := range report.Services {
if service.Name == target {
valid = true
break
}
}
}
case "docker-restart", "docker-check", "docker-update", "docker-remove":
if safeContainerName.MatchString(target) {
for _, container := range report.Containers {
if (container.ID == target || container.Name == target) && (action != "docker-remove" || container.State != "running") {
valid = true
break
}
}
}
case "updates-refresh", "system-update", "system-reboot":
valid = target == ""
}
if !valid {
return AgentAction{}, fmt.Errorf("действие или цель не входят в разрешённый список")
}
now := time.Now().Unix()
result, err := s.db.Exec(`INSERT INTO agent_actions(agent_id,action,target,requested_at,status) VALUES(?,?,?,?, 'queued')`, agentID, action, target, now)
if err != nil {
return AgentAction{}, err
}
id, _ := result.LastInsertId()
return AgentAction{ID: id, AgentID: agentID, AgentName: name, Action: action, Target: target, RequestedAt: now, Status: "queued"}, nil
}
func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
// Обновление пакетов и Docker-образов может быть долгим. Не выдаём одну
// команду повторно, пока предыдущий запуск ещё имеет шанс завершиться.
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-45*time.Minute).Unix())
rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
if err != nil {
return nil, err
}
var commands []AgentCommand
for rows.Next() {
var c AgentCommand
if err = rows.Scan(&c.ID, &c.Action, &c.Target); err != nil {
rows.Close()
return nil, err
}
commands = append(commands, c)
}
rows.Close()
now := time.Now().Unix()
for _, c := range commands {
if _, err = tx.Exec(`UPDATE agent_actions SET status='running',started_at=? WHERE id=?`, now, c.ID); err != nil {
return nil, err
}
}
return commands, tx.Commit()
}
func (s *Store) CompleteAgentActions(agentID string, results []AgentActionResult) {
now := time.Now().Unix()
for _, result := range results {
status := "failed"
if result.Success {
status = "completed"
}
message := result.Message
if len(message) > 4000 {
message = message[:4000]
}
_, _ = s.db.Exec(`UPDATE agent_actions SET status=?,result=?,completed_at=? WHERE id=? AND agent_id=? AND status='running'`, status, message, now, result.ID, agentID)
}
}
func (s *Store) AgentActions(limit int) ([]AgentAction, error) {
if limit < 1 || limit > 200 {
limit = 100
}
rows, err := s.db.Query(`SELECT a.id,a.agent_id,g.name,a.action,a.target,a.requested_at,a.started_at,a.completed_at,a.status,a.result FROM agent_actions a JOIN agents g ON g.id=a.agent_id ORDER BY a.id DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []AgentAction
for rows.Next() {
var v AgentAction
if err = rows.Scan(&v.ID, &v.AgentID, &v.AgentName, &v.Action, &v.Target, &v.RequestedAt, &v.StartedAt, &v.CompletedAt, &v.Status, &v.Result); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func executeAgentCommand(command AgentCommand) AgentActionResult {
result := AgentActionResult{ID: command.ID}
var output string
switch command.Action {
case "systemd-restart":
output = runAgentCombinedCommand(30*time.Second, "systemctl", "restart", command.Target)
case "systemd-reset-failed":
output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
case "docker-restart":
output = restartAgentContainer(command.Target)
case "docker-update":
output = updateAgentContainer(command.Target, true)
case "docker-check":
output = updateAgentContainer(command.Target, false)
case "docker-remove":
output = removeAgentContainer(command.Target)
case "system-update":
output = updateAgentSystem()
case "system-reboot":
output = scheduleAgentReboot()
case "updates-refresh":
guestInventory.Lock()
guestInventory.value.CollectedAt = time.Time{}
guestInventory.Unlock()
result.Success = true
result.Message = "Проверка обновлений поставлена в очередь"
return result
default:
result.Message = "Команда не разрешена"
return result
}
result.Success = !strings.HasPrefix(output, "ERROR: ")
result.Message = strings.TrimSpace(output)
if result.Message == "" && result.Success {
result.Message = "Выполнено успешно"
}
return result
}
func removeAgentContainer(target string) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
request, _ := http.NewRequest(http.MethodDelete, "http://docker/containers/"+target+"?v=false&force=false", nil)
response, err := (&http.Client{Transport: transport, Timeout: 20 * time.Second}).Do(request)
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
if response.StatusCode != http.StatusNoContent {
message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))
return fmt.Sprintf("ERROR: Docker вернул %s: %s", response.Status, strings.TrimSpace(string(message)))
}
return "Остановленный контейнер удалён; volumes сохранены"
}
func updateAgentSystem() string {
aptGet, err := exec.LookPath("apt-get")
if err != nil {
return "ERROR: apt-get не найден; обновление поддерживается для Debian/Ubuntu"
}
if output := runPrivilegedTransient(5*time.Minute, "update-lists", aptGet, "update"); strings.HasPrefix(output, "ERROR: ") {
return explainSystemUpdateError(output)
}
message := runPrivilegedTransient(30*time.Minute, "upgrade", aptGet, "-y", "upgrade")
if strings.HasPrefix(message, "ERROR: ") {
return explainSystemUpdateError(message)
}
guestInventory.Lock()
guestInventory.value.CollectedAt = time.Time{}
guestInventory.Unlock()
return "Обновления системы установлены; список пакетов будет обновлён в следующем отчёте агента"
}
func explainSystemUpdateError(message string) string {
if strings.Contains(strings.ToLower(message), "read-only file system") {
return "ERROR: файловая система агента доступна только для чтения. Обновите Dashboard и агент до v0.19.2 или новее, затем повторите операцию."
}
if len(message) > 1800 {
message = message[len(message)-1800:]
}
return message
}
func runPrivilegedTransient(timeout time.Duration, suffix, name string, args ...string) string {
systemdRun, err := exec.LookPath("systemd-run")
if err != nil {
return "ERROR: systemd-run не найден; невозможно безопасно выйти из read-only sandbox агента"
}
unit := fmt.Sprintf("proxmox-dashboard-%s-%d", suffix, time.Now().UnixNano())
commandArgs := []string{"--quiet", "--wait", "--collect", "--pipe", "--unit=" + unit, "--setenv=DEBIAN_FRONTEND=noninteractive", name}
commandArgs = append(commandArgs, args...)
return runAgentCombinedCommand(timeout, systemdRun, commandArgs...)
}
func scheduleAgentReboot() string {
systemdRun, err := exec.LookPath("systemd-run")
if err != nil {
return "ERROR: systemd-run не найден"
}
systemctl, err := exec.LookPath("systemctl")
if err != nil {
return "ERROR: systemctl не найден"
}
unit := fmt.Sprintf("proxmox-dashboard-reboot-%d", time.Now().UnixNano())
output := runAgentCombinedCommand(15*time.Second, systemdRun, "--quiet", "--collect", "--unit="+unit, "--on-active=20s", systemctl, "reboot")
if strings.HasPrefix(output, "ERROR: ") {
return output
}
return "Перезагрузка запланирована через 20 секунд; кратковременный offline агента ожидаем"
}
func updateAgentContainer(target string, recreate bool) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
response, err := client.Get("http://docker/containers/" + target + "/json")
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
var detail struct {
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"Config"`
}
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
return "ERROR: не удалось получить Compose-конфигурацию контейнера"
}
labels := detail.Config.Labels
service := labels["com.docker.compose.service"]
workdir := labels["com.docker.compose.project.working_dir"]
files := strings.Split(labels["com.docker.compose.project.config_files"], ",")
if service == "" || workdir == "" || len(files) == 0 || !filepath.IsAbs(workdir) || !filepath.IsAbs(files[0]) {
return "ERROR: безопасное обновление доступно только для Docker Compose"
}
if _, err = exec.LookPath("docker"); err != nil {
return "ERROR: Docker CLI не найден"
}
commands := [][]string{{"compose", "-f", files[0], "pull", service}}
if recreate {
commands = append(commands, []string{"compose", "-f", files[0], "up", "-d", "--no-deps", service})
}
for _, args := range commands {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
command := exec.CommandContext(ctx, "docker", args...)
command.Dir = workdir
output, commandErr := command.CombinedOutput()
cancel()
if commandErr != nil {
return "ERROR: " + commandErr.Error() + ": " + strings.TrimSpace(string(output))
}
}
if recreate {
return "Образ загружен, Compose-контейнер пересоздан"
}
return "Проверка завершена: образ загружен; если digest изменился, карточка покажет доступное обновление"
}
func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
output, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
message := strings.TrimSpace(string(output))
if err != nil {
return "ERROR: " + err.Error() + ": " + message
}
return message
}
func restartAgentContainer(target string) string {
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 35 * time.Second}
request, _ := http.NewRequest(http.MethodPost, "http://docker/containers/"+target+"/restart?t=10", bytes.NewReader(nil))
response, err := client.Do(request)
if err != nil {
return "ERROR: " + err.Error()
}
defer response.Body.Close()
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
if response.StatusCode >= 300 {
return "ERROR: Docker вернул " + response.Status + ": " + strings.TrimSpace(string(body))
}
return "Контейнер перезапущен"
}

77
agent_actions_test.go Normal file
View File

@@ -0,0 +1,77 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea", State: "running"}, {ID: "old123", Name: "old", State: "exited"}}})
_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
if err != nil {
t.Fatal(err)
}
if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
t.Fatal("unsafe unit accepted")
}
if _, err = store.QueueAgentAction("a", "docker-update", "gitea"); err != nil {
t.Fatalf("known Docker container rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "system-update", ""); err != nil {
t.Fatalf("system update rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "docker-remove", "gitea"); err == nil {
t.Fatal("running/unknown-state container accepted for removal")
}
action, err := store.QueueAgentAction("a", "docker-remove", "old")
if err != nil {
t.Fatalf("stopped container rejected for removal: %v", err)
}
commands, err := store.DispatchAgentActions("a")
if err != nil || len(commands) != 3 || commands[2].ID != action.ID {
t.Fatalf("commands=%#v err=%v", commands, err)
}
store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
items, err := store.AgentActions(10)
if err != nil || len(items) != 3 || items[0].Status != "completed" {
t.Fatalf("items=%#v err=%v", items, err)
}
if _, err = store.QueueAgentAction("a", "system-reboot", ""); err != nil {
t.Fatalf("system reboot rejected: %v", err)
}
if _, err = store.QueueAgentAction("a", "system-reboot", "now"); err == nil {
t.Fatal("system reboot with unexpected target accepted")
}
}
func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "a", Name: "app", VMID: 105, Online: false}}, ServiceLinks: []ServiceLink{{ServiceID: 7, AgentID: "a", VMID: 105}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{ID: 7, Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Kind: "local", Up: false}}}}}}
alerts := []Alert{{ID: "agent-offline-a-105"}, {ID: "service-7-local"}, {ID: "other"}}
grouped := groupLinkedAlerts(metrics, alerts)
if len(grouped) != 2 {
t.Fatalf("grouped=%#v", grouped)
}
found := false
for _, alert := range grouped {
if alert.ID == "group-agent-offline-a-105" {
found = true
}
}
if !found {
t.Fatalf("group alert missing: %#v", grouped)
}
}
func TestExplainReadOnlySystemUpdateError(t *testing.T) {
message := explainSystemUpdateError("ERROR: apt failed: Read-only file system")
if !strings.Contains(message, "v0.19.2") {
t.Fatalf("read-only failure is not actionable: %s", message)
}
}

View File

@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
@@ -25,6 +26,8 @@ type AgentProcess struct {
Name string `json:"name"`
State string `json:"state"`
CPUPercent float64 `json:"cpuPercent"`
CPUCores float64 `json:"cpuCores"`
HostPercent float64 `json:"hostPercent"`
MemoryBytes uint64 `json:"memoryBytes"`
ReadBytes uint64 `json:"readBytes"`
WriteBytes uint64 `json:"writeBytes"`
@@ -174,7 +177,11 @@ func collectAgentProcesses() []AgentProcess {
cpu, _ := strconv.ParseFloat(fields[3], 64)
rss, _ := strconv.ParseUint(fields[4], 10, 64)
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
cpuCount := runtime.NumCPU()
if cpuCount < 1 {
cpuCount = 1
}
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, CPUCores: cpu / 100, HostPercent: cpu / float64(cpuCount), MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
result = append(result, item)
added++

View File

@@ -53,6 +53,12 @@ func TestAgentOnlyInstallsNewerVersion(t *testing.T) {
}
}
func TestShortDockerDigest(t *testing.T) {
if got := shortDockerDigest("sha256:1234567890abcdef"); got != "1234567890ab" {
t.Fatalf("unexpected short digest: %q", got)
}
}
func TestParseAgentIssues(t *testing.T) {
issues := parseAgentIssues(`{"MESSAGE":"disk error","_SYSTEMD_UNIT":"worker.service","__REALTIME_TIMESTAMP":"1722960000000000"}` + "\n" + `{"MESSAGE":"network error","__REALTIME_TIMESTAMP":"1722960001000000"}`)
if len(issues) != 2 || issues[0].Unit != "worker.service" || issues[1].Unit != "system" || issues[0].Time != 1722960000 {

View File

@@ -124,10 +124,22 @@ func (s *Store) AcknowledgeAlert(eventID int64) error {
}
func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
return s.AlertHistoryBetween(limit, 0, 0)
}
func (s *Store) AlertHistoryBetween(limit int, from, to int64) ([]AlertEvent, error) {
if limit < 1 || limit > 500 {
limit = 200
}
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events ORDER BY first_seen DESC LIMIT ?`, limit)
query := `SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events`
args := []any{}
if from > 0 && to > from {
query += ` WHERE first_seen>=? AND first_seen<?`
args = append(args, from, to)
}
query += ` ORDER BY first_seen DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.Query(query, args...)
if err != nil {
return nil, err
}

View File

@@ -74,7 +74,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
if disk.SMARTStatus == "Ошибка" {
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
} else if disk.SMARTStatus == "Требует внимания" {
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(disk.AttentionReasons, "Проверьте SMART-показатели диска."))
reasons := withoutReason(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался")
if len(reasons) > 0 {
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(reasons, "Проверьте SMART-показатели диска."))
}
}
if disk.Temperature != nil {
limit := thresholds.DiskTempWarning
@@ -260,10 +263,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
}
var networkErrors uint64
for _, iface := range metrics.Network.Interfaces {
networkErrors += iface.ReceiveErrors + iface.TransmitErrors + iface.ReceiveDrops + iface.TransmitDrops
networkErrors += iface.NewErrors + iface.NewDrops
}
if networkErrors > 0 {
add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors))
add("network-errors", "warning", "Сеть", "Появились новые сетевые ошибки или потери", fmt.Sprintf("За последний интервал добавилось: %d. Старые счётчики не создают активный алерт.", networkErrors))
}
if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp {
add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён."))
@@ -386,3 +389,13 @@ func joinReasons(reasons []string, fallback string) string {
}
return strings.Join(reasons, " · ")
}
func withoutReason(reasons []string, excluded string) []string {
filtered := make([]string, 0, len(reasons))
for _, reason := range reasons {
if reason != excluded {
filtered = append(filtered, reason)
}
}
return filtered
}

View File

@@ -25,3 +25,17 @@ func TestSlowCollectionDoesNotCreateAlert(t *testing.T) {
}
}
}
func TestNeverRunSelfTestCreatesOneAlert(t *testing.T) {
metrics := dashboardMetrics{Disks: []DiskMetrics{{
Name: "nvme0n1",
Model: "Test NVMe",
SMARTStatus: "Требует внимания",
SelfTestNeverRun: true,
AttentionReasons: []string{"SMART self-test ещё ни разу не запускался"},
}}}
alerts := evaluateAlerts(metrics, defaultThresholds())
if len(alerts) != 1 || alerts[0].ID != "disk-selftest-nvme0n1" {
t.Fatalf("ожидалось одно точное уведомление о self-test, получено: %+v", alerts)
}
}

19
cpu.go
View File

@@ -33,14 +33,15 @@ type CPUStatic struct {
type CPUMetrics struct {
CPUStatic
UsagePercent float64 `json:"usagePercent"`
FrequencyMHz float64 `json:"frequencyMhz"`
Load1 float64 `json:"load1"`
Load5 float64 `json:"load5"`
Load15 float64 `json:"load15"`
Uptime uint64 `json:"uptimeSeconds"`
Temperatures []Temperature `json:"temperatures"`
CollectedAt time.Time `json:"collectedAt"`
UsagePercent float64 `json:"usagePercent"`
FrequencyMHz float64 `json:"frequencyMhz"`
Load1 float64 `json:"load1"`
Load5 float64 `json:"load5"`
Load15 float64 `json:"load15"`
Uptime uint64 `json:"uptimeSeconds"`
Temperatures []Temperature `json:"temperatures"`
TopProcesses []AgentProcess `json:"topProcesses"`
CollectedAt time.Time `json:"collectedAt"`
}
type Temperature struct {
@@ -92,7 +93,7 @@ func (c *cpuCollector) collect() (CPUMetrics, error) {
return CPUMetrics{
CPUStatic: c.static, UsagePercent: usage, FrequencyMHz: frequency,
Load1: load1, Load5: load5, Load15: load15, Uptime: uptime,
Temperatures: readTemperatures(), CollectedAt: time.Now(),
Temperatures: readTemperatures(), TopProcesses: collectAgentProcesses(), CollectedAt: time.Now(),
}, nil
}

View File

@@ -391,6 +391,17 @@ type smartctlJSON struct {
} `json:"table"`
} `json:"standard"`
} `json:"ata_smart_self_test_log"`
NVMeSelfTestLog struct {
Table []struct {
SelfTestCode struct {
String string `json:"string"`
} `json:"self_test_code"`
SelfTestResult struct {
String string `json:"string"`
} `json:"self_test_result"`
PowerOnHours uint64 `json:"power_on_hours"`
} `json:"table"`
} `json:"nvme_self_test_log"`
NVMeHealth *struct {
Temperature float64 `json:"temperature"`
PercentageUsed float64 `json:"percentage_used"`
@@ -410,6 +421,10 @@ func applySMART(disk *DiskMetrics) {
if len(output) == 0 {
return
}
applySMARTJSON(disk, output)
}
func applySMARTJSON(disk *DiskMetrics, output []byte) {
var smart smartctlJSON
if json.Unmarshal(output, &smart) != nil {
return
@@ -451,12 +466,20 @@ func applySMART(disk *DiskMetrics) {
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
}
}
if len(smart.ATASelfTestLog.Standard.Table) > 0 {
if len(smart.NVMeSelfTestLog.Table) > 0 {
last := smart.NVMeSelfTestLog.Table[0]
description := strings.TrimSpace(last.SelfTestCode.String)
result := firstNonEmpty(strings.TrimSpace(last.SelfTestResult.String), "Результат неизвестен")
if description != "" {
disk.LastSelfTest = description + " · " + result
} else {
disk.LastSelfTest = result
}
setLastSelfTestAt(disk, last.PowerOnHours)
} else if len(smart.ATASelfTestLog.Standard.Table) > 0 {
last := smart.ATASelfTestLog.Standard.Table[0]
disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours {
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339)
}
setLastSelfTestAt(disk, last.LifetimeHours)
} else {
disk.SelfTestNeverRun = true
disk.LastSelfTest = "Никогда не запускался"
@@ -469,6 +492,12 @@ func applySMART(disk *DiskMetrics) {
}
}
func setLastSelfTestAt(disk *DiskMetrics, testPowerOnHours uint64) {
if disk.PowerOnHours != nil && *disk.PowerOnHours >= testPowerOnHours {
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-testPowerOnHours) * time.Hour).Format(time.RFC3339)
}
}
func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
wearNames := map[string]bool{
"Percent_Lifetime_Remain": true, "SSD_Life_Left": true,

View File

@@ -6,8 +6,44 @@ import (
"path/filepath"
"strings"
"testing"
"time"
)
func TestApplySMARTJSONReadsNVMeSelfTestHistory(t *testing.T) {
disk := DiskMetrics{Path: "/dev/nvme0n1"}
output := []byte(`{
"model_name":"ADATA LEGEND 900",
"smart_status":{"passed":true},
"power_on_time":{"hours":960},
"nvme_self_test_log":{
"current_self_test_operation":{"value":0,"string":"No self-test in progress"},
"table":[{
"self_test_code":{"value":1,"string":"Short"},
"self_test_result":{"value":0,"string":"Completed without error"},
"power_on_hours":959
}]
}
}`)
before := time.Now().Add(-2 * time.Hour)
applySMARTJSON(&disk, output)
after := time.Now()
if disk.SelfTestNeverRun {
t.Fatal("завершённый NVMe self-test ошибочно отмечен как никогда не запускавшийся")
}
if disk.LastSelfTest != "Short · Completed without error" {
t.Fatalf("неожиданный результат self-test: %q", disk.LastSelfTest)
}
parsed, err := time.Parse(time.RFC3339, disk.LastSelfTestAt)
if err != nil || parsed.Before(before) || parsed.After(after) {
t.Fatalf("неверно вычислено время self-test: %q, err=%v", disk.LastSelfTestAt, err)
}
if len(disk.AttentionReasons) != 0 || disk.SMARTStatus != "Исправен" {
t.Fatalf("исправный NVMe требует внимания: %+v", disk)
}
}
func TestIsPhysicalDiskName(t *testing.T) {
tests := map[string]bool{
"sda": true, "sdb": true, "nvme0n1": true,

View File

@@ -64,6 +64,30 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
}
}
for _, agent := range metrics.Agents {
if !agent.Online {
continue
}
name := agent.ID + " · " + agent.Name
cpu := agent.Report.CPUPercent
memory := float64(0)
if agent.Report.MemoryTotal > 0 {
memory = float64(agent.Report.MemoryUsed) / float64(agent.Report.MemoryTotal) * 100
}
swap := float64(0)
if agent.Report.SwapTotal > 0 {
swap = float64(agent.Report.SwapUsed) / float64(agent.Report.SwapTotal) * 100
}
s.AddEntityHistory("agent-resource", name, ts, &cpu, &memory)
s.AddEntityHistory("agent-swap", name, ts, &swap, nil)
pressure := agent.Report.Pressure.IO.Avg10
s.AddEntityHistory("agent-pressure", name, ts, &pressure, nil)
for _, filesystem := range agent.Report.Filesystems {
used := filesystem.UsedPercent
inodes := filesystem.InodesPercent
s.AddEntityHistory("agent-filesystem", name+" · "+filesystem.Mountpoint, ts, &used, &inodes)
}
}
s.PruneEntityHistory()
}

View File

@@ -17,6 +17,51 @@ type MaintenanceWindow struct {
AllAlerts bool `json:"allAlerts"`
ServiceIDs []int64 `json:"serviceIds"`
VMIDs []int `json:"vmids"`
Automatic bool `json:"automatic"`
Source string `json:"source,omitempty"`
}
func automaticBackupMaintenance(activity ProxmoxActivity, links []ServiceLink, now time.Time) MaintenanceWindow {
vmids := []int{}
backupRunning := false
for _, task := range activity.Tasks {
if task.Type != "vzdump" || task.EndTime > 0 || (task.Status != "" && !strings.EqualFold(task.Status, "running")) {
continue
}
backupRunning = true
if vmid := backupTaskVMID(task); vmid > 0 {
vmids = append(vmids, vmid)
}
}
vmids = uniqueInt(vmids)
if !backupRunning {
return MaintenanceWindow{}
}
selected := map[int]bool{}
for _, vmid := range vmids {
selected[vmid] = true
}
serviceIDs := []int64{}
for _, link := range links {
if selected[link.VMID] {
serviceIDs = append(serviceIDs, link.ServiceID)
}
}
return MaintenanceWindow{Enabled: true, Active: true, StartedAt: now.Unix(), EndsAt: now.Add(2 * time.Minute).Unix(), Reason: "Автоматически: выполняется backup VM/LXC", VMIDs: vmids, ServiceIDs: uniqueInt64(serviceIDs), AllAlerts: len(vmids) == 0, Automatic: true, Source: "backup"}
}
func backupTaskVMID(task ProxmoxTask) int {
if vmid, err := strconv.Atoi(strings.TrimSpace(task.ID)); err == nil && vmid > 0 {
return vmid
}
parts := strings.Split(task.UPID, ":")
for i, part := range parts {
if part == "vzdump" && i+1 < len(parts) {
vmid, _ := strconv.Atoi(parts[i+1])
return vmid
}
}
return 0
}
func (s *Store) MaintenanceWindow() MaintenanceWindow {

View File

@@ -38,6 +38,21 @@ func TestMaintenanceExpiresAutomatically(t *testing.T) {
}
}
func TestRunningBackupCreatesScopedAutomaticMaintenance(t *testing.T) {
now := time.Now()
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", ID: "105"}, {Type: "vzdump", ID: "106", EndTime: now.Unix()}}}, []ServiceLink{{ServiceID: 7, VMID: 105}, {ServiceID: 8, VMID: 999}}, now)
if !window.Active || !window.Automatic || window.Source != "backup" || len(window.VMIDs) != 1 || window.VMIDs[0] != 105 || len(window.ServiceIDs) != 1 || window.ServiceIDs[0] != 7 {
t.Fatalf("неверное автоматическое окно: %+v", window)
}
}
func TestRunningBackupWithoutVMIDSuppressesAllAlerts(t *testing.T) {
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", UPID: "unknown"}}}, nil, time.Now())
if !window.Active || !window.AllAlerts {
t.Fatalf("backup с неизвестной целью должен безопасно подавить алерты: %+v", window)
}
}
func TestPlannedAlertOnlyAppearsInHistory(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {

View File

@@ -3,6 +3,7 @@ package main
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
@@ -36,6 +37,9 @@ type NetworkHealth struct {
ExternalIPChanged bool `json:"externalIpChanged"`
DisappearedInterfaces []string `json:"disappearedInterfaces"`
CheckedAt time.Time `json:"checkedAt"`
Summary string `json:"summary"`
LikelyCause string `json:"likelyCause"`
RecommendedAction string `json:"recommendedAction"`
}
type NetworkInterface struct {
@@ -46,6 +50,8 @@ type NetworkInterface struct {
TransmitErrors uint64 `json:"transmitErrors"`
ReceiveDrops uint64 `json:"receiveDrops"`
TransmitDrops uint64 `json:"transmitDrops"`
NewErrors uint64 `json:"newErrors"`
NewDrops uint64 `json:"newDrops"`
ReceivePerSec float64 `json:"receivePerSec"`
TransmitPerSec float64 `json:"transmitPerSec"`
VMID int `json:"vmid,omitempty"`
@@ -113,6 +119,12 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes {
v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed
v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed
if v.ReceiveErrors >= old.ReceiveErrors && v.TransmitErrors >= old.TransmitErrors {
v.NewErrors = v.ReceiveErrors - old.ReceiveErrors + v.TransmitErrors - old.TransmitErrors
}
if v.ReceiveDrops >= old.ReceiveDrops && v.TransmitDrops >= old.TransmitDrops {
v.NewDrops = v.ReceiveDrops - old.ReceiveDrops + v.TransmitDrops - old.TransmitDrops
}
}
}
c.previousInterfaces = map[string]NetworkInterface{}
@@ -138,12 +150,41 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
}
}
c.knownInterfaces = currentNames
explainNetworkHealth(&c.health, details)
return NetworkMetrics{
Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
}, nil
}
func explainNetworkHealth(health *NetworkHealth, interfaces []NetworkInterface) {
health.Summary, health.LikelyCause, health.RecommendedAction = "Сеть работает штатно", "Проблем не обнаружено", "Действий не требуется"
switch {
case health.Gateway == "":
health.Summary, health.LikelyCause, health.RecommendedAction = "Не найден маршрут в локальную сеть", "На хосте отсутствует default gateway", "Проверьте адрес шлюза и конфигурацию bridge/VLAN"
case !health.GatewayUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Хост не видит локальный шлюз", "Проблема между Proxmox и роутером: линк, bridge, bond или VLAN", "Проверьте кабель, carrier и настройки vmbr/bond/VLAN"
case health.PacketLoss > 0:
health.Summary = fmt.Sprintf("Нестабильная локальная сеть: потеряно %.1f%% пакетов", health.PacketLoss)
health.LikelyCause, health.RecommendedAction = "Перегрузка или ошибки линка между Proxmox и шлюзом", "Проверьте errors/drops, кабель, порт коммутатора и duplex"
case !health.DNSUp && health.InternetUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Интернет доступен, но DNS не отвечает", "Проблема DNS-сервера или /etc/resolv.conf", "Проверьте настроенные DNS и попробуйте разрешить домен вручную"
case health.DNSUp && !health.InternetUp:
health.Summary, health.LikelyCause, health.RecommendedAction = "Локальная сеть и DNS работают, но внешняя проверка недоступна", "Проблема выше роутера, блокировка HTTPS или недоступность провайдера", "Проверьте маршрут наружу, firewall и доступ к внешним адресам"
}
for _, v := range interfaces {
if v.Kind == "Physical" && (!v.Carrier || v.State == "down") {
health.Summary, health.LikelyCause, health.RecommendedAction = v.Name+": физический линк не поднят", "Нет carrier либо интерфейс административно выключен", "Проверьте кабель, порт коммутатора и состояние интерфейса"
return
}
if v.NewErrors+v.NewDrops > 0 {
health.Summary = fmt.Sprintf("%s: за последний интервал появилось %d сетевых ошибок/потерь", v.Name, v.NewErrors+v.NewDrops)
health.LikelyCause, health.RecommendedAction = "Счётчики интерфейса сейчас растут из-за перегрузки, кабеля, MTU или duplex", "Проверьте скорость и duplex, кабель, порт коммутатора и MTU на обоих концах"
return
}
}
}
func decorateInterface(v *NetworkInterface) {
base := filepath.Join("/sys/class/net", v.Name)
v.State = readTrimmed(filepath.Join(base, "operstate"))

View File

@@ -3,6 +3,7 @@ package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
@@ -33,6 +34,26 @@ func TestReadNetworkSnapshot(t *testing.T) {
}
}
func TestExplainNetworkHealthDistinguishesDNSFailure(t *testing.T) {
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: false}
explainNetworkHealth(&health, nil)
if !strings.Contains(health.Summary, "DNS") || health.RecommendedAction == "" {
t.Fatalf("непонятная диагностика: %+v", health)
}
}
func TestExplainNetworkHealthIgnoresOldInterfaceCounters(t *testing.T) {
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: true}
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 42}})
if health.Summary != "Сеть работает штатно" {
t.Fatalf("старый накопленный счётчик вызвал текущую тревогу: %+v", health)
}
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 43, NewDrops: 1}})
if !strings.Contains(health.Summary, "последний интервал") {
t.Fatalf("рост счётчика не объяснён: %+v", health)
}
}
func TestInterfaceVMID(t *testing.T) {
for name, want := range map[string]int{"tap100i0": 100, "veth108i0": 108, "fwbr113i0": 113, "vmbr0": 0, "enp6s0": 0} {
if got := interfaceVMID(name); got != want {

110
service_links.go Normal file
View File

@@ -0,0 +1,110 @@
package main
import (
"fmt"
"sort"
"strings"
)
type ServiceLink struct {
ServiceID int64 `json:"serviceId"`
AgentID string `json:"agentId"`
VMID int `json:"vmid"`
ComponentType string `json:"componentType"`
ComponentName string `json:"componentName"`
Port int `json:"port"`
}
func (s *Store) ServiceLinks() ([]ServiceLink, error) {
rows, err := s.db.Query(`SELECT service_id,agent_id,vmid,component_type,component_name,port FROM service_links ORDER BY service_id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ServiceLink
for rows.Next() {
var v ServiceLink
if err = rows.Scan(&v.ServiceID, &v.AgentID, &v.VMID, &v.ComponentType, &v.ComponentName, &v.Port); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
func (s *Store) SaveServiceLink(v ServiceLink) (ServiceLink, error) {
v.AgentID = strings.TrimSpace(v.AgentID)
v.ComponentType = strings.TrimSpace(v.ComponentType)
v.ComponentName = strings.TrimSpace(v.ComponentName)
if v.ServiceID <= 0 {
return v, fmt.Errorf("выберите сервис")
}
if v.AgentID == "" && v.VMID <= 0 {
return v, fmt.Errorf("выберите агент или VMID")
}
if v.ComponentType != "" && v.ComponentType != "systemd" && v.ComponentType != "docker" && v.ComponentType != "process" {
return v, fmt.Errorf("неизвестный тип компонента")
}
if v.Port < 0 || v.Port > 65535 {
return v, fmt.Errorf("некорректный порт")
}
_, err := s.db.Exec(`INSERT INTO service_links(service_id,agent_id,vmid,component_type,component_name,port) VALUES(?,?,?,?,?,?) ON CONFLICT(service_id) DO UPDATE SET agent_id=excluded.agent_id,vmid=excluded.vmid,component_type=excluded.component_type,component_name=excluded.component_name,port=excluded.port`, v.ServiceID, v.AgentID, v.VMID, v.ComponentType, v.ComponentName, v.Port)
return v, err
}
func (s *Store) DeleteServiceLink(serviceID int64) error {
_, err := s.db.Exec(`DELETE FROM service_links WHERE service_id=?`, serviceID)
return err
}
func groupLinkedAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
if len(metrics.ServiceLinks) == 0 {
return alerts
}
agentOffline := map[string]ManagedAgent{}
for _, agent := range metrics.Agents {
if !agent.Online {
agentOffline[agent.ID] = agent
}
}
affected := map[string][]string{}
remove := map[string]bool{}
for _, link := range metrics.ServiceLinks {
agent, offline := agentOffline[link.AgentID]
if !offline {
continue
}
for _, service := range metrics.Services.Services {
if service.ID != link.ServiceID {
continue
}
for _, endpoint := range service.Endpoints {
if !endpoint.Up {
key := fmt.Sprintf("service-%d-%s", service.ID, endpoint.Kind)
remove[key] = true
affected[agent.ID] = append(affected[agent.ID], service.Name)
}
}
}
}
if len(affected) == 0 {
return alerts
}
out := make([]Alert, 0, len(alerts))
for _, alert := range alerts {
skip := remove[alert.ID]
for agentID := range affected {
if alert.ID == "agent-offline-"+agentID+fmt.Sprintf("-%d", agentOffline[agentID].VMID) {
skip = true
}
}
if !skip {
out = append(out, alert)
}
}
for agentID, names := range affected {
sort.Strings(names)
agent := agentOffline[agentID]
out = append(out, Alert{ID: "group-agent-offline-" + agentID + fmt.Sprintf("-%d", agent.VMID), Severity: "critical", Source: "Корреляция", Title: agent.Name + ": агент недоступен, связанные сервисы не отвечают", Message: fmt.Sprintf("VMID %d. Затронуто сервисов: %d — %s. Вероятная первопричина: недоступна гостевая система или агент.", agent.VMID, len(names), strings.Join(names, ", "))})
}
return out
}

View File

@@ -140,6 +140,9 @@ func openStore(path string) (*Store, error) {
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
`CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`,
`CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`,
`CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`,
}
for _, statement := range statements {
if _, err := db.Exec(statement); err != nil {

117
web.go

File diff suppressed because one or more lines are too long