From a570537e64bcb2793e938ff8203c13b1c2554d4d Mon Sep 17 00:00:00 2001 From: Maxim Date: Sat, 1 Aug 2026 16:58:25 +0300 Subject: [PATCH] Add infrastructure reliability monitoring --- README.md | 11 +- alerts.go | 74 +++++++++++ disks.go | 109 +++++++++++++++- main.go | 1 + network.go | 127 ++++++++++++++++++- proxmox-cpu-dashboard.service | 4 +- reliability.go | 226 ++++++++++++++++++++++++++++++++++ reliability_test.go | 38 ++++++ store.go | 47 ++++++- ups.go | 57 ++++++--- web.go | 78 ++++++++++-- zfs.go | 205 +++++++++++++++++++++++++++--- 12 files changed, 926 insertions(+), 51 deletions(-) create mode 100644 reliability.go create mode 100644 reliability_test.go diff --git a/README.md b/README.md index f96addd..4b66abd 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ ssh root@192.168.0.10 ```bash apt-get update -apt-get install -y curl ca-certificates tar smartmontools +apt-get install -y curl ca-certificates tar smartmontools iproute2 iputils-ping cd /tmp curl -fL \ @@ -120,6 +120,15 @@ ss -lntp | grep 9105 Панель показывает: +- изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков; +- автоматический короткий SMART self-test раз в неделю для дисков без свежего теста; +- datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS; +- доступность шлюза, DNS и интернета, packet loss, latency и внешний IP; +- тип, состояние, скорость, duplex и master сетевых интерфейсов; +- состояние связи с NUT/apcupsd, возраст батареи и расчётную автономность UPS; +- настраиваемый сценарий корректного выключения VM/LXC и Proxmox с безопасным dry-run; +- самоконтроль сборщика, записи истории, свободного места базы и systemd watchdog; + - модель и текущую загрузку CPU; - количество сокетов, физических ядер и логических потоков; - среднюю текущую частоту; diff --git a/alerts.go b/alerts.go index 8f617b5..3a83956 100644 --- a/alerts.go +++ b/alerts.go @@ -75,6 +75,15 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler add("disk-temp-"+disk.Name, "warning", "Диски", disk.Model+": высокая температура", fmt.Sprintf("Температура %.1f °C.", *disk.Temperature)) } } + if len(disk.CounterChanges) > 0 { + add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · ")) + } + if disk.SelfTestNeverRun { + add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.") + } + } + for _, missing := range metrics.MissingDisks { + add("disk-missing-"+missing, "critical", "Диски", "Физический диск исчез", missing+" больше не обнаруживается в /sys/block.") } for _, pool := range metrics.ZFS.Pools { @@ -89,6 +98,25 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler if strings.Contains(strings.ToLower(pool.Scan), "in progress") { add("zfs-scan-"+pool.Name, "info", "ZFS", pool.Name+": выполняется scrub/resilver", pool.Scan) } + if pool.ReadErrors+pool.WriteErrors+pool.ChecksumErrors > 0 { + add("zfs-vdev-errors-"+pool.Name, "critical", "ZFS", pool.Name+": ошибки чтения/записи/checksum", fmt.Sprintf("read=%d, write=%d, checksum=%d.", pool.ReadErrors, pool.WriteErrors, pool.ChecksumErrors)) + } + if len(pool.ErrorChanges) > 0 { + add("zfs-new-errors-"+pool.Name, "critical", "ZFS", pool.Name+": появились новые ошибки", strings.Join(pool.ErrorChanges, " · ")) + } + if pool.ScrubTooOld { + add("zfs-scrub-old-"+pool.Name, "warning", "ZFS", pool.Name+": scrub просрочен", "Scrub не выполнялся более 35 дней либо ни разу не запускался.") + } + } + for _, dataset := range metrics.ZFS.Datasets { + if dataset.CapacityPercent >= 90 { + add("zfs-dataset-"+dataset.Name, "critical", "ZFS", dataset.Name+": dataset почти заполнен", fmt.Sprintf("Заполнение %.1f%%, доступно %.1f GB.", dataset.CapacityPercent, float64(dataset.AvailableBytes)/1073741824)) + } else if dataset.CapacityPercent >= 80 { + add("zfs-dataset-"+dataset.Name, "warning", "ZFS", dataset.Name+": мало свободного места", fmt.Sprintf("Заполнение %.1f%%.", dataset.CapacityPercent)) + } + } + if metrics.ZFS.ARC.Available && metrics.ZFS.ARC.Hits+metrics.ZFS.ARC.Misses > 10000 && metrics.ZFS.ARC.HitRatio < 70 { + add("zfs-arc-hit", "warning", "ZFS", "Низкий ARC hit ratio", fmt.Sprintf("Попадания ARC: %.1f%%.", metrics.ZFS.ARC.HitRatio)) } if metrics.UPS.Available { @@ -101,6 +129,14 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler if metrics.UPS.ChargePercent != nil && *metrics.UPS.ChargePercent < thresholds.UPSChargeCritical { add("ups-charge", "critical", "UPS", "Низкий заряд батареи", fmt.Sprintf("Осталось %.1f%%.", *metrics.UPS.ChargePercent)) } + if metrics.UPS.ReplaceBattery { + add("ups-replace-battery", "critical", "UPS", "UPS требует заменить батарею", "NUT/APC сообщает флаг Replace Battery.") + } + if metrics.UPS.BatteryAgeDays > 3*365 { + add("ups-battery-age", "warning", "UPS", "Батарея UPS старше трёх лет", fmt.Sprintf("Возраст батареи примерно %d дней.", metrics.UPS.BatteryAgeDays)) + } + } else if metrics.UPS.Source != "" { + add("ups-communication", "warning", "UPS", "Нет связи с UPS", "NUT и apcupsd не вернули данные. Проверьте USB/сетевое подключение и службу UPS.") } if metrics.Backups.Available && metrics.Guests.Available { @@ -164,6 +200,44 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler if networkErrors > 0 { add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors)) } + if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp { + add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён.")) + } + if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.DNSUp { + add("network-dns", "critical", "Сеть", "Не работает DNS", "Не удалось разрешить deb.debian.org.") + } + if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.InternetUp { + add("network-internet", "warning", "Сеть", "Нет доступа в интернет", "Внешняя проверка IP не выполнена.") + } + if metrics.Network.Health.PacketLoss > 0 { + add("network-loss", "warning", "Сеть", "Потери пакетов до шлюза", fmt.Sprintf("Потери %.1f%%, задержка %.1f ms.", metrics.Network.Health.PacketLoss, metrics.Network.Health.LatencyMs)) + } + if metrics.Network.Health.ExternalIPChanged { + add("network-public-ip", "info", "Сеть", "Изменился внешний IP", "Новый адрес: "+metrics.Network.Health.ExternalIP) + } + for _, name := range metrics.Network.Health.DisappearedInterfaces { + add("network-interface-missing-"+name, "warning", "Сеть", "Исчез сетевой интерфейс", name+" больше не присутствует в /proc/net/dev.") + } + for _, iface := range metrics.Network.Interfaces { + if iface.Kind == "Physical" && !iface.Carrier && (iface.Master != "" || iface.ReceiveBytes+iface.TransmitBytes > 0) { + add("network-link-"+iface.Name, "warning", "Сеть", iface.Name+": нет carrier", "Физический линк не поднят.") + } + if (iface.Kind == "Bridge" || iface.Kind == "Bond" || iface.Kind == "VLAN") && iface.State == "down" { + add("network-state-"+iface.Name, "warning", "Сеть", iface.Name+": интерфейс выключен", iface.Kind+" находится в состоянии down.") + } + } + if !metrics.Monitor.LastSuccessfulCollection.IsZero() && time.Since(metrics.Monitor.LastSuccessfulCollection) > 30*time.Second { + add("monitor-stale", "critical", "Dashboard", "Данные мониторинга устарели", "Последний успешный сбор был "+time.Since(metrics.Monitor.LastSuccessfulCollection).Round(time.Second).String()+" назад.") + } + if metrics.Monitor.SlowCollector { + add("monitor-slow", "warning", "Dashboard", "Сбор метрик выполняется слишком долго", fmt.Sprintf("Последний цикл занял %d ms.", metrics.Monitor.CollectionDurationMs)) + } + if metrics.Monitor.HistoryWriteError != "" { + add("monitor-history", "critical", "Dashboard", "Ошибка записи истории", metrics.Monitor.HistoryWriteError) + } + if metrics.Monitor.DatabaseFreeBytes > 0 && metrics.Monitor.DatabaseFreeBytes < 1024*1024*1024 { + add("monitor-db-space", "warning", "Dashboard", "Мало места для базы истории", fmt.Sprintf("На разделе базы свободно %.1f GB.", float64(metrics.Monitor.DatabaseFreeBytes)/1073741824)) + } oldSnapshots := 0 for _, snapshot := range metrics.Activity.Snapshots { if snapshot.AgeSeconds > 30*24*3600 { diff --git a/disks.go b/disks.go index 915211a..0c4584b 100644 --- a/disks.go +++ b/disks.go @@ -28,12 +28,25 @@ type DiskMetrics struct { Temperature *float64 `json:"temperatureCelsius"` PowerOnHours *uint64 `json:"powerOnHours"` WearUsedPercent *float64 `json:"wearUsedPercent"` + Reallocated uint64 `json:"reallocated"` + Pending uint64 `json:"pending"` + Uncorrectable uint64 `json:"uncorrectable"` + CRCErrors uint64 `json:"crcErrors"` + MediaErrors uint64 `json:"mediaErrors"` + CounterChanges []string `json:"counterChanges"` + LastSelfTest string `json:"lastSelfTest"` + LastSelfTestAt string `json:"lastSelfTestAt"` + SelfTestNeverRun bool `json:"selfTestNeverRun"` + ShortTestStarted bool `json:"shortTestStarted"` } type diskCollector struct { mu sync.Mutex cached []DiskMetrics updatedAt time.Time + previous map[string]DiskMetrics + missing []string + lastTests map[string]time.Time } func (c *diskCollector) collect() []DiskMetrics { @@ -42,11 +55,73 @@ func (c *diskCollector) collect() []DiskMetrics { if time.Since(c.updatedAt) < time.Minute && c.cached != nil { return c.cached } - c.cached = readPhysicalDisks() + current := readPhysicalDisks() + if c.previous == nil { + c.previous = map[string]DiskMetrics{} + c.lastTests = map[string]time.Time{} + } + seen := map[string]bool{} + c.missing = nil + for i := range current { + disk := ¤t[i] + key := disk.Serial + if key == "" || key == "Не указан" { + key = disk.Path + } + seen[key] = true + if old, ok := c.previous[key]; ok { + disk.CounterChanges = diskCounterChanges(old, *disk) + disk.AttentionReasons = append(disk.AttentionReasons, disk.CounterChanges...) + } + if disk.SMARTAvailable && (disk.SelfTestNeverRun || selfTestOlderThanWeek(disk.LastSelfTestAt)) && time.Since(c.lastTests[key]) >= 7*24*time.Hour { + disk.ShortTestStarted = startShortSMARTTest(disk.Path) + if disk.ShortTestStarted { + c.lastTests[key] = time.Now() + } + } + c.previous[key] = *disk + } + for key, disk := range c.previous { + if !seen[key] { + c.missing = append(c.missing, disk.Model+" ("+disk.Serial+")") + } + } + c.cached = current c.updatedAt = time.Now() return c.cached } +func (c *diskCollector) missingDisks() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.missing...) +} + +func diskCounterChanges(old, current DiskMetrics) []string { + checks := []struct { + label string + old, now uint64 + }{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}} + var changes []string + for _, check := range checks { + if check.now > check.old { + changes = append(changes, check.label+": "+strconv.FormatUint(check.old, 10)+" → "+strconv.FormatUint(check.now, 10)) + } + } + return changes +} + +func selfTestOlderThanWeek(value string) bool { + when, err := time.Parse(time.RFC3339, value) + return err == nil && time.Since(when) > 7*24*time.Hour +} + +func startShortSMARTTest(path string) bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return exec.CommandContext(ctx, "smartctl", "-t", "short", path).Run() == nil +} + func readPhysicalDisks() []DiskMetrics { entries, _ := os.ReadDir("/sys/block") usage := readDiskUsage() @@ -202,6 +277,16 @@ type smartctlJSON struct { Count int `json:"count"` } `json:"summary"` } `json:"ata_smart_error_log"` + ATASelfTestLog struct { + Standard struct { + Table []struct { + Status struct { + String string `json:"string"` + } `json:"status"` + LifetimeHours uint64 `json:"lifetime_hours"` + } `json:"table"` + } `json:"standard"` + } `json:"ata_smart_self_test_log"` NVMeHealth *struct { Temperature float64 `json:"temperature"` PercentageUsed float64 `json:"percentage_used"` @@ -254,6 +339,7 @@ func applySMART(disk *DiskMetrics) { } if smart.NVMeHealth != nil { disk.WearUsedPercent = &smart.NVMeHealth.PercentageUsed + disk.MediaErrors = smart.NVMeHealth.MediaErrors if smart.NVMeHealth.CriticalWarning != 0 { disk.AttentionReasons = append(disk.AttentionReasons, "NVMe сообщает критическое предупреждение") } @@ -261,6 +347,17 @@ func applySMART(disk *DiskMetrics) { disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10)) } } + if len(smart.ATASelfTestLog.Standard.Table) > 0 { + last := smart.ATASelfTestLog.Standard.Table[0] + disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен") + if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours { + disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339) + } + } else { + disk.SelfTestNeverRun = true + disk.LastSelfTest = "Никогда не запускался" + disk.AttentionReasons = append(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался") + } applyATAAttributes(disk, smart) applySMARTExitStatus(disk, smart.Smartctl.ExitStatus) if len(disk.AttentionReasons) > 0 && disk.SMARTStatus == "Исправен" { @@ -275,6 +372,16 @@ func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) { "Wear_Leveling_Count": true, } for _, attribute := range smart.ATAAttributes.Table { + switch attribute.Name { + case "Reallocated_Sector_Ct", "Reallocated_Event_Count": + disk.Reallocated += uint64(max(attribute.Raw.Value, 0)) + case "Current_Pending_Sector": + disk.Pending = uint64(max(attribute.Raw.Value, 0)) + case "Offline_Uncorrectable", "Reported_Uncorrect": + disk.Uncorrectable += uint64(max(attribute.Raw.Value, 0)) + case "UDMA_CRC_Error_Count", "Interface_CRC_Error_Count": + disk.CRCErrors += uint64(max(attribute.Raw.Value, 0)) + } if wearNames[attribute.Name] && attribute.Value >= 0 && attribute.Value <= 100 { wear := float64(100 - attribute.Value) disk.WearUsedPercent = &wear diff --git a/main.go b/main.go index d310558..c76be3f 100644 --- a/main.go +++ b/main.go @@ -50,6 +50,7 @@ func main() { Handler: routes(collector, store), ReadHeaderTimeout: 5 * time.Second, } + runSystemdWatchdog() go func() { log.Printf("Proxmox Dashboard %s запущен на http://0.0.0.0%s", version, address) diff --git a/network.go b/network.go index c446c67..8bb00f1 100644 --- a/network.go +++ b/network.go @@ -2,7 +2,13 @@ package main import ( "bufio" + "context" + "io" + "net" + "net/http" "os" + "os/exec" + "path/filepath" "strconv" "strings" "sync" @@ -16,6 +22,20 @@ type NetworkMetrics struct { ReceivePerSec float64 `json:"receivePerSec"` TransmitPerSec float64 `json:"transmitPerSec"` Interfaces []NetworkInterface `json:"interfaces"` + Health NetworkHealth `json:"health"` +} + +type NetworkHealth struct { + Gateway string `json:"gateway"` + GatewayUp bool `json:"gatewayUp"` + DNSUp bool `json:"dnsUp"` + InternetUp bool `json:"internetUp"` + PacketLoss float64 `json:"packetLoss"` + LatencyMs float64 `json:"latencyMs"` + ExternalIP string `json:"externalIp"` + ExternalIPChanged bool `json:"externalIpChanged"` + DisappearedInterfaces []string `json:"disappearedInterfaces"` + CheckedAt time.Time `json:"checkedAt"` } type NetworkInterface struct { @@ -31,6 +51,12 @@ type NetworkInterface struct { VMID int `json:"vmid,omitempty"` GuestName string `json:"guestName,omitempty"` GuestType string `json:"guestType,omitempty"` + Kind string `json:"kind"` + State string `json:"state"` + Carrier bool `json:"carrier"` + SpeedMbps int `json:"speedMbps,omitempty"` + Duplex string `json:"duplex,omitempty"` + Master string `json:"master,omitempty"` } type networkSnapshot struct { @@ -44,6 +70,10 @@ type networkCollector struct { path string previous networkSnapshot previousInterfaces map[string]NetworkInterface + health NetworkHealth + healthUpdatedAt time.Time + externalIP string + knownInterfaces map[string]bool } func newNetworkCollector(path string) (*networkCollector, error) { @@ -56,7 +86,11 @@ func newNetworkCollector(path string) (*networkCollector, error) { for _, v := range details { previous[v.Name] = v } - return &networkCollector{path: path, previous: snapshot, previousInterfaces: previous}, nil + known := map[string]bool{} + for _, v := range details { + known[v.Name] = true + } + return &networkCollector{path: path, previous: snapshot, previousInterfaces: previous, knownInterfaces: known}, nil } func (c *networkCollector) collect() (NetworkMetrics, error) { @@ -75,6 +109,7 @@ func (c *networkCollector) collect() (NetworkMetrics, error) { } for i := range details { v := &details[i] + decorateInterface(v) if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes { v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed @@ -85,12 +120,100 @@ func (c *networkCollector) collect() (NetworkMetrics, error) { c.previousInterfaces[v.Name] = v } c.previous = current + if time.Since(c.healthUpdatedAt) > 30*time.Second { + c.health = checkNetworkHealth(c.externalIP) + if c.health.ExternalIP != "" { + c.externalIP = c.health.ExternalIP + } + c.healthUpdatedAt = time.Now() + } + currentNames := map[string]bool{} + for _, v := range details { + currentNames[v.Name] = true + } + c.health.DisappearedInterfaces = nil + for name := range c.knownInterfaces { + if !currentNames[name] { + c.health.DisappearedInterfaces = append(c.health.DisappearedInterfaces, name) + } + } + c.knownInterfaces = currentNames return NetworkMetrics{ Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received, - TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, + TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health, }, nil } +func decorateInterface(v *NetworkInterface) { + base := filepath.Join("/sys/class/net", v.Name) + v.State = readTrimmed(filepath.Join(base, "operstate")) + v.Carrier = readTrimmed(filepath.Join(base, "carrier")) == "1" + v.SpeedMbps, _ = strconv.Atoi(readTrimmed(filepath.Join(base, "speed"))) + v.Duplex = readTrimmed(filepath.Join(base, "duplex")) + if target, err := filepath.EvalSymlinks(filepath.Join(base, "master")); err == nil { + v.Master = filepath.Base(target) + } + switch { + case strings.HasPrefix(v.Name, "vmbr"): + v.Kind = "Bridge" + case strings.HasPrefix(v.Name, "bond"): + v.Kind = "Bond" + case strings.Contains(v.Name, "."): + v.Kind = "VLAN" + case strings.HasPrefix(v.Name, "tap") || strings.HasPrefix(v.Name, "veth"): + v.Kind = "Guest" + default: + v.Kind = "Physical" + } +} + +func checkNetworkHealth(previousIP string) NetworkHealth { + result := NetworkHealth{CheckedAt: time.Now()} + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + if out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output(); err == nil { + f := strings.Fields(string(out)) + for i := range f { + if f[i] == "via" && i+1 < len(f) { + result.Gateway = f[i+1] + break + } + } + } + if result.Gateway != "" { + out, _ := exec.CommandContext(ctx, "ping", "-c", "3", "-W", "1", result.Gateway).CombinedOutput() + text := string(out) + result.GatewayUp = strings.Contains(text, "0% packet loss") + if i := strings.Index(text, "packet loss"); i > 0 { + start := strings.LastIndex(text[:i], " ") + if start >= 0 { + result.PacketLoss, _ = strconv.ParseFloat(strings.TrimSuffix(strings.TrimSpace(text[start:i]), "%"), 64) + } + } + if i := strings.Index(text, "min/avg/max"); i >= 0 { + if eq := strings.Index(text[i:], "="); eq >= 0 { + parts := strings.Split(strings.Fields(text[i+eq+1:])[0], "/") + if len(parts) > 1 { + result.LatencyMs, _ = strconv.ParseFloat(parts[1], 64) + } + } + } + } + resolver := net.Resolver{} + _, err := resolver.LookupHost(ctx, "deb.debian.org") + result.DNSUp = err == nil + client := http.Client{Timeout: 4 * time.Second} + request, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.ipify.org", nil) + if response, err := client.Do(request); err == nil { + defer response.Body.Close() + body, _ := io.ReadAll(io.LimitReader(response.Body, 128)) + result.ExternalIP = strings.TrimSpace(string(body)) + result.InternetUp = response.StatusCode == 200 && net.ParseIP(result.ExternalIP) != nil + } + result.ExternalIPChanged = previousIP != "" && result.ExternalIP != "" && previousIP != result.ExternalIP + return result +} + func readNetworkSnapshot(path string) (networkSnapshot, []string, []NetworkInterface, error) { file, err := os.Open(path) if err != nil { diff --git a/proxmox-cpu-dashboard.service b/proxmox-cpu-dashboard.service index df72dfa..5e63314 100644 --- a/proxmox-cpu-dashboard.service +++ b/proxmox-cpu-dashboard.service @@ -4,7 +4,7 @@ After=network-online.target Wants=network-online.target [Service] -Type=simple +Type=notify ExecStart=/usr/local/bin/proxmox-cpu-dashboard Environment=DASHBOARD_ADDR=:9105 Environment=DASHBOARD_DB_PATH=/var/lib/proxmox-dashboard/dashboard.db @@ -12,6 +12,8 @@ StateDirectory=proxmox-dashboard StateDirectoryMode=0750 Restart=on-failure RestartSec=3 +WatchdogSec=30 +TimeoutStartSec=20 NoNewPrivileges=true ProtectHome=true PrivateTmp=true diff --git a/reliability.go b/reliability.go new file mode 100644 index 0000000..4fc87cc --- /dev/null +++ b/reliability.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "fmt" + "log" + "net" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" +) + +func notifySystemd(message string) bool { + address := os.Getenv("NOTIFY_SOCKET") + if address == "" { + return false + } + if strings.HasPrefix(address, "@") { + address = "\x00" + address[1:] + } + connection, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: address, Net: "unixgram"}) + if err != nil { + return false + } + defer connection.Close() + _, err = connection.Write([]byte(message)) + return err == nil +} +func runSystemdWatchdog() { + if !notifySystemd("READY=1\nSTATUS=Dashboard collects Proxmox metrics") { + return + } + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for range ticker.C { + notifySystemd("WATCHDOG=1") + } + }() +} + +type MonitorHealth struct { + LastSuccessfulCollection time.Time `json:"lastSuccessfulCollection"` + LastCollectionError string `json:"lastCollectionError"` + CollectionDurationMs int64 `json:"collectionDurationMs"` + SlowCollector bool `json:"slowCollector"` + LastHistoryWrite time.Time `json:"lastHistoryWrite"` + HistoryWriteError string `json:"historyWriteError"` + DatabaseFreeBytes uint64 `json:"databaseFreeBytes"` + DatabasePath string `json:"databasePath"` + WatchdogEnabled bool `json:"watchdogEnabled"` +} + +type monitorTracker struct { + mu sync.RWMutex + lastSuccess time.Time + lastError string + duration time.Duration +} + +func (m *monitorTracker) record(start time.Time, err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.duration = time.Since(start) + if err != nil { + m.lastError = err.Error() + return + } + m.lastSuccess = time.Now() + m.lastError = "" +} + +func (m *monitorTracker) snapshot(store *Store) MonitorHealth { + m.mu.RLock() + result := MonitorHealth{LastSuccessfulCollection: m.lastSuccess, LastCollectionError: m.lastError, CollectionDurationMs: m.duration.Milliseconds(), SlowCollector: m.duration > 10*time.Second, DatabasePath: store.path, WatchdogEnabled: os.Getenv("NOTIFY_SOCKET") != ""} + m.mu.RUnlock() + result.LastHistoryWrite, result.HistoryWriteError = store.historyHealth() + path := store.path + if path == ":memory:" { + path = "/tmp" + } else { + path = filepath.Dir(path) + } + var stat syscall.Statfs_t + if syscall.Statfs(path, &stat) == nil { + result.DatabaseFreeBytes = stat.Bavail * uint64(stat.Bsize) + } + return result +} + +type UPSShutdownSettings struct { + Enabled bool `json:"enabled"` + ChargePercent float64 `json:"chargePercent"` + RuntimeSeconds float64 `json:"runtimeSeconds"` + GraceSeconds int `json:"graceSeconds"` + GuestOrder []int `json:"guestOrder"` +} + +func defaultUPSShutdownSettings() UPSShutdownSettings { + return UPSShutdownSettings{ChargePercent: 15, RuntimeSeconds: 600, GraceSeconds: 120} +} +func (s *Store) UPSShutdownSettings() UPSShutdownSettings { + value := defaultUPSShutdownSettings() + s.loadJSONSetting("ups_shutdown", &value) + return value +} +func (s *Store) SaveUPSShutdownSettings(value UPSShutdownSettings) error { + if value.ChargePercent < 1 || value.ChargePercent > 80 || value.RuntimeSeconds < 60 || value.RuntimeSeconds > 7200 || value.GraceSeconds < 30 || value.GraceSeconds > 1800 { + return fmt.Errorf("проверьте пороги UPS и задержку") + } + return s.saveJSONSetting("ups_shutdown", value) +} + +type ShutdownStep struct { + Order int `json:"order"` + Target string `json:"target"` + Action string `json:"action"` + Detail string `json:"detail"` +} + +func buildShutdownDryRun(settings UPSShutdownSettings, guests GuestsMetrics) []ShutdownStep { + byID := map[int]GuestMetrics{} + for _, g := range guests.Guests { + if g.Template == 0 && g.Status == "running" { + byID[g.VMID] = g + } + } + var ordered []GuestMetrics + for _, id := range settings.GuestOrder { + if g, ok := byID[id]; ok { + ordered = append(ordered, g) + delete(byID, id) + } + } + var rest []GuestMetrics + for _, g := range byID { + rest = append(rest, g) + } + sort.Slice(rest, func(i, j int) bool { return rest[i].VMID < rest[j].VMID }) + ordered = append(ordered, rest...) + steps := []ShutdownStep{{Order: 1, Target: "UPS", Action: "Ожидание порога", Detail: fmt.Sprintf("заряд ≤ %.0f%% или автономность ≤ %.0f сек.", settings.ChargePercent, settings.RuntimeSeconds)}} + for _, g := range ordered { + kind := "LXC" + if g.Type == "qemu" { + kind = "VM" + } + steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: fmt.Sprintf("%s %d · %s", kind, g.VMID, g.Name), Action: "Корректное завершение", Detail: fmt.Sprintf("ожидание до %d сек.", settings.GraceSeconds)}) + } + steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: "Proxmox host", Action: "Завершение работы", Detail: "только после остановки VM/LXC"}) + return steps +} + +func runUPSShutdownController(store *Store, ups *upsCollector, guests *guestCollector) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + triggered := false + for range ticker.C { + settings := store.UPSShutdownSettings() + metrics := ups.collect() + onBattery := strings.Contains(metrics.Status, "OB") || metrics.Status == "ONBATT" + if !settings.Enabled || !metrics.Available || !onBattery { + if metrics.Available && !onBattery { + triggered = false + } + continue + } + lowCharge := metrics.ChargePercent != nil && *metrics.ChargePercent <= settings.ChargePercent + lowRuntime := metrics.RuntimeSeconds != nil && *metrics.RuntimeSeconds <= settings.RuntimeSeconds + if triggered || (!lowCharge && !lowRuntime) { + continue + } + triggered = true + log.Printf("UPS достиг порога безопасного выключения; запускаем остановку гостей") + go executeUPSShutdown(settings, guests.collect()) + } +} + +func executeUPSShutdown(settings UPSShutdownSettings, guests GuestsMetrics) { + steps := buildShutdownDryRun(settings, guests) + _ = steps + byID := map[int]GuestMetrics{} + for _, g := range guests.Guests { + if g.Template == 0 && g.Status == "running" { + byID[g.VMID] = g + } + } + var order []int + for _, id := range settings.GuestOrder { + if _, ok := byID[id]; ok { + order = append(order, id) + delete(byID, id) + } + } + var remaining []int + for id := range byID { + remaining = append(remaining, id) + } + sort.Ints(remaining) + order = append(order, remaining...) + for _, id := range order { + g := GuestMetrics{} + for _, candidate := range guests.Guests { + if candidate.VMID == id { + g = candidate + break + } + } + command := "pct" + if g.Type == "qemu" { + command = "qm" + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(settings.GraceSeconds+30)*time.Second) + err := exec.CommandContext(ctx, command, "shutdown", fmt.Sprint(id), "--timeout", fmt.Sprint(settings.GraceSeconds)).Run() + cancel() + if err != nil { + log.Printf("UPS shutdown: не удалось остановить VMID %d: %v", id, err) + } + } + log.Printf("UPS shutdown: гости обработаны, выключаем Proxmox host") + _ = exec.Command("shutdown", "-h", "now").Run() +} diff --git a/reliability_test.go b/reliability_test.go new file mode 100644 index 0000000..7764324 --- /dev/null +++ b/reliability_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestDiskCounterChanges(t *testing.T) { + old := DiskMetrics{Reallocated: 1, Pending: 2, CRCErrors: 4} + current := DiskMetrics{Reallocated: 2, Pending: 2, CRCErrors: 7} + changes := strings.Join(diskCounterChanges(old, current), " ") + if !strings.Contains(changes, "Reallocated") || !strings.Contains(changes, "CRC") || strings.Contains(changes, "Pending") { + t.Fatalf("unexpected counter changes: %s", changes) + } +} + +func TestParseScrub(t *testing.T) { + status, at, duration, old := parseScrub("scrub repaired 0B in 01:12:03 with 0 errors on Fri Jan 2 03:04:05 2026") + if status != "Завершён" || at == "" || duration != "01:12:03" || !old { + t.Fatalf("unexpected scrub: %q %q %q %v", status, at, duration, old) + } +} + +func TestShutdownDryRunUsesConfiguredOrder(t *testing.T) { + settings := UPSShutdownSettings{ChargePercent: 15, RuntimeSeconds: 600, GraceSeconds: 120, GuestOrder: []int{105}} + guests := GuestsMetrics{Guests: []GuestMetrics{{VMID: 101, Name: "one", Type: "qemu", Status: "running"}, {VMID: 105, Name: "five", Type: "lxc", Status: "running"}}} + steps := buildShutdownDryRun(settings, guests) + if len(steps) != 4 || !strings.Contains(steps[1].Target, "105") || !strings.Contains(steps[2].Target, "101") || steps[3].Target != "Proxmox host" { + t.Fatalf("unexpected plan: %+v", steps) + } +} + +func TestSelfTestOlderThanWeek(t *testing.T) { + if !selfTestOlderThanWeek(time.Now().Add(-8*24*time.Hour).Format(time.RFC3339)) || selfTestOlderThanWeek(time.Now().Format(time.RFC3339)) { + t.Fatal("self-test age evaluation is incorrect") + } +} diff --git a/store.go b/store.go index f6ce353..1881810 100644 --- a/store.go +++ b/store.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" _ "modernc.org/sqlite" @@ -58,7 +59,13 @@ type HistoryPoint struct { GuestsMemory float64 `json:"guestsMemory"` } -type Store struct{ db *sql.DB } +type Store struct { + db *sql.DB + path string + healthMu sync.RWMutex + lastHistoryAt time.Time + lastHistoryError string +} type EmailSettings struct { Enabled bool `json:"enabled"` @@ -130,7 +137,7 @@ func openStore(path string) (*Store, error) { _, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`) _, _ = db.Exec(`ALTER TABLE service_checks ADD COLUMN endpoint TEXT NOT NULL DEFAULT 'public'`) _, _ = db.Exec(`PRAGMA foreign_keys=ON`) - return &Store{db: db}, nil + return &Store{db: db, path: path}, nil } type EntityHistoryPoint struct { @@ -285,9 +292,45 @@ func (s *Store) AddHistory(point HistoryPoint) error { _, err := s.db.Exec(`INSERT OR REPLACE INTO history(ts,cpu_usage,cpu_temp,memory_usage,swap_usage,root_usage,network_rx,network_tx,ups_charge,ups_load,zfs_usage,guests_cpu,guests_memory) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, point.Timestamp, point.CPUUsage, point.CPUTemperature, point.MemoryUsage, point.SwapUsage, point.RootUsage, point.NetworkReceive, point.NetworkTransmit, point.UPSCharge, point.UPSLoad, point.ZFSUsage, point.GuestsCPU, point.GuestsMemory) if err != nil { + s.setHistoryHealth(time.Time{}, err) return err } _, err = s.db.Exec(`DELETE FROM history WHERE ts < ?`, time.Now().Add(-historyRetention).Unix()) + s.setHistoryHealth(time.Now(), err) + return err +} + +func (s *Store) setHistoryHealth(at time.Time, err error) { + s.healthMu.Lock() + defer s.healthMu.Unlock() + if !at.IsZero() { + s.lastHistoryAt = at + } + if err != nil { + s.lastHistoryError = err.Error() + } else { + s.lastHistoryError = "" + } +} +func (s *Store) historyHealth() (time.Time, string) { + s.healthMu.RLock() + defer s.healthMu.RUnlock() + return s.lastHistoryAt, s.lastHistoryError +} + +func (s *Store) loadJSONSetting(key string, target any) bool { + var value string + if s.db.QueryRow(`SELECT value FROM settings WHERE key=?`, key).Scan(&value) != nil { + return false + } + return json.Unmarshal([]byte(value), target) == nil +} +func (s *Store) saveJSONSetting(key string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + _, err = s.db.Exec(`INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, string(data)) return err } diff --git a/ups.go b/ups.go index 04a07a5..63ce07d 100644 --- a/ups.go +++ b/ups.go @@ -10,20 +10,26 @@ import ( ) type UPSMetrics struct { - Available bool `json:"available"` - Source string `json:"source"` - Name string `json:"name"` - Model string `json:"model"` - Status string `json:"status"` - StatusLabel string `json:"statusLabel"` - ChargePercent *float64 `json:"chargePercent"` - RuntimeSeconds *float64 `json:"runtimeSeconds"` - LoadPercent *float64 `json:"loadPercent"` - InputVoltage *float64 `json:"inputVoltage"` - OutputVoltage *float64 `json:"outputVoltage"` - InputFrequency *float64 `json:"inputFrequency"` - BatteryVoltage *float64 `json:"batteryVoltage"` - Temperature *float64 `json:"temperatureCelsius"` + Available bool `json:"available"` + Source string `json:"source"` + Name string `json:"name"` + Model string `json:"model"` + Status string `json:"status"` + StatusLabel string `json:"statusLabel"` + ChargePercent *float64 `json:"chargePercent"` + RuntimeSeconds *float64 `json:"runtimeSeconds"` + LoadPercent *float64 `json:"loadPercent"` + InputVoltage *float64 `json:"inputVoltage"` + OutputVoltage *float64 `json:"outputVoltage"` + InputFrequency *float64 `json:"inputFrequency"` + BatteryVoltage *float64 `json:"batteryVoltage"` + Temperature *float64 `json:"temperatureCelsius"` + CommunicationOK bool `json:"communicationOk"` + BatteryDate string `json:"batteryDate"` + BatteryAgeDays int `json:"batteryAgeDays,omitempty"` + ReplaceBattery bool `json:"replaceBattery"` + LowBatteryCharge *float64 `json:"lowBatteryCharge,omitempty"` + LowBatteryRuntime *float64 `json:"lowBatteryRuntime,omitempty"` } type upsCollector struct { @@ -43,7 +49,13 @@ func (c *upsCollector) collect() UPSMetrics { } else if metrics, ok := readAPCUPSD(); ok { c.cached = metrics } else { - c.cached = UPSMetrics{} + if c.cached.Available || c.cached.Source != "" { + c.cached.Available = false + c.cached.CommunicationOK = false + c.cached.StatusLabel = "Нет связи" + } else { + c.cached = UPSMetrics{} + } } c.updatedAt = time.Now() return c.cached @@ -85,6 +97,7 @@ func readAPCUPSD() (UPSMetrics, bool) { "ups.load": raw["LOADPCT"], "input.voltage": raw["LINEV"], "output.voltage": raw["OUTPUTV"], "input.frequency": raw["LINEFREQ"], "battery.voltage": raw["BATTV"], "ups.temperature": raw["ITEMP"], + "battery.date": raw["BATTDATE"], "battery.charge.low": raw["MBATTCHG"], "battery.runtime.low": minutesToSeconds(raw["MINTIMEL"]), } return upsFromValues("apcupsd", raw["UPSNAME"], values), true } @@ -102,14 +115,24 @@ func parseColonValues(output string) map[string]string { func upsFromValues(source, name string, values map[string]string) UPSMetrics { status := values["ups.status"] - return UPSMetrics{ - Available: true, Source: source, Name: name, Model: values["device.model"], + metrics := UPSMetrics{ + Available: true, CommunicationOK: true, Source: source, Name: name, Model: values["device.model"], Status: status, StatusLabel: upsStatusLabel(status), ChargePercent: numberPointer(values["battery.charge"]), RuntimeSeconds: numberPointer(values["battery.runtime"]), LoadPercent: numberPointer(values["ups.load"]), InputVoltage: numberPointer(values["input.voltage"]), OutputVoltage: numberPointer(values["output.voltage"]), InputFrequency: numberPointer(values["input.frequency"]), BatteryVoltage: numberPointer(values["battery.voltage"]), Temperature: numberPointer(firstNonEmpty(values["ups.temperature"], values["battery.temperature"])), + BatteryDate: firstNonEmpty(values["battery.date"], values["battery.mfr.date"]), ReplaceBattery: strings.Contains(status, "RB"), LowBatteryCharge: numberPointer(values["battery.charge.low"]), LowBatteryRuntime: numberPointer(values["battery.runtime.low"]), } + if metrics.BatteryDate != "" { + for _, layout := range []string{"2006/01/02", "2006-01-02", "01/02/2006", "02/01/2006"} { + if date, err := time.Parse(layout, metrics.BatteryDate); err == nil { + metrics.BatteryAgeDays = int(time.Since(date).Hours() / 24) + break + } + } + } + return metrics } func numberPointer(value string) *float64 { diff --git a/web.go b/web.go index aec6af2..6baa4df 100644 --- a/web.go +++ b/web.go @@ -15,6 +15,7 @@ type dashboardMetrics struct { Memory MemoryMetrics `json:"memory"` Storage StorageMetrics `json:"storage"` Disks []DiskMetrics `json:"disks"` + MissingDisks []string `json:"missingDisks"` Network NetworkMetrics `json:"network"` ZFS ZFSMetrics `json:"zfs"` UPS UPSMetrics `json:"ups"` @@ -26,10 +27,12 @@ type dashboardMetrics struct { Services ServicesMetrics `json:"services"` SystemHealth SystemHealthMetrics `json:"systemHealth"` Activity ProxmoxActivity `json:"activity"` + Monitor MonitorHealth `json:"monitor"` + UPSShutdown UPSShutdownSettings `json:"upsShutdown"` Timestamp time.Time `json:"timestamp"` } -func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, store *Store) (dashboardMetrics, error) { +func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) { cpu, err := cpuCollector.collect() if err != nil { return dashboardMetrics{}, err @@ -52,6 +55,9 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle ZFS: zfsCollector.collect(), UPS: upsCollector.collect(), Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Services: serviceCollector.collect(), SystemHealth: systemHealth.collect(), Update: readUpdateStatus(), Timestamp: time.Now(), } + metrics.MissingDisks = diskCollector.missingDisks() + metrics.Monitor = monitor.snapshot(store) + metrics.UPSShutdown = store.UPSShutdownSettings() metrics.Activity = activity.collect(metrics.Guests) decorateNetworkInterfaces(&metrics.Network, metrics.Guests) metrics.Alerts = evaluateAlerts(metrics, store.Thresholds()) @@ -115,8 +121,13 @@ func routes(collector *cpuCollector, store *Store) http.Handler { services := newServiceCollector(store) systemHealth := newSystemHealthCollector() activity := newActivityCollector() + monitor := &monitorTracker{} + go runUPSShutdownController(store, ups, guests) collect := func() (dashboardMetrics, error) { - return collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, store) + started := time.Now() + metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, store, monitor) + monitor.record(started, err) + return metrics, err } go runHistoryCollector(collect, store) go runEmailNotifier(collect, store) @@ -219,6 +230,32 @@ func routes(collector *cpuCollector, store *Store) http.Handler { settings := store.EmailSettings(true) writeJSON(w, map[string]bool{"sent": true}, sendAlertEmail(settings, nil, true)) }) + mux.HandleFunc("/api/settings/ups-shutdown", func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + writeJSON(w, store.UPSShutdownSettings(), nil) + case http.MethodPut: + var settings UPSShutdownSettings + if json.NewDecoder(io.LimitReader(r.Body, 16<<10)).Decode(&settings) != nil { + http.Error(w, "Некорректные настройки", http.StatusBadRequest) + return + } + if err := store.SaveUPSShutdownSettings(settings); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, settings, nil) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + }) + mux.HandleFunc("/api/ups/shutdown-dry-run", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + writeJSON(w, buildShutdownDryRun(store.UPSShutdownSettings(), guests.collect()), nil) + }) mux.HandleFunc("/api/history", func(w http.ResponseWriter, r *http.Request) { hours, _ := strconv.Atoi(r.URL.Query().Get("hours")) points, err := store.History(hours) @@ -347,6 +384,7 @@ footer{color:#657581;font-size:12px;margin-top:17px} .services .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.system-health .fill{background:linear-gradient(90deg,#6ba8ff,#9d8cff)}.service-form{display:grid;grid-template-columns:1fr 1fr;gap:12px}.field{display:grid;gap:6px;min-width:0}.field>span{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.07em}.field small{color:var(--muted);font-size:11px}.field input,.field select{width:100%;min-width:0;padding:11px 12px;border:1px solid var(--line);border-radius:9px;background:var(--bg);color:var(--text)}.check-field{display:flex;align-items:center;gap:10px;padding:12px;border:1px solid var(--line);border-radius:9px}.check-field input{width:18px;height:18px;accent-color:var(--green)}.service-actions{display:flex;flex-wrap:wrap;gap:7px;margin-top:10px}.service-actions button,.open-address{padding:7px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);cursor:pointer;text-decoration:none}.status-up{color:var(--green)}.status-down{color:#ff665c}.issue-message{overflow-wrap:anywhere}.full-span{grid-column:1/-1}.endpoint-list{display:grid;gap:10px;margin-top:16px}.endpoint{padding:14px;border:1px solid var(--line);border-radius:12px;background:rgba(100,130,150,.04)}.endpoint-head{display:flex;align-items:flex-start;justify-content:space-between;gap:12px}.endpoint-address{color:var(--muted);font:12px ui-monospace,monospace;overflow-wrap:anywhere}.certificate{margin-top:11px;padding-top:11px;border-top:1px solid var(--line);color:var(--muted);font-size:12px}.certificate b{color:var(--text)}.maintenance-grid .detail{min-width:0}.maintenance-grid .detail strong{font-size:clamp(15px,2.2vw,21px);overflow-wrap:anywhere;word-break:break-word}.system-tabs{display:flex;gap:6px;padding:12px 26px;border-bottom:1px solid var(--line)}.system-panel[hidden]{display:none} .snapshots .fill{background:linear-gradient(90deg,#9d8cff,#ff78b8)}.tasks .fill{background:linear-gradient(90deg,#43c6d9,#6ba8ff)}.homelab-map .fill{background:linear-gradient(90deg,#55d58a,#9d8cff)}.map-board{display:grid;grid-template-columns:170px 1fr 1fr 1fr;gap:14px;padding:18px;min-width:680px}.map-column{display:grid;align-content:start;gap:9px}.map-column-title{color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em}.map-node{padding:12px;border:1px solid var(--line);border-radius:12px;background:var(--panel)}.map-node.good{border-color:#2d6245}.map-node.bad{border-color:#71362f}.map-node.warn{border-color:#6d4c2c}.map-node strong{display:block}.map-node small{display:block;color:var(--muted);margin-top:3px}.map-scroll{overflow-x:auto}.backup-check{margin-top:9px;color:var(--muted);font-size:12px}.task-filter{display:flex;gap:6px;padding:12px 16px;overflow-x:auto} .topology-board{padding:18px;min-width:620px}.topology-host{margin-bottom:14px}.topology-grid{display:grid;grid-template-columns:repeat(2,minmax(270px,1fr));gap:12px}.topology-group{padding:14px;border:1px solid var(--line);border-radius:15px;background:rgba(100,130,150,.04)}.topology-group-head{display:flex;justify-content:space-between;gap:10px;padding-bottom:10px;border-bottom:1px solid var(--line)}.topology-section{margin-top:11px}.topology-section-title{color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.08em;margin-bottom:6px}.topology-items{display:flex;flex-wrap:wrap;gap:6px}.topology-chip{padding:6px 9px;border:1px solid var(--line);border-radius:9px;background:var(--panel);font-size:12px}.topology-chip.good{border-color:#2d6245}.topology-chip.bad{border-color:#71362f}.topology-unassigned{margin-top:14px}@media(max-width:700px){.topology-grid{grid-template-columns:1fr}} +.monitor .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.shutdown-plan{display:grid;gap:8px;margin-top:12px}.shutdown-step{padding:10px 12px;border:1px solid var(--line);border-radius:10px}.infra-status{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:14px 0}.infra-status>div{padding:11px;border:1px solid var(--line);border-radius:10px}.infra-status b{display:block;margin-top:4px}@media(max-width:700px){.infra-status{grid-template-columns:1fr 1fr}}
@@ -394,6 +432,10 @@ footer{color:#657581;font-size:12px;margin-top:17px}
Состояние—
Автономность—
Нажмите, чтобы посмотреть подробности →
+ -
Поиск ZFS-пулов…
+
ARC
Размер—
Цель—
Hit ratio—
Datasets—
Поиск ZFS-пулов…
Заполнение каждого ZFS-пулаОтдельно по каждому пулу
@@ -500,11 +543,14 @@ footer{color:#657581;font-size:12px;margin-top:17px}
Состояние—
Заряд—
Автономность—
Нагрузка—
Входное напряжение—
Выходное напряжение—
Частота—
Батарея—
Температура—
+
Связь с UPS—
Дата батареи—
Возраст батареи—
Источник данных
Не настроен
Заряд и нагрузка UPSПоследние 7 дней
+
Dashboard health

Самоконтроль мониторинга

Последний успешный сбор—
Длительность цикла—
Последняя запись истории—
Свободно для базы—
Systemd watchdog—
Ошибка—
 +
Virtualization details

VM и LXC

Получение списка гостей…
@@ -547,18 +593,19 @@ footer{color:#657581;font-size:12px;margin-top:17px}
Dashboard

Настройки

-
+
Тема оформления
Видимые карточки
- +
Порядок карточек
Перетаскивайте строки или используйте стрелки.
+