From 8241fba6f44ceb88f82c8f85014cdb9aaf4c7466 Mon Sep 17 00:00:00 2001 From: Maxim Date: Mon, 3 Aug 2026 18:02:13 +0300 Subject: [PATCH] Add IO attribution anomaly detection and alert correlation --- README.md | 5 ++ activity.go | 2 +- alert_events.go | 4 +- alert_events_test.go | 2 +- alerts.go | 78 ++++++++++++++++-- disks.go | 92 +++++++++++---------- history.go | 8 ++ io_attribution.go | 178 +++++++++++++++++++++++++++++++++++++++++ io_attribution_test.go | 33 ++++++++ store.go | 25 ++++++ trends.go | 107 +++++++++++++++++++++++++ web.go | 27 +++++-- 12 files changed, 498 insertions(+), 63 deletions(-) create mode 100644 io_attribution.go create mode 100644 io_attribution_test.go create mode 100644 trends.go diff --git a/README.md b/README.md index 7929b41..9f5e0da 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,11 @@ ss -lntp | grep 9105 - изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков; - realtime read/write IOPS, MB/s, latency, utilization и глубину очереди каждого диска; - недельную историю IOPS/utilization и алерты при устойчивом I/O-давлении; +- вероятный источник I/O: VM/LXC, процесс хоста или активная backup/snapshot/migration-задача; +- отдельные пороги HDD, SATA SSD и NVMe с исключениями по имени или серийному номеру; +- недельные baseline-аномалии IOPS, latency, температуры, RAM, ZFS и длительности backup; +- корреляцию связанных событий в одну причинно-следственную цепочку; +- время первого появления и последнего наблюдения каждого уведомления; - автоматический короткий SMART self-test раз в неделю для дисков без свежего теста; - datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS; - доступность шлюза, DNS и интернета, packet loss, latency и внешний IP; diff --git a/activity.go b/activity.go index 8c5b8aa..b5c67c9 100644 --- a/activity.go +++ b/activity.go @@ -57,7 +57,7 @@ func (c *activityCollector) collect(guests GuestsMetrics) ProxmoxActivity { value := c.value c.mu.RUnlock() c.mu.Lock() - if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 10*time.Minute { + if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 15*time.Second { c.refreshing = true go c.refreshWithGuests(guests) } diff --git a/alert_events.go b/alert_events.go index 76af095..f4204a3 100644 --- a/alert_events.go +++ b/alert_events.go @@ -74,7 +74,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) { if err = tx.Commit(); err != nil { return alerts, err } - rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`) + rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`) if err != nil { return alerts, err } @@ -82,7 +82,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) { var visible []Alert for rows.Next() { var a Alert - if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message); err != nil { + if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message, &a.FirstSeen, &a.LastSeen); err != nil { return alerts, err } visible = append(visible, a) diff --git a/alert_events_test.go b/alert_events_test.go index 8837454..9a57766 100644 --- a/alert_events_test.go +++ b/alert_events_test.go @@ -10,7 +10,7 @@ func TestAlertLifecycle(t *testing.T) { defer store.Close() alert := Alert{ID: "disk", Severity: "warning", Source: "Диски", Title: "SMART", Message: "Проверьте диск"} visible, err := store.SyncAlerts([]Alert{alert}) - if err != nil || len(visible) != 1 || visible[0].EventID == 0 { + if err != nil || len(visible) != 1 || visible[0].EventID == 0 || visible[0].FirstSeen == 0 || visible[0].LastSeen == 0 { t.Fatalf("unexpected new alert: %+v %v", visible, err) } firstID := visible[0].EventID diff --git a/alerts.go b/alerts.go index 617f400..e772d2b 100644 --- a/alerts.go +++ b/alerts.go @@ -8,12 +8,14 @@ import ( ) type Alert struct { - EventID int64 `json:"eventId,omitempty"` - ID string `json:"id"` - Severity string `json:"severity"` - Source string `json:"source"` - Title string `json:"title"` - Message string `json:"message"` + EventID int64 `json:"eventId,omitempty"` + ID string `json:"id"` + Severity string `json:"severity"` + Source string `json:"source"` + Title string `json:"title"` + Message string `json:"message"` + FirstSeen int64 `json:"firstSeen,omitempty"` + LastSeen int64 `json:"lastSeen,omitempty"` } func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert { @@ -61,6 +63,14 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent)) } for _, disk := range metrics.Disks { + diskExcluded := false + for _, value := range strings.Split(thresholds.DiskIOExcluded, ",") { + value = strings.TrimSpace(value) + if value != "" && (strings.EqualFold(value, disk.Name) || strings.EqualFold(value, disk.Serial)) { + diskExcluded = true + break + } + } if disk.SMARTStatus == "Ошибка" { add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска.")) } else if disk.SMARTStatus == "Требует внимания" { @@ -78,12 +88,16 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler if len(disk.CounterChanges) > 0 { add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · ")) } - if disk.IOPressureSeconds >= 60 { + if !diskExcluded && disk.IOPressureSeconds >= thresholds.DiskIODuration { severity := "warning" if disk.Utilization >= 98 || disk.LatencyMs >= 100 { severity = "critical" } - add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("Нагрузка держится %.0f сек.: %.0f IOPS, %.1f MB/s, latency %.1f ms, util %.1f%%, очередь %.1f.", disk.IOPressureSeconds, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.Utilization, disk.AverageQueue)) + cause := disk.IOCause + if cause == "" { + cause = "источник пока не определён" + } + add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("%s занят на %.1f%%; вероятная причина: %s (уверенность %d%%). %.0f IOPS, %.1f MB/s, latency %.1f ms, очередь %.1f.", disk.Name, disk.Utilization, cause, disk.IOConfidence, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.AverageQueue)) } if disk.SelfTestNeverRun { add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.") @@ -266,6 +280,9 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status)) } } + for _, trend := range metrics.Trends { + add(trend.ID, trend.Severity, trend.Source, trend.Title, trend.Message) + } priority := map[string]int{"critical": 0, "warning": 1, "info": 2} sort.SliceStable(alerts, func(i, j int) bool { @@ -277,6 +294,51 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler return alerts } +func correlateAlerts(metrics dashboardMetrics, alerts []Alert) []Alert { + pressureAlerts := map[string]bool{} + for _, a := range alerts { + if strings.HasPrefix(a.ID, "disk-io-pressure-") { + pressureAlerts[strings.TrimPrefix(a.ID, "disk-io-pressure-")] = true + } + } + var pressure []DiskMetrics + for _, d := range metrics.Disks { + if pressureAlerts[d.Name] && d.IOCause != "" { + pressure = append(pressure, d) + } + } + if len(pressure) == 0 { + return alerts + } + var down []string + for _, service := range metrics.Services.Services { + for _, endpoint := range service.Endpoints { + if !endpoint.Up { + down = append(down, service.Name) + break + } + } + } + d := pressure[0] + chain := d.IOCause + " → " + d.Name + fmt.Sprintf(" util %.1f%% → latency %.1f ms", d.Utilization, d.LatencyMs) + if len(down) > 0 { + chain += " → недоступны/медленны сервисы: " + strings.Join(down, ", ") + } + severity := "warning" + if len(down) > 0 || d.Utilization >= 98 { + severity = "critical" + } + correlated := Alert{ID: "correlation-io-" + d.Name, Severity: severity, Source: "Корреляция", Title: "Обнаружена связанная цепочка I/O-событий", Message: chain + fmt.Sprintf(". Уверенность причины %d%%.", d.IOConfidence)} + filtered := []Alert{correlated} + for _, a := range alerts { + if strings.HasPrefix(a.ID, "disk-io-pressure-") || strings.HasPrefix(a.ID, "service-") { + continue + } + filtered = append(filtered, a) + } + return filtered +} + func joinReasons(reasons []string, fallback string) string { if len(reasons) == 0 { return fallback diff --git a/disks.go b/disks.go index f7593b5..c63446a 100644 --- a/disks.go +++ b/disks.go @@ -13,42 +13,46 @@ import ( ) type DiskMetrics struct { - Name string `json:"name"` - Path string `json:"path"` - Model string `json:"model"` - Serial string `json:"serial"` - Type string `json:"type"` - Protocol string `json:"protocol"` - SizeBytes uint64 `json:"sizeBytes"` - UsedBytes *uint64 `json:"usedBytes"` - UsagePercent *float64 `json:"usagePercent"` - SMARTStatus string `json:"smartStatus"` - SMARTAvailable bool `json:"smartAvailable"` - AttentionReasons []string `json:"attentionReasons"` - Temperature *float64 `json:"temperatureCelsius"` - PowerOnHours *uint64 `json:"powerOnHours"` - WearUsedPercent *float64 `json:"wearUsedPercent"` - Reallocated uint64 `json:"reallocated"` - Pending uint64 `json:"pending"` - Uncorrectable uint64 `json:"uncorrectable"` - CRCErrors uint64 `json:"crcErrors"` - MediaErrors uint64 `json:"mediaErrors"` - CounterChanges []string `json:"counterChanges"` - LastSelfTest string `json:"lastSelfTest"` - LastSelfTestAt string `json:"lastSelfTestAt"` - SelfTestNeverRun bool `json:"selfTestNeverRun"` - ShortTestStarted bool `json:"shortTestStarted"` - ReadIOPS float64 `json:"readIops"` - WriteIOPS float64 `json:"writeIops"` - ReadBytesPerSec float64 `json:"readBytesPerSec"` - WriteBytesPerSec float64 `json:"writeBytesPerSec"` - ReadLatencyMs float64 `json:"readLatencyMs"` - WriteLatencyMs float64 `json:"writeLatencyMs"` - LatencyMs float64 `json:"latencyMs"` - Utilization float64 `json:"utilizationPercent"` - QueueDepth uint64 `json:"queueDepth"` - AverageQueue float64 `json:"averageQueue"` - IOPressureSeconds float64 `json:"ioPressureSeconds"` + Name string `json:"name"` + Path string `json:"path"` + Model string `json:"model"` + Serial string `json:"serial"` + Type string `json:"type"` + Protocol string `json:"protocol"` + SizeBytes uint64 `json:"sizeBytes"` + UsedBytes *uint64 `json:"usedBytes"` + UsagePercent *float64 `json:"usagePercent"` + SMARTStatus string `json:"smartStatus"` + SMARTAvailable bool `json:"smartAvailable"` + AttentionReasons []string `json:"attentionReasons"` + Temperature *float64 `json:"temperatureCelsius"` + PowerOnHours *uint64 `json:"powerOnHours"` + WearUsedPercent *float64 `json:"wearUsedPercent"` + Reallocated uint64 `json:"reallocated"` + Pending uint64 `json:"pending"` + Uncorrectable uint64 `json:"uncorrectable"` + CRCErrors uint64 `json:"crcErrors"` + MediaErrors uint64 `json:"mediaErrors"` + CounterChanges []string `json:"counterChanges"` + LastSelfTest string `json:"lastSelfTest"` + LastSelfTestAt string `json:"lastSelfTestAt"` + SelfTestNeverRun bool `json:"selfTestNeverRun"` + ShortTestStarted bool `json:"shortTestStarted"` + ReadIOPS float64 `json:"readIops"` + WriteIOPS float64 `json:"writeIops"` + ReadBytesPerSec float64 `json:"readBytesPerSec"` + WriteBytesPerSec float64 `json:"writeBytesPerSec"` + ReadLatencyMs float64 `json:"readLatencyMs"` + WriteLatencyMs float64 `json:"writeLatencyMs"` + LatencyMs float64 `json:"latencyMs"` + Utilization float64 `json:"utilizationPercent"` + QueueDepth uint64 `json:"queueDepth"` + AverageQueue float64 `json:"averageQueue"` + IOPressureSeconds float64 `json:"ioPressureSeconds"` + IOCause string `json:"ioCause"` + IOCauseDetail string `json:"ioCauseDetail"` + IOConfidence int `json:"ioConfidence"` + TopConsumers []IOConsumer `json:"topConsumers"` } type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 } @@ -65,11 +69,11 @@ type diskCollector struct { ioPressureSince map[string]time.Time } -func (c *diskCollector) collect() []DiskMetrics { +func (c *diskCollector) collect(thresholds AlertThresholds) []DiskMetrics { c.mu.Lock() defer c.mu.Unlock() if time.Since(c.updatedAt) < time.Minute && c.cached != nil { - c.applyDiskIO(c.cached) + c.applyDiskIO(c.cached, thresholds) return c.cached } current := readPhysicalDisks() @@ -104,12 +108,12 @@ func (c *diskCollector) collect() []DiskMetrics { } } c.cached = current - c.applyDiskIO(c.cached) + c.applyDiskIO(c.cached, thresholds) c.updatedAt = time.Now() return c.cached } -func (c *diskCollector) applyDiskIO(disks []DiskMetrics) { +func (c *diskCollector) applyDiskIO(disks []DiskMetrics, thresholds AlertThresholds) { now := time.Now() snapshots := readDiskIOStats("/proc/diskstats") if c.ioPrevious == nil { @@ -144,11 +148,13 @@ func (c *diskCollector) applyDiskIO(disks []DiskMetrics) { } disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100) disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000) - latencyLimit := 20.0 + utilLimit, latencyLimit, queueLimit := thresholds.DiskSSDUtil, thresholds.DiskSSDLatency, thresholds.DiskSSDQueue if disk.Type == "HDD" { - latencyLimit = 50 + utilLimit, latencyLimit, queueLimit = thresholds.DiskHDDUtil, thresholds.DiskHDDLatency, thresholds.DiskHDDQueue + } else if strings.Contains(disk.Type, "NVMe") { + utilLimit, latencyLimit, queueLimit = thresholds.DiskNVMeUtil, thresholds.DiskNVMeLatency, thresholds.DiskNVMeQueue } - pressured := disk.Utilization >= 90 || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= 4 + pressured := disk.Utilization >= utilLimit || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= queueLimit if pressured { if c.ioPressureSince[disk.Name].IsZero() { c.ioPressureSince[disk.Name] = now diff --git a/history.go b/history.go index b46d168..2f31296 100644 --- a/history.go +++ b/history.go @@ -47,6 +47,8 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) { iops := d.ReadIOPS + d.WriteIOPS util := d.Utilization s.AddEntityHistory("disk-io", d.Name+" · "+d.Model, ts, &iops, &util) + latency := d.LatencyMs + s.AddEntityHistory("disk-latency", d.Name+" · "+d.Model, ts, &latency, nil) if d.Temperature == nil { continue } @@ -56,6 +58,12 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) { } s.AddEntityHistory("disk", d.Name+" · "+d.Model, ts, d.Temperature, wear) } + for _, task := range metrics.Activity.Tasks { + if task.Type == "vzdump" && task.Duration > 0 && task.EndTime >= time.Now().Add(-15*time.Minute).Unix() { + duration := float64(task.Duration) + s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil) + } + } s.PruneEntityHistory() } diff --git a/io_attribution.go b/io_attribution.go new file mode 100644 index 0000000..bf25929 --- /dev/null +++ b/io_attribution.go @@ -0,0 +1,178 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type IOConsumer struct { + Kind string `json:"kind"` + Name string `json:"name"` + VMID int `json:"vmid,omitempty"` + ReadBytesPerSec float64 `json:"readBytesPerSec"` + WriteBytesPerSec float64 `json:"writeBytesPerSec"` + SharePercent float64 `json:"sharePercent"` +} +type processIOSnapshot struct { + read, write uint64 + name, cmdline, cgroup string +} +type ioAttributionCollector struct { + mu sync.Mutex + previous map[int]processIOSnapshot + updatedAt time.Time + cached []IOConsumer +} + +var qemuIDPattern = regexp.MustCompile(`(?:^|\s)-id\s+(\d+)`) +var lxcIDPattern = regexp.MustCompile(`(?:lxc(?:\.payload)?[./-]|pve-container@)(\d+)`) + +func (c *ioAttributionCollector) collect(guests GuestsMetrics) []IOConsumer { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + current := readProcessIO() + elapsed := now.Sub(c.updatedAt).Seconds() + if c.previous == nil || elapsed <= 0 { + c.previous = current + c.updatedAt = now + return nil + } + names := map[int]string{} + for _, g := range guests.Guests { + names[g.VMID] = g.Name + } + grouped := map[string]*IOConsumer{} + var total float64 + for pid, snapshot := range current { + old, ok := c.previous[pid] + if !ok || snapshot.read < old.read || snapshot.write < old.write { + continue + } + read := float64(snapshot.read-old.read) / elapsed + write := float64(snapshot.write-old.write) / elapsed + if read+write < 1024 { + continue + } + kind, name, vmid := classifyIOProcess(snapshot, names) + key := kind + ":" + strconv.Itoa(vmid) + ":" + name + consumer := grouped[key] + if consumer == nil { + consumer = &IOConsumer{Kind: kind, Name: name, VMID: vmid} + grouped[key] = consumer + } + consumer.ReadBytesPerSec += read + consumer.WriteBytesPerSec += write + total += read + write + } + var result []IOConsumer + for _, v := range grouped { + if total > 0 { + v.SharePercent = (v.ReadBytesPerSec + v.WriteBytesPerSec) / total * 100 + } + result = append(result, *v) + } + sort.Slice(result, func(i, j int) bool { + return result[i].ReadBytesPerSec+result[i].WriteBytesPerSec > result[j].ReadBytesPerSec+result[j].WriteBytesPerSec + }) + if len(result) > 8 { + result = result[:8] + } + c.previous = current + c.updatedAt = now + c.cached = result + return result +} + +func classifyIOProcess(p processIOSnapshot, names map[int]string) (kind, name string, vmid int) { + if match := qemuIDPattern.FindStringSubmatch(p.cmdline); len(match) > 1 { + vmid, _ = strconv.Atoi(match[1]) + return "VM", firstNonEmpty(names[vmid], "VM "+match[1]), vmid + } + if match := lxcIDPattern.FindStringSubmatch(p.cgroup); len(match) > 1 { + vmid, _ = strconv.Atoi(match[1]) + return "LXC", firstNonEmpty(names[vmid], "LXC "+match[1]), vmid + } + return "Процесс", firstNonEmpty(p.name, "PID"), 0 +} + +func readProcessIO() map[int]processIOSnapshot { + result := map[int]processIOSnapshot{} + entries, _ := filepath.Glob("/proc/[0-9]*/io") + for _, path := range entries { + pid, _ := strconv.Atoi(filepath.Base(filepath.Dir(path))) + data, err := os.ReadFile(path) + if err != nil { + continue + } + var item processIOSnapshot + for _, line := range strings.Split(string(data), "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + n, _ := strconv.ParseUint(strings.TrimSpace(value), 10, 64) + if key == "read_bytes" { + item.read = n + } else if key == "write_bytes" { + item.write = n + } + } + comm, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "comm")) + cmd, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cmdline")) + cg, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cgroup")) + item.name = strings.TrimSpace(string(comm)) + item.cmdline = strings.ReplaceAll(string(cmd), "\x00", " ") + item.cgroup = string(cg) + result[pid] = item + } + return result +} + +func attributeDiskIO(metrics *dashboardMetrics, consumers []IOConsumer) { + active := activeIOOperation(metrics.Activity.Tasks) + for i := range metrics.Disks { + disk := &metrics.Disks[i] + disk.TopConsumers = append([]IOConsumer(nil), consumers...) + if active != "" { + disk.IOCause = active + disk.IOCauseDetail = "Активная задача Proxmox совпадает по времени с дисковой нагрузкой" + disk.IOConfidence = 75 + } else if len(consumers) > 0 { + top := consumers[0] + disk.IOCause = top.Kind + " · " + top.Name + disk.IOCauseDetail = "Крупнейший потребитель host I/O: " + strconv.FormatFloat(top.SharePercent, 'f', 1, 64) + "%" + disk.IOConfidence = 55 + } + } +} + +func activeIOOperation(tasks []ProxmoxTask) string { + for _, task := range tasks { + if task.EndTime != 0 && task.Status != "" { + continue + } + label := "" + switch task.Type { + case "vzdump": + label = "Backup" + case "qmigrate": + label = "Миграция" + case "qmsnapshot", "vzsnapshot": + label = "Snapshot" + } + if label != "" { + if task.ID != "" { + label += " VM/LXC " + task.ID + } + return label + } + } + return "" +} diff --git a/io_attribution_test.go b/io_attribution_test.go new file mode 100644 index 0000000..6cc8109 --- /dev/null +++ b/io_attribution_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "strings" + "testing" +) + +func TestClassifyQEMUAndLXCIO(t *testing.T) { + kind, name, vmid := classifyIOProcess(processIOSnapshot{name: "qemu", cmdline: "qemu-system-x86_64 -id 105 -name guest=test"}, map[int]string{105: "Gitea"}) + if kind != "VM" || name != "Gitea" || vmid != 105 { + t.Fatalf("unexpected qemu attribution: %s %s %d", kind, name, vmid) + } + kind, name, vmid = classifyIOProcess(processIOSnapshot{name: "worker", cgroup: "0::/lxc.payload.220/ns"}, map[int]string{220: "db"}) + if kind != "LXC" || name != "db" || vmid != 220 { + t.Fatalf("unexpected lxc attribution: %s %s %d", kind, name, vmid) + } +} + +func TestActiveIOOperation(t *testing.T) { + value := activeIOOperation([]ProxmoxTask{{Type: "vzdump", ID: "105", Status: "", EndTime: 0}}) + if !strings.Contains(value, "Backup") || !strings.Contains(value, "105") { + t.Fatalf("unexpected operation: %s", value) + } +} + +func TestCorrelateAlerts(t *testing.T) { + metrics := dashboardMetrics{Disks: []DiskMetrics{{Name: "sda", IOCause: "Backup VM/LXC 105", IOConfidence: 90, Utilization: 98, LatencyMs: 80}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Up: false}}}}}} + alerts := []Alert{{ID: "disk-io-pressure-sda", Severity: "critical"}, {ID: "service-1-public", Severity: "critical"}, {ID: "other", Severity: "warning"}} + result := correlateAlerts(metrics, alerts) + if len(result) != 2 || result[0].Source != "Корреляция" || !strings.Contains(result[0].Message, "Gitea") { + t.Fatalf("unexpected correlation: %+v", result) + } +} diff --git a/store.go b/store.go index 1881810..1b0a378 100644 --- a/store.go +++ b/store.go @@ -31,6 +31,17 @@ type AlertThresholds struct { NVMeTempWarning float64 `json:"nvmeTempWarning"` UPSChargeCritical float64 `json:"upsChargeCritical"` BackupMaxAgeHours float64 `json:"backupMaxAgeHours"` + DiskHDDUtil float64 `json:"diskHddUtil"` + DiskHDDLatency float64 `json:"diskHddLatency"` + DiskHDDQueue float64 `json:"diskHddQueue"` + DiskSSDUtil float64 `json:"diskSsdUtil"` + DiskSSDLatency float64 `json:"diskSsdLatency"` + DiskSSDQueue float64 `json:"diskSsdQueue"` + DiskNVMeUtil float64 `json:"diskNvmeUtil"` + DiskNVMeLatency float64 `json:"diskNvmeLatency"` + DiskNVMeQueue float64 `json:"diskNvmeQueue"` + DiskIODuration float64 `json:"diskIoDuration"` + DiskIOExcluded string `json:"diskIoExcluded"` } func defaultThresholds() AlertThresholds { @@ -40,6 +51,7 @@ func defaultThresholds() AlertThresholds { StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90, DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30, BackupMaxAgeHours: 48, + DiskHDDUtil: 90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60, } } @@ -285,6 +297,19 @@ func validateThresholds(t AlertThresholds) error { if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 { return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов") } + for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} { + if value < 1 || value > 100 { + return fmt.Errorf("пороги utilization должны быть от 1 до 100") + } + } + for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} { + if value < 0 { + return fmt.Errorf("пороги latency и очереди не могут быть отрицательными") + } + } + if t.DiskIODuration < 5 || t.DiskIODuration > 3600 { + return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд") + } return nil } diff --git a/trends.go b/trends.go new file mode 100644 index 0000000..f569673 --- /dev/null +++ b/trends.go @@ -0,0 +1,107 @@ +package main + +import ( + "database/sql" + "fmt" + "sync" + "time" +) + +type entityAverage struct { + Value1, Value2 float64 + Count int +} +type TrendAnomaly struct { + ID string `json:"id"` + Severity string `json:"severity"` + Source string `json:"source"` + Title string `json:"title"` + Message string `json:"message"` +} +type trendCollector struct { + mu sync.Mutex + updatedAt time.Time + cached []TrendAnomaly +} + +func (s *Store) entityAverages(kind string) map[string]entityAverage { + return s.entityAveragesSince(kind, 7*24*time.Hour) +} +func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage { + rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix()) + if err != nil { + return nil + } + defer rows.Close() + result := map[string]entityAverage{} + for rows.Next() { + var name string + var v1, v2 sql.NullFloat64 + var count int + if rows.Scan(&name, &v1, &v2, &count) == nil { + result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count} + } + } + return result +} + +func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly { + c.mu.Lock() + defer c.mu.Unlock() + if time.Since(c.updatedAt) < time.Minute { + return c.cached + } + var out []TrendAnomaly + iops := store.entityAverages("disk-io") + latency := store.entityAverages("disk-latency") + latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour) + temps := store.entityAverages("disk") + for _, d := range metrics.Disks { + name := d.Name + " · " + d.Model + currentIOPS := d.ReadIOPS + d.WriteIOPS + if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 { + out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)}) + } + if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 { + out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)}) + } + if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 { + out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)}) + } + if d.Temperature != nil { + if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 { + out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)}) + } + } + } + guestAvg := store.entityAverages("guest") + for _, g := range metrics.Guests.Guests { + if g.MaxMemory == 0 { + continue + } + name := fmt.Sprintf("%d · %s", g.VMID, g.Name) + current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100 + if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 { + out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)}) + } + } + zfsAverage := store.entityAverages("zfs") + for _, pool := range metrics.ZFS.Pools { + if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 { + out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)}) + } + } + backupAverage := store.entityAverages("backup-duration") + for _, task := range metrics.Activity.Tasks { + if task.Type != "vzdump" || task.Duration <= 0 { + continue + } + name := "VMID " + task.ID + if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 { + out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)}) + } + } + c.cached = out + c.updatedAt = time.Now() + return out +} diff --git a/web.go b/web.go index 21f90dd..3f7ea3e 100644 --- a/web.go +++ b/web.go @@ -29,10 +29,11 @@ type dashboardMetrics struct { Activity ProxmoxActivity `json:"activity"` Monitor MonitorHealth `json:"monitor"` UPSShutdown UPSShutdownSettings `json:"upsShutdown"` + Trends []TrendAnomaly `json:"trends"` Timestamp time.Time `json:"timestamp"` } -func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) { +func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, ioAttribution *ioAttributionCollector, trends *trendCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) { cpu, err := cpuCollector.collect() if err != nil { return dashboardMetrics{}, err @@ -49,9 +50,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle if err != nil { return dashboardMetrics{}, err } + thresholds := store.Thresholds() metrics := dashboardMetrics{ CPU: cpu, Memory: memory, Storage: storage, - Disks: diskCollector.collect(), Network: network, + Disks: diskCollector.collect(thresholds), Network: network, ZFS: zfsCollector.collect(), UPS: upsCollector.collect(), Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Services: serviceCollector.collect(), SystemHealth: systemHealth.collect(), Update: readUpdateStatus(), Timestamp: time.Now(), } @@ -59,8 +61,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle metrics.Monitor = monitor.snapshot(store) metrics.UPSShutdown = store.UPSShutdownSettings() metrics.Activity = activity.collect(metrics.Guests) + attributeDiskIO(&metrics, ioAttribution.collect(metrics.Guests)) + metrics.Trends = trends.collect(metrics, store) decorateNetworkInterfaces(&metrics.Network, metrics.Guests) - metrics.Alerts = evaluateAlerts(metrics, store.Thresholds()) + metrics.Alerts = correlateAlerts(metrics, evaluateAlerts(metrics, thresholds)) metrics.Alerts, err = store.SyncAlerts(metrics.Alerts) if err != nil { return dashboardMetrics{}, err @@ -121,12 +125,14 @@ func routes(collector *cpuCollector, store *Store) http.Handler { services := newServiceCollector(store) systemHealth := newSystemHealthCollector() activity := newActivityCollector() + ioAttribution := &ioAttributionCollector{} + trends := &trendCollector{} monitor := &monitorTracker{} go runCollectionWatchdog(monitor) go runUPSShutdownController(store, ups, guests) collect := func() (dashboardMetrics, error) { started := time.Now() - metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, store, monitor) + metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, ioAttribution, trends, store, monitor) monitor.record(started, err) return metrics, err } @@ -517,7 +523,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
Используется—
Свободно—
Общий объём—
Заполнение—
Точка монтирования/
-
Поиск накопителей…
+
Поиск накопителей…
Расшифровка источников I/O
Накопление данных…
Температура и I/O физических дисковОтдельно по каждому диску
Температура
IOPS
Utilization
@@ -617,6 +623,10 @@ footer{color:#657581;font-size:12px;margin-top:17px} + + + +
Настройки хранятся на сервере и применяются для всех браузеров.
@@ -663,6 +673,7 @@ let currentServiceStatuses=[]; function endpointHTML(v){const tls=v.tlsDays==null?'Нет TLS':v.tlsDays<0?'Сертификат истёк':v.tlsDays+' дн. до окончания',cert=v.tlsSubject?'
Сертификат: '+safe(v.tlsSubject)+'
Домены: '+safe((v.tlsDnsNames||[]).join(', ')||'не указаны')+'
Издатель: '+safe(v.tlsIssuer||'—')+'
Для этого адреса: '+(v.tlsHostnameValid?'подходит':'не подходит')+'
':'';return '
'+safe(v.label)+'
'+safe(v.address)+'
'+(v.up?'Доступен':'Недоступен')+'
Отклик'+fixed(v.latencyMs)+' ms
HTTP'+(v.statusCode||'—')+'
TLS'+tls+'
Uptime · сутки'+fixed(v.uptimeDay)+'%
Uptime · неделя'+fixed(v.uptimeWeek)+'%
'+cert+(v.error?'
'+safe(v.error)+'
':'')+'
'} function renderServices(svc){const items=svc.services||[];currentServiceStatuses=items;const avg=items.length?items.reduce((a,v)=>a+v.uptimeDay,0)/items.length:0,live=items.filter(v=>v.up),latency=live.length?live.reduce((a,v)=>a+v.latencyMs,0)/live.length:0;el('serviceUp').textContent=svc.up||0;el('serviceTotal').textContent=' / '+(svc.total||0);el('serviceBar').style.width=(svc.total?svc.up/svc.total*100:0)+'%';el('serviceUptime').textContent=svc.total?fixed(avg)+'%':'—';el('serviceLatency').textContent=svc.total?fixed(latency)+' ms':'—';el('serviceList').innerHTML=items.length?items.map(v=>'
'+safe(v.name)+'
'+(v.endpoints||[]).length+' '+((v.endpoints||[]).length===1?'адрес':'адреса')+' проверяются независимо
'+(v.up?'Всё доступно':'Есть проблема')+'
'+(v.endpoints||[]).map(endpointHTML).join('')+'
').join(''):'Добавьте первую проверку в настройках.'} function renderNetworkGroups(interfaces){const groups=new Map();interfaces.forEach(v=>{const key=v.vmid?'guest-'+v.vmid:'host',group=groups.get(key)||{vmid:v.vmid,name:v.vmid?(v.guestName||('VMID '+v.vmid)):'Интерфейсы хоста',type:v.guestType||'Host',items:[]};group.items.push(v);groups.set(key,group)});el('networkInterfacesList').innerHTML=[...groups.values()].map(g=>{const primary=g.vmid?g.items.filter(v=>v.name.startsWith('tap')||v.name.startsWith('veth')):g.items,rx=primary.reduce((a,v)=>a+v.receivePerSec,0),tx=primary.reduce((a,v)=>a+v.transmitPerSec,0),errors=g.items.reduce((a,v)=>a+v.receiveErrors+v.transmitErrors+v.receiveDrops+v.transmitDrops,0),down=primary.some(v=>v.kind==='Physical'&&!v.carrier&&(v.master||v.receiveBytes+v.transmitBytes>0));return '
'+safe(g.name)+'
'+(g.vmid?'VMID '+g.vmid+' · ':'')+safe(g.type)+' · '+safe(g.items.map(v=>v.name+' ['+(v.kind||'Interface')+(v.speedMbps?' · '+v.speedMbps+'Mb/s':'')+(v.duplex?' · '+v.duplex:'')+(v.master?' → '+v.master:'')+']').join(', '))+'
'+(down?'Нет линка':errors?'Есть потери':'Без ошибок')+'
Сейчас ↓'+rate(rx)+'
Сейчас ↑'+rate(tx)+'
Получено'+size(primary.reduce((a,v)=>a+v.receiveBytes,0))+'
Передано'+size(primary.reduce((a,v)=>a+v.transmitBytes,0))+'
Ошибки / drops'+errors+'
'}).join('')} +function renderIOAttribution(disks){const active=disks.filter(d=>d.ioCause||(d.topConsumers||[]).length);el('ioAttributionList').innerHTML=active.length?active.map(d=>'
'+safe(d.name)+' · '+safe(d.ioCause||'Источник не определён')+'
'+safe(d.ioCauseDetail||'Собираем статистику процессов')+'
'+(d.ioConfidence||0)+'%
'+(d.topConsumers||[]).slice(0,5).map(v=>''+safe(v.kind)+' · '+safe(v.name)+' · '+fixed(v.sharePercent)+'% · ↓'+rate(v.readBytesPerSec)+' ↑'+rate(v.writeBytesPerSec)+'').join('')+'
').join(''):'Для атрибуции нужно несколько realtime-циклов.'} const taskLabel=type=>({vzdump:'Backup',qmstart:'Запуск VM',qmstop:'Остановка VM',qmshutdown:'Выключение VM',qmsnapshot:'Snapshot VM',vzstart:'Запуск LXC',vzstop:'Остановка LXC',vzsnapshot:'Snapshot LXC',qmigrate:'Миграция',aptupdate:'Обновление пакетов'})[type]||type; function explainWarningBadges(){el('snapshotList').querySelectorAll('.badge.warn').forEach(v=>{v.tabIndex=0;v.dataset.reason='Snapshot старше 30 дней. Проверьте, нужен ли он: старые снимки могут занимать место в storage.'});el('taskList').querySelectorAll('.badge.bad').forEach(v=>{v.tabIndex=0;v.dataset.reason='Задача Proxmox завершилась не со статусом OK. Откройте её журнал в интерфейсе Proxmox для подробностей.'})} let currentTasks=[],currentTaskFilter='all'; @@ -674,7 +685,7 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av lastDashboardEvent=Date.now(); const update=x.update||{},updating=['queued','installing'].includes(update.state);el('updateCurrent').textContent=update.currentVersion||'—';el('updateAvailable').textContent=update.availableVersion||'—';el('updateMode').textContent=update.mode==='auto'?'Автоматически':'Уведомлять';el('updateMessage').textContent=update.message||'Проверка ещё не выполнялась';el('updateChecked').textContent=update.checkedAt?'Последняя проверка: '+new Date(update.checkedAt).toLocaleString('ru-RU'):'Проверка запускается автоматически раз в сутки.';el('installUpdate').disabled=updating;el('installUpdate').textContent=updating?'Обновление выполняется…':'Проверить и установить обновление'; const critical=alerts.filter(a=>a.severity==='critical').length,warnings=alerts.filter(a=>a.severity==='warning').length,infos=alerts.filter(a=>a.severity==='info').length;el('alertCount').textContent=alerts.length;el('alertCritical').textContent=critical;el('alertWarnings').textContent=warnings;el('alertHeadline').textContent=critical?'Требуется срочное внимание':warnings?'Есть предупреждения':infos?'Есть информационные события':'Система в норме';el('alertBar').style.width=(critical?100:warnings?65:infos?30:0)+'%'; - el('alertList').innerHTML=alerts.length?alerts.map(a=>'
'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+'
'+safe(a.title)+'
'+safe(a.message)+'
').join(''):'
Всё в порядкеНовых неподтверждённых событий нет.
'; + el('alertList').innerHTML=alerts.length?alerts.map(a=>'
'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+(a.firstSeen?' · С '+new Date(a.firstSeen*1000).toLocaleString('ru-RU'):'')+'
'+safe(a.title)+'
'+safe(a.message)+'
').join(''):'
Всё в порядкеНовых неподтверждённых событий нет.
'; el('cpuName').textContent=el('cpuDetailName').textContent=c.model;el('cpuUsage').textContent=fixed(c.usagePercent);el('cpuBar').style.width=Math.min(c.usagePercent,100)+'%';el('cpuTemp').textContent=avg===null?'Нет датчика':fixed(avg)+' °C';el('cpuTopology').textContent=c.physicalCores+' / '+c.logicalCpus; el('dCpuUsage').textContent=fixed(c.usagePercent)+'%';el('dSockets').textContent=c.sockets;el('dCores').textContent=c.physicalCores;el('dThreads').textContent=c.logicalCpus;el('dFreq').textContent=(c.frequencyMhz/1000).toFixed(2)+' GHz';el('dUptime').textContent=uptime(c.uptimeSeconds);el('dLoad1').textContent=fixed(c.load1);el('dLoad5').textContent=fixed(c.load5);el('dLoad15').textContent=fixed(c.load15);el('dTemps').innerHTML=c.temperatures.length?c.temperatures.map(t=>''+t.label+''+fixed(t.celsius)+'°C').join(''):'Датчики температуры не найдены'; el('memUsage').textContent=fixed(m.usagePercent);el('memBar').style.width=Math.min(m.usagePercent,100)+'%';el('memUsed').textContent=size(m.usedBytes);el('memAvailable').textContent=size(m.availableBytes);el('dMemUsed').textContent=size(m.usedBytes);el('dMemAvailable').textContent=size(m.availableBytes);el('dMemTotal').textContent=size(m.totalBytes);el('dMemCache').textContent=size(m.cachedBytes);el('dMemBuffers').textContent=size(m.buffersBytes);el('dMemPercent').textContent=fixed(m.usagePercent)+'%';el('dSwapUsed').textContent=size(m.swapUsedBytes);el('dSwapTotal').textContent=size(m.swapTotalBytes);el('dSwapPercent').textContent=fixed(m.swapPercent)+'%'; @@ -696,7 +707,7 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av const sh=x.systemHealth||{},issueCount=sh.systemErrors||0;el('systemIssues').textContent=issueCount;el('systemBar').style.width=Math.min(issueCount*10,100)+'%';el('failedUnits').textContent=sh.failedUnits??'—';el('kernelErrors').textContent=sh.kernelErrors??'—';el('dSystemErrors').textContent=issueCount;el('dKernelErrors').textContent=sh.kernelErrors||0;el('dFailedUnits').textContent=sh.failedUnits||0;el('issueList').innerHTML=(sh.issues||[]).length?sh.issues.map(v=>'
'+safe(v.unit)+'
'+new Date(v.time*1000).toLocaleString('ru-RU')+'
Ошибка
'+safe(v.message)+'
').join(''):'Критических сообщений за 24 часа нет.';el('failedServiceList').innerHTML=(sh.failedServices||[]).length?sh.failedServices.map(v=>'
'+safe(v.unit)+'
'+safe(v.description||'Описание отсутствует')+'
'+safe(v.state)+'
Загрузка'+safe(v.load)+'
Состояние'+safe(v.active)+'
Подсостояние'+safe(v.state)+'
').join(''):'Упавших systemd-служб нет.'; el('issueList').querySelectorAll('.badge').forEach((badge,index)=>{const count=(sh.issues||[])[index]?.count||1;badge.textContent=count>1?'Повторений: '+count:'1 раз'}); renderNetworkGroups(n.interfaces||[]); - renderActivity(x.activity||{},g,svc,z);explainWarningBadges(); + renderActivity(x.activity||{},g,svc,z);renderIOAttribution(disks);explainWarningBadges(); el('dot').classList.add('live');el('status').textContent='Данные поступают'; } const percent=v=>fixed(v)+'%',temperature=v=>fixed(v)+' °C',megabytes=v=>fixed(v)+' MB/s'; @@ -728,7 +739,7 @@ el('cardOrder').onclick=event=>{const button=event.target.closest('[data-move]') function setTheme(theme){document.documentElement.dataset.theme=theme;localStorage.setItem('proxmox-theme',theme);document.querySelectorAll('[data-theme-choice]').forEach(b=>b.classList.toggle('active',b.dataset.themeChoice===theme));document.querySelectorAll('.chart-section:not([hidden])').forEach(showHistory)}document.querySelectorAll('[data-theme-choice]').forEach(b=>b.onclick=()=>setTheme(b.dataset.themeChoice));setTheme(document.documentElement.dataset.theme);el('settingsButton').onclick=()=>el('settingsDialog').showModal(); document.querySelectorAll('[data-settings-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-settings-tab]').forEach(t=>t.classList.toggle('active',t===tab));document.querySelectorAll('[data-settings-panel]').forEach(p=>p.hidden=p.dataset.settingsPanel!==tab.dataset.settingsTab)}); async function loadThresholds(){const response=await fetch('/api/settings/alerts');if(!response.ok)return;alertSettings=await response.json();document.querySelectorAll('[data-threshold]').forEach(input=>input.value=alertSettings[input.dataset.threshold]??'')} -el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{}); +el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=input.dataset.threshold==='diskIoExcluded'?input.value:Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{}); let configuredServices=[]; async function loadServices(){const response=await fetch('/api/services');if(!response.ok)return;configuredServices=await response.json()||[];el('serviceSettingsList').innerHTML=configuredServices.length?configuredServices.map(v=>'
'+safe(v.name)+'
'+safe([v.url&&('Внешний: '+v.url),v.localAddress&&('Локальный: '+v.localAddress)].filter(Boolean).join(' · '))+'
'+(v.enabled?'Включён':'Выключен')+'
').join(''):'Список пуст.'} function resetServiceForm(){el('serviceId').value='';el('serviceName').value='';el('serviceType').value='http';el('serviceURL').value='';el('serviceLocal').value='';el('serviceCode').value=200;el('serviceText').value='';el('serviceTimeout').value=5;el('serviceEnabled').checked=true}