maxim/ProxmoxDash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity

Add IO attribution anomaly detection and alert correlation

Browse Source
This commit is contained in:
Maxim
2026-08-03 18:02:13 +03:00
parent 98afa2517b
commit 8241fba6f4
12 changed files with 498 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
alerts.go
View File

@@ -8,12 +8,14 @@ import (
)
type Alert struct {
EventID int64 `json:"eventId,omitempty"`
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
EventID int64 `json:"eventId,omitempty"`
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
FirstSeen int64 `json:"firstSeen,omitempty"`
LastSeen int64 `json:"lastSeen,omitempty"`
}
func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert {
@@ -61,6 +63,14 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent))
}
for _, disk := range metrics.Disks {
diskExcluded := false
for _, value := range strings.Split(thresholds.DiskIOExcluded, ",") {
value = strings.TrimSpace(value)
if value != "" && (strings.EqualFold(value, disk.Name) || strings.EqualFold(value, disk.Serial)) {
diskExcluded = true
break
}
}
if disk.SMARTStatus == "Ошибка" {
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
} else if disk.SMARTStatus == "Требует внимания" {
@@ -78,12 +88,16 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
if len(disk.CounterChanges) > 0 {
add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · "))
}
if disk.IOPressureSeconds >= 60 {
if !diskExcluded && disk.IOPressureSeconds >= thresholds.DiskIODuration {
severity := "warning"
if disk.Utilization >= 98 || disk.LatencyMs >= 100 {
severity = "critical"
}
add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("Нагрузка держится %.0f сек.: %.0f IOPS, %.1f MB/s, latency %.1f ms, util %.1f%%, очередь %.1f.", disk.IOPressureSeconds, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.Utilization, disk.AverageQueue))
cause := disk.IOCause
if cause == "" {
cause = "источник пока не определён"
}
add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("%s занят на %.1f%%; вероятная причина: %s (уверенность %d%%). %.0f IOPS, %.1f MB/s, latency %.1f ms, очередь %.1f.", disk.Name, disk.Utilization, cause, disk.IOConfidence, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.AverageQueue))
}
if disk.SelfTestNeverRun {
add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.")
@@ -266,6 +280,9 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status))
}
}
for _, trend := range metrics.Trends {
add(trend.ID, trend.Severity, trend.Source, trend.Title, trend.Message)
}
priority := map[string]int{"critical": 0, "warning": 1, "info": 2}
sort.SliceStable(alerts, func(i, j int) bool {
@@ -277,6 +294,51 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
return alerts
}
func correlateAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
pressureAlerts := map[string]bool{}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") {
pressureAlerts[strings.TrimPrefix(a.ID, "disk-io-pressure-")] = true
}
}
var pressure []DiskMetrics
for _, d := range metrics.Disks {
if pressureAlerts[d.Name] && d.IOCause != "" {
pressure = append(pressure, d)
}
}
if len(pressure) == 0 {
return alerts
}
var down []string
for _, service := range metrics.Services.Services {
for _, endpoint := range service.Endpoints {
if !endpoint.Up {
down = append(down, service.Name)
break
}
}
}
d := pressure[0]
chain := d.IOCause + " → " + d.Name + fmt.Sprintf(" util %.1f%% → latency %.1f ms", d.Utilization, d.LatencyMs)
if len(down) > 0 {
chain += " → недоступны/медленны сервисы: " + strings.Join(down, ", ")
}
severity := "warning"
if len(down) > 0 || d.Utilization >= 98 {
severity = "critical"
}
correlated := Alert{ID: "correlation-io-" + d.Name, Severity: severity, Source: "Корреляция", Title: "Обнаружена связанная цепочка I/O-событий", Message: chain + fmt.Sprintf(". Уверенность причины %d%%.", d.IOConfidence)}
filtered := []Alert{correlated}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") || strings.HasPrefix(a.ID, "service-") {
continue
}
filtered = append(filtered, a)
}
return filtered
}
func joinReasons(reasons []string, fallback string) string {
if len(reasons) == 0 {
return fallback