Add IO attribution anomaly detection and alert correlation

2026-08-03 18:02:13 +03:00
parent 98afa2517b
commit 8241fba6f4
12 changed files with 498 additions and 63 deletions
--- a/trends.go
+++ b/trends.go
@@ -0,0 +1,107 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type entityAverage struct {
+	Value1, Value2 float64
+	Count          int
+}
+type TrendAnomaly struct {
+	ID       string `json:"id"`
+	Severity string `json:"severity"`
+	Source   string `json:"source"`
+	Title    string `json:"title"`
+	Message  string `json:"message"`
+}
+type trendCollector struct {
+	mu        sync.Mutex
+	updatedAt time.Time
+	cached    []TrendAnomaly
+}
+
+func (s *Store) entityAverages(kind string) map[string]entityAverage {
+	return s.entityAveragesSince(kind, 7*24*time.Hour)
+}
+func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage {
+	rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix())
+	if err != nil {
+		return nil
+	}
+	defer rows.Close()
+	result := map[string]entityAverage{}
+	for rows.Next() {
+		var name string
+		var v1, v2 sql.NullFloat64
+		var count int
+		if rows.Scan(&name, &v1, &v2, &count) == nil {
+			result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count}
+		}
+	}
+	return result
+}
+
+func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if time.Since(c.updatedAt) < time.Minute {
+		return c.cached
+	}
+	var out []TrendAnomaly
+	iops := store.entityAverages("disk-io")
+	latency := store.entityAverages("disk-latency")
+	latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour)
+	temps := store.entityAverages("disk")
+	for _, d := range metrics.Disks {
+		name := d.Name + " · " + d.Model
+		currentIOPS := d.ReadIOPS + d.WriteIOPS
+		if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 {
+			out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)})
+		}
+		if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 {
+			out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)})
+		}
+		if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 {
+			out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)})
+		}
+		if d.Temperature != nil {
+			if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 {
+				out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)})
+			}
+		}
+	}
+	guestAvg := store.entityAverages("guest")
+	for _, g := range metrics.Guests.Guests {
+		if g.MaxMemory == 0 {
+			continue
+		}
+		name := fmt.Sprintf("%d · %s", g.VMID, g.Name)
+		current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100
+		if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 {
+			out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)})
+		}
+	}
+	zfsAverage := store.entityAverages("zfs")
+	for _, pool := range metrics.ZFS.Pools {
+		if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 {
+			out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)})
+		}
+	}
+	backupAverage := store.entityAverages("backup-duration")
+	for _, task := range metrics.Activity.Tasks {
+		if task.Type != "vzdump" || task.Duration <= 0 {
+			continue
+		}
+		name := "VMID " + task.ID
+		if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 {
+			out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)})
+		}
+	}
+	c.cached = out
+	c.updatedAt = time.Now()
+	return out
+}