108 lines
5.0 KiB
Go
108 lines
5.0 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type entityAverage struct {
|
|
Value1, Value2 float64
|
|
Count int
|
|
}
|
|
type TrendAnomaly struct {
|
|
ID string `json:"id"`
|
|
Severity string `json:"severity"`
|
|
Source string `json:"source"`
|
|
Title string `json:"title"`
|
|
Message string `json:"message"`
|
|
}
|
|
type trendCollector struct {
|
|
mu sync.Mutex
|
|
updatedAt time.Time
|
|
cached []TrendAnomaly
|
|
}
|
|
|
|
func (s *Store) entityAverages(kind string) map[string]entityAverage {
|
|
return s.entityAveragesSince(kind, 7*24*time.Hour)
|
|
}
|
|
func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage {
|
|
rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix())
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer rows.Close()
|
|
result := map[string]entityAverage{}
|
|
for rows.Next() {
|
|
var name string
|
|
var v1, v2 sql.NullFloat64
|
|
var count int
|
|
if rows.Scan(&name, &v1, &v2, &count) == nil {
|
|
result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count}
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if time.Since(c.updatedAt) < time.Minute {
|
|
return c.cached
|
|
}
|
|
var out []TrendAnomaly
|
|
iops := store.entityAverages("disk-io")
|
|
latency := store.entityAverages("disk-latency")
|
|
latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour)
|
|
temps := store.entityAverages("disk")
|
|
for _, d := range metrics.Disks {
|
|
name := d.Name + " · " + d.Model
|
|
currentIOPS := d.ReadIOPS + d.WriteIOPS
|
|
if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 {
|
|
out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)})
|
|
}
|
|
if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 {
|
|
out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)})
|
|
}
|
|
if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 {
|
|
out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)})
|
|
}
|
|
if d.Temperature != nil {
|
|
if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 {
|
|
out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)})
|
|
}
|
|
}
|
|
}
|
|
guestAvg := store.entityAverages("guest")
|
|
for _, g := range metrics.Guests.Guests {
|
|
if g.MaxMemory == 0 {
|
|
continue
|
|
}
|
|
name := fmt.Sprintf("%d · %s", g.VMID, g.Name)
|
|
current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100
|
|
if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 {
|
|
out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)})
|
|
}
|
|
}
|
|
zfsAverage := store.entityAverages("zfs")
|
|
for _, pool := range metrics.ZFS.Pools {
|
|
if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 {
|
|
out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)})
|
|
}
|
|
}
|
|
backupAverage := store.entityAverages("backup-duration")
|
|
for _, task := range metrics.Activity.Tasks {
|
|
if task.Type != "vzdump" || task.Duration <= 0 {
|
|
continue
|
|
}
|
|
name := "VMID " + task.ID
|
|
if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 {
|
|
out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)})
|
|
}
|
|
}
|
|
c.cached = out
|
|
c.updatedAt = time.Now()
|
|
return out
|
|
}
|