Add IO attribution anomaly detection and alert correlation

README.md

@@ -123,6 +123,11 @@ ss -lntp | grep 9105
 - изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков;
 - realtime read/write IOPS, MB/s, latency, utilization и глубину очереди каждого диска;
 - недельную историю IOPS/utilization и алерты при устойчивом I/O-давлении;
+- вероятный источник I/O: VM/LXC, процесс хоста или активная backup/snapshot/migration-задача;
+- отдельные пороги HDD, SATA SSD и NVMe с исключениями по имени или серийному номеру;
+- недельные baseline-аномалии IOPS, latency, температуры, RAM, ZFS и длительности backup;
+- корреляцию связанных событий в одну причинно-следственную цепочку;
+- время первого появления и последнего наблюдения каждого уведомления;
 - автоматический короткий SMART self-test раз в неделю для дисков без свежего теста;
 - datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS;
 - доступность шлюза, DNS и интернета, packet loss, latency и внешний IP;

activity.go

@@ -57,7 +57,7 @@ func (c *activityCollector) collect(guests GuestsMetrics) ProxmoxActivity {
 	value := c.value
 	c.mu.RUnlock()
 	c.mu.Lock()
-	if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 10*time.Minute {
+	if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 15*time.Second {
 		c.refreshing = true
 		go c.refreshWithGuests(guests)
 	}

alert_events.go

@@ -74,7 +74,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
 	if err = tx.Commit(); err != nil {
 		return alerts, err
 	}
-	rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`)
+	rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`)
 	if err != nil {
 		return alerts, err
 	}
@@ -82,7 +82,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
 	var visible []Alert
 	for rows.Next() {
 		var a Alert
-		if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message); err != nil {
+		if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message, &a.FirstSeen, &a.LastSeen); err != nil {
 			return alerts, err
 		}
 		visible = append(visible, a)

alert_events_test.go

@@ -10,7 +10,7 @@ func TestAlertLifecycle(t *testing.T) {
 	defer store.Close()
 	alert := Alert{ID: "disk", Severity: "warning", Source: "Диски", Title: "SMART", Message: "Проверьте диск"}
 	visible, err := store.SyncAlerts([]Alert{alert})
-	if err != nil || len(visible) != 1 || visible[0].EventID == 0 {
+	if err != nil || len(visible) != 1 || visible[0].EventID == 0 || visible[0].FirstSeen == 0 || visible[0].LastSeen == 0 {
 		t.Fatalf("unexpected new alert: %+v %v", visible, err)
 	}
 	firstID := visible[0].EventID

alerts.go

@@ -8,12 +8,14 @@ import (
 )
 
 type Alert struct {
-	EventID  int64  `json:"eventId,omitempty"`
+	EventID   int64  `json:"eventId,omitempty"`
-	ID       string `json:"id"`
+	ID        string `json:"id"`
-	Severity string `json:"severity"`
+	Severity  string `json:"severity"`
-	Source   string `json:"source"`
+	Source    string `json:"source"`
-	Title    string `json:"title"`
+	Title     string `json:"title"`
-	Message  string `json:"message"`
+	Message   string `json:"message"`
+	FirstSeen int64  `json:"firstSeen,omitempty"`
+	LastSeen  int64  `json:"lastSeen,omitempty"`
 }
 
 func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert {
@@ -61,6 +63,14 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 		add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent))
 	}
 	for _, disk := range metrics.Disks {
+		diskExcluded := false
+		for _, value := range strings.Split(thresholds.DiskIOExcluded, ",") {
+			value = strings.TrimSpace(value)
+			if value != "" && (strings.EqualFold(value, disk.Name) || strings.EqualFold(value, disk.Serial)) {
+				diskExcluded = true
+				break
+			}
+		}
 		if disk.SMARTStatus == "Ошибка" {
 			add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
 		} else if disk.SMARTStatus == "Требует внимания" {
@@ -78,12 +88,16 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 		if len(disk.CounterChanges) > 0 {
 			add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · "))
 		}
-		if disk.IOPressureSeconds >= 60 {
+		if !diskExcluded && disk.IOPressureSeconds >= thresholds.DiskIODuration {
 			severity := "warning"
 			if disk.Utilization >= 98 || disk.LatencyMs >= 100 {
 				severity = "critical"
 			}
-			add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("Нагрузка держится %.0f сек.: %.0f IOPS, %.1f MB/s, latency %.1f ms, util %.1f%%, очередь %.1f.", disk.IOPressureSeconds, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.Utilization, disk.AverageQueue))
+			cause := disk.IOCause
+			if cause == "" {
+				cause = "источник пока не определён"
+			}
+			add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("%s занят на %.1f%%; вероятная причина: %s (уверенность %d%%). %.0f IOPS, %.1f MB/s, latency %.1f ms, очередь %.1f.", disk.Name, disk.Utilization, cause, disk.IOConfidence, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.AverageQueue))
 		}
 		if disk.SelfTestNeverRun {
 			add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.")
@@ -266,6 +280,9 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 			add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status))
 		}
 	}
+	for _, trend := range metrics.Trends {
+		add(trend.ID, trend.Severity, trend.Source, trend.Title, trend.Message)
+	}
 
 	priority := map[string]int{"critical": 0, "warning": 1, "info": 2}
 	sort.SliceStable(alerts, func(i, j int) bool {
@@ -277,6 +294,51 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 	return alerts
 }
 
+func correlateAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
+	pressureAlerts := map[string]bool{}
+	for _, a := range alerts {
+		if strings.HasPrefix(a.ID, "disk-io-pressure-") {
+			pressureAlerts[strings.TrimPrefix(a.ID, "disk-io-pressure-")] = true
+		}
+	}
+	var pressure []DiskMetrics
+	for _, d := range metrics.Disks {
+		if pressureAlerts[d.Name] && d.IOCause != "" {
+			pressure = append(pressure, d)
+		}
+	}
+	if len(pressure) == 0 {
+		return alerts
+	}
+	var down []string
+	for _, service := range metrics.Services.Services {
+		for _, endpoint := range service.Endpoints {
+			if !endpoint.Up {
+				down = append(down, service.Name)
+				break
+			}
+		}
+	}
+	d := pressure[0]
+	chain := d.IOCause + " → " + d.Name + fmt.Sprintf(" util %.1f%% → latency %.1f ms", d.Utilization, d.LatencyMs)
+	if len(down) > 0 {
+		chain += " → недоступны/медленны сервисы: " + strings.Join(down, ", ")
+	}
+	severity := "warning"
+	if len(down) > 0 || d.Utilization >= 98 {
+		severity = "critical"
+	}
+	correlated := Alert{ID: "correlation-io-" + d.Name, Severity: severity, Source: "Корреляция", Title: "Обнаружена связанная цепочка I/O-событий", Message: chain + fmt.Sprintf(". Уверенность причины %d%%.", d.IOConfidence)}
+	filtered := []Alert{correlated}
+	for _, a := range alerts {
+		if strings.HasPrefix(a.ID, "disk-io-pressure-") || strings.HasPrefix(a.ID, "service-") {
+			continue
+		}
+		filtered = append(filtered, a)
+	}
+	return filtered
+}
+
 func joinReasons(reasons []string, fallback string) string {
 	if len(reasons) == 0 {
 		return fallback

disks.go

@@ -13,42 +13,46 @@ import (
 )
 
 type DiskMetrics struct {
-	Name              string   `json:"name"`
+	Name              string       `json:"name"`
-	Path              string   `json:"path"`
+	Path              string       `json:"path"`
-	Model             string   `json:"model"`
+	Model             string       `json:"model"`
-	Serial            string   `json:"serial"`
+	Serial            string       `json:"serial"`
-	Type              string   `json:"type"`
+	Type              string       `json:"type"`
-	Protocol          string   `json:"protocol"`
+	Protocol          string       `json:"protocol"`
-	SizeBytes         uint64   `json:"sizeBytes"`
+	SizeBytes         uint64       `json:"sizeBytes"`
-	UsedBytes         *uint64  `json:"usedBytes"`
+	UsedBytes         *uint64      `json:"usedBytes"`
-	UsagePercent      *float64 `json:"usagePercent"`
+	UsagePercent      *float64     `json:"usagePercent"`
-	SMARTStatus       string   `json:"smartStatus"`
+	SMARTStatus       string       `json:"smartStatus"`
-	SMARTAvailable    bool     `json:"smartAvailable"`
+	SMARTAvailable    bool         `json:"smartAvailable"`
-	AttentionReasons  []string `json:"attentionReasons"`
+	AttentionReasons  []string     `json:"attentionReasons"`
-	Temperature       *float64 `json:"temperatureCelsius"`
+	Temperature       *float64     `json:"temperatureCelsius"`
-	PowerOnHours      *uint64  `json:"powerOnHours"`
+	PowerOnHours      *uint64      `json:"powerOnHours"`
-	WearUsedPercent   *float64 `json:"wearUsedPercent"`
+	WearUsedPercent   *float64     `json:"wearUsedPercent"`
-	Reallocated       uint64   `json:"reallocated"`
+	Reallocated       uint64       `json:"reallocated"`
-	Pending           uint64   `json:"pending"`
+	Pending           uint64       `json:"pending"`
-	Uncorrectable     uint64   `json:"uncorrectable"`
+	Uncorrectable     uint64       `json:"uncorrectable"`
-	CRCErrors         uint64   `json:"crcErrors"`
+	CRCErrors         uint64       `json:"crcErrors"`
-	MediaErrors       uint64   `json:"mediaErrors"`
+	MediaErrors       uint64       `json:"mediaErrors"`
-	CounterChanges    []string `json:"counterChanges"`
+	CounterChanges    []string     `json:"counterChanges"`
-	LastSelfTest      string   `json:"lastSelfTest"`
+	LastSelfTest      string       `json:"lastSelfTest"`
-	LastSelfTestAt    string   `json:"lastSelfTestAt"`
+	LastSelfTestAt    string       `json:"lastSelfTestAt"`
-	SelfTestNeverRun  bool     `json:"selfTestNeverRun"`
+	SelfTestNeverRun  bool         `json:"selfTestNeverRun"`
-	ShortTestStarted  bool     `json:"shortTestStarted"`
+	ShortTestStarted  bool         `json:"shortTestStarted"`
-	ReadIOPS          float64  `json:"readIops"`
+	ReadIOPS          float64      `json:"readIops"`
-	WriteIOPS         float64  `json:"writeIops"`
+	WriteIOPS         float64      `json:"writeIops"`
-	ReadBytesPerSec   float64  `json:"readBytesPerSec"`
+	ReadBytesPerSec   float64      `json:"readBytesPerSec"`
-	WriteBytesPerSec  float64  `json:"writeBytesPerSec"`
+	WriteBytesPerSec  float64      `json:"writeBytesPerSec"`
-	ReadLatencyMs     float64  `json:"readLatencyMs"`
+	ReadLatencyMs     float64      `json:"readLatencyMs"`
-	WriteLatencyMs    float64  `json:"writeLatencyMs"`
+	WriteLatencyMs    float64      `json:"writeLatencyMs"`
-	LatencyMs         float64  `json:"latencyMs"`
+	LatencyMs         float64      `json:"latencyMs"`
-	Utilization       float64  `json:"utilizationPercent"`
+	Utilization       float64      `json:"utilizationPercent"`
-	QueueDepth        uint64   `json:"queueDepth"`
+	QueueDepth        uint64       `json:"queueDepth"`
-	AverageQueue      float64  `json:"averageQueue"`
+	AverageQueue      float64      `json:"averageQueue"`
-	IOPressureSeconds float64  `json:"ioPressureSeconds"`
+	IOPressureSeconds float64      `json:"ioPressureSeconds"`
+	IOCause           string       `json:"ioCause"`
+	IOCauseDetail     string       `json:"ioCauseDetail"`
+	IOConfidence      int          `json:"ioConfidence"`
+	TopConsumers      []IOConsumer `json:"topConsumers"`
 }
 
 type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 }
@@ -65,11 +69,11 @@ type diskCollector struct {
 	ioPressureSince map[string]time.Time
 }
 
-func (c *diskCollector) collect() []DiskMetrics {
+func (c *diskCollector) collect(thresholds AlertThresholds) []DiskMetrics {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
-		c.applyDiskIO(c.cached)
+		c.applyDiskIO(c.cached, thresholds)
 		return c.cached
 	}
 	current := readPhysicalDisks()
@@ -104,12 +108,12 @@ func (c *diskCollector) collect() []DiskMetrics {
 		}
 	}
 	c.cached = current
-	c.applyDiskIO(c.cached)
+	c.applyDiskIO(c.cached, thresholds)
 	c.updatedAt = time.Now()
 	return c.cached
 }
 
-func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
+func (c *diskCollector) applyDiskIO(disks []DiskMetrics, thresholds AlertThresholds) {
 	now := time.Now()
 	snapshots := readDiskIOStats("/proc/diskstats")
 	if c.ioPrevious == nil {
@@ -144,11 +148,13 @@ func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
 			}
 			disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100)
 			disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000)
-			latencyLimit := 20.0
+			utilLimit, latencyLimit, queueLimit := thresholds.DiskSSDUtil, thresholds.DiskSSDLatency, thresholds.DiskSSDQueue
 			if disk.Type == "HDD" {
-				latencyLimit = 50
+				utilLimit, latencyLimit, queueLimit = thresholds.DiskHDDUtil, thresholds.DiskHDDLatency, thresholds.DiskHDDQueue
+			} else if strings.Contains(disk.Type, "NVMe") {
+				utilLimit, latencyLimit, queueLimit = thresholds.DiskNVMeUtil, thresholds.DiskNVMeLatency, thresholds.DiskNVMeQueue
 			}
-			pressured := disk.Utilization >= 90 || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= 4
+			pressured := disk.Utilization >= utilLimit || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= queueLimit
 			if pressured {
 				if c.ioPressureSince[disk.Name].IsZero() {
 					c.ioPressureSince[disk.Name] = now

history.go

@@ -47,6 +47,8 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
 		iops := d.ReadIOPS + d.WriteIOPS
 		util := d.Utilization
 		s.AddEntityHistory("disk-io", d.Name+" · "+d.Model, ts, &iops, &util)
+		latency := d.LatencyMs
+		s.AddEntityHistory("disk-latency", d.Name+" · "+d.Model, ts, &latency, nil)
 		if d.Temperature == nil {
 			continue
 		}
@@ -56,6 +58,12 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
 		}
 		s.AddEntityHistory("disk", d.Name+" · "+d.Model, ts, d.Temperature, wear)
 	}
+	for _, task := range metrics.Activity.Tasks {
+		if task.Type == "vzdump" && task.Duration > 0 && task.EndTime >= time.Now().Add(-15*time.Minute).Unix() {
+			duration := float64(task.Duration)
+			s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
+		}
+	}
 	s.PruneEntityHistory()
 }
 

io_attribution.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type IOConsumer struct {
+	Kind             string  `json:"kind"`
+	Name             string  `json:"name"`
+	VMID             int     `json:"vmid,omitempty"`
+	ReadBytesPerSec  float64 `json:"readBytesPerSec"`
+	WriteBytesPerSec float64 `json:"writeBytesPerSec"`
+	SharePercent     float64 `json:"sharePercent"`
+}
+type processIOSnapshot struct {
+	read, write           uint64
+	name, cmdline, cgroup string
+}
+type ioAttributionCollector struct {
+	mu        sync.Mutex
+	previous  map[int]processIOSnapshot
+	updatedAt time.Time
+	cached    []IOConsumer
+}
+
+var qemuIDPattern = regexp.MustCompile(`(?:^|\s)-id\s+(\d+)`)
+var lxcIDPattern = regexp.MustCompile(`(?:lxc(?:\.payload)?[./-]|pve-container@)(\d+)`)
+
+func (c *ioAttributionCollector) collect(guests GuestsMetrics) []IOConsumer {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := time.Now()
+	current := readProcessIO()
+	elapsed := now.Sub(c.updatedAt).Seconds()
+	if c.previous == nil || elapsed <= 0 {
+		c.previous = current
+		c.updatedAt = now
+		return nil
+	}
+	names := map[int]string{}
+	for _, g := range guests.Guests {
+		names[g.VMID] = g.Name
+	}
+	grouped := map[string]*IOConsumer{}
+	var total float64
+	for pid, snapshot := range current {
+		old, ok := c.previous[pid]
+		if !ok || snapshot.read < old.read || snapshot.write < old.write {
+			continue
+		}
+		read := float64(snapshot.read-old.read) / elapsed
+		write := float64(snapshot.write-old.write) / elapsed
+		if read+write < 1024 {
+			continue
+		}
+		kind, name, vmid := classifyIOProcess(snapshot, names)
+		key := kind + ":" + strconv.Itoa(vmid) + ":" + name
+		consumer := grouped[key]
+		if consumer == nil {
+			consumer = &IOConsumer{Kind: kind, Name: name, VMID: vmid}
+			grouped[key] = consumer
+		}
+		consumer.ReadBytesPerSec += read
+		consumer.WriteBytesPerSec += write
+		total += read + write
+	}
+	var result []IOConsumer
+	for _, v := range grouped {
+		if total > 0 {
+			v.SharePercent = (v.ReadBytesPerSec + v.WriteBytesPerSec) / total * 100
+		}
+		result = append(result, *v)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].ReadBytesPerSec+result[i].WriteBytesPerSec > result[j].ReadBytesPerSec+result[j].WriteBytesPerSec
+	})
+	if len(result) > 8 {
+		result = result[:8]
+	}
+	c.previous = current
+	c.updatedAt = now
+	c.cached = result
+	return result
+}
+
+func classifyIOProcess(p processIOSnapshot, names map[int]string) (kind, name string, vmid int) {
+	if match := qemuIDPattern.FindStringSubmatch(p.cmdline); len(match) > 1 {
+		vmid, _ = strconv.Atoi(match[1])
+		return "VM", firstNonEmpty(names[vmid], "VM "+match[1]), vmid
+	}
+	if match := lxcIDPattern.FindStringSubmatch(p.cgroup); len(match) > 1 {
+		vmid, _ = strconv.Atoi(match[1])
+		return "LXC", firstNonEmpty(names[vmid], "LXC "+match[1]), vmid
+	}
+	return "Процесс", firstNonEmpty(p.name, "PID"), 0
+}
+
+func readProcessIO() map[int]processIOSnapshot {
+	result := map[int]processIOSnapshot{}
+	entries, _ := filepath.Glob("/proc/[0-9]*/io")
+	for _, path := range entries {
+		pid, _ := strconv.Atoi(filepath.Base(filepath.Dir(path)))
+		data, err := os.ReadFile(path)
+		if err != nil {
+			continue
+		}
+		var item processIOSnapshot
+		for _, line := range strings.Split(string(data), "\n") {
+			key, value, ok := strings.Cut(line, ":")
+			if !ok {
+				continue
+			}
+			n, _ := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
+			if key == "read_bytes" {
+				item.read = n
+			} else if key == "write_bytes" {
+				item.write = n
+			}
+		}
+		comm, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "comm"))
+		cmd, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cmdline"))
+		cg, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cgroup"))
+		item.name = strings.TrimSpace(string(comm))
+		item.cmdline = strings.ReplaceAll(string(cmd), "\x00", " ")
+		item.cgroup = string(cg)
+		result[pid] = item
+	}
+	return result
+}
+
+func attributeDiskIO(metrics *dashboardMetrics, consumers []IOConsumer) {
+	active := activeIOOperation(metrics.Activity.Tasks)
+	for i := range metrics.Disks {
+		disk := &metrics.Disks[i]
+		disk.TopConsumers = append([]IOConsumer(nil), consumers...)
+		if active != "" {
+			disk.IOCause = active
+			disk.IOCauseDetail = "Активная задача Proxmox совпадает по времени с дисковой нагрузкой"
+			disk.IOConfidence = 75
+		} else if len(consumers) > 0 {
+			top := consumers[0]
+			disk.IOCause = top.Kind + " · " + top.Name
+			disk.IOCauseDetail = "Крупнейший потребитель host I/O: " + strconv.FormatFloat(top.SharePercent, 'f', 1, 64) + "%"
+			disk.IOConfidence = 55
+		}
+	}
+}
+
+func activeIOOperation(tasks []ProxmoxTask) string {
+	for _, task := range tasks {
+		if task.EndTime != 0 && task.Status != "" {
+			continue
+		}
+		label := ""
+		switch task.Type {
+		case "vzdump":
+			label = "Backup"
+		case "qmigrate":
+			label = "Миграция"
+		case "qmsnapshot", "vzsnapshot":
+			label = "Snapshot"
+		}
+		if label != "" {
+			if task.ID != "" {
+				label += " VM/LXC " + task.ID
+			}
+			return label
+		}
+	}
+	return ""
+}

io_attribution_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestClassifyQEMUAndLXCIO(t *testing.T) {
+	kind, name, vmid := classifyIOProcess(processIOSnapshot{name: "qemu", cmdline: "qemu-system-x86_64 -id 105 -name guest=test"}, map[int]string{105: "Gitea"})
+	if kind != "VM" || name != "Gitea" || vmid != 105 {
+		t.Fatalf("unexpected qemu attribution: %s %s %d", kind, name, vmid)
+	}
+	kind, name, vmid = classifyIOProcess(processIOSnapshot{name: "worker", cgroup: "0::/lxc.payload.220/ns"}, map[int]string{220: "db"})
+	if kind != "LXC" || name != "db" || vmid != 220 {
+		t.Fatalf("unexpected lxc attribution: %s %s %d", kind, name, vmid)
+	}
+}
+
+func TestActiveIOOperation(t *testing.T) {
+	value := activeIOOperation([]ProxmoxTask{{Type: "vzdump", ID: "105", Status: "", EndTime: 0}})
+	if !strings.Contains(value, "Backup") || !strings.Contains(value, "105") {
+		t.Fatalf("unexpected operation: %s", value)
+	}
+}
+
+func TestCorrelateAlerts(t *testing.T) {
+	metrics := dashboardMetrics{Disks: []DiskMetrics{{Name: "sda", IOCause: "Backup VM/LXC 105", IOConfidence: 90, Utilization: 98, LatencyMs: 80}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Up: false}}}}}}
+	alerts := []Alert{{ID: "disk-io-pressure-sda", Severity: "critical"}, {ID: "service-1-public", Severity: "critical"}, {ID: "other", Severity: "warning"}}
+	result := correlateAlerts(metrics, alerts)
+	if len(result) != 2 || result[0].Source != "Корреляция" || !strings.Contains(result[0].Message, "Gitea") {
+		t.Fatalf("unexpected correlation: %+v", result)
+	}
+}

store.go

@@ -31,6 +31,17 @@ type AlertThresholds struct {
 	NVMeTempWarning   float64 `json:"nvmeTempWarning"`
 	UPSChargeCritical float64 `json:"upsChargeCritical"`
 	BackupMaxAgeHours float64 `json:"backupMaxAgeHours"`
+	DiskHDDUtil       float64 `json:"diskHddUtil"`
+	DiskHDDLatency    float64 `json:"diskHddLatency"`
+	DiskHDDQueue      float64 `json:"diskHddQueue"`
+	DiskSSDUtil       float64 `json:"diskSsdUtil"`
+	DiskSSDLatency    float64 `json:"diskSsdLatency"`
+	DiskSSDQueue      float64 `json:"diskSsdQueue"`
+	DiskNVMeUtil      float64 `json:"diskNvmeUtil"`
+	DiskNVMeLatency   float64 `json:"diskNvmeLatency"`
+	DiskNVMeQueue     float64 `json:"diskNvmeQueue"`
+	DiskIODuration    float64 `json:"diskIoDuration"`
+	DiskIOExcluded    string  `json:"diskIoExcluded"`
 }
 
 func defaultThresholds() AlertThresholds {
@@ -40,6 +51,7 @@ func defaultThresholds() AlertThresholds {
 		StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90,
 		DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30,
 		BackupMaxAgeHours: 48,
+		DiskHDDUtil:       90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60,
 	}
 }
 
@@ -285,6 +297,19 @@ func validateThresholds(t AlertThresholds) error {
 	if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 {
 		return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов")
 	}
+	for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} {
+		if value < 1 || value > 100 {
+			return fmt.Errorf("пороги utilization должны быть от 1 до 100")
+		}
+	}
+	for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} {
+		if value < 0 {
+			return fmt.Errorf("пороги latency и очереди не могут быть отрицательными")
+		}
+	}
+	if t.DiskIODuration < 5 || t.DiskIODuration > 3600 {
+		return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд")
+	}
 	return nil
 }
 

trends.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type entityAverage struct {
+	Value1, Value2 float64
+	Count          int
+}
+type TrendAnomaly struct {
+	ID       string `json:"id"`
+	Severity string `json:"severity"`
+	Source   string `json:"source"`
+	Title    string `json:"title"`
+	Message  string `json:"message"`
+}
+type trendCollector struct {
+	mu        sync.Mutex
+	updatedAt time.Time
+	cached    []TrendAnomaly
+}
+
+func (s *Store) entityAverages(kind string) map[string]entityAverage {
+	return s.entityAveragesSince(kind, 7*24*time.Hour)
+}
+func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage {
+	rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix())
+	if err != nil {
+		return nil
+	}
+	defer rows.Close()
+	result := map[string]entityAverage{}
+	for rows.Next() {
+		var name string
+		var v1, v2 sql.NullFloat64
+		var count int
+		if rows.Scan(&name, &v1, &v2, &count) == nil {
+			result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count}
+		}
+	}
+	return result
+}
+
+func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if time.Since(c.updatedAt) < time.Minute {
+		return c.cached
+	}
+	var out []TrendAnomaly
+	iops := store.entityAverages("disk-io")
+	latency := store.entityAverages("disk-latency")
+	latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour)
+	temps := store.entityAverages("disk")
+	for _, d := range metrics.Disks {
+		name := d.Name + " · " + d.Model
+		currentIOPS := d.ReadIOPS + d.WriteIOPS
+		if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 {
+			out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)})
+		}
+		if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 {
+			out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)})
+		}
+		if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 {
+			out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)})
+		}
+		if d.Temperature != nil {
+			if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 {
+				out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)})
+			}
+		}
+	}
+	guestAvg := store.entityAverages("guest")
+	for _, g := range metrics.Guests.Guests {
+		if g.MaxMemory == 0 {
+			continue
+		}
+		name := fmt.Sprintf("%d · %s", g.VMID, g.Name)
+		current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100
+		if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 {
+			out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)})
+		}
+	}
+	zfsAverage := store.entityAverages("zfs")
+	for _, pool := range metrics.ZFS.Pools {
+		if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 {
+			out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)})
+		}
+	}
+	backupAverage := store.entityAverages("backup-duration")
+	for _, task := range metrics.Activity.Tasks {
+		if task.Type != "vzdump" || task.Duration <= 0 {
+			continue
+		}
+		name := "VMID " + task.ID
+		if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 {
+			out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)})
+		}
+	}
+	c.cached = out
+	c.updatedAt = time.Now()
+	return out
+}

web.go

@@ -29,10 +29,11 @@ type dashboardMetrics struct {
 	Activity     ProxmoxActivity     `json:"activity"`
 	Monitor      MonitorHealth       `json:"monitor"`
 	UPSShutdown  UPSShutdownSettings `json:"upsShutdown"`
+	Trends       []TrendAnomaly      `json:"trends"`
 	Timestamp    time.Time           `json:"timestamp"`
 }
 
-func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) {
+func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, ioAttribution *ioAttributionCollector, trends *trendCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) {
 	cpu, err := cpuCollector.collect()
 	if err != nil {
 		return dashboardMetrics{}, err
@@ -49,9 +50,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle
 	if err != nil {
 		return dashboardMetrics{}, err
 	}
+	thresholds := store.Thresholds()
 	metrics := dashboardMetrics{
 		CPU: cpu, Memory: memory, Storage: storage,
-		Disks: diskCollector.collect(), Network: network,
+		Disks: diskCollector.collect(thresholds), Network: network,
 		ZFS: zfsCollector.collect(), UPS: upsCollector.collect(),
 		Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Services: serviceCollector.collect(), SystemHealth: systemHealth.collect(), Update: readUpdateStatus(), Timestamp: time.Now(),
 	}
@@ -59,8 +61,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle
 	metrics.Monitor = monitor.snapshot(store)
 	metrics.UPSShutdown = store.UPSShutdownSettings()
 	metrics.Activity = activity.collect(metrics.Guests)
+	attributeDiskIO(&metrics, ioAttribution.collect(metrics.Guests))
+	metrics.Trends = trends.collect(metrics, store)
 	decorateNetworkInterfaces(&metrics.Network, metrics.Guests)
-	metrics.Alerts = evaluateAlerts(metrics, store.Thresholds())
+	metrics.Alerts = correlateAlerts(metrics, evaluateAlerts(metrics, thresholds))
 	metrics.Alerts, err = store.SyncAlerts(metrics.Alerts)
 	if err != nil {
 		return dashboardMetrics{}, err
@@ -121,12 +125,14 @@ func routes(collector *cpuCollector, store *Store) http.Handler {
 	services := newServiceCollector(store)
 	systemHealth := newSystemHealthCollector()
 	activity := newActivityCollector()
+	ioAttribution := &ioAttributionCollector{}
+	trends := &trendCollector{}
 	monitor := &monitorTracker{}
 	go runCollectionWatchdog(monitor)
 	go runUPSShutdownController(store, ups, guests)
 	collect := func() (dashboardMetrics, error) {
 		started := time.Now()
-		metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, store, monitor)
+		metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, ioAttribution, trends, store, monitor)
 		monitor.record(started, err)
 		return metrics, err
 	}
@@ -517,7 +523,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
   <div class="detail"><span class="label">Используется</span><strong id="dDiskUsed">—</strong></div><div class="detail"><span class="label">Свободно</span><strong id="dDiskFree">—</strong></div><div class="detail"><span class="label">Общий объём</span><strong id="dDiskTotal">—</strong></div>
   <div class="detail"><span class="label">Заполнение</span><strong id="dDiskPercent">—</strong></div><div class="detail"><span class="label">Точка монтирования</span><strong>/</strong></div>
  </div>
- <div class="disk-list" id="diskList"><span class="empty">Поиск накопителей…</span></div>
+ <div class="disk-list" id="diskList"><span class="empty">Поиск накопителей…</span></div><div class="wide"><span class="label">Расшифровка источников I/O</span><div class="disk-list" id="ioAttributionList"><span class="empty">Накопление данных…</span></div></div>
  <div class="chart-section"><div class="chart-head"><strong>Температура и I/O физических дисков</strong><span>Отдельно по каждому диску</span></div><div class="label">Температура</div><canvas class="metric-chart" id="storageChart"></canvas><div class="label">IOPS</div><canvas class="metric-chart" id="diskIOChart"></canvas><div class="label">Utilization</div><canvas class="metric-chart" id="diskUtilChart"></canvas></div>
 </dialog>
 
@@ -617,6 +623,10 @@ footer{color:#657581;font-size:12px;margin-top:17px}
    <label class="threshold-item"><span>ZFS · критично, %</span><input type="number" data-threshold="zfsCritical"></label><label class="threshold-item"><span>Температура HDD/SSD</span><input type="number" data-threshold="diskTempWarning"></label>
    <label class="threshold-item"><span>Температура NVMe</span><input type="number" data-threshold="nvmeTempWarning"></label><label class="threshold-item"><span>UPS заряд · критично, %</span><input type="number" data-threshold="upsChargeCritical"></label>
    <label class="threshold-item"><span>Максимальный возраст backup, ч</span><input type="number" data-threshold="backupMaxAgeHours"></label>
+   <label class="threshold-item"><span>HDD utilization, %</span><input type="number" data-threshold="diskHddUtil"></label><label class="threshold-item"><span>HDD latency, ms</span><input type="number" data-threshold="diskHddLatency"></label><label class="threshold-item"><span>HDD очередь</span><input type="number" data-threshold="diskHddQueue"></label>
+   <label class="threshold-item"><span>SATA SSD utilization, %</span><input type="number" data-threshold="diskSsdUtil"></label><label class="threshold-item"><span>SATA SSD latency, ms</span><input type="number" data-threshold="diskSsdLatency"></label><label class="threshold-item"><span>SATA SSD очередь</span><input type="number" data-threshold="diskSsdQueue"></label>
+   <label class="threshold-item"><span>NVMe utilization, %</span><input type="number" data-threshold="diskNvmeUtil"></label><label class="threshold-item"><span>NVMe latency, ms</span><input type="number" data-threshold="diskNvmeLatency"></label><label class="threshold-item"><span>NVMe очередь</span><input type="number" data-threshold="diskNvmeQueue"></label>
+   <label class="threshold-item"><span>Минимальная длительность, сек.</span><input type="number" data-threshold="diskIoDuration"></label><label class="field full-span"><span>Исключить диски из I/O-алертов</span><input type="text" data-threshold="diskIoExcluded" placeholder="sda, SERIAL123"><small>Имена или серийные номера через запятую.</small></label>
   </div>
   <button class="save-settings" id="saveThresholds">Сохранить пороги</button><div class="settings-note" id="thresholdStatus">Настройки хранятся на сервере и применяются для всех браузеров.</div>
  </div>
@@ -663,6 +673,7 @@ let currentServiceStatuses=[];
 function endpointHTML(v){const tls=v.tlsDays==null?'Нет TLS':v.tlsDays<0?'Сертификат истёк':v.tlsDays+' дн. до окончания',cert=v.tlsSubject?'<div class="certificate"><div>Сертификат: <b>'+safe(v.tlsSubject)+'</b></div><div>Домены: <b>'+safe((v.tlsDnsNames||[]).join(', ')||'не указаны')+'</b></div><div>Издатель: <b>'+safe(v.tlsIssuer||'—')+'</b></div><div>Для этого адреса: <b class="'+(v.tlsHostnameValid?'status-up':'status-down')+'">'+(v.tlsHostnameValid?'подходит':'не подходит')+'</b></div></div>':'';return '<div class="endpoint"><div class="endpoint-head"><div><b>'+safe(v.label)+'</b><div class="endpoint-address">'+safe(v.address)+'</div></div><span class="badge '+(v.up?'':'bad')+'">'+(v.up?'Доступен':'Недоступен')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Отклик</span><b>'+fixed(v.latencyMs)+' ms</b></div><div class="disk-stat"><span>HTTP</span><b>'+(v.statusCode||'—')+'</b></div><div class="disk-stat"><span>TLS</span><b>'+tls+'</b></div><div class="disk-stat"><span>Uptime · сутки</span><b>'+fixed(v.uptimeDay)+'%</b></div><div class="disk-stat"><span>Uptime · неделя</span><b>'+fixed(v.uptimeWeek)+'%</b></div></div>'+cert+(v.error?'<div class="settings-note status-down">'+safe(v.error)+'</div>':'')+'</div>'}
 function renderServices(svc){const items=svc.services||[];currentServiceStatuses=items;const avg=items.length?items.reduce((a,v)=>a+v.uptimeDay,0)/items.length:0,live=items.filter(v=>v.up),latency=live.length?live.reduce((a,v)=>a+v.latencyMs,0)/live.length:0;el('serviceUp').textContent=svc.up||0;el('serviceTotal').textContent=' / '+(svc.total||0);el('serviceBar').style.width=(svc.total?svc.up/svc.total*100:0)+'%';el('serviceUptime').textContent=svc.total?fixed(avg)+'%':'—';el('serviceLatency').textContent=svc.total?fixed(latency)+' ms':'—';el('serviceList').innerHTML=items.length?items.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+(v.endpoints||[]).length+' '+((v.endpoints||[]).length===1?'адрес':'адреса')+' проверяются независимо</div></div><span class="badge '+(v.up?'':'bad')+'">'+(v.up?'Всё доступно':'Есть проблема')+'</span></div><div class="endpoint-list">'+(v.endpoints||[]).map(endpointHTML).join('')+'</div><div class="service-actions"><button data-service-open="'+v.id+'">Открыть сервис ↗</button></div></article>').join(''):'<span class="empty">Добавьте первую проверку в настройках.</span>'}
 function renderNetworkGroups(interfaces){const groups=new Map();interfaces.forEach(v=>{const key=v.vmid?'guest-'+v.vmid:'host',group=groups.get(key)||{vmid:v.vmid,name:v.vmid?(v.guestName||('VMID '+v.vmid)):'Интерфейсы хоста',type:v.guestType||'Host',items:[]};group.items.push(v);groups.set(key,group)});el('networkInterfacesList').innerHTML=[...groups.values()].map(g=>{const primary=g.vmid?g.items.filter(v=>v.name.startsWith('tap')||v.name.startsWith('veth')):g.items,rx=primary.reduce((a,v)=>a+v.receivePerSec,0),tx=primary.reduce((a,v)=>a+v.transmitPerSec,0),errors=g.items.reduce((a,v)=>a+v.receiveErrors+v.transmitErrors+v.receiveDrops+v.transmitDrops,0),down=primary.some(v=>v.kind==='Physical'&&!v.carrier&&(v.master||v.receiveBytes+v.transmitBytes>0));return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(g.name)+'</div><div class="disk-meta">'+(g.vmid?'VMID '+g.vmid+' · ':'')+safe(g.type)+' · '+safe(g.items.map(v=>v.name+' ['+(v.kind||'Interface')+(v.speedMbps?' · '+v.speedMbps+'Mb/s':'')+(v.duplex?' · '+v.duplex:'')+(v.master?' → '+v.master:'')+']').join(', '))+'</div></div><span class="badge '+(errors||down?'warn':'')+'"'+(errors||down?' tabindex="0" data-reason="'+safe(down?'Нет carrier на используемом физическом интерфейсе':'Счётчики errors/drops больше нуля')+'"':'')+'>'+(down?'Нет линка':errors?'Есть потери':'Без ошибок')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Сейчас ↓</span><b>'+rate(rx)+'</b></div><div class="disk-stat"><span>Сейчас ↑</span><b>'+rate(tx)+'</b></div><div class="disk-stat"><span>Получено</span><b>'+size(primary.reduce((a,v)=>a+v.receiveBytes,0))+'</b></div><div class="disk-stat"><span>Передано</span><b>'+size(primary.reduce((a,v)=>a+v.transmitBytes,0))+'</b></div><div class="disk-stat"><span>Ошибки / drops</span><b>'+errors+'</b></div></div></article>'}).join('')}
+function renderIOAttribution(disks){const active=disks.filter(d=>d.ioCause||(d.topConsumers||[]).length);el('ioAttributionList').innerHTML=active.length?active.map(d=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(d.name)+' · '+safe(d.ioCause||'Источник не определён')+'</div><div class="disk-meta">'+safe(d.ioCauseDetail||'Собираем статистику процессов')+'</div></div><span class="badge '+(d.ioConfidence>=80?'':'warn')+'" tabindex="0" data-reason="Уверенность учитывает совпадение активной задачи Proxmox и долю I/O процессов">'+(d.ioConfidence||0)+'%</span></div><div class="topology-items">'+(d.topConsumers||[]).slice(0,5).map(v=>'<span class="topology-chip">'+safe(v.kind)+' · '+safe(v.name)+' · '+fixed(v.sharePercent)+'% · ↓'+rate(v.readBytesPerSec)+' ↑'+rate(v.writeBytesPerSec)+'</span>').join('')+'</div></article>').join(''):'<span class="empty">Для атрибуции нужно несколько realtime-циклов.</span>'}
 const taskLabel=type=>({vzdump:'Backup',qmstart:'Запуск VM',qmstop:'Остановка VM',qmshutdown:'Выключение VM',qmsnapshot:'Snapshot VM',vzstart:'Запуск LXC',vzstop:'Остановка LXC',vzsnapshot:'Snapshot LXC',qmigrate:'Миграция',aptupdate:'Обновление пакетов'})[type]||type;
 function explainWarningBadges(){el('snapshotList').querySelectorAll('.badge.warn').forEach(v=>{v.tabIndex=0;v.dataset.reason='Snapshot старше 30 дней. Проверьте, нужен ли он: старые снимки могут занимать место в storage.'});el('taskList').querySelectorAll('.badge.bad').forEach(v=>{v.tabIndex=0;v.dataset.reason='Задача Proxmox завершилась не со статусом OK. Откройте её журнал в интерфейсе Proxmox для подробностей.'})}
 let currentTasks=[],currentTaskFilter='all';
@@ -674,7 +685,7 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av
  lastDashboardEvent=Date.now();
  const update=x.update||{},updating=['queued','installing'].includes(update.state);el('updateCurrent').textContent=update.currentVersion||'—';el('updateAvailable').textContent=update.availableVersion||'—';el('updateMode').textContent=update.mode==='auto'?'Автоматически':'Уведомлять';el('updateMessage').textContent=update.message||'Проверка ещё не выполнялась';el('updateChecked').textContent=update.checkedAt?'Последняя проверка: '+new Date(update.checkedAt).toLocaleString('ru-RU'):'Проверка запускается автоматически раз в сутки.';el('installUpdate').disabled=updating;el('installUpdate').textContent=updating?'Обновление выполняется…':'Проверить и установить обновление';
  const critical=alerts.filter(a=>a.severity==='critical').length,warnings=alerts.filter(a=>a.severity==='warning').length,infos=alerts.filter(a=>a.severity==='info').length;el('alertCount').textContent=alerts.length;el('alertCritical').textContent=critical;el('alertWarnings').textContent=warnings;el('alertHeadline').textContent=critical?'Требуется срочное внимание':warnings?'Есть предупреждения':infos?'Есть информационные события':'Система в норме';el('alertBar').style.width=(critical?100:warnings?65:infos?30:0)+'%';
- el('alertList').innerHTML=alerts.length?alerts.map(a=>'<article class="alert-item '+safe(a.severity)+'"><i class="alert-icon"></i><div><div class="alert-source">'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+'</div><div class="alert-title">'+safe(a.title)+'</div><div class="alert-message">'+safe(a.message)+'</div><button class="ack-alert" data-ack="'+a.eventId+'">Принято</button></div></article>').join(''):'<div class="all-good"><strong>Всё в порядке</strong><span>Новых неподтверждённых событий нет.</span></div>';
+ el('alertList').innerHTML=alerts.length?alerts.map(a=>'<article class="alert-item '+safe(a.severity)+'"><i class="alert-icon"></i><div><div class="alert-source">'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+(a.firstSeen?' · С '+new Date(a.firstSeen*1000).toLocaleString('ru-RU'):'')+'</div><div class="alert-title">'+safe(a.title)+'</div><div class="alert-message">'+safe(a.message)+'</div><button class="ack-alert" data-ack="'+a.eventId+'">Принято</button></div></article>').join(''):'<div class="all-good"><strong>Всё в порядке</strong><span>Новых неподтверждённых событий нет.</span></div>';
  el('cpuName').textContent=el('cpuDetailName').textContent=c.model;el('cpuUsage').textContent=fixed(c.usagePercent);el('cpuBar').style.width=Math.min(c.usagePercent,100)+'%';el('cpuTemp').textContent=avg===null?'Нет датчика':fixed(avg)+' °C';el('cpuTopology').textContent=c.physicalCores+' / '+c.logicalCpus;
  el('dCpuUsage').textContent=fixed(c.usagePercent)+'%';el('dSockets').textContent=c.sockets;el('dCores').textContent=c.physicalCores;el('dThreads').textContent=c.logicalCpus;el('dFreq').textContent=(c.frequencyMhz/1000).toFixed(2)+' GHz';el('dUptime').textContent=uptime(c.uptimeSeconds);el('dLoad1').textContent=fixed(c.load1);el('dLoad5').textContent=fixed(c.load5);el('dLoad15').textContent=fixed(c.load15);el('dTemps').innerHTML=c.temperatures.length?c.temperatures.map(t=>'<span class="temp">'+t.label+'<b>'+fixed(t.celsius)+'°C</b></span>').join(''):'<span class="empty">Датчики температуры не найдены</span>';
  el('memUsage').textContent=fixed(m.usagePercent);el('memBar').style.width=Math.min(m.usagePercent,100)+'%';el('memUsed').textContent=size(m.usedBytes);el('memAvailable').textContent=size(m.availableBytes);el('dMemUsed').textContent=size(m.usedBytes);el('dMemAvailable').textContent=size(m.availableBytes);el('dMemTotal').textContent=size(m.totalBytes);el('dMemCache').textContent=size(m.cachedBytes);el('dMemBuffers').textContent=size(m.buffersBytes);el('dMemPercent').textContent=fixed(m.usagePercent)+'%';el('dSwapUsed').textContent=size(m.swapUsedBytes);el('dSwapTotal').textContent=size(m.swapTotalBytes);el('dSwapPercent').textContent=fixed(m.swapPercent)+'%';
@@ -696,7 +707,7 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av
  const sh=x.systemHealth||{},issueCount=sh.systemErrors||0;el('systemIssues').textContent=issueCount;el('systemBar').style.width=Math.min(issueCount*10,100)+'%';el('failedUnits').textContent=sh.failedUnits??'—';el('kernelErrors').textContent=sh.kernelErrors??'—';el('dSystemErrors').textContent=issueCount;el('dKernelErrors').textContent=sh.kernelErrors||0;el('dFailedUnits').textContent=sh.failedUnits||0;el('issueList').innerHTML=(sh.issues||[]).length?sh.issues.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+new Date(v.time*1000).toLocaleString('ru-RU')+'</div></div><span class="badge bad">Ошибка</span></div><div class="settings-note issue-message">'+safe(v.message)+'</div></article>').join(''):'<span class="empty">Критических сообщений за 24 часа нет.</span>';el('failedServiceList').innerHTML=(sh.failedServices||[]).length?sh.failedServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+safe(v.description||'Описание отсутствует')+'</div></div><span class="badge bad">'+safe(v.state)+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Загрузка</span><b>'+safe(v.load)+'</b></div><div class="disk-stat"><span>Состояние</span><b>'+safe(v.active)+'</b></div><div class="disk-stat"><span>Подсостояние</span><b>'+safe(v.state)+'</b></div></div></article>').join(''):'<span class="empty">Упавших systemd-служб нет.</span>';
  el('issueList').querySelectorAll('.badge').forEach((badge,index)=>{const count=(sh.issues||[])[index]?.count||1;badge.textContent=count>1?'Повторений: '+count:'1 раз'});
  renderNetworkGroups(n.interfaces||[]);
- renderActivity(x.activity||{},g,svc,z);explainWarningBadges();
+ renderActivity(x.activity||{},g,svc,z);renderIOAttribution(disks);explainWarningBadges();
  el('dot').classList.add('live');el('status').textContent='Данные поступают';
 }
 const percent=v=>fixed(v)+'%',temperature=v=>fixed(v)+' °C',megabytes=v=>fixed(v)+' MB/s';
@@ -728,7 +739,7 @@ el('cardOrder').onclick=event=>{const button=event.target.closest('[data-move]')
 function setTheme(theme){document.documentElement.dataset.theme=theme;localStorage.setItem('proxmox-theme',theme);document.querySelectorAll('[data-theme-choice]').forEach(b=>b.classList.toggle('active',b.dataset.themeChoice===theme));document.querySelectorAll('.chart-section:not([hidden])').forEach(showHistory)}document.querySelectorAll('[data-theme-choice]').forEach(b=>b.onclick=()=>setTheme(b.dataset.themeChoice));setTheme(document.documentElement.dataset.theme);el('settingsButton').onclick=()=>el('settingsDialog').showModal();
 document.querySelectorAll('[data-settings-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-settings-tab]').forEach(t=>t.classList.toggle('active',t===tab));document.querySelectorAll('[data-settings-panel]').forEach(p=>p.hidden=p.dataset.settingsPanel!==tab.dataset.settingsTab)});
 async function loadThresholds(){const response=await fetch('/api/settings/alerts');if(!response.ok)return;alertSettings=await response.json();document.querySelectorAll('[data-threshold]').forEach(input=>input.value=alertSettings[input.dataset.threshold]??'')}
-el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{});
+el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=input.dataset.threshold==='diskIoExcluded'?input.value:Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{});
 let configuredServices=[];
 async function loadServices(){const response=await fetch('/api/services');if(!response.ok)return;configuredServices=await response.json()||[];el('serviceSettingsList').innerHTML=configuredServices.length?configuredServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+safe([v.url&&('Внешний: '+v.url),v.localAddress&&('Локальный: '+v.localAddress)].filter(Boolean).join(' · '))+'</div></div><span class="badge '+(v.enabled?'':'warn')+'">'+(v.enabled?'Включён':'Выключен')+'</span></div><div class="service-actions"><button data-service-edit="'+v.id+'">Изменить</button><button data-service-delete="'+v.id+'">Удалить</button></div></article>').join(''):'<span class="empty">Список пуст.</span>'}
 function resetServiceForm(){el('serviceId').value='';el('serviceName').value='';el('serviceType').value='http';el('serviceURL').value='';el('serviceLocal').value='';el('serviceCode').value=200;el('serviceText').value='';el('serviceTimeout').value=5;el('serviceEnabled').checked=true}