5 Commits

Author SHA1 Message Date
Maxim
a14001b373 Treat historical SMART failures as resolved 2026-08-03 21:17:15 +03:00
Maxim
05450bc949 Improve system health sorting and dashboard spacing 2026-08-03 21:09:30 +03:00
Maxim
8241fba6f4 Add IO attribution anomaly detection and alert correlation 2026-08-03 18:02:13 +03:00
Maxim
98afa2517b Add per-disk IO performance monitoring 2026-08-03 17:32:28 +03:00
Maxim
96af2faeb1 Add binary-compatible collection watchdog 2026-08-01 17:03:15 +03:00
14 changed files with 701 additions and 75 deletions

View File

@@ -121,6 +121,13 @@ ss -lntp | grep 9105
Панель показывает:
- изменение критичных SMART-счётчиков, результаты self-test и исчезновение дисков;
- realtime read/write IOPS, MB/s, latency, utilization и глубину очереди каждого диска;
- недельную историю IOPS/utilization и алерты при устойчивом I/O-давлении;
- вероятный источник I/O: VM/LXC, процесс хоста или активная backup/snapshot/migration-задача;
- отдельные пороги HDD, SATA SSD и NVMe с исключениями по имени или серийному номеру;
- недельные baseline-аномалии IOPS, latency, температуры, RAM, ZFS и длительности backup;
- корреляцию связанных событий в одну причинно-следственную цепочку;
- время первого появления и последнего наблюдения каждого уведомления;
- автоматический короткий SMART self-test раз в неделю для дисков без свежего теста;
- datasets, vdev-классы, ошибки read/write/checksum, scrub и ARC ZFS;
- доступность шлюза, DNS и интернета, packet loss, latency и внешний IP;

View File

@@ -57,7 +57,7 @@ func (c *activityCollector) collect(guests GuestsMetrics) ProxmoxActivity {
value := c.value
c.mu.RUnlock()
c.mu.Lock()
if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 10*time.Minute {
if !c.refreshing && time.Since(time.Unix(value.CheckedAt, 0)) > 15*time.Second {
c.refreshing = true
go c.refreshWithGuests(guests)
}

View File

@@ -74,7 +74,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
if err = tx.Commit(); err != nil {
return alerts, err
}
rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`)
rows, err = s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'warning' THEN 1 ELSE 2 END,first_seen DESC`)
if err != nil {
return alerts, err
}
@@ -82,7 +82,7 @@ func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
var visible []Alert
for rows.Next() {
var a Alert
if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message); err != nil {
if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message, &a.FirstSeen, &a.LastSeen); err != nil {
return alerts, err
}
visible = append(visible, a)

View File

@@ -10,7 +10,7 @@ func TestAlertLifecycle(t *testing.T) {
defer store.Close()
alert := Alert{ID: "disk", Severity: "warning", Source: "Диски", Title: "SMART", Message: "Проверьте диск"}
visible, err := store.SyncAlerts([]Alert{alert})
if err != nil || len(visible) != 1 || visible[0].EventID == 0 {
if err != nil || len(visible) != 1 || visible[0].EventID == 0 || visible[0].FirstSeen == 0 || visible[0].LastSeen == 0 {
t.Fatalf("unexpected new alert: %+v %v", visible, err)
}
firstID := visible[0].EventID

View File

@@ -8,12 +8,14 @@ import (
)
type Alert struct {
EventID int64 `json:"eventId,omitempty"`
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
EventID int64 `json:"eventId,omitempty"`
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
FirstSeen int64 `json:"firstSeen,omitempty"`
LastSeen int64 `json:"lastSeen,omitempty"`
}
func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Alert {
@@ -61,6 +63,14 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("root-storage", "warning", "Storage", "Мало места на системном разделе", fmt.Sprintf("Раздел / заполнен на %.1f%%.", metrics.Storage.UsagePercent))
}
for _, disk := range metrics.Disks {
diskExcluded := false
for _, value := range strings.Split(thresholds.DiskIOExcluded, ",") {
value = strings.TrimSpace(value)
if value != "" && (strings.EqualFold(value, disk.Name) || strings.EqualFold(value, disk.Serial)) {
diskExcluded = true
break
}
}
if disk.SMARTStatus == "Ошибка" {
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
} else if disk.SMARTStatus == "Требует внимания" {
@@ -78,6 +88,17 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
if len(disk.CounterChanges) > 0 {
add("disk-growth-"+disk.Name, "critical", "Диски", disk.Model+": ухудшились SMART-счётчики", strings.Join(disk.CounterChanges, " · "))
}
if !diskExcluded && disk.IOPressureSeconds >= thresholds.DiskIODuration {
severity := "warning"
if disk.Utilization >= 98 || disk.LatencyMs >= 100 {
severity = "critical"
}
cause := disk.IOCause
if cause == "" {
cause = "источник пока не определён"
}
add("disk-io-pressure-"+disk.Name, severity, "Диски", disk.Model+": длительная I/O-нагрузка", fmt.Sprintf("%s занят на %.1f%%; вероятная причина: %s (уверенность %d%%). %.0f IOPS, %.1f MB/s, latency %.1f ms, очередь %.1f.", disk.Name, disk.Utilization, cause, disk.IOConfidence, disk.ReadIOPS+disk.WriteIOPS, (disk.ReadBytesPerSec+disk.WriteBytesPerSec)/1048576, disk.LatencyMs, disk.AverageQueue))
}
if disk.SelfTestNeverRun {
add("disk-selftest-"+disk.Name, "warning", "Диски", disk.Model+": нет завершённого SMART self-test", "Короткий тест будет автоматически запущен; проверьте его результат после завершения.")
}
@@ -259,6 +280,9 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
add("task-failed-"+task.UPID, severity, "Proxmox", task.Type+": задача завершилась ошибкой", fmt.Sprintf("VMID %s, пользователь %s, статус %s.", task.ID, task.User, task.Status))
}
}
for _, trend := range metrics.Trends {
add(trend.ID, trend.Severity, trend.Source, trend.Title, trend.Message)
}
priority := map[string]int{"critical": 0, "warning": 1, "info": 2}
sort.SliceStable(alerts, func(i, j int) bool {
@@ -270,6 +294,51 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
return alerts
}
func correlateAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
pressureAlerts := map[string]bool{}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") {
pressureAlerts[strings.TrimPrefix(a.ID, "disk-io-pressure-")] = true
}
}
var pressure []DiskMetrics
for _, d := range metrics.Disks {
if pressureAlerts[d.Name] && d.IOCause != "" {
pressure = append(pressure, d)
}
}
if len(pressure) == 0 {
return alerts
}
var down []string
for _, service := range metrics.Services.Services {
for _, endpoint := range service.Endpoints {
if !endpoint.Up {
down = append(down, service.Name)
break
}
}
}
d := pressure[0]
chain := d.IOCause + " → " + d.Name + fmt.Sprintf(" util %.1f%% → latency %.1f ms", d.Utilization, d.LatencyMs)
if len(down) > 0 {
chain += " → недоступны/медленны сервисы: " + strings.Join(down, ", ")
}
severity := "warning"
if len(down) > 0 || d.Utilization >= 98 {
severity = "critical"
}
correlated := Alert{ID: "correlation-io-" + d.Name, Severity: severity, Source: "Корреляция", Title: "Обнаружена связанная цепочка I/O-событий", Message: chain + fmt.Sprintf(". Уверенность причины %d%%.", d.IOConfidence)}
filtered := []Alert{correlated}
for _, a := range alerts {
if strings.HasPrefix(a.ID, "disk-io-pressure-") || strings.HasPrefix(a.ID, "service-") {
continue
}
filtered = append(filtered, a)
}
return filtered
}
func joinReasons(reasons []string, fallback string) string {
if len(reasons) == 0 {
return fallback

187
disks.go
View File

@@ -13,46 +13,69 @@ import (
)
type DiskMetrics struct {
Name string `json:"name"`
Path string `json:"path"`
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"`
Protocol string `json:"protocol"`
SizeBytes uint64 `json:"sizeBytes"`
UsedBytes *uint64 `json:"usedBytes"`
UsagePercent *float64 `json:"usagePercent"`
SMARTStatus string `json:"smartStatus"`
SMARTAvailable bool `json:"smartAvailable"`
AttentionReasons []string `json:"attentionReasons"`
Temperature *float64 `json:"temperatureCelsius"`
PowerOnHours *uint64 `json:"powerOnHours"`
WearUsedPercent *float64 `json:"wearUsedPercent"`
Reallocated uint64 `json:"reallocated"`
Pending uint64 `json:"pending"`
Uncorrectable uint64 `json:"uncorrectable"`
CRCErrors uint64 `json:"crcErrors"`
MediaErrors uint64 `json:"mediaErrors"`
CounterChanges []string `json:"counterChanges"`
LastSelfTest string `json:"lastSelfTest"`
LastSelfTestAt string `json:"lastSelfTestAt"`
SelfTestNeverRun bool `json:"selfTestNeverRun"`
ShortTestStarted bool `json:"shortTestStarted"`
Name string `json:"name"`
Path string `json:"path"`
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"`
Protocol string `json:"protocol"`
SizeBytes uint64 `json:"sizeBytes"`
UsedBytes *uint64 `json:"usedBytes"`
UsagePercent *float64 `json:"usagePercent"`
SMARTStatus string `json:"smartStatus"`
SMARTAvailable bool `json:"smartAvailable"`
AttentionReasons []string `json:"attentionReasons"`
SMARTHistory []string `json:"smartHistory"`
Temperature *float64 `json:"temperatureCelsius"`
PowerOnHours *uint64 `json:"powerOnHours"`
WearUsedPercent *float64 `json:"wearUsedPercent"`
Reallocated uint64 `json:"reallocated"`
Pending uint64 `json:"pending"`
Uncorrectable uint64 `json:"uncorrectable"`
CRCErrors uint64 `json:"crcErrors"`
MediaErrors uint64 `json:"mediaErrors"`
SMARTErrorCount uint64 `json:"smartErrorCount"`
CounterChanges []string `json:"counterChanges"`
LastSelfTest string `json:"lastSelfTest"`
LastSelfTestAt string `json:"lastSelfTestAt"`
SelfTestNeverRun bool `json:"selfTestNeverRun"`
ShortTestStarted bool `json:"shortTestStarted"`
ReadIOPS float64 `json:"readIops"`
WriteIOPS float64 `json:"writeIops"`
ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
ReadLatencyMs float64 `json:"readLatencyMs"`
WriteLatencyMs float64 `json:"writeLatencyMs"`
LatencyMs float64 `json:"latencyMs"`
Utilization float64 `json:"utilizationPercent"`
QueueDepth uint64 `json:"queueDepth"`
AverageQueue float64 `json:"averageQueue"`
IOPressureSeconds float64 `json:"ioPressureSeconds"`
IOCause string `json:"ioCause"`
IOCauseDetail string `json:"ioCauseDetail"`
IOConfidence int `json:"ioConfidence"`
TopConsumers []IOConsumer `json:"topConsumers"`
}
type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 }
type diskCollector struct {
mu sync.Mutex
cached []DiskMetrics
updatedAt time.Time
previous map[string]DiskMetrics
missing []string
lastTests map[string]time.Time
mu sync.Mutex
cached []DiskMetrics
updatedAt time.Time
previous map[string]DiskMetrics
missing []string
lastTests map[string]time.Time
ioPrevious map[string]diskIOSnapshot
ioUpdatedAt time.Time
ioPressureSince map[string]time.Time
}
func (c *diskCollector) collect() []DiskMetrics {
func (c *diskCollector) collect(thresholds AlertThresholds) []DiskMetrics {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
c.applyDiskIO(c.cached, thresholds)
return c.cached
}
current := readPhysicalDisks()
@@ -87,10 +110,91 @@ func (c *diskCollector) collect() []DiskMetrics {
}
}
c.cached = current
c.applyDiskIO(c.cached, thresholds)
c.updatedAt = time.Now()
return c.cached
}
func (c *diskCollector) applyDiskIO(disks []DiskMetrics, thresholds AlertThresholds) {
now := time.Now()
snapshots := readDiskIOStats("/proc/diskstats")
if c.ioPrevious == nil {
c.ioPrevious = map[string]diskIOSnapshot{}
c.ioPressureSince = map[string]time.Time{}
}
elapsed := now.Sub(c.ioUpdatedAt).Seconds()
for i := range disks {
disk := &disks[i]
disk.ReadIOPS, disk.WriteIOPS, disk.ReadBytesPerSec, disk.WriteBytesPerSec = 0, 0, 0, 0
disk.ReadLatencyMs, disk.WriteLatencyMs, disk.LatencyMs, disk.Utilization, disk.AverageQueue, disk.IOPressureSeconds = 0, 0, 0, 0, 0, 0
current, ok := snapshots[disk.Name]
if !ok {
continue
}
disk.QueueDepth = current.inFlight
if old, found := c.ioPrevious[disk.Name]; found && elapsed > 0 && diskIOCountersMonotonic(old, current) {
readOps := current.reads - old.reads
writeOps := current.writes - old.writes
disk.ReadIOPS = float64(readOps) / elapsed
disk.WriteIOPS = float64(writeOps) / elapsed
disk.ReadBytesPerSec = float64(current.readSectors-old.readSectors) * 512 / elapsed
disk.WriteBytesPerSec = float64(current.writeSectors-old.writeSectors) * 512 / elapsed
if readOps > 0 {
disk.ReadLatencyMs = float64(current.readMS-old.readMS) / float64(readOps)
}
if writeOps > 0 {
disk.WriteLatencyMs = float64(current.writeMS-old.writeMS) / float64(writeOps)
}
if readOps+writeOps > 0 {
disk.LatencyMs = float64(current.readMS-old.readMS+current.writeMS-old.writeMS) / float64(readOps+writeOps)
}
disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100)
disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000)
utilLimit, latencyLimit, queueLimit := thresholds.DiskSSDUtil, thresholds.DiskSSDLatency, thresholds.DiskSSDQueue
if disk.Type == "HDD" {
utilLimit, latencyLimit, queueLimit = thresholds.DiskHDDUtil, thresholds.DiskHDDLatency, thresholds.DiskHDDQueue
} else if strings.Contains(disk.Type, "NVMe") {
utilLimit, latencyLimit, queueLimit = thresholds.DiskNVMeUtil, thresholds.DiskNVMeLatency, thresholds.DiskNVMeQueue
}
pressured := disk.Utilization >= utilLimit || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= queueLimit
if pressured {
if c.ioPressureSince[disk.Name].IsZero() {
c.ioPressureSince[disk.Name] = now
}
disk.IOPressureSeconds = now.Sub(c.ioPressureSince[disk.Name]).Seconds()
} else {
delete(c.ioPressureSince, disk.Name)
}
}
c.ioPrevious[disk.Name] = current
}
c.ioUpdatedAt = now
}
func diskIOCountersMonotonic(old, current diskIOSnapshot) bool {
return current.reads >= old.reads && current.readSectors >= old.readSectors && current.readMS >= old.readMS && current.writes >= old.writes && current.writeSectors >= old.writeSectors && current.writeMS >= old.writeMS && current.ioMS >= old.ioMS && current.weightedMS >= old.weightedMS
}
func readDiskIOStats(path string) map[string]diskIOSnapshot {
data, err := os.ReadFile(path)
if err != nil {
return nil
}
result := map[string]diskIOSnapshot{}
for _, line := range strings.Split(string(data), "\n") {
f := strings.Fields(line)
if len(f) < 14 || !isPhysicalDiskName(f[2]) {
continue
}
values := make([]uint64, 11)
for i := 0; i < 11; i++ {
values[i], _ = strconv.ParseUint(f[i+3], 10, 64)
}
result[f[2]] = diskIOSnapshot{reads: values[0], readSectors: values[2], readMS: values[3], writes: values[4], writeSectors: values[6], writeMS: values[7], inFlight: values[8], ioMS: values[9], weightedMS: values[10]}
}
return result
}
func (c *diskCollector) missingDisks() []string {
c.mu.Lock()
defer c.mu.Unlock()
@@ -101,7 +205,7 @@ func diskCounterChanges(old, current DiskMetrics) []string {
checks := []struct {
label string
old, now uint64
}{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}}
}{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}, {"Ошибки SMART-журнала выросли", old.SMARTErrorCount, current.SMARTErrorCount}}
var changes []string
for _, check := range checks {
if check.now > check.old {
@@ -386,16 +490,23 @@ func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
wear := float64(100 - attribute.Value)
disk.WearUsedPercent = &wear
}
if attribute.WhenFailed != "" && attribute.WhenFailed != "-" {
if strings.EqualFold(attribute.WhenFailed, "FAILING_NOW") {
reason := attribute.Name + " вышел за порог"
if attribute.Raw.String != "" {
reason += " (" + attribute.Raw.String + ")"
}
disk.AttentionReasons = append(disk.AttentionReasons, reason)
} else if attribute.WhenFailed != "" && attribute.WhenFailed != "-" {
reason := attribute.Name + " ранее выходил за порог"
if attribute.Raw.String != "" {
reason += " (" + attribute.Raw.String + ")"
}
disk.SMARTHistory = append(disk.SMARTHistory, reason)
}
}
if smart.ATAErrorLog.Summary.Count > 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибок в SMART-журнале: "+strconv.Itoa(smart.ATAErrorLog.Summary.Count))
disk.SMARTErrorCount = uint64(smart.ATAErrorLog.Summary.Count)
disk.SMARTHistory = append(disk.SMARTHistory, "Записей в SMART-журнале: "+strconv.Itoa(smart.ATAErrorLog.Summary.Count))
}
}
@@ -409,12 +520,12 @@ func applySMARTExitStatus(disk *DiskMetrics, status int) {
disk.AttentionReasons = append(disk.AttentionReasons, "Критический SMART-атрибут достиг порога")
}
if status&32 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "SMART-атрибут ранее находился ниже порога")
disk.SMARTHistory = append(disk.SMARTHistory, "SMART-атрибут ранее находился ниже порога")
}
if status&64 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "В журнале SMART обнаружены ошибки")
disk.SMARTHistory = append(disk.SMARTHistory, "В журнале SMART есть старые записи об ошибках")
}
if status&128 != 0 {
disk.AttentionReasons = append(disk.AttentionReasons, "В журнале самотестирования есть ошибки")
disk.SMARTHistory = append(disk.SMARTHistory, "В журнале самотестирования есть старые ошибки")
}
}

View File

@@ -2,6 +2,9 @@ package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -17,6 +20,22 @@ func TestIsPhysicalDiskName(t *testing.T) {
}
}
func TestReadDiskIOStats(t *testing.T) {
path := filepath.Join(t.TempDir(), "diskstats")
data := " 8 0 sda 100 2 300 40 50 3 700 80 4 90 120 0 0 0 0 0 0\n 7 0 loop0 999 0 999 0 999 0 999 0 0 0 0\n"
if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
t.Fatal(err)
}
stats := readDiskIOStats(path)
disk, ok := stats["sda"]
if !ok || disk.reads != 100 || disk.readSectors != 300 || disk.writes != 50 || disk.writeSectors != 700 || disk.inFlight != 4 || disk.ioMS != 90 || disk.weightedMS != 120 {
t.Fatalf("unexpected stats: %+v", disk)
}
if _, exists := stats["loop0"]; exists {
t.Fatal("virtual device must be ignored")
}
}
func TestSumFilesystemUsage(t *testing.T) {
device := lsblkDevice{Children: []lsblkDevice{
{FSUsed: json.RawMessage(`100`), FSAvail: json.RawMessage(`300`)},
@@ -50,3 +69,37 @@ func TestApplyATAWearAndReason(t *testing.T) {
t.Fatalf("причина предупреждения не добавлена: %+v", disk)
}
}
func TestHistoricalATAFailureDoesNotRequireAttention(t *testing.T) {
var smart smartctlJSON
attribute := struct {
Name string `json:"name"`
Value int `json:"value"`
Threshold int `json:"thresh"`
WhenFailed string `json:"when_failed"`
Raw struct {
String string `json:"string"`
Value int64 `json:"value"`
} `json:"raw"`
}{Name: "Airflow_Temperature_Cel", WhenFailed: "In_the_past"}
attribute.Raw.String = "46 (Min/Max 26/47 #15)"
smart.ATAAttributes.Table = append(smart.ATAAttributes.Table, attribute)
disk := DiskMetrics{}
applyATAAttributes(&disk, smart)
applySMARTExitStatus(&disk, 32|64|128)
if len(disk.AttentionReasons) != 0 {
t.Fatalf("историческая ошибка стала активной: %+v", disk.AttentionReasons)
}
if len(disk.SMARTHistory) != 4 {
t.Fatalf("история SMART потеряна: %+v", disk.SMARTHistory)
}
}
func TestSMARTErrorLogGrowthRequiresAttention(t *testing.T) {
old := DiskMetrics{SMARTErrorCount: 2}
current := DiskMetrics{SMARTErrorCount: 3}
changes := diskCounterChanges(old, current)
if len(changes) != 1 || !strings.Contains(changes[0], "2 → 3") {
t.Fatalf("рост журнала не обнаружен: %+v", changes)
}
}

View File

@@ -44,6 +44,11 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
s.AddEntityHistory("guest", fmt.Sprintf("%d · %s", g.VMID, g.Name), ts, &cpu, &mem)
}
for _, d := range metrics.Disks {
iops := d.ReadIOPS + d.WriteIOPS
util := d.Utilization
s.AddEntityHistory("disk-io", d.Name+" · "+d.Model, ts, &iops, &util)
latency := d.LatencyMs
s.AddEntityHistory("disk-latency", d.Name+" · "+d.Model, ts, &latency, nil)
if d.Temperature == nil {
continue
}
@@ -53,6 +58,12 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
}
s.AddEntityHistory("disk", d.Name+" · "+d.Model, ts, d.Temperature, wear)
}
for _, task := range metrics.Activity.Tasks {
if task.Type == "vzdump" && task.Duration > 0 && task.EndTime >= time.Now().Add(-15*time.Minute).Unix() {
duration := float64(task.Duration)
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
}
}
s.PruneEntityHistory()
}

178
io_attribution.go Normal file
View File

@@ -0,0 +1,178 @@
package main
import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type IOConsumer struct {
Kind string `json:"kind"`
Name string `json:"name"`
VMID int `json:"vmid,omitempty"`
ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
SharePercent float64 `json:"sharePercent"`
}
type processIOSnapshot struct {
read, write uint64
name, cmdline, cgroup string
}
type ioAttributionCollector struct {
mu sync.Mutex
previous map[int]processIOSnapshot
updatedAt time.Time
cached []IOConsumer
}
var qemuIDPattern = regexp.MustCompile(`(?:^|\s)-id\s+(\d+)`)
var lxcIDPattern = regexp.MustCompile(`(?:lxc(?:\.payload)?[./-]|pve-container@)(\d+)`)
func (c *ioAttributionCollector) collect(guests GuestsMetrics) []IOConsumer {
c.mu.Lock()
defer c.mu.Unlock()
now := time.Now()
current := readProcessIO()
elapsed := now.Sub(c.updatedAt).Seconds()
if c.previous == nil || elapsed <= 0 {
c.previous = current
c.updatedAt = now
return nil
}
names := map[int]string{}
for _, g := range guests.Guests {
names[g.VMID] = g.Name
}
grouped := map[string]*IOConsumer{}
var total float64
for pid, snapshot := range current {
old, ok := c.previous[pid]
if !ok || snapshot.read < old.read || snapshot.write < old.write {
continue
}
read := float64(snapshot.read-old.read) / elapsed
write := float64(snapshot.write-old.write) / elapsed
if read+write < 1024 {
continue
}
kind, name, vmid := classifyIOProcess(snapshot, names)
key := kind + ":" + strconv.Itoa(vmid) + ":" + name
consumer := grouped[key]
if consumer == nil {
consumer = &IOConsumer{Kind: kind, Name: name, VMID: vmid}
grouped[key] = consumer
}
consumer.ReadBytesPerSec += read
consumer.WriteBytesPerSec += write
total += read + write
}
var result []IOConsumer
for _, v := range grouped {
if total > 0 {
v.SharePercent = (v.ReadBytesPerSec + v.WriteBytesPerSec) / total * 100
}
result = append(result, *v)
}
sort.Slice(result, func(i, j int) bool {
return result[i].ReadBytesPerSec+result[i].WriteBytesPerSec > result[j].ReadBytesPerSec+result[j].WriteBytesPerSec
})
if len(result) > 8 {
result = result[:8]
}
c.previous = current
c.updatedAt = now
c.cached = result
return result
}
func classifyIOProcess(p processIOSnapshot, names map[int]string) (kind, name string, vmid int) {
if match := qemuIDPattern.FindStringSubmatch(p.cmdline); len(match) > 1 {
vmid, _ = strconv.Atoi(match[1])
return "VM", firstNonEmpty(names[vmid], "VM "+match[1]), vmid
}
if match := lxcIDPattern.FindStringSubmatch(p.cgroup); len(match) > 1 {
vmid, _ = strconv.Atoi(match[1])
return "LXC", firstNonEmpty(names[vmid], "LXC "+match[1]), vmid
}
return "Процесс", firstNonEmpty(p.name, "PID"), 0
}
func readProcessIO() map[int]processIOSnapshot {
result := map[int]processIOSnapshot{}
entries, _ := filepath.Glob("/proc/[0-9]*/io")
for _, path := range entries {
pid, _ := strconv.Atoi(filepath.Base(filepath.Dir(path)))
data, err := os.ReadFile(path)
if err != nil {
continue
}
var item processIOSnapshot
for _, line := range strings.Split(string(data), "\n") {
key, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
n, _ := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
if key == "read_bytes" {
item.read = n
} else if key == "write_bytes" {
item.write = n
}
}
comm, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "comm"))
cmd, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cmdline"))
cg, _ := os.ReadFile(filepath.Join(filepath.Dir(path), "cgroup"))
item.name = strings.TrimSpace(string(comm))
item.cmdline = strings.ReplaceAll(string(cmd), "\x00", " ")
item.cgroup = string(cg)
result[pid] = item
}
return result
}
func attributeDiskIO(metrics *dashboardMetrics, consumers []IOConsumer) {
active := activeIOOperation(metrics.Activity.Tasks)
for i := range metrics.Disks {
disk := &metrics.Disks[i]
disk.TopConsumers = append([]IOConsumer(nil), consumers...)
if active != "" {
disk.IOCause = active
disk.IOCauseDetail = "Активная задача Proxmox совпадает по времени с дисковой нагрузкой"
disk.IOConfidence = 75
} else if len(consumers) > 0 {
top := consumers[0]
disk.IOCause = top.Kind + " · " + top.Name
disk.IOCauseDetail = "Крупнейший потребитель host I/O: " + strconv.FormatFloat(top.SharePercent, 'f', 1, 64) + "%"
disk.IOConfidence = 55
}
}
}
func activeIOOperation(tasks []ProxmoxTask) string {
for _, task := range tasks {
if task.EndTime != 0 && task.Status != "" {
continue
}
label := ""
switch task.Type {
case "vzdump":
label = "Backup"
case "qmigrate":
label = "Миграция"
case "qmsnapshot", "vzsnapshot":
label = "Snapshot"
}
if label != "" {
if task.ID != "" {
label += " VM/LXC " + task.ID
}
return label
}
}
return ""
}

33
io_attribution_test.go Normal file
View File

@@ -0,0 +1,33 @@
package main
import (
"strings"
"testing"
)
func TestClassifyQEMUAndLXCIO(t *testing.T) {
kind, name, vmid := classifyIOProcess(processIOSnapshot{name: "qemu", cmdline: "qemu-system-x86_64 -id 105 -name guest=test"}, map[int]string{105: "Gitea"})
if kind != "VM" || name != "Gitea" || vmid != 105 {
t.Fatalf("unexpected qemu attribution: %s %s %d", kind, name, vmid)
}
kind, name, vmid = classifyIOProcess(processIOSnapshot{name: "worker", cgroup: "0::/lxc.payload.220/ns"}, map[int]string{220: "db"})
if kind != "LXC" || name != "db" || vmid != 220 {
t.Fatalf("unexpected lxc attribution: %s %s %d", kind, name, vmid)
}
}
func TestActiveIOOperation(t *testing.T) {
value := activeIOOperation([]ProxmoxTask{{Type: "vzdump", ID: "105", Status: "", EndTime: 0}})
if !strings.Contains(value, "Backup") || !strings.Contains(value, "105") {
t.Fatalf("unexpected operation: %s", value)
}
}
func TestCorrelateAlerts(t *testing.T) {
metrics := dashboardMetrics{Disks: []DiskMetrics{{Name: "sda", IOCause: "Backup VM/LXC 105", IOConfidence: 90, Utilization: 98, LatencyMs: 80}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Up: false}}}}}}
alerts := []Alert{{ID: "disk-io-pressure-sda", Severity: "critical"}, {ID: "service-1-public", Severity: "critical"}, {ID: "other", Severity: "warning"}}
result := correlateAlerts(metrics, alerts)
if len(result) != 2 || result[0].Source != "Корреляция" || !strings.Contains(result[0].Message, "Gitea") {
t.Fatalf("unexpected correlation: %+v", result)
}
}

View File

@@ -63,6 +63,24 @@ type monitorTracker struct {
duration time.Duration
}
func runCollectionWatchdog(monitor *monitorTracker) {
started := time.Now()
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for range ticker.C {
monitor.mu.RLock()
last := monitor.lastSuccess
monitor.mu.RUnlock()
if time.Since(started) < 2*time.Minute {
continue
}
if last.IsZero() || time.Since(last) > 90*time.Second {
log.Printf("Встроенный watchdog: сбор метрик не завершался более 90 секунд; перезапускаем процесс")
os.Exit(1)
}
}
}
func (m *monitorTracker) record(start time.Time, err error) {
m.mu.Lock()
defer m.mu.Unlock()

View File

@@ -31,6 +31,17 @@ type AlertThresholds struct {
NVMeTempWarning float64 `json:"nvmeTempWarning"`
UPSChargeCritical float64 `json:"upsChargeCritical"`
BackupMaxAgeHours float64 `json:"backupMaxAgeHours"`
DiskHDDUtil float64 `json:"diskHddUtil"`
DiskHDDLatency float64 `json:"diskHddLatency"`
DiskHDDQueue float64 `json:"diskHddQueue"`
DiskSSDUtil float64 `json:"diskSsdUtil"`
DiskSSDLatency float64 `json:"diskSsdLatency"`
DiskSSDQueue float64 `json:"diskSsdQueue"`
DiskNVMeUtil float64 `json:"diskNvmeUtil"`
DiskNVMeLatency float64 `json:"diskNvmeLatency"`
DiskNVMeQueue float64 `json:"diskNvmeQueue"`
DiskIODuration float64 `json:"diskIoDuration"`
DiskIOExcluded string `json:"diskIoExcluded"`
}
func defaultThresholds() AlertThresholds {
@@ -40,6 +51,7 @@ func defaultThresholds() AlertThresholds {
StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90,
DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30,
BackupMaxAgeHours: 48,
DiskHDDUtil: 90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60,
}
}
@@ -285,6 +297,19 @@ func validateThresholds(t AlertThresholds) error {
if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 {
return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов")
}
for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} {
if value < 1 || value > 100 {
return fmt.Errorf("пороги utilization должны быть от 1 до 100")
}
}
for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} {
if value < 0 {
return fmt.Errorf("пороги latency и очереди не могут быть отрицательными")
}
}
if t.DiskIODuration < 5 || t.DiskIODuration > 3600 {
return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд")
}
return nil
}

107
trends.go Normal file
View File

@@ -0,0 +1,107 @@
package main
import (
"database/sql"
"fmt"
"sync"
"time"
)
type entityAverage struct {
Value1, Value2 float64
Count int
}
type TrendAnomaly struct {
ID string `json:"id"`
Severity string `json:"severity"`
Source string `json:"source"`
Title string `json:"title"`
Message string `json:"message"`
}
type trendCollector struct {
mu sync.Mutex
updatedAt time.Time
cached []TrendAnomaly
}
func (s *Store) entityAverages(kind string) map[string]entityAverage {
return s.entityAveragesSince(kind, 7*24*time.Hour)
}
func (s *Store) entityAveragesSince(kind string, duration time.Duration) map[string]entityAverage {
rows, err := s.db.Query(`SELECT name,AVG(value1),AVG(value2),COUNT(*) FROM entity_history WHERE kind=? AND ts>=? GROUP BY name`, kind, time.Now().Add(-duration).Unix())
if err != nil {
return nil
}
defer rows.Close()
result := map[string]entityAverage{}
for rows.Next() {
var name string
var v1, v2 sql.NullFloat64
var count int
if rows.Scan(&name, &v1, &v2, &count) == nil {
result[name] = entityAverage{Value1: v1.Float64, Value2: v2.Float64, Count: count}
}
}
return result
}
func (c *trendCollector) collect(metrics dashboardMetrics, store *Store) []TrendAnomaly {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.updatedAt) < time.Minute {
return c.cached
}
var out []TrendAnomaly
iops := store.entityAverages("disk-io")
latency := store.entityAverages("disk-latency")
latencyDay := store.entityAveragesSince("disk-latency", 24*time.Hour)
temps := store.entityAverages("disk")
for _, d := range metrics.Disks {
name := d.Name + " · " + d.Model
currentIOPS := d.ReadIOPS + d.WriteIOPS
if base := iops[name]; base.Count >= 30 && base.Value1 >= 1 && currentIOPS >= base.Value1*5 {
out = append(out, TrendAnomaly{"disk-iops-" + d.Name, "warning", "Аномалии", d.Model + ": IOPS резко выше обычного", fmt.Sprintf("Сейчас %.0f IOPS, недельный baseline %.0f — рост в %.1f раза.", currentIOPS, base.Value1, currentIOPS/base.Value1)})
}
if base := latency[name]; base.Count >= 30 && base.Value1 > 0 && d.LatencyMs >= base.Value1*2 && d.LatencyMs-base.Value1 >= 10 {
out = append(out, TrendAnomaly{"disk-latency-" + d.Name, "warning", "Аномалии", d.Model + ": latency ухудшилась", fmt.Sprintf("Сейчас %.1f ms при baseline %.1f ms.", d.LatencyMs, base.Value1)})
}
if week, day := latency[name], latencyDay[name]; week.Count >= 100 && day.Count >= 30 && week.Value1 > 0 && day.Value1 >= week.Value1*1.5 && day.Value1-week.Value1 >= 5 {
out = append(out, TrendAnomaly{"disk-latency-degrading-" + d.Name, "warning", "Аномалии", d.Model + ": latency постепенно ухудшается", fmt.Sprintf("Среднее за сутки %.1f ms против недельного %.1f ms.", day.Value1, week.Value1)})
}
if d.Temperature != nil {
if base := temps[name]; base.Count >= 30 && *d.Temperature >= base.Value1+10 {
out = append(out, TrendAnomaly{"disk-temp-trend-" + d.Name, "warning", "Аномалии", d.Model + ": температура выше обычной", fmt.Sprintf("Сейчас %.1f °C, baseline %.1f °C.", *d.Temperature, base.Value1)})
}
}
}
guestAvg := store.entityAverages("guest")
for _, g := range metrics.Guests.Guests {
if g.MaxMemory == 0 {
continue
}
name := fmt.Sprintf("%d · %s", g.VMID, g.Name)
current := float64(g.MemoryBytes) / float64(g.MaxMemory) * 100
if base := guestAvg[name]; base.Count >= 30 && current >= base.Value2*1.5 && current-base.Value2 >= 15 {
out = append(out, TrendAnomaly{fmt.Sprintf("guest-memory-%d", g.VMID), "warning", "Аномалии", g.Name + ": память выше обычной", fmt.Sprintf("Сейчас %.1f%%, baseline %.1f%%.", current, base.Value2)})
}
}
zfsAverage := store.entityAverages("zfs")
for _, pool := range metrics.ZFS.Pools {
if base := zfsAverage[pool.Name]; base.Count >= 30 && pool.CapacityPercent-base.Value1 >= 5 {
out = append(out, TrendAnomaly{"zfs-growth-" + pool.Name, "warning", "Аномалии", pool.Name + ": заполнение выше недельного уровня", fmt.Sprintf("Сейчас %.1f%%, недельный baseline %.1f%%.", pool.CapacityPercent, base.Value1)})
}
}
backupAverage := store.entityAverages("backup-duration")
for _, task := range metrics.Activity.Tasks {
if task.Type != "vzdump" || task.Duration <= 0 {
continue
}
name := "VMID " + task.ID
if base := backupAverage[name]; base.Count >= 3 && float64(task.Duration) >= base.Value1*1.5 && float64(task.Duration)-base.Value1 >= 60 {
out = append(out, TrendAnomaly{"backup-duration-" + task.ID, "warning", "Аномалии", "Backup " + name + " выполнялся дольше обычного", fmt.Sprintf("Последняя длительность %s, baseline %s.", time.Duration(task.Duration)*time.Second, time.Duration(base.Value1)*time.Second)})
}
}
c.cached = out
c.updatedAt = time.Now()
return out
}

68
web.go
View File

@@ -29,10 +29,11 @@ type dashboardMetrics struct {
Activity ProxmoxActivity `json:"activity"`
Monitor MonitorHealth `json:"monitor"`
UPSShutdown UPSShutdownSettings `json:"upsShutdown"`
Trends []TrendAnomaly `json:"trends"`
Timestamp time.Time `json:"timestamp"`
}
func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) {
func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, activity *activityCollector, ioAttribution *ioAttributionCollector, trends *trendCollector, store *Store, monitor *monitorTracker) (dashboardMetrics, error) {
cpu, err := cpuCollector.collect()
if err != nil {
return dashboardMetrics{}, err
@@ -49,9 +50,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle
if err != nil {
return dashboardMetrics{}, err
}
thresholds := store.Thresholds()
metrics := dashboardMetrics{
CPU: cpu, Memory: memory, Storage: storage,
Disks: diskCollector.collect(), Network: network,
Disks: diskCollector.collect(thresholds), Network: network,
ZFS: zfsCollector.collect(), UPS: upsCollector.collect(),
Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Services: serviceCollector.collect(), SystemHealth: systemHealth.collect(), Update: readUpdateStatus(), Timestamp: time.Now(),
}
@@ -59,8 +61,10 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle
metrics.Monitor = monitor.snapshot(store)
metrics.UPSShutdown = store.UPSShutdownSettings()
metrics.Activity = activity.collect(metrics.Guests)
attributeDiskIO(&metrics, ioAttribution.collect(metrics.Guests))
metrics.Trends = trends.collect(metrics, store)
decorateNetworkInterfaces(&metrics.Network, metrics.Guests)
metrics.Alerts = evaluateAlerts(metrics, store.Thresholds())
metrics.Alerts = correlateAlerts(metrics, evaluateAlerts(metrics, thresholds))
metrics.Alerts, err = store.SyncAlerts(metrics.Alerts)
if err != nil {
return dashboardMetrics{}, err
@@ -121,11 +125,14 @@ func routes(collector *cpuCollector, store *Store) http.Handler {
services := newServiceCollector(store)
systemHealth := newSystemHealthCollector()
activity := newActivityCollector()
ioAttribution := &ioAttributionCollector{}
trends := &trendCollector{}
monitor := &monitorTracker{}
go runCollectionWatchdog(monitor)
go runUPSShutdownController(store, ups, guests)
collect := func() (dashboardMetrics, error) {
started := time.Now()
metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, store, monitor)
metrics, err := collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, activity, ioAttribution, trends, store, monitor)
monitor.record(started, err)
return metrics, err
}
@@ -366,7 +373,7 @@ button{font:inherit}main{width:min(1060px,calc(100% - 32px));margin:auto;padding
header{display:flex;align-items:end;justify-content:space-between;margin-bottom:22px}.eyebrow{color:var(--green);font:700 11px/1 monospace;letter-spacing:.16em;text-transform:uppercase}
h1{font-size:clamp(28px,5vw,44px);letter-spacing:-.045em;margin:9px 0 0}.header-actions,.status{display:flex;align-items:center;gap:10px}.status{color:var(--muted);font-size:13px}.dot{width:8px;height:8px;border-radius:50%;background:#f39a56}.dot.live{background:var(--green);box-shadow:0 0 15px var(--green)}.settings-button{width:38px;height:38px;border:1px solid var(--line);border-radius:12px;background:var(--panel);color:var(--text);cursor:pointer;font-size:18px}.settings-button:hover{border-color:var(--muted)}
.cards{display:grid;grid-template-columns:1fr 1fr;grid-auto-rows:1fr;gap:16px}.card{appearance:none;display:flex;flex-direction:column;text-align:left;color:inherit;width:100%;padding:0;background:linear-gradient(145deg,rgba(18,28,36,.98),rgba(12,19,25,.98));border:1px solid var(--line);border-radius:20px;overflow:hidden;cursor:pointer;transition:transform .18s,border-color .18s,box-shadow .18s}.card:hover{transform:translateY(-2px);border-color:#3d4e5c;box-shadow:0 20px 55px rgba(0,0,0,.22)}.card:focus-visible{outline:2px solid var(--green);outline-offset:3px}
.card-head{display:flex;justify-content:space-between;align-items:flex-start;gap:20px;min-height:106px;padding:23px 24px 18px}.card-head>div:first-child{min-width:0}.label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em}.name{font-size:17px;font-weight:650;margin-top:6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:350px}.percent{flex:0 0 auto;font:700 35px/.95 ui-monospace,monospace;letter-spacing:-.07em}.percent small{font-size:16px;color:var(--green)}
.card-head{display:flex;justify-content:space-between;align-items:flex-start;gap:20px;min-height:106px;padding:23px 24px 18px}.card-head>div:first-child{min-width:0}.label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.1em}.name{font-size:17px;font-weight:650;margin-top:6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:350px}.percent{display:inline-flex;align-items:baseline;gap:.22em;flex:0 0 auto;font:700 35px/.95 ui-monospace,monospace;letter-spacing:-.04em}.percent small{font-size:16px;color:var(--green);letter-spacing:-.02em}
.bar{height:5px;background:#222e37;margin:0 24px;overflow:hidden;border-radius:20px}.fill{height:100%;width:0;background:linear-gradient(90deg,var(--green),#afe36d);transition:width .5s ease}.memory .fill{background:linear-gradient(90deg,var(--violet),#72b9ff)}
.storage .fill{background:linear-gradient(90deg,#ffb35c,#ff755c)}.network .fill{background:linear-gradient(90deg,#43c6d9,#65ddaa)}.zfs .fill{background:linear-gradient(90deg,#46c0ff,#6577ff)}.ups .fill{background:linear-gradient(90deg,#d5e85a,#55d58a)}.guests .fill{background:linear-gradient(90deg,#ff78b8,#9d8cff)}.backups .fill{background:linear-gradient(90deg,#ffcf5c,#55d58a)}.alert-center{grid-column:1/-1}.alert-center .fill{background:linear-gradient(90deg,#ffb35c,#ff665c)}
.quick{display:grid;grid-template-columns:1fr 1fr;margin-top:19px;border-top:1px solid var(--line)}.quick div{padding:16px 24px}.quick div+div{border-left:1px solid var(--line)}.quick strong{display:block;font:650 20px/1.2 ui-monospace,monospace;margin-top:5px}.hint{color:#657581;font-size:11px;margin:auto 24px 18px;padding-top:14px}
@@ -381,7 +388,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
@media(max-width:700px){main{padding:28px 0}.cards{grid-template-columns:1fr}header{align-items:flex-start;flex-direction:column;gap:14px}.detail-grid{grid-template-columns:repeat(2,1fr)}.detail:nth-child(3n){border-right:1px solid var(--line)}.detail:nth-child(2n){border-right:0}.name{max-width:220px}.disk-stats,.setting-list,.threshold-grid{grid-template-columns:repeat(2,1fr)}.service-form,.maintenance-grid{grid-template-columns:1fr}.maintenance-grid .detail{border-right:0}.chart-head{align-items:flex-start;flex-direction:column}.detail-tabs,.settings-tabs,.system-tabs{overflow-x:auto}}
.save-settings:disabled{cursor:wait;opacity:.55}[data-settings-panel="email"] .threshold-item input{width:min(210px,52vw)}.ack-alert{margin-top:11px;padding:6px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);font-size:11px;cursor:pointer}.history-state{display:inline-block;margin-top:8px;color:var(--muted);font-size:11px}.alert-panel[hidden]{display:none}
.maintenance .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.card-order{display:grid;gap:7px;margin-top:11px}.order-item{display:flex;align-items:center;gap:10px;padding:9px 10px;border:1px solid var(--line);border-radius:10px;background:var(--panel)}.order-item.dragging{opacity:.45}.order-handle{cursor:grab;color:var(--muted);font-size:17px}.order-item span:nth-child(2){flex:1}.order-button{width:30px;height:28px;border:1px solid var(--line);border-radius:7px;background:var(--bg);color:var(--text);cursor:pointer}
.services .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.system-health .fill{background:linear-gradient(90deg,#6ba8ff,#9d8cff)}.service-form{display:grid;grid-template-columns:1fr 1fr;gap:12px}.field{display:grid;gap:6px;min-width:0}.field>span{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.07em}.field small{color:var(--muted);font-size:11px}.field input,.field select{width:100%;min-width:0;padding:11px 12px;border:1px solid var(--line);border-radius:9px;background:var(--bg);color:var(--text)}.check-field{display:flex;align-items:center;gap:10px;padding:12px;border:1px solid var(--line);border-radius:9px}.check-field input{width:18px;height:18px;accent-color:var(--green)}.service-actions{display:flex;flex-wrap:wrap;gap:7px;margin-top:10px}.service-actions button,.open-address{padding:7px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);cursor:pointer;text-decoration:none}.status-up{color:var(--green)}.status-down{color:#ff665c}.issue-message{overflow-wrap:anywhere}.full-span{grid-column:1/-1}.endpoint-list{display:grid;gap:10px;margin-top:16px}.endpoint{padding:14px;border:1px solid var(--line);border-radius:12px;background:rgba(100,130,150,.04)}.endpoint-head{display:flex;align-items:flex-start;justify-content:space-between;gap:12px}.endpoint-address{color:var(--muted);font:12px ui-monospace,monospace;overflow-wrap:anywhere}.certificate{margin-top:11px;padding-top:11px;border-top:1px solid var(--line);color:var(--muted);font-size:12px}.certificate b{color:var(--text)}.maintenance-grid .detail{min-width:0}.maintenance-grid .detail strong{font-size:clamp(15px,2.2vw,21px);overflow-wrap:anywhere;word-break:break-word}.system-tabs{display:flex;gap:6px;padding:12px 26px;border-bottom:1px solid var(--line)}.system-panel[hidden]{display:none}
.services .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.system-health .fill{background:linear-gradient(90deg,#6ba8ff,#9d8cff)}.service-form{display:grid;grid-template-columns:1fr 1fr;gap:12px}.field{display:grid;gap:6px;min-width:0}.field>span{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.07em}.field small{color:var(--muted);font-size:11px}.field input,.field select{width:100%;min-width:0;padding:11px 12px;border:1px solid var(--line);border-radius:9px;background:var(--bg);color:var(--text)}.check-field{display:flex;align-items:center;gap:10px;padding:12px;border:1px solid var(--line);border-radius:9px}.check-field input{width:18px;height:18px;accent-color:var(--green)}.service-actions{display:flex;flex-wrap:wrap;gap:7px;margin-top:10px}.service-actions button,.open-address{padding:7px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);cursor:pointer;text-decoration:none}.status-up{color:var(--green)}.status-down{color:#ff665c}.issue-message{overflow-wrap:anywhere;line-height:1.6;margin-top:12px}.full-span{grid-column:1/-1}.endpoint-list{display:grid;gap:10px;margin-top:16px}.endpoint{padding:14px;border:1px solid var(--line);border-radius:12px;background:rgba(100,130,150,.04)}.endpoint-head{display:flex;align-items:flex-start;justify-content:space-between;gap:12px}.endpoint-address{color:var(--muted);font:12px ui-monospace,monospace;overflow-wrap:anywhere}.certificate{margin-top:11px;padding-top:11px;border-top:1px solid var(--line);color:var(--muted);font-size:12px}.certificate b{color:var(--text)}.maintenance-grid .detail{min-width:0}.maintenance-grid .detail strong{font-size:clamp(15px,2.2vw,21px);overflow-wrap:anywhere;word-break:break-word}.system-tabs{display:flex;align-items:center;gap:6px;padding:12px 26px;border-bottom:1px solid var(--line)}.system-panel[hidden]{display:none}.system-sort{margin-left:auto;padding:8px 11px;border:1px solid var(--line);border-radius:9px;background:var(--panel);color:var(--text);cursor:pointer;white-space:nowrap}
.snapshots .fill{background:linear-gradient(90deg,#9d8cff,#ff78b8)}.tasks .fill{background:linear-gradient(90deg,#43c6d9,#6ba8ff)}.homelab-map .fill{background:linear-gradient(90deg,#55d58a,#9d8cff)}.map-board{display:grid;grid-template-columns:170px 1fr 1fr 1fr;gap:14px;padding:18px;min-width:680px}.map-column{display:grid;align-content:start;gap:9px}.map-column-title{color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.1em}.map-node{padding:12px;border:1px solid var(--line);border-radius:12px;background:var(--panel)}.map-node.good{border-color:#2d6245}.map-node.bad{border-color:#71362f}.map-node.warn{border-color:#6d4c2c}.map-node strong{display:block}.map-node small{display:block;color:var(--muted);margin-top:3px}.map-scroll{overflow-x:auto}.backup-check{margin-top:9px;color:var(--muted);font-size:12px}.task-filter{display:flex;gap:6px;padding:12px 16px;overflow-x:auto}
.topology-board{padding:18px;min-width:620px}.topology-host{margin-bottom:14px}.topology-grid{display:grid;grid-template-columns:repeat(2,minmax(270px,1fr));gap:12px}.topology-group{padding:14px;border:1px solid var(--line);border-radius:15px;background:rgba(100,130,150,.04)}.topology-group-head{display:flex;justify-content:space-between;gap:10px;padding-bottom:10px;border-bottom:1px solid var(--line)}.topology-section{margin-top:11px}.topology-section-title{color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.08em;margin-bottom:6px}.topology-items{display:flex;flex-wrap:wrap;gap:6px}.topology-chip{padding:6px 9px;border:1px solid var(--line);border-radius:9px;background:var(--panel);font-size:12px}.topology-chip.good{border-color:#2d6245}.topology-chip.bad{border-color:#71362f}.topology-unassigned{margin-top:14px}@media(max-width:700px){.topology-grid{grid-template-columns:1fr}}
.monitor .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.shutdown-plan{display:grid;gap:8px;margin-top:12px}.shutdown-step{padding:10px 12px;border:1px solid var(--line);border-radius:10px}.infra-status{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:14px 0}.infra-status>div{padding:11px;border:1px solid var(--line);border-radius:10px}.infra-status b{display:block;margin-top:4px}@media(max-width:700px){.infra-status{grid-template-columns:1fr 1fr}}
@@ -439,7 +446,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
<button class="card guests" data-card="guests" data-open="guestsDialog" aria-label="Открыть подробности VM и LXC">
<div class="card-head"><div><div class="label">Virtualization</div><div class="name">VM и LXC</div></div><div class="percent"><span id="guestRunning">—</span><small id="guestTotal"> / —</small></div></div>
<div class="bar"><div class="fill" id="guestBar"></div></div>
<div class="quick"><div><span class="label">CPU гостей</span><strong id="guestCPU">—</strong></div><div><span class="label">Память гостей</span><strong id="guestMemory">—</strong></div></div>
<div class="quick"><div><span class="label">CPU VM и LXC</span><strong id="guestCPU">—</strong></div><div><span class="label">Память VM и LXC</span><strong id="guestMemory">—</strong></div></div>
<div class="hint">Нажмите, чтобы посмотреть подробности →</div>
</button>
<button class="card maintenance" data-card="maintenance" data-open="maintenanceDialog" aria-label="Открыть сведения Proxmox">
@@ -468,7 +475,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
</button>
<button class="card snapshots" data-card="snapshots" data-open="snapshotsDialog" aria-label="Открыть snapshots">
<div class="card-head"><div><div class="label">Snapshots</div><div class="name">Снимки VM и LXC</div></div><div class="percent"><span id="snapshotCount">—</span><small> шт.</small></div></div><div class="bar"><div class="fill" id="snapshotBar"></div></div>
<div class="quick"><div><span class="label">Старше 30 дней</span><strong id="oldSnapshots">—</strong></div><div><span class="label">Гостей со снимками</span><strong id="snapshotGuests">—</strong></div></div><div class="hint">Нажмите, чтобы посмотреть снимки →</div>
<div class="quick"><div><span class="label">Старше 30 дней</span><strong id="oldSnapshots">—</strong></div><div><span class="label">VM/LXC со снимками</span><strong id="snapshotGuests">—</strong></div></div><div class="hint">Нажмите, чтобы посмотреть снимки →</div>
</button>
<button class="card tasks" data-card="tasks" data-open="tasksDialog" aria-label="Открыть задачи Proxmox">
<div class="card-head"><div><div class="label">Proxmox activity</div><div class="name">События и задачи</div></div><div class="percent"><span id="taskCount">—</span><small> задач</small></div></div><div class="bar"><div class="fill" id="taskBar"></div></div>
@@ -476,7 +483,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
</button>
<button class="card homelab-map" data-card="map" data-open="mapDialog" aria-label="Открыть карту homelab">
<div class="card-head"><div><div class="label">Topology</div><div class="name">Карта homelab</div></div><div class="percent"><span id="mapNodes">—</span><small> узлов</small></div></div><div class="bar"><div class="fill" id="mapBar"></div></div>
<div class="quick"><div><span class="label">VM / LXC</span><strong id="mapGuests">—</strong></div><div><span class="label">Сервисы / storage</span><strong id="mapResources">—</strong></div></div><div class="hint">Proxmox → гости → сервисы → storage →</div>
<div class="quick"><div><span class="label">VM / LXC</span><strong id="mapGuests">—</strong></div><div><span class="label">Сервисы / storage</span><strong id="mapResources">—</strong></div></div><div class="hint">Proxmox → VM/LXC → сервисы → storage →</div>
</button>
</section>
<footer>Обновление каждые 2 секунды · данные читаются напрямую с хоста</footer>
@@ -516,8 +523,8 @@ footer{color:#657581;font-size:12px;margin-top:17px}
<div class="detail"><span class="label">Используется</span><strong id="dDiskUsed">—</strong></div><div class="detail"><span class="label">Свободно</span><strong id="dDiskFree">—</strong></div><div class="detail"><span class="label">Общий объём</span><strong id="dDiskTotal">—</strong></div>
<div class="detail"><span class="label">Заполнение</span><strong id="dDiskPercent">—</strong></div><div class="detail"><span class="label">Точка монтирования</span><strong>/</strong></div>
</div>
<div class="disk-list" id="diskList"><span class="empty">Поиск накопителей…</span></div>
<div class="chart-section"><div class="chart-head"><strong>Температура физических дисков</strong><span>Отдельно по каждому диску</span></div><canvas class="metric-chart" id="storageChart"></canvas></div>
<div class="disk-list" id="diskList"><span class="empty">Поиск накопителей…</span></div><div class="wide"><span class="label">Расшифровка источников I/O</span><div class="disk-list" id="ioAttributionList"><span class="empty">Накопление данных…</span></div></div>
<div class="chart-section"><div class="chart-head"><strong>Температура и I/O физических дисков</strong><span>Отдельно по каждому диску</span></div><div class="label">Температура</div><canvas class="metric-chart" id="storageChart"></canvas><div class="label">IOPS</div><canvas class="metric-chart" id="diskIOChart"></canvas><div class="label">Utilization</div><canvas class="metric-chart" id="diskUtilChart"></canvas></div>
</dialog>
<dialog id="networkDialog">
@@ -553,8 +560,8 @@ footer{color:#657581;font-size:12px;margin-top:17px}
<dialog id="guestsDialog">
<div class="modal-head"><div><div class="label">Virtualization details</div><h2>VM и LXC</h2></div><button class="close" aria-label="Закрыть">×</button></div>
<div class="disk-list" id="guestList"><span class="empty">Получение списка гостей…</span></div>
<div class="chart-section"><div class="chart-head"><strong>CPU каждой VM и LXC</strong><span>Отдельно по каждому гостю</span></div><canvas class="metric-chart" id="guestsChart"></canvas></div>
<div class="disk-list" id="guestList"><span class="empty">Получение списка VM и LXC…</span></div>
<div class="chart-section"><div class="chart-head"><strong>CPU каждой VM и LXC</strong><span>Отдельно по каждой VM и LXC</span></div><canvas class="metric-chart" id="guestsChart"></canvas></div>
</dialog>
<dialog id="backupsDialog">
@@ -582,14 +589,14 @@ footer{color:#657581;font-size:12px;margin-top:17px}
<dialog id="systemDialog">
<div class="modal-head"><div><div class="label">System health</div><h2>Ошибки системы и ядра</h2></div><button class="close" aria-label="Закрыть">×</button></div>
<div class="detail-grid"><div class="detail"><span class="label">Ошибки · 24 часа</span><strong id="dSystemErrors">—</strong></div><div class="detail"><span class="label">Ядро</span><strong id="dKernelErrors">—</strong></div><div class="detail"><span class="label">Упавшие службы</span><strong id="dFailedUnits">—</strong></div></div>
<div class="system-tabs"><button class="detail-tab active" data-system-tab="issues">Ошибки журнала</button><button class="detail-tab" data-system-tab="services">Упавшие службы</button></div>
<div class="system-tabs"><button class="detail-tab active" data-system-tab="issues">Ошибки журнала</button><button class="detail-tab" data-system-tab="services">Упавшие службы</button><button class="system-sort" id="systemSort" type="button" aria-label="Сначала показаны свежие события">Сначала свежие ↓</button></div>
<div class="disk-list system-panel" data-system-panel="issues" id="issueList"><span class="empty">Системных ошибок нет.</span></div>
<div class="disk-list system-panel" data-system-panel="services" id="failedServiceList" hidden><span class="empty">Упавших служб нет.</span></div>
</dialog>
<dialog id="snapshotsDialog"><div class="modal-head"><div><div class="label">Snapshots</div><h2>Снимки VM и LXC</h2></div><button class="close" aria-label="Закрыть">×</button></div><div class="disk-list" id="snapshotList"><span class="empty">Получение snapshots…</span></div></dialog>
<dialog id="tasksDialog"><div class="modal-head"><div><div class="label">Proxmox activity</div><h2>События и задачи</h2></div><button class="close" aria-label="Закрыть">×</button></div><div class="task-filter"><button class="detail-tab active" data-task-filter="all">Все</button><button class="detail-tab" data-task-filter="vzdump">Backup</button><button class="detail-tab" data-task-filter="qm">VM/LXC</button><button class="detail-tab" data-task-filter="error">Ошибки</button></div><div class="disk-list" id="taskList"><span class="empty">Получение задач…</span></div></dialog>
<dialog id="mapDialog"><div class="modal-head"><div><div class="label">Homelab topology</div><h2>Карта homelab</h2></div><button class="close" aria-label="Закрыть">×</button></div><div class="map-scroll"><div class="topology-board" id="mapBoard"></div></div><div class="wide settings-note">Порт от локального адреса отбрасывается. Полученный IP сравнивается с адресами VM/LXC из Proxmox. Storage берётся из конфигурации каждого гостя.</div></dialog>
<dialog id="mapDialog"><div class="modal-head"><div><div class="label">Homelab topology</div><h2>Карта homelab</h2></div><button class="close" aria-label="Закрыть">×</button></div><div class="map-scroll"><div class="topology-board" id="mapBoard"></div></div><div class="wide settings-note">Порт от локального адреса отбрасывается. Полученный IP сравнивается с адресами VM/LXC из Proxmox. Storage берётся из конфигурации каждой VM и LXC.</div></dialog>
<dialog id="settingsDialog">
<div class="modal-head"><div><div class="label">Dashboard</div><h2>Настройки</h2></div><button class="close" aria-label="Закрыть">×</button></div>
@@ -605,7 +612,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
</div></div>
<div class="settings-group"><div class="label">Порядок карточек</div><div class="settings-note">Перетаскивайте строки или используйте стрелки.</div><div class="card-order" id="cardOrder"></div></div>
</div>
<div class="settings-body settings-panel" data-settings-panel="ups" hidden><div class="settings-group"><div class="label">Безопасное выключение при работе от батареи</div><div class="settings-note">Автоматическое выключение по умолчанию отключено. Dry-run ничего не останавливает и показывает будущий порядок.</div><div class="threshold-grid"><label class="setting-item"><input type="checkbox" id="upsShutdownEnabled"> Разрешить сценарий</label><label class="threshold-item"><span>Заряд, %</span><input type="number" id="upsShutdownCharge"></label><label class="threshold-item"><span>Автономность, сек.</span><input type="number" id="upsShutdownRuntime"></label><label class="threshold-item"><span>Ожидание гостя, сек.</span><input type="number" id="upsShutdownGrace"></label><label class="field full-span"><span>Порядок VMID через запятую</span><input id="upsShutdownOrder" placeholder="101, 105, 220"><small>Неуказанные запущенные гости добавятся после них по VMID.</small></label></div><button class="save-settings" id="saveUPSShutdown">Сохранить сценарий</button><button class="theme-option" id="dryRunUPSShutdown" style="margin-top:10px;width:100%">Показать dry-run</button><div class="shutdown-plan" id="upsShutdownPlan"></div><div class="settings-note" id="upsShutdownStatus"></div></div></div>
<div class="settings-body settings-panel" data-settings-panel="ups" hidden><div class="settings-group"><div class="label">Безопасное выключение при работе от батареи</div><div class="settings-note">Автоматическое выключение по умолчанию отключено. Dry-run ничего не останавливает и показывает будущий порядок.</div><div class="threshold-grid"><label class="setting-item"><input type="checkbox" id="upsShutdownEnabled"> Разрешить сценарий</label><label class="threshold-item"><span>Заряд, %</span><input type="number" id="upsShutdownCharge"></label><label class="threshold-item"><span>Автономность, сек.</span><input type="number" id="upsShutdownRuntime"></label><label class="threshold-item"><span>Ожидание VM/LXC, сек.</span><input type="number" id="upsShutdownGrace"></label><label class="field full-span"><span>Порядок VMID через запятую</span><input id="upsShutdownOrder" placeholder="101, 105, 220"><small>Неуказанные запущенные VM и LXC добавятся после них по VMID.</small></label></div><button class="save-settings" id="saveUPSShutdown">Сохранить сценарий</button><button class="theme-option" id="dryRunUPSShutdown" style="margin-top:10px;width:100%">Показать dry-run</button><div class="shutdown-plan" id="upsShutdownPlan"></div><div class="settings-note" id="upsShutdownStatus"></div></div></div>
<div class="settings-body settings-panel" data-settings-panel="alerts" hidden>
<div class="threshold-grid">
<label class="threshold-item"><span>CPU · предупреждение, %</span><input type="number" data-threshold="cpuWarning"></label><label class="threshold-item"><span>CPU · критично, %</span><input type="number" data-threshold="cpuCritical"></label>
@@ -616,6 +623,10 @@ footer{color:#657581;font-size:12px;margin-top:17px}
<label class="threshold-item"><span>ZFS · критично, %</span><input type="number" data-threshold="zfsCritical"></label><label class="threshold-item"><span>Температура HDD/SSD</span><input type="number" data-threshold="diskTempWarning"></label>
<label class="threshold-item"><span>Температура NVMe</span><input type="number" data-threshold="nvmeTempWarning"></label><label class="threshold-item"><span>UPS заряд · критично, %</span><input type="number" data-threshold="upsChargeCritical"></label>
<label class="threshold-item"><span>Максимальный возраст backup, ч</span><input type="number" data-threshold="backupMaxAgeHours"></label>
<label class="threshold-item"><span>HDD utilization, %</span><input type="number" data-threshold="diskHddUtil"></label><label class="threshold-item"><span>HDD latency, ms</span><input type="number" data-threshold="diskHddLatency"></label><label class="threshold-item"><span>HDD очередь</span><input type="number" data-threshold="diskHddQueue"></label>
<label class="threshold-item"><span>SATA SSD utilization, %</span><input type="number" data-threshold="diskSsdUtil"></label><label class="threshold-item"><span>SATA SSD latency, ms</span><input type="number" data-threshold="diskSsdLatency"></label><label class="threshold-item"><span>SATA SSD очередь</span><input type="number" data-threshold="diskSsdQueue"></label>
<label class="threshold-item"><span>NVMe utilization, %</span><input type="number" data-threshold="diskNvmeUtil"></label><label class="threshold-item"><span>NVMe latency, ms</span><input type="number" data-threshold="diskNvmeLatency"></label><label class="threshold-item"><span>NVMe очередь</span><input type="number" data-threshold="diskNvmeQueue"></label>
<label class="threshold-item"><span>Минимальная длительность, сек.</span><input type="number" data-threshold="diskIoDuration"></label><label class="field full-span"><span>Исключить диски из I/O-алертов</span><input type="text" data-threshold="diskIoExcluded" placeholder="sda, SERIAL123"><small>Имена или серийные номера через запятую.</small></label>
</div>
<button class="save-settings" id="saveThresholds">Сохранить пороги</button><div class="settings-note" id="thresholdStatus">Настройки хранятся на сервере и применяются для всех браузеров.</div>
</div>
@@ -659,27 +670,30 @@ const uptime=s=>{const d=Math.floor(s/86400),h=Math.floor(s%86400/3600),m=Math.f
let alertSettings={backupMaxAgeHours:48};
let lastDashboardEvent=0;
let currentServiceStatuses=[];
let currentSystemHealth={issues:[],failedServices:[]},systemSortNewest=true;
function renderSystemHealth(sh){currentSystemHealth=sh||{issues:[],failedServices:[]};const issues=[...(sh.issues||[])].sort((a,b)=>systemSortNewest?b.time-a.time:a.time-b.time);el('issueList').innerHTML=issues.length?issues.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+new Date(v.time*1000).toLocaleString('ru-RU')+'</div></div><span class="badge bad">'+(v.count>1?'Повторений: '+v.count:'1 раз')+'</span></div><div class="settings-note issue-message">'+safe(v.message)+'</div></article>').join(''):'<span class="empty">Критических сообщений за 24 часа нет.</span>';el('failedServiceList').innerHTML=(sh.failedServices||[]).length?sh.failedServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+safe(v.description||'Описание отсутствует')+'</div></div><span class="badge bad">'+safe(v.state)+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Загрузка</span><b>'+safe(v.load)+'</b></div><div class="disk-stat"><span>Состояние</span><b>'+safe(v.active)+'</b></div><div class="disk-stat"><span>Подсостояние</span><b>'+safe(v.state)+'</b></div></div></article>').join(''):'<span class="empty">Упавших systemd-служб нет.</span>';el('systemSort').textContent=systemSortNewest?'Сначала свежие ↓':'Сначала старые ↑';el('systemSort').setAttribute('aria-label',systemSortNewest?'Сначала показаны свежие события':'Сначала показаны старые события')}
function endpointHTML(v){const tls=v.tlsDays==null?'Нет TLS':v.tlsDays<0?'Сертификат истёк':v.tlsDays+' дн. до окончания',cert=v.tlsSubject?'<div class="certificate"><div>Сертификат: <b>'+safe(v.tlsSubject)+'</b></div><div>Домены: <b>'+safe((v.tlsDnsNames||[]).join(', ')||'не указаны')+'</b></div><div>Издатель: <b>'+safe(v.tlsIssuer||'—')+'</b></div><div>Для этого адреса: <b class="'+(v.tlsHostnameValid?'status-up':'status-down')+'">'+(v.tlsHostnameValid?'подходит':'не подходит')+'</b></div></div>':'';return '<div class="endpoint"><div class="endpoint-head"><div><b>'+safe(v.label)+'</b><div class="endpoint-address">'+safe(v.address)+'</div></div><span class="badge '+(v.up?'':'bad')+'">'+(v.up?'Доступен':'Недоступен')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Отклик</span><b>'+fixed(v.latencyMs)+' ms</b></div><div class="disk-stat"><span>HTTP</span><b>'+(v.statusCode||'—')+'</b></div><div class="disk-stat"><span>TLS</span><b>'+tls+'</b></div><div class="disk-stat"><span>Uptime · сутки</span><b>'+fixed(v.uptimeDay)+'%</b></div><div class="disk-stat"><span>Uptime · неделя</span><b>'+fixed(v.uptimeWeek)+'%</b></div></div>'+cert+(v.error?'<div class="settings-note status-down">'+safe(v.error)+'</div>':'')+'</div>'}
function renderServices(svc){const items=svc.services||[];currentServiceStatuses=items;const avg=items.length?items.reduce((a,v)=>a+v.uptimeDay,0)/items.length:0,live=items.filter(v=>v.up),latency=live.length?live.reduce((a,v)=>a+v.latencyMs,0)/live.length:0;el('serviceUp').textContent=svc.up||0;el('serviceTotal').textContent=' / '+(svc.total||0);el('serviceBar').style.width=(svc.total?svc.up/svc.total*100:0)+'%';el('serviceUptime').textContent=svc.total?fixed(avg)+'%':'—';el('serviceLatency').textContent=svc.total?fixed(latency)+' ms':'—';el('serviceList').innerHTML=items.length?items.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+(v.endpoints||[]).length+' '+((v.endpoints||[]).length===1?'адрес':'адреса')+' проверяются независимо</div></div><span class="badge '+(v.up?'':'bad')+'">'+(v.up?'Всё доступно':'Есть проблема')+'</span></div><div class="endpoint-list">'+(v.endpoints||[]).map(endpointHTML).join('')+'</div><div class="service-actions"><button data-service-open="'+v.id+'">Открыть сервис ↗</button></div></article>').join(''):'<span class="empty">Добавьте первую проверку в настройках.</span>'}
function renderNetworkGroups(interfaces){const groups=new Map();interfaces.forEach(v=>{const key=v.vmid?'guest-'+v.vmid:'host',group=groups.get(key)||{vmid:v.vmid,name:v.vmid?(v.guestName||('VMID '+v.vmid)):'Интерфейсы хоста',type:v.guestType||'Host',items:[]};group.items.push(v);groups.set(key,group)});el('networkInterfacesList').innerHTML=[...groups.values()].map(g=>{const primary=g.vmid?g.items.filter(v=>v.name.startsWith('tap')||v.name.startsWith('veth')):g.items,rx=primary.reduce((a,v)=>a+v.receivePerSec,0),tx=primary.reduce((a,v)=>a+v.transmitPerSec,0),errors=g.items.reduce((a,v)=>a+v.receiveErrors+v.transmitErrors+v.receiveDrops+v.transmitDrops,0),down=primary.some(v=>v.kind==='Physical'&&!v.carrier&&(v.master||v.receiveBytes+v.transmitBytes>0));return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(g.name)+'</div><div class="disk-meta">'+(g.vmid?'VMID '+g.vmid+' · ':'')+safe(g.type)+' · '+safe(g.items.map(v=>v.name+' ['+(v.kind||'Interface')+(v.speedMbps?' · '+v.speedMbps+'Mb/s':'')+(v.duplex?' · '+v.duplex:'')+(v.master?' → '+v.master:'')+']').join(', '))+'</div></div><span class="badge '+(errors||down?'warn':'')+'"'+(errors||down?' tabindex="0" data-reason="'+safe(down?'Нет carrier на используемом физическом интерфейсе':'Счётчики errors/drops больше нуля')+'"':'')+'>'+(down?'Нет линка':errors?'Есть потери':'Без ошибок')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Сейчас ↓</span><b>'+rate(rx)+'</b></div><div class="disk-stat"><span>Сейчас ↑</span><b>'+rate(tx)+'</b></div><div class="disk-stat"><span>Получено</span><b>'+size(primary.reduce((a,v)=>a+v.receiveBytes,0))+'</b></div><div class="disk-stat"><span>Передано</span><b>'+size(primary.reduce((a,v)=>a+v.transmitBytes,0))+'</b></div><div class="disk-stat"><span>Ошибки / drops</span><b>'+errors+'</b></div></div></article>'}).join('')}
function renderIOAttribution(disks){const active=disks.filter(d=>d.ioCause||(d.topConsumers||[]).length);el('ioAttributionList').innerHTML=active.length?active.map(d=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(d.name)+' · '+safe(d.ioCause||'Источник не определён')+'</div><div class="disk-meta">'+safe(d.ioCauseDetail||'Собираем статистику процессов')+'</div></div><span class="badge '+(d.ioConfidence>=80?'':'warn')+'" tabindex="0" data-reason="Уверенность учитывает совпадение активной задачи Proxmox и долю I/O процессов">'+(d.ioConfidence||0)+'%</span></div><div class="topology-items">'+(d.topConsumers||[]).slice(0,5).map(v=>'<span class="topology-chip">'+safe(v.kind)+' · '+safe(v.name)+' · '+fixed(v.sharePercent)+'% · ↓'+rate(v.readBytesPerSec)+' ↑'+rate(v.writeBytesPerSec)+'</span>').join('')+'</div></article>').join(''):'<span class="empty">Для атрибуции нужно несколько realtime-циклов.</span>'}
const taskLabel=type=>({vzdump:'Backup',qmstart:'Запуск VM',qmstop:'Остановка VM',qmshutdown:'Выключение VM',qmsnapshot:'Snapshot VM',vzstart:'Запуск LXC',vzstop:'Остановка LXC',vzsnapshot:'Snapshot LXC',qmigrate:'Миграция',aptupdate:'Обновление пакетов'})[type]||type;
function explainWarningBadges(){el('snapshotList').querySelectorAll('.badge.warn').forEach(v=>{v.tabIndex=0;v.dataset.reason='Snapshot старше 30 дней. Проверьте, нужен ли он: старые снимки могут занимать место в storage.'});el('taskList').querySelectorAll('.badge.bad').forEach(v=>{v.tabIndex=0;v.dataset.reason='Задача Proxmox завершилась не со статусом OK. Откройте её журнал в интерфейсе Proxmox для подробностей.'})}
let currentTasks=[],currentTaskFilter='all';
function renderTasks(tasks,filter='all'){currentTasks=tasks;const shown=tasks.filter(v=>filter==='all'||filter==='error'&&v.status!=='OK'||filter==='qm'&&(/^(qm|vz)/.test(v.type))||v.type===filter);el('taskList').innerHTML=shown.length?shown.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(taskLabel(v.type))+(v.id?' · '+safe(v.id):'')+'</div><div class="disk-meta">'+new Date(v.starttime*1000).toLocaleString('ru-RU')+' · '+safe(v.user||'—')+'</div></div><span class="badge '+(v.status==='OK'?'':'bad')+'">'+safe(v.status||'Выполняется')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Длительность</span><b>'+uptime(v.duration||Math.max(0,Date.now()/1000-v.starttime))+'</b></div><div class="disk-stat"><span>Узел</span><b>'+safe(v.node)+'</b></div><div class="disk-stat"><span>VMID</span><b>'+safe(v.id||'—')+'</b></div></div></article>').join(''):'<span class="empty">Задач этого типа нет.</span>'}
function renderActivity(activity,guests,services,zfs){const snapshots=activity.snapshots||[],tasks=activity.tasks||[],old=snapshots.filter(v=>v.ageSeconds>30*86400),guestCount=new Set(snapshots.map(v=>v.vmid)).size,failed=tasks.filter(v=>v.status&&v.status!=='OK');el('snapshotCount').textContent=snapshots.length;el('oldSnapshots').textContent=old.length;el('snapshotGuests').textContent=guestCount;el('snapshotBar').style.width=Math.min(old.length*20,100)+'%';el('snapshotList').innerHTML=snapshots.length?snapshots.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.guestName||('VMID '+v.vmid))+' · '+safe(v.name)+'</div><div class="disk-meta">VMID '+v.vmid+' · '+safe(v.guestType)+(v.createdAt?' · '+new Date(v.createdAt*1000).toLocaleString('ru-RU'):'')+'</div></div><span class="badge '+(v.ageSeconds>30*86400?'warn':'')+'">'+(v.ageSeconds?uptime(v.ageSeconds):'Без даты')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Размер</span><b>'+(v.sizeBytes?size(v.sizeBytes):'Storage не сообщает')+'</b></div><div class="disk-stat"><span>Состояние VM</span><b>'+(v.running?'Запущена':'Остановлена')+'</b></div><div class="disk-stat"><span>Описание</span><b>'+safe(v.description||'—')+'</b></div></div></article>').join(''):'<span class="empty">Snapshots не найдены.</span>';el('taskCount').textContent=tasks.length;el('failedTasks').textContent=failed.length;el('lastTask').textContent=tasks.length?taskLabel(tasks[0].type):'—';el('taskBar').style.width=Math.min(failed.length*20,100)+'%';renderTasks(tasks,currentTaskFilter);renderHomelabMap(guests,services,zfs)}
function topologyHost(address){if(!address)return '';try{return new URL(address.includes('://')?address:'http://'+address).hostname.replace(/^\[|\]$/g,'')}catch(e){return address.replace(/^[a-z]+:\/\//i,'').split('/')[0].replace(/:\d+$/,'')}}
function renderHomelabMap(guests,services,zfs){const guestItems=(guests.guests||[]).filter(v=>!v.template),serviceItems=services.services||[],pools=zfs.pools||[],serviceOwners=new Map();serviceItems.forEach(s=>{const host=topologyHost(s.localAddress||'');const owner=guestItems.find(g=>(g.ipAddresses||[]).some(ip=>ip===host));serviceOwners.set(s.id,owner?.vmid||0)});const serviceChip=s=>{const healthy=(s.endpoints||[]).every(v=>v.up);return '<span class="topology-chip '+(healthy?'good':'bad')+'"'+(!healthy?' tabindex="0" data-reason="Один или несколько адресов сервиса недоступны"':'')+'>'+safe(s.name)+'</span>'},storageChip=name=>{const pool=pools.find(p=>p.name===name),healthy=!pool||pool.health==='ONLINE';return '<span class="topology-chip '+(healthy?'good':'bad')+'">'+safe(name)+(pool?' · '+fixed(pool.capacityPercent)+'%':'')+'</span>'};const groups=guestItems.map(g=>{const owned=serviceItems.filter(s=>serviceOwners.get(s.id)===g.vmid),storage=[...new Set(g.storage||[])],running=g.status==='running';return '<section class="topology-group"><div class="topology-group-head"><div><strong>'+safe(g.name||('VMID '+g.vmid))+'</strong><div class="disk-meta">VMID '+g.vmid+' · '+safe(g.type==='qemu'?'VM':'LXC')+' · '+safe((g.ipAddresses||[]).join(', ')||'IP не получен')+'</div></div><span class="badge '+(running?'':'warn')+'"'+(!running?' tabindex="0" data-reason="VM/LXC сейчас остановлена"':'')+'>'+(running?'Работает':'Остановлена')+'</span></div><div class="topology-section"><div class="topology-section-title">Сервисы</div><div class="topology-items">'+(owned.length?owned.map(serviceChip).join(''):'<span class="empty">Не сопоставлены</span>')+'</div></div><div class="topology-section"><div class="topology-section-title">Storage</div><div class="topology-items">'+(storage.length?storage.map(storageChip).join(''):'<span class="empty">Не определён</span>')+'</div></div></section>'}).join(''),unassigned=serviceItems.filter(s=>!serviceOwners.get(s.id)),usedStorage=new Set(guestItems.flatMap(g=>g.storage||[])),unassignedPools=pools.filter(p=>!usedStorage.has(p.name));el('mapBoard').innerHTML='<div class="topology-host map-node good"><strong>Proxmox host</strong><small>'+safe(location.hostname)+' · '+guestItems.length+' VM/LXC</small></div><div class="topology-grid">'+groups+'</div>'+((unassigned.length||unassignedPools.length)?'<section class="topology-group topology-unassigned"><div class="topology-group-head"><strong>Без автоматической привязки</strong><span class="badge warn" tabindex="0" data-reason="Для сервиса не найден совпадающий IP гостя либо storage не используется в конфигурации VM/LXC">Проверьте связи</span></div><div class="topology-section"><div class="topology-section-title">Сервисы</div><div class="topology-items">'+unassigned.map(serviceChip).join('')+'</div></div><div class="topology-section"><div class="topology-section-title">Остальные ZFS-пулы</div><div class="topology-items">'+unassignedPools.map(p=>storageChip(p.name)).join('')+'</div></div></section>':'');const storage=new Set([...usedStorage,...pools.map(p=>p.name)]);el('mapGuests').textContent=guestItems.length;el('mapResources').textContent=serviceItems.length+' / '+storage.size;el('mapNodes').textContent=1+guestItems.length+serviceItems.length+storage.size;el('mapBar').style.width='100%'}
function renderHomelabMap(guests,services,zfs){const guestItems=(guests.guests||[]).filter(v=>!v.template),serviceItems=services.services||[],pools=zfs.pools||[],serviceOwners=new Map();serviceItems.forEach(s=>{const host=topologyHost(s.localAddress||'');const owner=guestItems.find(g=>(g.ipAddresses||[]).some(ip=>ip===host));serviceOwners.set(s.id,owner?.vmid||0)});const serviceChip=s=>{const healthy=(s.endpoints||[]).every(v=>v.up);return '<span class="topology-chip '+(healthy?'good':'bad')+'"'+(!healthy?' tabindex="0" data-reason="Один или несколько адресов сервиса недоступны"':'')+'>'+safe(s.name)+'</span>'},storageChip=name=>{const pool=pools.find(p=>p.name===name),healthy=!pool||pool.health==='ONLINE';return '<span class="topology-chip '+(healthy?'good':'bad')+'">'+safe(name)+(pool?' · '+fixed(pool.capacityPercent)+'%':'')+'</span>'};const groups=guestItems.map(g=>{const owned=serviceItems.filter(s=>serviceOwners.get(s.id)===g.vmid),storage=[...new Set(g.storage||[])],running=g.status==='running';return '<section class="topology-group"><div class="topology-group-head"><div><strong>'+safe(g.name||('VMID '+g.vmid))+'</strong><div class="disk-meta">VMID '+g.vmid+' · '+safe(g.type==='qemu'?'VM':'LXC')+' · '+safe((g.ipAddresses||[]).join(', ')||'IP не получен')+'</div></div><span class="badge '+(running?'':'warn')+'"'+(!running?' tabindex="0" data-reason="VM/LXC сейчас остановлена"':'')+'>'+(running?'Работает':'Остановлена')+'</span></div><div class="topology-section"><div class="topology-section-title">Сервисы</div><div class="topology-items">'+(owned.length?owned.map(serviceChip).join(''):'<span class="empty">Не сопоставлены</span>')+'</div></div><div class="topology-section"><div class="topology-section-title">Storage</div><div class="topology-items">'+(storage.length?storage.map(storageChip).join(''):'<span class="empty">Не определён</span>')+'</div></div></section>'}).join(''),unassigned=serviceItems.filter(s=>!serviceOwners.get(s.id)),usedStorage=new Set(guestItems.flatMap(g=>g.storage||[])),unassignedPools=pools.filter(p=>!usedStorage.has(p.name));el('mapBoard').innerHTML='<div class="topology-host map-node good"><strong>Proxmox host</strong><small>'+safe(location.hostname)+' · '+guestItems.length+' VM/LXC</small></div><div class="topology-grid">'+groups+'</div>'+((unassigned.length||unassignedPools.length)?'<section class="topology-group topology-unassigned"><div class="topology-group-head"><strong>Без автоматической привязки</strong><span class="badge warn" tabindex="0" data-reason="Для сервиса не найден совпадающий IP VM/LXC либо storage не используется в конфигурации VM/LXC">Проверьте связи</span></div><div class="topology-section"><div class="topology-section-title">Сервисы</div><div class="topology-items">'+unassigned.map(serviceChip).join('')+'</div></div><div class="topology-section"><div class="topology-section-title">Остальные ZFS-пулы</div><div class="topology-items">'+unassignedPools.map(p=>storageChip(p.name)).join('')+'</div></div></section>':'');const storage=new Set([...usedStorage,...pools.map(p=>p.name)]);el('mapGuests').textContent=guestItems.length;el('mapResources').textContent=serviceItems.length+' / '+storage.size;el('mapNodes').textContent=1+guestItems.length+serviceItems.length+storage.size;el('mapBar').style.width='100%'}
function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{available:false,pools:[]},u=x.ups||{available:false},g=x.guests||{available:false,guests:[]},b=x.backups||{available:false,items:[]},mt=x.maintenance||{},alerts=x.alerts||[],disks=x.disks||[],avg=c.temperatures.length?c.temperatures.reduce((a,t)=>a+t.celsius,0)/c.temperatures.length:null;
lastDashboardEvent=Date.now();
const update=x.update||{},updating=['queued','installing'].includes(update.state);el('updateCurrent').textContent=update.currentVersion||'—';el('updateAvailable').textContent=update.availableVersion||'—';el('updateMode').textContent=update.mode==='auto'?'Автоматически':'Уведомлять';el('updateMessage').textContent=update.message||'Проверка ещё не выполнялась';el('updateChecked').textContent=update.checkedAt?'Последняя проверка: '+new Date(update.checkedAt).toLocaleString('ru-RU'):'Проверка запускается автоматически раз в сутки.';el('installUpdate').disabled=updating;el('installUpdate').textContent=updating?'Обновление выполняется…':'Проверить и установить обновление';
const critical=alerts.filter(a=>a.severity==='critical').length,warnings=alerts.filter(a=>a.severity==='warning').length,infos=alerts.filter(a=>a.severity==='info').length;el('alertCount').textContent=alerts.length;el('alertCritical').textContent=critical;el('alertWarnings').textContent=warnings;el('alertHeadline').textContent=critical?'Требуется срочное внимание':warnings?'Есть предупреждения':infos?'Есть информационные события':'Система в норме';el('alertBar').style.width=(critical?100:warnings?65:infos?30:0)+'%';
el('alertList').innerHTML=alerts.length?alerts.map(a=>'<article class="alert-item '+safe(a.severity)+'"><i class="alert-icon"></i><div><div class="alert-source">'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+'</div><div class="alert-title">'+safe(a.title)+'</div><div class="alert-message">'+safe(a.message)+'</div><button class="ack-alert" data-ack="'+a.eventId+'">Принято</button></div></article>').join(''):'<div class="all-good"><strong>Всё в порядке</strong><span>Новых неподтверждённых событий нет.</span></div>';
el('alertList').innerHTML=alerts.length?alerts.map(a=>'<article class="alert-item '+safe(a.severity)+'"><i class="alert-icon"></i><div><div class="alert-source">'+safe(a.source)+' · '+(a.severity==='critical'?'Критическое':a.severity==='warning'?'Предупреждение':'Информация')+(a.firstSeen?' · С '+new Date(a.firstSeen*1000).toLocaleString('ru-RU'):'')+'</div><div class="alert-title">'+safe(a.title)+'</div><div class="alert-message">'+safe(a.message)+'</div><button class="ack-alert" data-ack="'+a.eventId+'">Принято</button></div></article>').join(''):'<div class="all-good"><strong>Всё в порядке</strong><span>Новых неподтверждённых событий нет.</span></div>';
el('cpuName').textContent=el('cpuDetailName').textContent=c.model;el('cpuUsage').textContent=fixed(c.usagePercent);el('cpuBar').style.width=Math.min(c.usagePercent,100)+'%';el('cpuTemp').textContent=avg===null?'Нет датчика':fixed(avg)+' °C';el('cpuTopology').textContent=c.physicalCores+' / '+c.logicalCpus;
el('dCpuUsage').textContent=fixed(c.usagePercent)+'%';el('dSockets').textContent=c.sockets;el('dCores').textContent=c.physicalCores;el('dThreads').textContent=c.logicalCpus;el('dFreq').textContent=(c.frequencyMhz/1000).toFixed(2)+' GHz';el('dUptime').textContent=uptime(c.uptimeSeconds);el('dLoad1').textContent=fixed(c.load1);el('dLoad5').textContent=fixed(c.load5);el('dLoad15').textContent=fixed(c.load15);el('dTemps').innerHTML=c.temperatures.length?c.temperatures.map(t=>'<span class="temp">'+t.label+'<b>'+fixed(t.celsius)+'°C</b></span>').join(''):'<span class="empty">Датчики температуры не найдены</span>';
el('memUsage').textContent=fixed(m.usagePercent);el('memBar').style.width=Math.min(m.usagePercent,100)+'%';el('memUsed').textContent=size(m.usedBytes);el('memAvailable').textContent=size(m.availableBytes);el('dMemUsed').textContent=size(m.usedBytes);el('dMemAvailable').textContent=size(m.availableBytes);el('dMemTotal').textContent=size(m.totalBytes);el('dMemCache').textContent=size(m.cachedBytes);el('dMemBuffers').textContent=size(m.buffersBytes);el('dMemPercent').textContent=fixed(m.usagePercent)+'%';el('dSwapUsed').textContent=size(m.swapUsedBytes);el('dSwapTotal').textContent=size(m.swapTotalBytes);el('dSwapPercent').textContent=fixed(m.swapPercent)+'%';
const failed=disks.filter(d=>d.smartStatus==='Ошибка').length,attention=disks.filter(d=>d.smartStatus==='Требует внимания').length,available=disks.filter(d=>d.smartAvailable).length;let health=failed?'Ошибка':attention?'Внимание':available===disks.length&&disks.length?'Исправны':available?'Частично':'Нет данных';
el('diskCount').textContent=disks.length;el('diskCountLabel').textContent=disks.length===1?' диск':' дисков';el('diskHealth').textContent=health;el('diskUsage').textContent=fixed(s.usagePercent)+'%';el('diskBar').style.width=Math.min(s.usagePercent,100)+'%';el('dDiskUsed').textContent=size(s.usedBytes);el('dDiskFree').textContent=size(s.freeBytes);el('dDiskTotal').textContent=size(s.totalBytes);el('dDiskPercent').textContent=fixed(s.usagePercent)+'%';
el('diskList').innerHTML=disks.length?disks.map(d=>{const badge=d.smartStatus==='Ошибка'?'bad':d.smartStatus==='Исправен'?'':'warn',reason=(d.attentionReasons||[]).join(' · '),tip=reason?' tabindex="0" data-reason="'+safe(reason)+'"':'';return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(d.model)+'</div><div class="disk-meta">'+safe(d.path)+' · SN: '+safe(d.serial)+'</div></div><span class="badge '+badge+'"'+tip+'>'+safe(d.smartStatus)+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Тип / объём</span><b>'+safe(d.type)+' · '+size(d.sizeBytes)+'</b></div><div class="disk-stat"><span>Занято</span><b>'+(d.usedBytes==null?'—':size(d.usedBytes)+' · '+fixed(d.usagePercent)+'%')+'</b></div><div class="disk-stat"><span>Температура / износ</span><b>'+(d.temperatureCelsius==null?'—':fixed(d.temperatureCelsius)+' °C')+' · '+(d.wearUsedPercent==null?'—':fixed(d.wearUsedPercent)+'%')+'</b></div><div class="disk-stat"><span>Reallocated / Pending</span><b>'+d.reallocated+' / '+d.pending+'</b></div><div class="disk-stat"><span>Uncorrectable / CRC</span><b>'+d.uncorrectable+' / '+d.crcErrors+'</b></div><div class="disk-stat"><span>Последний self-test</span><b>'+safe(d.lastSelfTest||'—')+(d.lastSelfTestAt?' · '+new Date(d.lastSelfTestAt).toLocaleDateString('ru-RU'):'')+(d.shortTestStarted?' · запущен новый':'')+'</b></div></div></article>'}).join(''):'<span class="empty">Физические накопители не найдены</span>';
el('diskList').innerHTML=disks.length?disks.map(d=>{const ioWarn=d.ioPressureSeconds>=60,badge=d.smartStatus==='Ошибка'?'bad':d.smartStatus==='Исправен'?(ioWarn?'warn':''):'warn',reasons=[...(d.attentionReasons||[]),...(ioWarn?['Длительная I/O-нагрузка: '+fixed(d.ioPressureSeconds)+' сек.']:[])],reason=reasons.join(' · '),tip=reason?' tabindex="0" data-reason="'+safe(reason)+'"':'';return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(d.model)+'</div><div class="disk-meta">'+safe(d.path)+' · SN: '+safe(d.serial)+'</div></div><span class="badge '+badge+'"'+tip+'>'+(ioWarn?'Высокий I/O':safe(d.smartStatus))+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Тип / объём</span><b>'+safe(d.type)+' · '+size(d.sizeBytes)+'</b></div><div class="disk-stat"><span>Занято</span><b>'+(d.usedBytes==null?'—':size(d.usedBytes)+' · '+fixed(d.usagePercent)+'%')+'</b></div><div class="disk-stat"><span>Температура / износ</span><b>'+(d.temperatureCelsius==null?'—':fixed(d.temperatureCelsius)+' °C')+' · '+(d.wearUsedPercent==null?'—':fixed(d.wearUsedPercent)+'%')+'</b></div><div class="disk-stat"><span>IOPS · чтение / запись</span><b>'+fixed(d.readIops)+' / '+fixed(d.writeIops)+'</b></div><div class="disk-stat"><span>Скорость · чтение / запись</span><b>'+rate(d.readBytesPerSec)+' / '+rate(d.writeBytesPerSec)+'</b></div><div class="disk-stat"><span>Latency · чтение / запись</span><b>'+fixed(d.readLatencyMs)+' / '+fixed(d.writeLatencyMs)+' ms</b></div><div class="disk-stat"><span>Utilization / очередь</span><b>'+fixed(d.utilizationPercent)+'% / '+fixed(d.averageQueue)+' ('+d.queueDepth+' сейчас)</b></div><div class="disk-stat"><span>Reallocated / Pending</span><b>'+d.reallocated+' / '+d.pending+'</b></div><div class="disk-stat"><span>Uncorrectable / CRC</span><b>'+d.uncorrectable+' / '+d.crcErrors+'</b></div><div class="disk-stat"><span>Последний self-test</span><b>'+safe(d.lastSelfTest||'—')+(d.lastSelfTestAt?' · '+new Date(d.lastSelfTestAt).toLocaleDateString('ru-RU'):'')+(d.shortTestStarted?' · запущен новый':'')+'</b></div></div></article>'}).join(''):'<span class="empty">Физические накопители не найдены</span>';
const totalRate=n.receivePerSec+n.transmitPerSec,nh=n.health||{};el('netInterfaces').textContent=n.interface||'Сетевые интерфейсы';el('netTotal').textContent=(totalRate/1048576).toFixed(2);el('netBar').style.width=Math.min(totalRate/125000000*100,100)+'%';el('netDown').textContent=rate(n.receivePerSec);el('netUp').textContent=rate(n.transmitPerSec);el('dNetDown').textContent=rate(n.receivePerSec);el('dNetUp').textContent=rate(n.transmitPerSec);el('dNetTotal').textContent=rate(totalRate);el('dNetReceived').textContent=size(n.receiveBytes);el('dNetSent').textContent=size(n.transmitBytes);el('dNetInterfaces').textContent=n.interface||'—';el('dNetGateway').textContent=(nh.gateway||'Не определён')+' · '+(nh.gatewayUp?'доступен':'нет ответа')+(nh.latencyMs?' · '+fixed(nh.latencyMs)+' ms':'');el('dNetDNS').textContent=nh.dnsUp?'Работает':'Ошибка';el('dNetInternet').textContent=nh.internetUp?'Доступен':'Недоступен';el('dNetExternalIP').textContent=(nh.externalIp||'—')+(nh.externalIpChanged?' · изменился':'');
const pools=z.pools||[],zTotal=pools.reduce((a,p)=>a+p.sizeBytes,0),zUsed=pools.reduce((a,p)=>a+p.allocatedBytes,0),zPercent=zTotal?zUsed/zTotal*100:0,zBad=pools.filter(p=>p.health!=='ONLINE').length;el('zfsCount').textContent=pools.length;el('zfsHealth').textContent=!z.available?'Не найден':zBad?zBad+' требуют внимания':pools.length+' ONLINE';el('zfsUsed').textContent=zTotal?size(zUsed)+' / '+size(zTotal)+' · '+fixed(zPercent)+'%':'—';el('zfsBar').style.width=Math.min(zPercent,100)+'%';
const datasets=z.datasets||[],arc=z.arc||{};el('zfsArcSize').textContent=arc.available?size(arc.sizeBytes):'—';el('zfsArcTarget').textContent=arc.available?size(arc.targetBytes):'—';el('zfsArcHit').textContent=arc.available?fixed(arc.hitRatio)+'%':'—';el('zfsDatasetCount').textContent=datasets.length;el('zfsList').innerHTML=pools.length?pools.map(p=>{const bad=p.health!=='ONLINE'||p.readErrors+p.writeErrors+p.checksumErrors>0,reason=(p.attentionReasons||[]).join(' · '),tip=reason?' tabindex="0" data-reason="'+safe(reason)+'"':'',poolDatasets=datasets.filter(d=>d.pool===p.name),vdevs=p.vdevs||[];return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(p.name)+'</div><div class="disk-meta">Scrub: '+safe(p.scrubStatus||p.scan||'нет данных')+(p.lastScrubAt?' · '+new Date(p.lastScrubAt).toLocaleString('ru-RU'):'')+(p.scrubDuration?' · '+safe(p.scrubDuration):'')+'</div></div><span class="badge '+(bad?'bad':p.scrubTooOld?'warn':'')+'"'+tip+'>'+safe(p.health)+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Объём / свободно</span><b>'+size(p.sizeBytes)+' / '+size(p.freeBytes)+'</b></div><div class="disk-stat"><span>Заполнение</span><b>'+fixed(p.capacityPercent)+'%</b></div><div class="disk-stat"><span>Read / write / checksum</span><b>'+p.readErrors+' / '+p.writeErrors+' / '+p.checksumErrors+'</b></div></div><div class="topology-section"><div class="topology-section-title">Vdev · data / special / cache / log</div><div class="topology-items">'+(vdevs.length?vdevs.map(v=>'<span class="topology-chip '+(v.state==='ONLINE'?'good':'bad')+'">'+safe(v.class)+' · '+safe(v.name)+' · '+safe(v.state)+' · '+v.read+'/'+v.write+'/'+v.checksum+'</span>').join(''):'<span class="empty">Нет данных</span>')+'</div></div><div class="topology-section"><div class="topology-section-title">Datasets</div><div class="topology-items">'+(poolDatasets.length?poolDatasets.map(d=>'<span class="topology-chip '+(d.capacityPercent>=80?'bad':'good')+'">'+safe(d.name)+' · '+fixed(d.capacityPercent)+'% · '+size(d.usedBytes)+'</span>').join(''):'<span class="empty">Нет datasets</span>')+'</div></div></article>'}).join(''):'<span class="empty">'+(z.available?'ZFS-пулы не найдены':'Команда zpool недоступна')+'</span>';
@@ -692,26 +706,26 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av
const backupLimitHours=alertSettings.backupMaxAgeHours||48,backupLimit=backupLimitHours*3600,backupMap=new Map((b.items||[]).map(item=>[item.vmid,item])),now=Date.now()/1000,protectedGuests=guests.filter(v=>backupMap.has(v.vmid)),freshGuests=protectedGuests.filter(v=>now-backupMap.get(v.vmid).ctime<=backupLimit),missing=guests.length-protectedGuests.length;el('backupFreshLabel').textContent='Свежие · '+backupLimitHours+' часов';el('backupProtected').textContent=protectedGuests.length;el('backupTotal').textContent=' / '+guests.length;el('backupBar').style.width=guests.length?(protectedGuests.length/guests.length*100)+'%':'0%';el('backupFresh').textContent=b.available?freshGuests.length:'Не найдено';el('backupMissing').textContent=b.available?missing:'—';
el('backupList').innerHTML=guests.length?guests.map(v=>{const item=backupMap.get(v.vmid),type=v.type==='qemu'?'VM':'LXC';if(!item)return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name||('Guest '+v.vmid))+'</div><div class="disk-meta">VMID '+v.vmid+' · '+type+'</div></div><span class="badge bad">Копии нет</span></div></article>';const age=now-item.ctime,fresh=age<=backupLimit,date=new Date(item.ctime*1000).toLocaleString('ru-RU'),task=(x.activity?.tasks||[]).find(t=>t.type==='vzdump'&&String(t.id)===String(v.vmid)),change=item.sizeChangePercent==null?'Нет предыдущей':(item.sizeChangePercent>=0?'+':'')+fixed(item.sizeChangePercent)+'%';return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name||('Guest '+v.vmid))+'</div><div class="disk-meta">VMID '+v.vmid+' · '+type+' · '+date+'</div></div><span class="badge '+(fresh?'':'warn')+'">'+(fresh?'Актуальна':'Старше '+backupLimitHours+' ч')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Хранилище</span><b>'+safe(item.storage)+'</b></div><div class="disk-stat"><span>Размер</span><b>'+size(item.size)+'</b></div><div class="disk-stat"><span>К предыдущей</span><b>'+change+'</b></div><div class="disk-stat"><span>Последнее задание</span><b>'+(task?safe(task.status)+' · '+uptime(task.duration):'Общее задание')+'</b></div><div class="disk-stat"><span>Возраст</span><b>'+uptime(age)+'</b></div><div class="disk-stat"><span>Формат</span><b>'+safe(item.format)+'</b></div></div><div class="service-actions"><button data-verify-backup="'+safe(item.volid)+'">Проверить читаемость</button></div><div class="backup-check" data-backup-result="'+safe(item.volid)+'"></div></article>'}).join(''):'<span class="empty">'+(b.available?'VM и LXC не найдены':'Данные резервных копий недоступны')+'</span>';
const svc=x.services||{services:[],up:0,total:0};renderServices(svc);
const sh=x.systemHealth||{},issueCount=sh.systemErrors||0;el('systemIssues').textContent=issueCount;el('systemBar').style.width=Math.min(issueCount*10,100)+'%';el('failedUnits').textContent=sh.failedUnits??'—';el('kernelErrors').textContent=sh.kernelErrors??'—';el('dSystemErrors').textContent=issueCount;el('dKernelErrors').textContent=sh.kernelErrors||0;el('dFailedUnits').textContent=sh.failedUnits||0;el('issueList').innerHTML=(sh.issues||[]).length?sh.issues.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+new Date(v.time*1000).toLocaleString('ru-RU')+'</div></div><span class="badge bad">Ошибка</span></div><div class="settings-note issue-message">'+safe(v.message)+'</div></article>').join(''):'<span class="empty">Критических сообщений за 24 часа нет.</span>';el('failedServiceList').innerHTML=(sh.failedServices||[]).length?sh.failedServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+safe(v.description||'Описание отсутствует')+'</div></div><span class="badge bad">'+safe(v.state)+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Загрузка</span><b>'+safe(v.load)+'</b></div><div class="disk-stat"><span>Состояние</span><b>'+safe(v.active)+'</b></div><div class="disk-stat"><span>Подсостояние</span><b>'+safe(v.state)+'</b></div></div></article>').join(''):'<span class="empty">Упавших systemd-служб нет.</span>';
el('issueList').querySelectorAll('.badge').forEach((badge,index)=>{const count=(sh.issues||[])[index]?.count||1;badge.textContent=count>1?'Повторений: '+count:'1 раз'});
const sh=x.systemHealth||{},issueCount=sh.systemErrors||0;el('systemIssues').textContent=issueCount;el('systemBar').style.width=Math.min(issueCount*10,100)+'%';el('failedUnits').textContent=sh.failedUnits??'—';el('kernelErrors').textContent=sh.kernelErrors??'—';el('dSystemErrors').textContent=issueCount;el('dKernelErrors').textContent=sh.kernelErrors||0;el('dFailedUnits').textContent=sh.failedUnits||0;renderSystemHealth(sh);
renderNetworkGroups(n.interfaces||[]);
renderActivity(x.activity||{},g,svc,z);explainWarningBadges();
renderActivity(x.activity||{},g,svc,z);renderIOAttribution(disks);explainWarningBadges();
el('dot').classList.add('live');el('status').textContent='Данные поступают';
}
const percent=v=>fixed(v)+'%',temperature=v=>fixed(v)+' °C',megabytes=v=>fixed(v)+' MB/s';
const chartConfigs={cpuChart:{series:[{label:'CPU',color:'#55d58a',value:p=>p.cpuUsage,format:percent},{label:'Температура',color:'#ff9b5c',value:p=>p.cpuTemperature,format:temperature}]},memoryChart:{series:[{label:'RAM',color:'#9d8cff',value:p=>p.memoryUsage,format:percent},{label:'Swap',color:'#ff78b8',value:p=>p.swapUsage,format:percent}]},storageChart:{series:[{label:'Раздел /',color:'#ff9b5c',value:p=>p.rootUsage,format:percent}]},networkChart:{max:0,series:[{label:'Приём',color:'#43c6d9',value:p=>p.networkReceive/1048576,format:megabytes},{label:'Передача',color:'#65ddaa',value:p=>p.networkTransmit/1048576,format:megabytes}]},zfsChart:{series:[{label:'ZFS',color:'#6577ff',value:p=>p.zfsUsage,format:percent}]},upsChart:{series:[{label:'Заряд',color:'#55d58a',value:p=>p.upsCharge,format:percent},{label:'Нагрузка',color:'#d5e85a',value:p=>p.upsLoad,format:percent}]},guestsChart:{series:[{label:'CPU',color:'#ff78b8',value:p=>p.guestsCpu,format:percent},{label:'RAM',color:'#9d8cff',value:p=>p.guestsMemory,format:percent}]}};
const historyCache=new Map();
async function loadHistory(hours){const cached=historyCache.get(hours);if(cached&&Date.now()-cached.loadedAt<60000)return cached.points;const response=await fetch('/api/history?hours='+hours);if(!response.ok)throw new Error('history');const points=await response.json()||[];historyCache.set(hours,{points,loadedAt:Date.now()});return points}
const entityChartKinds={storageChart:{kind:'disk',format:temperature},zfsChart:{kind:'zfs',format:percent},guestsChart:{kind:'guest',format:percent}},chartColors=['#55d58a','#6ba8ff','#ff9b5c','#9d8cff','#ff78b8','#43c6d9','#d5e85a','#ff665c','#6577ff','#65ddaa'];
async function loadEntityChart(id,hours){const definition=entityChartKinds[id],response=await fetch('/api/entity-history?kind='+definition.kind+'&hours='+hours);if(!response.ok)throw new Error('entity history');const data=await response.json()||[],timestamps=new Map(),series=data.map((item,index)=>{const key='entity'+index;item.points.forEach(p=>{const point=timestamps.get(p.timestamp)||{timestamp:p.timestamp};point[key]=p.value1;timestamps.set(p.timestamp,point)});return {label:item.name,color:chartColors[index%chartColors.length],value:p=>p[key],format:definition.format}});chartConfigs[id]={series};return [...timestamps.values()].sort((a,b)=>a.timestamp-b.timestamp)}
const entityChartKinds={storageChart:{kind:'disk',format:temperature},diskIOChart:{kind:'disk-io',format:v=>fixed(v)+' IOPS',dynamic:true},diskUtilChart:{kind:'disk-io',format:percent,value:'value2'},zfsChart:{kind:'zfs',format:percent},guestsChart:{kind:'guest',format:percent}},chartColors=['#55d58a','#6ba8ff','#ff9b5c','#9d8cff','#ff78b8','#43c6d9','#d5e85a','#ff665c','#6577ff','#65ddaa'];
async function loadEntityChart(id,hours){const definition=entityChartKinds[id],response=await fetch('/api/entity-history?kind='+definition.kind+'&hours='+hours);if(!response.ok)throw new Error('entity history');const data=await response.json()||[],timestamps=new Map(),series=data.map((item,index)=>{const key='entity'+index;item.points.forEach(p=>{const point=timestamps.get(p.timestamp)||{timestamp:p.timestamp};point[key]=p[definition.value||'value1'];timestamps.set(p.timestamp,point)});return {label:item.name,color:chartColors[index%chartColors.length],value:p=>p[key],format:definition.format}});chartConfigs[id]={series,max:definition.dynamic?0:undefined};return [...timestamps.values()].sort((a,b)=>a.timestamp-b.timestamp)}
function drawChart(id,points){const canvas=el(id),config=chartConfigs[id];if(!canvas||!config||!canvas.offsetWidth)return;const series=config.series,dpr=window.devicePixelRatio||1,w=canvas.offsetWidth,h=200;canvas.width=w*dpr;canvas.height=h*dpr;const ctx=canvas.getContext('2d');ctx.scale(dpr,dpr);const styles=getComputedStyle(document.documentElement),text=styles.getPropertyValue('--muted').trim(),line=styles.getPropertyValue('--line').trim();ctx.clearRect(0,0,w,h);ctx.font='11px system-ui';ctx.fillStyle=text;if(points.length<2){ctx.textAlign='center';ctx.fillText('История появится после нескольких минут работы',w/2,h/2);canvas._chartState=null;return}const values=series.flatMap(s=>points.map(p=>s.value(p)).filter(v=>v!=null&&Number.isFinite(v))),max=config.max===0?Math.max(...values,1)*1.1:100,pad={l:38,r:12,t:30,b:25},pw=w-pad.l-pad.r,ph=h-pad.t-pad.b;ctx.strokeStyle=line;ctx.lineWidth=1;for(let i=0;i<=4;i++){const y=pad.t+ph*i/4;ctx.beginPath();ctx.moveTo(pad.l,y);ctx.lineTo(w-pad.r,y);ctx.stroke();ctx.fillStyle=text;ctx.textAlign='right';ctx.fillText((max*(1-i/4)).toFixed(max<10?1:0),pad.l-7,y+4)}series.forEach((s,index)=>{ctx.strokeStyle=s.color;ctx.lineWidth=2;ctx.beginPath();let started=false;points.forEach((p,i)=>{const v=s.value(p);if(v==null||!Number.isFinite(v))return;const x=pad.l+pw*i/(points.length-1),y=pad.t+ph*(1-v/(max||1));if(!started){ctx.moveTo(x,y);started=true}else ctx.lineTo(x,y)});ctx.stroke();ctx.fillStyle=s.color;ctx.textAlign='left';ctx.fillText(s.label,pad.l+index*130,15)});const first=new Date(points[0].timestamp*1000),last=new Date(points[points.length-1].timestamp*1000),timeOptions={hour:'2-digit',minute:'2-digit'};ctx.fillStyle=text;ctx.textAlign='left';ctx.fillText(first.toLocaleString('ru-RU',timeOptions),pad.l,h-6);ctx.textAlign='right';ctx.fillText(last.toLocaleString('ru-RU',timeOptions),w-pad.r,h-6);canvas._chartState={points,series,pad,pw,w}}
function setupChartHover(canvas){canvas.addEventListener('mousemove',event=>{const state=canvas._chartState,section=canvas.closest('.chart-section'),tooltip=section&&section.querySelector('.chart-tooltip');if(!state||!tooltip)return;const rect=canvas.getBoundingClientRect(),x=event.clientX-rect.left,index=Math.max(0,Math.min(state.points.length-1,Math.round((x-state.pad.l)/state.pw*(state.points.length-1)))),point=state.points[index],date=new Date(point.timestamp*1000);tooltip.innerHTML='<div class="chart-tooltip-time">'+date.toLocaleString('ru-RU')+'</div>'+state.series.map(s=>{const value=s.value(point),shown=value==null||!Number.isFinite(value)?'—':s.format(value);return '<div class="chart-tooltip-value"><span><i style="background:'+s.color+'"></i>'+s.label+'</span><strong>'+shown+'</strong></div>'}).join('');tooltip.classList.add('visible');const sectionRect=section.getBoundingClientRect(),left=Math.min(event.clientX-sectionRect.left+12,section.clientWidth-tooltip.offsetWidth-10);tooltip.style.left=Math.max(10,left)+'px';tooltip.style.top=Math.max(55,event.clientY-sectionRect.top-tooltip.offsetHeight-12)+'px'});canvas.addEventListener('mouseleave',()=>{const tooltip=canvas.closest('.chart-section').querySelector('.chart-tooltip');tooltip.classList.remove('visible')})}
async function showHistory(section){const hours=Number(section.dataset.hours||1),canvas=section.querySelector('canvas');try{drawChart(canvas.id,entityChartKinds[canvas.id]?await loadEntityChart(canvas.id,hours):await loadHistory(hours))}catch(e){drawChart(canvas.id,[])}}
function setupDetailTabs(){document.querySelectorAll('dialog').forEach(dialog=>{const chart=dialog.querySelector('.chart-section');if(!chart)return;const head=dialog.querySelector('.modal-head'),tabs=document.createElement('div'),realtime=document.createElement('div');tabs.className='detail-tabs';tabs.innerHTML='<button class="detail-tab active" data-detail-tab="realtime">Реальное время</button><button class="detail-tab" data-detail-tab="history">История</button>';realtime.className='detail-panel';realtime.dataset.detailPanel='realtime';while(head.nextElementSibling&&head.nextElementSibling!==chart)realtime.appendChild(head.nextElementSibling);head.after(tabs);tabs.after(realtime);chart.classList.add('detail-panel');chart.dataset.detailPanel='history';chart.dataset.hours='1';chart.hidden=true;const title=chart.querySelector('.chart-head strong'),chartHead=chart.querySelector('.chart-head');chartHead.innerHTML='';chartHead.appendChild(title);const ranges=document.createElement('div');ranges.className='range-switch';ranges.innerHTML='<button class="range-button active" data-hours="1">1 час</button><button class="range-button" data-hours="24">24 часа</button><button class="range-button" data-hours="168">7 дней</button>';chartHead.appendChild(ranges);const tooltip=document.createElement('div');tooltip.className='chart-tooltip';chart.appendChild(tooltip);setupChartHover(chart.querySelector('canvas'));tabs.querySelectorAll('[data-detail-tab]').forEach(tab=>tab.onclick=()=>{tabs.querySelectorAll('.detail-tab').forEach(t=>t.classList.toggle('active',t===tab));realtime.hidden=tab.dataset.detailTab!=='realtime';chart.hidden=tab.dataset.detailTab!=='history';if(!chart.hidden)setTimeout(()=>showHistory(chart),20)});ranges.querySelectorAll('[data-hours]').forEach(button=>button.onclick=()=>{chart.dataset.hours=button.dataset.hours;ranges.querySelectorAll('.range-button').forEach(b=>b.classList.toggle('active',b===button));showHistory(chart)})})}
async function showHistory(section){const hours=Number(section.dataset.hours||1);await Promise.all([...section.querySelectorAll('canvas')].map(async canvas=>{try{drawChart(canvas.id,entityChartKinds[canvas.id]?await loadEntityChart(canvas.id,hours):await loadHistory(hours))}catch(e){drawChart(canvas.id,[])}}))}
function setupDetailTabs(){document.querySelectorAll('dialog').forEach(dialog=>{const chart=dialog.querySelector('.chart-section');if(!chart)return;const head=dialog.querySelector('.modal-head'),tabs=document.createElement('div'),realtime=document.createElement('div');tabs.className='detail-tabs';tabs.innerHTML='<button class="detail-tab active" data-detail-tab="realtime">Реальное время</button><button class="detail-tab" data-detail-tab="history">История</button>';realtime.className='detail-panel';realtime.dataset.detailPanel='realtime';while(head.nextElementSibling&&head.nextElementSibling!==chart)realtime.appendChild(head.nextElementSibling);head.after(tabs);tabs.after(realtime);chart.classList.add('detail-panel');chart.dataset.detailPanel='history';chart.dataset.hours='1';chart.hidden=true;const title=chart.querySelector('.chart-head strong'),chartHead=chart.querySelector('.chart-head');chartHead.innerHTML='';chartHead.appendChild(title);const ranges=document.createElement('div');ranges.className='range-switch';ranges.innerHTML='<button class="range-button active" data-hours="1">1 час</button><button class="range-button" data-hours="24">24 часа</button><button class="range-button" data-hours="168">7 дней</button>';chartHead.appendChild(ranges);const tooltip=document.createElement('div');tooltip.className='chart-tooltip';chart.appendChild(tooltip);chart.querySelectorAll('canvas').forEach(setupChartHover);tabs.querySelectorAll('[data-detail-tab]').forEach(tab=>tab.onclick=()=>{tabs.querySelectorAll('.detail-tab').forEach(t=>t.classList.toggle('active',t===tab));realtime.hidden=tab.dataset.detailTab!=='realtime';chart.hidden=tab.dataset.detailTab!=='history';if(!chart.hidden)setTimeout(()=>showHistory(chart),20)});ranges.querySelectorAll('[data-hours]').forEach(button=>button.onclick=()=>{chart.dataset.hours=button.dataset.hours;ranges.querySelectorAll('.range-button').forEach(b=>b.classList.toggle('active',b===button));showHistory(chart)})})}
setupDetailTabs();
document.querySelectorAll('[data-open]').forEach(b=>b.onclick=()=>el(b.dataset.open).showModal());
document.querySelectorAll('dialog').forEach(d=>{d.querySelector('.close').onclick=()=>d.close();d.onclick=e=>{if(e.target===d)d.close()}});
document.querySelectorAll('[data-system-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-system-tab]').forEach(v=>v.classList.toggle('active',v===tab));document.querySelectorAll('[data-system-panel]').forEach(v=>v.hidden=v.dataset.systemPanel!==tab.dataset.systemTab)});
el('systemSort').onclick=()=>{systemSortNewest=!systemSortNewest;renderSystemHealth(currentSystemHealth)};
el('serviceList').onclick=event=>{const button=event.target.closest('[data-service-open]');if(!button)return;const service=currentServiceStatuses.find(v=>v.id===Number(button.dataset.serviceOpen));if(!service)return;const choices=(service.endpoints||[]).filter(v=>/^https?:\/\//.test(v.address));if(choices.length===1){window.open(choices[0].address,'_blank','noopener');return}if(!choices.length)return;el('openServiceTitle').textContent=service.name;el('openServiceChoices').innerHTML=choices.map(v=>'<a class="theme-option" href="'+safe(v.address)+'" target="_blank" rel="noopener">'+safe(v.label)+'<small style="display:block;color:var(--muted);margin-top:4px">'+safe(v.address)+'</small></a>').join('');el('openServiceDialog').showModal()};
document.querySelectorAll('[data-task-filter]').forEach(button=>button.onclick=()=>{currentTaskFilter=button.dataset.taskFilter;document.querySelectorAll('[data-task-filter]').forEach(v=>v.classList.toggle('active',v===button));renderTasks(currentTasks,currentTaskFilter);explainWarningBadges()});
el('backupList').onclick=async event=>{const button=event.target.closest('[data-verify-backup]');if(!button)return;const target=el('backupList').querySelector('[data-backup-result="'+CSS.escape(button.dataset.verifyBackup)+'"]');button.disabled=true;target.textContent='Проверяем метаданные и читаемость…';try{const response=await fetch('/api/backups/verify',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({volid:button.dataset.verifyBackup})}),result=await response.json();target.textContent=(result.ok?'✓ ':'⚠ ')+result.message;target.className='backup-check '+(result.ok?'status-up':'status-down')}catch(e){target.textContent='Не удалось выполнить проверку.'}finally{button.disabled=false}};
@@ -727,7 +741,7 @@ el('cardOrder').onclick=event=>{const button=event.target.closest('[data-move]')
function setTheme(theme){document.documentElement.dataset.theme=theme;localStorage.setItem('proxmox-theme',theme);document.querySelectorAll('[data-theme-choice]').forEach(b=>b.classList.toggle('active',b.dataset.themeChoice===theme));document.querySelectorAll('.chart-section:not([hidden])').forEach(showHistory)}document.querySelectorAll('[data-theme-choice]').forEach(b=>b.onclick=()=>setTheme(b.dataset.themeChoice));setTheme(document.documentElement.dataset.theme);el('settingsButton').onclick=()=>el('settingsDialog').showModal();
document.querySelectorAll('[data-settings-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-settings-tab]').forEach(t=>t.classList.toggle('active',t===tab));document.querySelectorAll('[data-settings-panel]').forEach(p=>p.hidden=p.dataset.settingsPanel!==tab.dataset.settingsTab)});
async function loadThresholds(){const response=await fetch('/api/settings/alerts');if(!response.ok)return;alertSettings=await response.json();document.querySelectorAll('[data-threshold]').forEach(input=>input.value=alertSettings[input.dataset.threshold]??'')}
el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{});
el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=input.dataset.threshold==='diskIoExcluded'?input.value:Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{});
let configuredServices=[];
async function loadServices(){const response=await fetch('/api/services');if(!response.ok)return;configuredServices=await response.json()||[];el('serviceSettingsList').innerHTML=configuredServices.length?configuredServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+safe([v.url&&('Внешний: '+v.url),v.localAddress&&('Локальный: '+v.localAddress)].filter(Boolean).join(' · '))+'</div></div><span class="badge '+(v.enabled?'':'warn')+'">'+(v.enabled?'Включён':'Выключен')+'</span></div><div class="service-actions"><button data-service-edit="'+v.id+'">Изменить</button><button data-service-delete="'+v.id+'">Удалить</button></div></article>').join(''):'<span class="empty">Список пуст.</span>'}
function resetServiceForm(){el('serviceId').value='';el('serviceName').value='';el('serviceType').value='http';el('serviceURL').value='';el('serviceLocal').value='';el('serviceCode').value=200;el('serviceText').value='';el('serviceTimeout').value=5;el('serviceEnabled').checked=true}