Add infrastructure reliability monitoring

disks.go

@@ -28,12 +28,25 @@ type DiskMetrics struct {
 	Temperature      *float64 `json:"temperatureCelsius"`
 	PowerOnHours     *uint64  `json:"powerOnHours"`
 	WearUsedPercent  *float64 `json:"wearUsedPercent"`
+	Reallocated      uint64   `json:"reallocated"`
+	Pending          uint64   `json:"pending"`
+	Uncorrectable    uint64   `json:"uncorrectable"`
+	CRCErrors        uint64   `json:"crcErrors"`
+	MediaErrors      uint64   `json:"mediaErrors"`
+	CounterChanges   []string `json:"counterChanges"`
+	LastSelfTest     string   `json:"lastSelfTest"`
+	LastSelfTestAt   string   `json:"lastSelfTestAt"`
+	SelfTestNeverRun bool     `json:"selfTestNeverRun"`
+	ShortTestStarted bool     `json:"shortTestStarted"`
 }
 
 type diskCollector struct {
 	mu        sync.Mutex
 	cached    []DiskMetrics
 	updatedAt time.Time
+	previous  map[string]DiskMetrics
+	missing   []string
+	lastTests map[string]time.Time
 }
 
 func (c *diskCollector) collect() []DiskMetrics {
@@ -42,11 +55,73 @@ func (c *diskCollector) collect() []DiskMetrics {
 	if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
 		return c.cached
 	}
-	c.cached = readPhysicalDisks()
+	current := readPhysicalDisks()
+	if c.previous == nil {
+		c.previous = map[string]DiskMetrics{}
+		c.lastTests = map[string]time.Time{}
+	}
+	seen := map[string]bool{}
+	c.missing = nil
+	for i := range current {
+		disk := &current[i]
+		key := disk.Serial
+		if key == "" || key == "Не указан" {
+			key = disk.Path
+		}
+		seen[key] = true
+		if old, ok := c.previous[key]; ok {
+			disk.CounterChanges = diskCounterChanges(old, *disk)
+			disk.AttentionReasons = append(disk.AttentionReasons, disk.CounterChanges...)
+		}
+		if disk.SMARTAvailable && (disk.SelfTestNeverRun || selfTestOlderThanWeek(disk.LastSelfTestAt)) && time.Since(c.lastTests[key]) >= 7*24*time.Hour {
+			disk.ShortTestStarted = startShortSMARTTest(disk.Path)
+			if disk.ShortTestStarted {
+				c.lastTests[key] = time.Now()
+			}
+		}
+		c.previous[key] = *disk
+	}
+	for key, disk := range c.previous {
+		if !seen[key] {
+			c.missing = append(c.missing, disk.Model+" ("+disk.Serial+")")
+		}
+	}
+	c.cached = current
 	c.updatedAt = time.Now()
 	return c.cached
 }
 
+func (c *diskCollector) missingDisks() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]string(nil), c.missing...)
+}
+
+func diskCounterChanges(old, current DiskMetrics) []string {
+	checks := []struct {
+		label    string
+		old, now uint64
+	}{{"Reallocated вырос", old.Reallocated, current.Reallocated}, {"Pending вырос", old.Pending, current.Pending}, {"Uncorrectable вырос", old.Uncorrectable, current.Uncorrectable}, {"CRC-ошибки выросли", old.CRCErrors, current.CRCErrors}, {"Media errors выросли", old.MediaErrors, current.MediaErrors}}
+	var changes []string
+	for _, check := range checks {
+		if check.now > check.old {
+			changes = append(changes, check.label+": "+strconv.FormatUint(check.old, 10)+" → "+strconv.FormatUint(check.now, 10))
+		}
+	}
+	return changes
+}
+
+func selfTestOlderThanWeek(value string) bool {
+	when, err := time.Parse(time.RFC3339, value)
+	return err == nil && time.Since(when) > 7*24*time.Hour
+}
+
+func startShortSMARTTest(path string) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return exec.CommandContext(ctx, "smartctl", "-t", "short", path).Run() == nil
+}
+
 func readPhysicalDisks() []DiskMetrics {
 	entries, _ := os.ReadDir("/sys/block")
 	usage := readDiskUsage()
@@ -202,6 +277,16 @@ type smartctlJSON struct {
 			Count int `json:"count"`
 		} `json:"summary"`
 	} `json:"ata_smart_error_log"`
+	ATASelfTestLog struct {
+		Standard struct {
+			Table []struct {
+				Status struct {
+					String string `json:"string"`
+				} `json:"status"`
+				LifetimeHours uint64 `json:"lifetime_hours"`
+			} `json:"table"`
+		} `json:"standard"`
+	} `json:"ata_smart_self_test_log"`
 	NVMeHealth *struct {
 		Temperature     float64 `json:"temperature"`
 		PercentageUsed  float64 `json:"percentage_used"`
@@ -254,6 +339,7 @@ func applySMART(disk *DiskMetrics) {
 	}
 	if smart.NVMeHealth != nil {
 		disk.WearUsedPercent = &smart.NVMeHealth.PercentageUsed
+		disk.MediaErrors = smart.NVMeHealth.MediaErrors
 		if smart.NVMeHealth.CriticalWarning != 0 {
 			disk.AttentionReasons = append(disk.AttentionReasons, "NVMe сообщает критическое предупреждение")
 		}
@@ -261,6 +347,17 @@ func applySMART(disk *DiskMetrics) {
 			disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
 		}
 	}
+	if len(smart.ATASelfTestLog.Standard.Table) > 0 {
+		last := smart.ATASelfTestLog.Standard.Table[0]
+		disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
+		if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours {
+			disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339)
+		}
+	} else {
+		disk.SelfTestNeverRun = true
+		disk.LastSelfTest = "Никогда не запускался"
+		disk.AttentionReasons = append(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался")
+	}
 	applyATAAttributes(disk, smart)
 	applySMARTExitStatus(disk, smart.Smartctl.ExitStatus)
 	if len(disk.AttentionReasons) > 0 && disk.SMARTStatus == "Исправен" {
@@ -275,6 +372,16 @@ func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
 		"Wear_Leveling_Count": true,
 	}
 	for _, attribute := range smart.ATAAttributes.Table {
+		switch attribute.Name {
+		case "Reallocated_Sector_Ct", "Reallocated_Event_Count":
+			disk.Reallocated += uint64(max(attribute.Raw.Value, 0))
+		case "Current_Pending_Sector":
+			disk.Pending = uint64(max(attribute.Raw.Value, 0))
+		case "Offline_Uncorrectable", "Reported_Uncorrect":
+			disk.Uncorrectable += uint64(max(attribute.Raw.Value, 0))
+		case "UDMA_CRC_Error_Count", "Interface_CRC_Error_Count":
+			disk.CRCErrors += uint64(max(attribute.Raw.Value, 0))
+		}
 		if wearNames[attribute.Name] && attribute.Value >= 0 && attribute.Value <= 100 {
 			wear := float64(100 - attribute.Value)
 			disk.WearUsedPercent = &wear