maxim/ProxmoxDash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity

Add IO attribution anomaly detection and alert correlation

Browse Source
This commit is contained in:
Maxim
2026-08-03 18:02:13 +03:00
parent 98afa2517b
commit 8241fba6f4
12 changed files with 498 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
disks.go
View File

@@ -13,42 +13,46 @@ import (
)
type DiskMetrics struct {
Name string `json:"name"`
Path string `json:"path"`
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"`
Protocol string `json:"protocol"`
SizeBytes uint64 `json:"sizeBytes"`
UsedBytes *uint64 `json:"usedBytes"`
UsagePercent *float64 `json:"usagePercent"`
SMARTStatus string `json:"smartStatus"`
SMARTAvailable bool `json:"smartAvailable"`
AttentionReasons []string `json:"attentionReasons"`
Temperature *float64 `json:"temperatureCelsius"`
PowerOnHours *uint64 `json:"powerOnHours"`
WearUsedPercent *float64 `json:"wearUsedPercent"`
Reallocated uint64 `json:"reallocated"`
Pending uint64 `json:"pending"`
Uncorrectable uint64 `json:"uncorrectable"`
CRCErrors uint64 `json:"crcErrors"`
MediaErrors uint64 `json:"mediaErrors"`
CounterChanges []string `json:"counterChanges"`
LastSelfTest string `json:"lastSelfTest"`
LastSelfTestAt string `json:"lastSelfTestAt"`
SelfTestNeverRun bool `json:"selfTestNeverRun"`
ShortTestStarted bool `json:"shortTestStarted"`
ReadIOPS float64 `json:"readIops"`
WriteIOPS float64 `json:"writeIops"`
ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
ReadLatencyMs float64 `json:"readLatencyMs"`
WriteLatencyMs float64 `json:"writeLatencyMs"`
LatencyMs float64 `json:"latencyMs"`
Utilization float64 `json:"utilizationPercent"`
QueueDepth uint64 `json:"queueDepth"`
AverageQueue float64 `json:"averageQueue"`
IOPressureSeconds float64 `json:"ioPressureSeconds"`
Name string `json:"name"`
Path string `json:"path"`
Model string `json:"model"`
Serial string `json:"serial"`
Type string `json:"type"`
Protocol string `json:"protocol"`
SizeBytes uint64 `json:"sizeBytes"`
UsedBytes *uint64 `json:"usedBytes"`
UsagePercent *float64 `json:"usagePercent"`
SMARTStatus string `json:"smartStatus"`
SMARTAvailable bool `json:"smartAvailable"`
AttentionReasons []string `json:"attentionReasons"`
Temperature *float64 `json:"temperatureCelsius"`
PowerOnHours *uint64 `json:"powerOnHours"`
WearUsedPercent *float64 `json:"wearUsedPercent"`
Reallocated uint64 `json:"reallocated"`
Pending uint64 `json:"pending"`
Uncorrectable uint64 `json:"uncorrectable"`
CRCErrors uint64 `json:"crcErrors"`
MediaErrors uint64 `json:"mediaErrors"`
CounterChanges []string `json:"counterChanges"`
LastSelfTest string `json:"lastSelfTest"`
LastSelfTestAt string `json:"lastSelfTestAt"`
SelfTestNeverRun bool `json:"selfTestNeverRun"`
ShortTestStarted bool `json:"shortTestStarted"`
ReadIOPS float64 `json:"readIops"`
WriteIOPS float64 `json:"writeIops"`
ReadBytesPerSec float64 `json:"readBytesPerSec"`
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
ReadLatencyMs float64 `json:"readLatencyMs"`
WriteLatencyMs float64 `json:"writeLatencyMs"`
LatencyMs float64 `json:"latencyMs"`
Utilization float64 `json:"utilizationPercent"`
QueueDepth uint64 `json:"queueDepth"`
AverageQueue float64 `json:"averageQueue"`
IOPressureSeconds float64 `json:"ioPressureSeconds"`
IOCause string `json:"ioCause"`
IOCauseDetail string `json:"ioCauseDetail"`
IOConfidence int `json:"ioConfidence"`
TopConsumers []IOConsumer `json:"topConsumers"`
}
type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 }
@@ -65,11 +69,11 @@ type diskCollector struct {
ioPressureSince map[string]time.Time
}
func (c *diskCollector) collect() []DiskMetrics {
func (c *diskCollector) collect(thresholds AlertThresholds) []DiskMetrics {
c.mu.Lock()
defer c.mu.Unlock()
if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
c.applyDiskIO(c.cached)
c.applyDiskIO(c.cached, thresholds)
return c.cached
}
current := readPhysicalDisks()
@@ -104,12 +108,12 @@ func (c *diskCollector) collect() []DiskMetrics {
}
}
c.cached = current
c.applyDiskIO(c.cached)
c.applyDiskIO(c.cached, thresholds)
c.updatedAt = time.Now()
return c.cached
}
func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
func (c *diskCollector) applyDiskIO(disks []DiskMetrics, thresholds AlertThresholds) {
now := time.Now()
snapshots := readDiskIOStats("/proc/diskstats")
if c.ioPrevious == nil {
@@ -144,11 +148,13 @@ func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
}
disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100)
disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000)
latencyLimit := 20.0
utilLimit, latencyLimit, queueLimit := thresholds.DiskSSDUtil, thresholds.DiskSSDLatency, thresholds.DiskSSDQueue
if disk.Type == "HDD" {
latencyLimit = 50
utilLimit, latencyLimit, queueLimit = thresholds.DiskHDDUtil, thresholds.DiskHDDLatency, thresholds.DiskHDDQueue
} else if strings.Contains(disk.Type, "NVMe") {
utilLimit, latencyLimit, queueLimit = thresholds.DiskNVMeUtil, thresholds.DiskNVMeLatency, thresholds.DiskNVMeQueue
}
pressured := disk.Utilization >= 90 || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= 4
pressured := disk.Utilization >= utilLimit || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= queueLimit
if pressured {
if c.ioPressureSince[disk.Name].IsZero() {
c.ioPressureSince[disk.Name] = now