Add per-disk IO performance monitoring
This commit is contained in:
Maxim
158
disks.go
158
disks.go
@@ -13,46 +13,63 @@ import (
|
||||
)
|
||||
|
||||
type DiskMetrics struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Model string `json:"model"`
|
||||
Serial string `json:"serial"`
|
||||
Type string `json:"type"`
|
||||
Protocol string `json:"protocol"`
|
||||
SizeBytes uint64 `json:"sizeBytes"`
|
||||
UsedBytes *uint64 `json:"usedBytes"`
|
||||
UsagePercent *float64 `json:"usagePercent"`
|
||||
SMARTStatus string `json:"smartStatus"`
|
||||
SMARTAvailable bool `json:"smartAvailable"`
|
||||
AttentionReasons []string `json:"attentionReasons"`
|
||||
Temperature *float64 `json:"temperatureCelsius"`
|
||||
PowerOnHours *uint64 `json:"powerOnHours"`
|
||||
WearUsedPercent *float64 `json:"wearUsedPercent"`
|
||||
Reallocated uint64 `json:"reallocated"`
|
||||
Pending uint64 `json:"pending"`
|
||||
Uncorrectable uint64 `json:"uncorrectable"`
|
||||
CRCErrors uint64 `json:"crcErrors"`
|
||||
MediaErrors uint64 `json:"mediaErrors"`
|
||||
CounterChanges []string `json:"counterChanges"`
|
||||
LastSelfTest string `json:"lastSelfTest"`
|
||||
LastSelfTestAt string `json:"lastSelfTestAt"`
|
||||
SelfTestNeverRun bool `json:"selfTestNeverRun"`
|
||||
ShortTestStarted bool `json:"shortTestStarted"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Model string `json:"model"`
|
||||
Serial string `json:"serial"`
|
||||
Type string `json:"type"`
|
||||
Protocol string `json:"protocol"`
|
||||
SizeBytes uint64 `json:"sizeBytes"`
|
||||
UsedBytes *uint64 `json:"usedBytes"`
|
||||
UsagePercent *float64 `json:"usagePercent"`
|
||||
SMARTStatus string `json:"smartStatus"`
|
||||
SMARTAvailable bool `json:"smartAvailable"`
|
||||
AttentionReasons []string `json:"attentionReasons"`
|
||||
Temperature *float64 `json:"temperatureCelsius"`
|
||||
PowerOnHours *uint64 `json:"powerOnHours"`
|
||||
WearUsedPercent *float64 `json:"wearUsedPercent"`
|
||||
Reallocated uint64 `json:"reallocated"`
|
||||
Pending uint64 `json:"pending"`
|
||||
Uncorrectable uint64 `json:"uncorrectable"`
|
||||
CRCErrors uint64 `json:"crcErrors"`
|
||||
MediaErrors uint64 `json:"mediaErrors"`
|
||||
CounterChanges []string `json:"counterChanges"`
|
||||
LastSelfTest string `json:"lastSelfTest"`
|
||||
LastSelfTestAt string `json:"lastSelfTestAt"`
|
||||
SelfTestNeverRun bool `json:"selfTestNeverRun"`
|
||||
ShortTestStarted bool `json:"shortTestStarted"`
|
||||
ReadIOPS float64 `json:"readIops"`
|
||||
WriteIOPS float64 `json:"writeIops"`
|
||||
ReadBytesPerSec float64 `json:"readBytesPerSec"`
|
||||
WriteBytesPerSec float64 `json:"writeBytesPerSec"`
|
||||
ReadLatencyMs float64 `json:"readLatencyMs"`
|
||||
WriteLatencyMs float64 `json:"writeLatencyMs"`
|
||||
LatencyMs float64 `json:"latencyMs"`
|
||||
Utilization float64 `json:"utilizationPercent"`
|
||||
QueueDepth uint64 `json:"queueDepth"`
|
||||
AverageQueue float64 `json:"averageQueue"`
|
||||
IOPressureSeconds float64 `json:"ioPressureSeconds"`
|
||||
}
|
||||
|
||||
type diskIOSnapshot struct{ reads, readSectors, readMS, writes, writeSectors, writeMS, inFlight, ioMS, weightedMS uint64 }
|
||||
|
||||
type diskCollector struct {
|
||||
mu sync.Mutex
|
||||
cached []DiskMetrics
|
||||
updatedAt time.Time
|
||||
previous map[string]DiskMetrics
|
||||
missing []string
|
||||
lastTests map[string]time.Time
|
||||
mu sync.Mutex
|
||||
cached []DiskMetrics
|
||||
updatedAt time.Time
|
||||
previous map[string]DiskMetrics
|
||||
missing []string
|
||||
lastTests map[string]time.Time
|
||||
ioPrevious map[string]diskIOSnapshot
|
||||
ioUpdatedAt time.Time
|
||||
ioPressureSince map[string]time.Time
|
||||
}
|
||||
|
||||
func (c *diskCollector) collect() []DiskMetrics {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if time.Since(c.updatedAt) < time.Minute && c.cached != nil {
|
||||
c.applyDiskIO(c.cached)
|
||||
return c.cached
|
||||
}
|
||||
current := readPhysicalDisks()
|
||||
@@ -87,10 +104,89 @@ func (c *diskCollector) collect() []DiskMetrics {
|
||||
}
|
||||
}
|
||||
c.cached = current
|
||||
c.applyDiskIO(c.cached)
|
||||
c.updatedAt = time.Now()
|
||||
return c.cached
|
||||
}
|
||||
|
||||
func (c *diskCollector) applyDiskIO(disks []DiskMetrics) {
|
||||
now := time.Now()
|
||||
snapshots := readDiskIOStats("/proc/diskstats")
|
||||
if c.ioPrevious == nil {
|
||||
c.ioPrevious = map[string]diskIOSnapshot{}
|
||||
c.ioPressureSince = map[string]time.Time{}
|
||||
}
|
||||
elapsed := now.Sub(c.ioUpdatedAt).Seconds()
|
||||
for i := range disks {
|
||||
disk := &disks[i]
|
||||
disk.ReadIOPS, disk.WriteIOPS, disk.ReadBytesPerSec, disk.WriteBytesPerSec = 0, 0, 0, 0
|
||||
disk.ReadLatencyMs, disk.WriteLatencyMs, disk.LatencyMs, disk.Utilization, disk.AverageQueue, disk.IOPressureSeconds = 0, 0, 0, 0, 0, 0
|
||||
current, ok := snapshots[disk.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
disk.QueueDepth = current.inFlight
|
||||
if old, found := c.ioPrevious[disk.Name]; found && elapsed > 0 && diskIOCountersMonotonic(old, current) {
|
||||
readOps := current.reads - old.reads
|
||||
writeOps := current.writes - old.writes
|
||||
disk.ReadIOPS = float64(readOps) / elapsed
|
||||
disk.WriteIOPS = float64(writeOps) / elapsed
|
||||
disk.ReadBytesPerSec = float64(current.readSectors-old.readSectors) * 512 / elapsed
|
||||
disk.WriteBytesPerSec = float64(current.writeSectors-old.writeSectors) * 512 / elapsed
|
||||
if readOps > 0 {
|
||||
disk.ReadLatencyMs = float64(current.readMS-old.readMS) / float64(readOps)
|
||||
}
|
||||
if writeOps > 0 {
|
||||
disk.WriteLatencyMs = float64(current.writeMS-old.writeMS) / float64(writeOps)
|
||||
}
|
||||
if readOps+writeOps > 0 {
|
||||
disk.LatencyMs = float64(current.readMS-old.readMS+current.writeMS-old.writeMS) / float64(readOps+writeOps)
|
||||
}
|
||||
disk.Utilization = min(float64(current.ioMS-old.ioMS)/(elapsed*10), 100)
|
||||
disk.AverageQueue = float64(current.weightedMS-old.weightedMS) / (elapsed * 1000)
|
||||
latencyLimit := 20.0
|
||||
if disk.Type == "HDD" {
|
||||
latencyLimit = 50
|
||||
}
|
||||
pressured := disk.Utilization >= 90 || (disk.LatencyMs >= latencyLimit && disk.ReadIOPS+disk.WriteIOPS >= 1) || disk.AverageQueue >= 4
|
||||
if pressured {
|
||||
if c.ioPressureSince[disk.Name].IsZero() {
|
||||
c.ioPressureSince[disk.Name] = now
|
||||
}
|
||||
disk.IOPressureSeconds = now.Sub(c.ioPressureSince[disk.Name]).Seconds()
|
||||
} else {
|
||||
delete(c.ioPressureSince, disk.Name)
|
||||
}
|
||||
}
|
||||
c.ioPrevious[disk.Name] = current
|
||||
}
|
||||
c.ioUpdatedAt = now
|
||||
}
|
||||
|
||||
func diskIOCountersMonotonic(old, current diskIOSnapshot) bool {
|
||||
return current.reads >= old.reads && current.readSectors >= old.readSectors && current.readMS >= old.readMS && current.writes >= old.writes && current.writeSectors >= old.writeSectors && current.writeMS >= old.writeMS && current.ioMS >= old.ioMS && current.weightedMS >= old.weightedMS
|
||||
}
|
||||
|
||||
func readDiskIOStats(path string) map[string]diskIOSnapshot {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
result := map[string]diskIOSnapshot{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 14 || !isPhysicalDiskName(f[2]) {
|
||||
continue
|
||||
}
|
||||
values := make([]uint64, 11)
|
||||
for i := 0; i < 11; i++ {
|
||||
values[i], _ = strconv.ParseUint(f[i+3], 10, 64)
|
||||
}
|
||||
result[f[2]] = diskIOSnapshot{reads: values[0], readSectors: values[2], readMS: values[3], writes: values[4], writeSectors: values[6], writeMS: values[7], inFlight: values[8], ioMS: values[9], weightedMS: values[10]}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (c *diskCollector) missingDisks() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
Reference in New Issue
Block a user