Add infrastructure reliability monitoring
This commit is contained in:
Maxim
205
zfs.go
205
zfs.go
@@ -2,7 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -10,28 +12,66 @@ import (
|
||||
)
|
||||
|
||||
type ZFSPoolMetrics struct {
|
||||
Name string `json:"name"`
|
||||
Health string `json:"health"`
|
||||
SizeBytes uint64 `json:"sizeBytes"`
|
||||
AllocatedBytes uint64 `json:"allocatedBytes"`
|
||||
FreeBytes uint64 `json:"freeBytes"`
|
||||
CapacityPercent float64 `json:"capacityPercent"`
|
||||
Fragmentation float64 `json:"fragmentationPercent"`
|
||||
DedupRatio string `json:"dedupRatio"`
|
||||
Scan string `json:"scan"`
|
||||
Errors string `json:"errors"`
|
||||
AttentionReasons []string `json:"attentionReasons"`
|
||||
Name string `json:"name"`
|
||||
Health string `json:"health"`
|
||||
SizeBytes uint64 `json:"sizeBytes"`
|
||||
AllocatedBytes uint64 `json:"allocatedBytes"`
|
||||
FreeBytes uint64 `json:"freeBytes"`
|
||||
CapacityPercent float64 `json:"capacityPercent"`
|
||||
Fragmentation float64 `json:"fragmentationPercent"`
|
||||
DedupRatio string `json:"dedupRatio"`
|
||||
Scan string `json:"scan"`
|
||||
Errors string `json:"errors"`
|
||||
AttentionReasons []string `json:"attentionReasons"`
|
||||
ReadErrors uint64 `json:"readErrors"`
|
||||
WriteErrors uint64 `json:"writeErrors"`
|
||||
ChecksumErrors uint64 `json:"checksumErrors"`
|
||||
VDevs []ZFSVDev `json:"vdevs"`
|
||||
ScrubStatus string `json:"scrubStatus"`
|
||||
LastScrubAt string `json:"lastScrubAt"`
|
||||
ScrubDuration string `json:"scrubDuration"`
|
||||
ScrubTooOld bool `json:"scrubTooOld"`
|
||||
ErrorChanges []string `json:"errorChanges"`
|
||||
}
|
||||
|
||||
type ZFSVDev struct {
|
||||
Name string `json:"name"`
|
||||
Class string `json:"class"`
|
||||
State string `json:"state"`
|
||||
Read uint64 `json:"read"`
|
||||
Write uint64 `json:"write"`
|
||||
Checksum uint64 `json:"checksum"`
|
||||
}
|
||||
type ZFSDataset struct {
|
||||
Name string `json:"name"`
|
||||
Pool string `json:"pool"`
|
||||
UsedBytes uint64 `json:"usedBytes"`
|
||||
AvailableBytes uint64 `json:"availableBytes"`
|
||||
ReferencedBytes uint64 `json:"referencedBytes"`
|
||||
Mountpoint string `json:"mountpoint"`
|
||||
CapacityPercent float64 `json:"capacityPercent"`
|
||||
}
|
||||
type ZFSARC struct {
|
||||
Available bool `json:"available"`
|
||||
SizeBytes uint64 `json:"sizeBytes"`
|
||||
TargetBytes uint64 `json:"targetBytes"`
|
||||
HitRatio float64 `json:"hitRatio"`
|
||||
Hits uint64 `json:"hits"`
|
||||
Misses uint64 `json:"misses"`
|
||||
}
|
||||
|
||||
type ZFSMetrics struct {
|
||||
Available bool `json:"available"`
|
||||
Pools []ZFSPoolMetrics `json:"pools"`
|
||||
Datasets []ZFSDataset `json:"datasets"`
|
||||
ARC ZFSARC `json:"arc"`
|
||||
}
|
||||
|
||||
type zfsCollector struct {
|
||||
mu sync.Mutex
|
||||
cached ZFSMetrics
|
||||
updatedAt time.Time
|
||||
previous map[string]ZFSPoolMetrics
|
||||
}
|
||||
|
||||
func (c *zfsCollector) collect() ZFSMetrics {
|
||||
@@ -40,7 +80,26 @@ func (c *zfsCollector) collect() ZFSMetrics {
|
||||
if time.Since(c.updatedAt) < 30*time.Second {
|
||||
return c.cached
|
||||
}
|
||||
c.cached = readZFSPools()
|
||||
current := readZFSPools()
|
||||
if c.previous == nil {
|
||||
c.previous = map[string]ZFSPoolMetrics{}
|
||||
}
|
||||
for i := range current.Pools {
|
||||
pool := ¤t.Pools[i]
|
||||
if old, ok := c.previous[pool.Name]; ok {
|
||||
checks := []struct {
|
||||
label string
|
||||
before, now uint64
|
||||
}{{"read", old.ReadErrors, pool.ReadErrors}, {"write", old.WriteErrors, pool.WriteErrors}, {"checksum", old.ChecksumErrors, pool.ChecksumErrors}}
|
||||
for _, check := range checks {
|
||||
if check.now > check.before {
|
||||
pool.ErrorChanges = append(pool.ErrorChanges, check.label+": "+strconv.FormatUint(check.before, 10)+" → "+strconv.FormatUint(check.now, 10))
|
||||
}
|
||||
}
|
||||
}
|
||||
c.previous[pool.Name] = *pool
|
||||
}
|
||||
c.cached = current
|
||||
c.updatedAt = time.Now()
|
||||
return c.cached
|
||||
}
|
||||
@@ -67,15 +126,35 @@ func readZFSPools() ZFSMetrics {
|
||||
pool.FreeBytes, _ = strconv.ParseUint(fields[3], 10, 64)
|
||||
pool.Fragmentation = parsePercent(fields[4])
|
||||
pool.CapacityPercent = parsePercent(fields[5])
|
||||
pool.Scan, pool.Errors = readZFSStatus(pool.Name)
|
||||
pool.Scan, pool.Errors, pool.VDevs = readZFSStatus(pool.Name)
|
||||
for _, vdev := range pool.VDevs {
|
||||
pool.ReadErrors += vdev.Read
|
||||
pool.WriteErrors += vdev.Write
|
||||
pool.ChecksumErrors += vdev.Checksum
|
||||
if vdev.State != "ONLINE" {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, vdev.Class+" "+vdev.Name+": "+vdev.State)
|
||||
}
|
||||
}
|
||||
pool.ScrubStatus, pool.LastScrubAt, pool.ScrubDuration, pool.ScrubTooOld = parseScrub(pool.Scan)
|
||||
if pool.Health != "ONLINE" {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, "Состояние пула: "+pool.Health)
|
||||
}
|
||||
if pool.Errors != "" && !strings.Contains(pool.Errors, "No known data errors") {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, pool.Errors)
|
||||
}
|
||||
if pool.ReadErrors+pool.WriteErrors+pool.ChecksumErrors > 0 {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, "Ошибки vdev: read="+strconv.FormatUint(pool.ReadErrors, 10)+", write="+strconv.FormatUint(pool.WriteErrors, 10)+", checksum="+strconv.FormatUint(pool.ChecksumErrors, 10))
|
||||
}
|
||||
if pool.ScrubTooOld {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, "Scrub не выполнялся более 35 дней")
|
||||
}
|
||||
if pool.CapacityPercent >= 80 {
|
||||
pool.AttentionReasons = append(pool.AttentionReasons, "Мало свободного места для безопасной CoW-работы ZFS")
|
||||
}
|
||||
result.Pools = append(result.Pools, pool)
|
||||
}
|
||||
result.Datasets = readZFSDatasets()
|
||||
result.ARC = readZFSARC()
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -84,11 +163,13 @@ func parsePercent(value string) float64 {
|
||||
return parsed
|
||||
}
|
||||
|
||||
func readZFSStatus(pool string) (string, string) {
|
||||
func readZFSStatus(pool string) (string, string, []ZFSVDev) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
output, _ := exec.CommandContext(ctx, "zpool", "status", pool).Output()
|
||||
var scan, errors string
|
||||
var scan, errors, currentClass string
|
||||
var vdevs []ZFSVDev
|
||||
inConfig := false
|
||||
for _, line := range strings.Split(string(output), "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "scan:") {
|
||||
@@ -97,6 +178,98 @@ func readZFSStatus(pool string) (string, string) {
|
||||
if strings.HasPrefix(trimmed, "errors:") {
|
||||
errors = strings.TrimSpace(strings.TrimPrefix(trimmed, "errors:"))
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "config:") {
|
||||
inConfig = true
|
||||
continue
|
||||
}
|
||||
if !inConfig || trimmed == "" || strings.HasPrefix(trimmed, "NAME") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(trimmed)
|
||||
if len(fields) == 1 && (fields[0] == "logs" || fields[0] == "cache" || fields[0] == "spares" || fields[0] == "special") {
|
||||
currentClass = fields[0]
|
||||
continue
|
||||
}
|
||||
if len(fields) >= 5 && (fields[1] == "ONLINE" || fields[1] == "DEGRADED" || fields[1] == "OFFLINE" || fields[1] == "UNAVAIL" || fields[1] == "FAULTED" || fields[1] == "REMOVED") {
|
||||
read, _ := strconv.ParseUint(fields[2], 10, 64)
|
||||
write, _ := strconv.ParseUint(fields[3], 10, 64)
|
||||
checksum, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
class := currentClass
|
||||
if class == "" {
|
||||
class = "data"
|
||||
}
|
||||
vdevs = append(vdevs, ZFSVDev{Name: fields[0], Class: class, State: fields[1], Read: read, Write: write, Checksum: checksum})
|
||||
}
|
||||
}
|
||||
return scan, errors
|
||||
return scan, errors, vdevs
|
||||
}
|
||||
|
||||
var scrubDatePattern = regexp.MustCompile(`(?:on|since) (\w{3} \w{3}\s+\d+ \d\d:\d\d:\d\d \d{4})`)
|
||||
|
||||
func parseScrub(scan string) (status, lastAt, duration string, tooOld bool) {
|
||||
lower := strings.ToLower(scan)
|
||||
status = scan
|
||||
if strings.Contains(lower, "in progress") {
|
||||
status = "Выполняется"
|
||||
} else if strings.Contains(lower, "repaired") {
|
||||
status = "Завершён"
|
||||
} else if strings.Contains(lower, "none requested") || scan == "" {
|
||||
status = "Никогда не запускался"
|
||||
return status, "", "", true
|
||||
}
|
||||
if match := scrubDatePattern.FindStringSubmatch(scan); len(match) > 1 {
|
||||
if t, err := time.Parse("Mon Jan 2 15:04:05 2006", match[1]); err == nil {
|
||||
lastAt = t.Format(time.RFC3339)
|
||||
tooOld = time.Since(t) > 35*24*time.Hour
|
||||
}
|
||||
}
|
||||
if parts := strings.Split(scan, " in "); len(parts) > 1 {
|
||||
duration = strings.Fields(parts[1])[0]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func readZFSDatasets() []ZFSDataset {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-p", "-o", "name,used,avail,refer,mountpoint").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var result []ZFSDataset
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
used, _ := strconv.ParseUint(f[1], 10, 64)
|
||||
avail, _ := strconv.ParseUint(f[2], 10, 64)
|
||||
refer, _ := strconv.ParseUint(f[3], 10, 64)
|
||||
pct := 0.0
|
||||
if used+avail > 0 {
|
||||
pct = float64(used) / float64(used+avail) * 100
|
||||
}
|
||||
result = append(result, ZFSDataset{Name: f[0], Pool: strings.Split(f[0], "/")[0], UsedBytes: used, AvailableBytes: avail, ReferencedBytes: refer, Mountpoint: strings.Join(f[4:], " "), CapacityPercent: pct})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readZFSARC() ZFSARC {
|
||||
data, err := os.ReadFile("/proc/spl/kstat/zfs/arcstats")
|
||||
if err != nil {
|
||||
return ZFSARC{}
|
||||
}
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 3 {
|
||||
values[f[0]], _ = strconv.ParseUint(f[2], 10, 64)
|
||||
}
|
||||
}
|
||||
hits, misses := values["hits"], values["misses"]
|
||||
ratio := 0.0
|
||||
if hits+misses > 0 {
|
||||
ratio = float64(hits) / float64(hits+misses) * 100
|
||||
}
|
||||
return ZFSARC{Available: true, SizeBytes: values["size"], TargetBytes: values["c"], HitRatio: ratio, Hits: hits, Misses: misses}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user