276 lines
9.5 KiB
Go
276 lines
9.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type ZFSPoolMetrics struct {
|
|
Name string `json:"name"`
|
|
Health string `json:"health"`
|
|
SizeBytes uint64 `json:"sizeBytes"`
|
|
AllocatedBytes uint64 `json:"allocatedBytes"`
|
|
FreeBytes uint64 `json:"freeBytes"`
|
|
CapacityPercent float64 `json:"capacityPercent"`
|
|
Fragmentation float64 `json:"fragmentationPercent"`
|
|
DedupRatio string `json:"dedupRatio"`
|
|
Scan string `json:"scan"`
|
|
Errors string `json:"errors"`
|
|
AttentionReasons []string `json:"attentionReasons"`
|
|
ReadErrors uint64 `json:"readErrors"`
|
|
WriteErrors uint64 `json:"writeErrors"`
|
|
ChecksumErrors uint64 `json:"checksumErrors"`
|
|
VDevs []ZFSVDev `json:"vdevs"`
|
|
ScrubStatus string `json:"scrubStatus"`
|
|
LastScrubAt string `json:"lastScrubAt"`
|
|
ScrubDuration string `json:"scrubDuration"`
|
|
ScrubTooOld bool `json:"scrubTooOld"`
|
|
ErrorChanges []string `json:"errorChanges"`
|
|
}
|
|
|
|
type ZFSVDev struct {
|
|
Name string `json:"name"`
|
|
Class string `json:"class"`
|
|
State string `json:"state"`
|
|
Read uint64 `json:"read"`
|
|
Write uint64 `json:"write"`
|
|
Checksum uint64 `json:"checksum"`
|
|
}
|
|
type ZFSDataset struct {
|
|
Name string `json:"name"`
|
|
Pool string `json:"pool"`
|
|
UsedBytes uint64 `json:"usedBytes"`
|
|
AvailableBytes uint64 `json:"availableBytes"`
|
|
ReferencedBytes uint64 `json:"referencedBytes"`
|
|
Mountpoint string `json:"mountpoint"`
|
|
CapacityPercent float64 `json:"capacityPercent"`
|
|
}
|
|
type ZFSARC struct {
|
|
Available bool `json:"available"`
|
|
SizeBytes uint64 `json:"sizeBytes"`
|
|
TargetBytes uint64 `json:"targetBytes"`
|
|
HitRatio float64 `json:"hitRatio"`
|
|
Hits uint64 `json:"hits"`
|
|
Misses uint64 `json:"misses"`
|
|
}
|
|
|
|
type ZFSMetrics struct {
|
|
Available bool `json:"available"`
|
|
Pools []ZFSPoolMetrics `json:"pools"`
|
|
Datasets []ZFSDataset `json:"datasets"`
|
|
ARC ZFSARC `json:"arc"`
|
|
}
|
|
|
|
type zfsCollector struct {
|
|
mu sync.Mutex
|
|
cached ZFSMetrics
|
|
updatedAt time.Time
|
|
previous map[string]ZFSPoolMetrics
|
|
}
|
|
|
|
func (c *zfsCollector) collect() ZFSMetrics {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if time.Since(c.updatedAt) < 30*time.Second {
|
|
return c.cached
|
|
}
|
|
current := readZFSPools()
|
|
if c.previous == nil {
|
|
c.previous = map[string]ZFSPoolMetrics{}
|
|
}
|
|
for i := range current.Pools {
|
|
pool := ¤t.Pools[i]
|
|
if old, ok := c.previous[pool.Name]; ok {
|
|
checks := []struct {
|
|
label string
|
|
before, now uint64
|
|
}{{"read", old.ReadErrors, pool.ReadErrors}, {"write", old.WriteErrors, pool.WriteErrors}, {"checksum", old.ChecksumErrors, pool.ChecksumErrors}}
|
|
for _, check := range checks {
|
|
if check.now > check.before {
|
|
pool.ErrorChanges = append(pool.ErrorChanges, check.label+": "+strconv.FormatUint(check.before, 10)+" → "+strconv.FormatUint(check.now, 10))
|
|
}
|
|
}
|
|
}
|
|
c.previous[pool.Name] = *pool
|
|
}
|
|
c.cached = current
|
|
c.updatedAt = time.Now()
|
|
return c.cached
|
|
}
|
|
|
|
func readZFSPools() ZFSMetrics {
|
|
if _, err := exec.LookPath("zpool"); err != nil {
|
|
return ZFSMetrics{}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
|
defer cancel()
|
|
output, err := exec.CommandContext(ctx, "zpool", "list", "-H", "-p", "-o", "name,size,allocated,free,fragmentation,capacity,dedupratio,health").Output()
|
|
result := ZFSMetrics{Available: true}
|
|
if err != nil {
|
|
return result
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 8 {
|
|
continue
|
|
}
|
|
pool := ZFSPoolMetrics{Name: fields[0], DedupRatio: fields[6], Health: fields[7]}
|
|
pool.SizeBytes, _ = strconv.ParseUint(fields[1], 10, 64)
|
|
pool.AllocatedBytes, _ = strconv.ParseUint(fields[2], 10, 64)
|
|
pool.FreeBytes, _ = strconv.ParseUint(fields[3], 10, 64)
|
|
pool.Fragmentation = parsePercent(fields[4])
|
|
pool.CapacityPercent = parsePercent(fields[5])
|
|
pool.Scan, pool.Errors, pool.VDevs = readZFSStatus(pool.Name)
|
|
for _, vdev := range pool.VDevs {
|
|
pool.ReadErrors += vdev.Read
|
|
pool.WriteErrors += vdev.Write
|
|
pool.ChecksumErrors += vdev.Checksum
|
|
if vdev.State != "ONLINE" {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, vdev.Class+" "+vdev.Name+": "+vdev.State)
|
|
}
|
|
}
|
|
pool.ScrubStatus, pool.LastScrubAt, pool.ScrubDuration, pool.ScrubTooOld = parseScrub(pool.Scan)
|
|
if pool.Health != "ONLINE" {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, "Состояние пула: "+pool.Health)
|
|
}
|
|
if pool.Errors != "" && !strings.Contains(pool.Errors, "No known data errors") {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, pool.Errors)
|
|
}
|
|
if pool.ReadErrors+pool.WriteErrors+pool.ChecksumErrors > 0 {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, "Ошибки vdev: read="+strconv.FormatUint(pool.ReadErrors, 10)+", write="+strconv.FormatUint(pool.WriteErrors, 10)+", checksum="+strconv.FormatUint(pool.ChecksumErrors, 10))
|
|
}
|
|
if pool.ScrubTooOld {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, "Scrub не выполнялся более 35 дней")
|
|
}
|
|
if pool.CapacityPercent >= 80 {
|
|
pool.AttentionReasons = append(pool.AttentionReasons, "Мало свободного места для безопасной CoW-работы ZFS")
|
|
}
|
|
result.Pools = append(result.Pools, pool)
|
|
}
|
|
result.Datasets = readZFSDatasets()
|
|
result.ARC = readZFSARC()
|
|
return result
|
|
}
|
|
|
|
func parsePercent(value string) float64 {
|
|
parsed, _ := strconv.ParseFloat(strings.TrimSuffix(value, "%"), 64)
|
|
return parsed
|
|
}
|
|
|
|
func readZFSStatus(pool string) (string, string, []ZFSVDev) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
output, _ := exec.CommandContext(ctx, "zpool", "status", pool).Output()
|
|
var scan, errors, currentClass string
|
|
var vdevs []ZFSVDev
|
|
inConfig := false
|
|
for _, line := range strings.Split(string(output), "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "scan:") {
|
|
scan = strings.TrimSpace(strings.TrimPrefix(trimmed, "scan:"))
|
|
}
|
|
if strings.HasPrefix(trimmed, "errors:") {
|
|
errors = strings.TrimSpace(strings.TrimPrefix(trimmed, "errors:"))
|
|
}
|
|
if strings.HasPrefix(trimmed, "config:") {
|
|
inConfig = true
|
|
continue
|
|
}
|
|
if !inConfig || trimmed == "" || strings.HasPrefix(trimmed, "NAME") {
|
|
continue
|
|
}
|
|
fields := strings.Fields(trimmed)
|
|
if len(fields) == 1 && (fields[0] == "logs" || fields[0] == "cache" || fields[0] == "spares" || fields[0] == "special") {
|
|
currentClass = fields[0]
|
|
continue
|
|
}
|
|
if len(fields) >= 5 && (fields[1] == "ONLINE" || fields[1] == "DEGRADED" || fields[1] == "OFFLINE" || fields[1] == "UNAVAIL" || fields[1] == "FAULTED" || fields[1] == "REMOVED") {
|
|
read, _ := strconv.ParseUint(fields[2], 10, 64)
|
|
write, _ := strconv.ParseUint(fields[3], 10, 64)
|
|
checksum, _ := strconv.ParseUint(fields[4], 10, 64)
|
|
class := currentClass
|
|
if class == "" {
|
|
class = "data"
|
|
}
|
|
vdevs = append(vdevs, ZFSVDev{Name: fields[0], Class: class, State: fields[1], Read: read, Write: write, Checksum: checksum})
|
|
}
|
|
}
|
|
return scan, errors, vdevs
|
|
}
|
|
|
|
var scrubDatePattern = regexp.MustCompile(`(?:on|since) (\w{3} \w{3}\s+\d+ \d\d:\d\d:\d\d \d{4})`)
|
|
|
|
func parseScrub(scan string) (status, lastAt, duration string, tooOld bool) {
|
|
lower := strings.ToLower(scan)
|
|
status = scan
|
|
if strings.Contains(lower, "in progress") {
|
|
status = "Выполняется"
|
|
} else if strings.Contains(lower, "repaired") {
|
|
status = "Завершён"
|
|
} else if strings.Contains(lower, "none requested") || scan == "" {
|
|
status = "Никогда не запускался"
|
|
return status, "", "", true
|
|
}
|
|
if match := scrubDatePattern.FindStringSubmatch(scan); len(match) > 1 {
|
|
if t, err := time.Parse("Mon Jan 2 15:04:05 2006", match[1]); err == nil {
|
|
lastAt = t.Format(time.RFC3339)
|
|
tooOld = time.Since(t) > 35*24*time.Hour
|
|
}
|
|
}
|
|
if parts := strings.Split(scan, " in "); len(parts) > 1 {
|
|
duration = strings.Fields(parts[1])[0]
|
|
}
|
|
return
|
|
}
|
|
|
|
func readZFSDatasets() []ZFSDataset {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-p", "-o", "name,used,avail,refer,mountpoint").Output()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var result []ZFSDataset
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 5 {
|
|
continue
|
|
}
|
|
used, _ := strconv.ParseUint(f[1], 10, 64)
|
|
avail, _ := strconv.ParseUint(f[2], 10, 64)
|
|
refer, _ := strconv.ParseUint(f[3], 10, 64)
|
|
pct := 0.0
|
|
if used+avail > 0 {
|
|
pct = float64(used) / float64(used+avail) * 100
|
|
}
|
|
result = append(result, ZFSDataset{Name: f[0], Pool: strings.Split(f[0], "/")[0], UsedBytes: used, AvailableBytes: avail, ReferencedBytes: refer, Mountpoint: strings.Join(f[4:], " "), CapacityPercent: pct})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func readZFSARC() ZFSARC {
|
|
data, err := os.ReadFile("/proc/spl/kstat/zfs/arcstats")
|
|
if err != nil {
|
|
return ZFSARC{}
|
|
}
|
|
values := map[string]uint64{}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) >= 3 {
|
|
values[f[0]], _ = strconv.ParseUint(f[2], 10, 64)
|
|
}
|
|
}
|
|
hits, misses := values["hits"], values["misses"]
|
|
ratio := 0.0
|
|
if hits+misses > 0 {
|
|
ratio = float64(hits) / float64(hits+misses) * 100
|
|
}
|
|
return ZFSARC{Available: true, SizeBytes: values["size"], TargetBytes: values["c"], HitRatio: ratio, Hits: hits, Misses: misses}
|
|
}
|