Expand guest metrics and add Proxmox maintenance overview
This commit is contained in:
Maxim
264
maintenance.go
Normal file
264
maintenance.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MaintenanceMetrics struct {
|
||||
Available bool `json:"available"`
|
||||
ProxmoxVersion string `json:"proxmoxVersion"`
|
||||
KernelVersion string `json:"kernelVersion"`
|
||||
LatestInstalledKernel string `json:"latestInstalledKernel"`
|
||||
NewKernelAvailable bool `json:"newKernelAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
AvailableUpdates int `json:"availableUpdates"`
|
||||
LastSystemUpdate time.Time `json:"lastSystemUpdate"`
|
||||
NextBackup string `json:"nextBackup"`
|
||||
NextScrub string `json:"nextScrub"`
|
||||
CheckedAt time.Time `json:"checkedAt"`
|
||||
}
|
||||
|
||||
type maintenanceCollector struct {
|
||||
mu sync.Mutex
|
||||
cached MaintenanceMetrics
|
||||
updatedAt time.Time
|
||||
refreshing bool
|
||||
}
|
||||
|
||||
func (c *maintenanceCollector) collect() MaintenanceMetrics {
|
||||
c.mu.Lock()
|
||||
if time.Since(c.updatedAt) >= 15*time.Minute && !c.refreshing {
|
||||
c.refreshing = true
|
||||
go func() {
|
||||
metrics := readMaintenance()
|
||||
c.mu.Lock()
|
||||
c.cached = metrics
|
||||
c.updatedAt = time.Now()
|
||||
c.refreshing = false
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
result := c.cached
|
||||
c.mu.Unlock()
|
||||
return result
|
||||
}
|
||||
|
||||
func commandOutput(timeout time.Duration, name string, args ...string) string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, name, args...)
|
||||
command.Env = append(os.Environ(), "LC_ALL=C")
|
||||
output, err := command.Output()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(output))
|
||||
}
|
||||
|
||||
func readMaintenance() MaintenanceMetrics {
|
||||
result := MaintenanceMetrics{Available: true, CheckedAt: time.Now()}
|
||||
result.ProxmoxVersion = firstLine(commandOutput(5*time.Second, "pveversion"))
|
||||
result.KernelVersion = commandOutput(3*time.Second, "uname", "-r")
|
||||
result.RebootRequired = fileExists("/var/run/reboot-required")
|
||||
apt := commandOutput(45*time.Second, "apt-get", "-s", "-o", "Debug::NoLocking=1", "upgrade")
|
||||
for _, line := range strings.Split(apt, "\n") {
|
||||
if strings.HasPrefix(line, "Inst ") {
|
||||
result.AvailableUpdates++
|
||||
}
|
||||
}
|
||||
result.LastSystemUpdate = lastAPTUpdate("/var/log/apt/history.log")
|
||||
installed := commandOutput(8*time.Second, "dpkg-query", "-W", "-f=${Package} ${Version}\\n", "pve-kernel-*", "proxmox-kernel-*")
|
||||
result.LatestInstalledKernel = newestKernel(installed)
|
||||
result.NewKernelAvailable = result.LatestInstalledKernel != "" && !strings.Contains(result.KernelVersion, result.LatestInstalledKernel)
|
||||
result.NextBackup = nextBackupSchedule()
|
||||
result.NextScrub = nextScrubTimer(commandOutput(8*time.Second, "systemctl", "list-timers", "--all", "--no-legend", "--plain"))
|
||||
if result.NextScrub == "" {
|
||||
result.NextScrub = nextZFSScrubCron("/etc/cron.d/zfsutils-linux", time.Now())
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func firstLine(value string) string {
|
||||
if i := strings.IndexByte(value, '\n'); i >= 0 {
|
||||
return value[:i]
|
||||
}
|
||||
return value
|
||||
}
|
||||
func fileExists(path string) bool { _, err := os.Stat(path); return err == nil }
|
||||
|
||||
func lastAPTUpdate(path string) time.Time {
|
||||
file, err := os.Open(path)
|
||||
if err == nil {
|
||||
defer file.Close()
|
||||
if result := scanAPTUpdate(file); !result.IsZero() {
|
||||
return result
|
||||
}
|
||||
}
|
||||
files, _ := filepath.Glob(path + ".*.gz")
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
a, _ := os.Stat(files[i])
|
||||
b, _ := os.Stat(files[j])
|
||||
return a.ModTime().After(b.ModTime())
|
||||
})
|
||||
for _, name := range files {
|
||||
compressed, err := os.Open(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reader, err := gzip.NewReader(compressed)
|
||||
if err != nil {
|
||||
compressed.Close()
|
||||
continue
|
||||
}
|
||||
result := scanAPTUpdate(reader)
|
||||
reader.Close()
|
||||
compressed.Close()
|
||||
if !result.IsZero() {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
func scanAPTUpdate(reader io.Reader) time.Time {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
var result time.Time
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if strings.HasPrefix(line, "End-Date:") {
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, "End-Date:"))
|
||||
if parsed, err := time.ParseInLocation("2006-01-02 15:04:05", value, time.Local); err == nil {
|
||||
result = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
var versionNumbers = regexp.MustCompile(`\d+`)
|
||||
|
||||
func newestKernel(output string) string {
|
||||
var versions []string
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 1 && !strings.Contains(fields[0], "helper") {
|
||||
versions = append(versions, fields[1])
|
||||
}
|
||||
}
|
||||
sort.Slice(versions, func(i, j int) bool { return compareNumericVersion(versions[i], versions[j]) > 0 })
|
||||
if len(versions) > 0 {
|
||||
return versions[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func compareNumericVersion(a, b string) int {
|
||||
aa, bb := versionNumbers.FindAllString(a, -1), versionNumbers.FindAllString(b, -1)
|
||||
for i := 0; i < len(aa) || i < len(bb); i++ {
|
||||
av, bv := "0", "0"
|
||||
if i < len(aa) {
|
||||
av = aa[i]
|
||||
}
|
||||
if i < len(bb) {
|
||||
bv = bb[i]
|
||||
}
|
||||
if len(av) != len(bv) {
|
||||
if len(av) > len(bv) {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
if av != bv {
|
||||
if av > bv {
|
||||
return 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func nextBackupSchedule() string {
|
||||
data := commandOutput(8*time.Second, "pvesh", "get", "/cluster/backup", "--output-format", "json")
|
||||
if data == "" {
|
||||
return ""
|
||||
}
|
||||
var jobs []struct {
|
||||
Enabled *int `json:"enabled"`
|
||||
Schedule string `json:"schedule"`
|
||||
}
|
||||
if json.Unmarshal([]byte(data), &jobs) != nil {
|
||||
return ""
|
||||
}
|
||||
var next []string
|
||||
for _, job := range jobs {
|
||||
if (job.Enabled != nil && *job.Enabled == 0) || job.Schedule == "" {
|
||||
continue
|
||||
}
|
||||
calendar := commandOutput(5*time.Second, "systemd-analyze", "calendar", "--iterations=1", job.Schedule)
|
||||
for _, line := range strings.Split(calendar, "\n") {
|
||||
if strings.Contains(line, "Next elapse:") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
next = append(next, strings.TrimSpace(strings.TrimPrefix(trimmed, "Next elapse:")))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(next)
|
||||
if len(next) > 0 {
|
||||
return next[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func nextScrubTimer(output string) string {
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
if strings.Contains(strings.ToLower(line), "zfs") && strings.Contains(strings.ToLower(line), "scrub") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 4 {
|
||||
return strings.Join(fields[:4], " ")
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func nextZFSScrubCron(path string, now time.Time) string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if !strings.Contains(line, "/usr/lib/zfs-linux/scrub") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
return ""
|
||||
}
|
||||
minute, _ := strconv.Atoi(fields[0])
|
||||
hour, _ := strconv.Atoi(fields[1])
|
||||
rangeParts := strings.SplitN(fields[2], "-", 2)
|
||||
if len(rangeParts) != 2 {
|
||||
return ""
|
||||
}
|
||||
from, _ := strconv.Atoi(rangeParts[0])
|
||||
to, _ := strconv.Atoi(rangeParts[1])
|
||||
for offset := 0; offset < 70; offset++ {
|
||||
day := time.Date(now.Year(), now.Month(), now.Day()+offset, hour, minute, 0, 0, now.Location())
|
||||
if day.After(now) && day.Day() >= from && day.Day() <= to && day.Weekday() == time.Sunday {
|
||||
return day.Format("Mon 2006-01-02 15:04 MST")
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user