Add infrastructure reliability monitoring
This commit is contained in:
Maxim
226
reliability.go
Normal file
226
reliability.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func notifySystemd(message string) bool {
|
||||
address := os.Getenv("NOTIFY_SOCKET")
|
||||
if address == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(address, "@") {
|
||||
address = "\x00" + address[1:]
|
||||
}
|
||||
connection, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: address, Net: "unixgram"})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer connection.Close()
|
||||
_, err = connection.Write([]byte(message))
|
||||
return err == nil
|
||||
}
|
||||
func runSystemdWatchdog() {
|
||||
if !notifySystemd("READY=1\nSTATUS=Dashboard collects Proxmox metrics") {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
notifySystemd("WATCHDOG=1")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
type MonitorHealth struct {
|
||||
LastSuccessfulCollection time.Time `json:"lastSuccessfulCollection"`
|
||||
LastCollectionError string `json:"lastCollectionError"`
|
||||
CollectionDurationMs int64 `json:"collectionDurationMs"`
|
||||
SlowCollector bool `json:"slowCollector"`
|
||||
LastHistoryWrite time.Time `json:"lastHistoryWrite"`
|
||||
HistoryWriteError string `json:"historyWriteError"`
|
||||
DatabaseFreeBytes uint64 `json:"databaseFreeBytes"`
|
||||
DatabasePath string `json:"databasePath"`
|
||||
WatchdogEnabled bool `json:"watchdogEnabled"`
|
||||
}
|
||||
|
||||
type monitorTracker struct {
|
||||
mu sync.RWMutex
|
||||
lastSuccess time.Time
|
||||
lastError string
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
func (m *monitorTracker) record(start time.Time, err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.duration = time.Since(start)
|
||||
if err != nil {
|
||||
m.lastError = err.Error()
|
||||
return
|
||||
}
|
||||
m.lastSuccess = time.Now()
|
||||
m.lastError = ""
|
||||
}
|
||||
|
||||
func (m *monitorTracker) snapshot(store *Store) MonitorHealth {
|
||||
m.mu.RLock()
|
||||
result := MonitorHealth{LastSuccessfulCollection: m.lastSuccess, LastCollectionError: m.lastError, CollectionDurationMs: m.duration.Milliseconds(), SlowCollector: m.duration > 10*time.Second, DatabasePath: store.path, WatchdogEnabled: os.Getenv("NOTIFY_SOCKET") != ""}
|
||||
m.mu.RUnlock()
|
||||
result.LastHistoryWrite, result.HistoryWriteError = store.historyHealth()
|
||||
path := store.path
|
||||
if path == ":memory:" {
|
||||
path = "/tmp"
|
||||
} else {
|
||||
path = filepath.Dir(path)
|
||||
}
|
||||
var stat syscall.Statfs_t
|
||||
if syscall.Statfs(path, &stat) == nil {
|
||||
result.DatabaseFreeBytes = stat.Bavail * uint64(stat.Bsize)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type UPSShutdownSettings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
ChargePercent float64 `json:"chargePercent"`
|
||||
RuntimeSeconds float64 `json:"runtimeSeconds"`
|
||||
GraceSeconds int `json:"graceSeconds"`
|
||||
GuestOrder []int `json:"guestOrder"`
|
||||
}
|
||||
|
||||
func defaultUPSShutdownSettings() UPSShutdownSettings {
|
||||
return UPSShutdownSettings{ChargePercent: 15, RuntimeSeconds: 600, GraceSeconds: 120}
|
||||
}
|
||||
func (s *Store) UPSShutdownSettings() UPSShutdownSettings {
|
||||
value := defaultUPSShutdownSettings()
|
||||
s.loadJSONSetting("ups_shutdown", &value)
|
||||
return value
|
||||
}
|
||||
func (s *Store) SaveUPSShutdownSettings(value UPSShutdownSettings) error {
|
||||
if value.ChargePercent < 1 || value.ChargePercent > 80 || value.RuntimeSeconds < 60 || value.RuntimeSeconds > 7200 || value.GraceSeconds < 30 || value.GraceSeconds > 1800 {
|
||||
return fmt.Errorf("проверьте пороги UPS и задержку")
|
||||
}
|
||||
return s.saveJSONSetting("ups_shutdown", value)
|
||||
}
|
||||
|
||||
type ShutdownStep struct {
|
||||
Order int `json:"order"`
|
||||
Target string `json:"target"`
|
||||
Action string `json:"action"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
func buildShutdownDryRun(settings UPSShutdownSettings, guests GuestsMetrics) []ShutdownStep {
|
||||
byID := map[int]GuestMetrics{}
|
||||
for _, g := range guests.Guests {
|
||||
if g.Template == 0 && g.Status == "running" {
|
||||
byID[g.VMID] = g
|
||||
}
|
||||
}
|
||||
var ordered []GuestMetrics
|
||||
for _, id := range settings.GuestOrder {
|
||||
if g, ok := byID[id]; ok {
|
||||
ordered = append(ordered, g)
|
||||
delete(byID, id)
|
||||
}
|
||||
}
|
||||
var rest []GuestMetrics
|
||||
for _, g := range byID {
|
||||
rest = append(rest, g)
|
||||
}
|
||||
sort.Slice(rest, func(i, j int) bool { return rest[i].VMID < rest[j].VMID })
|
||||
ordered = append(ordered, rest...)
|
||||
steps := []ShutdownStep{{Order: 1, Target: "UPS", Action: "Ожидание порога", Detail: fmt.Sprintf("заряд ≤ %.0f%% или автономность ≤ %.0f сек.", settings.ChargePercent, settings.RuntimeSeconds)}}
|
||||
for _, g := range ordered {
|
||||
kind := "LXC"
|
||||
if g.Type == "qemu" {
|
||||
kind = "VM"
|
||||
}
|
||||
steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: fmt.Sprintf("%s %d · %s", kind, g.VMID, g.Name), Action: "Корректное завершение", Detail: fmt.Sprintf("ожидание до %d сек.", settings.GraceSeconds)})
|
||||
}
|
||||
steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: "Proxmox host", Action: "Завершение работы", Detail: "только после остановки VM/LXC"})
|
||||
return steps
|
||||
}
|
||||
|
||||
func runUPSShutdownController(store *Store, ups *upsCollector, guests *guestCollector) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
triggered := false
|
||||
for range ticker.C {
|
||||
settings := store.UPSShutdownSettings()
|
||||
metrics := ups.collect()
|
||||
onBattery := strings.Contains(metrics.Status, "OB") || metrics.Status == "ONBATT"
|
||||
if !settings.Enabled || !metrics.Available || !onBattery {
|
||||
if metrics.Available && !onBattery {
|
||||
triggered = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
lowCharge := metrics.ChargePercent != nil && *metrics.ChargePercent <= settings.ChargePercent
|
||||
lowRuntime := metrics.RuntimeSeconds != nil && *metrics.RuntimeSeconds <= settings.RuntimeSeconds
|
||||
if triggered || (!lowCharge && !lowRuntime) {
|
||||
continue
|
||||
}
|
||||
triggered = true
|
||||
log.Printf("UPS достиг порога безопасного выключения; запускаем остановку гостей")
|
||||
go executeUPSShutdown(settings, guests.collect())
|
||||
}
|
||||
}
|
||||
|
||||
func executeUPSShutdown(settings UPSShutdownSettings, guests GuestsMetrics) {
|
||||
steps := buildShutdownDryRun(settings, guests)
|
||||
_ = steps
|
||||
byID := map[int]GuestMetrics{}
|
||||
for _, g := range guests.Guests {
|
||||
if g.Template == 0 && g.Status == "running" {
|
||||
byID[g.VMID] = g
|
||||
}
|
||||
}
|
||||
var order []int
|
||||
for _, id := range settings.GuestOrder {
|
||||
if _, ok := byID[id]; ok {
|
||||
order = append(order, id)
|
||||
delete(byID, id)
|
||||
}
|
||||
}
|
||||
var remaining []int
|
||||
for id := range byID {
|
||||
remaining = append(remaining, id)
|
||||
}
|
||||
sort.Ints(remaining)
|
||||
order = append(order, remaining...)
|
||||
for _, id := range order {
|
||||
g := GuestMetrics{}
|
||||
for _, candidate := range guests.Guests {
|
||||
if candidate.VMID == id {
|
||||
g = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
command := "pct"
|
||||
if g.Type == "qemu" {
|
||||
command = "qm"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(settings.GraceSeconds+30)*time.Second)
|
||||
err := exec.CommandContext(ctx, command, "shutdown", fmt.Sprint(id), "--timeout", fmt.Sprint(settings.GraceSeconds)).Run()
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("UPS shutdown: не удалось остановить VMID %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
log.Printf("UPS shutdown: гости обработаны, выключаем Proxmox host")
|
||||
_ = exec.Command("shutdown", "-h", "now").Run()
|
||||
}
|
||||
Reference in New Issue
Block a user