245 lines
7.6 KiB
Go
245 lines
7.6 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log"
|
||
"net"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"syscall"
|
||
"time"
|
||
)
|
||
|
||
func notifySystemd(message string) bool {
|
||
address := os.Getenv("NOTIFY_SOCKET")
|
||
if address == "" {
|
||
return false
|
||
}
|
||
if strings.HasPrefix(address, "@") {
|
||
address = "\x00" + address[1:]
|
||
}
|
||
connection, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: address, Net: "unixgram"})
|
||
if err != nil {
|
||
return false
|
||
}
|
||
defer connection.Close()
|
||
_, err = connection.Write([]byte(message))
|
||
return err == nil
|
||
}
|
||
func runSystemdWatchdog() {
|
||
if !notifySystemd("READY=1\nSTATUS=Dashboard collects Proxmox metrics") {
|
||
return
|
||
}
|
||
go func() {
|
||
ticker := time.NewTicker(10 * time.Second)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
notifySystemd("WATCHDOG=1")
|
||
}
|
||
}()
|
||
}
|
||
|
||
type MonitorHealth struct {
|
||
LastSuccessfulCollection time.Time `json:"lastSuccessfulCollection"`
|
||
LastCollectionError string `json:"lastCollectionError"`
|
||
CollectionDurationMs int64 `json:"collectionDurationMs"`
|
||
SlowCollector bool `json:"slowCollector"`
|
||
LastHistoryWrite time.Time `json:"lastHistoryWrite"`
|
||
HistoryWriteError string `json:"historyWriteError"`
|
||
DatabaseFreeBytes uint64 `json:"databaseFreeBytes"`
|
||
DatabasePath string `json:"databasePath"`
|
||
WatchdogEnabled bool `json:"watchdogEnabled"`
|
||
}
|
||
|
||
type monitorTracker struct {
|
||
mu sync.RWMutex
|
||
lastSuccess time.Time
|
||
lastError string
|
||
duration time.Duration
|
||
}
|
||
|
||
func runCollectionWatchdog(monitor *monitorTracker) {
|
||
started := time.Now()
|
||
ticker := time.NewTicker(15 * time.Second)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
monitor.mu.RLock()
|
||
last := monitor.lastSuccess
|
||
monitor.mu.RUnlock()
|
||
if time.Since(started) < 2*time.Minute {
|
||
continue
|
||
}
|
||
if last.IsZero() || time.Since(last) > 90*time.Second {
|
||
log.Printf("Встроенный watchdog: сбор метрик не завершался более 90 секунд; перезапускаем процесс")
|
||
os.Exit(1)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (m *monitorTracker) record(start time.Time, err error) {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
m.duration = time.Since(start)
|
||
if err != nil {
|
||
m.lastError = err.Error()
|
||
return
|
||
}
|
||
m.lastSuccess = time.Now()
|
||
m.lastError = ""
|
||
}
|
||
|
||
func (m *monitorTracker) snapshot(store *Store) MonitorHealth {
|
||
m.mu.RLock()
|
||
result := MonitorHealth{LastSuccessfulCollection: m.lastSuccess, LastCollectionError: m.lastError, CollectionDurationMs: m.duration.Milliseconds(), SlowCollector: m.duration > 10*time.Second, DatabasePath: store.path, WatchdogEnabled: os.Getenv("NOTIFY_SOCKET") != ""}
|
||
m.mu.RUnlock()
|
||
result.LastHistoryWrite, result.HistoryWriteError = store.historyHealth()
|
||
path := store.path
|
||
if path == ":memory:" {
|
||
path = "/tmp"
|
||
} else {
|
||
path = filepath.Dir(path)
|
||
}
|
||
var stat syscall.Statfs_t
|
||
if syscall.Statfs(path, &stat) == nil {
|
||
result.DatabaseFreeBytes = stat.Bavail * uint64(stat.Bsize)
|
||
}
|
||
return result
|
||
}
|
||
|
||
type UPSShutdownSettings struct {
|
||
Enabled bool `json:"enabled"`
|
||
ChargePercent float64 `json:"chargePercent"`
|
||
RuntimeSeconds float64 `json:"runtimeSeconds"`
|
||
GraceSeconds int `json:"graceSeconds"`
|
||
GuestOrder []int `json:"guestOrder"`
|
||
}
|
||
|
||
func defaultUPSShutdownSettings() UPSShutdownSettings {
|
||
return UPSShutdownSettings{ChargePercent: 15, RuntimeSeconds: 600, GraceSeconds: 120}
|
||
}
|
||
func (s *Store) UPSShutdownSettings() UPSShutdownSettings {
|
||
value := defaultUPSShutdownSettings()
|
||
s.loadJSONSetting("ups_shutdown", &value)
|
||
return value
|
||
}
|
||
func (s *Store) SaveUPSShutdownSettings(value UPSShutdownSettings) error {
|
||
if value.ChargePercent < 1 || value.ChargePercent > 80 || value.RuntimeSeconds < 60 || value.RuntimeSeconds > 7200 || value.GraceSeconds < 30 || value.GraceSeconds > 1800 {
|
||
return fmt.Errorf("проверьте пороги UPS и задержку")
|
||
}
|
||
return s.saveJSONSetting("ups_shutdown", value)
|
||
}
|
||
|
||
type ShutdownStep struct {
|
||
Order int `json:"order"`
|
||
Target string `json:"target"`
|
||
Action string `json:"action"`
|
||
Detail string `json:"detail"`
|
||
}
|
||
|
||
func buildShutdownDryRun(settings UPSShutdownSettings, guests GuestsMetrics) []ShutdownStep {
|
||
byID := map[int]GuestMetrics{}
|
||
for _, g := range guests.Guests {
|
||
if g.Template == 0 && g.Status == "running" {
|
||
byID[g.VMID] = g
|
||
}
|
||
}
|
||
var ordered []GuestMetrics
|
||
for _, id := range settings.GuestOrder {
|
||
if g, ok := byID[id]; ok {
|
||
ordered = append(ordered, g)
|
||
delete(byID, id)
|
||
}
|
||
}
|
||
var rest []GuestMetrics
|
||
for _, g := range byID {
|
||
rest = append(rest, g)
|
||
}
|
||
sort.Slice(rest, func(i, j int) bool { return rest[i].VMID < rest[j].VMID })
|
||
ordered = append(ordered, rest...)
|
||
steps := []ShutdownStep{{Order: 1, Target: "UPS", Action: "Ожидание порога", Detail: fmt.Sprintf("заряд ≤ %.0f%% или автономность ≤ %.0f сек.", settings.ChargePercent, settings.RuntimeSeconds)}}
|
||
for _, g := range ordered {
|
||
kind := "LXC"
|
||
if g.Type == "qemu" {
|
||
kind = "VM"
|
||
}
|
||
steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: fmt.Sprintf("%s %d · %s", kind, g.VMID, g.Name), Action: "Корректное завершение", Detail: fmt.Sprintf("ожидание до %d сек.", settings.GraceSeconds)})
|
||
}
|
||
steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: "Proxmox host", Action: "Завершение работы", Detail: "только после остановки VM/LXC"})
|
||
return steps
|
||
}
|
||
|
||
func runUPSShutdownController(store *Store, ups *upsCollector, guests *guestCollector) {
|
||
ticker := time.NewTicker(10 * time.Second)
|
||
defer ticker.Stop()
|
||
triggered := false
|
||
for range ticker.C {
|
||
settings := store.UPSShutdownSettings()
|
||
metrics := ups.collect()
|
||
onBattery := strings.Contains(metrics.Status, "OB") || metrics.Status == "ONBATT"
|
||
if !settings.Enabled || !metrics.Available || !onBattery {
|
||
if metrics.Available && !onBattery {
|
||
triggered = false
|
||
}
|
||
continue
|
||
}
|
||
lowCharge := metrics.ChargePercent != nil && *metrics.ChargePercent <= settings.ChargePercent
|
||
lowRuntime := metrics.RuntimeSeconds != nil && *metrics.RuntimeSeconds <= settings.RuntimeSeconds
|
||
if triggered || (!lowCharge && !lowRuntime) {
|
||
continue
|
||
}
|
||
triggered = true
|
||
log.Printf("UPS достиг порога безопасного выключения; запускаем остановку гостей")
|
||
go executeUPSShutdown(settings, guests.collect())
|
||
}
|
||
}
|
||
|
||
func executeUPSShutdown(settings UPSShutdownSettings, guests GuestsMetrics) {
|
||
steps := buildShutdownDryRun(settings, guests)
|
||
_ = steps
|
||
byID := map[int]GuestMetrics{}
|
||
for _, g := range guests.Guests {
|
||
if g.Template == 0 && g.Status == "running" {
|
||
byID[g.VMID] = g
|
||
}
|
||
}
|
||
var order []int
|
||
for _, id := range settings.GuestOrder {
|
||
if _, ok := byID[id]; ok {
|
||
order = append(order, id)
|
||
delete(byID, id)
|
||
}
|
||
}
|
||
var remaining []int
|
||
for id := range byID {
|
||
remaining = append(remaining, id)
|
||
}
|
||
sort.Ints(remaining)
|
||
order = append(order, remaining...)
|
||
for _, id := range order {
|
||
g := GuestMetrics{}
|
||
for _, candidate := range guests.Guests {
|
||
if candidate.VMID == id {
|
||
g = candidate
|
||
break
|
||
}
|
||
}
|
||
command := "pct"
|
||
if g.Type == "qemu" {
|
||
command = "qm"
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(settings.GraceSeconds+30)*time.Second)
|
||
err := exec.CommandContext(ctx, command, "shutdown", fmt.Sprint(id), "--timeout", fmt.Sprint(settings.GraceSeconds)).Run()
|
||
cancel()
|
||
if err != nil {
|
||
log.Printf("UPS shutdown: не удалось остановить VMID %d: %v", id, err)
|
||
}
|
||
}
|
||
log.Printf("UPS shutdown: гости обработаны, выключаем Proxmox host")
|
||
_ = exec.Command("shutdown", "-h", "now").Run()
|
||
}
|