package main import ( "context" "fmt" "log" "net" "os" "os/exec" "path/filepath" "sort" "strings" "sync" "syscall" "time" ) func notifySystemd(message string) bool { address := os.Getenv("NOTIFY_SOCKET") if address == "" { return false } if strings.HasPrefix(address, "@") { address = "\x00" + address[1:] } connection, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: address, Net: "unixgram"}) if err != nil { return false } defer connection.Close() _, err = connection.Write([]byte(message)) return err == nil } func runSystemdWatchdog() { if !notifySystemd("READY=1\nSTATUS=Dashboard collects Proxmox metrics") { return } go func() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for range ticker.C { notifySystemd("WATCHDOG=1") } }() } type MonitorHealth struct { LastSuccessfulCollection time.Time `json:"lastSuccessfulCollection"` LastCollectionError string `json:"lastCollectionError"` CollectionDurationMs int64 `json:"collectionDurationMs"` SlowCollector bool `json:"slowCollector"` LastHistoryWrite time.Time `json:"lastHistoryWrite"` HistoryWriteError string `json:"historyWriteError"` DatabaseFreeBytes uint64 `json:"databaseFreeBytes"` DatabasePath string `json:"databasePath"` WatchdogEnabled bool `json:"watchdogEnabled"` } type monitorTracker struct { mu sync.RWMutex lastSuccess time.Time lastError string duration time.Duration } func runCollectionWatchdog(monitor *monitorTracker) { started := time.Now() ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() for range ticker.C { monitor.mu.RLock() last := monitor.lastSuccess monitor.mu.RUnlock() if time.Since(started) < 2*time.Minute { continue } if last.IsZero() || time.Since(last) > 90*time.Second { log.Printf("Встроенный watchdog: сбор метрик не завершался более 90 секунд; перезапускаем процесс") os.Exit(1) } } } func (m *monitorTracker) record(start time.Time, err error) { m.mu.Lock() defer m.mu.Unlock() m.duration = time.Since(start) if err != nil { m.lastError = err.Error() return } m.lastSuccess = time.Now() m.lastError = "" } func (m *monitorTracker) snapshot(store *Store) MonitorHealth { m.mu.RLock() result := MonitorHealth{LastSuccessfulCollection: m.lastSuccess, LastCollectionError: m.lastError, CollectionDurationMs: m.duration.Milliseconds(), SlowCollector: m.duration > 10*time.Second, DatabasePath: store.path, WatchdogEnabled: os.Getenv("NOTIFY_SOCKET") != ""} m.mu.RUnlock() result.LastHistoryWrite, result.HistoryWriteError = store.historyHealth() path := store.path if path == ":memory:" { path = "/tmp" } else { path = filepath.Dir(path) } var stat syscall.Statfs_t if syscall.Statfs(path, &stat) == nil { result.DatabaseFreeBytes = stat.Bavail * uint64(stat.Bsize) } return result } type UPSShutdownSettings struct { Enabled bool `json:"enabled"` ChargePercent float64 `json:"chargePercent"` RuntimeSeconds float64 `json:"runtimeSeconds"` GraceSeconds int `json:"graceSeconds"` GuestOrder []int `json:"guestOrder"` } func defaultUPSShutdownSettings() UPSShutdownSettings { return UPSShutdownSettings{ChargePercent: 15, RuntimeSeconds: 600, GraceSeconds: 120} } func (s *Store) UPSShutdownSettings() UPSShutdownSettings { value := defaultUPSShutdownSettings() s.loadJSONSetting("ups_shutdown", &value) return value } func (s *Store) SaveUPSShutdownSettings(value UPSShutdownSettings) error { if value.ChargePercent < 1 || value.ChargePercent > 80 || value.RuntimeSeconds < 60 || value.RuntimeSeconds > 7200 || value.GraceSeconds < 30 || value.GraceSeconds > 1800 { return fmt.Errorf("проверьте пороги UPS и задержку") } return s.saveJSONSetting("ups_shutdown", value) } type ShutdownStep struct { Order int `json:"order"` Target string `json:"target"` Action string `json:"action"` Detail string `json:"detail"` } func buildShutdownDryRun(settings UPSShutdownSettings, guests GuestsMetrics) []ShutdownStep { byID := map[int]GuestMetrics{} for _, g := range guests.Guests { if g.Template == 0 && g.Status == "running" { byID[g.VMID] = g } } var ordered []GuestMetrics for _, id := range settings.GuestOrder { if g, ok := byID[id]; ok { ordered = append(ordered, g) delete(byID, id) } } var rest []GuestMetrics for _, g := range byID { rest = append(rest, g) } sort.Slice(rest, func(i, j int) bool { return rest[i].VMID < rest[j].VMID }) ordered = append(ordered, rest...) steps := []ShutdownStep{{Order: 1, Target: "UPS", Action: "Ожидание порога", Detail: fmt.Sprintf("заряд ≤ %.0f%% или автономность ≤ %.0f сек.", settings.ChargePercent, settings.RuntimeSeconds)}} for _, g := range ordered { kind := "LXC" if g.Type == "qemu" { kind = "VM" } steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: fmt.Sprintf("%s %d · %s", kind, g.VMID, g.Name), Action: "Корректное завершение", Detail: fmt.Sprintf("ожидание до %d сек.", settings.GraceSeconds)}) } steps = append(steps, ShutdownStep{Order: len(steps) + 1, Target: "Proxmox host", Action: "Завершение работы", Detail: "только после остановки VM/LXC"}) return steps } func runUPSShutdownController(store *Store, ups *upsCollector, guests *guestCollector) { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() triggered := false for range ticker.C { settings := store.UPSShutdownSettings() metrics := ups.collect() onBattery := strings.Contains(metrics.Status, "OB") || metrics.Status == "ONBATT" if !settings.Enabled || !metrics.Available || !onBattery { if metrics.Available && !onBattery { triggered = false } continue } lowCharge := metrics.ChargePercent != nil && *metrics.ChargePercent <= settings.ChargePercent lowRuntime := metrics.RuntimeSeconds != nil && *metrics.RuntimeSeconds <= settings.RuntimeSeconds if triggered || (!lowCharge && !lowRuntime) { continue } triggered = true log.Printf("UPS достиг порога безопасного выключения; запускаем остановку гостей") go executeUPSShutdown(settings, guests.collect()) } } func executeUPSShutdown(settings UPSShutdownSettings, guests GuestsMetrics) { steps := buildShutdownDryRun(settings, guests) _ = steps byID := map[int]GuestMetrics{} for _, g := range guests.Guests { if g.Template == 0 && g.Status == "running" { byID[g.VMID] = g } } var order []int for _, id := range settings.GuestOrder { if _, ok := byID[id]; ok { order = append(order, id) delete(byID, id) } } var remaining []int for id := range byID { remaining = append(remaining, id) } sort.Ints(remaining) order = append(order, remaining...) for _, id := range order { g := GuestMetrics{} for _, candidate := range guests.Guests { if candidate.VMID == id { g = candidate break } } command := "pct" if g.Type == "qemu" { command = "qm" } ctx, cancel := context.WithTimeout(context.Background(), time.Duration(settings.GraceSeconds+30)*time.Second) err := exec.CommandContext(ctx, command, "shutdown", fmt.Sprint(id), "--timeout", fmt.Sprint(settings.GraceSeconds)).Run() cancel() if err != nil { log.Printf("UPS shutdown: не удалось остановить VMID %d: %v", id, err) } } log.Printf("UPS shutdown: гости обработаны, выключаем Proxmox host") _ = exec.Command("shutdown", "-h", "now").Run() }