diff --git a/reliability.go b/reliability.go index 4fc87cc..d0bd89d 100644 --- a/reliability.go +++ b/reliability.go @@ -63,6 +63,24 @@ type monitorTracker struct { duration time.Duration } +func runCollectionWatchdog(monitor *monitorTracker) { + started := time.Now() + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for range ticker.C { + monitor.mu.RLock() + last := monitor.lastSuccess + monitor.mu.RUnlock() + if time.Since(started) < 2*time.Minute { + continue + } + if last.IsZero() || time.Since(last) > 90*time.Second { + log.Printf("Встроенный watchdog: сбор метрик не завершался более 90 секунд; перезапускаем процесс") + os.Exit(1) + } + } +} + func (m *monitorTracker) record(start time.Time, err error) { m.mu.Lock() defer m.mu.Unlock() diff --git a/web.go b/web.go index 6baa4df..442085d 100644 --- a/web.go +++ b/web.go @@ -122,6 +122,7 @@ func routes(collector *cpuCollector, store *Store) http.Handler { systemHealth := newSystemHealthCollector() activity := newActivityCollector() monitor := &monitorTracker{} + go runCollectionWatchdog(monitor) go runUPSShutdownController(store, ups, guests) collect := func() (dashboardMetrics, error) { started := time.Now()