package main import ( "database/sql" "encoding/json" "fmt" "os" "path/filepath" "strings" "sync" "time" _ "modernc.org/sqlite" ) const historyRetention = 7 * 24 * time.Hour type AlertThresholds struct { CPUWarning float64 `json:"cpuWarning"` CPUCritical float64 `json:"cpuCritical"` CPUTempWarning float64 `json:"cpuTempWarning"` CPUTempCritical float64 `json:"cpuTempCritical"` MemoryWarning float64 `json:"memoryWarning"` MemoryCritical float64 `json:"memoryCritical"` SwapWarning float64 `json:"swapWarning"` StorageWarning float64 `json:"storageWarning"` StorageCritical float64 `json:"storageCritical"` ZFSWarning float64 `json:"zfsWarning"` ZFSCritical float64 `json:"zfsCritical"` DiskTempWarning float64 `json:"diskTempWarning"` NVMeTempWarning float64 `json:"nvmeTempWarning"` UPSChargeCritical float64 `json:"upsChargeCritical"` BackupMaxAgeHours float64 `json:"backupMaxAgeHours"` DiskHDDUtil float64 `json:"diskHddUtil"` DiskHDDLatency float64 `json:"diskHddLatency"` DiskHDDQueue float64 `json:"diskHddQueue"` DiskSSDUtil float64 `json:"diskSsdUtil"` DiskSSDLatency float64 `json:"diskSsdLatency"` DiskSSDQueue float64 `json:"diskSsdQueue"` DiskNVMeUtil float64 `json:"diskNvmeUtil"` DiskNVMeLatency float64 `json:"diskNvmeLatency"` DiskNVMeQueue float64 `json:"diskNvmeQueue"` DiskIODuration float64 `json:"diskIoDuration"` DiskIOExcluded string `json:"diskIoExcluded"` } func defaultThresholds() AlertThresholds { return AlertThresholds{ CPUWarning: 85, CPUCritical: 95, CPUTempWarning: 80, CPUTempCritical: 90, MemoryWarning: 85, MemoryCritical: 95, SwapWarning: 50, StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90, DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30, BackupMaxAgeHours: 48, DiskHDDUtil: 90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60, } } type HistoryPoint struct { Timestamp int64 `json:"timestamp"` CPUUsage float64 `json:"cpuUsage"` CPUTemperature float64 `json:"cpuTemperature"` MemoryUsage float64 `json:"memoryUsage"` SwapUsage float64 `json:"swapUsage"` RootUsage float64 `json:"rootUsage"` NetworkReceive float64 `json:"networkReceive"` NetworkTransmit float64 `json:"networkTransmit"` UPSCharge *float64 `json:"upsCharge"` UPSLoad *float64 `json:"upsLoad"` ZFSUsage float64 `json:"zfsUsage"` GuestsCPU float64 `json:"guestsCpu"` GuestsMemory float64 `json:"guestsMemory"` } type Store struct { db *sql.DB path string healthMu sync.RWMutex lastHistoryAt time.Time lastHistoryError string } type EmailSettings struct { Enabled bool `json:"enabled"` Host string `json:"host"` Port int `json:"port"` Username string `json:"username"` Password string `json:"password,omitempty"` HasPassword bool `json:"hasPassword"` SkipTLSVerify bool `json:"skipTlsVerify"` From string `json:"from"` To string `json:"to"` IntervalMinutes int `json:"intervalMinutes"` Severities []string `json:"severities"` Sources []string `json:"sources"` } func openStore(path string) (*Store, error) { if path != ":memory:" { if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, err } } db, err := sql.Open("sqlite", path) if err != nil { return nil, err } db.SetMaxOpenConns(1) statements := []string{ trafficSchema, `PRAGMA journal_mode=WAL`, `PRAGMA busy_timeout=5000`, `CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)`, `CREATE TABLE IF NOT EXISTS history ( ts INTEGER PRIMARY KEY, cpu_usage REAL NOT NULL, cpu_temp REAL NOT NULL, memory_usage REAL NOT NULL, swap_usage REAL NOT NULL, root_usage REAL NOT NULL, network_rx REAL NOT NULL, network_tx REAL NOT NULL, ups_charge REAL, ups_load REAL, zfs_usage REAL NOT NULL, guests_cpu REAL NOT NULL, guests_memory REAL NOT NULL )`, `CREATE INDEX IF NOT EXISTS history_ts_idx ON history(ts)`, `CREATE TABLE IF NOT EXISTS alert_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, alert_key TEXT NOT NULL, severity TEXT NOT NULL, source TEXT NOT NULL, title TEXT NOT NULL, message TEXT NOT NULL, first_seen INTEGER NOT NULL, last_seen INTEGER NOT NULL, resolved_at INTEGER, acknowledged_at INTEGER, email_sent_at INTEGER )`, `CREATE INDEX IF NOT EXISTS alert_events_active_idx ON alert_events(alert_key,resolved_at)`, `CREATE TABLE IF NOT EXISTS services ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, url TEXT NOT NULL, local_address TEXT NOT NULL DEFAULT '', check_type TEXT NOT NULL DEFAULT 'http', expected_code INTEGER NOT NULL DEFAULT 200, expected_text TEXT NOT NULL DEFAULT '', timeout_seconds INTEGER NOT NULL DEFAULT 5, enabled INTEGER NOT NULL DEFAULT 1 )`, `CREATE TABLE IF NOT EXISTS service_checks ( id INTEGER PRIMARY KEY AUTOINCREMENT, service_id INTEGER NOT NULL, ts INTEGER NOT NULL, up INTEGER NOT NULL, latency_ms REAL NOT NULL, status_code INTEGER NOT NULL DEFAULT 0, error TEXT NOT NULL DEFAULT '', tls_expiry INTEGER, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE )`, `CREATE INDEX IF NOT EXISTS service_checks_service_ts_idx ON service_checks(service_id,ts)`, `CREATE TABLE IF NOT EXISTS entity_history (kind TEXT NOT NULL,name TEXT NOT NULL,ts INTEGER NOT NULL,value1 REAL,value2 REAL,PRIMARY KEY(kind,name,ts))`, `CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`, `CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`, `CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`, `CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`, `CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`, `CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`, } for _, statement := range statements { if _, err := db.Exec(statement); err != nil { db.Close() return nil, err } } // Older databases get this column without requiring a separate migration tool. _, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`) _, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN planned INTEGER NOT NULL DEFAULT 0`) _, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN maintenance_reason TEXT NOT NULL DEFAULT ''`) _, _ = db.Exec(`DELETE FROM alert_events WHERE alert_key='monitor-slow'`) _, _ = db.Exec(`ALTER TABLE service_checks ADD COLUMN endpoint TEXT NOT NULL DEFAULT 'public'`) _, _ = db.Exec(`PRAGMA foreign_keys=ON`) return &Store{db: db, path: path}, nil } type EntityHistoryPoint struct { Timestamp int64 `json:"timestamp"` Value1 *float64 `json:"value1,omitempty"` Value2 *float64 `json:"value2,omitempty"` } type EntityHistorySeries struct { Name string `json:"name"` Points []EntityHistoryPoint `json:"points"` } func (s *Store) AddEntityHistory(kind, name string, ts int64, v1, v2 *float64) { _, _ = s.db.Exec(`INSERT OR REPLACE INTO entity_history(kind,name,ts,value1,value2) VALUES(?,?,?,?,?)`, kind, name, ts, v1, v2) } func (s *Store) EntityHistory(kind string, hours int) ([]EntityHistorySeries, error) { if hours < 1 { hours = 24 } if hours > 168 { hours = 168 } rows, err := s.db.Query(`SELECT name,ts,value1,value2 FROM entity_history WHERE kind=? AND ts>=? ORDER BY name,ts`, kind, time.Now().Add(-time.Duration(hours)*time.Hour).Unix()) if err != nil { return nil, err } defer rows.Close() var out []EntityHistorySeries index := map[string]int{} for rows.Next() { var name string var p EntityHistoryPoint var v1, v2 sql.NullFloat64 if err = rows.Scan(&name, &p.Timestamp, &v1, &v2); err != nil { return nil, err } if v1.Valid { p.Value1 = &v1.Float64 } if v2.Valid { p.Value2 = &v2.Float64 } i, ok := index[name] if !ok { i = len(out) index[name] = i out = append(out, EntityHistorySeries{Name: name}) } out[i].Points = append(out[i].Points, p) } return out, rows.Err() } func (s *Store) PruneEntityHistory() { _, _ = s.db.Exec(`DELETE FROM entity_history WHERE ts 65535 || settings.IntervalMinutes < 5 || settings.IntervalMinutes > 10080 { return fmt.Errorf("проверьте порт и интервал (от 5 минут до 7 дней)") } if settings.Enabled && (settings.Host == "" || settings.From == "" || settings.To == "") { return fmt.Errorf("для отправки нужны SMTP-сервер, отправитель и получатель") } if strings.ContainsAny(settings.Host+settings.From+settings.To+settings.Username, "\r\n") { return fmt.Errorf("поля почты содержат недопустимые символы") } if settings.Password == "" { settings.Password = s.EmailSettings(true).Password } settings.HasPassword = settings.Password != "" data, _ := json.Marshal(settings) _, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('email_notifications',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data)) return err } func (s *Store) LastEmailAt() time.Time { var value string if s.db.QueryRow(`SELECT value FROM settings WHERE key='email_last_sent'`).Scan(&value) != nil { return time.Time{} } result, _ := time.Parse(time.RFC3339, value) return result } func (s *Store) SetLastEmailAt(value time.Time) { _, _ = s.db.Exec(`INSERT INTO settings(key,value) VALUES('email_last_sent',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, value.Format(time.RFC3339)) } func validateThresholds(t AlertThresholds) error { percentages := []float64{t.CPUWarning, t.CPUCritical, t.MemoryWarning, t.MemoryCritical, t.SwapWarning, t.StorageWarning, t.StorageCritical, t.ZFSWarning, t.ZFSCritical, t.UPSChargeCritical} for _, value := range percentages { if value < 0 || value > 100 { return fmt.Errorf("процентные пороги должны быть от 0 до 100") } } temperatures := []float64{t.CPUTempWarning, t.CPUTempCritical, t.DiskTempWarning, t.NVMeTempWarning} for _, value := range temperatures { if value < 0 || value > 150 { return fmt.Errorf("температурные пороги должны быть от 0 до 150") } } if t.CPUWarning >= t.CPUCritical || t.CPUTempWarning >= t.CPUTempCritical || t.MemoryWarning >= t.MemoryCritical || t.StorageWarning >= t.StorageCritical || t.ZFSWarning >= t.ZFSCritical { return fmt.Errorf("порог предупреждения должен быть ниже критического") } if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 { return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов") } for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} { if value < 1 || value > 100 { return fmt.Errorf("пороги utilization должны быть от 1 до 100") } } for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} { if value < 0 { return fmt.Errorf("пороги latency и очереди не могут быть отрицательными") } } if t.DiskIODuration < 5 || t.DiskIODuration > 3600 { return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд") } return nil } func (s *Store) AddHistory(point HistoryPoint) error { _, err := s.db.Exec(`INSERT OR REPLACE INTO history(ts,cpu_usage,cpu_temp,memory_usage,swap_usage,root_usage,network_rx,network_tx,ups_charge,ups_load,zfs_usage,guests_cpu,guests_memory) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, point.Timestamp, point.CPUUsage, point.CPUTemperature, point.MemoryUsage, point.SwapUsage, point.RootUsage, point.NetworkReceive, point.NetworkTransmit, point.UPSCharge, point.UPSLoad, point.ZFSUsage, point.GuestsCPU, point.GuestsMemory) if err != nil { s.setHistoryHealth(time.Time{}, err) return err } _, err = s.db.Exec(`DELETE FROM history WHERE ts < ?`, time.Now().Add(-historyRetention).Unix()) s.setHistoryHealth(time.Now(), err) return err } func (s *Store) setHistoryHealth(at time.Time, err error) { s.healthMu.Lock() defer s.healthMu.Unlock() if !at.IsZero() { s.lastHistoryAt = at } if err != nil { s.lastHistoryError = err.Error() } else { s.lastHistoryError = "" } } func (s *Store) historyHealth() (time.Time, string) { s.healthMu.RLock() defer s.healthMu.RUnlock() return s.lastHistoryAt, s.lastHistoryError } func (s *Store) loadJSONSetting(key string, target any) bool { var value string if s.db.QueryRow(`SELECT value FROM settings WHERE key=?`, key).Scan(&value) != nil { return false } return json.Unmarshal([]byte(value), target) == nil } func (s *Store) saveJSONSetting(key string, value any) error { data, err := json.Marshal(value) if err != nil { return err } _, err = s.db.Exec(`INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, string(data)) return err } func (s *Store) History(hours int) ([]HistoryPoint, error) { if hours < 1 { hours = 24 } if hours > 168 { hours = 168 } bucket := int64(60) if hours > 48 { bucket = 600 } rows, err := s.db.Query(`SELECT (ts/?)*?, AVG(cpu_usage),AVG(cpu_temp),AVG(memory_usage),AVG(swap_usage),AVG(root_usage),AVG(network_rx),AVG(network_tx),AVG(ups_charge),AVG(ups_load),AVG(zfs_usage),AVG(guests_cpu),AVG(guests_memory) FROM history WHERE ts>=? GROUP BY (ts/?) ORDER BY ts`, bucket, bucket, time.Now().Add(-time.Duration(hours)*time.Hour).Unix(), bucket) if err != nil { return nil, err } defer rows.Close() var points []HistoryPoint for rows.Next() { var point HistoryPoint var upsCharge, upsLoad sql.NullFloat64 if err := rows.Scan(&point.Timestamp, &point.CPUUsage, &point.CPUTemperature, &point.MemoryUsage, &point.SwapUsage, &point.RootUsage, &point.NetworkReceive, &point.NetworkTransmit, &upsCharge, &upsLoad, &point.ZFSUsage, &point.GuestsCPU, &point.GuestsMemory); err != nil { return nil, err } if upsCharge.Valid { point.UPSCharge = &upsCharge.Float64 } if upsLoad.Valid { point.UPSLoad = &upsLoad.Float64 } points = append(points, point) } return points, rows.Err() }