403 lines
17 KiB
Go
403 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
const historyRetention = 7 * 24 * time.Hour
|
|
|
|
type AlertThresholds struct {
|
|
CPUWarning float64 `json:"cpuWarning"`
|
|
CPUCritical float64 `json:"cpuCritical"`
|
|
CPUTempWarning float64 `json:"cpuTempWarning"`
|
|
CPUTempCritical float64 `json:"cpuTempCritical"`
|
|
MemoryWarning float64 `json:"memoryWarning"`
|
|
MemoryCritical float64 `json:"memoryCritical"`
|
|
SwapWarning float64 `json:"swapWarning"`
|
|
StorageWarning float64 `json:"storageWarning"`
|
|
StorageCritical float64 `json:"storageCritical"`
|
|
ZFSWarning float64 `json:"zfsWarning"`
|
|
ZFSCritical float64 `json:"zfsCritical"`
|
|
DiskTempWarning float64 `json:"diskTempWarning"`
|
|
NVMeTempWarning float64 `json:"nvmeTempWarning"`
|
|
UPSChargeCritical float64 `json:"upsChargeCritical"`
|
|
BackupMaxAgeHours float64 `json:"backupMaxAgeHours"`
|
|
DiskHDDUtil float64 `json:"diskHddUtil"`
|
|
DiskHDDLatency float64 `json:"diskHddLatency"`
|
|
DiskHDDQueue float64 `json:"diskHddQueue"`
|
|
DiskSSDUtil float64 `json:"diskSsdUtil"`
|
|
DiskSSDLatency float64 `json:"diskSsdLatency"`
|
|
DiskSSDQueue float64 `json:"diskSsdQueue"`
|
|
DiskNVMeUtil float64 `json:"diskNvmeUtil"`
|
|
DiskNVMeLatency float64 `json:"diskNvmeLatency"`
|
|
DiskNVMeQueue float64 `json:"diskNvmeQueue"`
|
|
DiskIODuration float64 `json:"diskIoDuration"`
|
|
DiskIOExcluded string `json:"diskIoExcluded"`
|
|
}
|
|
|
|
func defaultThresholds() AlertThresholds {
|
|
return AlertThresholds{
|
|
CPUWarning: 85, CPUCritical: 95, CPUTempWarning: 80, CPUTempCritical: 90,
|
|
MemoryWarning: 85, MemoryCritical: 95, SwapWarning: 50,
|
|
StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90,
|
|
DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30,
|
|
BackupMaxAgeHours: 48,
|
|
DiskHDDUtil: 90, DiskHDDLatency: 50, DiskHDDQueue: 4, DiskSSDUtil: 90, DiskSSDLatency: 20, DiskSSDQueue: 4, DiskNVMeUtil: 95, DiskNVMeLatency: 10, DiskNVMeQueue: 8, DiskIODuration: 60,
|
|
}
|
|
}
|
|
|
|
type HistoryPoint struct {
|
|
Timestamp int64 `json:"timestamp"`
|
|
CPUUsage float64 `json:"cpuUsage"`
|
|
CPUTemperature float64 `json:"cpuTemperature"`
|
|
MemoryUsage float64 `json:"memoryUsage"`
|
|
SwapUsage float64 `json:"swapUsage"`
|
|
RootUsage float64 `json:"rootUsage"`
|
|
NetworkReceive float64 `json:"networkReceive"`
|
|
NetworkTransmit float64 `json:"networkTransmit"`
|
|
UPSCharge *float64 `json:"upsCharge"`
|
|
UPSLoad *float64 `json:"upsLoad"`
|
|
ZFSUsage float64 `json:"zfsUsage"`
|
|
GuestsCPU float64 `json:"guestsCpu"`
|
|
GuestsMemory float64 `json:"guestsMemory"`
|
|
}
|
|
|
|
type Store struct {
|
|
db *sql.DB
|
|
path string
|
|
healthMu sync.RWMutex
|
|
lastHistoryAt time.Time
|
|
lastHistoryError string
|
|
}
|
|
|
|
type EmailSettings struct {
|
|
Enabled bool `json:"enabled"`
|
|
Host string `json:"host"`
|
|
Port int `json:"port"`
|
|
Username string `json:"username"`
|
|
Password string `json:"password,omitempty"`
|
|
HasPassword bool `json:"hasPassword"`
|
|
SkipTLSVerify bool `json:"skipTlsVerify"`
|
|
From string `json:"from"`
|
|
To string `json:"to"`
|
|
IntervalMinutes int `json:"intervalMinutes"`
|
|
Severities []string `json:"severities"`
|
|
Sources []string `json:"sources"`
|
|
}
|
|
|
|
func openStore(path string) (*Store, error) {
|
|
if path != ":memory:" {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
db.SetMaxOpenConns(1)
|
|
statements := []string{
|
|
`PRAGMA journal_mode=WAL`,
|
|
`PRAGMA busy_timeout=5000`,
|
|
`CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
|
|
`CREATE TABLE IF NOT EXISTS history (
|
|
ts INTEGER PRIMARY KEY, cpu_usage REAL NOT NULL, cpu_temp REAL NOT NULL,
|
|
memory_usage REAL NOT NULL, swap_usage REAL NOT NULL, root_usage REAL NOT NULL,
|
|
network_rx REAL NOT NULL, network_tx REAL NOT NULL, ups_charge REAL, ups_load REAL,
|
|
zfs_usage REAL NOT NULL, guests_cpu REAL NOT NULL, guests_memory REAL NOT NULL
|
|
)`,
|
|
`CREATE INDEX IF NOT EXISTS history_ts_idx ON history(ts)`,
|
|
`CREATE TABLE IF NOT EXISTS alert_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, alert_key TEXT NOT NULL, severity TEXT NOT NULL,
|
|
source TEXT NOT NULL, title TEXT NOT NULL, message TEXT NOT NULL,
|
|
first_seen INTEGER NOT NULL, last_seen INTEGER NOT NULL, resolved_at INTEGER,
|
|
acknowledged_at INTEGER, email_sent_at INTEGER
|
|
)`,
|
|
`CREATE INDEX IF NOT EXISTS alert_events_active_idx ON alert_events(alert_key,resolved_at)`,
|
|
`CREATE TABLE IF NOT EXISTS services (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, url TEXT NOT NULL,
|
|
local_address TEXT NOT NULL DEFAULT '', check_type TEXT NOT NULL DEFAULT 'http',
|
|
expected_code INTEGER NOT NULL DEFAULT 200, expected_text TEXT NOT NULL DEFAULT '',
|
|
timeout_seconds INTEGER NOT NULL DEFAULT 5, enabled INTEGER NOT NULL DEFAULT 1
|
|
)`,
|
|
`CREATE TABLE IF NOT EXISTS service_checks (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, service_id INTEGER NOT NULL, ts INTEGER NOT NULL,
|
|
up INTEGER NOT NULL, latency_ms REAL NOT NULL, status_code INTEGER NOT NULL DEFAULT 0,
|
|
error TEXT NOT NULL DEFAULT '', tls_expiry INTEGER,
|
|
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
|
|
)`,
|
|
`CREATE INDEX IF NOT EXISTS service_checks_service_ts_idx ON service_checks(service_id,ts)`,
|
|
`CREATE TABLE IF NOT EXISTS entity_history (kind TEXT NOT NULL,name TEXT NOT NULL,ts INTEGER NOT NULL,value1 REAL,value2 REAL,PRIMARY KEY(kind,name,ts))`,
|
|
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
|
|
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
|
|
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
|
|
`CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`,
|
|
`CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`,
|
|
`CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`,
|
|
}
|
|
for _, statement := range statements {
|
|
if _, err := db.Exec(statement); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
}
|
|
// Older databases get this column without requiring a separate migration tool.
|
|
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`)
|
|
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN planned INTEGER NOT NULL DEFAULT 0`)
|
|
_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN maintenance_reason TEXT NOT NULL DEFAULT ''`)
|
|
_, _ = db.Exec(`DELETE FROM alert_events WHERE alert_key='monitor-slow'`)
|
|
_, _ = db.Exec(`ALTER TABLE service_checks ADD COLUMN endpoint TEXT NOT NULL DEFAULT 'public'`)
|
|
_, _ = db.Exec(`PRAGMA foreign_keys=ON`)
|
|
return &Store{db: db, path: path}, nil
|
|
}
|
|
|
|
type EntityHistoryPoint struct {
|
|
Timestamp int64 `json:"timestamp"`
|
|
Value1 *float64 `json:"value1,omitempty"`
|
|
Value2 *float64 `json:"value2,omitempty"`
|
|
}
|
|
type EntityHistorySeries struct {
|
|
Name string `json:"name"`
|
|
Points []EntityHistoryPoint `json:"points"`
|
|
}
|
|
|
|
func (s *Store) AddEntityHistory(kind, name string, ts int64, v1, v2 *float64) {
|
|
_, _ = s.db.Exec(`INSERT OR REPLACE INTO entity_history(kind,name,ts,value1,value2) VALUES(?,?,?,?,?)`, kind, name, ts, v1, v2)
|
|
}
|
|
func (s *Store) EntityHistory(kind string, hours int) ([]EntityHistorySeries, error) {
|
|
if hours < 1 {
|
|
hours = 24
|
|
}
|
|
if hours > 168 {
|
|
hours = 168
|
|
}
|
|
rows, err := s.db.Query(`SELECT name,ts,value1,value2 FROM entity_history WHERE kind=? AND ts>=? ORDER BY name,ts`, kind, time.Now().Add(-time.Duration(hours)*time.Hour).Unix())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []EntityHistorySeries
|
|
index := map[string]int{}
|
|
for rows.Next() {
|
|
var name string
|
|
var p EntityHistoryPoint
|
|
var v1, v2 sql.NullFloat64
|
|
if err = rows.Scan(&name, &p.Timestamp, &v1, &v2); err != nil {
|
|
return nil, err
|
|
}
|
|
if v1.Valid {
|
|
p.Value1 = &v1.Float64
|
|
}
|
|
if v2.Valid {
|
|
p.Value2 = &v2.Float64
|
|
}
|
|
i, ok := index[name]
|
|
if !ok {
|
|
i = len(out)
|
|
index[name] = i
|
|
out = append(out, EntityHistorySeries{Name: name})
|
|
}
|
|
out[i].Points = append(out[i].Points, p)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
func (s *Store) PruneEntityHistory() {
|
|
_, _ = s.db.Exec(`DELETE FROM entity_history WHERE ts<?`, time.Now().Add(-historyRetention).Unix())
|
|
}
|
|
|
|
func (s *Store) Close() error { return s.db.Close() }
|
|
|
|
func (s *Store) Thresholds() AlertThresholds {
|
|
defaults := defaultThresholds()
|
|
var value string
|
|
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key='alert_thresholds'`).Scan(&value); err != nil {
|
|
return defaults
|
|
}
|
|
if json.Unmarshal([]byte(value), &defaults) != nil {
|
|
return defaultThresholds()
|
|
}
|
|
return defaults
|
|
}
|
|
|
|
func (s *Store) SaveThresholds(value AlertThresholds) error {
|
|
if err := validateThresholds(value); err != nil {
|
|
return err
|
|
}
|
|
data, _ := json.Marshal(value)
|
|
_, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('alert_thresholds',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
|
|
return err
|
|
}
|
|
|
|
func defaultEmailSettings() EmailSettings {
|
|
return EmailSettings{Port: 587, IntervalMinutes: 60, Severities: []string{"critical", "warning"}, Sources: []string{"CPU", "Память", "Storage", "Диски", "ZFS", "UPS", "VM/LXC", "Proxmox", "Dashboard", "Сервисы", "Система", "Сеть", "Агенты", "Docker"}}
|
|
}
|
|
|
|
func (s *Store) EmailSettings(includePassword bool) EmailSettings {
|
|
settings := defaultEmailSettings()
|
|
var value string
|
|
if s.db.QueryRow(`SELECT value FROM settings WHERE key='email_notifications'`).Scan(&value) == nil {
|
|
_ = json.Unmarshal([]byte(value), &settings)
|
|
}
|
|
settings.HasPassword = settings.Password != ""
|
|
if !includePassword {
|
|
settings.Password = ""
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func (s *Store) SaveEmailSettings(settings EmailSettings) error {
|
|
if settings.Port < 1 || settings.Port > 65535 || settings.IntervalMinutes < 5 || settings.IntervalMinutes > 10080 {
|
|
return fmt.Errorf("проверьте порт и интервал (от 5 минут до 7 дней)")
|
|
}
|
|
if settings.Enabled && (settings.Host == "" || settings.From == "" || settings.To == "") {
|
|
return fmt.Errorf("для отправки нужны SMTP-сервер, отправитель и получатель")
|
|
}
|
|
if strings.ContainsAny(settings.Host+settings.From+settings.To+settings.Username, "\r\n") {
|
|
return fmt.Errorf("поля почты содержат недопустимые символы")
|
|
}
|
|
if settings.Password == "" {
|
|
settings.Password = s.EmailSettings(true).Password
|
|
}
|
|
settings.HasPassword = settings.Password != ""
|
|
data, _ := json.Marshal(settings)
|
|
_, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('email_notifications',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
|
|
return err
|
|
}
|
|
|
|
func (s *Store) LastEmailAt() time.Time {
|
|
var value string
|
|
if s.db.QueryRow(`SELECT value FROM settings WHERE key='email_last_sent'`).Scan(&value) != nil {
|
|
return time.Time{}
|
|
}
|
|
result, _ := time.Parse(time.RFC3339, value)
|
|
return result
|
|
}
|
|
|
|
func (s *Store) SetLastEmailAt(value time.Time) {
|
|
_, _ = s.db.Exec(`INSERT INTO settings(key,value) VALUES('email_last_sent',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, value.Format(time.RFC3339))
|
|
}
|
|
|
|
func validateThresholds(t AlertThresholds) error {
|
|
percentages := []float64{t.CPUWarning, t.CPUCritical, t.MemoryWarning, t.MemoryCritical, t.SwapWarning, t.StorageWarning, t.StorageCritical, t.ZFSWarning, t.ZFSCritical, t.UPSChargeCritical}
|
|
for _, value := range percentages {
|
|
if value < 0 || value > 100 {
|
|
return fmt.Errorf("процентные пороги должны быть от 0 до 100")
|
|
}
|
|
}
|
|
temperatures := []float64{t.CPUTempWarning, t.CPUTempCritical, t.DiskTempWarning, t.NVMeTempWarning}
|
|
for _, value := range temperatures {
|
|
if value < 0 || value > 150 {
|
|
return fmt.Errorf("температурные пороги должны быть от 0 до 150")
|
|
}
|
|
}
|
|
if t.CPUWarning >= t.CPUCritical || t.CPUTempWarning >= t.CPUTempCritical || t.MemoryWarning >= t.MemoryCritical || t.StorageWarning >= t.StorageCritical || t.ZFSWarning >= t.ZFSCritical {
|
|
return fmt.Errorf("порог предупреждения должен быть ниже критического")
|
|
}
|
|
if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 {
|
|
return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов")
|
|
}
|
|
for _, value := range []float64{t.DiskHDDUtil, t.DiskSSDUtil, t.DiskNVMeUtil} {
|
|
if value < 1 || value > 100 {
|
|
return fmt.Errorf("пороги utilization должны быть от 1 до 100")
|
|
}
|
|
}
|
|
for _, value := range []float64{t.DiskHDDLatency, t.DiskSSDLatency, t.DiskNVMeLatency, t.DiskHDDQueue, t.DiskSSDQueue, t.DiskNVMeQueue} {
|
|
if value < 0 {
|
|
return fmt.Errorf("пороги latency и очереди не могут быть отрицательными")
|
|
}
|
|
}
|
|
if t.DiskIODuration < 5 || t.DiskIODuration > 3600 {
|
|
return fmt.Errorf("длительность I/O-проблемы должна быть от 5 до 3600 секунд")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) AddHistory(point HistoryPoint) error {
|
|
_, err := s.db.Exec(`INSERT OR REPLACE INTO history(ts,cpu_usage,cpu_temp,memory_usage,swap_usage,root_usage,network_rx,network_tx,ups_charge,ups_load,zfs_usage,guests_cpu,guests_memory) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
point.Timestamp, point.CPUUsage, point.CPUTemperature, point.MemoryUsage, point.SwapUsage, point.RootUsage, point.NetworkReceive, point.NetworkTransmit, point.UPSCharge, point.UPSLoad, point.ZFSUsage, point.GuestsCPU, point.GuestsMemory)
|
|
if err != nil {
|
|
s.setHistoryHealth(time.Time{}, err)
|
|
return err
|
|
}
|
|
_, err = s.db.Exec(`DELETE FROM history WHERE ts < ?`, time.Now().Add(-historyRetention).Unix())
|
|
s.setHistoryHealth(time.Now(), err)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) setHistoryHealth(at time.Time, err error) {
|
|
s.healthMu.Lock()
|
|
defer s.healthMu.Unlock()
|
|
if !at.IsZero() {
|
|
s.lastHistoryAt = at
|
|
}
|
|
if err != nil {
|
|
s.lastHistoryError = err.Error()
|
|
} else {
|
|
s.lastHistoryError = ""
|
|
}
|
|
}
|
|
func (s *Store) historyHealth() (time.Time, string) {
|
|
s.healthMu.RLock()
|
|
defer s.healthMu.RUnlock()
|
|
return s.lastHistoryAt, s.lastHistoryError
|
|
}
|
|
|
|
func (s *Store) loadJSONSetting(key string, target any) bool {
|
|
var value string
|
|
if s.db.QueryRow(`SELECT value FROM settings WHERE key=?`, key).Scan(&value) != nil {
|
|
return false
|
|
}
|
|
return json.Unmarshal([]byte(value), target) == nil
|
|
}
|
|
func (s *Store) saveJSONSetting(key string, value any) error {
|
|
data, err := json.Marshal(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.db.Exec(`INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, string(data))
|
|
return err
|
|
}
|
|
|
|
func (s *Store) History(hours int) ([]HistoryPoint, error) {
|
|
if hours < 1 {
|
|
hours = 24
|
|
}
|
|
if hours > 168 {
|
|
hours = 168
|
|
}
|
|
bucket := int64(60)
|
|
if hours > 48 {
|
|
bucket = 600
|
|
}
|
|
rows, err := s.db.Query(`SELECT (ts/?)*?, AVG(cpu_usage),AVG(cpu_temp),AVG(memory_usage),AVG(swap_usage),AVG(root_usage),AVG(network_rx),AVG(network_tx),AVG(ups_charge),AVG(ups_load),AVG(zfs_usage),AVG(guests_cpu),AVG(guests_memory) FROM history WHERE ts>=? GROUP BY (ts/?) ORDER BY ts`, bucket, bucket, time.Now().Add(-time.Duration(hours)*time.Hour).Unix(), bucket)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var points []HistoryPoint
|
|
for rows.Next() {
|
|
var point HistoryPoint
|
|
var upsCharge, upsLoad sql.NullFloat64
|
|
if err := rows.Scan(&point.Timestamp, &point.CPUUsage, &point.CPUTemperature, &point.MemoryUsage, &point.SwapUsage, &point.RootUsage, &point.NetworkReceive, &point.NetworkTransmit, &upsCharge, &upsLoad, &point.ZFSUsage, &point.GuestsCPU, &point.GuestsMemory); err != nil {
|
|
return nil, err
|
|
}
|
|
if upsCharge.Valid {
|
|
point.UPSCharge = &upsCharge.Float64
|
|
}
|
|
if upsLoad.Valid {
|
|
point.UPSLoad = &upsLoad.Float64
|
|
}
|
|
points = append(points, point)
|
|
}
|
|
return points, rows.Err()
|
|
}
|