Add persistent thresholds and seven-day metrics history
This commit is contained in:
maxim
181
store.go
Normal file
181
store.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const historyRetention = 7 * 24 * time.Hour
|
||||
|
||||
type AlertThresholds struct {
|
||||
CPUWarning float64 `json:"cpuWarning"`
|
||||
CPUCritical float64 `json:"cpuCritical"`
|
||||
CPUTempWarning float64 `json:"cpuTempWarning"`
|
||||
CPUTempCritical float64 `json:"cpuTempCritical"`
|
||||
MemoryWarning float64 `json:"memoryWarning"`
|
||||
MemoryCritical float64 `json:"memoryCritical"`
|
||||
SwapWarning float64 `json:"swapWarning"`
|
||||
StorageWarning float64 `json:"storageWarning"`
|
||||
StorageCritical float64 `json:"storageCritical"`
|
||||
ZFSWarning float64 `json:"zfsWarning"`
|
||||
ZFSCritical float64 `json:"zfsCritical"`
|
||||
DiskTempWarning float64 `json:"diskTempWarning"`
|
||||
NVMeTempWarning float64 `json:"nvmeTempWarning"`
|
||||
UPSChargeCritical float64 `json:"upsChargeCritical"`
|
||||
BackupMaxAgeHours float64 `json:"backupMaxAgeHours"`
|
||||
}
|
||||
|
||||
func defaultThresholds() AlertThresholds {
|
||||
return AlertThresholds{
|
||||
CPUWarning: 85, CPUCritical: 95, CPUTempWarning: 80, CPUTempCritical: 90,
|
||||
MemoryWarning: 85, MemoryCritical: 95, SwapWarning: 50,
|
||||
StorageWarning: 85, StorageCritical: 95, ZFSWarning: 80, ZFSCritical: 90,
|
||||
DiskTempWarning: 60, NVMeTempWarning: 75, UPSChargeCritical: 30,
|
||||
BackupMaxAgeHours: 48,
|
||||
}
|
||||
}
|
||||
|
||||
type HistoryPoint struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
CPUUsage float64 `json:"cpuUsage"`
|
||||
CPUTemperature float64 `json:"cpuTemperature"`
|
||||
MemoryUsage float64 `json:"memoryUsage"`
|
||||
SwapUsage float64 `json:"swapUsage"`
|
||||
RootUsage float64 `json:"rootUsage"`
|
||||
NetworkReceive float64 `json:"networkReceive"`
|
||||
NetworkTransmit float64 `json:"networkTransmit"`
|
||||
UPSCharge *float64 `json:"upsCharge"`
|
||||
UPSLoad *float64 `json:"upsLoad"`
|
||||
ZFSUsage float64 `json:"zfsUsage"`
|
||||
GuestsCPU float64 `json:"guestsCpu"`
|
||||
GuestsMemory float64 `json:"guestsMemory"`
|
||||
}
|
||||
|
||||
type Store struct{ db *sql.DB }
|
||||
|
||||
func openStore(path string) (*Store, error) {
|
||||
if path != ":memory:" {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
statements := []string{
|
||||
`PRAGMA journal_mode=WAL`,
|
||||
`PRAGMA busy_timeout=5000`,
|
||||
`CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
|
||||
`CREATE TABLE IF NOT EXISTS history (
|
||||
ts INTEGER PRIMARY KEY, cpu_usage REAL NOT NULL, cpu_temp REAL NOT NULL,
|
||||
memory_usage REAL NOT NULL, swap_usage REAL NOT NULL, root_usage REAL NOT NULL,
|
||||
network_rx REAL NOT NULL, network_tx REAL NOT NULL, ups_charge REAL, ups_load REAL,
|
||||
zfs_usage REAL NOT NULL, guests_cpu REAL NOT NULL, guests_memory REAL NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS history_ts_idx ON history(ts)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := db.Exec(statement); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
func (s *Store) Thresholds() AlertThresholds {
|
||||
defaults := defaultThresholds()
|
||||
var value string
|
||||
if err := s.db.QueryRow(`SELECT value FROM settings WHERE key='alert_thresholds'`).Scan(&value); err != nil {
|
||||
return defaults
|
||||
}
|
||||
if json.Unmarshal([]byte(value), &defaults) != nil {
|
||||
return defaultThresholds()
|
||||
}
|
||||
return defaults
|
||||
}
|
||||
|
||||
func (s *Store) SaveThresholds(value AlertThresholds) error {
|
||||
if err := validateThresholds(value); err != nil {
|
||||
return err
|
||||
}
|
||||
data, _ := json.Marshal(value)
|
||||
_, err := s.db.Exec(`INSERT INTO settings(key,value) VALUES('alert_thresholds',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, string(data))
|
||||
return err
|
||||
}
|
||||
|
||||
func validateThresholds(t AlertThresholds) error {
|
||||
percentages := []float64{t.CPUWarning, t.CPUCritical, t.MemoryWarning, t.MemoryCritical, t.SwapWarning, t.StorageWarning, t.StorageCritical, t.ZFSWarning, t.ZFSCritical, t.UPSChargeCritical}
|
||||
for _, value := range percentages {
|
||||
if value < 0 || value > 100 {
|
||||
return fmt.Errorf("процентные пороги должны быть от 0 до 100")
|
||||
}
|
||||
}
|
||||
temperatures := []float64{t.CPUTempWarning, t.CPUTempCritical, t.DiskTempWarning, t.NVMeTempWarning}
|
||||
for _, value := range temperatures {
|
||||
if value < 0 || value > 150 {
|
||||
return fmt.Errorf("температурные пороги должны быть от 0 до 150")
|
||||
}
|
||||
}
|
||||
if t.CPUWarning >= t.CPUCritical || t.CPUTempWarning >= t.CPUTempCritical || t.MemoryWarning >= t.MemoryCritical || t.StorageWarning >= t.StorageCritical || t.ZFSWarning >= t.ZFSCritical {
|
||||
return fmt.Errorf("порог предупреждения должен быть ниже критического")
|
||||
}
|
||||
if t.BackupMaxAgeHours < 1 || t.BackupMaxAgeHours > 24*365 {
|
||||
return fmt.Errorf("возраст backup должен быть от 1 до 8760 часов")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AddHistory(point HistoryPoint) error {
|
||||
_, err := s.db.Exec(`INSERT OR REPLACE INTO history(ts,cpu_usage,cpu_temp,memory_usage,swap_usage,root_usage,network_rx,network_tx,ups_charge,ups_load,zfs_usage,guests_cpu,guests_memory) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
point.Timestamp, point.CPUUsage, point.CPUTemperature, point.MemoryUsage, point.SwapUsage, point.RootUsage, point.NetworkReceive, point.NetworkTransmit, point.UPSCharge, point.UPSLoad, point.ZFSUsage, point.GuestsCPU, point.GuestsMemory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`DELETE FROM history WHERE ts < ?`, time.Now().Add(-historyRetention).Unix())
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) History(hours int) ([]HistoryPoint, error) {
|
||||
if hours < 1 {
|
||||
hours = 24
|
||||
}
|
||||
if hours > 168 {
|
||||
hours = 168
|
||||
}
|
||||
bucket := int64(60)
|
||||
if hours > 48 {
|
||||
bucket = 600
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT (ts/?)*?, AVG(cpu_usage),AVG(cpu_temp),AVG(memory_usage),AVG(swap_usage),AVG(root_usage),AVG(network_rx),AVG(network_tx),AVG(ups_charge),AVG(ups_load),AVG(zfs_usage),AVG(guests_cpu),AVG(guests_memory) FROM history WHERE ts>=? GROUP BY (ts/?) ORDER BY ts`, bucket, bucket, time.Now().Add(-time.Duration(hours)*time.Hour).Unix(), bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var points []HistoryPoint
|
||||
for rows.Next() {
|
||||
var point HistoryPoint
|
||||
var upsCharge, upsLoad sql.NullFloat64
|
||||
if err := rows.Scan(&point.Timestamp, &point.CPUUsage, &point.CPUTemperature, &point.MemoryUsage, &point.SwapUsage, &point.RootUsage, &point.NetworkReceive, &point.NetworkTransmit, &upsCharge, &upsLoad, &point.ZFSUsage, &point.GuestsCPU, &point.GuestsMemory); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if upsCharge.Valid {
|
||||
point.UPSCharge = &upsCharge.Float64
|
||||
}
|
||||
if upsLoad.Valid {
|
||||
point.UPSLoad = &upsLoad.Float64
|
||||
}
|
||||
points = append(points, point)
|
||||
}
|
||||
return points, rows.Err()
|
||||
}
|
||||
Reference in New Issue
Block a user