Add homelab service and system health monitoring
This commit is contained in:
Maxim
281
services.go
Normal file
281
services.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MonitoredService struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
LocalAddress string `json:"localAddress"`
|
||||
CheckType string `json:"checkType"`
|
||||
ExpectedCode int `json:"expectedCode"`
|
||||
ExpectedText string `json:"expectedText"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type ServiceStatus struct {
|
||||
MonitoredService
|
||||
Up bool `json:"up"`
|
||||
CheckedAt int64 `json:"checkedAt"`
|
||||
LatencyMS float64 `json:"latencyMs"`
|
||||
StatusCode int `json:"statusCode"`
|
||||
Error string `json:"error,omitempty"`
|
||||
TLSExpiry *int64 `json:"tlsExpiry,omitempty"`
|
||||
TLSDays *int `json:"tlsDays,omitempty"`
|
||||
UptimeDay float64 `json:"uptimeDay"`
|
||||
UptimeWeek float64 `json:"uptimeWeek"`
|
||||
}
|
||||
|
||||
type ServicesMetrics struct {
|
||||
Available bool `json:"available"`
|
||||
Up int `json:"up"`
|
||||
Total int `json:"total"`
|
||||
Services []ServiceStatus `json:"services"`
|
||||
}
|
||||
type ServiceHistoryPoint struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
Up bool `json:"up"`
|
||||
LatencyMS float64 `json:"latencyMs"`
|
||||
}
|
||||
|
||||
func (s *Store) Services() ([]MonitoredService, error) {
|
||||
rows, err := s.db.Query(`SELECT id,name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled FROM services ORDER BY id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []MonitoredService
|
||||
for rows.Next() {
|
||||
var v MonitoredService
|
||||
if err = rows.Scan(&v.ID, &v.Name, &v.URL, &v.LocalAddress, &v.CheckType, &v.ExpectedCode, &v.ExpectedText, &v.TimeoutSeconds, &v.Enabled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func validateService(v MonitoredService) error {
|
||||
v.Name = strings.TrimSpace(v.Name)
|
||||
v.URL = strings.TrimSpace(v.URL)
|
||||
v.LocalAddress = strings.TrimSpace(v.LocalAddress)
|
||||
if v.Name == "" || (v.URL == "" && v.LocalAddress == "") {
|
||||
return fmt.Errorf("укажите название и хотя бы один адрес")
|
||||
}
|
||||
if v.CheckType != "http" && v.CheckType != "tcp" && v.CheckType != "ping" {
|
||||
return fmt.Errorf("неизвестный тип проверки")
|
||||
}
|
||||
if v.TimeoutSeconds < 1 || v.TimeoutSeconds > 30 {
|
||||
return fmt.Errorf("таймаут должен быть от 1 до 30 секунд")
|
||||
}
|
||||
if v.CheckType == "http" {
|
||||
target := v.URL
|
||||
if target == "" {
|
||||
target = v.LocalAddress
|
||||
}
|
||||
u, err := url.Parse(target)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return fmt.Errorf("укажите полный HTTP/HTTPS-адрес")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveService(v MonitoredService) (MonitoredService, error) {
|
||||
if err := validateService(v); err != nil {
|
||||
return v, err
|
||||
}
|
||||
if v.ExpectedCode == 0 {
|
||||
v.ExpectedCode = 200
|
||||
}
|
||||
if v.ID == 0 {
|
||||
r, err := s.db.Exec(`INSERT INTO services(name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled) VALUES(?,?,?,?,?,?,?,?)`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
v.ID, _ = r.LastInsertId()
|
||||
} else {
|
||||
_, err := s.db.Exec(`UPDATE services SET name=?,url=?,local_address=?,check_type=?,expected_code=?,expected_text=?,timeout_seconds=?,enabled=? WHERE id=?`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled, v.ID)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
func (s *Store) DeleteService(id int64) error {
|
||||
_, err := s.db.Exec(`DELETE FROM services WHERE id=?`, id)
|
||||
return err
|
||||
}
|
||||
func (s *Store) addServiceCheck(id int64, v ServiceStatus) {
|
||||
var expiry any
|
||||
if v.TLSExpiry != nil {
|
||||
expiry = *v.TLSExpiry
|
||||
}
|
||||
_, _ = s.db.Exec(`INSERT INTO service_checks(service_id,ts,up,latency_ms,status_code,error,tls_expiry) VALUES(?,?,?,?,?,?,?)`, id, v.CheckedAt, v.Up, v.LatencyMS, v.StatusCode, v.Error, expiry)
|
||||
_, _ = s.db.Exec(`DELETE FROM service_checks WHERE ts<?`, time.Now().Add(-historyRetention).Unix())
|
||||
}
|
||||
func (s *Store) serviceUptime(id int64, since time.Time) float64 {
|
||||
var value sql.NullFloat64
|
||||
_ = s.db.QueryRow(`SELECT AVG(up)*100 FROM service_checks WHERE service_id=? AND ts>=?`, id, since.Unix()).Scan(&value)
|
||||
if value.Valid {
|
||||
return value.Float64
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func (s *Store) ServiceHistory(id int64, hours int) ([]ServiceHistoryPoint, error) {
|
||||
if hours < 1 {
|
||||
hours = 24
|
||||
}
|
||||
if hours > 168 {
|
||||
hours = 168
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT ts,up,latency_ms FROM service_checks WHERE service_id=? AND ts>=? ORDER BY ts`, id, time.Now().Add(-time.Duration(hours)*time.Hour).Unix())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ServiceHistoryPoint
|
||||
for rows.Next() {
|
||||
var p ServiceHistoryPoint
|
||||
if err = rows.Scan(&p.Timestamp, &p.Up, &p.LatencyMS); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
type serviceCollector struct {
|
||||
mu sync.RWMutex
|
||||
metrics ServicesMetrics
|
||||
store *Store
|
||||
}
|
||||
|
||||
func newServiceCollector(store *Store) *serviceCollector {
|
||||
c := &serviceCollector{store: store}
|
||||
go c.run()
|
||||
return c
|
||||
}
|
||||
func (c *serviceCollector) collect() ServicesMetrics {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.metrics
|
||||
}
|
||||
func (c *serviceCollector) run() {
|
||||
c.checkAll()
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
c.checkAll()
|
||||
}
|
||||
}
|
||||
func (c *serviceCollector) checkAll() {
|
||||
items, err := c.store.Services()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
m := ServicesMetrics{Available: true}
|
||||
for _, v := range items {
|
||||
if !v.Enabled {
|
||||
continue
|
||||
}
|
||||
st := checkService(v)
|
||||
st.UptimeDay = c.store.serviceUptime(v.ID, time.Now().Add(-24*time.Hour))
|
||||
st.UptimeWeek = c.store.serviceUptime(v.ID, time.Now().Add(-7*24*time.Hour))
|
||||
c.store.addServiceCheck(v.ID, st)
|
||||
m.Total++
|
||||
if st.Up {
|
||||
m.Up++
|
||||
}
|
||||
m.Services = append(m.Services, st)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.metrics = m
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func checkService(v MonitoredService) ServiceStatus {
|
||||
st := ServiceStatus{MonitoredService: v, CheckedAt: time.Now().Unix()}
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(v.TimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
switch v.CheckType {
|
||||
case "tcp":
|
||||
target := v.URL
|
||||
if target == "" {
|
||||
target = v.LocalAddress
|
||||
}
|
||||
d := net.Dialer{}
|
||||
conn, err := d.DialContext(ctx, "tcp", target)
|
||||
if err == nil {
|
||||
st.Up = true
|
||||
conn.Close()
|
||||
} else {
|
||||
st.Error = err.Error()
|
||||
}
|
||||
case "ping":
|
||||
target := v.URL
|
||||
if target == "" {
|
||||
target = v.LocalAddress
|
||||
}
|
||||
host := strings.TrimPrefix(strings.TrimPrefix(target, "https://"), "http://")
|
||||
host = strings.Split(host, "/")[0]
|
||||
host = strings.Split(host, ":")[0]
|
||||
arg := "-W"
|
||||
timeout := strconv.Itoa(v.TimeoutSeconds)
|
||||
if _, err := exec.LookPath("ping"); err != nil {
|
||||
st.Error = "ping не установлен"
|
||||
} else if out, err := exec.CommandContext(ctx, "ping", "-c", "1", arg, timeout, host).CombinedOutput(); err == nil {
|
||||
st.Up = true
|
||||
} else {
|
||||
st.Error = strings.TrimSpace(string(out))
|
||||
}
|
||||
default:
|
||||
target := v.URL
|
||||
if target == "" {
|
||||
target = v.LocalAddress
|
||||
}
|
||||
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}}
|
||||
client := &http.Client{Transport: tr, Timeout: time.Duration(v.TimeoutSeconds) * time.Second}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
st.Error = err.Error()
|
||||
} else {
|
||||
st.StatusCode = resp.StatusCode
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
resp.Body.Close()
|
||||
st.Up = resp.StatusCode == v.ExpectedCode
|
||||
if st.Up && v.ExpectedText != "" && !strings.Contains(string(body), v.ExpectedText) {
|
||||
st.Up = false
|
||||
st.Error = "ожидаемый текст не найден"
|
||||
}
|
||||
if !st.Up && st.Error == "" {
|
||||
st.Error = fmt.Sprintf("HTTP %d, ожидался %d", resp.StatusCode, v.ExpectedCode)
|
||||
}
|
||||
if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
|
||||
expiry := resp.TLS.PeerCertificates[0].NotAfter.Unix()
|
||||
days := int(time.Until(resp.TLS.PeerCertificates[0].NotAfter).Hours() / 24)
|
||||
st.TLSExpiry = &expiry
|
||||
st.TLSDays = &days
|
||||
}
|
||||
}
|
||||
}
|
||||
st.LatencyMS = float64(time.Since(start).Microseconds()) / 1000
|
||||
return st
|
||||
}
|
||||
Reference in New Issue
Block a user