Add homelab service and system health monitoring

README.md

@@ -25,6 +25,10 @@
 - версию Proxmox, запущенное и последнее установленное ядро, доступные
   обновления, необходимость перезагрузки, последнее обновление системы,
   ближайшие backup и ZFS scrub.
+- управляемый мониторинг HTTP/HTTPS, TCP и ping для homelab-сервисов: отклик,
+  ожидаемый код и текст, TLS-сертификат, uptime за сутки и неделю;
+- ошибки systemd и ядра за 24 часа, упавшие службы, ошибки и drops каждого
+  сетевого интерфейса.
 
 Через кнопку настроек можно скрывать неиспользуемые карточки и переключать
 светлую или тёмную тему. Карточки можно переставлять перетаскиванием или
@@ -48,6 +52,12 @@ SMTP поверх TLS (порт 465).
 событие из активных, не удаляя его из журнала; повторное возникновение проблемы
 создаёт новое событие.
 
+Критические события отправляются сразу, не ожидая очередной сводки. К ним
+относятся недоступность сервиса, отказ SMART/ZFS/systemd, критический перегрев и
+низкий заряд UPS. Переход UPS на батарею также отправляется немедленно. После
+устранения уже отправленной проблемы приходит отдельное уведомление о
+восстановлении.
+
 Пороги уведомлений сохраняются в SQLite и настраиваются через вкладку
 `Настройки → Уведомления`. Панель раз в минуту сохраняет историю CPU,
 температуры, памяти, swap, сети, storage, ZFS, UPS и гостей. Данные старше семи
@@ -173,6 +183,8 @@ apcaccess status
 - `GET /api/cpu` — один JSON-снимок.
 - `GET /api/system` — единый JSON-снимок CPU и памяти.
 - `GET /api/events` — поток метрик SSE с интервалом две секунды.
+- `GET/POST/PUT/DELETE /api/services` — управление проверками сервисов.
+- `GET /api/services/history` — семидневная история доступности и отклика.
 
 ## Сборка
 

alert_events.go

@@ -18,6 +18,11 @@ type AlertEvent struct {
 	AcknowledgedAt *int64 `json:"acknowledgedAt,omitempty"`
 }
 
+type RecoveryEvent struct {
+	EventID                              int64
+	ID, Severity, Source, Title, Message string
+}
+
 func (s *Store) SyncAlerts(alerts []Alert) ([]Alert, error) {
 	tx, err := s.db.Begin()
 	if err != nil {
@@ -148,3 +153,50 @@ func (s *Store) MarkAlertsEmailed(alerts []Alert) error {
 	}
 	return tx.Commit()
 }
+
+func (s *Store) PendingImmediateAlerts() ([]Alert, error) {
+	rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NULL AND acknowledged_at IS NULL AND email_sent_at IS NULL AND (severity='critical' OR alert_key='ups-battery') ORDER BY first_seen`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Alert
+	for rows.Next() {
+		var a Alert
+		if err = rows.Scan(&a.EventID, &a.ID, &a.Severity, &a.Source, &a.Title, &a.Message); err != nil {
+			return nil, err
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+func (s *Store) PendingRecoveries() ([]RecoveryEvent, error) {
+	rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message FROM alert_events WHERE resolved_at IS NOT NULL AND email_sent_at IS NOT NULL AND recovery_sent_at IS NULL ORDER BY resolved_at`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []RecoveryEvent
+	for rows.Next() {
+		var v RecoveryEvent
+		if err = rows.Scan(&v.EventID, &v.ID, &v.Severity, &v.Source, &v.Title, &v.Message); err != nil {
+			return nil, err
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}
+func (s *Store) MarkRecoveriesEmailed(items []RecoveryEvent) error {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	now := time.Now().Unix()
+	for _, v := range items {
+		if _, err = tx.Exec(`UPDATE alert_events SET recovery_sent_at=? WHERE id=?`, now, v.EventID); err != nil {
+			return err
+		}
+	}
+	return tx.Commit()
+}

alerts.go

@@ -140,6 +140,28 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 			add("guests-stopped", "info", "VM/LXC", "Есть остановленные гости", fmt.Sprintf("Остановлено VM/LXC: %d. Это уведомление информационное.", stopped))
 		}
 	}
+	for _, service := range metrics.Services.Services {
+		if !service.Up {
+			add(fmt.Sprintf("service-%d", service.ID), "critical", "Сервисы", service.Name+": недоступен", service.Error)
+		} else if service.TLSDays != nil && *service.TLSDays < 0 {
+			add(fmt.Sprintf("service-tls-%d", service.ID), "critical", "Сервисы", service.Name+": сертификат истёк", fmt.Sprintf("Истёк %d дн. назад.", -*service.TLSDays))
+		} else if service.TLSDays != nil && *service.TLSDays < 14 {
+			add(fmt.Sprintf("service-tls-%d", service.ID), "warning", "Сервисы", service.Name+": сертификат скоро истечёт", fmt.Sprintf("Осталось %d дн.", *service.TLSDays))
+		}
+	}
+	if metrics.SystemHealth.FailedUnits > 0 {
+		add("systemd-failed", "critical", "Система", "Есть упавшие systemd-службы", fmt.Sprintf("Не запущено служб: %d.", metrics.SystemHealth.FailedUnits))
+	}
+	if metrics.SystemHealth.KernelErrors > 0 {
+		add("kernel-errors", "warning", "Система", "Ошибки ядра за 24 часа", fmt.Sprintf("Найдено сообщений: %d.", metrics.SystemHealth.KernelErrors))
+	}
+	var networkErrors uint64
+	for _, iface := range metrics.Network.Interfaces {
+		networkErrors += iface.ReceiveErrors + iface.TransmitErrors + iface.ReceiveDrops + iface.TransmitDrops
+	}
+	if networkErrors > 0 {
+		add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors))
+	}
 
 	priority := map[string]int{"critical": 0, "warning": 1, "info": 2}
 	sort.SliceStable(alerts, func(i, j int) bool {

email.go

@@ -111,6 +111,55 @@ func sendAlertEmail(settings EmailSettings, alerts []Alert, test bool) error {
 	return client.Quit()
 }
 
+func sendRecoveryEmail(settings EmailSettings, items []RecoveryEvent) error {
+	alerts := make([]Alert, 0, len(items))
+	for _, v := range items {
+		alerts = append(alerts, Alert{ID: v.ID, Severity: "info", Source: v.Source, Title: "Восстановлено: " + v.Title, Message: "Проблема больше не наблюдается."})
+	}
+	return sendAlertEmail(settings, alerts, false)
+}
+
+func runImmediateNotifier(collect func() (dashboardMetrics, error), store *Store) {
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+	for range ticker.C {
+		settings := store.EmailSettings(true)
+		if !settings.Enabled {
+			continue
+		}
+		if _, err := collect(); err != nil {
+			continue
+		}
+		urgent, err := store.PendingImmediateAlerts()
+		if err == nil && len(urgent) > 0 {
+			urgent = filterEmailAlerts(urgent, settings)
+			if len(urgent) > 0 && sendAlertEmail(settings, urgent, false) == nil {
+				_ = store.MarkAlertsEmailed(urgent)
+			}
+		}
+		recoveries, err := store.PendingRecoveries()
+		if err == nil && len(recoveries) > 0 {
+			var filtered []RecoveryEvent
+			for _, v := range recoveries {
+				if contains(settings.Sources, v.Source) {
+					filtered = append(filtered, v)
+				}
+			}
+			if len(filtered) > 0 && sendRecoveryEmail(settings, filtered) == nil {
+				_ = store.MarkRecoveriesEmailed(filtered)
+			}
+		}
+	}
+}
+func contains(items []string, want string) bool {
+	for _, v := range items {
+		if v == want {
+			return true
+		}
+	}
+	return false
+}
+
 func runEmailNotifier(collect func() (dashboardMetrics, error), store *Store) {
 	ticker := time.NewTicker(time.Minute)
 	defer ticker.Stop()

network.go

@@ -10,11 +10,22 @@ import (
 )
 
 type NetworkMetrics struct {
-	Interface      string  `json:"interface"`
-	ReceiveBytes   uint64  `json:"receiveBytes"`
-	TransmitBytes  uint64  `json:"transmitBytes"`
-	ReceivePerSec  float64 `json:"receivePerSec"`
-	TransmitPerSec float64 `json:"transmitPerSec"`
+	Interface      string             `json:"interface"`
+	ReceiveBytes   uint64             `json:"receiveBytes"`
+	TransmitBytes  uint64             `json:"transmitBytes"`
+	ReceivePerSec  float64            `json:"receivePerSec"`
+	TransmitPerSec float64            `json:"transmitPerSec"`
+	Interfaces     []NetworkInterface `json:"interfaces"`
+}
+
+type NetworkInterface struct {
+	Name           string `json:"name"`
+	ReceiveBytes   uint64 `json:"receiveBytes"`
+	TransmitBytes  uint64 `json:"transmitBytes"`
+	ReceiveErrors  uint64 `json:"receiveErrors"`
+	TransmitErrors uint64 `json:"transmitErrors"`
+	ReceiveDrops   uint64 `json:"receiveDrops"`
+	TransmitDrops  uint64 `json:"transmitDrops"`
 }
 
 type networkSnapshot struct {
@@ -30,7 +41,7 @@ type networkCollector struct {
 }
 
 func newNetworkCollector(path string) (*networkCollector, error) {
-	snapshot, _, err := readNetworkSnapshot(path)
+	snapshot, _, _, err := readNetworkSnapshot(path)
 	if err != nil {
 		return nil, err
 	}
@@ -41,7 +52,7 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	current, interfaces, err := readNetworkSnapshot(c.path)
+	current, interfaceNames, details, err := readNetworkSnapshot(c.path)
 	if err != nil {
 		return NetworkMetrics{}, err
 	}
@@ -53,20 +64,21 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 	}
 	c.previous = current
 	return NetworkMetrics{
-		Interface: strings.Join(interfaces, ", "), ReceiveBytes: current.received,
+		Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
 		TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate,
 	}, nil
 }
 
-func readNetworkSnapshot(path string) (networkSnapshot, []string, error) {
+func readNetworkSnapshot(path string) (networkSnapshot, []string, []NetworkInterface, error) {
 	file, err := os.Open(path)
 	if err != nil {
-		return networkSnapshot{}, nil, err
+		return networkSnapshot{}, nil, nil, err
 	}
 	defer file.Close()
 
 	var received, sent uint64
 	var interfaces []string
+	var details []NetworkInterface
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
@@ -84,7 +96,12 @@ func readNetworkSnapshot(path string) (networkSnapshot, []string, error) {
 			received += rx
 			sent += tx
 			interfaces = append(interfaces, strings.TrimSpace(name))
+			rxErr, _ := strconv.ParseUint(fields[2], 10, 64)
+			rxDrop, _ := strconv.ParseUint(fields[3], 10, 64)
+			txErr, _ := strconv.ParseUint(fields[10], 10, 64)
+			txDrop, _ := strconv.ParseUint(fields[11], 10, 64)
+			details = append(details, NetworkInterface{Name: strings.TrimSpace(name), ReceiveBytes: rx, TransmitBytes: tx, ReceiveErrors: rxErr, ReceiveDrops: rxDrop, TransmitErrors: txErr, TransmitDrops: txDrop})
 		}
 	}
-	return networkSnapshot{at: time.Now(), received: received, sent: sent}, interfaces, scanner.Err()
+	return networkSnapshot{at: time.Now(), received: received, sent: sent}, interfaces, details, scanner.Err()
 }

network_test.go

@@ -18,7 +18,7 @@ func TestReadNetworkSnapshot(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	snapshot, interfaces, err := readNetworkSnapshot(path)
+	snapshot, interfaces, details, err := readNetworkSnapshot(path)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -28,4 +28,7 @@ func TestReadNetworkSnapshot(t *testing.T) {
 	if len(interfaces) != 2 {
 		t.Fatalf("неожиданный список интерфейсов: %v", interfaces)
 	}
+	if len(details) != 2 {
+		t.Fatalf("unexpected interface details: %+v", details)
+	}
 }

services.go

@@ -0,0 +1,281 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"database/sql"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type MonitoredService struct {
+	ID             int64  `json:"id"`
+	Name           string `json:"name"`
+	URL            string `json:"url"`
+	LocalAddress   string `json:"localAddress"`
+	CheckType      string `json:"checkType"`
+	ExpectedCode   int    `json:"expectedCode"`
+	ExpectedText   string `json:"expectedText"`
+	TimeoutSeconds int    `json:"timeoutSeconds"`
+	Enabled        bool   `json:"enabled"`
+}
+
+type ServiceStatus struct {
+	MonitoredService
+	Up         bool    `json:"up"`
+	CheckedAt  int64   `json:"checkedAt"`
+	LatencyMS  float64 `json:"latencyMs"`
+	StatusCode int     `json:"statusCode"`
+	Error      string  `json:"error,omitempty"`
+	TLSExpiry  *int64  `json:"tlsExpiry,omitempty"`
+	TLSDays    *int    `json:"tlsDays,omitempty"`
+	UptimeDay  float64 `json:"uptimeDay"`
+	UptimeWeek float64 `json:"uptimeWeek"`
+}
+
+type ServicesMetrics struct {
+	Available bool            `json:"available"`
+	Up        int             `json:"up"`
+	Total     int             `json:"total"`
+	Services  []ServiceStatus `json:"services"`
+}
+type ServiceHistoryPoint struct {
+	Timestamp int64   `json:"timestamp"`
+	Up        bool    `json:"up"`
+	LatencyMS float64 `json:"latencyMs"`
+}
+
+func (s *Store) Services() ([]MonitoredService, error) {
+	rows, err := s.db.Query(`SELECT id,name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled FROM services ORDER BY id`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var result []MonitoredService
+	for rows.Next() {
+		var v MonitoredService
+		if err = rows.Scan(&v.ID, &v.Name, &v.URL, &v.LocalAddress, &v.CheckType, &v.ExpectedCode, &v.ExpectedText, &v.TimeoutSeconds, &v.Enabled); err != nil {
+			return nil, err
+		}
+		result = append(result, v)
+	}
+	return result, rows.Err()
+}
+
+func validateService(v MonitoredService) error {
+	v.Name = strings.TrimSpace(v.Name)
+	v.URL = strings.TrimSpace(v.URL)
+	v.LocalAddress = strings.TrimSpace(v.LocalAddress)
+	if v.Name == "" || (v.URL == "" && v.LocalAddress == "") {
+		return fmt.Errorf("укажите название и хотя бы один адрес")
+	}
+	if v.CheckType != "http" && v.CheckType != "tcp" && v.CheckType != "ping" {
+		return fmt.Errorf("неизвестный тип проверки")
+	}
+	if v.TimeoutSeconds < 1 || v.TimeoutSeconds > 30 {
+		return fmt.Errorf("таймаут должен быть от 1 до 30 секунд")
+	}
+	if v.CheckType == "http" {
+		target := v.URL
+		if target == "" {
+			target = v.LocalAddress
+		}
+		u, err := url.Parse(target)
+		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+			return fmt.Errorf("укажите полный HTTP/HTTPS-адрес")
+		}
+	}
+	return nil
+}
+
+func (s *Store) SaveService(v MonitoredService) (MonitoredService, error) {
+	if err := validateService(v); err != nil {
+		return v, err
+	}
+	if v.ExpectedCode == 0 {
+		v.ExpectedCode = 200
+	}
+	if v.ID == 0 {
+		r, err := s.db.Exec(`INSERT INTO services(name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled) VALUES(?,?,?,?,?,?,?,?)`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled)
+		if err != nil {
+			return v, err
+		}
+		v.ID, _ = r.LastInsertId()
+	} else {
+		_, err := s.db.Exec(`UPDATE services SET name=?,url=?,local_address=?,check_type=?,expected_code=?,expected_text=?,timeout_seconds=?,enabled=? WHERE id=?`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled, v.ID)
+		if err != nil {
+			return v, err
+		}
+	}
+	return v, nil
+}
+func (s *Store) DeleteService(id int64) error {
+	_, err := s.db.Exec(`DELETE FROM services WHERE id=?`, id)
+	return err
+}
+func (s *Store) addServiceCheck(id int64, v ServiceStatus) {
+	var expiry any
+	if v.TLSExpiry != nil {
+		expiry = *v.TLSExpiry
+	}
+	_, _ = s.db.Exec(`INSERT INTO service_checks(service_id,ts,up,latency_ms,status_code,error,tls_expiry) VALUES(?,?,?,?,?,?,?)`, id, v.CheckedAt, v.Up, v.LatencyMS, v.StatusCode, v.Error, expiry)
+	_, _ = s.db.Exec(`DELETE FROM service_checks WHERE ts<?`, time.Now().Add(-historyRetention).Unix())
+}
+func (s *Store) serviceUptime(id int64, since time.Time) float64 {
+	var value sql.NullFloat64
+	_ = s.db.QueryRow(`SELECT AVG(up)*100 FROM service_checks WHERE service_id=? AND ts>=?`, id, since.Unix()).Scan(&value)
+	if value.Valid {
+		return value.Float64
+	}
+	return 0
+}
+func (s *Store) ServiceHistory(id int64, hours int) ([]ServiceHistoryPoint, error) {
+	if hours < 1 {
+		hours = 24
+	}
+	if hours > 168 {
+		hours = 168
+	}
+	rows, err := s.db.Query(`SELECT ts,up,latency_ms FROM service_checks WHERE service_id=? AND ts>=? ORDER BY ts`, id, time.Now().Add(-time.Duration(hours)*time.Hour).Unix())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []ServiceHistoryPoint
+	for rows.Next() {
+		var p ServiceHistoryPoint
+		if err = rows.Scan(&p.Timestamp, &p.Up, &p.LatencyMS); err != nil {
+			return nil, err
+		}
+		out = append(out, p)
+	}
+	return out, rows.Err()
+}
+
+type serviceCollector struct {
+	mu      sync.RWMutex
+	metrics ServicesMetrics
+	store   *Store
+}
+
+func newServiceCollector(store *Store) *serviceCollector {
+	c := &serviceCollector{store: store}
+	go c.run()
+	return c
+}
+func (c *serviceCollector) collect() ServicesMetrics {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.metrics
+}
+func (c *serviceCollector) run() {
+	c.checkAll()
+	ticker := time.NewTicker(15 * time.Second)
+	defer ticker.Stop()
+	for range ticker.C {
+		c.checkAll()
+	}
+}
+func (c *serviceCollector) checkAll() {
+	items, err := c.store.Services()
+	if err != nil {
+		return
+	}
+	m := ServicesMetrics{Available: true}
+	for _, v := range items {
+		if !v.Enabled {
+			continue
+		}
+		st := checkService(v)
+		st.UptimeDay = c.store.serviceUptime(v.ID, time.Now().Add(-24*time.Hour))
+		st.UptimeWeek = c.store.serviceUptime(v.ID, time.Now().Add(-7*24*time.Hour))
+		c.store.addServiceCheck(v.ID, st)
+		m.Total++
+		if st.Up {
+			m.Up++
+		}
+		m.Services = append(m.Services, st)
+	}
+	c.mu.Lock()
+	c.metrics = m
+	c.mu.Unlock()
+}
+
+func checkService(v MonitoredService) ServiceStatus {
+	st := ServiceStatus{MonitoredService: v, CheckedAt: time.Now().Unix()}
+	start := time.Now()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(v.TimeoutSeconds)*time.Second)
+	defer cancel()
+	switch v.CheckType {
+	case "tcp":
+		target := v.URL
+		if target == "" {
+			target = v.LocalAddress
+		}
+		d := net.Dialer{}
+		conn, err := d.DialContext(ctx, "tcp", target)
+		if err == nil {
+			st.Up = true
+			conn.Close()
+		} else {
+			st.Error = err.Error()
+		}
+	case "ping":
+		target := v.URL
+		if target == "" {
+			target = v.LocalAddress
+		}
+		host := strings.TrimPrefix(strings.TrimPrefix(target, "https://"), "http://")
+		host = strings.Split(host, "/")[0]
+		host = strings.Split(host, ":")[0]
+		arg := "-W"
+		timeout := strconv.Itoa(v.TimeoutSeconds)
+		if _, err := exec.LookPath("ping"); err != nil {
+			st.Error = "ping не установлен"
+		} else if out, err := exec.CommandContext(ctx, "ping", "-c", "1", arg, timeout, host).CombinedOutput(); err == nil {
+			st.Up = true
+		} else {
+			st.Error = strings.TrimSpace(string(out))
+		}
+	default:
+		target := v.URL
+		if target == "" {
+			target = v.LocalAddress
+		}
+		tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}}
+		client := &http.Client{Transport: tr, Timeout: time.Duration(v.TimeoutSeconds) * time.Second}
+		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+		resp, err := client.Do(req)
+		if err != nil {
+			st.Error = err.Error()
+		} else {
+			st.StatusCode = resp.StatusCode
+			body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+			resp.Body.Close()
+			st.Up = resp.StatusCode == v.ExpectedCode
+			if st.Up && v.ExpectedText != "" && !strings.Contains(string(body), v.ExpectedText) {
+				st.Up = false
+				st.Error = "ожидаемый текст не найден"
+			}
+			if !st.Up && st.Error == "" {
+				st.Error = fmt.Sprintf("HTTP %d, ожидался %d", resp.StatusCode, v.ExpectedCode)
+			}
+			if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
+				expiry := resp.TLS.PeerCertificates[0].NotAfter.Unix()
+				days := int(time.Until(resp.TLS.PeerCertificates[0].NotAfter).Hours() / 24)
+				st.TLSExpiry = &expiry
+				st.TLSDays = &days
+			}
+		}
+	}
+	st.LatencyMS = float64(time.Since(start).Microseconds()) / 1000
+	return st
+}

services_test.go

@@ -0,0 +1,29 @@
+package main
+
+import "testing"
+
+func TestServiceCRUD(t *testing.T) {
+	store, err := openStore(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+	v, err := store.SaveService(MonitoredService{Name: "Test", URL: "https://example.test", CheckType: "http", ExpectedCode: 204, TimeoutSeconds: 2, Enabled: true})
+	if err != nil || v.ID == 0 {
+		t.Fatalf("save: %+v %v", v, err)
+	}
+	items, err := store.Services()
+	if err != nil || len(items) != 1 {
+		t.Fatalf("list: %+v %v", items, err)
+	}
+	if err := store.DeleteService(v.ID); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestValidateServiceAllowsLocalAddress(t *testing.T) {
+	v := MonitoredService{Name: "Host", LocalAddress: "192.168.0.12", CheckType: "ping", TimeoutSeconds: 5}
+	if err := validateService(v); err != nil {
+		t.Fatal(err)
+	}
+}

store.go

@@ -104,6 +104,19 @@ func openStore(path string) (*Store, error) {
 			acknowledged_at INTEGER, email_sent_at INTEGER
 		)`,
 		`CREATE INDEX IF NOT EXISTS alert_events_active_idx ON alert_events(alert_key,resolved_at)`,
+		`CREATE TABLE IF NOT EXISTS services (
+			id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, url TEXT NOT NULL,
+			local_address TEXT NOT NULL DEFAULT '', check_type TEXT NOT NULL DEFAULT 'http',
+			expected_code INTEGER NOT NULL DEFAULT 200, expected_text TEXT NOT NULL DEFAULT '',
+			timeout_seconds INTEGER NOT NULL DEFAULT 5, enabled INTEGER NOT NULL DEFAULT 1
+		)`,
+		`CREATE TABLE IF NOT EXISTS service_checks (
+			id INTEGER PRIMARY KEY AUTOINCREMENT, service_id INTEGER NOT NULL, ts INTEGER NOT NULL,
+			up INTEGER NOT NULL, latency_ms REAL NOT NULL, status_code INTEGER NOT NULL DEFAULT 0,
+			error TEXT NOT NULL DEFAULT '', tls_expiry INTEGER,
+			FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
+		)`,
+		`CREATE INDEX IF NOT EXISTS service_checks_service_ts_idx ON service_checks(service_id,ts)`,
 	}
 	for _, statement := range statements {
 		if _, err := db.Exec(statement); err != nil {
@@ -111,6 +124,9 @@ func openStore(path string) (*Store, error) {
 			return nil, err
 		}
 	}
+	// Older databases get this column without requiring a separate migration tool.
+	_, _ = db.Exec(`ALTER TABLE alert_events ADD COLUMN recovery_sent_at INTEGER`)
+	_, _ = db.Exec(`PRAGMA foreign_keys=ON`)
 	return &Store{db: db}, nil
 }
 
@@ -138,7 +154,7 @@ func (s *Store) SaveThresholds(value AlertThresholds) error {
 }
 
 func defaultEmailSettings() EmailSettings {
-	return EmailSettings{Port: 587, IntervalMinutes: 60, Severities: []string{"critical", "warning"}, Sources: []string{"CPU", "Память", "Storage", "Диски", "ZFS", "UPS", "VM/LXC", "Proxmox", "Dashboard"}}
+	return EmailSettings{Port: 587, IntervalMinutes: 60, Severities: []string{"critical", "warning"}, Sources: []string{"CPU", "Память", "Storage", "Диски", "ZFS", "UPS", "VM/LXC", "Proxmox", "Dashboard", "Сервисы", "Система", "Сеть"}}
 }
 
 func (s *Store) EmailSettings(includePassword bool) EmailSettings {

system_health.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type SystemIssue struct {
+	Time     int64  `json:"time"`
+	Unit     string `json:"unit"`
+	Priority string `json:"priority"`
+	Message  string `json:"message"`
+}
+type SystemHealthMetrics struct {
+	Available    bool          `json:"available"`
+	FailedUnits  int           `json:"failedUnits"`
+	KernelErrors int           `json:"kernelErrors"`
+	SystemErrors int           `json:"systemErrors"`
+	Issues       []SystemIssue `json:"issues"`
+	CheckedAt    int64         `json:"checkedAt"`
+}
+type systemHealthCollector struct {
+	mu    sync.RWMutex
+	value SystemHealthMetrics
+}
+
+func newSystemHealthCollector() *systemHealthCollector {
+	c := &systemHealthCollector{}
+	go c.run()
+	return c
+}
+func (c *systemHealthCollector) collect() SystemHealthMetrics {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.value
+}
+func (c *systemHealthCollector) run() {
+	c.refresh()
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+	for range t.C {
+		c.refresh()
+	}
+}
+func (c *systemHealthCollector) refresh() {
+	m := SystemHealthMetrics{CheckedAt: time.Now().Unix()}
+	if _, err := exec.LookPath("journalctl"); err != nil {
+		c.mu.Lock()
+		c.value = m
+		c.mu.Unlock()
+		return
+	}
+	m.Available = true
+	if out, err := exec.Command("systemctl", "--failed", "--no-legend", "--plain").Output(); err == nil {
+		for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+			if strings.TrimSpace(line) != "" {
+				m.FailedUnits++
+			}
+		}
+	}
+	out, _ := exec.Command("journalctl", "--since", "24 hours ago", "-p", "0..3", "--no-pager", "-o", "short-unix", "-n", "100").Output()
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		f := strings.Fields(line)
+		if len(f) < 2 {
+			continue
+		}
+		raw := strings.TrimSuffix(f[0], ":")
+		sec, _ := strconv.ParseFloat(raw, 64)
+		unit := strings.TrimSuffix(f[1], ":")
+		message := strings.Join(f[2:], " ")
+		issue := SystemIssue{Time: int64(sec), Unit: unit, Priority: "error", Message: message}
+		m.Issues = append(m.Issues, issue)
+		m.SystemErrors++
+		if strings.Contains(strings.ToLower(unit), "kernel") {
+			m.KernelErrors++
+		}
+	}
+	c.mu.Lock()
+	c.value = m
+	c.mu.Unlock()
+}

web.go

@@ -10,22 +10,24 @@ import (
 )
 
 type dashboardMetrics struct {
-	CPU         CPUMetrics         `json:"cpu"`
-	Memory      MemoryMetrics      `json:"memory"`
-	Storage     StorageMetrics     `json:"storage"`
-	Disks       []DiskMetrics      `json:"disks"`
-	Network     NetworkMetrics     `json:"network"`
-	ZFS         ZFSMetrics         `json:"zfs"`
-	UPS         UPSMetrics         `json:"ups"`
-	Guests      GuestsMetrics      `json:"guests"`
-	Backups     BackupMetrics      `json:"backups"`
-	Alerts      []Alert            `json:"alerts"`
-	Update      UpdateStatus       `json:"update"`
-	Maintenance MaintenanceMetrics `json:"maintenance"`
-	Timestamp   time.Time          `json:"timestamp"`
+	CPU          CPUMetrics          `json:"cpu"`
+	Memory       MemoryMetrics       `json:"memory"`
+	Storage      StorageMetrics      `json:"storage"`
+	Disks        []DiskMetrics       `json:"disks"`
+	Network      NetworkMetrics      `json:"network"`
+	ZFS          ZFSMetrics          `json:"zfs"`
+	UPS          UPSMetrics          `json:"ups"`
+	Guests       GuestsMetrics       `json:"guests"`
+	Backups      BackupMetrics       `json:"backups"`
+	Alerts       []Alert             `json:"alerts"`
+	Update       UpdateStatus        `json:"update"`
+	Maintenance  MaintenanceMetrics  `json:"maintenance"`
+	Services     ServicesMetrics     `json:"services"`
+	SystemHealth SystemHealthMetrics `json:"systemHealth"`
+	Timestamp    time.Time           `json:"timestamp"`
 }
 
-func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, store *Store) (dashboardMetrics, error) {
+func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkCollector, diskCollector *diskCollector, zfsCollector *zfsCollector, upsCollector *upsCollector, guestCollector *guestCollector, backupCollector *backupCollector, maintenanceCollector *maintenanceCollector, serviceCollector *serviceCollector, systemHealth *systemHealthCollector, store *Store) (dashboardMetrics, error) {
 	cpu, err := cpuCollector.collect()
 	if err != nil {
 		return dashboardMetrics{}, err
@@ -46,7 +48,7 @@ func collectDashboard(cpuCollector *cpuCollector, networkCollector *networkColle
 		CPU: cpu, Memory: memory, Storage: storage,
 		Disks: diskCollector.collect(), Network: network,
 		ZFS: zfsCollector.collect(), UPS: upsCollector.collect(),
-		Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Update: readUpdateStatus(), Timestamp: time.Now(),
+		Guests: guestCollector.collect(), Backups: backupCollector.collect(), Maintenance: maintenanceCollector.collect(), Services: serviceCollector.collect(), SystemHealth: systemHealth.collect(), Update: readUpdateStatus(), Timestamp: time.Now(),
 	}
 	metrics.Alerts = evaluateAlerts(metrics, store.Thresholds())
 	metrics.Alerts, err = store.SyncAlerts(metrics.Alerts)
@@ -65,11 +67,14 @@ func routes(collector *cpuCollector, store *Store) http.Handler {
 	guests := &guestCollector{}
 	backups := &backupCollector{}
 	maintenance := &maintenanceCollector{}
+	services := newServiceCollector(store)
+	systemHealth := newSystemHealthCollector()
 	collect := func() (dashboardMetrics, error) {
-		return collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, store)
+		return collectDashboard(collector, network, disks, zfs, ups, guests, backups, maintenance, services, systemHealth, store)
 	}
 	go runHistoryCollector(collect, store)
 	go runEmailNotifier(collect, store)
+	go runImmediateNotifier(collect, store)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/" {
 			http.NotFound(w, r)
@@ -173,6 +178,36 @@ func routes(collector *cpuCollector, store *Store) http.Handler {
 		points, err := store.History(hours)
 		writeJSON(w, points, err)
 	})
+	mux.HandleFunc("/api/services", func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodGet:
+			items, err := store.Services()
+			writeJSON(w, items, err)
+		case http.MethodPost, http.MethodPut:
+			var v MonitoredService
+			if err := json.NewDecoder(io.LimitReader(r.Body, 32<<10)).Decode(&v); err != nil {
+				http.Error(w, "Некорректные данные", 400)
+				return
+			}
+			saved, err := store.SaveService(v)
+			writeJSON(w, saved, err)
+		case http.MethodDelete:
+			id, _ := strconv.ParseInt(r.URL.Query().Get("id"), 10, 64)
+			if id < 1 {
+				http.Error(w, "Некорректный сервис", 400)
+				return
+			}
+			writeJSON(w, map[string]bool{"deleted": true}, store.DeleteService(id))
+		default:
+			w.WriteHeader(http.StatusMethodNotAllowed)
+		}
+	})
+	mux.HandleFunc("/api/services/history", func(w http.ResponseWriter, r *http.Request) {
+		id, _ := strconv.ParseInt(r.URL.Query().Get("id"), 10, 64)
+		hours, _ := strconv.Atoi(r.URL.Query().Get("hours"))
+		points, err := store.ServiceHistory(id, hours)
+		writeJSON(w, points, err)
+	})
 	mux.HandleFunc("/api/events", func(w http.ResponseWriter, r *http.Request) {
 		flusher, ok := w.(http.Flusher)
 		if !ok {
@@ -244,6 +279,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
 @media(max-width:700px){main{padding:28px 0}.cards{grid-template-columns:1fr}header{align-items:flex-start;flex-direction:column;gap:14px}.detail-grid{grid-template-columns:repeat(2,1fr)}.detail:nth-child(3n){border-right:1px solid var(--line)}.detail:nth-child(2n){border-right:0}.name{max-width:220px}.disk-stats,.setting-list,.threshold-grid{grid-template-columns:repeat(2,1fr)}.chart-head{align-items:flex-start;flex-direction:column}.detail-tabs{overflow-x:auto}}
 .save-settings:disabled{cursor:wait;opacity:.55}[data-settings-panel="email"] .threshold-item input{width:min(210px,52vw)}.ack-alert{margin-top:11px;padding:6px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);font-size:11px;cursor:pointer}.history-state{display:inline-block;margin-top:8px;color:var(--muted);font-size:11px}.alert-panel[hidden]{display:none}
 .maintenance .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.card-order{display:grid;gap:7px;margin-top:11px}.order-item{display:flex;align-items:center;gap:10px;padding:9px 10px;border:1px solid var(--line);border-radius:10px;background:var(--panel)}.order-item.dragging{opacity:.45}.order-handle{cursor:grab;color:var(--muted);font-size:17px}.order-item span:nth-child(2){flex:1}.order-button{width:30px;height:28px;border:1px solid var(--line);border-radius:7px;background:var(--bg);color:var(--text);cursor:pointer}
+.services .fill{background:linear-gradient(90deg,#55d58a,#43c6d9)}.system-health .fill{background:linear-gradient(90deg,#6ba8ff,#9d8cff)}.service-form{display:grid;grid-template-columns:1fr 1fr;gap:9px}.service-form input,.service-form select{width:100%;box-sizing:border-box;padding:10px;border:1px solid var(--line);border-radius:9px;background:var(--bg);color:var(--text)}.service-actions{display:flex;gap:7px;margin-top:10px}.service-actions button{padding:7px 10px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--text);cursor:pointer}.status-up{color:var(--green)}.status-down{color:#ff665c}.issue-message{overflow-wrap:anywhere}.service-history{height:160px}.full-span{grid-column:1/-1}
 </style>
 </head>
 <body><main>
@@ -309,6 +345,18 @@ footer{color:#657581;font-size:12px;margin-top:17px}
  <div class="quick"><div><span class="label" id="backupFreshLabel">Свежие · 48 часов</span><strong id="backupFresh">—</strong></div><div><span class="label">Без копии</span><strong id="backupMissing">—</strong></div></div>
  <div class="hint">Нажмите, чтобы посмотреть подробности →</div>
 </button>
+<button class="card services" data-card="services" data-open="servicesDialog" aria-label="Открыть мониторинг сервисов">
+ <div class="card-head"><div><div class="label">Homelab services</div><div class="name">Сервисы</div></div><div class="percent"><span id="serviceUp">—</span><small id="serviceTotal"> / —</small></div></div>
+ <div class="bar"><div class="fill" id="serviceBar"></div></div>
+ <div class="quick"><div><span class="label">Доступность · сутки</span><strong id="serviceUptime">—</strong></div><div><span class="label">Средний отклик</span><strong id="serviceLatency">—</strong></div></div>
+ <div class="hint">Нажмите, чтобы управлять проверками →</div>
+</button>
+<button class="card system-health" data-card="system" data-open="systemDialog" aria-label="Открыть системные события">
+ <div class="card-head"><div><div class="label">System health</div><div class="name">Система и ядро</div></div><div class="percent"><span id="systemIssues">—</span><small> ошибок</small></div></div>
+ <div class="bar"><div class="fill" id="systemBar"></div></div>
+ <div class="quick"><div><span class="label">Упавшие службы</span><strong id="failedUnits">—</strong></div><div><span class="label">Ошибки ядра</span><strong id="kernelErrors">—</strong></div></div>
+ <div class="hint">Нажмите, чтобы посмотреть журнал →</div>
+</button>
 </section>
 <footer>Обновление каждые 2 секунды · данные читаются напрямую с хоста</footer>
 </main>
@@ -357,6 +405,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
   <div class="detail"><span class="label">Получение сейчас</span><strong id="dNetDown">—</strong></div><div class="detail"><span class="label">Передача сейчас</span><strong id="dNetUp">—</strong></div><div class="detail"><span class="label">Суммарная скорость</span><strong id="dNetTotal">—</strong></div>
   <div class="detail"><span class="label">Получено с запуска</span><strong id="dNetReceived">—</strong></div><div class="detail"><span class="label">Передано с запуска</span><strong id="dNetSent">—</strong></div><div class="detail"><span class="label">Интерфейсы</span><strong id="dNetInterfaces">—</strong></div>
  </div>
+ <div class="disk-list" id="networkInterfacesList"></div>
  <div class="chart-section"><div class="chart-head"><strong>Входящий и исходящий трафик</strong><span>Последние 7 дней</span></div><canvas class="metric-chart" id="networkChart"></canvas></div>
 </dialog>
 
@@ -397,9 +446,19 @@ footer{color:#657581;font-size:12px;margin-top:17px}
  <div class="wide"><span class="label">Ближайшие задания</span><div class="disk-stats"><div class="disk-stat"><span>Backup</span><b id="dNextBackup">—</b></div><div class="disk-stat"><span>ZFS scrub</span><b id="dNextScrub">—</b></div><div class="disk-stat"><span>Проверено</span><b id="dMaintenanceChecked">—</b></div></div></div>
 </dialog>
 
+<dialog id="servicesDialog">
+ <div class="modal-head"><div><div class="label">Homelab monitoring</div><h2>Сервисы</h2></div><button class="close" aria-label="Закрыть">×</button></div>
+ <div class="disk-list" id="serviceList"><span class="empty">Добавьте первую проверку в настройках.</span></div>
+</dialog>
+<dialog id="systemDialog">
+ <div class="modal-head"><div><div class="label">System health</div><h2>Ошибки системы и ядра</h2></div><button class="close" aria-label="Закрыть">×</button></div>
+ <div class="detail-grid"><div class="detail"><span class="label">Ошибки · 24 часа</span><strong id="dSystemErrors">—</strong></div><div class="detail"><span class="label">Ядро</span><strong id="dKernelErrors">—</strong></div><div class="detail"><span class="label">Упавшие службы</span><strong id="dFailedUnits">—</strong></div></div>
+ <div class="disk-list" id="issueList"><span class="empty">Системных ошибок нет.</span></div>
+</dialog>
+
 <dialog id="settingsDialog">
  <div class="modal-head"><div><div class="label">Dashboard</div><h2>Настройки</h2></div><button class="close" aria-label="Закрыть">×</button></div>
- <div class="settings-tabs"><button class="settings-tab active" data-settings-tab="appearance">Интерфейс</button><button class="settings-tab" data-settings-tab="alerts">Пороги</button><button class="settings-tab" data-settings-tab="email">Почта</button><button class="settings-tab" data-settings-tab="updates">Обновления</button></div>
+ <div class="settings-tabs"><button class="settings-tab active" data-settings-tab="appearance">Интерфейс</button><button class="settings-tab" data-settings-tab="alerts">Пороги</button><button class="settings-tab" data-settings-tab="services">Сервисы</button><button class="settings-tab" data-settings-tab="email">Почта</button><button class="settings-tab" data-settings-tab="updates">Обновления</button></div>
  <div class="settings-body settings-panel" data-settings-panel="appearance">
    <div class="settings-group"><div class="label">Тема оформления</div><div class="theme-options"><button class="theme-option" data-theme-choice="dark">Тёмная</button><button class="theme-option" data-theme-choice="light">Светлая</button></div></div>
    <div class="settings-group"><div class="label">Видимые карточки</div><div class="setting-list">
@@ -407,7 +466,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
     <label class="setting-item"><input type="checkbox" data-card-toggle="cpu"> CPU</label><label class="setting-item"><input type="checkbox" data-card-toggle="memory"> Память</label>
     <label class="setting-item"><input type="checkbox" data-card-toggle="storage"> Диски</label><label class="setting-item"><input type="checkbox" data-card-toggle="network"> Сеть</label>
     <label class="setting-item"><input type="checkbox" data-card-toggle="zfs"> ZFS</label><label class="setting-item"><input type="checkbox" data-card-toggle="ups"> UPS</label>
-    <label class="setting-item"><input type="checkbox" data-card-toggle="guests"> VM и LXC</label><label class="setting-item"><input type="checkbox" data-card-toggle="maintenance"> Proxmox</label><label class="setting-item"><input type="checkbox" data-card-toggle="backups"> Резервные копии</label>
+    <label class="setting-item"><input type="checkbox" data-card-toggle="guests"> VM и LXC</label><label class="setting-item"><input type="checkbox" data-card-toggle="maintenance"> Proxmox</label><label class="setting-item"><input type="checkbox" data-card-toggle="backups"> Резервные копии</label><label class="setting-item"><input type="checkbox" data-card-toggle="services"> Сервисы</label><label class="setting-item"><input type="checkbox" data-card-toggle="system"> Система</label>
    </div></div>
    <div class="settings-group"><div class="label">Порядок карточек</div><div class="settings-note">Перетаскивайте строки или используйте стрелки.</div><div class="card-order" id="cardOrder"></div></div>
  </div>
@@ -424,6 +483,15 @@ footer{color:#657581;font-size:12px;margin-top:17px}
   </div>
   <button class="save-settings" id="saveThresholds">Сохранить пороги</button><div class="settings-note" id="thresholdStatus">Настройки хранятся на сервере и применяются для всех браузеров.</div>
  </div>
+ <div class="settings-body settings-panel" data-settings-panel="services" hidden>
+  <div class="settings-group"><div class="label">Новая или изменяемая проверка</div><div class="service-form">
+   <input type="hidden" id="serviceId"><input id="serviceName" placeholder="Название · Gitea"><select id="serviceType"><option value="http">HTTP / HTTPS</option><option value="tcp">TCP-порт</option><option value="ping">Ping</option></select>
+   <input id="serviceURL" placeholder="https://panel.myown.center или 192.168.0.12:22"><input id="serviceLocal" placeholder="Локальный адрес · необязательно">
+   <input id="serviceCode" type="number" value="200" placeholder="Ожидаемый HTTP-код"><input id="serviceText" placeholder="Текст на странице · необязательно">
+   <input id="serviceTimeout" type="number" value="5" min="1" max="30" placeholder="Таймаут"><label class="setting-item"><input id="serviceEnabled" type="checkbox" checked> Проверка включена</label>
+  </div><button class="save-settings" id="saveService">Сохранить сервис</button><div class="settings-note" id="serviceStatus">Проверка выполняется каждые 15 секунд, история хранится 7 дней.</div></div>
+  <div class="settings-group"><div class="label">Добавленные сервисы</div><div class="disk-list" id="serviceSettingsList"><span class="empty">Список пуст.</span></div></div>
+ </div>
  <div class="settings-body settings-panel" data-settings-panel="email" hidden>
   <div class="settings-group"><div class="setting-list"><label class="setting-item"><input type="checkbox" id="emailEnabled"> Включить почтовые дайджесты</label><label class="setting-item"><input type="checkbox" id="emailSkipTLS"> Не проверять TLS-сертификат</label></div><div class="settings-note">Только для доверенного сервера во внутренней сети.</div></div>
   <div class="settings-group"><div class="label">SMTP и получатель</div><div class="threshold-grid">
@@ -434,7 +502,7 @@ footer{color:#657581;font-size:12px;margin-top:17px}
   </div></div>
   <div class="settings-group"><div class="label">Важность</div><div class="setting-list"><label class="setting-item"><input type="checkbox" data-email-severity="critical"> Критические</label><label class="setting-item"><input type="checkbox" data-email-severity="warning"> Предупреждения</label><label class="setting-item"><input type="checkbox" data-email-severity="info"> Информация</label></div></div>
   <div class="settings-group"><div class="label">Какие разделы отправлять</div><div class="setting-list">
-   <label class="setting-item"><input type="checkbox" data-email-source="CPU"> CPU</label><label class="setting-item"><input type="checkbox" data-email-source="Память"> Память</label><label class="setting-item"><input type="checkbox" data-email-source="Storage"> Storage</label><label class="setting-item"><input type="checkbox" data-email-source="Диски"> Диски</label><label class="setting-item"><input type="checkbox" data-email-source="ZFS"> ZFS</label><label class="setting-item"><input type="checkbox" data-email-source="UPS"> UPS</label><label class="setting-item"><input type="checkbox" data-email-source="VM/LXC"> VM и LXC</label><label class="setting-item"><input type="checkbox" data-email-source="Proxmox"> Proxmox</label><label class="setting-item"><input type="checkbox" data-email-source="Backups"> Резервные копии</label><label class="setting-item"><input type="checkbox" data-email-source="Dashboard"> Dashboard</label>
+   <label class="setting-item"><input type="checkbox" data-email-source="CPU"> CPU</label><label class="setting-item"><input type="checkbox" data-email-source="Память"> Память</label><label class="setting-item"><input type="checkbox" data-email-source="Storage"> Storage</label><label class="setting-item"><input type="checkbox" data-email-source="Диски"> Диски</label><label class="setting-item"><input type="checkbox" data-email-source="ZFS"> ZFS</label><label class="setting-item"><input type="checkbox" data-email-source="UPS"> UPS</label><label class="setting-item"><input type="checkbox" data-email-source="VM/LXC"> VM и LXC</label><label class="setting-item"><input type="checkbox" data-email-source="Proxmox"> Proxmox</label><label class="setting-item"><input type="checkbox" data-email-source="Backups"> Резервные копии</label><label class="setting-item"><input type="checkbox" data-email-source="Dashboard"> Dashboard</label><label class="setting-item"><input type="checkbox" data-email-source="Сервисы"> Сервисы</label><label class="setting-item"><input type="checkbox" data-email-source="Система"> Система</label><label class="setting-item"><input type="checkbox" data-email-source="Сеть"> Сеть</label>
   </div></div>
   <button class="save-settings" id="saveEmail">Сохранить настройки почты</button><button class="theme-option" id="testEmail" style="width:100%;margin-top:9px">Отправить тестовое письмо</button><div class="settings-note" id="emailStatus">Пароль хранится в локальной базе сервера и не возвращается в браузер.</div>
  </div>
@@ -472,6 +540,9 @@ function render(x){const c=x.cpu,m=x.memory,s=x.storage,n=x.network,z=x.zfs||{av
  el('maintenanceVersion').textContent=mt.proxmoxVersion||'Proxmox VE';el('maintenanceUpdates').textContent=mt.availableUpdates??'—';el('maintenanceBar').style.width=Math.min((mt.availableUpdates||0)*5,100)+'%';el('maintenanceReboot').textContent=(mt.rebootRequired||mt.newKernelAvailable)?'Требуется':'Не требуется';el('maintenanceKernel').textContent=mt.kernelVersion||'—';el('dPveVersion').textContent=mt.proxmoxVersion||'—';el('dKernel').textContent=mt.kernelVersion||'—';el('dLatestKernel').textContent=mt.latestInstalledKernel||'—';el('dUpdates').textContent=(mt.availableUpdates??'—')+' пакетов';el('dReboot').textContent=(mt.rebootRequired||mt.newKernelAvailable)?'Требуется':'Не требуется';el('dLastUpdate').textContent=mt.lastSystemUpdate?new Date(mt.lastSystemUpdate).toLocaleString('ru-RU'):'—';el('dNextBackup').textContent=mt.nextBackup||'Расписание не найдено';el('dNextScrub').textContent=mt.nextScrub||'Таймер не найден';el('dMaintenanceChecked').textContent=mt.checkedAt?new Date(mt.checkedAt).toLocaleString('ru-RU'):'—';
  const backupLimitHours=alertSettings.backupMaxAgeHours||48,backupLimit=backupLimitHours*3600,backupMap=new Map((b.items||[]).map(item=>[item.vmid,item])),now=Date.now()/1000,protectedGuests=guests.filter(v=>backupMap.has(v.vmid)),freshGuests=protectedGuests.filter(v=>now-backupMap.get(v.vmid).ctime<=backupLimit),missing=guests.length-protectedGuests.length;el('backupFreshLabel').textContent='Свежие · '+backupLimitHours+' часов';el('backupProtected').textContent=protectedGuests.length;el('backupTotal').textContent=' / '+guests.length;el('backupBar').style.width=guests.length?(protectedGuests.length/guests.length*100)+'%':'0%';el('backupFresh').textContent=b.available?freshGuests.length:'Не найдено';el('backupMissing').textContent=b.available?missing:'—';
  el('backupList').innerHTML=guests.length?guests.map(v=>{const item=backupMap.get(v.vmid),type=v.type==='qemu'?'VM':'LXC';if(!item)return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name||('Guest '+v.vmid))+'</div><div class="disk-meta">VMID '+v.vmid+' · '+type+'</div></div><span class="badge bad">Копии нет</span></div></article>';const age=now-item.ctime,fresh=age<=backupLimit,date=new Date(item.ctime*1000).toLocaleString('ru-RU');return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name||('Guest '+v.vmid))+'</div><div class="disk-meta">VMID '+v.vmid+' · '+type+' · '+date+'</div></div><span class="badge '+(fresh?'':'warn')+'">'+(fresh?'Актуальна':'Старше '+backupLimitHours+' ч')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Хранилище</span><b>'+safe(item.storage)+'</b></div><div class="disk-stat"><span>Размер</span><b>'+size(item.size)+'</b></div><div class="disk-stat"><span>Формат</span><b>'+safe(item.format)+'</b></div><div class="disk-stat"><span>Возраст</span><b>'+uptime(age)+'</b></div><div class="disk-stat"><span>Создана</span><b>'+date+'</b></div><div class="disk-stat"><span>VMID</span><b>'+v.vmid+'</b></div></div></article>'}).join(''):'<span class="empty">'+(b.available?'VM и LXC не найдены':'Данные резервных копий недоступны')+'</span>';
+ const svc=x.services||{services:[],up:0,total:0},svcItems=svc.services||[],svcAvg=svcItems.length?svcItems.reduce((a,v)=>a+v.uptimeDay,0)/svcItems.length:0,svcLatency=svcItems.filter(v=>v.up).length?svcItems.filter(v=>v.up).reduce((a,v)=>a+v.latencyMs,0)/svcItems.filter(v=>v.up).length:0;el('serviceUp').textContent=svc.up||0;el('serviceTotal').textContent=' / '+(svc.total||0);el('serviceBar').style.width=(svc.total?svc.up/svc.total*100:0)+'%';el('serviceUptime').textContent=svc.total?fixed(svcAvg)+'%':'—';el('serviceLatency').textContent=svc.total?fixed(svcLatency)+' ms':'—';el('serviceList').innerHTML=svcItems.length?svcItems.map(v=>{const tls=v.tlsDays==null?'—':v.tlsDays<0?'Истёк':v.tlsDays+' дн.',address=v.localAddress||v.url;return '<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+safe(address)+' · '+safe(v.checkType.toUpperCase())+'</div></div><span class="badge '+(v.up?'':'bad')+'">'+(v.up?'Доступен':'Недоступен')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Отклик</span><b>'+fixed(v.latencyMs)+' ms</b></div><div class="disk-stat"><span>HTTP</span><b>'+(v.statusCode||'—')+'</b></div><div class="disk-stat"><span>TLS</span><b>'+tls+'</b></div><div class="disk-stat"><span>Uptime · сутки</span><b>'+fixed(v.uptimeDay)+'%</b></div><div class="disk-stat"><span>Uptime · неделя</span><b>'+fixed(v.uptimeWeek)+'%</b></div><div class="disk-stat"><span>Проверено</span><b>'+new Date(v.checkedAt*1000).toLocaleTimeString('ru-RU')+'</b></div></div>'+(v.error?'<div class="settings-note status-down">'+safe(v.error)+'</div>':'')+'</article>'}).join(''):'<span class="empty">Добавьте первую проверку в настройках.</span>';
+ const sh=x.systemHealth||{},issueCount=sh.systemErrors||0;el('systemIssues').textContent=issueCount;el('systemBar').style.width=Math.min(issueCount*10,100)+'%';el('failedUnits').textContent=sh.failedUnits??'—';el('kernelErrors').textContent=sh.kernelErrors??'—';el('dSystemErrors').textContent=issueCount;el('dKernelErrors').textContent=sh.kernelErrors||0;el('dFailedUnits').textContent=sh.failedUnits||0;el('issueList').innerHTML=(sh.issues||[]).length?sh.issues.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.unit)+'</div><div class="disk-meta">'+new Date(v.time*1000).toLocaleString('ru-RU')+'</div></div><span class="badge bad">Ошибка</span></div><div class="settings-note issue-message">'+safe(v.message)+'</div></article>').join(''):'<span class="empty">Критических сообщений за 24 часа нет.</span>';
+ el('networkInterfacesList').innerHTML=(n.interfaces||[]).map(v=>'<article class="disk-item"><div class="disk-top"><div class="disk-model">'+safe(v.name)+'</div><span class="badge '+((v.receiveErrors+v.transmitErrors+v.receiveDrops+v.transmitDrops)?'warn':'')+'">'+((v.receiveErrors+v.transmitErrors+v.receiveDrops+v.transmitDrops)?'Есть потери':'Без ошибок')+'</span></div><div class="disk-stats"><div class="disk-stat"><span>Получено</span><b>'+size(v.receiveBytes)+'</b></div><div class="disk-stat"><span>Передано</span><b>'+size(v.transmitBytes)+'</b></div><div class="disk-stat"><span>RX errors / drops</span><b>'+v.receiveErrors+' / '+v.receiveDrops+'</b></div><div class="disk-stat"><span>TX errors / drops</span><b>'+v.transmitErrors+' / '+v.transmitDrops+'</b></div></div></article>').join('');
  el('dot').classList.add('live');el('status').textContent='Данные поступают';
 }
 const percent=v=>fixed(v)+'%',temperature=v=>fixed(v)+' °C',megabytes=v=>fixed(v)+' MB/s';
@@ -488,7 +559,7 @@ document.querySelectorAll('dialog').forEach(d=>{d.querySelector('.close').onclic
 el('alertList').onclick=async event=>{const button=event.target.closest('[data-ack]');if(!button)return;button.disabled=true;const response=await fetch('/api/alerts/acknowledge',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({eventId:Number(button.dataset.ack)})});if(response.ok)button.closest('.alert-item').remove();else button.disabled=false};
 async function loadAlertHistory(){const response=await fetch('/api/alerts/history');if(!response.ok)return;const events=await response.json()||[];el('alertHistory').innerHTML=events.length?events.map(a=>{const state=a.acknowledgedAt?'Принято · '+new Date(a.acknowledgedAt*1000).toLocaleString('ru-RU'):a.resolvedAt?'Завершено · '+new Date(a.resolvedAt*1000).toLocaleString('ru-RU'):'Активно';return '<article class="alert-item '+safe(a.severity)+'"><i class="alert-icon"></i><div><div class="alert-source">'+safe(a.source)+' · '+new Date(a.firstSeen*1000).toLocaleString('ru-RU')+'</div><div class="alert-title">'+safe(a.title)+'</div><div class="alert-message">'+safe(a.message)+'</div><span class="history-state">'+state+'</span></div></article>'}).join(''):'<div class="all-good"><strong>История пока пуста</strong><span>Новые события появятся здесь автоматически.</span></div>'}
 document.querySelectorAll('[data-alert-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-alert-tab]').forEach(t=>t.classList.toggle('active',t===tab));document.querySelectorAll('[data-alert-panel]').forEach(p=>p.hidden=p.dataset.alertPanel!==tab.dataset.alertTab);if(tab.dataset.alertTab==='history')loadAlertHistory()});
-const cardNames=['alerts','cpu','memory','storage','network','zfs','ups','guests','maintenance','backups'],cardLabels={alerts:'Центр уведомлений',cpu:'CPU',memory:'Память',storage:'Диски',network:'Сеть',zfs:'ZFS',ups:'UPS',guests:'VM и LXC',maintenance:'Proxmox',backups:'Резервные копии'};let hiddenCards=[],cardOrder=[];try{hiddenCards=JSON.parse(localStorage.getItem('proxmox-hidden-cards')||'[]');cardOrder=JSON.parse(localStorage.getItem('proxmox-card-order')||'[]')}catch(e){}cardOrder=[...cardOrder.filter(x=>cardNames.includes(x)),...cardNames.filter(x=>!cardOrder.includes(x))];
+const cardNames=['alerts','cpu','memory','storage','network','zfs','ups','guests','maintenance','backups','services','system'],cardLabels={alerts:'Центр уведомлений',cpu:'CPU',memory:'Память',storage:'Диски',network:'Сеть',zfs:'ZFS',ups:'UPS',guests:'VM и LXC',maintenance:'Proxmox',backups:'Резервные копии',services:'Сервисы',system:'Система и ядро'};let hiddenCards=[],cardOrder=[];try{hiddenCards=JSON.parse(localStorage.getItem('proxmox-hidden-cards')||'[]');cardOrder=JSON.parse(localStorage.getItem('proxmox-card-order')||'[]')}catch(e){}cardOrder=[...cardOrder.filter(x=>cardNames.includes(x)),...cardNames.filter(x=>!cardOrder.includes(x))];
 function renderCardOrder(){el('cardOrder').innerHTML=cardOrder.map(name=>'<div class="order-item" draggable="true" data-order-card="'+name+'"><span class="order-handle">⠿</span><span>'+cardLabels[name]+'</span><button class="order-button" data-move="up" aria-label="Выше">↑</button><button class="order-button" data-move="down" aria-label="Ниже">↓</button></div>').join('')}
 function saveCardOrder(){localStorage.setItem('proxmox-card-order',JSON.stringify(cardOrder));applyCardSettings();renderCardOrder()}
 function applyCardSettings(){const container=document.querySelector('.cards');cardOrder.forEach(name=>{const card=document.querySelector('[data-card="'+name+'"]');if(card)container.appendChild(card)});cardNames.forEach(name=>{const visible=!hiddenCards.includes(name),card=document.querySelector('[data-card="'+name+'"]'),toggle=document.querySelector('[data-card-toggle="'+name+'"]');if(card)card.hidden=!visible;if(toggle)toggle.checked=visible})}
@@ -498,6 +569,11 @@ function setTheme(theme){document.documentElement.dataset.theme=theme;localStora
 document.querySelectorAll('[data-settings-tab]').forEach(tab=>tab.onclick=()=>{document.querySelectorAll('[data-settings-tab]').forEach(t=>t.classList.toggle('active',t===tab));document.querySelectorAll('[data-settings-panel]').forEach(p=>p.hidden=p.dataset.settingsPanel!==tab.dataset.settingsTab)});
 async function loadThresholds(){const response=await fetch('/api/settings/alerts');if(!response.ok)return;alertSettings=await response.json();document.querySelectorAll('[data-threshold]').forEach(input=>input.value=alertSettings[input.dataset.threshold]??'')}
 el('saveThresholds').onclick=async()=>{const values={};document.querySelectorAll('[data-threshold]').forEach(input=>values[input.dataset.threshold]=Number(input.value));const status=el('thresholdStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/alerts',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(values)});if(response.ok){alertSettings=await response.json();status.textContent='Пороги сохранены и уже применяются.'}else{status.textContent=await response.text()||'Не удалось сохранить настройки.'}};loadThresholds().catch(()=>{});
+let configuredServices=[];
+async function loadServices(){const response=await fetch('/api/services');if(!response.ok)return;configuredServices=await response.json()||[];el('serviceSettingsList').innerHTML=configuredServices.length?configuredServices.map(v=>'<article class="disk-item"><div class="disk-top"><div><div class="disk-model">'+safe(v.name)+'</div><div class="disk-meta">'+safe(v.url)+' · '+safe(v.checkType.toUpperCase())+'</div></div><span class="badge '+(v.enabled?'':'warn')+'">'+(v.enabled?'Включён':'Выключен')+'</span></div><div class="service-actions"><button data-service-edit="'+v.id+'">Изменить</button><button data-service-delete="'+v.id+'">Удалить</button></div></article>').join(''):'<span class="empty">Список пуст.</span>'}
+function resetServiceForm(){el('serviceId').value='';el('serviceName').value='';el('serviceType').value='http';el('serviceURL').value='';el('serviceLocal').value='';el('serviceCode').value=200;el('serviceText').value='';el('serviceTimeout').value=5;el('serviceEnabled').checked=true}
+el('saveService').onclick=async()=>{const value={id:Number(el('serviceId').value)||0,name:el('serviceName').value.trim(),checkType:el('serviceType').value,url:el('serviceURL').value.trim(),localAddress:el('serviceLocal').value.trim(),expectedCode:Number(el('serviceCode').value)||200,expectedText:el('serviceText').value,timeoutSeconds:Number(el('serviceTimeout').value)||5,enabled:el('serviceEnabled').checked},status=el('serviceStatus');status.textContent='Сохранение…';const response=await fetch('/api/services',{method:value.id?'PUT':'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(value)});status.textContent=response.ok?'Сервис сохранён. Первая проверка появится в течение 15 секунд.':await response.text();if(response.ok){resetServiceForm();loadServices()}};
+el('serviceSettingsList').onclick=async event=>{const edit=event.target.closest('[data-service-edit]'),del=event.target.closest('[data-service-delete]');if(edit){const v=configuredServices.find(x=>x.id===Number(edit.dataset.serviceEdit));if(!v)return;el('serviceId').value=v.id;el('serviceName').value=v.name;el('serviceType').value=v.checkType;el('serviceURL').value=v.url;el('serviceLocal').value=v.localAddress||'';el('serviceCode').value=v.expectedCode;el('serviceText').value=v.expectedText||'';el('serviceTimeout').value=v.timeoutSeconds;el('serviceEnabled').checked=v.enabled;return}if(del&&confirm('Удалить сервис и его историю?')){const response=await fetch('/api/services?id='+del.dataset.serviceDelete,{method:'DELETE'});if(response.ok)loadServices()}};loadServices().catch(()=>{});
 function emailForm(){return {enabled:el('emailEnabled').checked,skipTlsVerify:el('emailSkipTLS').checked,host:el('emailHost').value.trim(),port:Number(el('emailPort').value),username:el('emailUsername').value.trim(),password:el('emailPassword').value,from:el('emailFrom').value.trim(),to:el('emailTo').value.trim(),intervalMinutes:Number(el('emailInterval').value),severities:[...document.querySelectorAll('[data-email-severity]:checked')].map(x=>x.dataset.emailSeverity),sources:[...document.querySelectorAll('[data-email-source]:checked')].map(x=>x.dataset.emailSource)}}
 async function loadEmail(){const response=await fetch('/api/settings/email');if(!response.ok)return;const s=await response.json();el('emailEnabled').checked=s.enabled;el('emailSkipTLS').checked=!!s.skipTlsVerify;el('emailHost').value=s.host||'';el('emailPort').value=s.port||587;el('emailUsername').value=s.username||'';el('emailPassword').value='';el('emailPassword').placeholder=s.hasPassword?'Сохранён · не изменять':'Пароль';el('emailFrom').value=s.from||'';el('emailTo').value=s.to||'';el('emailInterval').value=s.intervalMinutes||60;document.querySelectorAll('[data-email-severity]').forEach(x=>x.checked=(s.severities||[]).includes(x.dataset.emailSeverity));document.querySelectorAll('[data-email-source]').forEach(x=>x.checked=(s.sources||[]).includes(x.dataset.emailSource))}
 el('saveEmail').onclick=async()=>{const status=el('emailStatus');status.textContent='Сохранение…';const response=await fetch('/api/settings/email',{method:'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(emailForm())});status.textContent=response.ok?'Настройки сохранены.':await response.text();if(response.ok)loadEmail()};