Add backup maintenance and guest update controls

2026-08-21 21:15:52 +03:00
parent 714c29c449
commit f6498a2fc1
10 changed files with 268 additions and 32 deletions
--- a/agent.go
+++ b/agent.go
@@ -13,6 +13,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -40,16 +41,18 @@ type agentConfig struct {
 }

 type AgentContainer struct {
-	ID          string   `json:"id"`
-	Name        string   `json:"name"`
-	Image       string   `json:"image"`
-	State       string   `json:"state"`
-	Status      string   `json:"status"`
-	Health      string   `json:"health,omitempty"`
-	Restarts    int64    `json:"restarts"`
-	CPUPercent  float64  `json:"cpuPercent"`
-	MemoryUsage uint64   `json:"memoryUsage"`
-	Ports       []string `json:"ports,omitempty"`
+	ID              string   `json:"id"`
+	Name            string   `json:"name"`
+	Image           string   `json:"image"`
+	State           string   `json:"state"`
+	Status          string   `json:"status"`
+	Health          string   `json:"health,omitempty"`
+	Restarts        int64    `json:"restarts"`
+	CPUPercent      float64  `json:"cpuPercent"`
+	MemoryUsage     uint64   `json:"memoryUsage"`
+	Ports           []string `json:"ports,omitempty"`
+	Compose         bool     `json:"compose"`
+	UpdateAvailable bool     `json:"updateAvailable"`
 }

 type AgentInterface struct {
@@ -913,14 +916,27 @@ func collectDockerContainers() ([]AgentContainer, string) {
 		inspect, inspectErr := client.Get("http://docker/containers/" + v.ID + "/json")
 		if inspectErr == nil {
 			var detail struct {
+				Image        string
 				RestartCount int64
 				State        struct{ Health *struct{ Status string } }
+				Config       struct {
+					Labels map[string]string
+				}
 			}
 			if json.NewDecoder(inspect.Body).Decode(&detail) == nil {
 				item.Restarts = detail.RestartCount
 				if detail.State.Health != nil {
 					item.Health = detail.State.Health.Status
 				}
+				item.Compose = detail.Config.Labels["com.docker.compose.service"] != ""
+				imageResponse, imageErr := client.Get("http://docker/images/" + url.PathEscape(v.Image) + "/json")
+				if imageErr == nil {
+					var imageDetail struct{ ID string }
+					if json.NewDecoder(imageResponse.Body).Decode(&imageDetail) == nil && detail.Image != "" && imageDetail.ID != "" {
+						item.UpdateAvailable = detail.Image != imageDetail.ID
+					}
+					imageResponse.Body.Close()
+				}
 			}
 			inspect.Body.Close()
 		}
--- a/agent_actions.go
+++ b/agent_actions.go
@@ -8,7 +8,9 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
@@ -61,7 +63,7 @@ func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, e
 				}
 			}
 		}
-	case "docker-restart":
+	case "docker-restart", "docker-check", "docker-update":
 		if safeContainerName.MatchString(target) {
 			for _, container := range report.Containers {
 				if container.ID == target || container.Name == target {
@@ -70,7 +72,7 @@ func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, e
 				}
 			}
 		}
-	case "updates-refresh":
+	case "updates-refresh", "system-update":
 		valid = target == ""
 	}
 	if !valid {
@@ -91,7 +93,9 @@ func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
 		return nil, err
 	}
 	defer tx.Rollback()
-	_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-5*time.Minute).Unix())
+	// Обновление пакетов и Docker-образов может быть долгим. Не выдаём одну
+	// команду повторно, пока предыдущий запуск ещё имеет шанс завершиться.
+	_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-45*time.Minute).Unix())
 	rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
 	if err != nil {
 		return nil, err
@@ -160,6 +164,12 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
 		output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
 	case "docker-restart":
 		output = restartAgentContainer(command.Target)
+	case "docker-update":
+		output = updateAgentContainer(command.Target, true)
+	case "docker-check":
+		output = updateAgentContainer(command.Target, false)
+	case "system-update":
+		output = updateAgentSystem()
 	case "updates-refresh":
 		guestInventory.Lock()
 		guestInventory.value.CollectedAt = time.Time{}
@@ -179,6 +189,77 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
 	return result
 }

+func updateAgentSystem() string {
+	if output := runAgentCombinedCommand(3*time.Minute, "apt-get", "update"); strings.HasPrefix(output, "ERROR: ") {
+		return output
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	command := exec.CommandContext(ctx, "apt-get", "-y", "upgrade")
+	command.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
+	output, err := command.CombinedOutput()
+	message := strings.TrimSpace(string(output))
+	if len(message) > 3500 {
+		message = message[len(message)-3500:]
+	}
+	if err != nil {
+		return "ERROR: " + err.Error() + ": " + message
+	}
+	guestInventory.Lock()
+	guestInventory.value.CollectedAt = time.Time{}
+	guestInventory.Unlock()
+	return firstNonEmpty(message, "Обновления системы установлены")
+}
+
+func updateAgentContainer(target string, recreate bool) string {
+	transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
+	}}
+	defer transport.CloseIdleConnections()
+	client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
+	response, err := client.Get("http://docker/containers/" + target + "/json")
+	if err != nil {
+		return "ERROR: " + err.Error()
+	}
+	defer response.Body.Close()
+	var detail struct {
+		Config struct {
+			Labels map[string]string `json:"Labels"`
+		} `json:"Config"`
+	}
+	if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
+		return "ERROR: не удалось получить Compose-конфигурацию контейнера"
+	}
+	labels := detail.Config.Labels
+	service := labels["com.docker.compose.service"]
+	workdir := labels["com.docker.compose.project.working_dir"]
+	files := strings.Split(labels["com.docker.compose.project.config_files"], ",")
+	if service == "" || workdir == "" || len(files) == 0 || !filepath.IsAbs(workdir) || !filepath.IsAbs(files[0]) {
+		return "ERROR: безопасное обновление доступно только для Docker Compose"
+	}
+	if _, err = exec.LookPath("docker"); err != nil {
+		return "ERROR: Docker CLI не найден"
+	}
+	commands := [][]string{{"compose", "-f", files[0], "pull", service}}
+	if recreate {
+		commands = append(commands, []string{"compose", "-f", files[0], "up", "-d", "--no-deps", service})
+	}
+	for _, args := range commands {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+		command := exec.CommandContext(ctx, "docker", args...)
+		command.Dir = workdir
+		output, commandErr := command.CombinedOutput()
+		cancel()
+		if commandErr != nil {
+			return "ERROR: " + commandErr.Error() + ": " + strings.TrimSpace(string(output))
+		}
+	}
+	if recreate {
+		return "Образ загружен, Compose-контейнер пересоздан"
+	}
+	return "Проверка завершена: образ загружен; если digest изменился, карточка покажет доступное обновление"
+}
+
 func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
--- a/agent_actions_test.go
+++ b/agent_actions_test.go
@@ -20,17 +20,23 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
 	if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
 		t.Fatal("unsafe unit accepted")
 	}
+	if _, err = store.QueueAgentAction("a", "docker-update", "gitea"); err != nil {
+		t.Fatalf("known Docker container rejected: %v", err)
+	}
+	if _, err = store.QueueAgentAction("a", "system-update", ""); err != nil {
+		t.Fatalf("system update rejected: %v", err)
+	}
 	action, err := store.QueueAgentAction("a", "systemd-restart", "gitea.service")
 	if err != nil {
 		t.Fatal(err)
 	}
 	commands, err := store.DispatchAgentActions("a")
-	if err != nil || len(commands) != 1 || commands[0].ID != action.ID {
+	if err != nil || len(commands) != 3 || commands[2].ID != action.ID {
 		t.Fatalf("commands=%#v err=%v", commands, err)
 	}
 	store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
 	items, err := store.AgentActions(10)
-	if err != nil || len(items) != 1 || items[0].Status != "completed" {
+	if err != nil || len(items) != 3 || items[0].Status != "completed" {
 		t.Fatalf("items=%#v err=%v", items, err)
 	}
 }
--- a/alerts.go
+++ b/alerts.go
@@ -263,10 +263,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
 	}
 	var networkErrors uint64
 	for _, iface := range metrics.Network.Interfaces {
-		networkErrors += iface.ReceiveErrors + iface.TransmitErrors + iface.ReceiveDrops + iface.TransmitDrops
+		networkErrors += iface.NewErrors + iface.NewDrops
 	}
 	if networkErrors > 0 {
-		add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors))
+		add("network-errors", "warning", "Сеть", "Появились новые сетевые ошибки или потери", fmt.Sprintf("За последний интервал добавилось: %d. Старые счётчики не создают активный алерт.", networkErrors))
 	}
 	if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp {
 		add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён."))
--- a/cpu.go
+++ b/cpu.go
@@ -33,14 +33,15 @@ type CPUStatic struct {

 type CPUMetrics struct {
 	CPUStatic
-	UsagePercent float64       `json:"usagePercent"`
-	FrequencyMHz float64       `json:"frequencyMhz"`
-	Load1        float64       `json:"load1"`
-	Load5        float64       `json:"load5"`
-	Load15       float64       `json:"load15"`
-	Uptime       uint64        `json:"uptimeSeconds"`
-	Temperatures []Temperature `json:"temperatures"`
-	CollectedAt  time.Time     `json:"collectedAt"`
+	UsagePercent float64        `json:"usagePercent"`
+	FrequencyMHz float64        `json:"frequencyMhz"`
+	Load1        float64        `json:"load1"`
+	Load5        float64        `json:"load5"`
+	Load15       float64        `json:"load15"`
+	Uptime       uint64         `json:"uptimeSeconds"`
+	Temperatures []Temperature  `json:"temperatures"`
+	TopProcesses []AgentProcess `json:"topProcesses"`
+	CollectedAt  time.Time      `json:"collectedAt"`
 }

 type Temperature struct {
@@ -92,7 +93,7 @@ func (c *cpuCollector) collect() (CPUMetrics, error) {
 	return CPUMetrics{
 		CPUStatic: c.static, UsagePercent: usage, FrequencyMHz: frequency,
 		Load1: load1, Load5: load5, Load15: load15, Uptime: uptime,
-		Temperatures: readTemperatures(), CollectedAt: time.Now(),
+		Temperatures: readTemperatures(), TopProcesses: collectAgentProcesses(), CollectedAt: time.Now(),
 	}, nil
 }

--- a/maintenance_window.go
+++ b/maintenance_window.go
@@ -17,6 +17,51 @@ type MaintenanceWindow struct {
 	AllAlerts  bool    `json:"allAlerts"`
 	ServiceIDs []int64 `json:"serviceIds"`
 	VMIDs      []int   `json:"vmids"`
+	Automatic  bool    `json:"automatic"`
+	Source     string  `json:"source,omitempty"`
+}
+
+func automaticBackupMaintenance(activity ProxmoxActivity, links []ServiceLink, now time.Time) MaintenanceWindow {
+	vmids := []int{}
+	backupRunning := false
+	for _, task := range activity.Tasks {
+		if task.Type != "vzdump" || task.EndTime > 0 || (task.Status != "" && !strings.EqualFold(task.Status, "running")) {
+			continue
+		}
+		backupRunning = true
+		if vmid := backupTaskVMID(task); vmid > 0 {
+			vmids = append(vmids, vmid)
+		}
+	}
+	vmids = uniqueInt(vmids)
+	if !backupRunning {
+		return MaintenanceWindow{}
+	}
+	selected := map[int]bool{}
+	for _, vmid := range vmids {
+		selected[vmid] = true
+	}
+	serviceIDs := []int64{}
+	for _, link := range links {
+		if selected[link.VMID] {
+			serviceIDs = append(serviceIDs, link.ServiceID)
+		}
+	}
+	return MaintenanceWindow{Enabled: true, Active: true, StartedAt: now.Unix(), EndsAt: now.Add(2 * time.Minute).Unix(), Reason: "Автоматически: выполняется backup VM/LXC", VMIDs: vmids, ServiceIDs: uniqueInt64(serviceIDs), AllAlerts: len(vmids) == 0, Automatic: true, Source: "backup"}
+}
+
+func backupTaskVMID(task ProxmoxTask) int {
+	if vmid, err := strconv.Atoi(strings.TrimSpace(task.ID)); err == nil && vmid > 0 {
+		return vmid
+	}
+	parts := strings.Split(task.UPID, ":")
+	for i, part := range parts {
+		if part == "vzdump" && i+1 < len(parts) {
+			vmid, _ := strconv.Atoi(parts[i+1])
+			return vmid
+		}
+	}
+	return 0
 }

 func (s *Store) MaintenanceWindow() MaintenanceWindow {
--- a/maintenance_window_test.go
+++ b/maintenance_window_test.go
@@ -38,6 +38,21 @@ func TestMaintenanceExpiresAutomatically(t *testing.T) {
 	}
 }

+func TestRunningBackupCreatesScopedAutomaticMaintenance(t *testing.T) {
+	now := time.Now()
+	window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", ID: "105"}, {Type: "vzdump", ID: "106", EndTime: now.Unix()}}}, []ServiceLink{{ServiceID: 7, VMID: 105}, {ServiceID: 8, VMID: 999}}, now)
+	if !window.Active || !window.Automatic || window.Source != "backup" || len(window.VMIDs) != 1 || window.VMIDs[0] != 105 || len(window.ServiceIDs) != 1 || window.ServiceIDs[0] != 7 {
+		t.Fatalf("неверное автоматическое окно: %+v", window)
+	}
+}
+
+func TestRunningBackupWithoutVMIDSuppressesAllAlerts(t *testing.T) {
+	window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", UPID: "unknown"}}}, nil, time.Now())
+	if !window.Active || !window.AllAlerts {
+		t.Fatalf("backup с неизвестной целью должен безопасно подавить алерты: %+v", window)
+	}
+}
+
 func TestPlannedAlertOnlyAppearsInHistory(t *testing.T) {
 	store, err := openStore(":memory:")
 	if err != nil {
--- a/network.go
+++ b/network.go
@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -36,6 +37,9 @@ type NetworkHealth struct {
 	ExternalIPChanged     bool      `json:"externalIpChanged"`
 	DisappearedInterfaces []string  `json:"disappearedInterfaces"`
 	CheckedAt             time.Time `json:"checkedAt"`
+	Summary               string    `json:"summary"`
+	LikelyCause           string    `json:"likelyCause"`
+	RecommendedAction     string    `json:"recommendedAction"`
 }

 type NetworkInterface struct {
@@ -46,6 +50,8 @@ type NetworkInterface struct {
 	TransmitErrors uint64  `json:"transmitErrors"`
 	ReceiveDrops   uint64  `json:"receiveDrops"`
 	TransmitDrops  uint64  `json:"transmitDrops"`
+	NewErrors      uint64  `json:"newErrors"`
+	NewDrops       uint64  `json:"newDrops"`
 	ReceivePerSec  float64 `json:"receivePerSec"`
 	TransmitPerSec float64 `json:"transmitPerSec"`
 	VMID           int     `json:"vmid,omitempty"`
@@ -113,6 +119,12 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 		if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes {
 			v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed
 			v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed
+			if v.ReceiveErrors >= old.ReceiveErrors && v.TransmitErrors >= old.TransmitErrors {
+				v.NewErrors = v.ReceiveErrors - old.ReceiveErrors + v.TransmitErrors - old.TransmitErrors
+			}
+			if v.ReceiveDrops >= old.ReceiveDrops && v.TransmitDrops >= old.TransmitDrops {
+				v.NewDrops = v.ReceiveDrops - old.ReceiveDrops + v.TransmitDrops - old.TransmitDrops
+			}
 		}
 	}
 	c.previousInterfaces = map[string]NetworkInterface{}
@@ -138,12 +150,41 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 		}
 	}
 	c.knownInterfaces = currentNames
+	explainNetworkHealth(&c.health, details)
 	return NetworkMetrics{
 		Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
 		TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
 	}, nil
 }

+func explainNetworkHealth(health *NetworkHealth, interfaces []NetworkInterface) {
+	health.Summary, health.LikelyCause, health.RecommendedAction = "Сеть работает штатно", "Проблем не обнаружено", "Действий не требуется"
+	switch {
+	case health.Gateway == "":
+		health.Summary, health.LikelyCause, health.RecommendedAction = "Не найден маршрут в локальную сеть", "На хосте отсутствует default gateway", "Проверьте адрес шлюза и конфигурацию bridge/VLAN"
+	case !health.GatewayUp:
+		health.Summary, health.LikelyCause, health.RecommendedAction = "Хост не видит локальный шлюз", "Проблема между Proxmox и роутером: линк, bridge, bond или VLAN", "Проверьте кабель, carrier и настройки vmbr/bond/VLAN"
+	case health.PacketLoss > 0:
+		health.Summary = fmt.Sprintf("Нестабильная локальная сеть: потеряно %.1f%% пакетов", health.PacketLoss)
+		health.LikelyCause, health.RecommendedAction = "Перегрузка или ошибки линка между Proxmox и шлюзом", "Проверьте errors/drops, кабель, порт коммутатора и duplex"
+	case !health.DNSUp && health.InternetUp:
+		health.Summary, health.LikelyCause, health.RecommendedAction = "Интернет доступен, но DNS не отвечает", "Проблема DNS-сервера или /etc/resolv.conf", "Проверьте настроенные DNS и попробуйте разрешить домен вручную"
+	case health.DNSUp && !health.InternetUp:
+		health.Summary, health.LikelyCause, health.RecommendedAction = "Локальная сеть и DNS работают, но внешняя проверка недоступна", "Проблема выше роутера, блокировка HTTPS или недоступность провайдера", "Проверьте маршрут наружу, firewall и доступ к внешним адресам"
+	}
+	for _, v := range interfaces {
+		if v.Kind == "Physical" && (!v.Carrier || v.State == "down") {
+			health.Summary, health.LikelyCause, health.RecommendedAction = v.Name+": физический линк не поднят", "Нет carrier либо интерфейс административно выключен", "Проверьте кабель, порт коммутатора и состояние интерфейса"
+			return
+		}
+		if v.NewErrors+v.NewDrops > 0 {
+			health.Summary = fmt.Sprintf("%s: за последний интервал появилось %d сетевых ошибок/потерь", v.Name, v.NewErrors+v.NewDrops)
+			health.LikelyCause, health.RecommendedAction = "Счётчики интерфейса сейчас растут из-за перегрузки, кабеля, MTU или duplex", "Проверьте скорость и duplex, кабель, порт коммутатора и MTU на обоих концах"
+			return
+		}
+	}
+}
+
 func decorateInterface(v *NetworkInterface) {
 	base := filepath.Join("/sys/class/net", v.Name)
 	v.State = readTrimmed(filepath.Join(base, "operstate"))
--- a/network_test.go
+++ b/network_test.go
@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )

@@ -33,6 +34,26 @@ func TestReadNetworkSnapshot(t *testing.T) {
 	}
 }

+func TestExplainNetworkHealthDistinguishesDNSFailure(t *testing.T) {
+	health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: false}
+	explainNetworkHealth(&health, nil)
+	if !strings.Contains(health.Summary, "DNS") || health.RecommendedAction == "" {
+		t.Fatalf("непонятная диагностика: %+v", health)
+	}
+}
+
+func TestExplainNetworkHealthIgnoresOldInterfaceCounters(t *testing.T) {
+	health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: true}
+	explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 42}})
+	if health.Summary != "Сеть работает штатно" {
+		t.Fatalf("старый накопленный счётчик вызвал текущую тревогу: %+v", health)
+	}
+	explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 43, NewDrops: 1}})
+	if !strings.Contains(health.Summary, "последний интервал") {
+		t.Fatalf("рост счётчика не объяснён: %+v", health)
+	}
+}
+
 func TestInterfaceVMID(t *testing.T) {
 	for name, want := range map[string]int{"tap100i0": 100, "veth108i0": 108, "fwbr113i0": 113, "vmbr0": 0, "enp6s0": 0} {
 		if got := interfaceVMID(name); got != want {
--- a/web.go
+++ b/web.go