Run guest updates outside agent sandbox

agent.go

@@ -41,18 +41,22 @@ type agentConfig struct {
 }
 
 type AgentContainer struct {
-	ID              string   `json:"id"`
-	Name            string   `json:"name"`
-	Image           string   `json:"image"`
-	State           string   `json:"state"`
-	Status          string   `json:"status"`
-	Health          string   `json:"health,omitempty"`
-	Restarts        int64    `json:"restarts"`
-	CPUPercent      float64  `json:"cpuPercent"`
-	MemoryUsage     uint64   `json:"memoryUsage"`
-	Ports           []string `json:"ports,omitempty"`
-	Compose         bool     `json:"compose"`
-	UpdateAvailable bool     `json:"updateAvailable"`
+	ID               string   `json:"id"`
+	Name             string   `json:"name"`
+	Image            string   `json:"image"`
+	State            string   `json:"state"`
+	Status           string   `json:"status"`
+	Health           string   `json:"health,omitempty"`
+	Restarts         int64    `json:"restarts"`
+	CPUPercent       float64  `json:"cpuPercent"`
+	MemoryUsage      uint64   `json:"memoryUsage"`
+	Ports            []string `json:"ports,omitempty"`
+	Compose          bool     `json:"compose"`
+	UpdateAvailable  bool     `json:"updateAvailable"`
+	CurrentVersion   string   `json:"currentVersion,omitempty"`
+	CurrentDigest    string   `json:"currentDigest,omitempty"`
+	AvailableVersion string   `json:"availableVersion,omitempty"`
+	AvailableDigest  string   `json:"availableDigest,omitempty"`
 }
 
 type AgentInterface struct {
@@ -99,6 +103,7 @@ type AgentReport struct {
 	UptimeSeconds    int64               `json:"uptimeSeconds"`
 	Load1            float64             `json:"load1"`
 	CPUPercent       float64             `json:"cpuPercent"`
+	CPUCount         int                 `json:"cpuCount"`
 	MemoryTotal      uint64              `json:"memoryTotal"`
 	MemoryUsed       uint64              `json:"memoryUsed"`
 	SwapTotal        uint64              `json:"swapTotal"`
@@ -394,7 +399,9 @@ func runGuestAgent(opts agentOptions) error {
 		if requestErr != nil {
 			fmt.Fprintf(os.Stderr, "agent: %v\n", requestErr)
 		}
-		time.Sleep(10 * time.Second)
+		// Короткий интервал нужен для интерактивных команд из панели. Тяжёлый
+		// системный инвентарь отдельно кэшируется и не запускается каждые 3 секунды.
+		time.Sleep(3 * time.Second)
 	}
 }
 
@@ -530,7 +537,7 @@ func saveAgentConfig(path string, c agentConfig) error {
 func collectAgentReport() AgentReport {
 	hostname, _ := os.Hostname()
 	machineID, _ := os.ReadFile("/etc/machine-id")
-	report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CollectedAt: time.Now().Unix()}
+	report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CPUCount: runtime.NumCPU(), CollectedAt: time.Now().Unix()}
 	if data, err := os.ReadFile("/proc/uptime"); err == nil {
 		fields := strings.Fields(string(data))
 		if len(fields) > 0 {
@@ -929,14 +936,10 @@ func collectDockerContainers() ([]AgentContainer, string) {
 					item.Health = detail.State.Health.Status
 				}
 				item.Compose = detail.Config.Labels["com.docker.compose.service"] != ""
-				imageResponse, imageErr := client.Get("http://docker/images/" + url.PathEscape(v.Image) + "/json")
-				if imageErr == nil {
-					var imageDetail struct{ ID string }
-					if json.NewDecoder(imageResponse.Body).Decode(&imageDetail) == nil && detail.Image != "" && imageDetail.ID != "" {
-						item.UpdateAvailable = detail.Image != imageDetail.ID
-					}
-					imageResponse.Body.Close()
-				}
+				item.CurrentVersion, item.CurrentDigest, _ = inspectDockerImage(client, detail.Image, v.Image)
+				var availableID string
+				item.AvailableVersion, item.AvailableDigest, availableID = inspectDockerImage(client, v.Image, v.Image)
+				item.UpdateAvailable = detail.Image != "" && availableID != "" && detail.Image != availableID
 			}
 			inspect.Body.Close()
 		}
@@ -987,6 +990,46 @@ func collectDockerContainers() ([]AgentContainer, string) {
 	}
 	return result, ""
 }
+
+func inspectDockerImage(client *http.Client, reference, fallback string) (string, string, string) {
+	if reference == "" {
+		return "", "", ""
+	}
+	response, err := client.Get("http://docker/images/" + url.PathEscape(reference) + "/json")
+	if err != nil {
+		return "", "", ""
+	}
+	defer response.Body.Close()
+	var detail struct {
+		ID          string
+		RepoTags    []string
+		RepoDigests []string
+		Config      struct{ Labels map[string]string }
+	}
+	if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
+		return "", "", ""
+	}
+	version := firstNonEmpty(detail.Config.Labels["org.opencontainers.image.version"], detail.Config.Labels["version"], firstString(detail.RepoTags), fallback, shortDockerDigest(detail.ID))
+	digest := ""
+	if len(detail.RepoDigests) > 0 {
+		digest = detail.RepoDigests[0]
+		if _, value, found := strings.Cut(digest, "@"); found {
+			digest = value
+		}
+	}
+	if digest == "" {
+		digest = detail.ID
+	}
+	return version, shortDockerDigest(digest), detail.ID
+}
+
+func shortDockerDigest(value string) string {
+	value = strings.TrimPrefix(value, "sha256:")
+	if len(value) > 12 {
+		value = value[:12]
+	}
+	return value
+}
 func firstString(v []string) string {
 	if len(v) > 0 {
 		return v[0]

agent_actions.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
@@ -63,16 +62,16 @@ func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, e
 				}
 			}
 		}
-	case "docker-restart", "docker-check", "docker-update":
+	case "docker-restart", "docker-check", "docker-update", "docker-remove":
 		if safeContainerName.MatchString(target) {
 			for _, container := range report.Containers {
-				if container.ID == target || container.Name == target {
+				if (container.ID == target || container.Name == target) && (action != "docker-remove" || container.State != "running") {
 					valid = true
 					break
 				}
 			}
 		}
-	case "updates-refresh", "system-update":
+	case "updates-refresh", "system-update", "system-reboot":
 		valid = target == ""
 	}
 	if !valid {
@@ -168,8 +167,12 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
 		output = updateAgentContainer(command.Target, true)
 	case "docker-check":
 		output = updateAgentContainer(command.Target, false)
+	case "docker-remove":
+		output = removeAgentContainer(command.Target)
 	case "system-update":
 		output = updateAgentSystem()
+	case "system-reboot":
+		output = scheduleAgentReboot()
 	case "updates-refresh":
 		guestInventory.Lock()
 		guestInventory.value.CollectedAt = time.Time{}
@@ -189,26 +192,78 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
 	return result
 }
 
-func updateAgentSystem() string {
-	if output := runAgentCombinedCommand(3*time.Minute, "apt-get", "update"); strings.HasPrefix(output, "ERROR: ") {
-		return output
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
-	defer cancel()
-	command := exec.CommandContext(ctx, "apt-get", "-y", "upgrade")
-	command.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive")
-	output, err := command.CombinedOutput()
-	message := strings.TrimSpace(string(output))
-	if len(message) > 3500 {
-		message = message[len(message)-3500:]
-	}
+func removeAgentContainer(target string) string {
+	transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
+	}}
+	defer transport.CloseIdleConnections()
+	request, _ := http.NewRequest(http.MethodDelete, "http://docker/containers/"+target+"?v=false&force=false", nil)
+	response, err := (&http.Client{Transport: transport, Timeout: 20 * time.Second}).Do(request)
 	if err != nil {
-		return "ERROR: " + err.Error() + ": " + message
+		return "ERROR: " + err.Error()
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusNoContent {
+		message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))
+		return fmt.Sprintf("ERROR: Docker вернул %s: %s", response.Status, strings.TrimSpace(string(message)))
+	}
+	return "Остановленный контейнер удалён; volumes сохранены"
+}
+
+func updateAgentSystem() string {
+	aptGet, err := exec.LookPath("apt-get")
+	if err != nil {
+		return "ERROR: apt-get не найден; обновление поддерживается для Debian/Ubuntu"
+	}
+	if output := runPrivilegedTransient(5*time.Minute, "update-lists", aptGet, "update"); strings.HasPrefix(output, "ERROR: ") {
+		return explainSystemUpdateError(output)
+	}
+	message := runPrivilegedTransient(30*time.Minute, "upgrade", aptGet, "-y", "upgrade")
+	if strings.HasPrefix(message, "ERROR: ") {
+		return explainSystemUpdateError(message)
 	}
 	guestInventory.Lock()
 	guestInventory.value.CollectedAt = time.Time{}
 	guestInventory.Unlock()
-	return firstNonEmpty(message, "Обновления системы установлены")
+	return "Обновления системы установлены; список пакетов будет обновлён в следующем отчёте агента"
+}
+
+func explainSystemUpdateError(message string) string {
+	if strings.Contains(strings.ToLower(message), "read-only file system") {
+		return "ERROR: файловая система агента доступна только для чтения. Обновите Dashboard и агент до v0.19.2 или новее, затем повторите операцию."
+	}
+	if len(message) > 1800 {
+		message = message[len(message)-1800:]
+	}
+	return message
+}
+
+func runPrivilegedTransient(timeout time.Duration, suffix, name string, args ...string) string {
+	systemdRun, err := exec.LookPath("systemd-run")
+	if err != nil {
+		return "ERROR: systemd-run не найден; невозможно безопасно выйти из read-only sandbox агента"
+	}
+	unit := fmt.Sprintf("proxmox-dashboard-%s-%d", suffix, time.Now().UnixNano())
+	commandArgs := []string{"--quiet", "--wait", "--collect", "--pipe", "--unit=" + unit, "--setenv=DEBIAN_FRONTEND=noninteractive", name}
+	commandArgs = append(commandArgs, args...)
+	return runAgentCombinedCommand(timeout, systemdRun, commandArgs...)
+}
+
+func scheduleAgentReboot() string {
+	systemdRun, err := exec.LookPath("systemd-run")
+	if err != nil {
+		return "ERROR: systemd-run не найден"
+	}
+	systemctl, err := exec.LookPath("systemctl")
+	if err != nil {
+		return "ERROR: systemctl не найден"
+	}
+	unit := fmt.Sprintf("proxmox-dashboard-reboot-%d", time.Now().UnixNano())
+	output := runAgentCombinedCommand(15*time.Second, systemdRun, "--quiet", "--collect", "--unit="+unit, "--on-active=20s", systemctl, "reboot")
+	if strings.HasPrefix(output, "ERROR: ") {
+		return output
+	}
+	return "Перезагрузка запланирована через 20 секунд; кратковременный offline агента ожидаем"
 }
 
 func updateAgentContainer(target string, recreate bool) string {

agent_actions_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 	"time"
 )
@@ -12,7 +13,7 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer store.Close()
-	report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea"}}})
+	report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea", State: "running"}, {ID: "old123", Name: "old", State: "exited"}}})
 	_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
 	if err != nil {
 		t.Fatal(err)
@@ -26,9 +27,12 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
 	if _, err = store.QueueAgentAction("a", "system-update", ""); err != nil {
 		t.Fatalf("system update rejected: %v", err)
 	}
-	action, err := store.QueueAgentAction("a", "systemd-restart", "gitea.service")
+	if _, err = store.QueueAgentAction("a", "docker-remove", "gitea"); err == nil {
+		t.Fatal("running/unknown-state container accepted for removal")
+	}
+	action, err := store.QueueAgentAction("a", "docker-remove", "old")
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("stopped container rejected for removal: %v", err)
 	}
 	commands, err := store.DispatchAgentActions("a")
 	if err != nil || len(commands) != 3 || commands[2].ID != action.ID {
@@ -39,6 +43,12 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
 	if err != nil || len(items) != 3 || items[0].Status != "completed" {
 		t.Fatalf("items=%#v err=%v", items, err)
 	}
+	if _, err = store.QueueAgentAction("a", "system-reboot", ""); err != nil {
+		t.Fatalf("system reboot rejected: %v", err)
+	}
+	if _, err = store.QueueAgentAction("a", "system-reboot", "now"); err == nil {
+		t.Fatal("system reboot with unexpected target accepted")
+	}
 }
 
 func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
@@ -58,3 +68,10 @@ func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
 		t.Fatalf("group alert missing: %#v", grouped)
 	}
 }
+
+func TestExplainReadOnlySystemUpdateError(t *testing.T) {
+	message := explainSystemUpdateError("ERROR: apt failed: Read-only file system")
+	if !strings.Contains(message, "v0.19.2") {
+		t.Fatalf("read-only failure is not actionable: %s", message)
+	}
+}

agent_diagnostics.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -25,6 +26,8 @@ type AgentProcess struct {
 	Name        string  `json:"name"`
 	State       string  `json:"state"`
 	CPUPercent  float64 `json:"cpuPercent"`
+	CPUCores    float64 `json:"cpuCores"`
+	HostPercent float64 `json:"hostPercent"`
 	MemoryBytes uint64  `json:"memoryBytes"`
 	ReadBytes   uint64  `json:"readBytes"`
 	WriteBytes  uint64  `json:"writeBytes"`
@@ -174,7 +177,11 @@ func collectAgentProcesses() []AgentProcess {
 			cpu, _ := strconv.ParseFloat(fields[3], 64)
 			rss, _ := strconv.ParseUint(fields[4], 10, 64)
 			elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
-			item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
+			cpuCount := runtime.NumCPU()
+			if cpuCount < 1 {
+				cpuCount = 1
+			}
+			item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, CPUCores: cpu / 100, HostPercent: cpu / float64(cpuCount), MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
 			item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
 			result = append(result, item)
 			added++

agent_test.go

@@ -53,6 +53,12 @@ func TestAgentOnlyInstallsNewerVersion(t *testing.T) {
 	}
 }
 
+func TestShortDockerDigest(t *testing.T) {
+	if got := shortDockerDigest("sha256:1234567890abcdef"); got != "1234567890ab" {
+		t.Fatalf("unexpected short digest: %q", got)
+	}
+}
+
 func TestParseAgentIssues(t *testing.T) {
 	issues := parseAgentIssues(`{"MESSAGE":"disk error","_SYSTEMD_UNIT":"worker.service","__REALTIME_TIMESTAMP":"1722960000000000"}` + "\n" + `{"MESSAGE":"network error","__REALTIME_TIMESTAMP":"1722960001000000"}`)
 	if len(issues) != 2 || issues[0].Unit != "worker.service" || issues[1].Unit != "system" || issues[0].Time != 1722960000 {