Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1296d052a5 | ||
|
|
f764195ba5 | ||
|
|
f6498a2fc1 | ||
|
|
714c29c449 |
83
agent.go
83
agent.go
@@ -13,6 +13,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -40,16 +41,22 @@ type agentConfig struct {
|
||||
}
|
||||
|
||||
type AgentContainer struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image"`
|
||||
State string `json:"state"`
|
||||
Status string `json:"status"`
|
||||
Health string `json:"health,omitempty"`
|
||||
Restarts int64 `json:"restarts"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryUsage uint64 `json:"memoryUsage"`
|
||||
Ports []string `json:"ports,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Image string `json:"image"`
|
||||
State string `json:"state"`
|
||||
Status string `json:"status"`
|
||||
Health string `json:"health,omitempty"`
|
||||
Restarts int64 `json:"restarts"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryUsage uint64 `json:"memoryUsage"`
|
||||
Ports []string `json:"ports,omitempty"`
|
||||
Compose bool `json:"compose"`
|
||||
UpdateAvailable bool `json:"updateAvailable"`
|
||||
CurrentVersion string `json:"currentVersion,omitempty"`
|
||||
CurrentDigest string `json:"currentDigest,omitempty"`
|
||||
AvailableVersion string `json:"availableVersion,omitempty"`
|
||||
AvailableDigest string `json:"availableDigest,omitempty"`
|
||||
}
|
||||
|
||||
type AgentInterface struct {
|
||||
@@ -96,6 +103,7 @@ type AgentReport struct {
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
CPUCount int `json:"cpuCount"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
SwapTotal uint64 `json:"swapTotal"`
|
||||
@@ -391,7 +399,9 @@ func runGuestAgent(opts agentOptions) error {
|
||||
if requestErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "agent: %v\n", requestErr)
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
// Короткий интервал нужен для интерактивных команд из панели. Тяжёлый
|
||||
// системный инвентарь отдельно кэшируется и не запускается каждые 3 секунды.
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -527,7 +537,7 @@ func saveAgentConfig(path string, c agentConfig) error {
|
||||
func collectAgentReport() AgentReport {
|
||||
hostname, _ := os.Hostname()
|
||||
machineID, _ := os.ReadFile("/etc/machine-id")
|
||||
report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CollectedAt: time.Now().Unix()}
|
||||
report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CPUCount: runtime.NumCPU(), CollectedAt: time.Now().Unix()}
|
||||
if data, err := os.ReadFile("/proc/uptime"); err == nil {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) > 0 {
|
||||
@@ -913,14 +923,23 @@ func collectDockerContainers() ([]AgentContainer, string) {
|
||||
inspect, inspectErr := client.Get("http://docker/containers/" + v.ID + "/json")
|
||||
if inspectErr == nil {
|
||||
var detail struct {
|
||||
Image string
|
||||
RestartCount int64
|
||||
State struct{ Health *struct{ Status string } }
|
||||
Config struct {
|
||||
Labels map[string]string
|
||||
}
|
||||
}
|
||||
if json.NewDecoder(inspect.Body).Decode(&detail) == nil {
|
||||
item.Restarts = detail.RestartCount
|
||||
if detail.State.Health != nil {
|
||||
item.Health = detail.State.Health.Status
|
||||
}
|
||||
item.Compose = detail.Config.Labels["com.docker.compose.service"] != ""
|
||||
item.CurrentVersion, item.CurrentDigest, _ = inspectDockerImage(client, detail.Image, v.Image)
|
||||
var availableID string
|
||||
item.AvailableVersion, item.AvailableDigest, availableID = inspectDockerImage(client, v.Image, v.Image)
|
||||
item.UpdateAvailable = detail.Image != "" && availableID != "" && detail.Image != availableID
|
||||
}
|
||||
inspect.Body.Close()
|
||||
}
|
||||
@@ -971,6 +990,46 @@ func collectDockerContainers() ([]AgentContainer, string) {
|
||||
}
|
||||
return result, ""
|
||||
}
|
||||
|
||||
func inspectDockerImage(client *http.Client, reference, fallback string) (string, string, string) {
|
||||
if reference == "" {
|
||||
return "", "", ""
|
||||
}
|
||||
response, err := client.Get("http://docker/images/" + url.PathEscape(reference) + "/json")
|
||||
if err != nil {
|
||||
return "", "", ""
|
||||
}
|
||||
defer response.Body.Close()
|
||||
var detail struct {
|
||||
ID string
|
||||
RepoTags []string
|
||||
RepoDigests []string
|
||||
Config struct{ Labels map[string]string }
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
|
||||
return "", "", ""
|
||||
}
|
||||
version := firstNonEmpty(detail.Config.Labels["org.opencontainers.image.version"], detail.Config.Labels["version"], firstString(detail.RepoTags), fallback, shortDockerDigest(detail.ID))
|
||||
digest := ""
|
||||
if len(detail.RepoDigests) > 0 {
|
||||
digest = detail.RepoDigests[0]
|
||||
if _, value, found := strings.Cut(digest, "@"); found {
|
||||
digest = value
|
||||
}
|
||||
}
|
||||
if digest == "" {
|
||||
digest = detail.ID
|
||||
}
|
||||
return version, shortDockerDigest(digest), detail.ID
|
||||
}
|
||||
|
||||
func shortDockerDigest(value string) string {
|
||||
value = strings.TrimPrefix(value, "sha256:")
|
||||
if len(value) > 12 {
|
||||
value = value[:12]
|
||||
}
|
||||
return value
|
||||
}
|
||||
func firstString(v []string) string {
|
||||
if len(v) > 0 {
|
||||
return v[0]
|
||||
|
||||
144
agent_actions.go
144
agent_actions.go
@@ -9,6 +9,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -61,16 +62,16 @@ func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, e
|
||||
}
|
||||
}
|
||||
}
|
||||
case "docker-restart":
|
||||
case "docker-restart", "docker-check", "docker-update", "docker-remove":
|
||||
if safeContainerName.MatchString(target) {
|
||||
for _, container := range report.Containers {
|
||||
if container.ID == target || container.Name == target {
|
||||
if (container.ID == target || container.Name == target) && (action != "docker-remove" || container.State != "running") {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
case "updates-refresh":
|
||||
case "updates-refresh", "system-update", "system-reboot":
|
||||
valid = target == ""
|
||||
}
|
||||
if !valid {
|
||||
@@ -91,7 +92,9 @@ func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-5*time.Minute).Unix())
|
||||
// Обновление пакетов и Docker-образов может быть долгим. Не выдаём одну
|
||||
// команду повторно, пока предыдущий запуск ещё имеет шанс завершиться.
|
||||
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-45*time.Minute).Unix())
|
||||
rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -160,6 +163,16 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
|
||||
output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
|
||||
case "docker-restart":
|
||||
output = restartAgentContainer(command.Target)
|
||||
case "docker-update":
|
||||
output = updateAgentContainer(command.Target, true)
|
||||
case "docker-check":
|
||||
output = updateAgentContainer(command.Target, false)
|
||||
case "docker-remove":
|
||||
output = removeAgentContainer(command.Target)
|
||||
case "system-update":
|
||||
output = updateAgentSystem()
|
||||
case "system-reboot":
|
||||
output = scheduleAgentReboot()
|
||||
case "updates-refresh":
|
||||
guestInventory.Lock()
|
||||
guestInventory.value.CollectedAt = time.Time{}
|
||||
@@ -179,6 +192,129 @@ func executeAgentCommand(command AgentCommand) AgentActionResult {
|
||||
return result
|
||||
}
|
||||
|
||||
func removeAgentContainer(target string) string {
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||
}}
|
||||
defer transport.CloseIdleConnections()
|
||||
request, _ := http.NewRequest(http.MethodDelete, "http://docker/containers/"+target+"?v=false&force=false", nil)
|
||||
response, err := (&http.Client{Transport: transport, Timeout: 20 * time.Second}).Do(request)
|
||||
if err != nil {
|
||||
return "ERROR: " + err.Error()
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusNoContent {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))
|
||||
return fmt.Sprintf("ERROR: Docker вернул %s: %s", response.Status, strings.TrimSpace(string(message)))
|
||||
}
|
||||
return "Остановленный контейнер удалён; volumes сохранены"
|
||||
}
|
||||
|
||||
func updateAgentSystem() string {
|
||||
aptGet, err := exec.LookPath("apt-get")
|
||||
if err != nil {
|
||||
return "ERROR: apt-get не найден; обновление поддерживается для Debian/Ubuntu"
|
||||
}
|
||||
if output := runPrivilegedTransient(5*time.Minute, "update-lists", aptGet, "update"); strings.HasPrefix(output, "ERROR: ") {
|
||||
return explainSystemUpdateError(output)
|
||||
}
|
||||
message := runPrivilegedTransient(30*time.Minute, "upgrade", aptGet, "-y", "upgrade")
|
||||
if strings.HasPrefix(message, "ERROR: ") {
|
||||
return explainSystemUpdateError(message)
|
||||
}
|
||||
guestInventory.Lock()
|
||||
guestInventory.value.CollectedAt = time.Time{}
|
||||
guestInventory.Unlock()
|
||||
return "Обновления системы установлены; список пакетов будет обновлён в следующем отчёте агента"
|
||||
}
|
||||
|
||||
func explainSystemUpdateError(message string) string {
|
||||
if strings.Contains(strings.ToLower(message), "read-only file system") {
|
||||
return "ERROR: файловая система агента доступна только для чтения. Обновите Dashboard и агент до v0.19.2 или новее, затем повторите операцию."
|
||||
}
|
||||
if len(message) > 1800 {
|
||||
message = message[len(message)-1800:]
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func runPrivilegedTransient(timeout time.Duration, suffix, name string, args ...string) string {
|
||||
systemdRun, err := exec.LookPath("systemd-run")
|
||||
if err != nil {
|
||||
return "ERROR: systemd-run не найден; невозможно безопасно выйти из read-only sandbox агента"
|
||||
}
|
||||
unit := fmt.Sprintf("proxmox-dashboard-%s-%d", suffix, time.Now().UnixNano())
|
||||
commandArgs := []string{"--quiet", "--wait", "--collect", "--pipe", "--unit=" + unit, "--setenv=DEBIAN_FRONTEND=noninteractive", name}
|
||||
commandArgs = append(commandArgs, args...)
|
||||
return runAgentCombinedCommand(timeout, systemdRun, commandArgs...)
|
||||
}
|
||||
|
||||
func scheduleAgentReboot() string {
|
||||
systemdRun, err := exec.LookPath("systemd-run")
|
||||
if err != nil {
|
||||
return "ERROR: systemd-run не найден"
|
||||
}
|
||||
systemctl, err := exec.LookPath("systemctl")
|
||||
if err != nil {
|
||||
return "ERROR: systemctl не найден"
|
||||
}
|
||||
unit := fmt.Sprintf("proxmox-dashboard-reboot-%d", time.Now().UnixNano())
|
||||
output := runAgentCombinedCommand(15*time.Second, systemdRun, "--quiet", "--collect", "--unit="+unit, "--on-active=20s", systemctl, "reboot")
|
||||
if strings.HasPrefix(output, "ERROR: ") {
|
||||
return output
|
||||
}
|
||||
return "Перезагрузка запланирована через 20 секунд; кратковременный offline агента ожидаем"
|
||||
}
|
||||
|
||||
func updateAgentContainer(target string, recreate bool) string {
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||
}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
|
||||
response, err := client.Get("http://docker/containers/" + target + "/json")
|
||||
if err != nil {
|
||||
return "ERROR: " + err.Error()
|
||||
}
|
||||
defer response.Body.Close()
|
||||
var detail struct {
|
||||
Config struct {
|
||||
Labels map[string]string `json:"Labels"`
|
||||
} `json:"Config"`
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil {
|
||||
return "ERROR: не удалось получить Compose-конфигурацию контейнера"
|
||||
}
|
||||
labels := detail.Config.Labels
|
||||
service := labels["com.docker.compose.service"]
|
||||
workdir := labels["com.docker.compose.project.working_dir"]
|
||||
files := strings.Split(labels["com.docker.compose.project.config_files"], ",")
|
||||
if service == "" || workdir == "" || len(files) == 0 || !filepath.IsAbs(workdir) || !filepath.IsAbs(files[0]) {
|
||||
return "ERROR: безопасное обновление доступно только для Docker Compose"
|
||||
}
|
||||
if _, err = exec.LookPath("docker"); err != nil {
|
||||
return "ERROR: Docker CLI не найден"
|
||||
}
|
||||
commands := [][]string{{"compose", "-f", files[0], "pull", service}}
|
||||
if recreate {
|
||||
commands = append(commands, []string{"compose", "-f", files[0], "up", "-d", "--no-deps", service})
|
||||
}
|
||||
for _, args := range commands {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
command := exec.CommandContext(ctx, "docker", args...)
|
||||
command.Dir = workdir
|
||||
output, commandErr := command.CombinedOutput()
|
||||
cancel()
|
||||
if commandErr != nil {
|
||||
return "ERROR: " + commandErr.Error() + ": " + strings.TrimSpace(string(output))
|
||||
}
|
||||
}
|
||||
if recreate {
|
||||
return "Образ загружен, Compose-контейнер пересоздан"
|
||||
}
|
||||
return "Проверка завершена: образ загружен; если digest изменился, карточка покажет доступное обновление"
|
||||
}
|
||||
|
||||
func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -12,7 +13,7 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea"}}})
|
||||
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea", State: "running"}, {ID: "old123", Name: "old", State: "exited"}}})
|
||||
_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -20,19 +21,34 @@ func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
|
||||
if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
|
||||
t.Fatal("unsafe unit accepted")
|
||||
}
|
||||
action, err := store.QueueAgentAction("a", "systemd-restart", "gitea.service")
|
||||
if _, err = store.QueueAgentAction("a", "docker-update", "gitea"); err != nil {
|
||||
t.Fatalf("known Docker container rejected: %v", err)
|
||||
}
|
||||
if _, err = store.QueueAgentAction("a", "system-update", ""); err != nil {
|
||||
t.Fatalf("system update rejected: %v", err)
|
||||
}
|
||||
if _, err = store.QueueAgentAction("a", "docker-remove", "gitea"); err == nil {
|
||||
t.Fatal("running/unknown-state container accepted for removal")
|
||||
}
|
||||
action, err := store.QueueAgentAction("a", "docker-remove", "old")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("stopped container rejected for removal: %v", err)
|
||||
}
|
||||
commands, err := store.DispatchAgentActions("a")
|
||||
if err != nil || len(commands) != 1 || commands[0].ID != action.ID {
|
||||
if err != nil || len(commands) != 3 || commands[2].ID != action.ID {
|
||||
t.Fatalf("commands=%#v err=%v", commands, err)
|
||||
}
|
||||
store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
|
||||
items, err := store.AgentActions(10)
|
||||
if err != nil || len(items) != 1 || items[0].Status != "completed" {
|
||||
if err != nil || len(items) != 3 || items[0].Status != "completed" {
|
||||
t.Fatalf("items=%#v err=%v", items, err)
|
||||
}
|
||||
if _, err = store.QueueAgentAction("a", "system-reboot", ""); err != nil {
|
||||
t.Fatalf("system reboot rejected: %v", err)
|
||||
}
|
||||
if _, err = store.QueueAgentAction("a", "system-reboot", "now"); err == nil {
|
||||
t.Fatal("system reboot with unexpected target accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
|
||||
@@ -52,3 +68,10 @@ func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
|
||||
t.Fatalf("group alert missing: %#v", grouped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplainReadOnlySystemUpdateError(t *testing.T) {
|
||||
message := explainSystemUpdateError("ERROR: apt failed: Read-only file system")
|
||||
if !strings.Contains(message, "v0.19.2") {
|
||||
t.Fatalf("read-only failure is not actionable: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -25,6 +26,8 @@ type AgentProcess struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
CPUCores float64 `json:"cpuCores"`
|
||||
HostPercent float64 `json:"hostPercent"`
|
||||
MemoryBytes uint64 `json:"memoryBytes"`
|
||||
ReadBytes uint64 `json:"readBytes"`
|
||||
WriteBytes uint64 `json:"writeBytes"`
|
||||
@@ -174,7 +177,11 @@ func collectAgentProcesses() []AgentProcess {
|
||||
cpu, _ := strconv.ParseFloat(fields[3], 64)
|
||||
rss, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
|
||||
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
|
||||
cpuCount := runtime.NumCPU()
|
||||
if cpuCount < 1 {
|
||||
cpuCount = 1
|
||||
}
|
||||
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, CPUCores: cpu / 100, HostPercent: cpu / float64(cpuCount), MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
|
||||
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
|
||||
result = append(result, item)
|
||||
added++
|
||||
|
||||
@@ -53,6 +53,12 @@ func TestAgentOnlyInstallsNewerVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortDockerDigest(t *testing.T) {
|
||||
if got := shortDockerDigest("sha256:1234567890abcdef"); got != "1234567890ab" {
|
||||
t.Fatalf("unexpected short digest: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAgentIssues(t *testing.T) {
|
||||
issues := parseAgentIssues(`{"MESSAGE":"disk error","_SYSTEMD_UNIT":"worker.service","__REALTIME_TIMESTAMP":"1722960000000000"}` + "\n" + `{"MESSAGE":"network error","__REALTIME_TIMESTAMP":"1722960001000000"}`)
|
||||
if len(issues) != 2 || issues[0].Unit != "worker.service" || issues[1].Unit != "system" || issues[0].Time != 1722960000 {
|
||||
|
||||
19
alerts.go
19
alerts.go
@@ -74,7 +74,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
|
||||
if disk.SMARTStatus == "Ошибка" {
|
||||
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
|
||||
} else if disk.SMARTStatus == "Требует внимания" {
|
||||
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(disk.AttentionReasons, "Проверьте SMART-показатели диска."))
|
||||
reasons := withoutReason(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался")
|
||||
if len(reasons) > 0 {
|
||||
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(reasons, "Проверьте SMART-показатели диска."))
|
||||
}
|
||||
}
|
||||
if disk.Temperature != nil {
|
||||
limit := thresholds.DiskTempWarning
|
||||
@@ -260,10 +263,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
|
||||
}
|
||||
var networkErrors uint64
|
||||
for _, iface := range metrics.Network.Interfaces {
|
||||
networkErrors += iface.ReceiveErrors + iface.TransmitErrors + iface.ReceiveDrops + iface.TransmitDrops
|
||||
networkErrors += iface.NewErrors + iface.NewDrops
|
||||
}
|
||||
if networkErrors > 0 {
|
||||
add("network-errors", "warning", "Сеть", "Сетевые ошибки и потери пакетов", fmt.Sprintf("Суммарный счётчик интерфейсов: %d.", networkErrors))
|
||||
add("network-errors", "warning", "Сеть", "Появились новые сетевые ошибки или потери", fmt.Sprintf("За последний интервал добавилось: %d. Старые счётчики не создают активный алерт.", networkErrors))
|
||||
}
|
||||
if !metrics.Network.Health.CheckedAt.IsZero() && !metrics.Network.Health.GatewayUp {
|
||||
add("network-gateway", "critical", "Сеть", "Недоступен локальный шлюз", firstNonEmpty(metrics.Network.Health.Gateway, "Default gateway не определён."))
|
||||
@@ -386,3 +389,13 @@ func joinReasons(reasons []string, fallback string) string {
|
||||
}
|
||||
return strings.Join(reasons, " · ")
|
||||
}
|
||||
|
||||
func withoutReason(reasons []string, excluded string) []string {
|
||||
filtered := make([]string, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
if reason != excluded {
|
||||
filtered = append(filtered, reason)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
@@ -25,3 +25,17 @@ func TestSlowCollectionDoesNotCreateAlert(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeverRunSelfTestCreatesOneAlert(t *testing.T) {
|
||||
metrics := dashboardMetrics{Disks: []DiskMetrics{{
|
||||
Name: "nvme0n1",
|
||||
Model: "Test NVMe",
|
||||
SMARTStatus: "Требует внимания",
|
||||
SelfTestNeverRun: true,
|
||||
AttentionReasons: []string{"SMART self-test ещё ни разу не запускался"},
|
||||
}}}
|
||||
alerts := evaluateAlerts(metrics, defaultThresholds())
|
||||
if len(alerts) != 1 || alerts[0].ID != "disk-selftest-nvme0n1" {
|
||||
t.Fatalf("ожидалось одно точное уведомление о self-test, получено: %+v", alerts)
|
||||
}
|
||||
}
|
||||
|
||||
19
cpu.go
19
cpu.go
@@ -33,14 +33,15 @@ type CPUStatic struct {
|
||||
|
||||
type CPUMetrics struct {
|
||||
CPUStatic
|
||||
UsagePercent float64 `json:"usagePercent"`
|
||||
FrequencyMHz float64 `json:"frequencyMhz"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
Uptime uint64 `json:"uptimeSeconds"`
|
||||
Temperatures []Temperature `json:"temperatures"`
|
||||
CollectedAt time.Time `json:"collectedAt"`
|
||||
UsagePercent float64 `json:"usagePercent"`
|
||||
FrequencyMHz float64 `json:"frequencyMhz"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
Uptime uint64 `json:"uptimeSeconds"`
|
||||
Temperatures []Temperature `json:"temperatures"`
|
||||
TopProcesses []AgentProcess `json:"topProcesses"`
|
||||
CollectedAt time.Time `json:"collectedAt"`
|
||||
}
|
||||
|
||||
type Temperature struct {
|
||||
@@ -92,7 +93,7 @@ func (c *cpuCollector) collect() (CPUMetrics, error) {
|
||||
return CPUMetrics{
|
||||
CPUStatic: c.static, UsagePercent: usage, FrequencyMHz: frequency,
|
||||
Load1: load1, Load5: load5, Load15: load15, Uptime: uptime,
|
||||
Temperatures: readTemperatures(), CollectedAt: time.Now(),
|
||||
Temperatures: readTemperatures(), TopProcesses: collectAgentProcesses(), CollectedAt: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
37
disks.go
37
disks.go
@@ -391,6 +391,17 @@ type smartctlJSON struct {
|
||||
} `json:"table"`
|
||||
} `json:"standard"`
|
||||
} `json:"ata_smart_self_test_log"`
|
||||
NVMeSelfTestLog struct {
|
||||
Table []struct {
|
||||
SelfTestCode struct {
|
||||
String string `json:"string"`
|
||||
} `json:"self_test_code"`
|
||||
SelfTestResult struct {
|
||||
String string `json:"string"`
|
||||
} `json:"self_test_result"`
|
||||
PowerOnHours uint64 `json:"power_on_hours"`
|
||||
} `json:"table"`
|
||||
} `json:"nvme_self_test_log"`
|
||||
NVMeHealth *struct {
|
||||
Temperature float64 `json:"temperature"`
|
||||
PercentageUsed float64 `json:"percentage_used"`
|
||||
@@ -410,6 +421,10 @@ func applySMART(disk *DiskMetrics) {
|
||||
if len(output) == 0 {
|
||||
return
|
||||
}
|
||||
applySMARTJSON(disk, output)
|
||||
}
|
||||
|
||||
func applySMARTJSON(disk *DiskMetrics, output []byte) {
|
||||
var smart smartctlJSON
|
||||
if json.Unmarshal(output, &smart) != nil {
|
||||
return
|
||||
@@ -451,12 +466,20 @@ func applySMART(disk *DiskMetrics) {
|
||||
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
|
||||
}
|
||||
}
|
||||
if len(smart.ATASelfTestLog.Standard.Table) > 0 {
|
||||
if len(smart.NVMeSelfTestLog.Table) > 0 {
|
||||
last := smart.NVMeSelfTestLog.Table[0]
|
||||
description := strings.TrimSpace(last.SelfTestCode.String)
|
||||
result := firstNonEmpty(strings.TrimSpace(last.SelfTestResult.String), "Результат неизвестен")
|
||||
if description != "" {
|
||||
disk.LastSelfTest = description + " · " + result
|
||||
} else {
|
||||
disk.LastSelfTest = result
|
||||
}
|
||||
setLastSelfTestAt(disk, last.PowerOnHours)
|
||||
} else if len(smart.ATASelfTestLog.Standard.Table) > 0 {
|
||||
last := smart.ATASelfTestLog.Standard.Table[0]
|
||||
disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
|
||||
if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours {
|
||||
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339)
|
||||
}
|
||||
setLastSelfTestAt(disk, last.LifetimeHours)
|
||||
} else {
|
||||
disk.SelfTestNeverRun = true
|
||||
disk.LastSelfTest = "Никогда не запускался"
|
||||
@@ -469,6 +492,12 @@ func applySMART(disk *DiskMetrics) {
|
||||
}
|
||||
}
|
||||
|
||||
func setLastSelfTestAt(disk *DiskMetrics, testPowerOnHours uint64) {
|
||||
if disk.PowerOnHours != nil && *disk.PowerOnHours >= testPowerOnHours {
|
||||
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-testPowerOnHours) * time.Hour).Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
|
||||
func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
|
||||
wearNames := map[string]bool{
|
||||
"Percent_Lifetime_Remain": true, "SSD_Life_Left": true,
|
||||
|
||||
@@ -6,8 +6,44 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestApplySMARTJSONReadsNVMeSelfTestHistory(t *testing.T) {
|
||||
disk := DiskMetrics{Path: "/dev/nvme0n1"}
|
||||
output := []byte(`{
|
||||
"model_name":"ADATA LEGEND 900",
|
||||
"smart_status":{"passed":true},
|
||||
"power_on_time":{"hours":960},
|
||||
"nvme_self_test_log":{
|
||||
"current_self_test_operation":{"value":0,"string":"No self-test in progress"},
|
||||
"table":[{
|
||||
"self_test_code":{"value":1,"string":"Short"},
|
||||
"self_test_result":{"value":0,"string":"Completed without error"},
|
||||
"power_on_hours":959
|
||||
}]
|
||||
}
|
||||
}`)
|
||||
|
||||
before := time.Now().Add(-2 * time.Hour)
|
||||
applySMARTJSON(&disk, output)
|
||||
after := time.Now()
|
||||
|
||||
if disk.SelfTestNeverRun {
|
||||
t.Fatal("завершённый NVMe self-test ошибочно отмечен как никогда не запускавшийся")
|
||||
}
|
||||
if disk.LastSelfTest != "Short · Completed without error" {
|
||||
t.Fatalf("неожиданный результат self-test: %q", disk.LastSelfTest)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, disk.LastSelfTestAt)
|
||||
if err != nil || parsed.Before(before) || parsed.After(after) {
|
||||
t.Fatalf("неверно вычислено время self-test: %q, err=%v", disk.LastSelfTestAt, err)
|
||||
}
|
||||
if len(disk.AttentionReasons) != 0 || disk.SMARTStatus != "Исправен" {
|
||||
t.Fatalf("исправный NVMe требует внимания: %+v", disk)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPhysicalDiskName(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
"sda": true, "sdb": true, "nvme0n1": true,
|
||||
|
||||
@@ -17,6 +17,51 @@ type MaintenanceWindow struct {
|
||||
AllAlerts bool `json:"allAlerts"`
|
||||
ServiceIDs []int64 `json:"serviceIds"`
|
||||
VMIDs []int `json:"vmids"`
|
||||
Automatic bool `json:"automatic"`
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
func automaticBackupMaintenance(activity ProxmoxActivity, links []ServiceLink, now time.Time) MaintenanceWindow {
|
||||
vmids := []int{}
|
||||
backupRunning := false
|
||||
for _, task := range activity.Tasks {
|
||||
if task.Type != "vzdump" || task.EndTime > 0 || (task.Status != "" && !strings.EqualFold(task.Status, "running")) {
|
||||
continue
|
||||
}
|
||||
backupRunning = true
|
||||
if vmid := backupTaskVMID(task); vmid > 0 {
|
||||
vmids = append(vmids, vmid)
|
||||
}
|
||||
}
|
||||
vmids = uniqueInt(vmids)
|
||||
if !backupRunning {
|
||||
return MaintenanceWindow{}
|
||||
}
|
||||
selected := map[int]bool{}
|
||||
for _, vmid := range vmids {
|
||||
selected[vmid] = true
|
||||
}
|
||||
serviceIDs := []int64{}
|
||||
for _, link := range links {
|
||||
if selected[link.VMID] {
|
||||
serviceIDs = append(serviceIDs, link.ServiceID)
|
||||
}
|
||||
}
|
||||
return MaintenanceWindow{Enabled: true, Active: true, StartedAt: now.Unix(), EndsAt: now.Add(2 * time.Minute).Unix(), Reason: "Автоматически: выполняется backup VM/LXC", VMIDs: vmids, ServiceIDs: uniqueInt64(serviceIDs), AllAlerts: len(vmids) == 0, Automatic: true, Source: "backup"}
|
||||
}
|
||||
|
||||
func backupTaskVMID(task ProxmoxTask) int {
|
||||
if vmid, err := strconv.Atoi(strings.TrimSpace(task.ID)); err == nil && vmid > 0 {
|
||||
return vmid
|
||||
}
|
||||
parts := strings.Split(task.UPID, ":")
|
||||
for i, part := range parts {
|
||||
if part == "vzdump" && i+1 < len(parts) {
|
||||
vmid, _ := strconv.Atoi(parts[i+1])
|
||||
return vmid
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *Store) MaintenanceWindow() MaintenanceWindow {
|
||||
|
||||
@@ -38,6 +38,21 @@ func TestMaintenanceExpiresAutomatically(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningBackupCreatesScopedAutomaticMaintenance(t *testing.T) {
|
||||
now := time.Now()
|
||||
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", ID: "105"}, {Type: "vzdump", ID: "106", EndTime: now.Unix()}}}, []ServiceLink{{ServiceID: 7, VMID: 105}, {ServiceID: 8, VMID: 999}}, now)
|
||||
if !window.Active || !window.Automatic || window.Source != "backup" || len(window.VMIDs) != 1 || window.VMIDs[0] != 105 || len(window.ServiceIDs) != 1 || window.ServiceIDs[0] != 7 {
|
||||
t.Fatalf("неверное автоматическое окно: %+v", window)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningBackupWithoutVMIDSuppressesAllAlerts(t *testing.T) {
|
||||
window := automaticBackupMaintenance(ProxmoxActivity{Tasks: []ProxmoxTask{{Type: "vzdump", UPID: "unknown"}}}, nil, time.Now())
|
||||
if !window.Active || !window.AllAlerts {
|
||||
t.Fatalf("backup с неизвестной целью должен безопасно подавить алерты: %+v", window)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlannedAlertOnlyAppearsInHistory(t *testing.T) {
|
||||
store, err := openStore(":memory:")
|
||||
if err != nil {
|
||||
|
||||
41
network.go
41
network.go
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -36,6 +37,9 @@ type NetworkHealth struct {
|
||||
ExternalIPChanged bool `json:"externalIpChanged"`
|
||||
DisappearedInterfaces []string `json:"disappearedInterfaces"`
|
||||
CheckedAt time.Time `json:"checkedAt"`
|
||||
Summary string `json:"summary"`
|
||||
LikelyCause string `json:"likelyCause"`
|
||||
RecommendedAction string `json:"recommendedAction"`
|
||||
}
|
||||
|
||||
type NetworkInterface struct {
|
||||
@@ -46,6 +50,8 @@ type NetworkInterface struct {
|
||||
TransmitErrors uint64 `json:"transmitErrors"`
|
||||
ReceiveDrops uint64 `json:"receiveDrops"`
|
||||
TransmitDrops uint64 `json:"transmitDrops"`
|
||||
NewErrors uint64 `json:"newErrors"`
|
||||
NewDrops uint64 `json:"newDrops"`
|
||||
ReceivePerSec float64 `json:"receivePerSec"`
|
||||
TransmitPerSec float64 `json:"transmitPerSec"`
|
||||
VMID int `json:"vmid,omitempty"`
|
||||
@@ -113,6 +119,12 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
|
||||
if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes {
|
||||
v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed
|
||||
v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed
|
||||
if v.ReceiveErrors >= old.ReceiveErrors && v.TransmitErrors >= old.TransmitErrors {
|
||||
v.NewErrors = v.ReceiveErrors - old.ReceiveErrors + v.TransmitErrors - old.TransmitErrors
|
||||
}
|
||||
if v.ReceiveDrops >= old.ReceiveDrops && v.TransmitDrops >= old.TransmitDrops {
|
||||
v.NewDrops = v.ReceiveDrops - old.ReceiveDrops + v.TransmitDrops - old.TransmitDrops
|
||||
}
|
||||
}
|
||||
}
|
||||
c.previousInterfaces = map[string]NetworkInterface{}
|
||||
@@ -138,12 +150,41 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
|
||||
}
|
||||
}
|
||||
c.knownInterfaces = currentNames
|
||||
explainNetworkHealth(&c.health, details)
|
||||
return NetworkMetrics{
|
||||
Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
|
||||
TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func explainNetworkHealth(health *NetworkHealth, interfaces []NetworkInterface) {
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = "Сеть работает штатно", "Проблем не обнаружено", "Действий не требуется"
|
||||
switch {
|
||||
case health.Gateway == "":
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = "Не найден маршрут в локальную сеть", "На хосте отсутствует default gateway", "Проверьте адрес шлюза и конфигурацию bridge/VLAN"
|
||||
case !health.GatewayUp:
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = "Хост не видит локальный шлюз", "Проблема между Proxmox и роутером: линк, bridge, bond или VLAN", "Проверьте кабель, carrier и настройки vmbr/bond/VLAN"
|
||||
case health.PacketLoss > 0:
|
||||
health.Summary = fmt.Sprintf("Нестабильная локальная сеть: потеряно %.1f%% пакетов", health.PacketLoss)
|
||||
health.LikelyCause, health.RecommendedAction = "Перегрузка или ошибки линка между Proxmox и шлюзом", "Проверьте errors/drops, кабель, порт коммутатора и duplex"
|
||||
case !health.DNSUp && health.InternetUp:
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = "Интернет доступен, но DNS не отвечает", "Проблема DNS-сервера или /etc/resolv.conf", "Проверьте настроенные DNS и попробуйте разрешить домен вручную"
|
||||
case health.DNSUp && !health.InternetUp:
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = "Локальная сеть и DNS работают, но внешняя проверка недоступна", "Проблема выше роутера, блокировка HTTPS или недоступность провайдера", "Проверьте маршрут наружу, firewall и доступ к внешним адресам"
|
||||
}
|
||||
for _, v := range interfaces {
|
||||
if v.Kind == "Physical" && (!v.Carrier || v.State == "down") {
|
||||
health.Summary, health.LikelyCause, health.RecommendedAction = v.Name+": физический линк не поднят", "Нет carrier либо интерфейс административно выключен", "Проверьте кабель, порт коммутатора и состояние интерфейса"
|
||||
return
|
||||
}
|
||||
if v.NewErrors+v.NewDrops > 0 {
|
||||
health.Summary = fmt.Sprintf("%s: за последний интервал появилось %d сетевых ошибок/потерь", v.Name, v.NewErrors+v.NewDrops)
|
||||
health.LikelyCause, health.RecommendedAction = "Счётчики интерфейса сейчас растут из-за перегрузки, кабеля, MTU или duplex", "Проверьте скорость и duplex, кабель, порт коммутатора и MTU на обоих концах"
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decorateInterface(v *NetworkInterface) {
|
||||
base := filepath.Join("/sys/class/net", v.Name)
|
||||
v.State = readTrimmed(filepath.Join(base, "operstate"))
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -33,6 +34,26 @@ func TestReadNetworkSnapshot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplainNetworkHealthDistinguishesDNSFailure(t *testing.T) {
|
||||
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: false}
|
||||
explainNetworkHealth(&health, nil)
|
||||
if !strings.Contains(health.Summary, "DNS") || health.RecommendedAction == "" {
|
||||
t.Fatalf("непонятная диагностика: %+v", health)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplainNetworkHealthIgnoresOldInterfaceCounters(t *testing.T) {
|
||||
health := NetworkHealth{Gateway: "192.168.0.1", GatewayUp: true, InternetUp: true, DNSUp: true}
|
||||
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 42}})
|
||||
if health.Summary != "Сеть работает штатно" {
|
||||
t.Fatalf("старый накопленный счётчик вызвал текущую тревогу: %+v", health)
|
||||
}
|
||||
explainNetworkHealth(&health, []NetworkInterface{{Name: "eno1", Carrier: true, State: "up", ReceiveDrops: 43, NewDrops: 1}})
|
||||
if !strings.Contains(health.Summary, "последний интервал") {
|
||||
t.Fatalf("рост счётчика не объяснён: %+v", health)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterfaceVMID(t *testing.T) {
|
||||
for name, want := range map[string]int{"tap100i0": 100, "veth108i0": 108, "fwbr113i0": 113, "vmbr0": 0, "enp6s0": 0} {
|
||||
if got := interfaceVMID(name); got != want {
|
||||
|
||||
Reference in New Issue
Block a user