maxim/ProxmoxDash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity

Add guest root cause diagnostics

Browse Source
This commit is contained in:
Maxim
2026-08-07 21:23:05 +03:00
parent a2525b53f0
commit 338d53cb2f
6 changed files with 483 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@@ -347,6 +347,17 @@ health-check, число перезапусков, CPU и память. Аген
unit-файла, время запуска, число перезапусков, последний exit code, память и
накопленное CPU-время, когда соответствующий accounting доступен в systemd.
Расширенная диагностика агента включает последние сообщения проблемных служб,
`Requires`/`After`, PID, команду запуска и слушающие порты; выявляет частые
рестарты, OOM-kill и процессы в D-state. Отдельно отображаются top процессов по
CPU и RAM с накопленным I/O, swap, PSI pressure, все постоянные файловые системы,
их заполнение и использование inode.
Панель «Диагностика homelab» и карта сопоставляют известные IP и порты с VM/LXC,
агентом, systemd-службой или Docker-контейнером и настроенной HTTP/TCP-проверкой.
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
помечаются как предположение.
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
минут и после регистрации заменяется индивидуальным секретом агента.

114
agent.go
View File

@@ -58,18 +58,25 @@ type AgentInterface struct {
}
type AgentService struct {
Name string `json:"name"`
State string `json:"state"`
SubState string `json:"subState"`
Description string `json:"description"`
Origin string `json:"origin"`
UnitPath string `json:"unitPath,omitempty"`
Package string `json:"package,omitempty"`
Restarts uint64 `json:"restarts"`
ExitCode int `json:"exitCode"`
StartedAt int64 `json:"startedAt,omitempty"`
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
Name string `json:"name"`
State string `json:"state"`
SubState string `json:"subState"`
Description string `json:"description"`
Origin string `json:"origin"`
UnitPath string `json:"unitPath,omitempty"`
Package string `json:"package,omitempty"`
Restarts uint64 `json:"restarts"`
ExitCode int `json:"exitCode"`
StartedAt int64 `json:"startedAt,omitempty"`
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
MainPID int `json:"mainPid,omitempty"`
ExecStart string `json:"execStart,omitempty"`
Requires []string `json:"requires,omitempty"`
After []string `json:"after,omitempty"`
ListenPorts []int `json:"listenPorts,omitempty"`
Logs []AgentLog `json:"logs,omitempty"`
Flapping bool `json:"flapping"`
}
type AgentIssue struct {
@@ -79,30 +86,37 @@ type AgentIssue struct {
}
type AgentReport struct {
Hostname string `json:"hostname"`
MachineID string `json:"machineId"`
Version string `json:"version"`
OS string `json:"os"`
Arch string `json:"arch"`
OSName string `json:"osName"`
Kernel string `json:"kernel"`
UptimeSeconds int64 `json:"uptimeSeconds"`
Load1 float64 `json:"load1"`
CPUPercent float64 `json:"cpuPercent"`
MemoryTotal uint64 `json:"memoryTotal"`
MemoryUsed uint64 `json:"memoryUsed"`
RootTotal uint64 `json:"rootTotal"`
RootUsed uint64 `json:"rootUsed"`
DockerAvailable bool `json:"dockerAvailable"`
DockerError string `json:"dockerError,omitempty"`
Containers []AgentContainer `json:"containers"`
Interfaces []AgentInterface `json:"interfaces"`
Services []AgentService `json:"services"`
Issues []AgentIssue `json:"issues"`
UpdatesAvailable int `json:"updatesAvailable"`
RebootRequired bool `json:"rebootRequired"`
InventoryAt int64 `json:"inventoryAt"`
CollectedAt int64 `json:"collectedAt"`
Hostname string `json:"hostname"`
MachineID string `json:"machineId"`
Version string `json:"version"`
OS string `json:"os"`
Arch string `json:"arch"`
OSName string `json:"osName"`
Kernel string `json:"kernel"`
UptimeSeconds int64 `json:"uptimeSeconds"`
Load1 float64 `json:"load1"`
CPUPercent float64 `json:"cpuPercent"`
MemoryTotal uint64 `json:"memoryTotal"`
MemoryUsed uint64 `json:"memoryUsed"`
SwapTotal uint64 `json:"swapTotal"`
SwapUsed uint64 `json:"swapUsed"`
RootTotal uint64 `json:"rootTotal"`
RootUsed uint64 `json:"rootUsed"`
DockerAvailable bool `json:"dockerAvailable"`
DockerError string `json:"dockerError,omitempty"`
Containers []AgentContainer `json:"containers"`
Interfaces []AgentInterface `json:"interfaces"`
Services []AgentService `json:"services"`
Issues []AgentIssue `json:"issues"`
Processes []AgentProcess `json:"processes"`
Filesystems []AgentFilesystem `json:"filesystems"`
ListenPorts []AgentListenPort `json:"listenPorts"`
Pressure AgentPressure `json:"pressure"`
OOMKills []AgentLog `json:"oomKills"`
UpdatesAvailable int `json:"updatesAvailable"`
RebootRequired bool `json:"rebootRequired"`
InventoryAt int64 `json:"inventoryAt"`
CollectedAt int64 `json:"collectedAt"`
}
type agentInventory struct {
@@ -501,6 +515,11 @@ func collectAgentReport() AgentReport {
}
report.CPUPercent = sampleAgentCPU()
report.Interfaces = collectAgentInterfaces()
report.Processes = collectAgentProcesses()
report.Filesystems = collectAgentFilesystems()
report.ListenPorts = collectAgentListenPorts()
report.Pressure = collectAgentPressure()
report.OOMKills = cachedAgentOOMKills()
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
values := map[string]uint64{}
for _, line := range strings.Split(string(data), "\n") {
@@ -515,6 +534,10 @@ func collectAgentReport() AgentReport {
if report.MemoryTotal > available {
report.MemoryUsed = report.MemoryTotal - available
}
report.SwapTotal = values["SwapTotal"]
if report.SwapTotal > values["SwapFree"] {
report.SwapUsed = report.SwapTotal - values["SwapFree"]
}
}
var stat syscall.Statfs_t
if syscall.Statfs("/", &stat) == nil {
@@ -637,7 +660,7 @@ func enrichAgentServices(services []AgentService) []AgentService {
if len(services) == 0 {
return services
}
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec"}
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec", "--property=MainPID", "--property=ExecStart", "--property=Requires", "--property=After"}
for _, service := range services {
args = append(args, service.Name)
}
@@ -706,6 +729,23 @@ func enrichAgentServices(services []AgentService) []AgentService {
service.MemoryBytes, _ = strconv.ParseUint(values["MemoryCurrent"], 10, 64)
cpu, _ := strconv.ParseUint(values["CPUUsageNSec"], 10, 64)
service.CPUSeconds = float64(cpu) / 1_000_000_000
pid, _ := strconv.Atoi(values["MainPID"])
service.MainPID = pid
service.ExecStart = compactSystemdExec(values["ExecStart"])
service.Requires = limitedFields(values["Requires"], 12)
service.After = limitedFields(values["After"], 12)
service.Flapping = service.Restarts >= 5 && service.StartedAt > time.Now().Add(-10*time.Minute).Unix()
if service.State == "failed" || service.SubState == "failed" || service.Flapping {
service.Logs = collectAgentJournal(service.Name, 8, "-2h")
}
}
ports := collectAgentListenPorts()
for i := range services {
for _, port := range ports {
if services[i].MainPID > 0 && port.PID == services[i].MainPID {
services[i].ListenPorts = appendUniqueInt(services[i].ListenPorts, port.Port)
}
}
}
return services
}

329
agent_diagnostics.go Normal file
View File

@@ -0,0 +1,329 @@
package main
import (
"bufio"
"encoding/json"
"net"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
)
type AgentLog struct {
Time int64 `json:"time"`
Message string `json:"message"`
}
type AgentProcess struct {
PID int `json:"pid"`
Name string `json:"name"`
State string `json:"state"`
CPUPercent float64 `json:"cpuPercent"`
MemoryBytes uint64 `json:"memoryBytes"`
ReadBytes uint64 `json:"readBytes"`
WriteBytes uint64 `json:"writeBytes"`
Elapsed int64 `json:"elapsedSeconds"`
Command string `json:"command"`
}
type AgentFilesystem struct {
Device string `json:"device"`
Mountpoint string `json:"mountpoint"`
Type string `json:"type"`
TotalBytes uint64 `json:"totalBytes"`
UsedBytes uint64 `json:"usedBytes"`
UsedPercent float64 `json:"usedPercent"`
InodesTotal uint64 `json:"inodesTotal"`
InodesUsed uint64 `json:"inodesUsed"`
InodesPercent float64 `json:"inodesPercent"`
}
type AgentListenPort struct {
Protocol string `json:"protocol"`
Address string `json:"address"`
Port int `json:"port"`
PID int `json:"pid,omitempty"`
Process string `json:"process,omitempty"`
}
type AgentPressureValue struct {
Avg10 float64 `json:"avg10"`
Avg60 float64 `json:"avg60"`
Avg300 float64 `json:"avg300"`
}
type AgentPressure struct {
CPU AgentPressureValue `json:"cpu"`
Memory AgentPressureValue `json:"memory"`
IO AgentPressureValue `json:"io"`
}
var agentOOMCache struct {
sync.Mutex
value []AgentLog
at time.Time
}
func compactSystemdExec(value string) string {
if pathAt := strings.Index(value, "path="); pathAt >= 0 {
value = value[pathAt+5:]
}
if end := strings.IndexAny(value, " ;}"); end > 0 {
value = value[:end]
}
if len(value) > 240 {
value = value[:240]
}
return strings.TrimSpace(value)
}
func limitedFields(value string, limit int) []string {
fields := strings.Fields(value)
if len(fields) > limit {
fields = fields[:limit]
}
return fields
}
func appendUniqueInt(items []int, value int) []int {
for _, item := range items {
if item == value {
return items
}
}
return append(items, value)
}
func collectAgentJournal(unit string, limit int, since string) []AgentLog {
args := []string{"--no-pager", "-o", "json", "--reverse", "-n", strconv.Itoa(limit), "--since", since}
if unit != "" {
args = append(args, "-u", unit)
}
output := runAgentCommand(6*time.Second, "journalctl", args...)
result := []AgentLog{}
for _, line := range strings.Split(output, "\n") {
var entry struct {
Message any `json:"MESSAGE"`
Timestamp string `json:"__REALTIME_TIMESTAMP"`
}
if json.Unmarshal([]byte(line), &entry) != nil {
continue
}
message, ok := entry.Message.(string)
if !ok || strings.TrimSpace(message) == "" {
continue
}
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
}
return result
}
func cachedAgentOOMKills() []AgentLog {
agentOOMCache.Lock()
defer agentOOMCache.Unlock()
if time.Since(agentOOMCache.at) < time.Minute {
return agentOOMCache.value
}
output := runAgentCommand(6*time.Second, "journalctl", "-k", "--since", "24 hours ago", "--reverse", "--no-pager", "-o", "json", "-g", "Out of memory|Killed process")
result := []AgentLog{}
for _, line := range strings.Split(output, "\n") {
var entry struct {
Message any `json:"MESSAGE"`
Timestamp string `json:"__REALTIME_TIMESTAMP"`
}
if json.Unmarshal([]byte(line), &entry) != nil {
continue
}
message, ok := entry.Message.(string)
if !ok {
continue
}
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
if len(result) >= 20 {
break
}
}
agentOOMCache.value, agentOOMCache.at = result, time.Now()
return result
}
func collectAgentProcesses() []AgentProcess {
result := []AgentProcess{}
seen := map[int]bool{}
for _, sortBy := range []string{"-pcpu", "-rss"} {
output := runAgentCommand(5*time.Second, "ps", "-eo", "pid=,comm=,state=,pcpu=,rss=,etimes=,args=", "--sort="+sortBy)
added := 0
for _, line := range strings.Split(output, "\n") {
fields := strings.Fields(line)
if len(fields) < 7 {
continue
}
pid, err := strconv.Atoi(fields[0])
if err != nil || seen[pid] {
continue
}
seen[pid] = true
cpu, _ := strconv.ParseFloat(fields[3], 64)
rss, _ := strconv.ParseUint(fields[4], 10, 64)
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
result = append(result, item)
added++
if added >= 12 || len(result) >= 24 {
break
}
}
}
return result
}
func readAgentProcessIO(pid int) (uint64, uint64) {
data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "io"))
if err != nil {
return 0, 0
}
values := map[string]uint64{}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) != 2 {
continue
}
values[strings.TrimSuffix(fields[0], ":")], _ = strconv.ParseUint(fields[1], 10, 64)
}
return values["read_bytes"], values["write_bytes"]
}
func collectAgentFilesystems() []AgentFilesystem {
file, err := os.Open("/proc/self/mounts")
if err != nil {
return nil
}
defer file.Close()
excluded := map[string]bool{"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "cgroup": true, "cgroup2": true, "overlay": true, "squashfs": true, "nsfs": true, "tracefs": true, "debugfs": true, "securityfs": true, "pstore": true, "mqueue": true, "hugetlbfs": true, "fusectl": true, "configfs": true, "autofs": true, "rpc_pipefs": true}
seen := map[string]bool{}
result := []AgentFilesystem{}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 3 || excluded[fields[2]] || seen[fields[1]] {
continue
}
mountpoint := strings.ReplaceAll(fields[1], `\040`, " ")
var stat syscall.Statfs_t
if syscall.Statfs(mountpoint, &stat) != nil || stat.Blocks == 0 {
continue
}
seen[mountpoint] = true
total := stat.Blocks * uint64(stat.Bsize)
free := stat.Bavail * uint64(stat.Bsize)
used := total - free
item := AgentFilesystem{Device: fields[0], Mountpoint: mountpoint, Type: fields[2], TotalBytes: total, UsedBytes: used, UsedPercent: float64(used) / float64(total) * 100, InodesTotal: stat.Files}
if stat.Files > stat.Ffree {
item.InodesUsed = stat.Files - stat.Ffree
item.InodesPercent = float64(item.InodesUsed) / float64(stat.Files) * 100
}
result = append(result, item)
if len(result) >= 30 {
break
}
}
sort.Slice(result, func(i, j int) bool { return result[i].Mountpoint < result[j].Mountpoint })
return result
}
var ssPIDPattern = regexp.MustCompile(`pid=([0-9]+)`)
var ssNamePattern = regexp.MustCompile(`users:\(\(\"([^\"]+)\"`)
func collectAgentListenPorts() []AgentListenPort {
output := runAgentCommand(5*time.Second, "ss", "-H", "-lntup")
result := []AgentListenPort{}
seen := map[string]bool{}
for _, line := range strings.Split(output, "\n") {
fields := strings.Fields(line)
if len(fields) < 5 {
continue
}
protocol := strings.ToLower(fields[0])
local := ""
for _, field := range fields[1:] {
if strings.Contains(field, ":") && !strings.Contains(field, "users:") {
local = field
break
}
}
if local == "" {
continue
}
address, portText, err := net.SplitHostPort(local)
if err != nil {
idx := strings.LastIndex(local, ":")
if idx < 0 {
continue
}
address, portText = local[:idx], local[idx+1:]
}
port, _ := strconv.Atoi(portText)
if port <= 0 {
continue
}
pid := 0
if match := ssPIDPattern.FindStringSubmatch(line); len(match) > 1 {
pid, _ = strconv.Atoi(match[1])
}
process := ""
if match := ssNamePattern.FindStringSubmatch(line); len(match) > 1 {
process = match[1]
}
key := protocol + "|" + address + "|" + portText + "|" + strconv.Itoa(pid)
if seen[key] {
continue
}
seen[key] = true
result = append(result, AgentListenPort{Protocol: protocol, Address: address, Port: port, PID: pid, Process: process})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Port == result[j].Port {
return result[i].Protocol < result[j].Protocol
}
return result[i].Port < result[j].Port
})
return result
}
func collectAgentPressure() AgentPressure {
return AgentPressure{CPU: readPressure("/proc/pressure/cpu"), Memory: readPressure("/proc/pressure/memory"), IO: readPressure("/proc/pressure/io")}
}
func readPressure(path string) AgentPressureValue {
data, err := os.ReadFile(path)
if err != nil {
return AgentPressureValue{}
}
line := strings.SplitN(string(data), "\n", 2)[0]
value := AgentPressureValue{}
for _, field := range strings.Fields(line) {
key, raw, ok := strings.Cut(field, "=")
if !ok {
continue
}
number, _ := strconv.ParseFloat(raw, 64)
switch key {
case "avg10":
value.Avg10 = number
case "avg60":
value.Avg60 = number
case "avg300":
value.Avg300 = number
}
}
return value
}

34
agent_test.go
View File

@@ -69,6 +69,21 @@ func TestClassifyAgentService(t *testing.T) {
}
}
func TestAgentServiceDiagnosticsHelpers(t *testing.T) {
if got := compactSystemdExec(`{ path=/usr/bin/example ; argv[]=/usr/bin/example --serve ; }`); got != "/usr/bin/example" {
t.Fatalf("unexpected executable: %q", got)
}
fields := limitedFields("a b c d", 2)
if len(fields) != 2 || fields[0] != "a" || fields[1] != "b" {
t.Fatalf("unexpected limited fields: %#v", fields)
}
ports := appendUniqueInt([]int{80}, 80)
ports = appendUniqueInt(ports, 443)
if len(ports) != 2 || ports[1] != 443 {
t.Fatalf("unexpected ports: %#v", ports)
}
}
func TestAgentReportAuthenticationAndOfflineState(t *testing.T) {
store, err := openStore(":memory:")
if err != nil {
@@ -110,3 +125,22 @@ func TestAgentProblemsBecomeMaintenanceAwareAlerts(t *testing.T) {
t.Fatalf("active=%d planned=%d alerts=%#v", len(active), len(planned), alerts)
}
}
func TestAgentDiagnosticsBecomeAlerts(t *testing.T) {
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "node", Name: "app", VMID: 105, Online: true, Report: AgentReport{
Services: []AgentService{{Name: "web.service", State: "active", Flapping: true, Restarts: 8, StartedAt: time.Now().Unix()}},
Filesystems: []AgentFilesystem{{Mountpoint: "/data", TotalBytes: 1000, UsedBytes: 950, UsedPercent: 95}},
}}}}
alerts := evaluateAlerts(metrics, defaultThresholds())
wanted := map[string]bool{"agent-flapping-node-web.service-105": false, "agent-filesystem-node-/data-105": false}
for _, alert := range alerts {
if _, ok := wanted[alert.ID]; ok {
wanted[alert.ID] = true
}
}
for id, found := range wanted {
if !found {
t.Fatalf("missing alert %s in %#v", id, alerts)
}
}
}

14
alerts.go
View File

@@ -215,8 +215,22 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
for _, service := range agent.Report.Services {
if service.State == "failed" || service.SubState == "failed" {
add("agent-systemd-"+agent.ID+"-"+service.Name+vmSuffix, "critical", "Агенты", agent.Name+": упала служба "+service.Name, fmt.Sprintf("%s · VMID %d.", firstNonEmpty(service.Description, service.SubState), agent.VMID))
} else if service.Flapping {
add("agent-flapping-"+agent.ID+"-"+service.Name+vmSuffix, "warning", "Агенты", agent.Name+": служба постоянно перезапускается", fmt.Sprintf("%s: %d рестартов, последний запуск %s · VMID %d.", service.Name, service.Restarts, time.Unix(service.StartedAt, 0).Format("15:04:05"), agent.VMID))
}
}
for _, filesystem := range agent.Report.Filesystems {
if filesystem.UsedPercent >= 90 {
add("agent-filesystem-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчивается место "+filesystem.Mountpoint, fmt.Sprintf("Заполнено %.1f%%, свободно %.1f GB · VMID %d.", filesystem.UsedPercent, float64(filesystem.TotalBytes-filesystem.UsedBytes)/1073741824, agent.VMID))
}
if filesystem.InodesPercent >= 90 {
add("agent-inodes-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчиваются inode "+filesystem.Mountpoint, fmt.Sprintf("Использовано %.1f%% inode · VMID %d.", filesystem.InodesPercent, agent.VMID))
}
}
if len(agent.Report.OOMKills) > 0 {
latest := agent.Report.OOMKills[0]
add("agent-oom-"+agent.ID+vmSuffix, "critical", "Агенты", agent.Name+": ядро завершало процессы из-за нехватки памяти", fmt.Sprintf("%s · %s · VMID %d.", time.Unix(latest.Time, 0).Format("02.01 15:04"), latest.Message, agent.VMID))
}
if agent.Report.RebootRequired {
add("agent-reboot-"+agent.ID+vmSuffix, "info", "Агенты", agent.Name+": требуется перезагрузка", fmt.Sprintf("Обновления системы запросили перезагрузку · VMID %d.", agent.VMID))
}

24
web.go
View File

File diff suppressed because one or more lines are too long