Add guest root cause diagnostics
This commit is contained in:
Maxim
114
agent.go
114
agent.go
@@ -58,18 +58,25 @@ type AgentInterface struct {
|
||||
}
|
||||
|
||||
type AgentService struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SubState string `json:"subState"`
|
||||
Description string `json:"description"`
|
||||
Origin string `json:"origin"`
|
||||
UnitPath string `json:"unitPath,omitempty"`
|
||||
Package string `json:"package,omitempty"`
|
||||
Restarts uint64 `json:"restarts"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
StartedAt int64 `json:"startedAt,omitempty"`
|
||||
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
|
||||
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SubState string `json:"subState"`
|
||||
Description string `json:"description"`
|
||||
Origin string `json:"origin"`
|
||||
UnitPath string `json:"unitPath,omitempty"`
|
||||
Package string `json:"package,omitempty"`
|
||||
Restarts uint64 `json:"restarts"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
StartedAt int64 `json:"startedAt,omitempty"`
|
||||
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
|
||||
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
|
||||
MainPID int `json:"mainPid,omitempty"`
|
||||
ExecStart string `json:"execStart,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
After []string `json:"after,omitempty"`
|
||||
ListenPorts []int `json:"listenPorts,omitempty"`
|
||||
Logs []AgentLog `json:"logs,omitempty"`
|
||||
Flapping bool `json:"flapping"`
|
||||
}
|
||||
|
||||
type AgentIssue struct {
|
||||
@@ -79,30 +86,37 @@ type AgentIssue struct {
|
||||
}
|
||||
|
||||
type AgentReport struct {
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
OSName string `json:"osName"`
|
||||
Kernel string `json:"kernel"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
Interfaces []AgentInterface `json:"interfaces"`
|
||||
Services []AgentService `json:"services"`
|
||||
Issues []AgentIssue `json:"issues"`
|
||||
UpdatesAvailable int `json:"updatesAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
InventoryAt int64 `json:"inventoryAt"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
OSName string `json:"osName"`
|
||||
Kernel string `json:"kernel"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
SwapTotal uint64 `json:"swapTotal"`
|
||||
SwapUsed uint64 `json:"swapUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
Interfaces []AgentInterface `json:"interfaces"`
|
||||
Services []AgentService `json:"services"`
|
||||
Issues []AgentIssue `json:"issues"`
|
||||
Processes []AgentProcess `json:"processes"`
|
||||
Filesystems []AgentFilesystem `json:"filesystems"`
|
||||
ListenPorts []AgentListenPort `json:"listenPorts"`
|
||||
Pressure AgentPressure `json:"pressure"`
|
||||
OOMKills []AgentLog `json:"oomKills"`
|
||||
UpdatesAvailable int `json:"updatesAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
InventoryAt int64 `json:"inventoryAt"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
}
|
||||
|
||||
type agentInventory struct {
|
||||
@@ -501,6 +515,11 @@ func collectAgentReport() AgentReport {
|
||||
}
|
||||
report.CPUPercent = sampleAgentCPU()
|
||||
report.Interfaces = collectAgentInterfaces()
|
||||
report.Processes = collectAgentProcesses()
|
||||
report.Filesystems = collectAgentFilesystems()
|
||||
report.ListenPorts = collectAgentListenPorts()
|
||||
report.Pressure = collectAgentPressure()
|
||||
report.OOMKills = cachedAgentOOMKills()
|
||||
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
@@ -515,6 +534,10 @@ func collectAgentReport() AgentReport {
|
||||
if report.MemoryTotal > available {
|
||||
report.MemoryUsed = report.MemoryTotal - available
|
||||
}
|
||||
report.SwapTotal = values["SwapTotal"]
|
||||
if report.SwapTotal > values["SwapFree"] {
|
||||
report.SwapUsed = report.SwapTotal - values["SwapFree"]
|
||||
}
|
||||
}
|
||||
var stat syscall.Statfs_t
|
||||
if syscall.Statfs("/", &stat) == nil {
|
||||
@@ -637,7 +660,7 @@ func enrichAgentServices(services []AgentService) []AgentService {
|
||||
if len(services) == 0 {
|
||||
return services
|
||||
}
|
||||
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec"}
|
||||
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec", "--property=MainPID", "--property=ExecStart", "--property=Requires", "--property=After"}
|
||||
for _, service := range services {
|
||||
args = append(args, service.Name)
|
||||
}
|
||||
@@ -706,6 +729,23 @@ func enrichAgentServices(services []AgentService) []AgentService {
|
||||
service.MemoryBytes, _ = strconv.ParseUint(values["MemoryCurrent"], 10, 64)
|
||||
cpu, _ := strconv.ParseUint(values["CPUUsageNSec"], 10, 64)
|
||||
service.CPUSeconds = float64(cpu) / 1_000_000_000
|
||||
pid, _ := strconv.Atoi(values["MainPID"])
|
||||
service.MainPID = pid
|
||||
service.ExecStart = compactSystemdExec(values["ExecStart"])
|
||||
service.Requires = limitedFields(values["Requires"], 12)
|
||||
service.After = limitedFields(values["After"], 12)
|
||||
service.Flapping = service.Restarts >= 5 && service.StartedAt > time.Now().Add(-10*time.Minute).Unix()
|
||||
if service.State == "failed" || service.SubState == "failed" || service.Flapping {
|
||||
service.Logs = collectAgentJournal(service.Name, 8, "-2h")
|
||||
}
|
||||
}
|
||||
ports := collectAgentListenPorts()
|
||||
for i := range services {
|
||||
for _, port := range ports {
|
||||
if services[i].MainPID > 0 && port.PID == services[i].MainPID {
|
||||
services[i].ListenPorts = appendUniqueInt(services[i].ListenPorts, port.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
return services
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user