Add guest root cause diagnostics
This commit is contained in:
Maxim
329
agent_diagnostics.go
Normal file
329
agent_diagnostics.go
Normal file
@@ -0,0 +1,329 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AgentLog struct {
|
||||
Time int64 `json:"time"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type AgentProcess struct {
|
||||
PID int `json:"pid"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryBytes uint64 `json:"memoryBytes"`
|
||||
ReadBytes uint64 `json:"readBytes"`
|
||||
WriteBytes uint64 `json:"writeBytes"`
|
||||
Elapsed int64 `json:"elapsedSeconds"`
|
||||
Command string `json:"command"`
|
||||
}
|
||||
|
||||
type AgentFilesystem struct {
|
||||
Device string `json:"device"`
|
||||
Mountpoint string `json:"mountpoint"`
|
||||
Type string `json:"type"`
|
||||
TotalBytes uint64 `json:"totalBytes"`
|
||||
UsedBytes uint64 `json:"usedBytes"`
|
||||
UsedPercent float64 `json:"usedPercent"`
|
||||
InodesTotal uint64 `json:"inodesTotal"`
|
||||
InodesUsed uint64 `json:"inodesUsed"`
|
||||
InodesPercent float64 `json:"inodesPercent"`
|
||||
}
|
||||
|
||||
type AgentListenPort struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
Process string `json:"process,omitempty"`
|
||||
}
|
||||
|
||||
type AgentPressureValue struct {
|
||||
Avg10 float64 `json:"avg10"`
|
||||
Avg60 float64 `json:"avg60"`
|
||||
Avg300 float64 `json:"avg300"`
|
||||
}
|
||||
|
||||
type AgentPressure struct {
|
||||
CPU AgentPressureValue `json:"cpu"`
|
||||
Memory AgentPressureValue `json:"memory"`
|
||||
IO AgentPressureValue `json:"io"`
|
||||
}
|
||||
|
||||
var agentOOMCache struct {
|
||||
sync.Mutex
|
||||
value []AgentLog
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func compactSystemdExec(value string) string {
|
||||
if pathAt := strings.Index(value, "path="); pathAt >= 0 {
|
||||
value = value[pathAt+5:]
|
||||
}
|
||||
if end := strings.IndexAny(value, " ;}"); end > 0 {
|
||||
value = value[:end]
|
||||
}
|
||||
if len(value) > 240 {
|
||||
value = value[:240]
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func limitedFields(value string, limit int) []string {
|
||||
fields := strings.Fields(value)
|
||||
if len(fields) > limit {
|
||||
fields = fields[:limit]
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func appendUniqueInt(items []int, value int) []int {
|
||||
for _, item := range items {
|
||||
if item == value {
|
||||
return items
|
||||
}
|
||||
}
|
||||
return append(items, value)
|
||||
}
|
||||
|
||||
func collectAgentJournal(unit string, limit int, since string) []AgentLog {
|
||||
args := []string{"--no-pager", "-o", "json", "--reverse", "-n", strconv.Itoa(limit), "--since", since}
|
||||
if unit != "" {
|
||||
args = append(args, "-u", unit)
|
||||
}
|
||||
output := runAgentCommand(6*time.Second, "journalctl", args...)
|
||||
result := []AgentLog{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
var entry struct {
|
||||
Message any `json:"MESSAGE"`
|
||||
Timestamp string `json:"__REALTIME_TIMESTAMP"`
|
||||
}
|
||||
if json.Unmarshal([]byte(line), &entry) != nil {
|
||||
continue
|
||||
}
|
||||
message, ok := entry.Message.(string)
|
||||
if !ok || strings.TrimSpace(message) == "" {
|
||||
continue
|
||||
}
|
||||
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
|
||||
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func cachedAgentOOMKills() []AgentLog {
|
||||
agentOOMCache.Lock()
|
||||
defer agentOOMCache.Unlock()
|
||||
if time.Since(agentOOMCache.at) < time.Minute {
|
||||
return agentOOMCache.value
|
||||
}
|
||||
output := runAgentCommand(6*time.Second, "journalctl", "-k", "--since", "24 hours ago", "--reverse", "--no-pager", "-o", "json", "-g", "Out of memory|Killed process")
|
||||
result := []AgentLog{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
var entry struct {
|
||||
Message any `json:"MESSAGE"`
|
||||
Timestamp string `json:"__REALTIME_TIMESTAMP"`
|
||||
}
|
||||
if json.Unmarshal([]byte(line), &entry) != nil {
|
||||
continue
|
||||
}
|
||||
message, ok := entry.Message.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
|
||||
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
|
||||
if len(result) >= 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
agentOOMCache.value, agentOOMCache.at = result, time.Now()
|
||||
return result
|
||||
}
|
||||
|
||||
func collectAgentProcesses() []AgentProcess {
|
||||
result := []AgentProcess{}
|
||||
seen := map[int]bool{}
|
||||
for _, sortBy := range []string{"-pcpu", "-rss"} {
|
||||
output := runAgentCommand(5*time.Second, "ps", "-eo", "pid=,comm=,state=,pcpu=,rss=,etimes=,args=", "--sort="+sortBy)
|
||||
added := 0
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 7 {
|
||||
continue
|
||||
}
|
||||
pid, err := strconv.Atoi(fields[0])
|
||||
if err != nil || seen[pid] {
|
||||
continue
|
||||
}
|
||||
seen[pid] = true
|
||||
cpu, _ := strconv.ParseFloat(fields[3], 64)
|
||||
rss, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
|
||||
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
|
||||
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
|
||||
result = append(result, item)
|
||||
added++
|
||||
if added >= 12 || len(result) >= 24 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readAgentProcessIO(pid int) (uint64, uint64) {
|
||||
data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "io"))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
values[strings.TrimSuffix(fields[0], ":")], _ = strconv.ParseUint(fields[1], 10, 64)
|
||||
}
|
||||
return values["read_bytes"], values["write_bytes"]
|
||||
}
|
||||
|
||||
func collectAgentFilesystems() []AgentFilesystem {
|
||||
file, err := os.Open("/proc/self/mounts")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
excluded := map[string]bool{"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "cgroup": true, "cgroup2": true, "overlay": true, "squashfs": true, "nsfs": true, "tracefs": true, "debugfs": true, "securityfs": true, "pstore": true, "mqueue": true, "hugetlbfs": true, "fusectl": true, "configfs": true, "autofs": true, "rpc_pipefs": true}
|
||||
seen := map[string]bool{}
|
||||
result := []AgentFilesystem{}
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) < 3 || excluded[fields[2]] || seen[fields[1]] {
|
||||
continue
|
||||
}
|
||||
mountpoint := strings.ReplaceAll(fields[1], `\040`, " ")
|
||||
var stat syscall.Statfs_t
|
||||
if syscall.Statfs(mountpoint, &stat) != nil || stat.Blocks == 0 {
|
||||
continue
|
||||
}
|
||||
seen[mountpoint] = true
|
||||
total := stat.Blocks * uint64(stat.Bsize)
|
||||
free := stat.Bavail * uint64(stat.Bsize)
|
||||
used := total - free
|
||||
item := AgentFilesystem{Device: fields[0], Mountpoint: mountpoint, Type: fields[2], TotalBytes: total, UsedBytes: used, UsedPercent: float64(used) / float64(total) * 100, InodesTotal: stat.Files}
|
||||
if stat.Files > stat.Ffree {
|
||||
item.InodesUsed = stat.Files - stat.Ffree
|
||||
item.InodesPercent = float64(item.InodesUsed) / float64(stat.Files) * 100
|
||||
}
|
||||
result = append(result, item)
|
||||
if len(result) >= 30 {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].Mountpoint < result[j].Mountpoint })
|
||||
return result
|
||||
}
|
||||
|
||||
var ssPIDPattern = regexp.MustCompile(`pid=([0-9]+)`)
|
||||
var ssNamePattern = regexp.MustCompile(`users:\(\(\"([^\"]+)\"`)
|
||||
|
||||
func collectAgentListenPorts() []AgentListenPort {
|
||||
output := runAgentCommand(5*time.Second, "ss", "-H", "-lntup")
|
||||
result := []AgentListenPort{}
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
continue
|
||||
}
|
||||
protocol := strings.ToLower(fields[0])
|
||||
local := ""
|
||||
for _, field := range fields[1:] {
|
||||
if strings.Contains(field, ":") && !strings.Contains(field, "users:") {
|
||||
local = field
|
||||
break
|
||||
}
|
||||
}
|
||||
if local == "" {
|
||||
continue
|
||||
}
|
||||
address, portText, err := net.SplitHostPort(local)
|
||||
if err != nil {
|
||||
idx := strings.LastIndex(local, ":")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
address, portText = local[:idx], local[idx+1:]
|
||||
}
|
||||
port, _ := strconv.Atoi(portText)
|
||||
if port <= 0 {
|
||||
continue
|
||||
}
|
||||
pid := 0
|
||||
if match := ssPIDPattern.FindStringSubmatch(line); len(match) > 1 {
|
||||
pid, _ = strconv.Atoi(match[1])
|
||||
}
|
||||
process := ""
|
||||
if match := ssNamePattern.FindStringSubmatch(line); len(match) > 1 {
|
||||
process = match[1]
|
||||
}
|
||||
key := protocol + "|" + address + "|" + portText + "|" + strconv.Itoa(pid)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
result = append(result, AgentListenPort{Protocol: protocol, Address: address, Port: port, PID: pid, Process: process})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Port == result[j].Port {
|
||||
return result[i].Protocol < result[j].Protocol
|
||||
}
|
||||
return result[i].Port < result[j].Port
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func collectAgentPressure() AgentPressure {
|
||||
return AgentPressure{CPU: readPressure("/proc/pressure/cpu"), Memory: readPressure("/proc/pressure/memory"), IO: readPressure("/proc/pressure/io")}
|
||||
}
|
||||
|
||||
func readPressure(path string) AgentPressureValue {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return AgentPressureValue{}
|
||||
}
|
||||
line := strings.SplitN(string(data), "\n", 2)[0]
|
||||
value := AgentPressureValue{}
|
||||
for _, field := range strings.Fields(line) {
|
||||
key, raw, ok := strings.Cut(field, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
number, _ := strconv.ParseFloat(raw, 64)
|
||||
switch key {
|
||||
case "avg10":
|
||||
value.Avg10 = number
|
||||
case "avg60":
|
||||
value.Avg60 = number
|
||||
case "avg300":
|
||||
value.Avg300 = number
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
Reference in New Issue
Block a user