Expand guest agent system monitoring
This commit is contained in:
185
agent.go
185
agent.go
@@ -14,10 +14,12 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
@@ -50,23 +52,56 @@ type AgentContainer struct {
|
||||
Ports []string `json:"ports,omitempty"`
|
||||
}
|
||||
|
||||
type AgentInterface struct {
|
||||
Name string `json:"name"`
|
||||
Addresses []string `json:"addresses"`
|
||||
}
|
||||
|
||||
type AgentService struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SubState string `json:"subState"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type AgentReport struct {
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
OSName string `json:"osName"`
|
||||
Kernel string `json:"kernel"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
Interfaces []AgentInterface `json:"interfaces"`
|
||||
Services []AgentService `json:"services"`
|
||||
UpdatesAvailable int `json:"updatesAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
InventoryAt int64 `json:"inventoryAt"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
}
|
||||
|
||||
type agentInventory struct {
|
||||
OSName string
|
||||
Kernel string
|
||||
Services []AgentService
|
||||
UpdatesAvailable int
|
||||
RebootRequired bool
|
||||
CollectedAt time.Time
|
||||
}
|
||||
|
||||
var guestInventory struct {
|
||||
sync.Mutex
|
||||
value agentInventory
|
||||
}
|
||||
|
||||
type ManagedAgent struct {
|
||||
@@ -287,6 +322,7 @@ func collectAgentReport() AgentReport {
|
||||
}
|
||||
}
|
||||
report.CPUPercent = sampleAgentCPU()
|
||||
report.Interfaces = collectAgentInterfaces()
|
||||
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
@@ -312,9 +348,113 @@ func collectAgentReport() AgentReport {
|
||||
}
|
||||
report.Containers, report.DockerError = collectDockerContainers()
|
||||
report.DockerAvailable = report.DockerError == ""
|
||||
inventory := cachedAgentInventory()
|
||||
report.OSName, report.Kernel, report.Services = inventory.OSName, inventory.Kernel, inventory.Services
|
||||
report.UpdatesAvailable, report.RebootRequired, report.InventoryAt = inventory.UpdatesAvailable, inventory.RebootRequired, inventory.CollectedAt.Unix()
|
||||
return report
|
||||
}
|
||||
|
||||
func collectAgentInterfaces() []AgentInterface {
|
||||
interfaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
result := []AgentInterface{}
|
||||
for _, iface := range interfaces {
|
||||
if iface.Flags&net.FlagLoopback != 0 {
|
||||
continue
|
||||
}
|
||||
addresses, _ := iface.Addrs()
|
||||
item := AgentInterface{Name: iface.Name}
|
||||
for _, address := range addresses {
|
||||
value := address.String()
|
||||
if host, _, err := net.ParseCIDR(value); err == nil {
|
||||
value = host.String()
|
||||
}
|
||||
item.Addresses = append(item.Addresses, value)
|
||||
}
|
||||
if len(item.Addresses) > 0 {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func cachedAgentInventory() agentInventory {
|
||||
guestInventory.Lock()
|
||||
defer guestInventory.Unlock()
|
||||
if time.Since(guestInventory.value.CollectedAt) < 15*time.Minute {
|
||||
return guestInventory.value
|
||||
}
|
||||
value := agentInventory{CollectedAt: time.Now()}
|
||||
value.OSName = readOSPrettyName()
|
||||
value.Kernel = strings.TrimSpace(runAgentCommand(3*time.Second, "uname", "-r"))
|
||||
value.Services = collectAgentServices()
|
||||
value.UpdatesAvailable = countAgentUpdates()
|
||||
_, err := os.Stat("/var/run/reboot-required")
|
||||
value.RebootRequired = err == nil
|
||||
guestInventory.value = value
|
||||
return value
|
||||
}
|
||||
|
||||
func readOSPrettyName() string {
|
||||
data, err := os.ReadFile("/etc/os-release")
|
||||
if err != nil {
|
||||
return runtime.GOOS
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, "PRETTY_NAME=") {
|
||||
return strings.Trim(strings.TrimPrefix(line, "PRETTY_NAME="), "\"")
|
||||
}
|
||||
}
|
||||
return runtime.GOOS
|
||||
}
|
||||
|
||||
func runAgentCommand(timeout time.Duration, name string, args ...string) string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, name, args...).Output()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func collectAgentServices() []AgentService {
|
||||
output := runAgentCommand(5*time.Second, "systemctl", "list-units", "--type=service", "--state=running,failed", "--no-legend", "--plain", "--no-pager")
|
||||
result := []AgentService{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
offset := 0
|
||||
if len(fields) > 0 && fields[0] == "●" {
|
||||
offset = 1
|
||||
}
|
||||
if len(fields) < offset+4 {
|
||||
continue
|
||||
}
|
||||
description := ""
|
||||
if len(fields) > offset+4 {
|
||||
description = strings.Join(fields[offset+4:], " ")
|
||||
}
|
||||
result = append(result, AgentService{Name: fields[offset], State: fields[offset+2], SubState: fields[offset+3], Description: description})
|
||||
if len(result) >= 150 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func countAgentUpdates() int {
|
||||
output := runAgentCommand(45*time.Second, "apt-get", "-s", "-o", "Debug::NoLocking=1", "upgrade")
|
||||
count := 0
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
if strings.HasPrefix(line, "Inst ") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func sampleAgentCPU() float64 {
|
||||
total1, idle1 := readAgentCPU()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
@@ -323,6 +463,9 @@ func sampleAgentCPU() float64 {
|
||||
return 0
|
||||
}
|
||||
totalDelta, idleDelta := total2-total1, idle2-idle1
|
||||
if idleDelta > totalDelta {
|
||||
return 0
|
||||
}
|
||||
return float64(totalDelta-idleDelta) / float64(totalDelta) * 100
|
||||
}
|
||||
|
||||
@@ -357,6 +500,7 @@ func collectDockerContainers() ([]AgentContainer, string) {
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||
}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
response, err := client.Get("http://docker/containers/json?all=1")
|
||||
if err != nil {
|
||||
@@ -423,8 +567,13 @@ func collectDockerContainers() ([]AgentContainer, string) {
|
||||
} `json:"memory_stats"`
|
||||
}
|
||||
if json.NewDecoder(statsResponse.Body).Decode(&stats) == nil {
|
||||
cpuDelta := stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage
|
||||
systemDelta := stats.CPUStats.SystemCPUUsage - stats.PreCPUStats.SystemCPUUsage
|
||||
cpuDelta, systemDelta := uint64(0), uint64(0)
|
||||
if stats.CPUStats.CPUUsage.TotalUsage >= stats.PreCPUStats.CPUUsage.TotalUsage {
|
||||
cpuDelta = stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage
|
||||
}
|
||||
if stats.CPUStats.SystemCPUUsage >= stats.PreCPUStats.SystemCPUUsage {
|
||||
systemDelta = stats.CPUStats.SystemCPUUsage - stats.PreCPUStats.SystemCPUUsage
|
||||
}
|
||||
cpus := stats.CPUStats.OnlineCPUs
|
||||
if cpus == 0 {
|
||||
cpus = 1
|
||||
|
||||
Reference in New Issue
Block a user