986 lines
31 KiB
Go
986 lines
31 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/hmac"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const agentOfflineAfter = 45 * time.Second
|
|
|
|
type agentOptions struct {
|
|
Server, EnrollmentToken, Name, ConfigPath string
|
|
VMID int
|
|
}
|
|
|
|
type agentConfig struct {
|
|
Server string `json:"server"`
|
|
ID string `json:"id"`
|
|
Secret string `json:"secret"`
|
|
Name string `json:"name"`
|
|
VMID int `json:"vmid"`
|
|
}
|
|
|
|
type AgentContainer struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Image string `json:"image"`
|
|
State string `json:"state"`
|
|
Status string `json:"status"`
|
|
Health string `json:"health,omitempty"`
|
|
Restarts int64 `json:"restarts"`
|
|
CPUPercent float64 `json:"cpuPercent"`
|
|
MemoryUsage uint64 `json:"memoryUsage"`
|
|
Ports []string `json:"ports,omitempty"`
|
|
}
|
|
|
|
type AgentInterface struct {
|
|
Name string `json:"name"`
|
|
Addresses []string `json:"addresses"`
|
|
}
|
|
|
|
type AgentService struct {
|
|
Name string `json:"name"`
|
|
State string `json:"state"`
|
|
SubState string `json:"subState"`
|
|
Description string `json:"description"`
|
|
Origin string `json:"origin"`
|
|
UnitPath string `json:"unitPath,omitempty"`
|
|
Package string `json:"package,omitempty"`
|
|
Restarts uint64 `json:"restarts"`
|
|
ExitCode int `json:"exitCode"`
|
|
StartedAt int64 `json:"startedAt,omitempty"`
|
|
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
|
|
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
|
|
MainPID int `json:"mainPid,omitempty"`
|
|
ExecStart string `json:"execStart,omitempty"`
|
|
Requires []string `json:"requires,omitempty"`
|
|
After []string `json:"after,omitempty"`
|
|
ListenPorts []int `json:"listenPorts,omitempty"`
|
|
Logs []AgentLog `json:"logs,omitempty"`
|
|
Flapping bool `json:"flapping"`
|
|
}
|
|
|
|
type AgentIssue struct {
|
|
Time int64 `json:"time"`
|
|
Unit string `json:"unit"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type AgentReport struct {
|
|
Hostname string `json:"hostname"`
|
|
MachineID string `json:"machineId"`
|
|
Version string `json:"version"`
|
|
OS string `json:"os"`
|
|
Arch string `json:"arch"`
|
|
OSName string `json:"osName"`
|
|
Kernel string `json:"kernel"`
|
|
UptimeSeconds int64 `json:"uptimeSeconds"`
|
|
Load1 float64 `json:"load1"`
|
|
CPUPercent float64 `json:"cpuPercent"`
|
|
MemoryTotal uint64 `json:"memoryTotal"`
|
|
MemoryUsed uint64 `json:"memoryUsed"`
|
|
SwapTotal uint64 `json:"swapTotal"`
|
|
SwapUsed uint64 `json:"swapUsed"`
|
|
RootTotal uint64 `json:"rootTotal"`
|
|
RootUsed uint64 `json:"rootUsed"`
|
|
DockerAvailable bool `json:"dockerAvailable"`
|
|
DockerError string `json:"dockerError,omitempty"`
|
|
Containers []AgentContainer `json:"containers"`
|
|
Interfaces []AgentInterface `json:"interfaces"`
|
|
Services []AgentService `json:"services"`
|
|
Issues []AgentIssue `json:"issues"`
|
|
Processes []AgentProcess `json:"processes"`
|
|
Filesystems []AgentFilesystem `json:"filesystems"`
|
|
ListenPorts []AgentListenPort `json:"listenPorts"`
|
|
Pressure AgentPressure `json:"pressure"`
|
|
OOMKills []AgentLog `json:"oomKills"`
|
|
UpdatesAvailable int `json:"updatesAvailable"`
|
|
RebootRequired bool `json:"rebootRequired"`
|
|
InventoryAt int64 `json:"inventoryAt"`
|
|
CollectedAt int64 `json:"collectedAt"`
|
|
ActionResults []AgentActionResult `json:"actionResults,omitempty"`
|
|
}
|
|
|
|
type agentInventory struct {
|
|
OSName string
|
|
Kernel string
|
|
Services []AgentService
|
|
UpdatesAvailable int
|
|
RebootRequired bool
|
|
CollectedAt time.Time
|
|
}
|
|
|
|
type AgentUpdateManifest struct {
|
|
Version string `json:"version"`
|
|
SHA256 string `json:"sha256"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
type agentReportResponse struct {
|
|
Accepted bool `json:"accepted"`
|
|
Update *AgentUpdateManifest `json:"update,omitempty"`
|
|
Commands []AgentCommand `json:"commands,omitempty"`
|
|
}
|
|
|
|
var guestInventory struct {
|
|
sync.Mutex
|
|
value agentInventory
|
|
}
|
|
|
|
var guestIssues struct {
|
|
sync.Mutex
|
|
value []AgentIssue
|
|
collectedAt time.Time
|
|
}
|
|
|
|
var guestActionResults struct {
|
|
sync.Mutex
|
|
value []AgentActionResult
|
|
}
|
|
|
|
var agentBinaryInfo struct {
|
|
sync.Once
|
|
manifest AgentUpdateManifest
|
|
path string
|
|
err error
|
|
}
|
|
|
|
func currentAgentUpdate() (AgentUpdateManifest, string, error) {
|
|
agentBinaryInfo.Do(func() {
|
|
agentBinaryInfo.path, agentBinaryInfo.err = os.Executable()
|
|
if agentBinaryInfo.err != nil {
|
|
return
|
|
}
|
|
file, err := os.Open(agentBinaryInfo.path)
|
|
if err != nil {
|
|
agentBinaryInfo.err = err
|
|
return
|
|
}
|
|
defer file.Close()
|
|
hash := sha256.New()
|
|
if _, err = io.Copy(hash, file); err != nil {
|
|
agentBinaryInfo.err = err
|
|
return
|
|
}
|
|
agentBinaryInfo.manifest = AgentUpdateManifest{Version: version, SHA256: hex.EncodeToString(hash.Sum(nil)), URL: "/api/agent/binary"}
|
|
})
|
|
return agentBinaryInfo.manifest, agentBinaryInfo.path, agentBinaryInfo.err
|
|
}
|
|
|
|
type ManagedAgent struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
VMID int `json:"vmid"`
|
|
Hostname string `json:"hostname"`
|
|
Version string `json:"version"`
|
|
LastSeen int64 `json:"lastSeen"`
|
|
EnrolledAt int64 `json:"enrolledAt"`
|
|
Online bool `json:"online"`
|
|
RemoteAddress string `json:"remoteAddress"`
|
|
Report AgentReport `json:"report"`
|
|
}
|
|
|
|
type agentEnrollmentRequest struct {
|
|
Token, Name, Hostname, MachineID string
|
|
VMID int
|
|
}
|
|
|
|
func randomHex(bytesCount int) (string, error) {
|
|
b := make([]byte, bytesCount)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
func tokenHash(value string) string {
|
|
sum := sha256.Sum256([]byte(value))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (s *Store) CreateAgentEnrollmentToken() (string, error) {
|
|
token, err := randomHex(24)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
_, err = s.db.Exec(`INSERT INTO agent_enrollment_tokens(token_hash,created_at,expires_at) VALUES(?,?,?)`, tokenHash(token), time.Now().Unix(), time.Now().Add(15*time.Minute).Unix())
|
|
return token, err
|
|
}
|
|
|
|
func (s *Store) EnrollAgent(req agentEnrollmentRequest) (agentConfig, error) {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
defer tx.Rollback()
|
|
var expires int64
|
|
if err = tx.QueryRow(`SELECT expires_at FROM agent_enrollment_tokens WHERE token_hash=? AND used_at IS NULL`, tokenHash(req.Token)).Scan(&expires); err != nil || expires < time.Now().Unix() {
|
|
return agentConfig{}, errors.New("одноразовый токен недействителен или истёк")
|
|
}
|
|
id, err := randomHex(12)
|
|
if err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
secret, err := randomHex(32)
|
|
if err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
name := strings.TrimSpace(req.Name)
|
|
if name == "" {
|
|
name = req.Hostname
|
|
}
|
|
now := time.Now().Unix()
|
|
if _, err = tx.Exec(`INSERT INTO agents(id,name,vmid,hostname,machine_id,secret_hash,enrolled_at,last_seen,report_json,remote_address) VALUES(?,?,?,?,?,?,?,?,?,?)`, id, name, req.VMID, req.Hostname, req.MachineID, tokenHash(secret), now, now, `{}`, ""); err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
if _, err = tx.Exec(`UPDATE agent_enrollment_tokens SET used_at=? WHERE token_hash=?`, now, tokenHash(req.Token)); err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
if err = tx.Commit(); err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
return agentConfig{ID: id, Secret: secret, Name: name, VMID: req.VMID}, nil
|
|
}
|
|
|
|
func (s *Store) SaveAgentReport(id, secret, remote string, report AgentReport) error {
|
|
if !s.AuthenticateAgent(id, secret) {
|
|
return errors.New("агент не авторизован")
|
|
}
|
|
s.CompleteAgentActions(id, report.ActionResults)
|
|
report.ActionResults = nil
|
|
data, _ := json.Marshal(report)
|
|
_, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) AuthenticateAgent(id, secret string) bool {
|
|
var expected string
|
|
return s.db.QueryRow(`SELECT secret_hash FROM agents WHERE id=?`, id).Scan(&expected) == nil && hmac.Equal([]byte(expected), []byte(tokenHash(secret)))
|
|
}
|
|
|
|
func (s *Store) Agents() ([]ManagedAgent, error) {
|
|
rows, err := s.db.Query(`SELECT id,name,vmid,hostname,version,enrolled_at,last_seen,report_json,remote_address FROM agents ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
result := []ManagedAgent{}
|
|
for rows.Next() {
|
|
var a ManagedAgent
|
|
var raw string
|
|
if err = rows.Scan(&a.ID, &a.Name, &a.VMID, &a.Hostname, &a.Version, &a.EnrolledAt, &a.LastSeen, &raw, &a.RemoteAddress); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = json.Unmarshal([]byte(raw), &a.Report)
|
|
a.Online = time.Since(time.Unix(a.LastSeen, 0)) < agentOfflineAfter
|
|
result = append(result, a)
|
|
}
|
|
return result, rows.Err()
|
|
}
|
|
|
|
func (s *Store) DeleteAgent(id string) error {
|
|
_, err := s.db.Exec(`DELETE FROM agents WHERE id=?`, id)
|
|
return err
|
|
}
|
|
|
|
func runGuestAgent(opts agentOptions) error {
|
|
config, err := loadAgentConfig(opts.ConfigPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if opts.Server != "" {
|
|
config.Server = strings.TrimRight(opts.Server, "/")
|
|
}
|
|
if opts.Name != "" {
|
|
config.Name = opts.Name
|
|
}
|
|
if opts.VMID > 0 {
|
|
config.VMID = opts.VMID
|
|
}
|
|
if config.Server == "" {
|
|
return errors.New("укажите --agent-server")
|
|
}
|
|
if config.ID == "" {
|
|
if opts.EnrollmentToken == "" {
|
|
return errors.New("для первого запуска нужен --agent-enroll-token")
|
|
}
|
|
config, err = enrollRemote(config, opts.EnrollmentToken)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = saveAgentConfig(opts.ConfigPath, config); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
client := &http.Client{Timeout: 12 * time.Second}
|
|
for {
|
|
report := collectAgentReport()
|
|
guestActionResults.Lock()
|
|
report.ActionResults = append([]AgentActionResult(nil), guestActionResults.value...)
|
|
guestActionResults.Unlock()
|
|
data, _ := json.Marshal(report)
|
|
req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret)
|
|
response, requestErr := client.Do(req)
|
|
if requestErr == nil {
|
|
responseData, _ := io.ReadAll(io.LimitReader(response.Body, 64<<10))
|
|
response.Body.Close()
|
|
if response.StatusCode == http.StatusUnauthorized && opts.EnrollmentToken != "" {
|
|
fresh, enrollmentErr := enrollRemote(config, opts.EnrollmentToken)
|
|
if enrollmentErr == nil {
|
|
config = fresh
|
|
if saveErr := saveAgentConfig(opts.ConfigPath, config); saveErr != nil {
|
|
return saveErr
|
|
}
|
|
continue
|
|
}
|
|
requestErr = fmt.Errorf("повторная регистрация: %v", enrollmentErr)
|
|
}
|
|
if response.StatusCode >= 300 {
|
|
if requestErr == nil {
|
|
requestErr = fmt.Errorf("сервер вернул %s", response.Status)
|
|
}
|
|
}
|
|
if response.StatusCode == http.StatusOK {
|
|
var result agentReportResponse
|
|
if json.Unmarshal(responseData, &result) == nil {
|
|
if len(report.ActionResults) > 0 {
|
|
guestActionResults.Lock()
|
|
guestActionResults.value = nil
|
|
guestActionResults.Unlock()
|
|
}
|
|
if len(result.Commands) > 0 {
|
|
executed := make([]AgentActionResult, 0, len(result.Commands))
|
|
for _, command := range result.Commands {
|
|
executed = append(executed, executeAgentCommand(command))
|
|
}
|
|
guestActionResults.Lock()
|
|
guestActionResults.value = append(guestActionResults.value, executed...)
|
|
guestActionResults.Unlock()
|
|
}
|
|
}
|
|
if result.Update != nil && newerAgentVersion(result.Update.Version, version) {
|
|
if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil {
|
|
requestErr = fmt.Errorf("автообновление: %v", updateErr)
|
|
} else {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if requestErr != nil {
|
|
fmt.Fprintf(os.Stderr, "agent: %v\n", requestErr)
|
|
}
|
|
time.Sleep(10 * time.Second)
|
|
}
|
|
}
|
|
|
|
func newerAgentVersion(candidate, current string) bool {
|
|
parse := func(value string) [3]int {
|
|
value = strings.TrimPrefix(strings.TrimSpace(value), "v")
|
|
value = strings.SplitN(value, "-", 2)[0]
|
|
parts := strings.Split(value, ".")
|
|
var result [3]int
|
|
for i := 0; i < len(parts) && i < 3; i++ {
|
|
result[i], _ = strconv.Atoi(parts[i])
|
|
}
|
|
return result
|
|
}
|
|
if candidate == "" || current == "" || current == "dev" {
|
|
return false
|
|
}
|
|
next, installed := parse(candidate), parse(current)
|
|
for i := 0; i < 3; i++ {
|
|
if next[i] > installed[i] {
|
|
return true
|
|
}
|
|
if next[i] < installed[i] {
|
|
return false
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func applyAgentUpdate(client *http.Client, config agentConfig, update AgentUpdateManifest) error {
|
|
if !strings.HasPrefix(update.URL, "/api/agent/") || len(update.SHA256) != 64 {
|
|
return errors.New("сервер вернул некорректный манифест")
|
|
}
|
|
request, err := http.NewRequest(http.MethodGet, config.Server+update.URL, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret)
|
|
updateClient := *client
|
|
updateClient.Timeout = 2 * time.Minute
|
|
response, err := updateClient.Do(request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("загрузка вернула %s", response.Status)
|
|
}
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
temporary := executable + ".update"
|
|
_ = os.Remove(temporary)
|
|
file, err := os.OpenFile(temporary, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
keep := false
|
|
defer func() {
|
|
file.Close()
|
|
if !keep {
|
|
_ = os.Remove(temporary)
|
|
}
|
|
}()
|
|
hash := sha256.New()
|
|
written, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(response.Body, (64<<20)+1))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if written > 64<<20 {
|
|
return errors.New("бинарник превышает допустимый размер")
|
|
}
|
|
if err = file.Sync(); err != nil {
|
|
return err
|
|
}
|
|
if err = file.Close(); err != nil {
|
|
return err
|
|
}
|
|
actual := hex.EncodeToString(hash.Sum(nil))
|
|
if !hmac.Equal([]byte(actual), []byte(strings.ToLower(update.SHA256))) {
|
|
return errors.New("SHA-256 обновления не совпал")
|
|
}
|
|
if err = os.Chmod(temporary, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err = os.Rename(temporary, executable); err != nil {
|
|
return err
|
|
}
|
|
keep = true
|
|
return nil
|
|
}
|
|
|
|
func enrollRemote(config agentConfig, token string) (agentConfig, error) {
|
|
hostname, _ := os.Hostname()
|
|
machineID, _ := os.ReadFile("/etc/machine-id")
|
|
payload := agentEnrollmentRequest{Token: token, Name: config.Name, VMID: config.VMID, Hostname: hostname, MachineID: strings.TrimSpace(string(machineID))}
|
|
data, _ := json.Marshal(payload)
|
|
response, err := http.Post(config.Server+"/api/agent/enroll", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
return config, err
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != http.StatusOK {
|
|
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
|
return config, fmt.Errorf("регистрация: %s", strings.TrimSpace(string(message)))
|
|
}
|
|
var credentials agentConfig
|
|
if err = json.NewDecoder(response.Body).Decode(&credentials); err != nil {
|
|
return config, err
|
|
}
|
|
credentials.Server = config.Server
|
|
return credentials, nil
|
|
}
|
|
|
|
func loadAgentConfig(path string) (agentConfig, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return agentConfig{}, err
|
|
}
|
|
var c agentConfig
|
|
err = json.Unmarshal(data, &c)
|
|
return c, err
|
|
}
|
|
func saveAgentConfig(path string, c agentConfig) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
return err
|
|
}
|
|
data, _ := json.MarshalIndent(c, "", " ")
|
|
return os.WriteFile(path, data, 0600)
|
|
}
|
|
|
|
func collectAgentReport() AgentReport {
|
|
hostname, _ := os.Hostname()
|
|
machineID, _ := os.ReadFile("/etc/machine-id")
|
|
report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CollectedAt: time.Now().Unix()}
|
|
if data, err := os.ReadFile("/proc/uptime"); err == nil {
|
|
fields := strings.Fields(string(data))
|
|
if len(fields) > 0 {
|
|
value, _ := strconv.ParseFloat(fields[0], 64)
|
|
report.UptimeSeconds = int64(value)
|
|
}
|
|
}
|
|
if data, err := os.ReadFile("/proc/loadavg"); err == nil {
|
|
fields := strings.Fields(string(data))
|
|
if len(fields) > 0 {
|
|
report.Load1, _ = strconv.ParseFloat(fields[0], 64)
|
|
}
|
|
}
|
|
report.CPUPercent = sampleAgentCPU()
|
|
report.Interfaces = collectAgentInterfaces()
|
|
report.Processes = collectAgentProcesses()
|
|
report.Filesystems = collectAgentFilesystems()
|
|
report.ListenPorts = collectAgentListenPorts()
|
|
report.Pressure = collectAgentPressure()
|
|
report.OOMKills = cachedAgentOOMKills()
|
|
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
|
values := map[string]uint64{}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) >= 2 {
|
|
v, _ := strconv.ParseUint(f[1], 10, 64)
|
|
values[strings.TrimSuffix(f[0], ":")] = v * 1024
|
|
}
|
|
}
|
|
report.MemoryTotal = values["MemTotal"]
|
|
available := values["MemAvailable"]
|
|
if report.MemoryTotal > available {
|
|
report.MemoryUsed = report.MemoryTotal - available
|
|
}
|
|
report.SwapTotal = values["SwapTotal"]
|
|
if report.SwapTotal > values["SwapFree"] {
|
|
report.SwapUsed = report.SwapTotal - values["SwapFree"]
|
|
}
|
|
}
|
|
var stat syscall.Statfs_t
|
|
if syscall.Statfs("/", &stat) == nil {
|
|
report.RootTotal = stat.Blocks * uint64(stat.Bsize)
|
|
free := stat.Bavail * uint64(stat.Bsize)
|
|
if report.RootTotal > free {
|
|
report.RootUsed = report.RootTotal - free
|
|
}
|
|
}
|
|
report.Containers, report.DockerError = collectDockerContainers()
|
|
report.DockerAvailable = report.DockerError == ""
|
|
inventory := cachedAgentInventory()
|
|
report.OSName, report.Kernel, report.Services = inventory.OSName, inventory.Kernel, inventory.Services
|
|
report.Issues = cachedAgentIssues()
|
|
report.UpdatesAvailable, report.RebootRequired, report.InventoryAt = inventory.UpdatesAvailable, inventory.RebootRequired, inventory.CollectedAt.Unix()
|
|
return report
|
|
}
|
|
|
|
func collectAgentInterfaces() []AgentInterface {
|
|
interfaces, err := net.Interfaces()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
result := []AgentInterface{}
|
|
for _, iface := range interfaces {
|
|
if iface.Flags&net.FlagLoopback != 0 {
|
|
continue
|
|
}
|
|
addresses, _ := iface.Addrs()
|
|
item := AgentInterface{Name: iface.Name}
|
|
for _, address := range addresses {
|
|
value := address.String()
|
|
if host, _, err := net.ParseCIDR(value); err == nil {
|
|
value = host.String()
|
|
}
|
|
item.Addresses = append(item.Addresses, value)
|
|
}
|
|
if len(item.Addresses) > 0 {
|
|
result = append(result, item)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func cachedAgentInventory() agentInventory {
|
|
guestInventory.Lock()
|
|
defer guestInventory.Unlock()
|
|
if time.Since(guestInventory.value.CollectedAt) < 15*time.Minute {
|
|
return guestInventory.value
|
|
}
|
|
value := agentInventory{CollectedAt: time.Now()}
|
|
value.OSName = readOSPrettyName()
|
|
value.Kernel = strings.TrimSpace(runAgentCommand(3*time.Second, "uname", "-r"))
|
|
value.Services = collectAgentServices()
|
|
value.UpdatesAvailable = countAgentUpdates()
|
|
_, err := os.Stat("/var/run/reboot-required")
|
|
value.RebootRequired = err == nil
|
|
guestInventory.value = value
|
|
return value
|
|
}
|
|
|
|
func cachedAgentIssues() []AgentIssue {
|
|
guestIssues.Lock()
|
|
defer guestIssues.Unlock()
|
|
if time.Since(guestIssues.collectedAt) >= time.Minute {
|
|
guestIssues.value = collectAgentIssues()
|
|
guestIssues.collectedAt = time.Now()
|
|
}
|
|
return guestIssues.value
|
|
}
|
|
|
|
func readOSPrettyName() string {
|
|
data, err := os.ReadFile("/etc/os-release")
|
|
if err != nil {
|
|
return runtime.GOOS
|
|
}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
if strings.HasPrefix(line, "PRETTY_NAME=") {
|
|
return strings.Trim(strings.TrimPrefix(line, "PRETTY_NAME="), "\"")
|
|
}
|
|
}
|
|
return runtime.GOOS
|
|
}
|
|
|
|
func runAgentCommand(timeout time.Duration, name string, args ...string) string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
output, err := exec.CommandContext(ctx, name, args...).Output()
|
|
if err != nil && len(output) == 0 {
|
|
return ""
|
|
}
|
|
return string(output)
|
|
}
|
|
|
|
func collectAgentServices() []AgentService {
|
|
output := runAgentCommand(5*time.Second, "systemctl", "list-units", "--type=service", "--state=running,failed", "--no-legend", "--plain", "--no-pager")
|
|
result := []AgentService{}
|
|
for _, line := range strings.Split(output, "\n") {
|
|
fields := strings.Fields(line)
|
|
offset := 0
|
|
if len(fields) > 0 && fields[0] == "●" {
|
|
offset = 1
|
|
}
|
|
if len(fields) < offset+4 {
|
|
continue
|
|
}
|
|
description := ""
|
|
if len(fields) > offset+4 {
|
|
description = strings.Join(fields[offset+4:], " ")
|
|
}
|
|
result = append(result, AgentService{Name: fields[offset], State: fields[offset+2], SubState: fields[offset+3], Description: description})
|
|
if len(result) >= 300 {
|
|
break
|
|
}
|
|
}
|
|
return enrichAgentServices(result)
|
|
}
|
|
|
|
func enrichAgentServices(services []AgentService) []AgentService {
|
|
if len(services) == 0 {
|
|
return services
|
|
}
|
|
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec", "--property=MainPID", "--property=ExecStart", "--property=Requires", "--property=After"}
|
|
for _, service := range services {
|
|
args = append(args, service.Name)
|
|
}
|
|
show := runAgentCommand(8*time.Second, "systemctl", args...)
|
|
properties := map[string]map[string]string{}
|
|
paths := map[string]string{}
|
|
for _, record := range strings.Split(show, "\n\n") {
|
|
values := map[string]string{}
|
|
for _, line := range strings.Split(record, "\n") {
|
|
key, value, ok := strings.Cut(line, "=")
|
|
if ok {
|
|
values[key] = value
|
|
}
|
|
}
|
|
if id := values["Id"]; id != "" {
|
|
properties[id] = values
|
|
paths[id] = values["FragmentPath"]
|
|
}
|
|
}
|
|
pathArgs := []string{"-S"}
|
|
for _, service := range services {
|
|
if path := paths[service.Name]; path != "" {
|
|
pathArgs = append(pathArgs, path)
|
|
}
|
|
}
|
|
owners := map[string]string{}
|
|
packages := []string{}
|
|
if len(pathArgs) > 1 {
|
|
output := runAgentCommand(8*time.Second, "dpkg-query", pathArgs...)
|
|
for _, line := range strings.Split(output, "\n") {
|
|
left, path, ok := strings.Cut(line, ": ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
pkg := strings.TrimSpace(strings.Split(left, ",")[0])
|
|
path = strings.TrimSpace(path)
|
|
if pkg != "" && path != "" {
|
|
owners[path] = pkg
|
|
packages = append(packages, pkg)
|
|
}
|
|
}
|
|
}
|
|
priorities := map[string]string{}
|
|
if len(packages) > 0 {
|
|
packageArgs := []string{"-W", "-f=${binary:Package}\t${Priority}\n"}
|
|
packageArgs = append(packageArgs, packages...)
|
|
output := runAgentCommand(8*time.Second, "dpkg-query", packageArgs...)
|
|
for _, line := range strings.Split(output, "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) >= 2 {
|
|
priorities[fields[0]] = fields[1]
|
|
}
|
|
}
|
|
}
|
|
for i := range services {
|
|
service := &services[i]
|
|
service.UnitPath = paths[service.Name]
|
|
service.Package = owners[service.UnitPath]
|
|
service.Origin = classifyAgentService(service.UnitPath, service.Package, priorities[service.Package])
|
|
values := properties[service.Name]
|
|
service.Restarts, _ = strconv.ParseUint(values["NRestarts"], 10, 64)
|
|
exit, _ := strconv.ParseInt(values["ExecMainStatus"], 10, 32)
|
|
service.ExitCode = int(exit)
|
|
started, _ := strconv.ParseInt(values["ActiveEnterTimestampUSec"], 10, 64)
|
|
service.StartedAt = started / 1_000_000
|
|
service.MemoryBytes, _ = strconv.ParseUint(values["MemoryCurrent"], 10, 64)
|
|
cpu, _ := strconv.ParseUint(values["CPUUsageNSec"], 10, 64)
|
|
service.CPUSeconds = float64(cpu) / 1_000_000_000
|
|
pid, _ := strconv.Atoi(values["MainPID"])
|
|
service.MainPID = pid
|
|
service.ExecStart = compactSystemdExec(values["ExecStart"])
|
|
service.Requires = limitedFields(values["Requires"], 12)
|
|
service.After = limitedFields(values["After"], 12)
|
|
service.Flapping = service.Restarts >= 5 && service.StartedAt > time.Now().Add(-10*time.Minute).Unix()
|
|
if service.State == "failed" || service.SubState == "failed" || service.Flapping {
|
|
service.Logs = collectAgentJournal(service.Name, 8, "-2h")
|
|
}
|
|
}
|
|
ports := collectAgentListenPorts()
|
|
for i := range services {
|
|
for _, port := range ports {
|
|
if services[i].MainPID > 0 && port.PID == services[i].MainPID {
|
|
services[i].ListenPorts = appendUniqueInt(services[i].ListenPorts, port.Port)
|
|
}
|
|
}
|
|
}
|
|
return services
|
|
}
|
|
|
|
func classifyAgentService(path, pkg, priority string) string {
|
|
switch {
|
|
case strings.HasPrefix(path, "/etc/systemd/") || strings.HasPrefix(path, "/usr/local/"):
|
|
return "local"
|
|
case priority == "required" || priority == "important" || priority == "standard":
|
|
return "system"
|
|
case pkg != "":
|
|
return "added"
|
|
case strings.HasPrefix(path, "/run/systemd/"):
|
|
return "runtime"
|
|
default:
|
|
return "system"
|
|
}
|
|
}
|
|
|
|
func collectAgentIssues() []AgentIssue {
|
|
output := runAgentCommand(8*time.Second, "journalctl", "-p", "err..alert", "--since", "24 hours ago", "--reverse", "-n", "50", "--no-pager", "-o", "json")
|
|
return parseAgentIssues(output)
|
|
}
|
|
|
|
func parseAgentIssues(output string) []AgentIssue {
|
|
result := []AgentIssue{}
|
|
for _, line := range strings.Split(output, "\n") {
|
|
var entry struct {
|
|
Message any `json:"MESSAGE"`
|
|
Unit string `json:"_SYSTEMD_UNIT"`
|
|
Timestamp string `json:"__REALTIME_TIMESTAMP"`
|
|
}
|
|
if json.Unmarshal([]byte(line), &entry) != nil {
|
|
continue
|
|
}
|
|
message, ok := entry.Message.(string)
|
|
if !ok || strings.TrimSpace(message) == "" {
|
|
continue
|
|
}
|
|
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
|
|
unit := entry.Unit
|
|
if unit == "" {
|
|
unit = "system"
|
|
}
|
|
result = append(result, AgentIssue{Time: microseconds / 1_000_000, Unit: unit, Message: message})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func countAgentUpdates() int {
|
|
output := runAgentCommand(45*time.Second, "apt-get", "-s", "-o", "Debug::NoLocking=1", "upgrade")
|
|
count := 0
|
|
for _, line := range strings.Split(output, "\n") {
|
|
if strings.HasPrefix(line, "Inst ") {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func sampleAgentCPU() float64 {
|
|
total1, idle1 := readAgentCPU()
|
|
time.Sleep(100 * time.Millisecond)
|
|
total2, idle2 := readAgentCPU()
|
|
if total2 <= total1 {
|
|
return 0
|
|
}
|
|
totalDelta, idleDelta := total2-total1, idle2-idle1
|
|
if idleDelta > totalDelta {
|
|
return 0
|
|
}
|
|
return float64(totalDelta-idleDelta) / float64(totalDelta) * 100
|
|
}
|
|
|
|
func readAgentCPU() (uint64, uint64) {
|
|
data, err := os.ReadFile("/proc/stat")
|
|
if err != nil {
|
|
return 0, 0
|
|
}
|
|
line := strings.SplitN(string(data), "\n", 2)[0]
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 5 {
|
|
return 0, 0
|
|
}
|
|
var total uint64
|
|
values := make([]uint64, 0, len(fields)-1)
|
|
for _, field := range fields[1:] {
|
|
v, _ := strconv.ParseUint(field, 10, 64)
|
|
values = append(values, v)
|
|
total += v
|
|
}
|
|
idle := values[3]
|
|
if len(values) > 4 {
|
|
idle += values[4]
|
|
}
|
|
return total, idle
|
|
}
|
|
|
|
func collectDockerContainers() ([]AgentContainer, string) {
|
|
if _, err := os.Stat("/var/run/docker.sock"); err != nil {
|
|
return []AgentContainer{}, "Docker socket недоступен"
|
|
}
|
|
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
|
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
|
}}
|
|
defer transport.CloseIdleConnections()
|
|
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
|
response, err := client.Get("http://docker/containers/json?all=1")
|
|
if err != nil {
|
|
return nil, err.Error()
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != 200 {
|
|
return nil, response.Status
|
|
}
|
|
var raw []struct {
|
|
ID, Image, State, Status string
|
|
Names []string
|
|
Ports []struct {
|
|
IP string
|
|
PrivatePort, PublicPort int
|
|
Type string
|
|
}
|
|
}
|
|
if err = json.NewDecoder(response.Body).Decode(&raw); err != nil {
|
|
return nil, err.Error()
|
|
}
|
|
result := make([]AgentContainer, 0, len(raw))
|
|
for _, v := range raw {
|
|
name := strings.TrimPrefix(firstString(v.Names), "/")
|
|
item := AgentContainer{ID: shortID(v.ID), Name: name, Image: v.Image, State: v.State, Status: v.Status}
|
|
for _, p := range v.Ports {
|
|
if p.PublicPort > 0 {
|
|
item.Ports = append(item.Ports, fmt.Sprintf("%d:%d/%s", p.PublicPort, p.PrivatePort, p.Type))
|
|
}
|
|
}
|
|
inspect, inspectErr := client.Get("http://docker/containers/" + v.ID + "/json")
|
|
if inspectErr == nil {
|
|
var detail struct {
|
|
RestartCount int64
|
|
State struct{ Health *struct{ Status string } }
|
|
}
|
|
if json.NewDecoder(inspect.Body).Decode(&detail) == nil {
|
|
item.Restarts = detail.RestartCount
|
|
if detail.State.Health != nil {
|
|
item.Health = detail.State.Health.Status
|
|
}
|
|
}
|
|
inspect.Body.Close()
|
|
}
|
|
statsResponse, statsErr := client.Get("http://docker/containers/" + v.ID + "/stats?stream=false")
|
|
if statsErr == nil {
|
|
var stats struct {
|
|
CPUStats struct {
|
|
CPUUsage struct {
|
|
TotalUsage uint64 `json:"total_usage"`
|
|
} `json:"cpu_usage"`
|
|
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
|
OnlineCPUs uint64 `json:"online_cpus"`
|
|
} `json:"cpu_stats"`
|
|
PreCPUStats struct {
|
|
CPUUsage struct {
|
|
TotalUsage uint64 `json:"total_usage"`
|
|
} `json:"cpu_usage"`
|
|
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
|
} `json:"precpu_stats"`
|
|
MemoryStats struct {
|
|
Usage uint64 `json:"usage"`
|
|
Stats map[string]uint64 `json:"stats"`
|
|
} `json:"memory_stats"`
|
|
}
|
|
if json.NewDecoder(statsResponse.Body).Decode(&stats) == nil {
|
|
cpuDelta, systemDelta := uint64(0), uint64(0)
|
|
if stats.CPUStats.CPUUsage.TotalUsage >= stats.PreCPUStats.CPUUsage.TotalUsage {
|
|
cpuDelta = stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage
|
|
}
|
|
if stats.CPUStats.SystemCPUUsage >= stats.PreCPUStats.SystemCPUUsage {
|
|
systemDelta = stats.CPUStats.SystemCPUUsage - stats.PreCPUStats.SystemCPUUsage
|
|
}
|
|
cpus := stats.CPUStats.OnlineCPUs
|
|
if cpus == 0 {
|
|
cpus = 1
|
|
}
|
|
if systemDelta > 0 {
|
|
item.CPUPercent = float64(cpuDelta) / float64(systemDelta) * float64(cpus) * 100
|
|
}
|
|
item.MemoryUsage = stats.MemoryStats.Usage
|
|
if cache := stats.MemoryStats.Stats["inactive_file"]; item.MemoryUsage > cache {
|
|
item.MemoryUsage -= cache
|
|
}
|
|
}
|
|
statsResponse.Body.Close()
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
return result, ""
|
|
}
|
|
func firstString(v []string) string {
|
|
if len(v) > 0 {
|
|
return v[0]
|
|
}
|
|
return ""
|
|
}
|
|
func shortID(v string) string {
|
|
if len(v) > 12 {
|
|
return v[:12]
|
|
}
|
|
return v
|
|
}
|