package main import ( "bytes" "context" "crypto/hmac" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "sync" "syscall" "time" ) const agentOfflineAfter = 45 * time.Second type agentOptions struct { Server, EnrollmentToken, Name, ConfigPath string VMID int } type agentConfig struct { Server string `json:"server"` ID string `json:"id"` Secret string `json:"secret"` Name string `json:"name"` VMID int `json:"vmid"` } type AgentContainer struct { ID string `json:"id"` Name string `json:"name"` Image string `json:"image"` State string `json:"state"` Status string `json:"status"` Health string `json:"health,omitempty"` Restarts int64 `json:"restarts"` CPUPercent float64 `json:"cpuPercent"` MemoryUsage uint64 `json:"memoryUsage"` Ports []string `json:"ports,omitempty"` Compose bool `json:"compose"` UpdateAvailable bool `json:"updateAvailable"` CurrentVersion string `json:"currentVersion,omitempty"` CurrentDigest string `json:"currentDigest,omitempty"` AvailableVersion string `json:"availableVersion,omitempty"` AvailableDigest string `json:"availableDigest,omitempty"` } type AgentInterface struct { Name string `json:"name"` Addresses []string `json:"addresses"` } type AgentService struct { Name string `json:"name"` State string `json:"state"` SubState string `json:"subState"` Description string `json:"description"` Origin string `json:"origin"` UnitPath string `json:"unitPath,omitempty"` Package string `json:"package,omitempty"` Restarts uint64 `json:"restarts"` ExitCode int `json:"exitCode"` StartedAt int64 `json:"startedAt,omitempty"` MemoryBytes uint64 `json:"memoryBytes,omitempty"` CPUSeconds float64 `json:"cpuSeconds,omitempty"` MainPID int `json:"mainPid,omitempty"` ExecStart string `json:"execStart,omitempty"` Requires []string `json:"requires,omitempty"` After []string `json:"after,omitempty"` ListenPorts []int `json:"listenPorts,omitempty"` Logs []AgentLog `json:"logs,omitempty"` Flapping bool `json:"flapping"` } type AgentIssue struct { Time int64 `json:"time"` Unit string `json:"unit"` Message string `json:"message"` } type AgentReport struct { Hostname string `json:"hostname"` MachineID string `json:"machineId"` Version string `json:"version"` OS string `json:"os"` Arch string `json:"arch"` OSName string `json:"osName"` Kernel string `json:"kernel"` UptimeSeconds int64 `json:"uptimeSeconds"` Load1 float64 `json:"load1"` CPUPercent float64 `json:"cpuPercent"` CPUCount int `json:"cpuCount"` MemoryTotal uint64 `json:"memoryTotal"` MemoryUsed uint64 `json:"memoryUsed"` SwapTotal uint64 `json:"swapTotal"` SwapUsed uint64 `json:"swapUsed"` RootTotal uint64 `json:"rootTotal"` RootUsed uint64 `json:"rootUsed"` DockerAvailable bool `json:"dockerAvailable"` DockerError string `json:"dockerError,omitempty"` Containers []AgentContainer `json:"containers"` Interfaces []AgentInterface `json:"interfaces"` Services []AgentService `json:"services"` Issues []AgentIssue `json:"issues"` Processes []AgentProcess `json:"processes"` Filesystems []AgentFilesystem `json:"filesystems"` ListenPorts []AgentListenPort `json:"listenPorts"` Pressure AgentPressure `json:"pressure"` OOMKills []AgentLog `json:"oomKills"` UpdatesAvailable int `json:"updatesAvailable"` RebootRequired bool `json:"rebootRequired"` InventoryAt int64 `json:"inventoryAt"` CollectedAt int64 `json:"collectedAt"` ActionResults []AgentActionResult `json:"actionResults,omitempty"` } type agentInventory struct { OSName string Kernel string Services []AgentService UpdatesAvailable int RebootRequired bool CollectedAt time.Time } type AgentUpdateManifest struct { Version string `json:"version"` SHA256 string `json:"sha256"` URL string `json:"url"` } type agentReportResponse struct { Accepted bool `json:"accepted"` NetworkEnabled bool `json:"networkEnabled"` TrafficEnabled bool `json:"trafficEnabled"` Update *AgentUpdateManifest `json:"update,omitempty"` Commands []AgentCommand `json:"commands,omitempty"` } var guestInventory struct { sync.Mutex value agentInventory } var guestIssues struct { sync.Mutex value []AgentIssue collectedAt time.Time } var guestActionResults struct { sync.Mutex value []AgentActionResult } var agentBinaryInfo struct { sync.Once manifest AgentUpdateManifest path string err error } func currentAgentUpdate() (AgentUpdateManifest, string, error) { agentBinaryInfo.Do(func() { agentBinaryInfo.path, agentBinaryInfo.err = os.Executable() if agentBinaryInfo.err != nil { return } file, err := os.Open(agentBinaryInfo.path) if err != nil { agentBinaryInfo.err = err return } defer file.Close() hash := sha256.New() if _, err = io.Copy(hash, file); err != nil { agentBinaryInfo.err = err return } agentBinaryInfo.manifest = AgentUpdateManifest{Version: version, SHA256: hex.EncodeToString(hash.Sum(nil)), URL: "/api/agent/binary"} }) return agentBinaryInfo.manifest, agentBinaryInfo.path, agentBinaryInfo.err } type ManagedAgent struct { ID string `json:"id"` Name string `json:"name"` VMID int `json:"vmid"` Hostname string `json:"hostname"` Version string `json:"version"` LastSeen int64 `json:"lastSeen"` EnrolledAt int64 `json:"enrolledAt"` Online bool `json:"online"` RemoteAddress string `json:"remoteAddress"` Report AgentReport `json:"report"` } type agentEnrollmentRequest struct { Token, Name, Hostname, MachineID string VMID int } func randomHex(bytesCount int) (string, error) { b := make([]byte, bytesCount) if _, err := rand.Read(b); err != nil { return "", err } return hex.EncodeToString(b), nil } func tokenHash(value string) string { sum := sha256.Sum256([]byte(value)) return hex.EncodeToString(sum[:]) } func (s *Store) CreateAgentEnrollmentToken() (string, error) { token, err := randomHex(24) if err != nil { return "", err } _, err = s.db.Exec(`INSERT INTO agent_enrollment_tokens(token_hash,created_at,expires_at) VALUES(?,?,?)`, tokenHash(token), time.Now().Unix(), time.Now().Add(15*time.Minute).Unix()) return token, err } func (s *Store) EnrollAgent(req agentEnrollmentRequest) (agentConfig, error) { tx, err := s.db.Begin() if err != nil { return agentConfig{}, err } defer tx.Rollback() var expires int64 if err = tx.QueryRow(`SELECT expires_at FROM agent_enrollment_tokens WHERE token_hash=? AND used_at IS NULL`, tokenHash(req.Token)).Scan(&expires); err != nil || expires < time.Now().Unix() { return agentConfig{}, errors.New("одноразовый токен недействителен или истёк") } id, err := randomHex(12) if err != nil { return agentConfig{}, err } secret, err := randomHex(32) if err != nil { return agentConfig{}, err } name := strings.TrimSpace(req.Name) if name == "" { name = req.Hostname } now := time.Now().Unix() if _, err = tx.Exec(`INSERT INTO agents(id,name,vmid,hostname,machine_id,secret_hash,enrolled_at,last_seen,report_json,remote_address) VALUES(?,?,?,?,?,?,?,?,?,?)`, id, name, req.VMID, req.Hostname, req.MachineID, tokenHash(secret), now, now, `{}`, ""); err != nil { return agentConfig{}, err } if _, err = tx.Exec(`UPDATE agent_enrollment_tokens SET used_at=? WHERE token_hash=?`, now, tokenHash(req.Token)); err != nil { return agentConfig{}, err } if err = tx.Commit(); err != nil { return agentConfig{}, err } return agentConfig{ID: id, Secret: secret, Name: name, VMID: req.VMID}, nil } func (s *Store) SaveAgentReport(id, secret, remote string, report AgentReport) error { if !s.AuthenticateAgent(id, secret) { return errors.New("агент не авторизован") } s.CompleteAgentActions(id, report.ActionResults) report.ActionResults = nil data, _ := json.Marshal(report) _, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id) return err } func (s *Store) AuthenticateAgent(id, secret string) bool { var expected string return s.db.QueryRow(`SELECT secret_hash FROM agents WHERE id=?`, id).Scan(&expected) == nil && hmac.Equal([]byte(expected), []byte(tokenHash(secret))) } func (s *Store) Agents() ([]ManagedAgent, error) { rows, err := s.db.Query(`SELECT id,name,vmid,hostname,version,enrolled_at,last_seen,report_json,remote_address FROM agents ORDER BY name`) if err != nil { return nil, err } defer rows.Close() result := []ManagedAgent{} for rows.Next() { var a ManagedAgent var raw string if err = rows.Scan(&a.ID, &a.Name, &a.VMID, &a.Hostname, &a.Version, &a.EnrolledAt, &a.LastSeen, &raw, &a.RemoteAddress); err != nil { return nil, err } _ = json.Unmarshal([]byte(raw), &a.Report) a.Online = time.Since(time.Unix(a.LastSeen, 0)) < agentOfflineAfter result = append(result, a) } return result, rows.Err() } func (s *Store) DeleteAgent(id string) error { _, err := s.db.Exec(`DELETE FROM agents WHERE id=?`, id) return err } func runGuestAgent(opts agentOptions) error { config, err := loadAgentConfig(opts.ConfigPath) if err != nil && !os.IsNotExist(err) { return err } if opts.Server != "" { config.Server = strings.TrimRight(opts.Server, "/") } if opts.Name != "" { config.Name = opts.Name } if opts.VMID > 0 { config.VMID = opts.VMID } if config.Server == "" { return errors.New("укажите --agent-server") } if config.ID == "" { if opts.EnrollmentToken == "" { return errors.New("для первого запуска нужен --agent-enroll-token") } config, err = enrollRemote(config, opts.EnrollmentToken) if err != nil { return err } if err = saveAgentConfig(opts.ConfigPath, config); err != nil { return err } } var networkStop func() stopNetwork := func() { if networkStop != nil { networkStop() networkStop = nil } } defer stopNetwork() var trafficStop func() stopTraffic := func() { if trafficStop != nil { trafficStop() trafficStop = nil } } defer stopTraffic() client := &http.Client{Timeout: 12 * time.Second} for { report := collectAgentReport() guestActionResults.Lock() report.ActionResults = append([]AgentActionResult(nil), guestActionResults.value...) guestActionResults.Unlock() data, _ := json.Marshal(report) req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data)) req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret) response, requestErr := client.Do(req) if requestErr == nil { responseData, _ := io.ReadAll(io.LimitReader(response.Body, 64<<10)) response.Body.Close() if response.StatusCode == http.StatusUnauthorized && opts.EnrollmentToken != "" { fresh, enrollmentErr := enrollRemote(config, opts.EnrollmentToken) if enrollmentErr == nil { stopTraffic() stopNetwork() config = fresh if saveErr := saveAgentConfig(opts.ConfigPath, config); saveErr != nil { return saveErr } continue } requestErr = fmt.Errorf("повторная регистрация: %v", enrollmentErr) } if response.StatusCode >= 300 { if requestErr == nil { requestErr = fmt.Errorf("сервер вернул %s", response.Status) } } if response.StatusCode == http.StatusOK { var result agentReportResponse if json.Unmarshal(responseData, &result) == nil { if !result.NetworkEnabled { stopNetwork() } else if networkStop == nil { networkStop = startAgentNetwork(config) } if !result.TrafficEnabled { stopTraffic() } else if trafficStop == nil { trafficStop = startAgentTraffic(config, filepath.Dir(opts.ConfigPath)) } if len(report.ActionResults) > 0 { guestActionResults.Lock() guestActionResults.value = nil guestActionResults.Unlock() } if len(result.Commands) > 0 { executed := make([]AgentActionResult, 0, len(result.Commands)) for _, command := range result.Commands { executed = append(executed, executeAgentCommand(command)) } guestActionResults.Lock() guestActionResults.value = append(guestActionResults.value, executed...) guestActionResults.Unlock() } } if result.Update != nil && newerAgentVersion(result.Update.Version, version) { if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil { requestErr = fmt.Errorf("автообновление: %v", updateErr) } else { return nil } } } } if requestErr != nil { fmt.Fprintf(os.Stderr, "agent: %v\n", requestErr) } // Короткий интервал нужен для интерактивных команд из панели. Тяжёлый // системный инвентарь отдельно кэшируется и не запускается каждые 3 секунды. time.Sleep(3 * time.Second) } } func newerAgentVersion(candidate, current string) bool { parse := func(value string) [3]int { value = strings.TrimPrefix(strings.TrimSpace(value), "v") value = strings.SplitN(value, "-", 2)[0] parts := strings.Split(value, ".") var result [3]int for i := 0; i < len(parts) && i < 3; i++ { result[i], _ = strconv.Atoi(parts[i]) } return result } if candidate == "" || current == "" || current == "dev" { return false } next, installed := parse(candidate), parse(current) for i := 0; i < 3; i++ { if next[i] > installed[i] { return true } if next[i] < installed[i] { return false } } return false } func applyAgentUpdate(client *http.Client, config agentConfig, update AgentUpdateManifest) error { if !strings.HasPrefix(update.URL, "/api/agent/") || len(update.SHA256) != 64 { return errors.New("сервер вернул некорректный манифест") } request, err := http.NewRequest(http.MethodGet, config.Server+update.URL, nil) if err != nil { return err } request.Header.Set("Authorization", "Bearer "+config.ID+"."+config.Secret) updateClient := *client updateClient.Timeout = 2 * time.Minute response, err := updateClient.Do(request) if err != nil { return err } defer response.Body.Close() if response.StatusCode != http.StatusOK { return fmt.Errorf("загрузка вернула %s", response.Status) } executable, err := os.Executable() if err != nil { return err } temporary := executable + ".update" _ = os.Remove(temporary) file, err := os.OpenFile(temporary, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755) if err != nil { return err } keep := false defer func() { file.Close() if !keep { _ = os.Remove(temporary) } }() hash := sha256.New() written, err := io.Copy(io.MultiWriter(file, hash), io.LimitReader(response.Body, (64<<20)+1)) if err != nil { return err } if written > 64<<20 { return errors.New("бинарник превышает допустимый размер") } if err = file.Sync(); err != nil { return err } if err = file.Close(); err != nil { return err } actual := hex.EncodeToString(hash.Sum(nil)) if !hmac.Equal([]byte(actual), []byte(strings.ToLower(update.SHA256))) { return errors.New("SHA-256 обновления не совпал") } if err = os.Chmod(temporary, 0755); err != nil { return err } if err = os.Rename(temporary, executable); err != nil { return err } keep = true return nil } func enrollRemote(config agentConfig, token string) (agentConfig, error) { hostname, _ := os.Hostname() machineID, _ := os.ReadFile("/etc/machine-id") payload := agentEnrollmentRequest{Token: token, Name: config.Name, VMID: config.VMID, Hostname: hostname, MachineID: strings.TrimSpace(string(machineID))} data, _ := json.Marshal(payload) response, err := http.Post(config.Server+"/api/agent/enroll", "application/json", bytes.NewReader(data)) if err != nil { return config, err } defer response.Body.Close() if response.StatusCode != http.StatusOK { message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) return config, fmt.Errorf("регистрация: %s", strings.TrimSpace(string(message))) } var credentials agentConfig if err = json.NewDecoder(response.Body).Decode(&credentials); err != nil { return config, err } credentials.Server = config.Server return credentials, nil } func loadAgentConfig(path string) (agentConfig, error) { data, err := os.ReadFile(path) if err != nil { return agentConfig{}, err } var c agentConfig err = json.Unmarshal(data, &c) return c, err } func saveAgentConfig(path string, c agentConfig) error { if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return err } data, _ := json.MarshalIndent(c, "", " ") return os.WriteFile(path, data, 0600) } func collectAgentReport() AgentReport { hostname, _ := os.Hostname() machineID, _ := os.ReadFile("/etc/machine-id") report := AgentReport{Hostname: hostname, MachineID: strings.TrimSpace(string(machineID)), Version: version, OS: runtime.GOOS, Arch: runtime.GOARCH, CPUCount: runtime.NumCPU(), CollectedAt: time.Now().Unix()} if data, err := os.ReadFile("/proc/uptime"); err == nil { fields := strings.Fields(string(data)) if len(fields) > 0 { value, _ := strconv.ParseFloat(fields[0], 64) report.UptimeSeconds = int64(value) } } if data, err := os.ReadFile("/proc/loadavg"); err == nil { fields := strings.Fields(string(data)) if len(fields) > 0 { report.Load1, _ = strconv.ParseFloat(fields[0], 64) } } report.CPUPercent = sampleAgentCPU() report.Interfaces = collectAgentInterfaces() report.Processes = collectAgentProcesses() report.Filesystems = collectAgentFilesystems() report.ListenPorts = collectAgentListenPorts() report.Pressure = collectAgentPressure() report.OOMKills = cachedAgentOOMKills() if data, err := os.ReadFile("/proc/meminfo"); err == nil { values := map[string]uint64{} for _, line := range strings.Split(string(data), "\n") { f := strings.Fields(line) if len(f) >= 2 { v, _ := strconv.ParseUint(f[1], 10, 64) values[strings.TrimSuffix(f[0], ":")] = v * 1024 } } report.MemoryTotal = values["MemTotal"] available := values["MemAvailable"] if report.MemoryTotal > available { report.MemoryUsed = report.MemoryTotal - available } report.SwapTotal = values["SwapTotal"] if report.SwapTotal > values["SwapFree"] { report.SwapUsed = report.SwapTotal - values["SwapFree"] } } var stat syscall.Statfs_t if syscall.Statfs("/", &stat) == nil { report.RootTotal = stat.Blocks * uint64(stat.Bsize) free := stat.Bavail * uint64(stat.Bsize) if report.RootTotal > free { report.RootUsed = report.RootTotal - free } } report.Containers, report.DockerError = collectDockerContainers() report.DockerAvailable = report.DockerError == "" inventory := cachedAgentInventory() report.OSName, report.Kernel, report.Services = inventory.OSName, inventory.Kernel, inventory.Services report.Issues = cachedAgentIssues() report.UpdatesAvailable, report.RebootRequired, report.InventoryAt = inventory.UpdatesAvailable, inventory.RebootRequired, inventory.CollectedAt.Unix() return report } func collectAgentInterfaces() []AgentInterface { interfaces, err := net.Interfaces() if err != nil { return nil } result := []AgentInterface{} for _, iface := range interfaces { if iface.Flags&net.FlagLoopback != 0 { continue } addresses, _ := iface.Addrs() item := AgentInterface{Name: iface.Name} for _, address := range addresses { value := address.String() if host, _, err := net.ParseCIDR(value); err == nil { value = host.String() } item.Addresses = append(item.Addresses, value) } if len(item.Addresses) > 0 { result = append(result, item) } } return result } func cachedAgentInventory() agentInventory { guestInventory.Lock() defer guestInventory.Unlock() if time.Since(guestInventory.value.CollectedAt) < 15*time.Minute { return guestInventory.value } value := agentInventory{CollectedAt: time.Now()} value.OSName = readOSPrettyName() value.Kernel = strings.TrimSpace(runAgentCommand(3*time.Second, "uname", "-r")) value.Services = collectAgentServices() value.UpdatesAvailable = countAgentUpdates() _, err := os.Stat("/var/run/reboot-required") value.RebootRequired = err == nil guestInventory.value = value return value } func cachedAgentIssues() []AgentIssue { guestIssues.Lock() defer guestIssues.Unlock() if time.Since(guestIssues.collectedAt) >= time.Minute { guestIssues.value = collectAgentIssues() guestIssues.collectedAt = time.Now() } return guestIssues.value } func readOSPrettyName() string { data, err := os.ReadFile("/etc/os-release") if err != nil { return runtime.GOOS } for _, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(line, "PRETTY_NAME=") { return strings.Trim(strings.TrimPrefix(line, "PRETTY_NAME="), "\"") } } return runtime.GOOS } func runAgentCommand(timeout time.Duration, name string, args ...string) string { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() output, err := exec.CommandContext(ctx, name, args...).Output() if err != nil && len(output) == 0 { return "" } return string(output) } func collectAgentServices() []AgentService { output := runAgentCommand(5*time.Second, "systemctl", "list-units", "--type=service", "--state=running,failed", "--no-legend", "--plain", "--no-pager") result := []AgentService{} for _, line := range strings.Split(output, "\n") { fields := strings.Fields(line) offset := 0 if len(fields) > 0 && fields[0] == "●" { offset = 1 } if len(fields) < offset+4 { continue } description := "" if len(fields) > offset+4 { description = strings.Join(fields[offset+4:], " ") } result = append(result, AgentService{Name: fields[offset], State: fields[offset+2], SubState: fields[offset+3], Description: description}) if len(result) >= 300 { break } } return enrichAgentServices(result) } func enrichAgentServices(services []AgentService) []AgentService { if len(services) == 0 { return services } args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec", "--property=MainPID", "--property=ExecStart", "--property=Requires", "--property=After"} for _, service := range services { args = append(args, service.Name) } show := runAgentCommand(8*time.Second, "systemctl", args...) properties := map[string]map[string]string{} paths := map[string]string{} for _, record := range strings.Split(show, "\n\n") { values := map[string]string{} for _, line := range strings.Split(record, "\n") { key, value, ok := strings.Cut(line, "=") if ok { values[key] = value } } if id := values["Id"]; id != "" { properties[id] = values paths[id] = values["FragmentPath"] } } pathArgs := []string{"-S"} for _, service := range services { if path := paths[service.Name]; path != "" { pathArgs = append(pathArgs, path) } } owners := map[string]string{} packages := []string{} if len(pathArgs) > 1 { output := runAgentCommand(8*time.Second, "dpkg-query", pathArgs...) for _, line := range strings.Split(output, "\n") { left, path, ok := strings.Cut(line, ": ") if !ok { continue } pkg := strings.TrimSpace(strings.Split(left, ",")[0]) path = strings.TrimSpace(path) if pkg != "" && path != "" { owners[path] = pkg packages = append(packages, pkg) } } } priorities := map[string]string{} if len(packages) > 0 { packageArgs := []string{"-W", "-f=${binary:Package}\t${Priority}\n"} packageArgs = append(packageArgs, packages...) output := runAgentCommand(8*time.Second, "dpkg-query", packageArgs...) for _, line := range strings.Split(output, "\n") { fields := strings.Fields(line) if len(fields) >= 2 { priorities[fields[0]] = fields[1] } } } for i := range services { service := &services[i] service.UnitPath = paths[service.Name] service.Package = owners[service.UnitPath] service.Origin = classifyAgentService(service.UnitPath, service.Package, priorities[service.Package]) values := properties[service.Name] service.Restarts, _ = strconv.ParseUint(values["NRestarts"], 10, 64) exit, _ := strconv.ParseInt(values["ExecMainStatus"], 10, 32) service.ExitCode = int(exit) started, _ := strconv.ParseInt(values["ActiveEnterTimestampUSec"], 10, 64) service.StartedAt = started / 1_000_000 service.MemoryBytes, _ = strconv.ParseUint(values["MemoryCurrent"], 10, 64) cpu, _ := strconv.ParseUint(values["CPUUsageNSec"], 10, 64) service.CPUSeconds = float64(cpu) / 1_000_000_000 pid, _ := strconv.Atoi(values["MainPID"]) service.MainPID = pid service.ExecStart = compactSystemdExec(values["ExecStart"]) service.Requires = limitedFields(values["Requires"], 12) service.After = limitedFields(values["After"], 12) service.Flapping = service.Restarts >= 5 && service.StartedAt > time.Now().Add(-10*time.Minute).Unix() if service.State == "failed" || service.SubState == "failed" || service.Flapping { service.Logs = collectAgentJournal(service.Name, 8, "-2h") } } ports := collectAgentListenPorts() for i := range services { for _, port := range ports { if services[i].MainPID > 0 && port.PID == services[i].MainPID { services[i].ListenPorts = appendUniqueInt(services[i].ListenPorts, port.Port) } } } return services } func classifyAgentService(path, pkg, priority string) string { switch { case strings.HasPrefix(path, "/etc/systemd/") || strings.HasPrefix(path, "/usr/local/"): return "local" case priority == "required" || priority == "important" || priority == "standard": return "system" case pkg != "": return "added" case strings.HasPrefix(path, "/run/systemd/"): return "runtime" default: return "system" } } func collectAgentIssues() []AgentIssue { output := runAgentCommand(8*time.Second, "journalctl", "-p", "err..alert", "--since", "24 hours ago", "--reverse", "-n", "50", "--no-pager", "-o", "json") return parseAgentIssues(output) } func parseAgentIssues(output string) []AgentIssue { result := []AgentIssue{} for _, line := range strings.Split(output, "\n") { var entry struct { Message any `json:"MESSAGE"` Unit string `json:"_SYSTEMD_UNIT"` Timestamp string `json:"__REALTIME_TIMESTAMP"` } if json.Unmarshal([]byte(line), &entry) != nil { continue } message, ok := entry.Message.(string) if !ok || strings.TrimSpace(message) == "" { continue } microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64) unit := entry.Unit if unit == "" { unit = "system" } result = append(result, AgentIssue{Time: microseconds / 1_000_000, Unit: unit, Message: message}) } return result } func countAgentUpdates() int { output := runAgentCommand(45*time.Second, "apt-get", "-s", "-o", "Debug::NoLocking=1", "upgrade") count := 0 for _, line := range strings.Split(output, "\n") { if strings.HasPrefix(line, "Inst ") { count++ } } return count } func sampleAgentCPU() float64 { total1, idle1 := readAgentCPU() time.Sleep(100 * time.Millisecond) total2, idle2 := readAgentCPU() if total2 <= total1 { return 0 } totalDelta, idleDelta := total2-total1, idle2-idle1 if idleDelta > totalDelta { return 0 } return float64(totalDelta-idleDelta) / float64(totalDelta) * 100 } func readAgentCPU() (uint64, uint64) { data, err := os.ReadFile("/proc/stat") if err != nil { return 0, 0 } line := strings.SplitN(string(data), "\n", 2)[0] fields := strings.Fields(line) if len(fields) < 5 { return 0, 0 } var total uint64 values := make([]uint64, 0, len(fields)-1) for _, field := range fields[1:] { v, _ := strconv.ParseUint(field, 10, 64) values = append(values, v) total += v } idle := values[3] if len(values) > 4 { idle += values[4] } return total, idle } func collectDockerContainers() ([]AgentContainer, string) { if _, err := os.Stat("/var/run/docker.sock"); err != nil { return []AgentContainer{}, "Docker socket недоступен" } transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock") }} defer transport.CloseIdleConnections() client := &http.Client{Transport: transport, Timeout: 5 * time.Second} response, err := client.Get("http://docker/containers/json?all=1") if err != nil { return nil, err.Error() } defer response.Body.Close() if response.StatusCode != 200 { return nil, response.Status } var raw []struct { ID, Image, State, Status string Names []string Ports []struct { IP string PrivatePort, PublicPort int Type string } } if err = json.NewDecoder(response.Body).Decode(&raw); err != nil { return nil, err.Error() } result := make([]AgentContainer, 0, len(raw)) for _, v := range raw { name := strings.TrimPrefix(firstString(v.Names), "/") item := AgentContainer{ID: shortID(v.ID), Name: name, Image: v.Image, State: v.State, Status: v.Status} for _, p := range v.Ports { if p.PublicPort > 0 { item.Ports = append(item.Ports, fmt.Sprintf("%d:%d/%s", p.PublicPort, p.PrivatePort, p.Type)) } } inspect, inspectErr := client.Get("http://docker/containers/" + v.ID + "/json") if inspectErr == nil { var detail struct { Image string RestartCount int64 State struct{ Health *struct{ Status string } } Config struct { Labels map[string]string } } if json.NewDecoder(inspect.Body).Decode(&detail) == nil { item.Restarts = detail.RestartCount if detail.State.Health != nil { item.Health = detail.State.Health.Status } item.Compose = detail.Config.Labels["com.docker.compose.service"] != "" item.CurrentVersion, item.CurrentDigest, _ = inspectDockerImage(client, detail.Image, v.Image) var availableID string item.AvailableVersion, item.AvailableDigest, availableID = inspectDockerImage(client, v.Image, v.Image) item.UpdateAvailable = detail.Image != "" && availableID != "" && detail.Image != availableID } inspect.Body.Close() } statsResponse, statsErr := client.Get("http://docker/containers/" + v.ID + "/stats?stream=false") if statsErr == nil { var stats struct { CPUStats struct { CPUUsage struct { TotalUsage uint64 `json:"total_usage"` } `json:"cpu_usage"` SystemCPUUsage uint64 `json:"system_cpu_usage"` OnlineCPUs uint64 `json:"online_cpus"` } `json:"cpu_stats"` PreCPUStats struct { CPUUsage struct { TotalUsage uint64 `json:"total_usage"` } `json:"cpu_usage"` SystemCPUUsage uint64 `json:"system_cpu_usage"` } `json:"precpu_stats"` MemoryStats struct { Usage uint64 `json:"usage"` Stats map[string]uint64 `json:"stats"` } `json:"memory_stats"` } if json.NewDecoder(statsResponse.Body).Decode(&stats) == nil { cpuDelta, systemDelta := uint64(0), uint64(0) if stats.CPUStats.CPUUsage.TotalUsage >= stats.PreCPUStats.CPUUsage.TotalUsage { cpuDelta = stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage } if stats.CPUStats.SystemCPUUsage >= stats.PreCPUStats.SystemCPUUsage { systemDelta = stats.CPUStats.SystemCPUUsage - stats.PreCPUStats.SystemCPUUsage } cpus := stats.CPUStats.OnlineCPUs if cpus == 0 { cpus = 1 } if systemDelta > 0 { item.CPUPercent = float64(cpuDelta) / float64(systemDelta) * float64(cpus) * 100 } item.MemoryUsage = stats.MemoryStats.Usage if cache := stats.MemoryStats.Stats["inactive_file"]; item.MemoryUsage > cache { item.MemoryUsage -= cache } } statsResponse.Body.Close() } result = append(result, item) } return result, "" } func inspectDockerImage(client *http.Client, reference, fallback string) (string, string, string) { if reference == "" { return "", "", "" } response, err := client.Get("http://docker/images/" + url.PathEscape(reference) + "/json") if err != nil { return "", "", "" } defer response.Body.Close() var detail struct { ID string RepoTags []string RepoDigests []string Config struct{ Labels map[string]string } } if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&detail) != nil { return "", "", "" } version := firstNonEmpty(detail.Config.Labels["org.opencontainers.image.version"], detail.Config.Labels["version"], firstString(detail.RepoTags), fallback, shortDockerDigest(detail.ID)) digest := "" if len(detail.RepoDigests) > 0 { digest = detail.RepoDigests[0] if _, value, found := strings.Cut(digest, "@"); found { digest = value } } if digest == "" { digest = detail.ID } return version, shortDockerDigest(digest), detail.ID } func shortDockerDigest(value string) string { value = strings.TrimPrefix(value, "sha256:") if len(value) > 12 { value = value[:12] } return value } func firstString(v []string) string { if len(v) > 0 { return v[0] } return "" } func shortID(v string) string { if len(v) > 12 { return v[:12] } return v }