Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21b4f528e0 | ||
|
|
c50b4cf986 | ||
|
|
338d53cb2f | ||
|
|
a2525b53f0 |
27
README.md
27
README.md
@@ -342,6 +342,33 @@ APT-обновления и необходимость перезагрузки.
|
||||
health-check, число перезапусков, CPU и память. Агент сам подключается к Dashboard
|
||||
каждые 10 секунд; открывать входящий порт внутри гостевой системы не нужно.
|
||||
|
||||
В подробностях агента systemd-службы разделяются на системные, установленные
|
||||
дополнительно и локальные unit-файлы. Для каждой службы показываются пакет и путь
|
||||
unit-файла, время запуска, число перезапусков, последний exit code, память и
|
||||
накопленное CPU-время, когда соответствующий accounting доступен в systemd.
|
||||
|
||||
Расширенная диагностика агента включает последние сообщения проблемных служб,
|
||||
`Requires`/`After`, PID, команду запуска и слушающие порты; выявляет частые
|
||||
рестарты, OOM-kill и процессы в D-state. Отдельно отображаются top процессов по
|
||||
CPU и RAM с накопленным I/O, swap, PSI pressure, все постоянные файловые системы,
|
||||
их заполнение и использование inode.
|
||||
|
||||
Панель «Диагностика homelab» и карта сопоставляют известные IP и порты с VM/LXC,
|
||||
агентом, systemd-службой или Docker-контейнером и настроенной HTTP/TCP-проверкой.
|
||||
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
|
||||
помечаются как предположение.
|
||||
|
||||
Для сложных reverse proxy и Docker-сетей в управлении сервисами можно сохранить
|
||||
ручную связь `сервис → агент/VMID → systemd/Docker/процесс → порт`. Она имеет
|
||||
приоритет над автоматическим определением и используется для группировки
|
||||
связанных уведомлений по первопричине.
|
||||
|
||||
Dashboard хранит семь дней истории CPU, RAM, swap, PSI и файловых систем каждого
|
||||
агента. Из подробностей агента доступны только заранее разрешённые действия:
|
||||
перезапуск известной systemd-службы или Docker-контейнера, `reset-failed` и
|
||||
обновление списка пакетов. Агент не принимает произвольные shell-команды; каждое
|
||||
действие требует подтверждения в браузере и сохраняется в журнале управления.
|
||||
|
||||
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
|
||||
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
|
||||
минут и после регистрации заменяется индивидуальным секретом агента.
|
||||
|
||||
230
agent.go
230
agent.go
@@ -58,10 +58,25 @@ type AgentInterface struct {
|
||||
}
|
||||
|
||||
type AgentService struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SubState string `json:"subState"`
|
||||
Description string `json:"description"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
SubState string `json:"subState"`
|
||||
Description string `json:"description"`
|
||||
Origin string `json:"origin"`
|
||||
UnitPath string `json:"unitPath,omitempty"`
|
||||
Package string `json:"package,omitempty"`
|
||||
Restarts uint64 `json:"restarts"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
StartedAt int64 `json:"startedAt,omitempty"`
|
||||
MemoryBytes uint64 `json:"memoryBytes,omitempty"`
|
||||
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
|
||||
MainPID int `json:"mainPid,omitempty"`
|
||||
ExecStart string `json:"execStart,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
After []string `json:"after,omitempty"`
|
||||
ListenPorts []int `json:"listenPorts,omitempty"`
|
||||
Logs []AgentLog `json:"logs,omitempty"`
|
||||
Flapping bool `json:"flapping"`
|
||||
}
|
||||
|
||||
type AgentIssue struct {
|
||||
@@ -71,30 +86,38 @@ type AgentIssue struct {
|
||||
}
|
||||
|
||||
type AgentReport struct {
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
OSName string `json:"osName"`
|
||||
Kernel string `json:"kernel"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
Interfaces []AgentInterface `json:"interfaces"`
|
||||
Services []AgentService `json:"services"`
|
||||
Issues []AgentIssue `json:"issues"`
|
||||
UpdatesAvailable int `json:"updatesAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
InventoryAt int64 `json:"inventoryAt"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
Hostname string `json:"hostname"`
|
||||
MachineID string `json:"machineId"`
|
||||
Version string `json:"version"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
OSName string `json:"osName"`
|
||||
Kernel string `json:"kernel"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Load1 float64 `json:"load1"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryTotal uint64 `json:"memoryTotal"`
|
||||
MemoryUsed uint64 `json:"memoryUsed"`
|
||||
SwapTotal uint64 `json:"swapTotal"`
|
||||
SwapUsed uint64 `json:"swapUsed"`
|
||||
RootTotal uint64 `json:"rootTotal"`
|
||||
RootUsed uint64 `json:"rootUsed"`
|
||||
DockerAvailable bool `json:"dockerAvailable"`
|
||||
DockerError string `json:"dockerError,omitempty"`
|
||||
Containers []AgentContainer `json:"containers"`
|
||||
Interfaces []AgentInterface `json:"interfaces"`
|
||||
Services []AgentService `json:"services"`
|
||||
Issues []AgentIssue `json:"issues"`
|
||||
Processes []AgentProcess `json:"processes"`
|
||||
Filesystems []AgentFilesystem `json:"filesystems"`
|
||||
ListenPorts []AgentListenPort `json:"listenPorts"`
|
||||
Pressure AgentPressure `json:"pressure"`
|
||||
OOMKills []AgentLog `json:"oomKills"`
|
||||
UpdatesAvailable int `json:"updatesAvailable"`
|
||||
RebootRequired bool `json:"rebootRequired"`
|
||||
InventoryAt int64 `json:"inventoryAt"`
|
||||
CollectedAt int64 `json:"collectedAt"`
|
||||
ActionResults []AgentActionResult `json:"actionResults,omitempty"`
|
||||
}
|
||||
|
||||
type agentInventory struct {
|
||||
@@ -115,6 +138,7 @@ type AgentUpdateManifest struct {
|
||||
type agentReportResponse struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Update *AgentUpdateManifest `json:"update,omitempty"`
|
||||
Commands []AgentCommand `json:"commands,omitempty"`
|
||||
}
|
||||
|
||||
var guestInventory struct {
|
||||
@@ -128,6 +152,11 @@ var guestIssues struct {
|
||||
collectedAt time.Time
|
||||
}
|
||||
|
||||
var guestActionResults struct {
|
||||
sync.Mutex
|
||||
value []AgentActionResult
|
||||
}
|
||||
|
||||
var agentBinaryInfo struct {
|
||||
sync.Once
|
||||
manifest AgentUpdateManifest
|
||||
@@ -236,6 +265,8 @@ func (s *Store) SaveAgentReport(id, secret, remote string, report AgentReport) e
|
||||
if !s.AuthenticateAgent(id, secret) {
|
||||
return errors.New("агент не авторизован")
|
||||
}
|
||||
s.CompleteAgentActions(id, report.ActionResults)
|
||||
report.ActionResults = nil
|
||||
data, _ := json.Marshal(report)
|
||||
_, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id)
|
||||
return err
|
||||
@@ -303,6 +334,9 @@ func runGuestAgent(opts agentOptions) error {
|
||||
client := &http.Client{Timeout: 12 * time.Second}
|
||||
for {
|
||||
report := collectAgentReport()
|
||||
guestActionResults.Lock()
|
||||
report.ActionResults = append([]AgentActionResult(nil), guestActionResults.value...)
|
||||
guestActionResults.Unlock()
|
||||
data, _ := json.Marshal(report)
|
||||
req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
@@ -329,7 +363,23 @@ func runGuestAgent(opts agentOptions) error {
|
||||
}
|
||||
if response.StatusCode == http.StatusOK {
|
||||
var result agentReportResponse
|
||||
if json.Unmarshal(responseData, &result) == nil && result.Update != nil && newerAgentVersion(result.Update.Version, version) {
|
||||
if json.Unmarshal(responseData, &result) == nil {
|
||||
if len(report.ActionResults) > 0 {
|
||||
guestActionResults.Lock()
|
||||
guestActionResults.value = nil
|
||||
guestActionResults.Unlock()
|
||||
}
|
||||
if len(result.Commands) > 0 {
|
||||
executed := make([]AgentActionResult, 0, len(result.Commands))
|
||||
for _, command := range result.Commands {
|
||||
executed = append(executed, executeAgentCommand(command))
|
||||
}
|
||||
guestActionResults.Lock()
|
||||
guestActionResults.value = append(guestActionResults.value, executed...)
|
||||
guestActionResults.Unlock()
|
||||
}
|
||||
}
|
||||
if result.Update != nil && newerAgentVersion(result.Update.Version, version) {
|
||||
if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil {
|
||||
requestErr = fmt.Errorf("автообновление: %v", updateErr)
|
||||
} else {
|
||||
@@ -493,6 +543,11 @@ func collectAgentReport() AgentReport {
|
||||
}
|
||||
report.CPUPercent = sampleAgentCPU()
|
||||
report.Interfaces = collectAgentInterfaces()
|
||||
report.Processes = collectAgentProcesses()
|
||||
report.Filesystems = collectAgentFilesystems()
|
||||
report.ListenPorts = collectAgentListenPorts()
|
||||
report.Pressure = collectAgentPressure()
|
||||
report.OOMKills = cachedAgentOOMKills()
|
||||
if data, err := os.ReadFile("/proc/meminfo"); err == nil {
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
@@ -507,6 +562,10 @@ func collectAgentReport() AgentReport {
|
||||
if report.MemoryTotal > available {
|
||||
report.MemoryUsed = report.MemoryTotal - available
|
||||
}
|
||||
report.SwapTotal = values["SwapTotal"]
|
||||
if report.SwapTotal > values["SwapFree"] {
|
||||
report.SwapUsed = report.SwapTotal - values["SwapFree"]
|
||||
}
|
||||
}
|
||||
var stat syscall.Statfs_t
|
||||
if syscall.Statfs("/", &stat) == nil {
|
||||
@@ -595,7 +654,7 @@ func runAgentCommand(timeout time.Duration, name string, args ...string) string
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, name, args...).Output()
|
||||
if err != nil {
|
||||
if err != nil && len(output) == 0 {
|
||||
return ""
|
||||
}
|
||||
return string(output)
|
||||
@@ -622,7 +681,116 @@ func collectAgentServices() []AgentService {
|
||||
break
|
||||
}
|
||||
}
|
||||
return result
|
||||
return enrichAgentServices(result)
|
||||
}
|
||||
|
||||
func enrichAgentServices(services []AgentService) []AgentService {
|
||||
if len(services) == 0 {
|
||||
return services
|
||||
}
|
||||
args := []string{"show", "--no-pager", "--property=Id", "--property=FragmentPath", "--property=NRestarts", "--property=ExecMainStatus", "--property=ActiveEnterTimestampUSec", "--property=MemoryCurrent", "--property=CPUUsageNSec", "--property=MainPID", "--property=ExecStart", "--property=Requires", "--property=After"}
|
||||
for _, service := range services {
|
||||
args = append(args, service.Name)
|
||||
}
|
||||
show := runAgentCommand(8*time.Second, "systemctl", args...)
|
||||
properties := map[string]map[string]string{}
|
||||
paths := map[string]string{}
|
||||
for _, record := range strings.Split(show, "\n\n") {
|
||||
values := map[string]string{}
|
||||
for _, line := range strings.Split(record, "\n") {
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if ok {
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
if id := values["Id"]; id != "" {
|
||||
properties[id] = values
|
||||
paths[id] = values["FragmentPath"]
|
||||
}
|
||||
}
|
||||
pathArgs := []string{"-S"}
|
||||
for _, service := range services {
|
||||
if path := paths[service.Name]; path != "" {
|
||||
pathArgs = append(pathArgs, path)
|
||||
}
|
||||
}
|
||||
owners := map[string]string{}
|
||||
packages := []string{}
|
||||
if len(pathArgs) > 1 {
|
||||
output := runAgentCommand(8*time.Second, "dpkg-query", pathArgs...)
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
left, path, ok := strings.Cut(line, ": ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
pkg := strings.TrimSpace(strings.Split(left, ",")[0])
|
||||
path = strings.TrimSpace(path)
|
||||
if pkg != "" && path != "" {
|
||||
owners[path] = pkg
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
}
|
||||
}
|
||||
priorities := map[string]string{}
|
||||
if len(packages) > 0 {
|
||||
packageArgs := []string{"-W", "-f=${binary:Package}\t${Priority}\n"}
|
||||
packageArgs = append(packageArgs, packages...)
|
||||
output := runAgentCommand(8*time.Second, "dpkg-query", packageArgs...)
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 {
|
||||
priorities[fields[0]] = fields[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range services {
|
||||
service := &services[i]
|
||||
service.UnitPath = paths[service.Name]
|
||||
service.Package = owners[service.UnitPath]
|
||||
service.Origin = classifyAgentService(service.UnitPath, service.Package, priorities[service.Package])
|
||||
values := properties[service.Name]
|
||||
service.Restarts, _ = strconv.ParseUint(values["NRestarts"], 10, 64)
|
||||
exit, _ := strconv.ParseInt(values["ExecMainStatus"], 10, 32)
|
||||
service.ExitCode = int(exit)
|
||||
started, _ := strconv.ParseInt(values["ActiveEnterTimestampUSec"], 10, 64)
|
||||
service.StartedAt = started / 1_000_000
|
||||
service.MemoryBytes, _ = strconv.ParseUint(values["MemoryCurrent"], 10, 64)
|
||||
cpu, _ := strconv.ParseUint(values["CPUUsageNSec"], 10, 64)
|
||||
service.CPUSeconds = float64(cpu) / 1_000_000_000
|
||||
pid, _ := strconv.Atoi(values["MainPID"])
|
||||
service.MainPID = pid
|
||||
service.ExecStart = compactSystemdExec(values["ExecStart"])
|
||||
service.Requires = limitedFields(values["Requires"], 12)
|
||||
service.After = limitedFields(values["After"], 12)
|
||||
service.Flapping = service.Restarts >= 5 && service.StartedAt > time.Now().Add(-10*time.Minute).Unix()
|
||||
if service.State == "failed" || service.SubState == "failed" || service.Flapping {
|
||||
service.Logs = collectAgentJournal(service.Name, 8, "-2h")
|
||||
}
|
||||
}
|
||||
ports := collectAgentListenPorts()
|
||||
for i := range services {
|
||||
for _, port := range ports {
|
||||
if services[i].MainPID > 0 && port.PID == services[i].MainPID {
|
||||
services[i].ListenPorts = appendUniqueInt(services[i].ListenPorts, port.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
return services
|
||||
}
|
||||
|
||||
func classifyAgentService(path, pkg, priority string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(path, "/etc/systemd/") || strings.HasPrefix(path, "/usr/local/"):
|
||||
return "local"
|
||||
case priority == "required" || priority == "important" || priority == "standard":
|
||||
return "system"
|
||||
case pkg != "":
|
||||
return "added"
|
||||
case strings.HasPrefix(path, "/run/systemd/"):
|
||||
return "runtime"
|
||||
default:
|
||||
return "system"
|
||||
}
|
||||
}
|
||||
|
||||
func collectAgentIssues() []AgentIssue {
|
||||
|
||||
210
agent_actions.go
Normal file
210
agent_actions.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AgentCommand struct {
|
||||
ID int64 `json:"id"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target,omitempty"`
|
||||
}
|
||||
|
||||
type AgentActionResult struct {
|
||||
ID int64 `json:"id"`
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type AgentAction struct {
|
||||
ID int64 `json:"id"`
|
||||
AgentID string `json:"agentId"`
|
||||
AgentName string `json:"agentName"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
RequestedAt int64 `json:"requestedAt"`
|
||||
StartedAt *int64 `json:"startedAt,omitempty"`
|
||||
CompletedAt *int64 `json:"completedAt,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Result string `json:"result"`
|
||||
}
|
||||
|
||||
var safeUnitName = regexp.MustCompile(`^[A-Za-z0-9_.@:-]+\.service$`)
|
||||
var safeContainerName = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
|
||||
|
||||
func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, error) {
|
||||
action, target = strings.TrimSpace(action), strings.TrimSpace(target)
|
||||
var raw, name string
|
||||
if err := s.db.QueryRow(`SELECT report_json,name FROM agents WHERE id=?`, agentID).Scan(&raw, &name); err != nil {
|
||||
return AgentAction{}, fmt.Errorf("агент не найден")
|
||||
}
|
||||
var report AgentReport
|
||||
_ = json.Unmarshal([]byte(raw), &report)
|
||||
valid := false
|
||||
switch action {
|
||||
case "systemd-restart", "systemd-reset-failed":
|
||||
if safeUnitName.MatchString(target) {
|
||||
for _, service := range report.Services {
|
||||
if service.Name == target {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
case "docker-restart":
|
||||
if safeContainerName.MatchString(target) {
|
||||
for _, container := range report.Containers {
|
||||
if container.ID == target || container.Name == target {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
case "updates-refresh":
|
||||
valid = target == ""
|
||||
}
|
||||
if !valid {
|
||||
return AgentAction{}, fmt.Errorf("действие или цель не входят в разрешённый список")
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
result, err := s.db.Exec(`INSERT INTO agent_actions(agent_id,action,target,requested_at,status) VALUES(?,?,?,?, 'queued')`, agentID, action, target, now)
|
||||
if err != nil {
|
||||
return AgentAction{}, err
|
||||
}
|
||||
id, _ := result.LastInsertId()
|
||||
return AgentAction{ID: id, AgentID: agentID, AgentName: name, Action: action, Target: target, RequestedAt: now, Status: "queued"}, nil
|
||||
}
|
||||
|
||||
func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-5*time.Minute).Unix())
|
||||
rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var commands []AgentCommand
|
||||
for rows.Next() {
|
||||
var c AgentCommand
|
||||
if err = rows.Scan(&c.ID, &c.Action, &c.Target); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
commands = append(commands, c)
|
||||
}
|
||||
rows.Close()
|
||||
now := time.Now().Unix()
|
||||
for _, c := range commands {
|
||||
if _, err = tx.Exec(`UPDATE agent_actions SET status='running',started_at=? WHERE id=?`, now, c.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return commands, tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) CompleteAgentActions(agentID string, results []AgentActionResult) {
|
||||
now := time.Now().Unix()
|
||||
for _, result := range results {
|
||||
status := "failed"
|
||||
if result.Success {
|
||||
status = "completed"
|
||||
}
|
||||
message := result.Message
|
||||
if len(message) > 4000 {
|
||||
message = message[:4000]
|
||||
}
|
||||
_, _ = s.db.Exec(`UPDATE agent_actions SET status=?,result=?,completed_at=? WHERE id=? AND agent_id=? AND status='running'`, status, message, now, result.ID, agentID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) AgentActions(limit int) ([]AgentAction, error) {
|
||||
if limit < 1 || limit > 200 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT a.id,a.agent_id,g.name,a.action,a.target,a.requested_at,a.started_at,a.completed_at,a.status,a.result FROM agent_actions a JOIN agents g ON g.id=a.agent_id ORDER BY a.id DESC LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []AgentAction
|
||||
for rows.Next() {
|
||||
var v AgentAction
|
||||
if err = rows.Scan(&v.ID, &v.AgentID, &v.AgentName, &v.Action, &v.Target, &v.RequestedAt, &v.StartedAt, &v.CompletedAt, &v.Status, &v.Result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func executeAgentCommand(command AgentCommand) AgentActionResult {
|
||||
result := AgentActionResult{ID: command.ID}
|
||||
var output string
|
||||
switch command.Action {
|
||||
case "systemd-restart":
|
||||
output = runAgentCombinedCommand(30*time.Second, "systemctl", "restart", command.Target)
|
||||
case "systemd-reset-failed":
|
||||
output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
|
||||
case "docker-restart":
|
||||
output = restartAgentContainer(command.Target)
|
||||
case "updates-refresh":
|
||||
guestInventory.Lock()
|
||||
guestInventory.value.CollectedAt = time.Time{}
|
||||
guestInventory.Unlock()
|
||||
result.Success = true
|
||||
result.Message = "Проверка обновлений поставлена в очередь"
|
||||
return result
|
||||
default:
|
||||
result.Message = "Команда не разрешена"
|
||||
return result
|
||||
}
|
||||
result.Success = !strings.HasPrefix(output, "ERROR: ")
|
||||
result.Message = strings.TrimSpace(output)
|
||||
if result.Message == "" && result.Success {
|
||||
result.Message = "Выполнено успешно"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
|
||||
message := strings.TrimSpace(string(output))
|
||||
if err != nil {
|
||||
return "ERROR: " + err.Error() + ": " + message
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func restartAgentContainer(target string) string {
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||
}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 35 * time.Second}
|
||||
request, _ := http.NewRequest(http.MethodPost, "http://docker/containers/"+target+"/restart?t=10", bytes.NewReader(nil))
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "ERROR: " + err.Error()
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
if response.StatusCode >= 300 {
|
||||
return "ERROR: Docker вернул " + response.Status + ": " + strings.TrimSpace(string(body))
|
||||
}
|
||||
return "Контейнер перезапущен"
|
||||
}
|
||||
54
agent_actions_test.go
Normal file
54
agent_actions_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
|
||||
store, err := openStore(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea"}}})
|
||||
_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
|
||||
t.Fatal("unsafe unit accepted")
|
||||
}
|
||||
action, err := store.QueueAgentAction("a", "systemd-restart", "gitea.service")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commands, err := store.DispatchAgentActions("a")
|
||||
if err != nil || len(commands) != 1 || commands[0].ID != action.ID {
|
||||
t.Fatalf("commands=%#v err=%v", commands, err)
|
||||
}
|
||||
store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
|
||||
items, err := store.AgentActions(10)
|
||||
if err != nil || len(items) != 1 || items[0].Status != "completed" {
|
||||
t.Fatalf("items=%#v err=%v", items, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
|
||||
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "a", Name: "app", VMID: 105, Online: false}}, ServiceLinks: []ServiceLink{{ServiceID: 7, AgentID: "a", VMID: 105}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{ID: 7, Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Kind: "local", Up: false}}}}}}
|
||||
alerts := []Alert{{ID: "agent-offline-a-105"}, {ID: "service-7-local"}, {ID: "other"}}
|
||||
grouped := groupLinkedAlerts(metrics, alerts)
|
||||
if len(grouped) != 2 {
|
||||
t.Fatalf("grouped=%#v", grouped)
|
||||
}
|
||||
found := false
|
||||
for _, alert := range grouped {
|
||||
if alert.ID == "group-agent-offline-a-105" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("group alert missing: %#v", grouped)
|
||||
}
|
||||
}
|
||||
329
agent_diagnostics.go
Normal file
329
agent_diagnostics.go
Normal file
@@ -0,0 +1,329 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AgentLog struct {
|
||||
Time int64 `json:"time"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type AgentProcess struct {
|
||||
PID int `json:"pid"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
CPUPercent float64 `json:"cpuPercent"`
|
||||
MemoryBytes uint64 `json:"memoryBytes"`
|
||||
ReadBytes uint64 `json:"readBytes"`
|
||||
WriteBytes uint64 `json:"writeBytes"`
|
||||
Elapsed int64 `json:"elapsedSeconds"`
|
||||
Command string `json:"command"`
|
||||
}
|
||||
|
||||
type AgentFilesystem struct {
|
||||
Device string `json:"device"`
|
||||
Mountpoint string `json:"mountpoint"`
|
||||
Type string `json:"type"`
|
||||
TotalBytes uint64 `json:"totalBytes"`
|
||||
UsedBytes uint64 `json:"usedBytes"`
|
||||
UsedPercent float64 `json:"usedPercent"`
|
||||
InodesTotal uint64 `json:"inodesTotal"`
|
||||
InodesUsed uint64 `json:"inodesUsed"`
|
||||
InodesPercent float64 `json:"inodesPercent"`
|
||||
}
|
||||
|
||||
type AgentListenPort struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
Process string `json:"process,omitempty"`
|
||||
}
|
||||
|
||||
type AgentPressureValue struct {
|
||||
Avg10 float64 `json:"avg10"`
|
||||
Avg60 float64 `json:"avg60"`
|
||||
Avg300 float64 `json:"avg300"`
|
||||
}
|
||||
|
||||
type AgentPressure struct {
|
||||
CPU AgentPressureValue `json:"cpu"`
|
||||
Memory AgentPressureValue `json:"memory"`
|
||||
IO AgentPressureValue `json:"io"`
|
||||
}
|
||||
|
||||
var agentOOMCache struct {
|
||||
sync.Mutex
|
||||
value []AgentLog
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func compactSystemdExec(value string) string {
|
||||
if pathAt := strings.Index(value, "path="); pathAt >= 0 {
|
||||
value = value[pathAt+5:]
|
||||
}
|
||||
if end := strings.IndexAny(value, " ;}"); end > 0 {
|
||||
value = value[:end]
|
||||
}
|
||||
if len(value) > 240 {
|
||||
value = value[:240]
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func limitedFields(value string, limit int) []string {
|
||||
fields := strings.Fields(value)
|
||||
if len(fields) > limit {
|
||||
fields = fields[:limit]
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func appendUniqueInt(items []int, value int) []int {
|
||||
for _, item := range items {
|
||||
if item == value {
|
||||
return items
|
||||
}
|
||||
}
|
||||
return append(items, value)
|
||||
}
|
||||
|
||||
func collectAgentJournal(unit string, limit int, since string) []AgentLog {
|
||||
args := []string{"--no-pager", "-o", "json", "--reverse", "-n", strconv.Itoa(limit), "--since", since}
|
||||
if unit != "" {
|
||||
args = append(args, "-u", unit)
|
||||
}
|
||||
output := runAgentCommand(6*time.Second, "journalctl", args...)
|
||||
result := []AgentLog{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
var entry struct {
|
||||
Message any `json:"MESSAGE"`
|
||||
Timestamp string `json:"__REALTIME_TIMESTAMP"`
|
||||
}
|
||||
if json.Unmarshal([]byte(line), &entry) != nil {
|
||||
continue
|
||||
}
|
||||
message, ok := entry.Message.(string)
|
||||
if !ok || strings.TrimSpace(message) == "" {
|
||||
continue
|
||||
}
|
||||
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
|
||||
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func cachedAgentOOMKills() []AgentLog {
|
||||
agentOOMCache.Lock()
|
||||
defer agentOOMCache.Unlock()
|
||||
if time.Since(agentOOMCache.at) < time.Minute {
|
||||
return agentOOMCache.value
|
||||
}
|
||||
output := runAgentCommand(6*time.Second, "journalctl", "-k", "--since", "24 hours ago", "--reverse", "--no-pager", "-o", "json", "-g", "Out of memory|Killed process")
|
||||
result := []AgentLog{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
var entry struct {
|
||||
Message any `json:"MESSAGE"`
|
||||
Timestamp string `json:"__REALTIME_TIMESTAMP"`
|
||||
}
|
||||
if json.Unmarshal([]byte(line), &entry) != nil {
|
||||
continue
|
||||
}
|
||||
message, ok := entry.Message.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
microseconds, _ := strconv.ParseInt(entry.Timestamp, 10, 64)
|
||||
result = append(result, AgentLog{Time: microseconds / 1_000_000, Message: message})
|
||||
if len(result) >= 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
agentOOMCache.value, agentOOMCache.at = result, time.Now()
|
||||
return result
|
||||
}
|
||||
|
||||
func collectAgentProcesses() []AgentProcess {
|
||||
result := []AgentProcess{}
|
||||
seen := map[int]bool{}
|
||||
for _, sortBy := range []string{"-pcpu", "-rss"} {
|
||||
output := runAgentCommand(5*time.Second, "ps", "-eo", "pid=,comm=,state=,pcpu=,rss=,etimes=,args=", "--sort="+sortBy)
|
||||
added := 0
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 7 {
|
||||
continue
|
||||
}
|
||||
pid, err := strconv.Atoi(fields[0])
|
||||
if err != nil || seen[pid] {
|
||||
continue
|
||||
}
|
||||
seen[pid] = true
|
||||
cpu, _ := strconv.ParseFloat(fields[3], 64)
|
||||
rss, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
elapsed, _ := strconv.ParseInt(fields[5], 10, 64)
|
||||
item := AgentProcess{PID: pid, Name: fields[1], State: fields[2], CPUPercent: cpu, MemoryBytes: rss * 1024, Elapsed: elapsed, Command: strings.Join(fields[6:], " ")}
|
||||
item.ReadBytes, item.WriteBytes = readAgentProcessIO(pid)
|
||||
result = append(result, item)
|
||||
added++
|
||||
if added >= 12 || len(result) >= 24 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readAgentProcessIO(pid int) (uint64, uint64) {
|
||||
data, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "io"))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
values[strings.TrimSuffix(fields[0], ":")], _ = strconv.ParseUint(fields[1], 10, 64)
|
||||
}
|
||||
return values["read_bytes"], values["write_bytes"]
|
||||
}
|
||||
|
||||
func collectAgentFilesystems() []AgentFilesystem {
|
||||
file, err := os.Open("/proc/self/mounts")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer file.Close()
|
||||
excluded := map[string]bool{"proc": true, "sysfs": true, "devtmpfs": true, "devpts": true, "tmpfs": true, "cgroup": true, "cgroup2": true, "overlay": true, "squashfs": true, "nsfs": true, "tracefs": true, "debugfs": true, "securityfs": true, "pstore": true, "mqueue": true, "hugetlbfs": true, "fusectl": true, "configfs": true, "autofs": true, "rpc_pipefs": true}
|
||||
seen := map[string]bool{}
|
||||
result := []AgentFilesystem{}
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) < 3 || excluded[fields[2]] || seen[fields[1]] {
|
||||
continue
|
||||
}
|
||||
mountpoint := strings.ReplaceAll(fields[1], `\040`, " ")
|
||||
var stat syscall.Statfs_t
|
||||
if syscall.Statfs(mountpoint, &stat) != nil || stat.Blocks == 0 {
|
||||
continue
|
||||
}
|
||||
seen[mountpoint] = true
|
||||
total := stat.Blocks * uint64(stat.Bsize)
|
||||
free := stat.Bavail * uint64(stat.Bsize)
|
||||
used := total - free
|
||||
item := AgentFilesystem{Device: fields[0], Mountpoint: mountpoint, Type: fields[2], TotalBytes: total, UsedBytes: used, UsedPercent: float64(used) / float64(total) * 100, InodesTotal: stat.Files}
|
||||
if stat.Files > stat.Ffree {
|
||||
item.InodesUsed = stat.Files - stat.Ffree
|
||||
item.InodesPercent = float64(item.InodesUsed) / float64(stat.Files) * 100
|
||||
}
|
||||
result = append(result, item)
|
||||
if len(result) >= 30 {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].Mountpoint < result[j].Mountpoint })
|
||||
return result
|
||||
}
|
||||
|
||||
var ssPIDPattern = regexp.MustCompile(`pid=([0-9]+)`)
|
||||
var ssNamePattern = regexp.MustCompile(`users:\(\(\"([^\"]+)\"`)
|
||||
|
||||
func collectAgentListenPorts() []AgentListenPort {
|
||||
output := runAgentCommand(5*time.Second, "ss", "-H", "-lntup")
|
||||
result := []AgentListenPort{}
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
continue
|
||||
}
|
||||
protocol := strings.ToLower(fields[0])
|
||||
local := ""
|
||||
for _, field := range fields[1:] {
|
||||
if strings.Contains(field, ":") && !strings.Contains(field, "users:") {
|
||||
local = field
|
||||
break
|
||||
}
|
||||
}
|
||||
if local == "" {
|
||||
continue
|
||||
}
|
||||
address, portText, err := net.SplitHostPort(local)
|
||||
if err != nil {
|
||||
idx := strings.LastIndex(local, ":")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
address, portText = local[:idx], local[idx+1:]
|
||||
}
|
||||
port, _ := strconv.Atoi(portText)
|
||||
if port <= 0 {
|
||||
continue
|
||||
}
|
||||
pid := 0
|
||||
if match := ssPIDPattern.FindStringSubmatch(line); len(match) > 1 {
|
||||
pid, _ = strconv.Atoi(match[1])
|
||||
}
|
||||
process := ""
|
||||
if match := ssNamePattern.FindStringSubmatch(line); len(match) > 1 {
|
||||
process = match[1]
|
||||
}
|
||||
key := protocol + "|" + address + "|" + portText + "|" + strconv.Itoa(pid)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
result = append(result, AgentListenPort{Protocol: protocol, Address: address, Port: port, PID: pid, Process: process})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Port == result[j].Port {
|
||||
return result[i].Protocol < result[j].Protocol
|
||||
}
|
||||
return result[i].Port < result[j].Port
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func collectAgentPressure() AgentPressure {
|
||||
return AgentPressure{CPU: readPressure("/proc/pressure/cpu"), Memory: readPressure("/proc/pressure/memory"), IO: readPressure("/proc/pressure/io")}
|
||||
}
|
||||
|
||||
func readPressure(path string) AgentPressureValue {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return AgentPressureValue{}
|
||||
}
|
||||
line := strings.SplitN(string(data), "\n", 2)[0]
|
||||
value := AgentPressureValue{}
|
||||
for _, field := range strings.Fields(line) {
|
||||
key, raw, ok := strings.Cut(field, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
number, _ := strconv.ParseFloat(raw, 64)
|
||||
switch key {
|
||||
case "avg10":
|
||||
value.Avg10 = number
|
||||
case "avg60":
|
||||
value.Avg60 = number
|
||||
case "avg300":
|
||||
value.Avg300 = number
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -60,6 +60,30 @@ func TestParseAgentIssues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyAgentService(t *testing.T) {
|
||||
cases := []struct{ path, pkg, priority, want string }{{"/lib/systemd/system/cron.service", "cron", "important", "system"}, {"/lib/systemd/system/jellyfin.service", "jellyfin", "optional", "added"}, {"/etc/systemd/system/my.service", "", "", "local"}, {"/run/systemd/system/transient.service", "", "", "runtime"}}
|
||||
for _, tc := range cases {
|
||||
if got := classifyAgentService(tc.path, tc.pkg, tc.priority); got != tc.want {
|
||||
t.Errorf("%s: got %s want %s", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServiceDiagnosticsHelpers(t *testing.T) {
|
||||
if got := compactSystemdExec(`{ path=/usr/bin/example ; argv[]=/usr/bin/example --serve ; }`); got != "/usr/bin/example" {
|
||||
t.Fatalf("unexpected executable: %q", got)
|
||||
}
|
||||
fields := limitedFields("a b c d", 2)
|
||||
if len(fields) != 2 || fields[0] != "a" || fields[1] != "b" {
|
||||
t.Fatalf("unexpected limited fields: %#v", fields)
|
||||
}
|
||||
ports := appendUniqueInt([]int{80}, 80)
|
||||
ports = appendUniqueInt(ports, 443)
|
||||
if len(ports) != 2 || ports[1] != 443 {
|
||||
t.Fatalf("unexpected ports: %#v", ports)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentReportAuthenticationAndOfflineState(t *testing.T) {
|
||||
store, err := openStore(":memory:")
|
||||
if err != nil {
|
||||
@@ -101,3 +125,22 @@ func TestAgentProblemsBecomeMaintenanceAwareAlerts(t *testing.T) {
|
||||
t.Fatalf("active=%d planned=%d alerts=%#v", len(active), len(planned), alerts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentDiagnosticsBecomeAlerts(t *testing.T) {
|
||||
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "node", Name: "app", VMID: 105, Online: true, Report: AgentReport{
|
||||
Services: []AgentService{{Name: "web.service", State: "active", Flapping: true, Restarts: 8, StartedAt: time.Now().Unix()}},
|
||||
Filesystems: []AgentFilesystem{{Mountpoint: "/data", TotalBytes: 1000, UsedBytes: 950, UsedPercent: 95}},
|
||||
}}}}
|
||||
alerts := evaluateAlerts(metrics, defaultThresholds())
|
||||
wanted := map[string]bool{"agent-flapping-node-web.service-105": false, "agent-filesystem-node-/data-105": false}
|
||||
for _, alert := range alerts {
|
||||
if _, ok := wanted[alert.ID]; ok {
|
||||
wanted[alert.ID] = true
|
||||
}
|
||||
}
|
||||
for id, found := range wanted {
|
||||
if !found {
|
||||
t.Fatalf("missing alert %s in %#v", id, alerts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,10 +124,22 @@ func (s *Store) AcknowledgeAlert(eventID int64) error {
|
||||
}
|
||||
|
||||
func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
|
||||
return s.AlertHistoryBetween(limit, 0, 0)
|
||||
}
|
||||
|
||||
func (s *Store) AlertHistoryBetween(limit int, from, to int64) ([]AlertEvent, error) {
|
||||
if limit < 1 || limit > 500 {
|
||||
limit = 200
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events ORDER BY first_seen DESC LIMIT ?`, limit)
|
||||
query := `SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events`
|
||||
args := []any{}
|
||||
if from > 0 && to > from {
|
||||
query += ` WHERE first_seen>=? AND first_seen<?`
|
||||
args = append(args, from, to)
|
||||
}
|
||||
query += ` ORDER BY first_seen DESC LIMIT ?`
|
||||
args = append(args, limit)
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
14
alerts.go
14
alerts.go
@@ -215,8 +215,22 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
|
||||
for _, service := range agent.Report.Services {
|
||||
if service.State == "failed" || service.SubState == "failed" {
|
||||
add("agent-systemd-"+agent.ID+"-"+service.Name+vmSuffix, "critical", "Агенты", agent.Name+": упала служба "+service.Name, fmt.Sprintf("%s · VMID %d.", firstNonEmpty(service.Description, service.SubState), agent.VMID))
|
||||
} else if service.Flapping {
|
||||
add("agent-flapping-"+agent.ID+"-"+service.Name+vmSuffix, "warning", "Агенты", agent.Name+": служба постоянно перезапускается", fmt.Sprintf("%s: %d рестартов, последний запуск %s · VMID %d.", service.Name, service.Restarts, time.Unix(service.StartedAt, 0).Format("15:04:05"), agent.VMID))
|
||||
}
|
||||
}
|
||||
for _, filesystem := range agent.Report.Filesystems {
|
||||
if filesystem.UsedPercent >= 90 {
|
||||
add("agent-filesystem-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчивается место "+filesystem.Mountpoint, fmt.Sprintf("Заполнено %.1f%%, свободно %.1f GB · VMID %d.", filesystem.UsedPercent, float64(filesystem.TotalBytes-filesystem.UsedBytes)/1073741824, agent.VMID))
|
||||
}
|
||||
if filesystem.InodesPercent >= 90 {
|
||||
add("agent-inodes-"+agent.ID+"-"+filesystem.Mountpoint+vmSuffix, "critical", "Агенты", agent.Name+": заканчиваются inode "+filesystem.Mountpoint, fmt.Sprintf("Использовано %.1f%% inode · VMID %d.", filesystem.InodesPercent, agent.VMID))
|
||||
}
|
||||
}
|
||||
if len(agent.Report.OOMKills) > 0 {
|
||||
latest := agent.Report.OOMKills[0]
|
||||
add("agent-oom-"+agent.ID+vmSuffix, "critical", "Агенты", agent.Name+": ядро завершало процессы из-за нехватки памяти", fmt.Sprintf("%s · %s · VMID %d.", time.Unix(latest.Time, 0).Format("02.01 15:04"), latest.Message, agent.VMID))
|
||||
}
|
||||
if agent.Report.RebootRequired {
|
||||
add("agent-reboot-"+agent.ID+vmSuffix, "info", "Агенты", agent.Name+": требуется перезагрузка", fmt.Sprintf("Обновления системы запросили перезагрузку · VMID %d.", agent.VMID))
|
||||
}
|
||||
|
||||
24
history.go
24
history.go
@@ -64,6 +64,30 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
|
||||
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
|
||||
}
|
||||
}
|
||||
for _, agent := range metrics.Agents {
|
||||
if !agent.Online {
|
||||
continue
|
||||
}
|
||||
name := agent.ID + " · " + agent.Name
|
||||
cpu := agent.Report.CPUPercent
|
||||
memory := float64(0)
|
||||
if agent.Report.MemoryTotal > 0 {
|
||||
memory = float64(agent.Report.MemoryUsed) / float64(agent.Report.MemoryTotal) * 100
|
||||
}
|
||||
swap := float64(0)
|
||||
if agent.Report.SwapTotal > 0 {
|
||||
swap = float64(agent.Report.SwapUsed) / float64(agent.Report.SwapTotal) * 100
|
||||
}
|
||||
s.AddEntityHistory("agent-resource", name, ts, &cpu, &memory)
|
||||
s.AddEntityHistory("agent-swap", name, ts, &swap, nil)
|
||||
pressure := agent.Report.Pressure.IO.Avg10
|
||||
s.AddEntityHistory("agent-pressure", name, ts, &pressure, nil)
|
||||
for _, filesystem := range agent.Report.Filesystems {
|
||||
used := filesystem.UsedPercent
|
||||
inodes := filesystem.InodesPercent
|
||||
s.AddEntityHistory("agent-filesystem", name+" · "+filesystem.Mountpoint, ts, &used, &inodes)
|
||||
}
|
||||
}
|
||||
s.PruneEntityHistory()
|
||||
}
|
||||
|
||||
|
||||
110
service_links.go
Normal file
110
service_links.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ServiceLink struct {
|
||||
ServiceID int64 `json:"serviceId"`
|
||||
AgentID string `json:"agentId"`
|
||||
VMID int `json:"vmid"`
|
||||
ComponentType string `json:"componentType"`
|
||||
ComponentName string `json:"componentName"`
|
||||
Port int `json:"port"`
|
||||
}
|
||||
|
||||
func (s *Store) ServiceLinks() ([]ServiceLink, error) {
|
||||
rows, err := s.db.Query(`SELECT service_id,agent_id,vmid,component_type,component_name,port FROM service_links ORDER BY service_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ServiceLink
|
||||
for rows.Next() {
|
||||
var v ServiceLink
|
||||
if err = rows.Scan(&v.ServiceID, &v.AgentID, &v.VMID, &v.ComponentType, &v.ComponentName, &v.Port); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) SaveServiceLink(v ServiceLink) (ServiceLink, error) {
|
||||
v.AgentID = strings.TrimSpace(v.AgentID)
|
||||
v.ComponentType = strings.TrimSpace(v.ComponentType)
|
||||
v.ComponentName = strings.TrimSpace(v.ComponentName)
|
||||
if v.ServiceID <= 0 {
|
||||
return v, fmt.Errorf("выберите сервис")
|
||||
}
|
||||
if v.AgentID == "" && v.VMID <= 0 {
|
||||
return v, fmt.Errorf("выберите агент или VMID")
|
||||
}
|
||||
if v.ComponentType != "" && v.ComponentType != "systemd" && v.ComponentType != "docker" && v.ComponentType != "process" {
|
||||
return v, fmt.Errorf("неизвестный тип компонента")
|
||||
}
|
||||
if v.Port < 0 || v.Port > 65535 {
|
||||
return v, fmt.Errorf("некорректный порт")
|
||||
}
|
||||
_, err := s.db.Exec(`INSERT INTO service_links(service_id,agent_id,vmid,component_type,component_name,port) VALUES(?,?,?,?,?,?) ON CONFLICT(service_id) DO UPDATE SET agent_id=excluded.agent_id,vmid=excluded.vmid,component_type=excluded.component_type,component_name=excluded.component_name,port=excluded.port`, v.ServiceID, v.AgentID, v.VMID, v.ComponentType, v.ComponentName, v.Port)
|
||||
return v, err
|
||||
}
|
||||
func (s *Store) DeleteServiceLink(serviceID int64) error {
|
||||
_, err := s.db.Exec(`DELETE FROM service_links WHERE service_id=?`, serviceID)
|
||||
return err
|
||||
}
|
||||
|
||||
func groupLinkedAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
|
||||
if len(metrics.ServiceLinks) == 0 {
|
||||
return alerts
|
||||
}
|
||||
agentOffline := map[string]ManagedAgent{}
|
||||
for _, agent := range metrics.Agents {
|
||||
if !agent.Online {
|
||||
agentOffline[agent.ID] = agent
|
||||
}
|
||||
}
|
||||
affected := map[string][]string{}
|
||||
remove := map[string]bool{}
|
||||
for _, link := range metrics.ServiceLinks {
|
||||
agent, offline := agentOffline[link.AgentID]
|
||||
if !offline {
|
||||
continue
|
||||
}
|
||||
for _, service := range metrics.Services.Services {
|
||||
if service.ID != link.ServiceID {
|
||||
continue
|
||||
}
|
||||
for _, endpoint := range service.Endpoints {
|
||||
if !endpoint.Up {
|
||||
key := fmt.Sprintf("service-%d-%s", service.ID, endpoint.Kind)
|
||||
remove[key] = true
|
||||
affected[agent.ID] = append(affected[agent.ID], service.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(affected) == 0 {
|
||||
return alerts
|
||||
}
|
||||
out := make([]Alert, 0, len(alerts))
|
||||
for _, alert := range alerts {
|
||||
skip := remove[alert.ID]
|
||||
for agentID := range affected {
|
||||
if alert.ID == "agent-offline-"+agentID+fmt.Sprintf("-%d", agentOffline[agentID].VMID) {
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
if !skip {
|
||||
out = append(out, alert)
|
||||
}
|
||||
}
|
||||
for agentID, names := range affected {
|
||||
sort.Strings(names)
|
||||
agent := agentOffline[agentID]
|
||||
out = append(out, Alert{ID: "group-agent-offline-" + agentID + fmt.Sprintf("-%d", agent.VMID), Severity: "critical", Source: "Корреляция", Title: agent.Name + ": агент недоступен, связанные сервисы не отвечают", Message: fmt.Sprintf("VMID %d. Затронуто сервисов: %d — %s. Вероятная первопричина: недоступна гостевая система или агент.", agent.VMID, len(names), strings.Join(names, ", "))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
3
store.go
3
store.go
@@ -140,6 +140,9 @@ func openStore(path string) (*Store, error) {
|
||||
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
|
||||
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
|
||||
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
|
||||
`CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`,
|
||||
`CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`,
|
||||
`CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := db.Exec(statement); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user