Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
714c29c449 | ||
|
|
21b4f528e0 |
11
README.md
11
README.md
@@ -358,6 +358,17 @@ CPU и RAM с накопленным I/O, swap, PSI pressure, все посто
|
|||||||
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
|
Связи по PID и порту считаются подтверждёнными; совпадения только по названию
|
||||||
помечаются как предположение.
|
помечаются как предположение.
|
||||||
|
|
||||||
|
Для сложных reverse proxy и Docker-сетей в управлении сервисами можно сохранить
|
||||||
|
ручную связь `сервис → агент/VMID → systemd/Docker/процесс → порт`. Она имеет
|
||||||
|
приоритет над автоматическим определением и используется для группировки
|
||||||
|
связанных уведомлений по первопричине.
|
||||||
|
|
||||||
|
Dashboard хранит семь дней истории CPU, RAM, swap, PSI и файловых систем каждого
|
||||||
|
агента. Из подробностей агента доступны только заранее разрешённые действия:
|
||||||
|
перезапуск известной systemd-службы или Docker-контейнера, `reset-failed` и
|
||||||
|
обновление списка пакетов. Агент не принимает произвольные shell-команды; каждое
|
||||||
|
действие требует подтверждения в браузере и сохраняется в журнале управления.
|
||||||
|
|
||||||
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
|
Откройте `Настройки → Агенты`, укажите адрес Dashboard, доступный из VM/LXC, и
|
||||||
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
|
VMID. Кнопка создаст готовую команду установки. Одноразовый токен действует 15
|
||||||
минут и после регистрации заменяется индивидуальным секретом агента.
|
минут и после регистрации заменяется индивидуальным секретом агента.
|
||||||
|
|||||||
30
agent.go
30
agent.go
@@ -117,6 +117,7 @@ type AgentReport struct {
|
|||||||
RebootRequired bool `json:"rebootRequired"`
|
RebootRequired bool `json:"rebootRequired"`
|
||||||
InventoryAt int64 `json:"inventoryAt"`
|
InventoryAt int64 `json:"inventoryAt"`
|
||||||
CollectedAt int64 `json:"collectedAt"`
|
CollectedAt int64 `json:"collectedAt"`
|
||||||
|
ActionResults []AgentActionResult `json:"actionResults,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type agentInventory struct {
|
type agentInventory struct {
|
||||||
@@ -137,6 +138,7 @@ type AgentUpdateManifest struct {
|
|||||||
type agentReportResponse struct {
|
type agentReportResponse struct {
|
||||||
Accepted bool `json:"accepted"`
|
Accepted bool `json:"accepted"`
|
||||||
Update *AgentUpdateManifest `json:"update,omitempty"`
|
Update *AgentUpdateManifest `json:"update,omitempty"`
|
||||||
|
Commands []AgentCommand `json:"commands,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var guestInventory struct {
|
var guestInventory struct {
|
||||||
@@ -150,6 +152,11 @@ var guestIssues struct {
|
|||||||
collectedAt time.Time
|
collectedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var guestActionResults struct {
|
||||||
|
sync.Mutex
|
||||||
|
value []AgentActionResult
|
||||||
|
}
|
||||||
|
|
||||||
var agentBinaryInfo struct {
|
var agentBinaryInfo struct {
|
||||||
sync.Once
|
sync.Once
|
||||||
manifest AgentUpdateManifest
|
manifest AgentUpdateManifest
|
||||||
@@ -258,6 +265,8 @@ func (s *Store) SaveAgentReport(id, secret, remote string, report AgentReport) e
|
|||||||
if !s.AuthenticateAgent(id, secret) {
|
if !s.AuthenticateAgent(id, secret) {
|
||||||
return errors.New("агент не авторизован")
|
return errors.New("агент не авторизован")
|
||||||
}
|
}
|
||||||
|
s.CompleteAgentActions(id, report.ActionResults)
|
||||||
|
report.ActionResults = nil
|
||||||
data, _ := json.Marshal(report)
|
data, _ := json.Marshal(report)
|
||||||
_, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id)
|
_, err := s.db.Exec(`UPDATE agents SET hostname=?,version=?,last_seen=?,report_json=?,remote_address=? WHERE id=?`, report.Hostname, report.Version, time.Now().Unix(), string(data), remote, id)
|
||||||
return err
|
return err
|
||||||
@@ -325,6 +334,9 @@ func runGuestAgent(opts agentOptions) error {
|
|||||||
client := &http.Client{Timeout: 12 * time.Second}
|
client := &http.Client{Timeout: 12 * time.Second}
|
||||||
for {
|
for {
|
||||||
report := collectAgentReport()
|
report := collectAgentReport()
|
||||||
|
guestActionResults.Lock()
|
||||||
|
report.ActionResults = append([]AgentActionResult(nil), guestActionResults.value...)
|
||||||
|
guestActionResults.Unlock()
|
||||||
data, _ := json.Marshal(report)
|
data, _ := json.Marshal(report)
|
||||||
req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data))
|
req, _ := http.NewRequest(http.MethodPost, config.Server+"/api/agent/report", bytes.NewReader(data))
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
@@ -351,7 +363,23 @@ func runGuestAgent(opts agentOptions) error {
|
|||||||
}
|
}
|
||||||
if response.StatusCode == http.StatusOK {
|
if response.StatusCode == http.StatusOK {
|
||||||
var result agentReportResponse
|
var result agentReportResponse
|
||||||
if json.Unmarshal(responseData, &result) == nil && result.Update != nil && newerAgentVersion(result.Update.Version, version) {
|
if json.Unmarshal(responseData, &result) == nil {
|
||||||
|
if len(report.ActionResults) > 0 {
|
||||||
|
guestActionResults.Lock()
|
||||||
|
guestActionResults.value = nil
|
||||||
|
guestActionResults.Unlock()
|
||||||
|
}
|
||||||
|
if len(result.Commands) > 0 {
|
||||||
|
executed := make([]AgentActionResult, 0, len(result.Commands))
|
||||||
|
for _, command := range result.Commands {
|
||||||
|
executed = append(executed, executeAgentCommand(command))
|
||||||
|
}
|
||||||
|
guestActionResults.Lock()
|
||||||
|
guestActionResults.value = append(guestActionResults.value, executed...)
|
||||||
|
guestActionResults.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result.Update != nil && newerAgentVersion(result.Update.Version, version) {
|
||||||
if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil {
|
if updateErr := applyAgentUpdate(client, config, *result.Update); updateErr != nil {
|
||||||
requestErr = fmt.Errorf("автообновление: %v", updateErr)
|
requestErr = fmt.Errorf("автообновление: %v", updateErr)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
210
agent_actions.go
Normal file
210
agent_actions.go
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AgentCommand struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
Target string `json:"target,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type AgentActionResult struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type AgentAction struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
AgentID string `json:"agentId"`
|
||||||
|
AgentName string `json:"agentName"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
RequestedAt int64 `json:"requestedAt"`
|
||||||
|
StartedAt *int64 `json:"startedAt,omitempty"`
|
||||||
|
CompletedAt *int64 `json:"completedAt,omitempty"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Result string `json:"result"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var safeUnitName = regexp.MustCompile(`^[A-Za-z0-9_.@:-]+\.service$`)
|
||||||
|
var safeContainerName = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`)
|
||||||
|
|
||||||
|
func (s *Store) QueueAgentAction(agentID, action, target string) (AgentAction, error) {
|
||||||
|
action, target = strings.TrimSpace(action), strings.TrimSpace(target)
|
||||||
|
var raw, name string
|
||||||
|
if err := s.db.QueryRow(`SELECT report_json,name FROM agents WHERE id=?`, agentID).Scan(&raw, &name); err != nil {
|
||||||
|
return AgentAction{}, fmt.Errorf("агент не найден")
|
||||||
|
}
|
||||||
|
var report AgentReport
|
||||||
|
_ = json.Unmarshal([]byte(raw), &report)
|
||||||
|
valid := false
|
||||||
|
switch action {
|
||||||
|
case "systemd-restart", "systemd-reset-failed":
|
||||||
|
if safeUnitName.MatchString(target) {
|
||||||
|
for _, service := range report.Services {
|
||||||
|
if service.Name == target {
|
||||||
|
valid = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "docker-restart":
|
||||||
|
if safeContainerName.MatchString(target) {
|
||||||
|
for _, container := range report.Containers {
|
||||||
|
if container.ID == target || container.Name == target {
|
||||||
|
valid = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "updates-refresh":
|
||||||
|
valid = target == ""
|
||||||
|
}
|
||||||
|
if !valid {
|
||||||
|
return AgentAction{}, fmt.Errorf("действие или цель не входят в разрешённый список")
|
||||||
|
}
|
||||||
|
now := time.Now().Unix()
|
||||||
|
result, err := s.db.Exec(`INSERT INTO agent_actions(agent_id,action,target,requested_at,status) VALUES(?,?,?,?, 'queued')`, agentID, action, target, now)
|
||||||
|
if err != nil {
|
||||||
|
return AgentAction{}, err
|
||||||
|
}
|
||||||
|
id, _ := result.LastInsertId()
|
||||||
|
return AgentAction{ID: id, AgentID: agentID, AgentName: name, Action: action, Target: target, RequestedAt: now, Status: "queued"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) DispatchAgentActions(agentID string) ([]AgentCommand, error) {
|
||||||
|
tx, err := s.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
_, _ = tx.Exec(`UPDATE agent_actions SET status='queued',started_at=NULL WHERE agent_id=? AND status='running' AND started_at<?`, agentID, time.Now().Add(-5*time.Minute).Unix())
|
||||||
|
rows, err := tx.Query(`SELECT id,action,target FROM agent_actions WHERE agent_id=? AND status='queued' ORDER BY id LIMIT 3`, agentID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var commands []AgentCommand
|
||||||
|
for rows.Next() {
|
||||||
|
var c AgentCommand
|
||||||
|
if err = rows.Scan(&c.ID, &c.Action, &c.Target); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
commands = append(commands, c)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for _, c := range commands {
|
||||||
|
if _, err = tx.Exec(`UPDATE agent_actions SET status='running',started_at=? WHERE id=?`, now, c.ID); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return commands, tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CompleteAgentActions(agentID string, results []AgentActionResult) {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for _, result := range results {
|
||||||
|
status := "failed"
|
||||||
|
if result.Success {
|
||||||
|
status = "completed"
|
||||||
|
}
|
||||||
|
message := result.Message
|
||||||
|
if len(message) > 4000 {
|
||||||
|
message = message[:4000]
|
||||||
|
}
|
||||||
|
_, _ = s.db.Exec(`UPDATE agent_actions SET status=?,result=?,completed_at=? WHERE id=? AND agent_id=? AND status='running'`, status, message, now, result.ID, agentID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AgentActions(limit int) ([]AgentAction, error) {
|
||||||
|
if limit < 1 || limit > 200 {
|
||||||
|
limit = 100
|
||||||
|
}
|
||||||
|
rows, err := s.db.Query(`SELECT a.id,a.agent_id,g.name,a.action,a.target,a.requested_at,a.started_at,a.completed_at,a.status,a.result FROM agent_actions a JOIN agents g ON g.id=a.agent_id ORDER BY a.id DESC LIMIT ?`, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []AgentAction
|
||||||
|
for rows.Next() {
|
||||||
|
var v AgentAction
|
||||||
|
if err = rows.Scan(&v.ID, &v.AgentID, &v.AgentName, &v.Action, &v.Target, &v.RequestedAt, &v.StartedAt, &v.CompletedAt, &v.Status, &v.Result); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeAgentCommand(command AgentCommand) AgentActionResult {
|
||||||
|
result := AgentActionResult{ID: command.ID}
|
||||||
|
var output string
|
||||||
|
switch command.Action {
|
||||||
|
case "systemd-restart":
|
||||||
|
output = runAgentCombinedCommand(30*time.Second, "systemctl", "restart", command.Target)
|
||||||
|
case "systemd-reset-failed":
|
||||||
|
output = runAgentCombinedCommand(15*time.Second, "systemctl", "reset-failed", command.Target)
|
||||||
|
case "docker-restart":
|
||||||
|
output = restartAgentContainer(command.Target)
|
||||||
|
case "updates-refresh":
|
||||||
|
guestInventory.Lock()
|
||||||
|
guestInventory.value.CollectedAt = time.Time{}
|
||||||
|
guestInventory.Unlock()
|
||||||
|
result.Success = true
|
||||||
|
result.Message = "Проверка обновлений поставлена в очередь"
|
||||||
|
return result
|
||||||
|
default:
|
||||||
|
result.Message = "Команда не разрешена"
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
result.Success = !strings.HasPrefix(output, "ERROR: ")
|
||||||
|
result.Message = strings.TrimSpace(output)
|
||||||
|
if result.Message == "" && result.Success {
|
||||||
|
result.Message = "Выполнено успешно"
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func runAgentCombinedCommand(timeout time.Duration, name string, args ...string) string {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
output, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
|
||||||
|
message := strings.TrimSpace(string(output))
|
||||||
|
if err != nil {
|
||||||
|
return "ERROR: " + err.Error() + ": " + message
|
||||||
|
}
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
func restartAgentContainer(target string) string {
|
||||||
|
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||||
|
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
||||||
|
}}
|
||||||
|
defer transport.CloseIdleConnections()
|
||||||
|
client := &http.Client{Transport: transport, Timeout: 35 * time.Second}
|
||||||
|
request, _ := http.NewRequest(http.MethodPost, "http://docker/containers/"+target+"/restart?t=10", bytes.NewReader(nil))
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return "ERROR: " + err.Error()
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||||
|
if response.StatusCode >= 300 {
|
||||||
|
return "ERROR: Docker вернул " + response.Status + ": " + strings.TrimSpace(string(body))
|
||||||
|
}
|
||||||
|
return "Контейнер перезапущен"
|
||||||
|
}
|
||||||
54
agent_actions_test.go
Normal file
54
agent_actions_test.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAgentActionAllowlistAndLifecycle(t *testing.T) {
|
||||||
|
store, err := openStore(":memory:")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer store.Close()
|
||||||
|
report, _ := json.Marshal(AgentReport{Services: []AgentService{{Name: "gitea.service"}}, Containers: []AgentContainer{{ID: "abc123", Name: "gitea"}}})
|
||||||
|
_, err = store.db.Exec(`INSERT INTO agents(id,name,secret_hash,enrolled_at,last_seen,report_json) VALUES('a','agent','x',?,?,?)`, time.Now().Unix(), time.Now().Unix(), string(report))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = store.QueueAgentAction("a", "systemd-restart", "../../bad.service"); err == nil {
|
||||||
|
t.Fatal("unsafe unit accepted")
|
||||||
|
}
|
||||||
|
action, err := store.QueueAgentAction("a", "systemd-restart", "gitea.service")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
commands, err := store.DispatchAgentActions("a")
|
||||||
|
if err != nil || len(commands) != 1 || commands[0].ID != action.ID {
|
||||||
|
t.Fatalf("commands=%#v err=%v", commands, err)
|
||||||
|
}
|
||||||
|
store.CompleteAgentActions("a", []AgentActionResult{{ID: action.ID, Success: true, Message: "ok"}})
|
||||||
|
items, err := store.AgentActions(10)
|
||||||
|
if err != nil || len(items) != 1 || items[0].Status != "completed" {
|
||||||
|
t.Fatalf("items=%#v err=%v", items, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManualServiceLinkGroupsOfflineAlerts(t *testing.T) {
|
||||||
|
metrics := dashboardMetrics{Agents: []ManagedAgent{{ID: "a", Name: "app", VMID: 105, Online: false}}, ServiceLinks: []ServiceLink{{ServiceID: 7, AgentID: "a", VMID: 105}}, Services: ServicesMetrics{Services: []ServiceStatus{{MonitoredService: MonitoredService{ID: 7, Name: "Gitea"}, Endpoints: []ServiceEndpointStatus{{Kind: "local", Up: false}}}}}}
|
||||||
|
alerts := []Alert{{ID: "agent-offline-a-105"}, {ID: "service-7-local"}, {ID: "other"}}
|
||||||
|
grouped := groupLinkedAlerts(metrics, alerts)
|
||||||
|
if len(grouped) != 2 {
|
||||||
|
t.Fatalf("grouped=%#v", grouped)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, alert := range grouped {
|
||||||
|
if alert.ID == "group-agent-offline-a-105" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("group alert missing: %#v", grouped)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -124,10 +124,22 @@ func (s *Store) AcknowledgeAlert(eventID int64) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
|
func (s *Store) AlertHistory(limit int) ([]AlertEvent, error) {
|
||||||
|
return s.AlertHistoryBetween(limit, 0, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AlertHistoryBetween(limit int, from, to int64) ([]AlertEvent, error) {
|
||||||
if limit < 1 || limit > 500 {
|
if limit < 1 || limit > 500 {
|
||||||
limit = 200
|
limit = 200
|
||||||
}
|
}
|
||||||
rows, err := s.db.Query(`SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events ORDER BY first_seen DESC LIMIT ?`, limit)
|
query := `SELECT id,alert_key,severity,source,title,message,first_seen,last_seen,resolved_at,acknowledged_at,planned,maintenance_reason FROM alert_events`
|
||||||
|
args := []any{}
|
||||||
|
if from > 0 && to > from {
|
||||||
|
query += ` WHERE first_seen>=? AND first_seen<?`
|
||||||
|
args = append(args, from, to)
|
||||||
|
}
|
||||||
|
query += ` ORDER BY first_seen DESC LIMIT ?`
|
||||||
|
args = append(args, limit)
|
||||||
|
rows, err := s.db.Query(query, args...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
15
alerts.go
15
alerts.go
@@ -74,7 +74,10 @@ func evaluateAlerts(metrics dashboardMetrics, thresholds AlertThresholds) []Aler
|
|||||||
if disk.SMARTStatus == "Ошибка" {
|
if disk.SMARTStatus == "Ошибка" {
|
||||||
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
|
add("disk-smart-"+disk.Name, "critical", "Диски", disk.Model+": ошибка SMART", joinReasons(disk.AttentionReasons, "SMART сообщает о неисправности диска."))
|
||||||
} else if disk.SMARTStatus == "Требует внимания" {
|
} else if disk.SMARTStatus == "Требует внимания" {
|
||||||
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(disk.AttentionReasons, "Проверьте SMART-показатели диска."))
|
reasons := withoutReason(disk.AttentionReasons, "SMART self-test ещё ни разу не запускался")
|
||||||
|
if len(reasons) > 0 {
|
||||||
|
add("disk-smart-"+disk.Name, "warning", "Диски", disk.Model+": SMART требует внимания", joinReasons(reasons, "Проверьте SMART-показатели диска."))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if disk.Temperature != nil {
|
if disk.Temperature != nil {
|
||||||
limit := thresholds.DiskTempWarning
|
limit := thresholds.DiskTempWarning
|
||||||
@@ -386,3 +389,13 @@ func joinReasons(reasons []string, fallback string) string {
|
|||||||
}
|
}
|
||||||
return strings.Join(reasons, " · ")
|
return strings.Join(reasons, " · ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func withoutReason(reasons []string, excluded string) []string {
|
||||||
|
filtered := make([]string, 0, len(reasons))
|
||||||
|
for _, reason := range reasons {
|
||||||
|
if reason != excluded {
|
||||||
|
filtered = append(filtered, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return filtered
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,3 +25,17 @@ func TestSlowCollectionDoesNotCreateAlert(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNeverRunSelfTestCreatesOneAlert(t *testing.T) {
|
||||||
|
metrics := dashboardMetrics{Disks: []DiskMetrics{{
|
||||||
|
Name: "nvme0n1",
|
||||||
|
Model: "Test NVMe",
|
||||||
|
SMARTStatus: "Требует внимания",
|
||||||
|
SelfTestNeverRun: true,
|
||||||
|
AttentionReasons: []string{"SMART self-test ещё ни разу не запускался"},
|
||||||
|
}}}
|
||||||
|
alerts := evaluateAlerts(metrics, defaultThresholds())
|
||||||
|
if len(alerts) != 1 || alerts[0].ID != "disk-selftest-nvme0n1" {
|
||||||
|
t.Fatalf("ожидалось одно точное уведомление о self-test, получено: %+v", alerts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
37
disks.go
37
disks.go
@@ -391,6 +391,17 @@ type smartctlJSON struct {
|
|||||||
} `json:"table"`
|
} `json:"table"`
|
||||||
} `json:"standard"`
|
} `json:"standard"`
|
||||||
} `json:"ata_smart_self_test_log"`
|
} `json:"ata_smart_self_test_log"`
|
||||||
|
NVMeSelfTestLog struct {
|
||||||
|
Table []struct {
|
||||||
|
SelfTestCode struct {
|
||||||
|
String string `json:"string"`
|
||||||
|
} `json:"self_test_code"`
|
||||||
|
SelfTestResult struct {
|
||||||
|
String string `json:"string"`
|
||||||
|
} `json:"self_test_result"`
|
||||||
|
PowerOnHours uint64 `json:"power_on_hours"`
|
||||||
|
} `json:"table"`
|
||||||
|
} `json:"nvme_self_test_log"`
|
||||||
NVMeHealth *struct {
|
NVMeHealth *struct {
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
PercentageUsed float64 `json:"percentage_used"`
|
PercentageUsed float64 `json:"percentage_used"`
|
||||||
@@ -410,6 +421,10 @@ func applySMART(disk *DiskMetrics) {
|
|||||||
if len(output) == 0 {
|
if len(output) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
applySMARTJSON(disk, output)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applySMARTJSON(disk *DiskMetrics, output []byte) {
|
||||||
var smart smartctlJSON
|
var smart smartctlJSON
|
||||||
if json.Unmarshal(output, &smart) != nil {
|
if json.Unmarshal(output, &smart) != nil {
|
||||||
return
|
return
|
||||||
@@ -451,12 +466,20 @@ func applySMART(disk *DiskMetrics) {
|
|||||||
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
|
disk.AttentionReasons = append(disk.AttentionReasons, "Ошибки целостности носителя: "+strconv.FormatUint(smart.NVMeHealth.MediaErrors, 10))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(smart.ATASelfTestLog.Standard.Table) > 0 {
|
if len(smart.NVMeSelfTestLog.Table) > 0 {
|
||||||
|
last := smart.NVMeSelfTestLog.Table[0]
|
||||||
|
description := strings.TrimSpace(last.SelfTestCode.String)
|
||||||
|
result := firstNonEmpty(strings.TrimSpace(last.SelfTestResult.String), "Результат неизвестен")
|
||||||
|
if description != "" {
|
||||||
|
disk.LastSelfTest = description + " · " + result
|
||||||
|
} else {
|
||||||
|
disk.LastSelfTest = result
|
||||||
|
}
|
||||||
|
setLastSelfTestAt(disk, last.PowerOnHours)
|
||||||
|
} else if len(smart.ATASelfTestLog.Standard.Table) > 0 {
|
||||||
last := smart.ATASelfTestLog.Standard.Table[0]
|
last := smart.ATASelfTestLog.Standard.Table[0]
|
||||||
disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
|
disk.LastSelfTest = firstNonEmpty(last.Status.String, "Результат неизвестен")
|
||||||
if disk.PowerOnHours != nil && *disk.PowerOnHours >= last.LifetimeHours {
|
setLastSelfTestAt(disk, last.LifetimeHours)
|
||||||
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-last.LifetimeHours) * time.Hour).Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
disk.SelfTestNeverRun = true
|
disk.SelfTestNeverRun = true
|
||||||
disk.LastSelfTest = "Никогда не запускался"
|
disk.LastSelfTest = "Никогда не запускался"
|
||||||
@@ -469,6 +492,12 @@ func applySMART(disk *DiskMetrics) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setLastSelfTestAt(disk *DiskMetrics, testPowerOnHours uint64) {
|
||||||
|
if disk.PowerOnHours != nil && *disk.PowerOnHours >= testPowerOnHours {
|
||||||
|
disk.LastSelfTestAt = time.Now().Add(-time.Duration(*disk.PowerOnHours-testPowerOnHours) * time.Hour).Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
|
func applyATAAttributes(disk *DiskMetrics, smart smartctlJSON) {
|
||||||
wearNames := map[string]bool{
|
wearNames := map[string]bool{
|
||||||
"Percent_Lifetime_Remain": true, "SSD_Life_Left": true,
|
"Percent_Lifetime_Remain": true, "SSD_Life_Left": true,
|
||||||
|
|||||||
@@ -6,8 +6,44 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestApplySMARTJSONReadsNVMeSelfTestHistory(t *testing.T) {
|
||||||
|
disk := DiskMetrics{Path: "/dev/nvme0n1"}
|
||||||
|
output := []byte(`{
|
||||||
|
"model_name":"ADATA LEGEND 900",
|
||||||
|
"smart_status":{"passed":true},
|
||||||
|
"power_on_time":{"hours":960},
|
||||||
|
"nvme_self_test_log":{
|
||||||
|
"current_self_test_operation":{"value":0,"string":"No self-test in progress"},
|
||||||
|
"table":[{
|
||||||
|
"self_test_code":{"value":1,"string":"Short"},
|
||||||
|
"self_test_result":{"value":0,"string":"Completed without error"},
|
||||||
|
"power_on_hours":959
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
|
||||||
|
before := time.Now().Add(-2 * time.Hour)
|
||||||
|
applySMARTJSON(&disk, output)
|
||||||
|
after := time.Now()
|
||||||
|
|
||||||
|
if disk.SelfTestNeverRun {
|
||||||
|
t.Fatal("завершённый NVMe self-test ошибочно отмечен как никогда не запускавшийся")
|
||||||
|
}
|
||||||
|
if disk.LastSelfTest != "Short · Completed without error" {
|
||||||
|
t.Fatalf("неожиданный результат self-test: %q", disk.LastSelfTest)
|
||||||
|
}
|
||||||
|
parsed, err := time.Parse(time.RFC3339, disk.LastSelfTestAt)
|
||||||
|
if err != nil || parsed.Before(before) || parsed.After(after) {
|
||||||
|
t.Fatalf("неверно вычислено время self-test: %q, err=%v", disk.LastSelfTestAt, err)
|
||||||
|
}
|
||||||
|
if len(disk.AttentionReasons) != 0 || disk.SMARTStatus != "Исправен" {
|
||||||
|
t.Fatalf("исправный NVMe требует внимания: %+v", disk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIsPhysicalDiskName(t *testing.T) {
|
func TestIsPhysicalDiskName(t *testing.T) {
|
||||||
tests := map[string]bool{
|
tests := map[string]bool{
|
||||||
"sda": true, "sdb": true, "nvme0n1": true,
|
"sda": true, "sdb": true, "nvme0n1": true,
|
||||||
|
|||||||
24
history.go
24
history.go
@@ -64,6 +64,30 @@ func (s *Store) AddEntityMetrics(metrics dashboardMetrics) {
|
|||||||
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
|
s.AddEntityHistory("backup-duration", "VMID "+task.ID, ts, &duration, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, agent := range metrics.Agents {
|
||||||
|
if !agent.Online {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := agent.ID + " · " + agent.Name
|
||||||
|
cpu := agent.Report.CPUPercent
|
||||||
|
memory := float64(0)
|
||||||
|
if agent.Report.MemoryTotal > 0 {
|
||||||
|
memory = float64(agent.Report.MemoryUsed) / float64(agent.Report.MemoryTotal) * 100
|
||||||
|
}
|
||||||
|
swap := float64(0)
|
||||||
|
if agent.Report.SwapTotal > 0 {
|
||||||
|
swap = float64(agent.Report.SwapUsed) / float64(agent.Report.SwapTotal) * 100
|
||||||
|
}
|
||||||
|
s.AddEntityHistory("agent-resource", name, ts, &cpu, &memory)
|
||||||
|
s.AddEntityHistory("agent-swap", name, ts, &swap, nil)
|
||||||
|
pressure := agent.Report.Pressure.IO.Avg10
|
||||||
|
s.AddEntityHistory("agent-pressure", name, ts, &pressure, nil)
|
||||||
|
for _, filesystem := range agent.Report.Filesystems {
|
||||||
|
used := filesystem.UsedPercent
|
||||||
|
inodes := filesystem.InodesPercent
|
||||||
|
s.AddEntityHistory("agent-filesystem", name+" · "+filesystem.Mountpoint, ts, &used, &inodes)
|
||||||
|
}
|
||||||
|
}
|
||||||
s.PruneEntityHistory()
|
s.PruneEntityHistory()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
110
service_links.go
Normal file
110
service_links.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ServiceLink struct {
|
||||||
|
ServiceID int64 `json:"serviceId"`
|
||||||
|
AgentID string `json:"agentId"`
|
||||||
|
VMID int `json:"vmid"`
|
||||||
|
ComponentType string `json:"componentType"`
|
||||||
|
ComponentName string `json:"componentName"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ServiceLinks() ([]ServiceLink, error) {
|
||||||
|
rows, err := s.db.Query(`SELECT service_id,agent_id,vmid,component_type,component_name,port FROM service_links ORDER BY service_id`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []ServiceLink
|
||||||
|
for rows.Next() {
|
||||||
|
var v ServiceLink
|
||||||
|
if err = rows.Scan(&v.ServiceID, &v.AgentID, &v.VMID, &v.ComponentType, &v.ComponentName, &v.Port); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) SaveServiceLink(v ServiceLink) (ServiceLink, error) {
|
||||||
|
v.AgentID = strings.TrimSpace(v.AgentID)
|
||||||
|
v.ComponentType = strings.TrimSpace(v.ComponentType)
|
||||||
|
v.ComponentName = strings.TrimSpace(v.ComponentName)
|
||||||
|
if v.ServiceID <= 0 {
|
||||||
|
return v, fmt.Errorf("выберите сервис")
|
||||||
|
}
|
||||||
|
if v.AgentID == "" && v.VMID <= 0 {
|
||||||
|
return v, fmt.Errorf("выберите агент или VMID")
|
||||||
|
}
|
||||||
|
if v.ComponentType != "" && v.ComponentType != "systemd" && v.ComponentType != "docker" && v.ComponentType != "process" {
|
||||||
|
return v, fmt.Errorf("неизвестный тип компонента")
|
||||||
|
}
|
||||||
|
if v.Port < 0 || v.Port > 65535 {
|
||||||
|
return v, fmt.Errorf("некорректный порт")
|
||||||
|
}
|
||||||
|
_, err := s.db.Exec(`INSERT INTO service_links(service_id,agent_id,vmid,component_type,component_name,port) VALUES(?,?,?,?,?,?) ON CONFLICT(service_id) DO UPDATE SET agent_id=excluded.agent_id,vmid=excluded.vmid,component_type=excluded.component_type,component_name=excluded.component_name,port=excluded.port`, v.ServiceID, v.AgentID, v.VMID, v.ComponentType, v.ComponentName, v.Port)
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
func (s *Store) DeleteServiceLink(serviceID int64) error {
|
||||||
|
_, err := s.db.Exec(`DELETE FROM service_links WHERE service_id=?`, serviceID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func groupLinkedAlerts(metrics dashboardMetrics, alerts []Alert) []Alert {
|
||||||
|
if len(metrics.ServiceLinks) == 0 {
|
||||||
|
return alerts
|
||||||
|
}
|
||||||
|
agentOffline := map[string]ManagedAgent{}
|
||||||
|
for _, agent := range metrics.Agents {
|
||||||
|
if !agent.Online {
|
||||||
|
agentOffline[agent.ID] = agent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
affected := map[string][]string{}
|
||||||
|
remove := map[string]bool{}
|
||||||
|
for _, link := range metrics.ServiceLinks {
|
||||||
|
agent, offline := agentOffline[link.AgentID]
|
||||||
|
if !offline {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, service := range metrics.Services.Services {
|
||||||
|
if service.ID != link.ServiceID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, endpoint := range service.Endpoints {
|
||||||
|
if !endpoint.Up {
|
||||||
|
key := fmt.Sprintf("service-%d-%s", service.ID, endpoint.Kind)
|
||||||
|
remove[key] = true
|
||||||
|
affected[agent.ID] = append(affected[agent.ID], service.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(affected) == 0 {
|
||||||
|
return alerts
|
||||||
|
}
|
||||||
|
out := make([]Alert, 0, len(alerts))
|
||||||
|
for _, alert := range alerts {
|
||||||
|
skip := remove[alert.ID]
|
||||||
|
for agentID := range affected {
|
||||||
|
if alert.ID == "agent-offline-"+agentID+fmt.Sprintf("-%d", agentOffline[agentID].VMID) {
|
||||||
|
skip = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !skip {
|
||||||
|
out = append(out, alert)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for agentID, names := range affected {
|
||||||
|
sort.Strings(names)
|
||||||
|
agent := agentOffline[agentID]
|
||||||
|
out = append(out, Alert{ID: "group-agent-offline-" + agentID + fmt.Sprintf("-%d", agent.VMID), Severity: "critical", Source: "Корреляция", Title: agent.Name + ": агент недоступен, связанные сервисы не отвечают", Message: fmt.Sprintf("VMID %d. Затронуто сервисов: %d — %s. Вероятная первопричина: недоступна гостевая система или агент.", agent.VMID, len(names), strings.Join(names, ", "))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
3
store.go
3
store.go
@@ -140,6 +140,9 @@ func openStore(path string) (*Store, error) {
|
|||||||
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
|
`CREATE INDEX IF NOT EXISTS entity_history_kind_ts_idx ON entity_history(kind,ts)`,
|
||||||
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
|
`CREATE TABLE IF NOT EXISTS agent_enrollment_tokens (token_hash TEXT PRIMARY KEY, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER)`,
|
||||||
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
|
`CREATE TABLE IF NOT EXISTS agents (id TEXT PRIMARY KEY, name TEXT NOT NULL, vmid INTEGER NOT NULL DEFAULT 0, hostname TEXT NOT NULL DEFAULT '', machine_id TEXT NOT NULL DEFAULT '', secret_hash TEXT NOT NULL, enrolled_at INTEGER NOT NULL, last_seen INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '', report_json TEXT NOT NULL DEFAULT '{}', remote_address TEXT NOT NULL DEFAULT '')`,
|
||||||
|
`CREATE TABLE IF NOT EXISTS agent_actions (id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, action TEXT NOT NULL, target TEXT NOT NULL DEFAULT '', requested_at INTEGER NOT NULL, started_at INTEGER, completed_at INTEGER, status TEXT NOT NULL DEFAULT 'queued', result TEXT NOT NULL DEFAULT '', FOREIGN KEY(agent_id) REFERENCES agents(id) ON DELETE CASCADE)`,
|
||||||
|
`CREATE INDEX IF NOT EXISTS agent_actions_agent_status_idx ON agent_actions(agent_id,status)`,
|
||||||
|
`CREATE TABLE IF NOT EXISTS service_links (service_id INTEGER PRIMARY KEY, agent_id TEXT NOT NULL DEFAULT '', vmid INTEGER NOT NULL DEFAULT 0, component_type TEXT NOT NULL DEFAULT '', component_name TEXT NOT NULL DEFAULT '', port INTEGER NOT NULL DEFAULT 0, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE)`,
|
||||||
}
|
}
|
||||||
for _, statement := range statements {
|
for _, statement := range statements {
|
||||||
if _, err := db.Exec(statement); err != nil {
|
if _, err := db.Exec(statement); err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user