299 lines
12 KiB
Go
299 lines
12 KiB
Go
package main
|
||
|
||
import (
|
||
"bufio"
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
type NetworkMetrics struct {
|
||
Interface string `json:"interface"`
|
||
ReceiveBytes uint64 `json:"receiveBytes"`
|
||
TransmitBytes uint64 `json:"transmitBytes"`
|
||
ReceivePerSec float64 `json:"receivePerSec"`
|
||
TransmitPerSec float64 `json:"transmitPerSec"`
|
||
Interfaces []NetworkInterface `json:"interfaces"`
|
||
Health NetworkHealth `json:"health"`
|
||
}
|
||
|
||
type NetworkHealth struct {
|
||
Gateway string `json:"gateway"`
|
||
GatewayUp bool `json:"gatewayUp"`
|
||
DNSUp bool `json:"dnsUp"`
|
||
InternetUp bool `json:"internetUp"`
|
||
PacketLoss float64 `json:"packetLoss"`
|
||
LatencyMs float64 `json:"latencyMs"`
|
||
ExternalIP string `json:"externalIp"`
|
||
ExternalIPChanged bool `json:"externalIpChanged"`
|
||
DisappearedInterfaces []string `json:"disappearedInterfaces"`
|
||
CheckedAt time.Time `json:"checkedAt"`
|
||
Summary string `json:"summary"`
|
||
LikelyCause string `json:"likelyCause"`
|
||
RecommendedAction string `json:"recommendedAction"`
|
||
}
|
||
|
||
type NetworkInterface struct {
|
||
Name string `json:"name"`
|
||
ReceiveBytes uint64 `json:"receiveBytes"`
|
||
TransmitBytes uint64 `json:"transmitBytes"`
|
||
ReceiveErrors uint64 `json:"receiveErrors"`
|
||
TransmitErrors uint64 `json:"transmitErrors"`
|
||
ReceiveDrops uint64 `json:"receiveDrops"`
|
||
TransmitDrops uint64 `json:"transmitDrops"`
|
||
NewErrors uint64 `json:"newErrors"`
|
||
NewReceiveDrops uint64 `json:"newReceiveDrops"`
|
||
NewTransmitDrops uint64 `json:"newTransmitDrops"`
|
||
FaultCounters map[string]uint64 `json:"faultCounters,omitempty"`
|
||
NewFaults uint64 `json:"newFaults"`
|
||
LinkDown bool `json:"linkDown"`
|
||
NewDrops uint64 `json:"newDrops"`
|
||
ReceivePerSec float64 `json:"receivePerSec"`
|
||
TransmitPerSec float64 `json:"transmitPerSec"`
|
||
VMID int `json:"vmid,omitempty"`
|
||
GuestName string `json:"guestName,omitempty"`
|
||
GuestType string `json:"guestType,omitempty"`
|
||
Kind string `json:"kind"`
|
||
State string `json:"state"`
|
||
Carrier bool `json:"carrier"`
|
||
SpeedMbps int `json:"speedMbps,omitempty"`
|
||
Duplex string `json:"duplex,omitempty"`
|
||
Master string `json:"master,omitempty"`
|
||
}
|
||
|
||
type networkSnapshot struct {
|
||
at time.Time
|
||
received uint64
|
||
sent uint64
|
||
}
|
||
|
||
type networkCollector struct {
|
||
mu sync.Mutex
|
||
path string
|
||
previous networkSnapshot
|
||
previousInterfaces map[string]NetworkInterface
|
||
health NetworkHealth
|
||
healthUpdatedAt time.Time
|
||
externalIP string
|
||
knownInterfaces map[string]bool
|
||
}
|
||
|
||
func newNetworkCollector(path string) (*networkCollector, error) {
|
||
snapshot, _, details, err := readNetworkSnapshot(path)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
previous := map[string]NetworkInterface{}
|
||
for _, v := range details {
|
||
decorateInterface(&v)
|
||
collectInterfaceFaults(&v)
|
||
previous[v.Name] = v
|
||
}
|
||
known := map[string]bool{}
|
||
for _, v := range details {
|
||
known[v.Name] = true
|
||
}
|
||
return &networkCollector{path: path, previous: snapshot, previousInterfaces: previous, knownInterfaces: known}, nil
|
||
}
|
||
|
||
func (c *networkCollector) collect() (NetworkMetrics, error) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
|
||
current, interfaceNames, details, err := readNetworkSnapshot(c.path)
|
||
if err != nil {
|
||
return NetworkMetrics{}, err
|
||
}
|
||
elapsed := current.at.Sub(c.previous.at).Seconds()
|
||
var receivedRate, sentRate float64
|
||
if elapsed > 0 && current.received >= c.previous.received && current.sent >= c.previous.sent {
|
||
receivedRate = float64(current.received-c.previous.received) / elapsed
|
||
sentRate = float64(current.sent-c.previous.sent) / elapsed
|
||
}
|
||
for i := range details {
|
||
v := &details[i]
|
||
decorateInterface(v)
|
||
collectInterfaceFaults(v)
|
||
if old, ok := c.previousInterfaces[v.Name]; ok {
|
||
updateInterfaceDeltas(v, old, elapsed)
|
||
}
|
||
v.LinkDown = interfaceLinkDown(*v)
|
||
}
|
||
c.previousInterfaces = map[string]NetworkInterface{}
|
||
for _, v := range details {
|
||
c.previousInterfaces[v.Name] = v
|
||
}
|
||
c.previous = current
|
||
if time.Since(c.healthUpdatedAt) > 30*time.Second {
|
||
c.health = checkNetworkHealth(c.externalIP)
|
||
if c.health.ExternalIP != "" {
|
||
c.externalIP = c.health.ExternalIP
|
||
}
|
||
c.healthUpdatedAt = time.Now()
|
||
}
|
||
currentNames := map[string]bool{}
|
||
for _, v := range details {
|
||
currentNames[v.Name] = true
|
||
}
|
||
c.health.DisappearedInterfaces = nil
|
||
for name := range c.knownInterfaces {
|
||
if !currentNames[name] {
|
||
c.health.DisappearedInterfaces = append(c.health.DisappearedInterfaces, name)
|
||
}
|
||
}
|
||
c.knownInterfaces = currentNames
|
||
explainNetworkHealth(&c.health, details)
|
||
return NetworkMetrics{
|
||
Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
|
||
TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
|
||
}, nil
|
||
}
|
||
|
||
func explainNetworkHealth(health *NetworkHealth, interfaces []NetworkInterface) {
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = "Сеть работает штатно", "Проблем не обнаружено", "Действий не требуется"
|
||
switch {
|
||
case health.Gateway == "":
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = "Не найден маршрут в локальную сеть", "На хосте отсутствует default gateway", "Проверьте адрес шлюза и конфигурацию bridge/VLAN"
|
||
case !health.GatewayUp:
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = "Хост не видит локальный шлюз", "Проблема между Proxmox и роутером: линк, bridge, bond или VLAN", "Проверьте кабель, carrier и настройки vmbr/bond/VLAN"
|
||
case health.PacketLoss > 0:
|
||
health.Summary = fmt.Sprintf("Нестабильная локальная сеть: потеряно %.1f%% пакетов", health.PacketLoss)
|
||
health.LikelyCause, health.RecommendedAction = "Перегрузка или ошибки линка между Proxmox и шлюзом", "Проверьте аппаратные ошибки, кабель, порт коммутатора и duplex"
|
||
case !health.DNSUp && health.InternetUp:
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = "Интернет доступен, но DNS не отвечает", "Проблема DNS-сервера или /etc/resolv.conf", "Проверьте настроенные DNS и попробуйте разрешить домен вручную"
|
||
case health.DNSUp && !health.InternetUp:
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = "Локальная сеть и DNS работают, но внешняя проверка недоступна", "Проблема выше роутера, блокировка HTTPS или недоступность провайдера", "Проверьте маршрут наружу, firewall и доступ к внешним адресам"
|
||
}
|
||
for _, v := range interfaces {
|
||
if v.LinkDown {
|
||
health.Summary, health.LikelyCause, health.RecommendedAction = v.Name+": линк не поднят", "Нет carrier либо интерфейс административно выключен", "Проверьте кабель, порт коммутатора и состояние интерфейса"
|
||
return
|
||
}
|
||
if v.NewErrors+v.NewFaults > 0 {
|
||
health.Summary = fmt.Sprintf("%s: за последний интервал появилось %d изменений счётчиков сетевых ошибок", v.Name, v.NewErrors+v.NewFaults)
|
||
health.LikelyCause, health.RecommendedAction = "Счётчики интерфейса сейчас растут из-за перегрузки, кабеля, MTU или duplex", "Проверьте скорость и duplex, кабель, порт коммутатора и MTU на обоих концах"
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
func decorateInterface(v *NetworkInterface) {
|
||
base := filepath.Join("/sys/class/net", v.Name)
|
||
v.State = readTrimmed(filepath.Join(base, "operstate"))
|
||
v.Carrier = readTrimmed(filepath.Join(base, "carrier")) == "1"
|
||
v.SpeedMbps, _ = strconv.Atoi(readTrimmed(filepath.Join(base, "speed")))
|
||
v.Duplex = readTrimmed(filepath.Join(base, "duplex"))
|
||
if target, err := filepath.EvalSymlinks(filepath.Join(base, "master")); err == nil {
|
||
v.Master = filepath.Base(target)
|
||
}
|
||
switch {
|
||
case strings.HasPrefix(v.Name, "fwbr"):
|
||
v.Kind = "Firewall bridge"
|
||
case strings.HasPrefix(v.Name, "fwpr") || strings.HasPrefix(v.Name, "fwln"):
|
||
v.Kind = "Firewall link"
|
||
case strings.HasPrefix(v.Name, "vmbr"):
|
||
v.Kind = "Bridge"
|
||
case strings.HasPrefix(v.Name, "bond"):
|
||
v.Kind = "Bond"
|
||
case strings.Contains(v.Name, "."):
|
||
v.Kind = "VLAN"
|
||
case strings.HasPrefix(v.Name, "tap") || strings.HasPrefix(v.Name, "veth"):
|
||
v.Kind = "Guest"
|
||
default:
|
||
v.Kind = "Physical"
|
||
}
|
||
}
|
||
|
||
func checkNetworkHealth(previousIP string) NetworkHealth {
|
||
result := NetworkHealth{CheckedAt: time.Now()}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
||
defer cancel()
|
||
if out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output(); err == nil {
|
||
f := strings.Fields(string(out))
|
||
for i := range f {
|
||
if f[i] == "via" && i+1 < len(f) {
|
||
result.Gateway = f[i+1]
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if result.Gateway != "" {
|
||
out, _ := exec.CommandContext(ctx, "ping", "-c", "3", "-W", "1", result.Gateway).CombinedOutput()
|
||
text := string(out)
|
||
result.GatewayUp = strings.Contains(text, "0% packet loss")
|
||
if i := strings.Index(text, "packet loss"); i > 0 {
|
||
start := strings.LastIndex(text[:i], " ")
|
||
if start >= 0 {
|
||
result.PacketLoss, _ = strconv.ParseFloat(strings.TrimSuffix(strings.TrimSpace(text[start:i]), "%"), 64)
|
||
}
|
||
}
|
||
if i := strings.Index(text, "min/avg/max"); i >= 0 {
|
||
if eq := strings.Index(text[i:], "="); eq >= 0 {
|
||
parts := strings.Split(strings.Fields(text[i+eq+1:])[0], "/")
|
||
if len(parts) > 1 {
|
||
result.LatencyMs, _ = strconv.ParseFloat(parts[1], 64)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
resolver := net.Resolver{}
|
||
_, err := resolver.LookupHost(ctx, "deb.debian.org")
|
||
result.DNSUp = err == nil
|
||
client := http.Client{Timeout: 4 * time.Second}
|
||
request, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.ipify.org", nil)
|
||
if response, err := client.Do(request); err == nil {
|
||
defer response.Body.Close()
|
||
body, _ := io.ReadAll(io.LimitReader(response.Body, 128))
|
||
result.ExternalIP = strings.TrimSpace(string(body))
|
||
result.InternetUp = response.StatusCode == 200 && net.ParseIP(result.ExternalIP) != nil
|
||
}
|
||
result.ExternalIPChanged = previousIP != "" && result.ExternalIP != "" && previousIP != result.ExternalIP
|
||
return result
|
||
}
|
||
|
||
func readNetworkSnapshot(path string) (networkSnapshot, []string, []NetworkInterface, error) {
|
||
file, err := os.Open(path)
|
||
if err != nil {
|
||
return networkSnapshot{}, nil, nil, err
|
||
}
|
||
defer file.Close()
|
||
|
||
var received, sent uint64
|
||
var interfaces []string
|
||
var details []NetworkInterface
|
||
scanner := bufio.NewScanner(file)
|
||
for scanner.Scan() {
|
||
line := strings.TrimSpace(scanner.Text())
|
||
name, values, found := strings.Cut(line, ":")
|
||
if !found || strings.TrimSpace(name) == "lo" {
|
||
continue
|
||
}
|
||
fields := strings.Fields(values)
|
||
if len(fields) < 16 {
|
||
continue
|
||
}
|
||
rx, errRX := strconv.ParseUint(fields[0], 10, 64)
|
||
tx, errTX := strconv.ParseUint(fields[8], 10, 64)
|
||
if errRX == nil && errTX == nil {
|
||
received += rx
|
||
sent += tx
|
||
interfaces = append(interfaces, strings.TrimSpace(name))
|
||
rxErr, _ := strconv.ParseUint(fields[2], 10, 64)
|
||
rxDrop, _ := strconv.ParseUint(fields[3], 10, 64)
|
||
txErr, _ := strconv.ParseUint(fields[10], 10, 64)
|
||
txDrop, _ := strconv.ParseUint(fields[11], 10, 64)
|
||
details = append(details, NetworkInterface{Name: strings.TrimSpace(name), ReceiveBytes: rx, TransmitBytes: tx, ReceiveErrors: rxErr, ReceiveDrops: rxDrop, TransmitErrors: txErr, TransmitDrops: txDrop})
|
||
}
|
||
}
|
||
return networkSnapshot{at: time.Now(), received: received, sent: sent}, interfaces, details, scanner.Err()
|
||
}
|