Add infrastructure reliability monitoring

network.go

@@ -2,7 +2,13 @@ package main
 
 import (
 	"bufio"
+	"context"
+	"io"
+	"net"
+	"net/http"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,6 +22,20 @@ type NetworkMetrics struct {
 	ReceivePerSec  float64            `json:"receivePerSec"`
 	TransmitPerSec float64            `json:"transmitPerSec"`
 	Interfaces     []NetworkInterface `json:"interfaces"`
+	Health         NetworkHealth      `json:"health"`
+}
+
+type NetworkHealth struct {
+	Gateway               string    `json:"gateway"`
+	GatewayUp             bool      `json:"gatewayUp"`
+	DNSUp                 bool      `json:"dnsUp"`
+	InternetUp            bool      `json:"internetUp"`
+	PacketLoss            float64   `json:"packetLoss"`
+	LatencyMs             float64   `json:"latencyMs"`
+	ExternalIP            string    `json:"externalIp"`
+	ExternalIPChanged     bool      `json:"externalIpChanged"`
+	DisappearedInterfaces []string  `json:"disappearedInterfaces"`
+	CheckedAt             time.Time `json:"checkedAt"`
 }
 
 type NetworkInterface struct {
@@ -31,6 +51,12 @@ type NetworkInterface struct {
 	VMID           int     `json:"vmid,omitempty"`
 	GuestName      string  `json:"guestName,omitempty"`
 	GuestType      string  `json:"guestType,omitempty"`
+	Kind           string  `json:"kind"`
+	State          string  `json:"state"`
+	Carrier        bool    `json:"carrier"`
+	SpeedMbps      int     `json:"speedMbps,omitempty"`
+	Duplex         string  `json:"duplex,omitempty"`
+	Master         string  `json:"master,omitempty"`
 }
 
 type networkSnapshot struct {
@@ -44,6 +70,10 @@ type networkCollector struct {
 	path               string
 	previous           networkSnapshot
 	previousInterfaces map[string]NetworkInterface
+	health             NetworkHealth
+	healthUpdatedAt    time.Time
+	externalIP         string
+	knownInterfaces    map[string]bool
 }
 
 func newNetworkCollector(path string) (*networkCollector, error) {
@@ -56,7 +86,11 @@ func newNetworkCollector(path string) (*networkCollector, error) {
 	for _, v := range details {
 		previous[v.Name] = v
 	}
-	return &networkCollector{path: path, previous: snapshot, previousInterfaces: previous}, nil
+	known := map[string]bool{}
+	for _, v := range details {
+		known[v.Name] = true
+	}
+	return &networkCollector{path: path, previous: snapshot, previousInterfaces: previous, knownInterfaces: known}, nil
 }
 
 func (c *networkCollector) collect() (NetworkMetrics, error) {
@@ -75,6 +109,7 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 	}
 	for i := range details {
 		v := &details[i]
+		decorateInterface(v)
 		if old, ok := c.previousInterfaces[v.Name]; ok && elapsed > 0 && v.ReceiveBytes >= old.ReceiveBytes && v.TransmitBytes >= old.TransmitBytes {
 			v.ReceivePerSec = float64(v.ReceiveBytes-old.ReceiveBytes) / elapsed
 			v.TransmitPerSec = float64(v.TransmitBytes-old.TransmitBytes) / elapsed
@@ -85,12 +120,100 @@ func (c *networkCollector) collect() (NetworkMetrics, error) {
 		c.previousInterfaces[v.Name] = v
 	}
 	c.previous = current
+	if time.Since(c.healthUpdatedAt) > 30*time.Second {
+		c.health = checkNetworkHealth(c.externalIP)
+		if c.health.ExternalIP != "" {
+			c.externalIP = c.health.ExternalIP
+		}
+		c.healthUpdatedAt = time.Now()
+	}
+	currentNames := map[string]bool{}
+	for _, v := range details {
+		currentNames[v.Name] = true
+	}
+	c.health.DisappearedInterfaces = nil
+	for name := range c.knownInterfaces {
+		if !currentNames[name] {
+			c.health.DisappearedInterfaces = append(c.health.DisappearedInterfaces, name)
+		}
+	}
+	c.knownInterfaces = currentNames
 	return NetworkMetrics{
 		Interface: strings.Join(interfaceNames, ", "), Interfaces: details, ReceiveBytes: current.received,
-		TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate,
+		TransmitBytes: current.sent, ReceivePerSec: receivedRate, TransmitPerSec: sentRate, Health: c.health,
 	}, nil
 }
 
+func decorateInterface(v *NetworkInterface) {
+	base := filepath.Join("/sys/class/net", v.Name)
+	v.State = readTrimmed(filepath.Join(base, "operstate"))
+	v.Carrier = readTrimmed(filepath.Join(base, "carrier")) == "1"
+	v.SpeedMbps, _ = strconv.Atoi(readTrimmed(filepath.Join(base, "speed")))
+	v.Duplex = readTrimmed(filepath.Join(base, "duplex"))
+	if target, err := filepath.EvalSymlinks(filepath.Join(base, "master")); err == nil {
+		v.Master = filepath.Base(target)
+	}
+	switch {
+	case strings.HasPrefix(v.Name, "vmbr"):
+		v.Kind = "Bridge"
+	case strings.HasPrefix(v.Name, "bond"):
+		v.Kind = "Bond"
+	case strings.Contains(v.Name, "."):
+		v.Kind = "VLAN"
+	case strings.HasPrefix(v.Name, "tap") || strings.HasPrefix(v.Name, "veth"):
+		v.Kind = "Guest"
+	default:
+		v.Kind = "Physical"
+	}
+}
+
+func checkNetworkHealth(previousIP string) NetworkHealth {
+	result := NetworkHealth{CheckedAt: time.Now()}
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
+	defer cancel()
+	if out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output(); err == nil {
+		f := strings.Fields(string(out))
+		for i := range f {
+			if f[i] == "via" && i+1 < len(f) {
+				result.Gateway = f[i+1]
+				break
+			}
+		}
+	}
+	if result.Gateway != "" {
+		out, _ := exec.CommandContext(ctx, "ping", "-c", "3", "-W", "1", result.Gateway).CombinedOutput()
+		text := string(out)
+		result.GatewayUp = strings.Contains(text, "0% packet loss")
+		if i := strings.Index(text, "packet loss"); i > 0 {
+			start := strings.LastIndex(text[:i], " ")
+			if start >= 0 {
+				result.PacketLoss, _ = strconv.ParseFloat(strings.TrimSuffix(strings.TrimSpace(text[start:i]), "%"), 64)
+			}
+		}
+		if i := strings.Index(text, "min/avg/max"); i >= 0 {
+			if eq := strings.Index(text[i:], "="); eq >= 0 {
+				parts := strings.Split(strings.Fields(text[i+eq+1:])[0], "/")
+				if len(parts) > 1 {
+					result.LatencyMs, _ = strconv.ParseFloat(parts[1], 64)
+				}
+			}
+		}
+	}
+	resolver := net.Resolver{}
+	_, err := resolver.LookupHost(ctx, "deb.debian.org")
+	result.DNSUp = err == nil
+	client := http.Client{Timeout: 4 * time.Second}
+	request, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.ipify.org", nil)
+	if response, err := client.Do(request); err == nil {
+		defer response.Body.Close()
+		body, _ := io.ReadAll(io.LimitReader(response.Body, 128))
+		result.ExternalIP = strings.TrimSpace(string(body))
+		result.InternetUp = response.StatusCode == 200 && net.ParseIP(result.ExternalIP) != nil
+	}
+	result.ExternalIPChanged = previousIP != "" && result.ExternalIP != "" && previousIP != result.ExternalIP
+	return result
+}
+
 func readNetworkSnapshot(path string) (networkSnapshot, []string, []NetworkInterface, error) {
 	file, err := os.Open(path)
 	if err != nil {