354 lines
11 KiB
Go
354 lines
11 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"crypto/tls"
|
||
"database/sql"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"os/exec"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
type MonitoredService struct {
|
||
ID int64 `json:"id"`
|
||
Name string `json:"name"`
|
||
URL string `json:"url"`
|
||
LocalAddress string `json:"localAddress"`
|
||
CheckType string `json:"checkType"`
|
||
ExpectedCode int `json:"expectedCode"`
|
||
ExpectedText string `json:"expectedText"`
|
||
TimeoutSeconds int `json:"timeoutSeconds"`
|
||
Enabled bool `json:"enabled"`
|
||
}
|
||
|
||
type ServiceStatus struct {
|
||
MonitoredService
|
||
Endpoints []ServiceEndpointStatus `json:"endpoints"`
|
||
Up bool `json:"up"`
|
||
CheckedAt int64 `json:"checkedAt"`
|
||
LatencyMS float64 `json:"latencyMs"`
|
||
StatusCode int `json:"statusCode"`
|
||
Error string `json:"error,omitempty"`
|
||
TLSExpiry *int64 `json:"tlsExpiry,omitempty"`
|
||
TLSDays *int `json:"tlsDays,omitempty"`
|
||
UptimeDay float64 `json:"uptimeDay"`
|
||
UptimeWeek float64 `json:"uptimeWeek"`
|
||
}
|
||
|
||
type ServiceEndpointStatus struct {
|
||
Kind string `json:"kind"`
|
||
Label string `json:"label"`
|
||
Address string `json:"address"`
|
||
Up bool `json:"up"`
|
||
CheckedAt int64 `json:"checkedAt"`
|
||
LatencyMS float64 `json:"latencyMs"`
|
||
StatusCode int `json:"statusCode"`
|
||
Error string `json:"error,omitempty"`
|
||
TLSExpiry *int64 `json:"tlsExpiry,omitempty"`
|
||
TLSDays *int `json:"tlsDays,omitempty"`
|
||
TLSSubject string `json:"tlsSubject,omitempty"`
|
||
TLSIssuer string `json:"tlsIssuer,omitempty"`
|
||
TLSDNSNames []string `json:"tlsDnsNames,omitempty"`
|
||
TLSHostnameValid bool `json:"tlsHostnameValid"`
|
||
TLSHostnameError string `json:"tlsHostnameError,omitempty"`
|
||
UptimeDay float64 `json:"uptimeDay"`
|
||
UptimeWeek float64 `json:"uptimeWeek"`
|
||
}
|
||
|
||
type ServicesMetrics struct {
|
||
Available bool `json:"available"`
|
||
Up int `json:"up"`
|
||
Total int `json:"total"`
|
||
Services []ServiceStatus `json:"services"`
|
||
}
|
||
type ServiceHistoryPoint struct {
|
||
Timestamp int64 `json:"timestamp"`
|
||
Up bool `json:"up"`
|
||
LatencyMS float64 `json:"latencyMs"`
|
||
}
|
||
|
||
var serviceHTTPTransport = &http.Transport{
|
||
Proxy: http.ProxyFromEnvironment,
|
||
DialContext: (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||
ForceAttemptHTTP2: true,
|
||
MaxIdleConns: 32,
|
||
MaxIdleConnsPerHost: 2,
|
||
IdleConnTimeout: 30 * time.Second,
|
||
TLSHandshakeTimeout: 10 * time.Second,
|
||
ExpectContinueTimeout: time.Second,
|
||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12},
|
||
}
|
||
|
||
var serviceHTTPClient = &http.Client{Transport: serviceHTTPTransport}
|
||
|
||
func (s *Store) Services() ([]MonitoredService, error) {
|
||
rows, err := s.db.Query(`SELECT id,name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled FROM services ORDER BY id`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var result []MonitoredService
|
||
for rows.Next() {
|
||
var v MonitoredService
|
||
if err = rows.Scan(&v.ID, &v.Name, &v.URL, &v.LocalAddress, &v.CheckType, &v.ExpectedCode, &v.ExpectedText, &v.TimeoutSeconds, &v.Enabled); err != nil {
|
||
return nil, err
|
||
}
|
||
result = append(result, v)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
func validateService(v MonitoredService) error {
|
||
v.Name = strings.TrimSpace(v.Name)
|
||
v.URL = strings.TrimSpace(v.URL)
|
||
v.LocalAddress = strings.TrimSpace(v.LocalAddress)
|
||
if v.Name == "" || (v.URL == "" && v.LocalAddress == "") {
|
||
return fmt.Errorf("укажите название и хотя бы один адрес")
|
||
}
|
||
if v.CheckType != "http" && v.CheckType != "tcp" && v.CheckType != "ping" {
|
||
return fmt.Errorf("неизвестный тип проверки")
|
||
}
|
||
if v.TimeoutSeconds < 1 || v.TimeoutSeconds > 30 {
|
||
return fmt.Errorf("таймаут должен быть от 1 до 30 секунд")
|
||
}
|
||
if v.CheckType == "http" {
|
||
for _, target := range []string{v.URL, v.LocalAddress} {
|
||
if target == "" {
|
||
continue
|
||
}
|
||
u, err := url.Parse(target)
|
||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||
return fmt.Errorf("оба адреса должны начинаться с http:// или https://")
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Store) SaveService(v MonitoredService) (MonitoredService, error) {
|
||
if err := validateService(v); err != nil {
|
||
return v, err
|
||
}
|
||
if v.ExpectedCode == 0 {
|
||
v.ExpectedCode = 200
|
||
}
|
||
if v.ID == 0 {
|
||
r, err := s.db.Exec(`INSERT INTO services(name,url,local_address,check_type,expected_code,expected_text,timeout_seconds,enabled) VALUES(?,?,?,?,?,?,?,?)`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled)
|
||
if err != nil {
|
||
return v, err
|
||
}
|
||
v.ID, _ = r.LastInsertId()
|
||
} else {
|
||
_, err := s.db.Exec(`UPDATE services SET name=?,url=?,local_address=?,check_type=?,expected_code=?,expected_text=?,timeout_seconds=?,enabled=? WHERE id=?`, v.Name, v.URL, v.LocalAddress, v.CheckType, v.ExpectedCode, v.ExpectedText, v.TimeoutSeconds, v.Enabled, v.ID)
|
||
if err != nil {
|
||
return v, err
|
||
}
|
||
}
|
||
return v, nil
|
||
}
|
||
func (s *Store) DeleteService(id int64) error {
|
||
_, err := s.db.Exec(`DELETE FROM services WHERE id=?`, id)
|
||
return err
|
||
}
|
||
func (s *Store) addServiceCheck(id int64, endpoint string, v ServiceEndpointStatus) {
|
||
var expiry any
|
||
if v.TLSExpiry != nil {
|
||
expiry = *v.TLSExpiry
|
||
}
|
||
_, _ = s.db.Exec(`INSERT INTO service_checks(service_id,ts,up,latency_ms,status_code,error,tls_expiry,endpoint) VALUES(?,?,?,?,?,?,?,?)`, id, v.CheckedAt, v.Up, v.LatencyMS, v.StatusCode, v.Error, expiry, endpoint)
|
||
_, _ = s.db.Exec(`DELETE FROM service_checks WHERE ts<?`, time.Now().Add(-historyRetention).Unix())
|
||
}
|
||
func (s *Store) serviceUptime(id int64, endpoint string, since time.Time) float64 {
|
||
var value sql.NullFloat64
|
||
_ = s.db.QueryRow(`SELECT AVG(up)*100 FROM service_checks WHERE service_id=? AND endpoint=? AND ts>=?`, id, endpoint, since.Unix()).Scan(&value)
|
||
if value.Valid {
|
||
return value.Float64
|
||
}
|
||
return 0
|
||
}
|
||
func (s *Store) ServiceHistory(id int64, hours int) ([]ServiceHistoryPoint, error) {
|
||
if hours < 1 {
|
||
hours = 24
|
||
}
|
||
if hours > 168 {
|
||
hours = 168
|
||
}
|
||
rows, err := s.db.Query(`SELECT ts,up,latency_ms FROM service_checks WHERE service_id=? AND ts>=? ORDER BY ts`, id, time.Now().Add(-time.Duration(hours)*time.Hour).Unix())
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []ServiceHistoryPoint
|
||
for rows.Next() {
|
||
var p ServiceHistoryPoint
|
||
if err = rows.Scan(&p.Timestamp, &p.Up, &p.LatencyMS); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, p)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
type serviceCollector struct {
|
||
mu sync.RWMutex
|
||
metrics ServicesMetrics
|
||
store *Store
|
||
}
|
||
|
||
func newServiceCollector(store *Store) *serviceCollector {
|
||
c := &serviceCollector{store: store}
|
||
go c.run()
|
||
return c
|
||
}
|
||
func (c *serviceCollector) collect() ServicesMetrics {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
return c.metrics
|
||
}
|
||
func (c *serviceCollector) run() {
|
||
c.checkAll()
|
||
ticker := time.NewTicker(15 * time.Second)
|
||
defer ticker.Stop()
|
||
for range ticker.C {
|
||
c.checkAll()
|
||
}
|
||
}
|
||
func (c *serviceCollector) checkAll() {
|
||
items, err := c.store.Services()
|
||
if err != nil {
|
||
return
|
||
}
|
||
m := ServicesMetrics{Available: true}
|
||
for _, v := range items {
|
||
if !v.Enabled {
|
||
continue
|
||
}
|
||
st := checkService(v)
|
||
for i := range st.Endpoints {
|
||
ep := &st.Endpoints[i]
|
||
ep.UptimeDay = c.store.serviceUptime(v.ID, ep.Kind, time.Now().Add(-24*time.Hour))
|
||
ep.UptimeWeek = c.store.serviceUptime(v.ID, ep.Kind, time.Now().Add(-7*24*time.Hour))
|
||
c.store.addServiceCheck(v.ID, ep.Kind, *ep)
|
||
}
|
||
if len(st.Endpoints) > 0 {
|
||
st.UptimeDay = st.Endpoints[0].UptimeDay
|
||
st.UptimeWeek = st.Endpoints[0].UptimeWeek
|
||
}
|
||
m.Total++
|
||
if st.Up {
|
||
m.Up++
|
||
}
|
||
m.Services = append(m.Services, st)
|
||
}
|
||
c.mu.Lock()
|
||
c.metrics = m
|
||
c.mu.Unlock()
|
||
}
|
||
|
||
func checkService(v MonitoredService) ServiceStatus {
|
||
st := ServiceStatus{MonitoredService: v, CheckedAt: time.Now().Unix(), Up: true}
|
||
if v.URL != "" {
|
||
st.Endpoints = append(st.Endpoints, checkServiceEndpoint(v, "public", "Внешний адрес", v.URL))
|
||
}
|
||
if v.LocalAddress != "" {
|
||
st.Endpoints = append(st.Endpoints, checkServiceEndpoint(v, "local", "Локальный адрес", v.LocalAddress))
|
||
}
|
||
for _, ep := range st.Endpoints {
|
||
if !ep.Up {
|
||
st.Up = false
|
||
}
|
||
st.LatencyMS += ep.LatencyMS
|
||
if st.StatusCode == 0 {
|
||
st.StatusCode = ep.StatusCode
|
||
}
|
||
if st.Error == "" && ep.Error != "" {
|
||
st.Error = ep.Label + ": " + ep.Error
|
||
}
|
||
}
|
||
if len(st.Endpoints) > 0 {
|
||
st.LatencyMS /= float64(len(st.Endpoints))
|
||
ep := st.Endpoints[0]
|
||
st.TLSExpiry = ep.TLSExpiry
|
||
st.TLSDays = ep.TLSDays
|
||
}
|
||
return st
|
||
}
|
||
|
||
func checkServiceEndpoint(v MonitoredService, kind, label, target string) ServiceEndpointStatus {
|
||
st := ServiceEndpointStatus{Kind: kind, Label: label, Address: target, CheckedAt: time.Now().Unix()}
|
||
start := time.Now()
|
||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(v.TimeoutSeconds)*time.Second)
|
||
defer cancel()
|
||
switch v.CheckType {
|
||
case "tcp":
|
||
d := net.Dialer{}
|
||
conn, err := d.DialContext(ctx, "tcp", target)
|
||
if err == nil {
|
||
st.Up = true
|
||
conn.Close()
|
||
} else {
|
||
st.Error = err.Error()
|
||
}
|
||
case "ping":
|
||
host := strings.TrimPrefix(strings.TrimPrefix(target, "https://"), "http://")
|
||
host = strings.Split(host, "/")[0]
|
||
host = strings.Split(host, ":")[0]
|
||
arg := "-W"
|
||
timeout := strconv.Itoa(v.TimeoutSeconds)
|
||
if _, err := exec.LookPath("ping"); err != nil {
|
||
st.Error = "ping не установлен"
|
||
} else if out, err := exec.CommandContext(ctx, "ping", "-c", "1", arg, timeout, host).CombinedOutput(); err == nil {
|
||
st.Up = true
|
||
} else {
|
||
st.Error = strings.TrimSpace(string(out))
|
||
}
|
||
default:
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||
if err != nil {
|
||
st.Error = err.Error()
|
||
break
|
||
}
|
||
resp, err := serviceHTTPClient.Do(req)
|
||
if err != nil {
|
||
st.Error = err.Error()
|
||
} else {
|
||
st.StatusCode = resp.StatusCode
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||
resp.Body.Close()
|
||
st.Up = resp.StatusCode == v.ExpectedCode
|
||
if st.Up && v.ExpectedText != "" && !strings.Contains(string(body), v.ExpectedText) {
|
||
st.Up = false
|
||
st.Error = "ожидаемый текст не найден"
|
||
}
|
||
if !st.Up && st.Error == "" {
|
||
st.Error = fmt.Sprintf("HTTP %d, ожидался %d", resp.StatusCode, v.ExpectedCode)
|
||
}
|
||
if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
|
||
cert := resp.TLS.PeerCertificates[0]
|
||
expiry := cert.NotAfter.Unix()
|
||
days := int(time.Until(cert.NotAfter).Hours() / 24)
|
||
st.TLSExpiry = &expiry
|
||
st.TLSDays = &days
|
||
st.TLSSubject = cert.Subject.CommonName
|
||
st.TLSIssuer = cert.Issuer.CommonName
|
||
st.TLSDNSNames = cert.DNSNames
|
||
host := req.URL.Hostname()
|
||
if err := cert.VerifyHostname(host); err == nil {
|
||
st.TLSHostnameValid = true
|
||
} else {
|
||
st.TLSHostnameError = err.Error()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
st.LatencyMS = float64(time.Since(start).Microseconds()) / 1000
|
||
return st
|
||
}
|