The smallest watchdog on earth. Tiny, monitoring-plugins compatible monitoring with a status page. https://cloud.docker.com/repository/docker/momar/chihuahua/general
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

check.go 3.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110
  1. package chihuahua
  2. import (
  3. "bytes"
  4. "context"
  5. "os/exec"
  6. "strings"
  7. "time"
  8. "codeberg.org/momar/chihuahua/types"
  9. "codeberg.org/momar/logg"
  10. )
  11. // MaxConnections defines the maximum number of simultaneous connections against a single server
  12. const MaxConnections = 5
  13. const ConnectionTimeout = 30 * time.Second
  14. // RunCheck runs a check and populates it with the interpreted result
  15. func RunCheck(checkName string, check *types.Check, shell []string) {
  16. logg.Tag("check", checkName).Debug("Executing command: %#v", append(shell, check.Command))
  17. ctx, cancel := context.WithTimeout(context.Background(), ConnectionTimeout)
  18. defer cancel()
  19. c := exec.CommandContext(ctx, shell[0], append(shell[1:], "[ -e ~/.chihuahuarc ] && eval \"$(cat ~/.chihuahuarc)\"; "+check.Command)...)
  20. var errbuf bytes.Buffer
  21. c.Stderr = &errbuf
  22. output, err := c.Output()
  23. stderr, _ := errbuf.ReadString(0)
  24. check.Error = strings.TrimSpace(string(stderr))
  25. if err == nil {
  26. check.Status = types.StatusOk
  27. } else if err.Error() == "exit status 1" {
  28. check.Status = types.StatusWarning
  29. } else if err.Error() == "exit status 2" {
  30. check.Status = types.StatusCritical
  31. } else {
  32. check.Status = types.StatusUnknown
  33. if err.Error() != "exit status 3" {
  34. check.Error = strings.TrimSpace(err.Error() + "\n" + check.Error)
  35. }
  36. }
  37. check.Details = strings.SplitN(strings.SplitN(string(output), "\n", 2)[0], "|", 2)[0]
  38. // TODO: parse performance data
  39. check.LastUpdate = time.Now()
  40. logg.Tag("check", checkName).Debug("Check completed, result %d", check.Status)
  41. }
  42. // RunServerChecks runs all checks on a server asynchronously (up to MaxConnections checks at the same time) and populates them with their interpreted results
  43. func RunServerChecks(serverName string, server *types.Server) {
  44. completed := 0
  45. channel := make(chan bool)
  46. connections := 0
  47. if len(server.Shell) <= 0 {
  48. server.Shell = []string{"sh", "-c"}
  49. }
  50. processCheck := func(checkName string, check *types.Check) {
  51. // limit to 5 simultaneous connections
  52. for connections > MaxConnections {
  53. time.Sleep(250 * time.Millisecond)
  54. }
  55. connections++
  56. logg.Tag("check", serverName).Debug("Processing check: %s", checkName)
  57. oldStatus := check.Status
  58. RunCheck(serverName+"/"+checkName, check, server.Shell)
  59. // repeat the check if the check failed to mitigate hiccups
  60. if oldStatus != check.Status && check.Status > 0 {
  61. RunCheck(serverName+"/"+checkName, check, server.Shell)
  62. }
  63. connections--
  64. completed++
  65. channel <- true
  66. }
  67. for checkName, check := range server.Checks {
  68. go processCheck(checkName, check)
  69. }
  70. // Wait for checks to complete
  71. for completed < len(server.Checks) && <-channel {
  72. logg.Tag("check", serverName).Debug("%d checks left", len(server.Checks)-completed)
  73. }
  74. }
  75. // RunAllChecks runs all checks on all servers asynchronously and populates them with their interpreted results
  76. func RunAllChecks(servers map[string]*types.Server) {
  77. completed := 0
  78. channel := make(chan bool)
  79. processServer := func(serverName string, server *types.Server) {
  80. logg.Tag("check").Debug("Processing server: %s", serverName)
  81. RunServerChecks(serverName, server)
  82. completed++
  83. channel <- true
  84. }
  85. for serverName, server := range servers {
  86. go processServer(serverName, server)
  87. }
  88. // Wait for checks to complete
  89. for completed < len(servers) && <-channel {
  90. logg.Tag("check").Debug("%d servers left", len(servers)-completed)
  91. }
  92. }