result_handler.go 2.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. package alerting
  2. import (
  3. "time"
  4. "github.com/grafana/grafana/pkg/bus"
  5. "github.com/grafana/grafana/pkg/log"
  6. "github.com/grafana/grafana/pkg/metrics"
  7. m "github.com/grafana/grafana/pkg/models"
  8. "github.com/grafana/grafana/pkg/services/annotations"
  9. )
  10. type ResultHandler interface {
  11. Handle(ctx *EvalContext)
  12. }
  13. type DefaultResultHandler struct {
  14. notifier Notifier
  15. log log.Logger
  16. }
  17. func NewResultHandler() *DefaultResultHandler {
  18. return &DefaultResultHandler{
  19. log: log.New("alerting.resultHandler"),
  20. notifier: NewRootNotifier(),
  21. }
  22. }
  23. func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
  24. oldState := ctx.Rule.State
  25. exeuctionError := ""
  26. if ctx.Error != nil {
  27. handler.log.Error("Alert Rule Result Error", "ruleId", ctx.Rule.Id, "error", ctx.Error)
  28. ctx.Rule.State = m.AlertStateExeuctionError
  29. exeuctionError = ctx.Error.Error()
  30. } else if ctx.Firing {
  31. ctx.Rule.State = m.AlertStateType(ctx.Rule.Severity)
  32. } else {
  33. ctx.Rule.State = m.AlertStateOK
  34. }
  35. countSeverity(ctx.Rule.Severity)
  36. if ctx.Rule.State != oldState {
  37. handler.log.Info("New state change", "alertId", ctx.Rule.Id, "newState", ctx.Rule.State, "oldState", oldState)
  38. cmd := &m.SetAlertStateCommand{
  39. AlertId: ctx.Rule.Id,
  40. OrgId: ctx.Rule.OrgId,
  41. State: ctx.Rule.State,
  42. Error: exeuctionError,
  43. }
  44. if err := bus.Dispatch(cmd); err != nil {
  45. handler.log.Error("Failed to save state", "error", err)
  46. }
  47. // save annotation
  48. item := annotations.Item{
  49. OrgId: ctx.Rule.OrgId,
  50. Type: annotations.AlertType,
  51. AlertId: ctx.Rule.Id,
  52. Title: ctx.Rule.Name,
  53. Text: ctx.GetStateText(),
  54. NewState: string(ctx.Rule.State),
  55. PrevState: string(oldState),
  56. Timestamp: time.Now(),
  57. }
  58. annotationRepo := annotations.GetRepository()
  59. if err := annotationRepo.Save(&item); err != nil {
  60. handler.log.Error("Failed to save annotation for new alert state", "error", err)
  61. }
  62. handler.notifier.Notify(ctx)
  63. }
  64. }
  65. func countSeverity(state m.AlertSeverityType) {
  66. switch state {
  67. case m.AlertSeverityOK:
  68. metrics.M_Alerting_Result_Ok.Inc(1)
  69. case m.AlertSeverityInfo:
  70. metrics.M_Alerting_Result_Info.Inc(1)
  71. case m.AlertSeverityWarning:
  72. metrics.M_Alerting_Result_Warning.Inc(1)
  73. case m.AlertSeverityCritical:
  74. metrics.M_Alerting_Result_Critical.Inc(1)
  75. }
  76. }