bosun-monitor · sipidronov · Jun 29, 2018 · Jun 29, 2018 · Jun 29, 2018 · Sep 20, 2018
diff --git a/cmd/bosun/conf/conf.go b/cmd/bosun/conf/conf.go
@@ -449,6 +449,10 @@ type Alert struct {
 	AlertTemplateKeys map[string]*template.Template `json:"-"`
 }
 
+func (a *Alert) GetName() string {
+	return a.Name
+}
+
 // A Locator stores the information about the location of the rule in the underlying
 // rule store
 type Locator interface{}

diff --git a/cmd/bosun/conf/unknownNotify.go b/cmd/bosun/conf/unknownNotify.go
@@ -3,6 +3,7 @@ package conf
 import (
 	"bytes"
 	"fmt"
+	"net/url"
 	"time"
 
 	"bosun.org/cmd/bosun/conf/template"
@@ -11,9 +12,17 @@ import (
 )
 
 type unknownContext struct {
-	Time  time.Time
-	Name  string
-	Group models.AlertKeys
+	Time     time.Time
+	Name     string
+	Group    models.AlertKeys
+	States   *models.IncidentState
+	makeLink func(string, *url.Values) string
+}
+
+func (u *unknownContext) IncidentUnknownLink(i int64) string {
+	return u.makeLink("/incident", &url.Values{
+		"id": []string{fmt.Sprint(i)},
+	})
 }
 
 var defaultUnknownTemplate = &Template{
@@ -44,11 +53,13 @@ func init() {
 	unknownDefaults.body = template.Must(template.New("body").Parse(body))
 }
 
-func (n *Notification) PrepareUnknown(t *Template, c SystemConfProvider, name string, aks []models.AlertKey) *PreparedNotifications {
+func (n *Notification) PrepareUnknown(t *Template, c SystemConfProvider, name string, aks []models.AlertKey, states *models.IncidentState) *PreparedNotifications {
 	ctx := &unknownContext{
-		Time:  time.Now().UTC(),
-		Name:  name,
-		Group: aks,
+		Time:     time.Now().UTC(),
+		Name:     name,
+		Group:    aks,
+		States:   states,
+		makeLink: c.MakeLink,
 	}
 	pn := &PreparedNotifications{}
 	buf := &bytes.Buffer{}
@@ -101,8 +112,8 @@ func (n *Notification) PrepareUnknown(t *Template, c SystemConfProvider, name st
 	return pn
 }
 
-func (n *Notification) NotifyUnknown(t *Template, c SystemConfProvider, name string, aks []models.AlertKey) {
-	go n.PrepareUnknown(t, c, name, aks).Send(c)
+func (n *Notification) NotifyUnknown(t *Template, c SystemConfProvider, name string, aks []models.AlertKey, states *models.IncidentState) {
+	go n.PrepareUnknown(t, c, name, aks, states).Send(c)
 }
 
 var unknownMultiDefaults defaultTemplates
@@ -111,6 +122,7 @@ type unknownMultiContext struct {
 	Time      time.Time
 	Threshold int
 	Groups    map[string]models.AlertKeys
+	States    []*models.IncidentState
 }
 
 func init() {
@@ -135,11 +147,12 @@ func init() {
 	unknownMultiDefaults.body = template.Must(template.New("body").Parse(body))
 }
 
-func (n *Notification) PrepareMultipleUnknowns(t *Template, c SystemConfProvider, groups map[string]models.AlertKeys) *PreparedNotifications {
+func (n *Notification) PrepareMultipleUnknowns(t *Template, c SystemConfProvider, groups map[string]models.AlertKeys, states []*models.IncidentState) *PreparedNotifications {
 	ctx := &unknownMultiContext{
 		Time:      time.Now().UTC(),
 		Threshold: c.GetUnknownThreshold(),
 		Groups:    groups,
+		States:    states,
 	}
 	pn := &PreparedNotifications{}
 	buf := &bytes.Buffer{}
@@ -180,8 +193,8 @@ func (n *Notification) PrepareMultipleUnknowns(t *Template, c SystemConfProvider
 	return pn
 }
 
-func (n *Notification) NotifyMultipleUnknowns(t *Template, c SystemConfProvider, groups map[string]models.AlertKeys) {
-	n.PrepareMultipleUnknowns(t, c, groups).Send(c)
+func (n *Notification) NotifyMultipleUnknowns(t *Template, c SystemConfProvider, groups map[string]models.AlertKeys, states []*models.IncidentState) {
+	n.PrepareMultipleUnknowns(t, c, groups, states).Send(c)
 }
 
 // code common to PrepareAction / PrepareUnknown / PrepareMultipleUnknowns

diff --git a/cmd/bosun/expr/expr.go b/cmd/bosun/expr/expr.go
@@ -33,6 +33,7 @@ type State struct {
 	unjoinedOk         bool
 	autods             int
 	vValue             float64
+	supplicant         interface{}
 
 	Timer miniprofiler.Timer
 
@@ -96,7 +97,7 @@ func New(expr string, funcs ...map[string]parse.Func) (*Expr, error) {
 
 // Execute applies a parse expression to the specified OpenTSDB context, and
 // returns one result per group. T may be nil to ignore timings.
-func (e *Expr) Execute(backends *Backends, providers *BosunProviders, T miniprofiler.Timer, now time.Time, autods int, unjoinedOk bool) (r *Results, queries []opentsdb.Request, err error) {
+func (e *Expr) Execute(backends *Backends, providers *BosunProviders, T miniprofiler.Timer, now time.Time, autods int, unjoinedOk bool, supplicant interface{}) (r *Results, queries []opentsdb.Request, err error) {
 	if providers.Squelched == nil {
 		providers.Squelched = func(tags opentsdb.TagSet) bool {
 			return false
@@ -109,7 +110,9 @@ func (e *Expr) Execute(backends *Backends, providers *BosunProviders, T miniprof
 		unjoinedOk:     unjoinedOk,
 		Backends:       backends,
 		BosunProviders: providers,
+		supplicant:     supplicant,
 		Timer:          T,
+		supplicant:     supplicant,
 	}
 	return e.ExecuteState(s)
 }

diff --git a/cmd/bosun/expr/expr_test.go b/cmd/bosun/expr/expr_test.go
@@ -68,7 +68,7 @@ func TestExprSimple(t *testing.T) {
 			InfluxConfig: client.HTTPConfig{},
 		}
 		providers := &BosunProviders{}
-		r, _, err := e.Execute(backends, providers, nil, time.Now(), 0, false)
+		r, _, err := e.Execute(backends, providers, nil, time.Now(), 0, false, nil)
 		if err != nil {
 			t.Error(err)
 			break
@@ -214,7 +214,7 @@ func TestQueryExpr(t *testing.T) {
 			InfluxConfig: client.HTTPConfig{},
 		}
 		providers := &BosunProviders{}
-		results, _, err := e.Execute(backends, providers, nil, queryTime, 0, false)
+		results, _, err := e.Execute(backends, providers, nil, queryTime, 0, false, nil)
 		if err != nil {
 			t.Fatal(err)
 		}

diff --git a/cmd/bosun/expr/funcs_test.go b/cmd/bosun/expr/funcs_test.go
@@ -32,7 +32,7 @@ func testExpression(eio exprInOut) error {
 		InfluxConfig: client.HTTPConfig{},
 	}
 	providers := &BosunProviders{}
-	r, _, err := e.Execute(backends, providers, nil, queryTime, 0, false)
+	r, _, err := e.Execute(backends, providers, nil, queryTime, 0, false, nil)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/bosun/expr/graphite.go b/cmd/bosun/expr/graphite.go
@@ -179,9 +179,10 @@ func GraphiteQuery(e *State, query string, sduration, eduration, format string)
 	st := e.now.Add(-time.Duration(sd))
 	et := e.now.Add(-time.Duration(ed))
 	req := &graphite.Request{
-		Targets: []string{query},
-		Start:   &st,
-		End:     &et,
+		Targets:  []string{query},
+		Start:    &st,
+		End:      &et,
+		DataProv: e.supplicant,
 	}
 	s, err := timeGraphiteRequest(e, req)
 	if err != nil {
@@ -215,6 +216,7 @@ func timeGraphiteRequest(e *State, req *graphite.Request) (resp graphite.Respons
 	e.Timer.StepCustomTiming("graphite", "query", string(b), func() {
 		key := req.CacheKey()
 		getFn := func() (interface{}, error) {
+			e.GraphiteContext.SubstitueHeaders(req)
 			return e.GraphiteContext.Query(req)
 		}
 		var val interface{}

diff --git a/cmd/bosun/sched/check.go b/cmd/bosun/sched/check.go
@@ -643,7 +643,7 @@ func (s *Schedule) executeExpr(T miniprofiler.Timer, rh *RunHistory, a *conf.Ale
 		History:   s,
 		Annotate:  s.annotate,
 	}
-	results, _, err := e.Execute(rh.Backends, providers, T, rh.Start, 0, a.UnjoinedOK)
+	results, _, err := e.Execute(rh.Backends, providers, T, rh.Start, 0, a.UnjoinedOK, a)
 	return results, err
 }
 

diff --git a/cmd/bosun/sched/notify.go b/cmd/bosun/sched/notify.go
@@ -186,6 +186,8 @@ func (s *Schedule) sendUnknownNotifications() {
 			ustates[st.AlertKey] = st
 		}
 		var c int
+		var multiUstates []*models.IncidentState
+
 		hitThreshold := false
 		overThresholdSets := make(map[string]models.AlertKeys)
 		minGroupSize := s.SystemConf.GetMinGroupSize()
@@ -199,20 +201,23 @@ func (s *Schedule) sendUnknownNotifications() {
 		}
 		for name, group := range groupSets {
 			c++
-			if c >= threshold && threshold > 0 {
-				if !hitThreshold && len(groupSets) == c {
-					// If the threshold is hit but only 1 email remains, just send the normal unknown
-					n.NotifyUnknown(gk.template, s.SystemConf, name, group)
-					break
+			for _, ak := range group {
+				if c >= threshold && threshold > 0 {
+					if !hitThreshold && len(groupSets) == c {
+						// If the threshold is hit but only 1 email remains, just send the normal unknown
+						n.NotifyUnknown(gk.template, s.SystemConf, name, group, ustates[ak])
+						break
+					}
+					hitThreshold = true
+					overThresholdSets[name] = group
+					multiUstates = append(multiUstates, ustates[ak])
+				} else {
+					n.NotifyUnknown(gk.template, s.SystemConf, name, group, ustates[ak])
 				}
-				hitThreshold = true
-				overThresholdSets[name] = group
-			} else {
-				n.NotifyUnknown(gk.template, s.SystemConf, name, group)
 			}
 		}
 		if len(overThresholdSets) > 0 {
-			n.NotifyMultipleUnknowns(gk.template, s.SystemConf, overThresholdSets)
+			n.NotifyMultipleUnknowns(gk.template, s.SystemConf, overThresholdSets, multiUstates)
 		}
 	}
 	s.pendingUnknowns = make(map[notificationGroupKey][]*models.IncidentState)

diff --git a/cmd/bosun/sched/template.go b/cmd/bosun/sched/template.go
@@ -303,7 +303,7 @@ func (c *Context) evalExpr(e *expr.Expr, filter bool, series bool, autods int) (
 		Squelched: c.schedule.RuleConf.AlertSquelched(c.Alert),
 		History:   c.schedule,
 	}
-	res, _, err := e.Execute(c.runHistory.Backends, providers, nil, c.runHistory.Start, autods, c.Alert.UnjoinedOK)
+	res, _, err := e.Execute(c.runHistory.Backends, providers, nil, c.runHistory.Start, autods, c.Alert.UnjoinedOK, c.Alert)
 	if err != nil {
 		return nil, "", fmt.Errorf("%s: %v", e, err)
 	}

diff --git a/cmd/bosun/web/chart.go b/cmd/bosun/web/chart.go
@@ -253,7 +253,7 @@ func ExprGraph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (in
 		Squelched: nil,
 		History:   nil,
 	}
-	res, _, err := e.Execute(backends, providers, t, now, autods, false)
+	res, _, err := e.Execute(backends, providers, t, now, autods, false, nil)
 	if err != nil {
 		return nil, err
 	}

diff --git a/cmd/bosun/web/expr.go b/cmd/bosun/web/expr.go
@@ -95,7 +95,7 @@ func Expr(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (v inter
 		History:   nil,
 		Annotate:  AnnotateBackend,
 	}
-	res, queries, err := e.Execute(backends, providers, t, now, 0, false)
+	res, queries, err := e.Execute(backends, providers, t, now, 0, false, nil)
 	if err != nil {
 		return nil, err
 	}