diff --git a/cmd/bosun/expr/expr.go b/cmd/bosun/expr/expr.go index 9e9f627b76..e8db477078 100644 --- a/cmd/bosun/expr/expr.go +++ b/cmd/bosun/expr/expr.go @@ -29,6 +29,7 @@ type State struct { enableComputations bool unjoinedOk bool autods int + vValue float64 *Backends @@ -76,6 +77,7 @@ func (e *Expr) MarshalJSON() ([]byte, error) { return json.Marshal(e.String()) } +// New creates a new expression tree func New(expr string, funcs ...map[string]parse.Func) (*Expr, error) { funcs = append(funcs, builtins) t, err := parse.Parse(expr, funcs...) @@ -172,6 +174,11 @@ type String string func (s String) Type() models.FuncType { return models.TypeString } func (s String) Value() interface{} { return s } +type NumberExpr Expr + +func (s NumberExpr) Type() models.FuncType { return models.TypeNumberExpr } +func (s NumberExpr) Value() interface{} { return s } + //func (s String) MarshalJSON() ([]byte, error) { return json.Marshal(s) } // Series is the standard form within bosun to represent timeseries data. @@ -460,12 +467,24 @@ func (e *State) walk(node parse.Node, T miniprofiler.Timer) *Results { res = e.walkUnary(node, T) case *parse.FuncNode: res = e.walkFunc(node, T) + case *parse.ExprNode: + res = e.walkExpr(node, T) default: panic(fmt.Errorf("expr: unknown node type")) } return res } +func (e *State) walkExpr(node *parse.ExprNode, T miniprofiler.Timer) *Results { + return &Results{ + Results: ResultSlice{ + &Result{ + Value: NumberExpr{node.Tree}, + }, + }, + } +} + func (e *State) walkBinary(node *parse.BinaryNode, T miniprofiler.Timer) *Results { ar := e.walk(node.Args[0], T) br := e.walk(node.Args[1], T) @@ -692,6 +711,8 @@ func (e *State) walkFunc(node *parse.FuncNode, T miniprofiler.Timer) *Results { v = extract(e.walkUnary(t, T)) case *parse.BinaryNode: v = extract(e.walkBinary(t, T)) + case *parse.ExprNode: + v = e.walkExpr(t, T) default: panic(fmt.Errorf("expr: unknown func arg type")) } @@ -741,5 +762,8 @@ func extract(res *Results) interface{} { if len(res.Results) == 1 && res.Results[0].Type() == models.TypeString { return string(res.Results[0].Value.Value().(String)) } + if len(res.Results) == 1 && res.Results[0].Type() == models.TypeNumberExpr { + return res.Results[0].Value.Value() + } return res } diff --git a/cmd/bosun/expr/expr_test.go b/cmd/bosun/expr/expr_test.go index dffe77b99a..fbbb45c7ab 100644 --- a/cmd/bosun/expr/expr_test.go +++ b/cmd/bosun/expr/expr_test.go @@ -278,6 +278,7 @@ func TestSeriesOperations(t *testing.T) { }, }, }, + false, }, { fmt.Sprintf(template, seriesA, "+", seriesC), @@ -291,6 +292,7 @@ func TestSeriesOperations(t *testing.T) { }, }, }, + false, }, { fmt.Sprintf(template, seriesA, "/", seriesB), @@ -306,6 +308,7 @@ func TestSeriesOperations(t *testing.T) { }, }, }, + false, }, } for _, test := range tests { diff --git a/cmd/bosun/expr/funcs.go b/cmd/bosun/expr/funcs.go index fcf1f6ad49..ac044eec07 100644 --- a/cmd/bosun/expr/funcs.go +++ b/cmd/bosun/expr/funcs.go @@ -334,6 +334,49 @@ var builtins = map[string]parse.Func{ Tags: tagFirst, F: Tail, }, + "map": { + Args: []models.FuncType{models.TypeSeriesSet, models.TypeNumberExpr}, + Return: models.TypeSeriesSet, + Tags: tagFirst, + F: Map, + }, + "v": { + Return: models.TypeScalar, + F: V, + MapFunc: true, + }, +} + +func V(e *State, T miniprofiler.Timer) (*Results, error) { + return fromScalar(e.vValue), nil +} + +func Map(e *State, T miniprofiler.Timer, series *Results, expr *Results) (*Results, error) { + newExpr := Expr{expr.Results[0].Value.Value().(NumberExpr).Tree} + for _, result := range series.Results { + newSeries := make(Series) + for t, v := range result.Value.Value().(Series) { + e.vValue = v + subResults, _, err := newExpr.ExecuteState(e, T) + if err != nil { + return series, err + } + for _, res := range subResults.Results { + var v float64 + switch res.Value.Value().(type) { + case Number: + v = float64(res.Value.Value().(Number)) + case Scalar: + v = float64(res.Value.Value().(Scalar)) + default: + return series, fmt.Errorf("wrong return type for map expr: %v", res.Type()) + } + newSeries[t] = v + } + } + result.Value = newSeries + } + return series, nil } func SeriesFunc(e *State, T miniprofiler.Timer, tags string, pairs ...float64) (*Results, error) { diff --git a/cmd/bosun/expr/funcs_test.go b/cmd/bosun/expr/funcs_test.go index cebd506a91..e0808f63dd 100644 --- a/cmd/bosun/expr/funcs_test.go +++ b/cmd/bosun/expr/funcs_test.go @@ -11,12 +11,19 @@ import ( ) type exprInOut struct { - expr string - out Results + expr string + out Results + shouldParseErr bool } func testExpression(eio exprInOut) error { e, err := New(eio.expr, builtins) + if eio.shouldParseErr { + if err == nil { + return fmt.Errorf("no error when expected error on %v", eio.expr) + } + return nil + } if err != nil { return err } @@ -44,6 +51,7 @@ func TestDuration(t *testing.T) { }, }, }, + false, } err := testExpression(d) if err != nil { @@ -81,6 +89,7 @@ func TestToDuration(t *testing.T) { }, }, }, + false, } err := testExpression(d) if err != nil { @@ -103,6 +112,7 @@ func TestUngroup(t *testing.T) { }, }, }, + false, }) if err != nil { @@ -131,6 +141,7 @@ func TestMerge(t *testing.T) { }, }, }, + false, }) if err != nil { t.Error(err) @@ -155,6 +166,7 @@ func TestMerge(t *testing.T) { }, }, }, + false, }) if err == nil { t.Errorf("error expected due to identical groups in merge but did not get one") @@ -191,6 +203,7 @@ func TestTimedelta(t *testing.T) { }, }, }, + false, }) if err != nil { @@ -236,6 +249,7 @@ func TestTail(t *testing.T) { }, }, }, + false, }) if err != nil { diff --git a/cmd/bosun/expr/map_test.go b/cmd/bosun/expr/map_test.go new file mode 100644 index 0000000000..1bb98a843c --- /dev/null +++ b/cmd/bosun/expr/map_test.go @@ -0,0 +1,166 @@ +package expr + +import ( + "testing" + "time" + + "bosun.org/opentsdb" +) + +func TestMap(t *testing.T) { + err := testExpression(exprInOut{ + `map(series("test=test", 0, 1, 1, 3), expr(v()+1))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{ + time.Unix(0, 0): 2, + time.Unix(1, 0): 4, + }, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `avg(map(series("test=test", 0, 1, 1, 3), expr(v()+1)))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Number(3), + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `1 + avg(map(series("test=test", 0, 1, 1, 3), expr(v()+1))) + 1`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Number(5), + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `max(map(series("test=test", 0, 1, 1, 3), expr(v()+v())))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Number(6), + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `map(series("test=test", 0, -2, 1, 3), expr(1+1))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{ + time.Unix(0, 0): 2, + time.Unix(1, 0): 2, + }, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `map(series("test=test", 0, -2, 1, 3), expr(abs(v())))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{ + time.Unix(0, 0): 2, + time.Unix(1, 0): 3, + }, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `map(series("test=test", 0, -2, 1, 3), expr(series("test=test", 0, v())))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{}, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + true, // expect parse error here, series result not valid as TypeNumberExpr + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `v()`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{}, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + true, // v() is not valid outside a map expression + }) + if err != nil { + t.Error(err) + } + + err = testExpression(exprInOut{ + `map(series("test=test", 0, -2, 1, 0), expr(!v()))`, + Results{ + Results: ResultSlice{ + &Result{ + Value: Series{ + time.Unix(0, 0): 0, + time.Unix(1, 0): 1, + }, + Group: opentsdb.TagSet{"test": "test"}, + }, + }, + }, + false, + }) + if err != nil { + t.Error(err) + } +} diff --git a/cmd/bosun/expr/parse/lex.go b/cmd/bosun/expr/parse/lex.go index 152ca26430..16569675e4 100644 --- a/cmd/bosun/expr/parse/lex.go +++ b/cmd/bosun/expr/parse/lex.go @@ -58,6 +58,7 @@ const ( itemFunc itemTripleQuotedString itemPow // '**' + itemExpr ) const eof = -1 @@ -278,6 +279,10 @@ func lexFunc(l *lexer) stateFn { // absorb default: l.backup() + if l.input[l.start:l.pos] == "expr" { + l.emit(itemExpr) + return lexItem + } l.emit(itemFunc) return lexItem } diff --git a/cmd/bosun/expr/parse/node.go b/cmd/bosun/expr/parse/node.go index e818d0086a..a8a9b2cb4d 100644 --- a/cmd/bosun/expr/parse/node.go +++ b/cmd/bosun/expr/parse/node.go @@ -58,6 +58,7 @@ const ( NodeUnary // Unary operator: !, - NodeString // A string constant. NodeNumber // A numerical constant. + NodeExpr // A sub expression ) // Nodes. @@ -104,6 +105,9 @@ func (f *FuncNode) StringAST() string { } func (f *FuncNode) Check(t *Tree) error { + if f.F.MapFunc && !t.mapExpr { + return fmt.Errorf("%v is only valid in a map expression", f.Name) + } const errFuncType = "parse: bad argument type in %s, expected %s, got %s" // For VArgs we make sure they are all of the expected type if f.F.VArgs { @@ -164,6 +168,48 @@ type NumberNode struct { Text string // The original textual representation from the input. } +type ExprNode struct { + NodeType + Pos + Text string + Tree *Tree +} + +func newExprNode(text string, pos Pos) (*ExprNode, error) { + return &ExprNode{ + NodeType: NodeExpr, + Text: text, + Pos: pos, + }, nil +} + +func (s *ExprNode) String() string { + return fmt.Sprintf("%v", s.Text) +} + +func (s *ExprNode) StringAST() string { + return s.String() +} + +func (s *ExprNode) Check(*Tree) error { + return nil +} + +func (s *ExprNode) Return() models.FuncType { + switch s.Tree.Root.Return() { + case models.TypeNumberSet, models.TypeScalar: + return models.TypeNumberExpr + case models.TypeSeriesSet: + return models.TypeSeriesExpr + default: + return models.TypeUnexpected + } +} + +func (s *ExprNode) Tags() (Tags, error) { + return nil, nil +} + func newNumber(pos Pos, text string) (*NumberNode, error) { n := &NumberNode{NodeType: NodeNumber, Pos: pos, Text: text} // Do integer test first so we get 0x123 etc. @@ -365,7 +411,7 @@ func Walk(n Node, f func(Node)) { for _, a := range n.Args { Walk(a, f) } - case *NumberNode, *StringNode: + case *NumberNode, *StringNode, *ExprNode: // Ignore. case *UnaryNode: Walk(n.Arg, f) diff --git a/cmd/bosun/expr/parse/parse.go b/cmd/bosun/expr/parse/parse.go index b354a5bdc6..b96052acab 100644 --- a/cmd/bosun/expr/parse/parse.go +++ b/cmd/bosun/expr/parse/parse.go @@ -22,7 +22,8 @@ type Tree struct { Text string // text parsed to create the expression. Root Node // top-level root of the tree, returns a number. - funcs []map[string]Func + funcs []map[string]Func + mapExpr bool // Parsing only; cleared after parse. lex *lexer @@ -38,6 +39,7 @@ type Func struct { VArgs bool VArgsPos int VArgsOmit bool + MapFunc bool // Func is only valid in map expressions Check func(*Tree, *FuncNode) error } @@ -95,6 +97,14 @@ func Parse(text string, funcs ...map[string]Func) (t *Tree, err error) { return } +func ParseSub(text string, funcs ...map[string]Func) (t *Tree, err error) { + t = New() + t.mapExpr = true + t.Text = text + err = t.Parse(text, funcs...) + return +} + // next returns the next token. func (t *Tree) next() item { if t.peekCount > 0 { @@ -220,7 +230,7 @@ func (t *Tree) Parse(text string, funcs ...map[string]Func) (err error) { // It runs to EOF. func (t *Tree) parse() { t.Root = t.O() - t.expect(itemEOF, "input") + t.expect(itemEOF, "root input") if err := t.Root.Check(t); err != nil { t.error(err) } @@ -236,7 +246,7 @@ E -> F {( "**" ) F} F -> v | "(" O ")" | "!" O | "-" O v -> number | func(..) Func -> name "(" param {"," param} ")" -param -> number | "string" | [query] +param -> number | "string" | subExpr | [query] */ // expr: @@ -321,10 +331,10 @@ func (t *Tree) F() Node { case itemLeftParen: t.next() n := t.O() - t.expect(itemRightParen, "input") + t.expect(itemRightParen, "input: F()") return n default: - t.unexpected(token, "input") + t.unexpected(token, "input: F()") } return nil } @@ -341,7 +351,7 @@ func (t *Tree) v() Node { t.backup() return t.Func() default: - t.unexpected(token, "input") + t.unexpected(token, "input: v()") } return nil } @@ -369,6 +379,38 @@ func (t *Tree) Func() (f *FuncNode) { f.append(newString(token.pos, token.val, s)) case itemRightParen: return + case itemExpr: + t.expect(itemLeftParen, "v() expect left paran in itemExpr") + start := t.lex.lastPos + leftCount := 1 + TOKENS: + for { + switch token = t.next(); token.typ { + case itemLeftParen: + leftCount++ + case itemFunc: + case itemRightParen: + leftCount-- + if leftCount == 0 { + t.expect(itemRightParen, "v() expect right paren in itemExpr") + t.backup() + break TOKENS + } + case itemEOF: + t.unexpected(token, "input: v()") + default: + // continue + } + } + n, err := newExprNode(t.lex.input[start:t.lex.lastPos], t.lex.lastPos) + if err != nil { + t.error(err) + } + n.Tree, err = ParseSub(n.Text, t.funcs...) + if err != nil { + t.error(err) + } + f.append(n) } switch token = t.next(); token.typ { case itemComma: @@ -393,6 +435,20 @@ func (t *Tree) GetFunction(name string) (v Func, ok bool) { return } +func (t *Tree) SetFunction(name string, F interface{}) error { + for i, funcMap := range t.funcs { + if funcMap == nil { + continue + } + if v, ok := funcMap[name]; ok { + v.F = F + t.funcs[i][name] = v + return nil + } + } + return fmt.Errorf("can not set function, function %v not found", name) +} + func (t *Tree) String() string { return t.Root.String() } diff --git a/cmd/bosun/expr/parse/parse_test.go b/cmd/bosun/expr/parse/parse_test.go index aaefac139a..8ec01e6432 100644 --- a/cmd/bosun/expr/parse/parse_test.go +++ b/cmd/bosun/expr/parse/parse_test.go @@ -159,6 +159,7 @@ var builtins = map[string]Func{ false, 0, false, + false, nil, }, "band": { @@ -169,6 +170,7 @@ var builtins = map[string]Func{ false, 0, false, + false, nil, }, "q": { @@ -179,6 +181,7 @@ var builtins = map[string]Func{ false, 0, false, + false, nil, }, "forecastlr": { @@ -189,6 +192,7 @@ var builtins = map[string]Func{ false, 0, false, + false, nil, }, } diff --git a/docs/expressions.md b/docs/expressions.md index 1f6fcef310..34a2bc7f1f 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -562,6 +562,30 @@ Returns the first key from the given lookup table with matching tags, this searc Returns the first key from the given lookup table with matching tags. The first argument is a series to use from which to derive the tag information. This is good for alternative storage backends such as graphite and influxdb. +## map(series seriesSet, subExpr numberSetExpr) seriesSet + +map applies the subExpr to each value in each series in the set. A special function `v()` which is only available in a numberSetExpr and it gives you the value for each item in the series. + +For example you can do something like the following to get the absolute value for each item in the series (since the normal `abs()` function works on normal numbers, not series: + +``` +$q = q("avg:rate:os.cpu{host=*bosun*}", "5m", "") +map($q, expr(abs(v()))) +``` + +Or for another example, this would get you the absolute difference of each datapoint from the series average as a new series: + +``` +$q = q("avg:rate:os.cpu{host=*bosun*}", "5m", "") +map($q, expr(abs(v()-avg($q)))) +``` + +Since this function is not optimized for a particular operation on a seriesSet it may not be very efficent. If you find you are doing things that involve more complex expressions within the `expr(...)` inside map (for example, having query functions in there) than you may want to consider requesting a new function to be added to bosun's DSL. + +## expr(expression) + +expr takes an expression and returns either a numberSetExpr or a seriesSetExpr depending on the resulting type of the inner expression. This exists for functions like `map` - it is currently not valid in the expression language outside of function arguments. + ## month(offset scalar, startEnd string) scalar Returns the epoch of either the start or end of the month. Offset is the timezone offset from UTC that the month starts/ends at (but the returned epoch is representitive of UTC). startEnd must be either `"start"` or `"end"`. Useful for things like monthly billing, for example: diff --git a/models/incidents.go b/models/incidents.go index 4cb4c27dfd..26ddf8be28 100644 --- a/models/incidents.go +++ b/models/incidents.go @@ -118,6 +118,10 @@ func (f FuncType) String() string { return "esquery" case TypeESIndexer: return "esindexer" + case TypeNumberExpr: + return "numberexpr" + case TypeSeriesExpr: + return "seriesexpr" default: return "unknown" } @@ -130,6 +134,9 @@ const ( TypeSeriesSet TypeESQuery TypeESIndexer + TypeNumberExpr + TypeSeriesExpr // No implmentation yet + TypeUnexpected ) type Status int