Funcfile Support (#104)

Implement funcfile, `len` and `@len`

cmd/expressions.go

@@ -8,7 +8,7 @@ import (
 	"rare/pkg/color"
 	"rare/pkg/expressions"
 	"rare/pkg/expressions/exprofiler"
-	"rare/pkg/expressions/stdlib"
+	"rare/pkg/expressions/funclib"
 	"rare/pkg/humanize"
 	"rare/pkg/minijson"
 	"strconv"
@@ -45,7 +45,7 @@ func expressionFunction(c *cli.Context) error {
 		expString = string(b)
 	}
 
-	builder := stdlib.NewStdKeyBuilderEx(!noOptimize)
+	builder := funclib.NewKeyBuilderEx(!noOptimize)
 	compiled, err := builder.Compile(expString)
 
 	if err != nil {

cmd/reduce.go

@@ -7,7 +7,7 @@ import (
 	"rare/pkg/aggregation/sorting"
 	"rare/pkg/color"
 	"rare/pkg/csv"
-	"rare/pkg/expressions/stdlib"
+	"rare/pkg/expressions/funclib"
 	"rare/pkg/logger"
 	"rare/pkg/multiterm/termrenderers"
 	"strings"
@@ -31,7 +31,7 @@ func reduceFunction(c *cli.Context) error {
 	batcher := helpers.BuildBatcherFromArguments(c)
 	extractor := helpers.BuildExtractorFromArguments(c, batcher)
 
-	aggr := aggregation.NewAccumulatingGroup(stdlib.NewStdKeyBuilder())
+	aggr := aggregation.NewAccumulatingGroup(funclib.NewKeyBuilder())
 
 	// Set up groups
 	for _, group := range groupExpr {

docs/usage/expressions.md

@@ -42,6 +42,10 @@ $ rare expression -d 15 -d 20 -k key=30 "The sum is {sumi {0} {1} {key}}"
 The sum is 65
 ```
 
+### Extending
+
+You can add custom functions to *rare* using a [Funcs File](funcsfile.md)
+
 ## Examples
 
 ### Parsing an nginx access.log file
@@ -150,6 +154,12 @@ Clamps a given input `intVal` between `min` and `max`.  If falls outside bucket,
 the word "min" or "max" as appropriate.  If you wish to not see these values, you can
 filter with `--ignore`
 
+#### Len
+
+Syntax: `{len string}`
+
+Returns the length of the provided `string`. eg. the string of `hello` returns 5.
+
 ### Logic
 
 #### If, Unless
@@ -314,9 +324,17 @@ Syntax: `{@ ele0 ele1 ele2}` (`{$ ele0 ele1 ele2}` is equivalent)
 
 Creates an array with the provided elements. Use `{@}` for an array of all matches.
 
+#### @len
+
+Syntax: `{@len <arr>}`
+
+Returns the length of an array.  Empty "" returns 0, a literal will be 1.
+
+**Note:** This is a linear-time operation.
+
 #### @in
 
-Syntax: `{@in <val> array}` or `{@in <val> {@ val0 val1 val2 ...}}`
+Syntax: `{@in <val> <arr>}` or `{@in <val> {@ val0 val1 val2 ...}}`
 
 Returns truthy if a given `val` is contained within the array.
 

docs/usage/funcsfile.md

@@ -0,0 +1,48 @@
+# Expression Functions File
+
+A *functions file* allows you to specify additional expression
+helpers that can be loaded and used in *rare*.
+
+## Example
+
+For example, if you frequently need to double a number you
+could create a function:
+
+```funcfile title="custom.funcs"
+double {multi {0} 2}
+```
+
+And use it in rare with argument `--funcs`:
+```sh
+rare --funcs custom.funcs filter -m '(\d+)' -e '{double {1}}' access.log
+```
+
+Or via environment variable `RARE_FUNC_FILES`:
+```sh
+export RARE_FUNC_FILES=/path/to/custom.funcs
+rare filter -m '(\d+)' -e '{double {1}}' access.log
+```
+
+You can load multiple files by providing `--funcs` argument multiple
+times, or providing a comma-separated list to `$RARE_FUNC_FILES`
+
+## Format
+
+A functions file is key-value pairs of `name` to `expression`. Lines
+starting with `#`, or any characters after `#`, are considered comments.
+
+*Expressions* can be multi-line by ending the previous line with a `\`.
+
+```funcsfile
+# Allows comments that start with '#'
+name-of-func {sumi {0} {1}} # comments can also go here
+
+# Multi-line ends with '\'
+classifylen {switch \
+    # short string
+    {lt {len {0}} 5} short \
+    # long string
+    {gt {len {0}} 15} long \
+    medium \ # else, medium
+}
+```

main.go

@@ -7,6 +7,8 @@ import (
 	"rare/cmd"
 	"rare/cmd/helpers"
 	"rare/pkg/color"
+	"rare/pkg/expressions/funcfile"
+	"rare/pkg/expressions/funclib"
 	"rare/pkg/expressions/stdlib"
 	"rare/pkg/fastregex"
 	"rare/pkg/humanize"
@@ -64,6 +66,11 @@ func buildApp() *cli.App {
 			Aliases: []string{"nl"},
 			Usage:   "Disable external file loading in expressions",
 		},
+		&cli.StringSliceFlag{
+			Name:    "funcs",
+			EnvVars: []string{"RARE_FUNC_FILES"},
+			Usage:   "Specify filenames to load expressions from",
+		},
 		&cli.BoolFlag{
 			Name:  "color",
 			Usage: "Force-enable color output",
@@ -111,6 +118,12 @@ func buildApp() *cli.App {
 		if c.Bool("noload") {
 			stdlib.DisableLoad = true
 		}
+		if funcs := c.StringSlice("funcs"); len(funcs) > 0 {
+			cmplr := funclib.NewKeyBuilder()
+			for _, ff := range funcs {
+				funclib.TryAddFunctions(funcfile.LoadDefinitionsFile(cmplr, ff))
+			}
+		}
 		return nil
 	})
 

mkdocs.yml

@@ -17,6 +17,7 @@ nav:
     - Examples: usage/examples.md
     - Advanced:
       - JSON: usage/json.md
+      - Funcs File: usage/funcsfile.md
       - Regular Expressions: usage/regexp.md
     - CLI Docs: cli-help.md
   - Benchmarks: benchmarks.md

pkg/expressions/funcfile/example.funcfile

@@ -0,0 +1,17 @@
+# test func
+double {sumi {0} {0}}
+
+# Blank space above
+multipline {switch \
+    {eq {0} a} isa \
+    {eq {0} b} isb \
+}
+
+multipline2 {switch \
+    {eq {0} a} isa \ #inline
+
+    #Blank above
+    {eq {0} b} isb \
+    # else
+    b \
+}

pkg/expressions/funcfile/loader.go

@@ -0,0 +1,92 @@
+package funcfile
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"rare/pkg/expressions"
+	"rare/pkg/logger"
+	"strings"
+)
+
+func LoadDefinitionsFile(compiler *expressions.KeyBuilder, filename string) (map[string]expressions.KeyBuilderFunction, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return LoadDefinitions(compiler, f)
+}
+
+func LoadDefinitions(compiler *expressions.KeyBuilder, r io.Reader) (map[string]expressions.KeyBuilderFunction, error) {
+	scanner := bufio.NewScanner(r)
+	ret := make(map[string]expressions.KeyBuilderFunction)
+
+	errors := 0
+	linenum := 0
+	for {
+		// read possible multiline
+		var sb strings.Builder
+		for scanner.Scan() {
+			linenum++
+
+			// Get line and split out comments
+			line := strings.TrimSpace(trimAfter(scanner.Text(), '#'))
+			if line == "" {
+				continue
+			}
+
+			if strings.HasSuffix(line, "\\") { // multiline
+				sb.WriteString(strings.TrimSuffix(line, "\\"))
+			} else {
+				sb.WriteString(line)
+				break
+			}
+		}
+		if sb.Len() == 0 {
+			break
+		}
+		phrase := sb.String()
+
+		// Split arguments
+		args := strings.SplitN(phrase, " ", 2)
+		if len(args) != 2 {
+			continue
+		}
+
+		// Compile and save
+		fnc, err := createAndAddFunc(compiler, args[0], args[1])
+		if err != nil {
+			logger.Printf("Error creating function '%s', line %d: %s", args[0], linenum, err)
+			errors++
+		} else {
+			ret[args[0]] = fnc
+		}
+	}
+
+	if errors > 0 {
+		return ret, fmt.Errorf("%d compile errors while loading func spec", errors)
+	}
+	return ret, nil
+}
+
+func trimAfter(s string, r rune) string {
+	idx := strings.IndexRune(s, r)
+	if idx < 0 {
+		return s
+	}
+	return s[:idx]
+}
+
+func createAndAddFunc(compiler *expressions.KeyBuilder, name, expr string) (expressions.KeyBuilderFunction, error) {
+	kb, err := compiler.Compile(expr)
+	if err != nil {
+		return nil, err
+	}
+
+	fnc := keyBuilderToFunction(kb)
+	compiler.Func(name, fnc)
+	return fnc, nil
+}

pkg/expressions/funcfile/loader_test.go

@@ -0,0 +1,55 @@
+package funcfile
+
+import (
+	"rare/pkg/expressions/stdlib"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadDefinitions(t *testing.T) {
+	data := `# a comment
+	double {sumi {0} {0}}
+	
+	# Another comment
+	quad this equals: \
+	  {multi \
+		{0} \
+		{0} \
+		#Mixed comment
+		{0} {0}}
+	`
+	kb := stdlib.NewStdKeyBuilder()
+	funcs, err := LoadDefinitions(kb, strings.NewReader(data))
+	assert.NoError(t, err)
+	assert.Len(t, funcs, 2)
+	assert.Contains(t, funcs, "quad")
+	assert.Contains(t, funcs, "double")
+
+	val, err := kb.Compile("{quad 5}")
+	assert.Nil(t, err)
+	assert.Equal(t, "this equals: 625", val.BuildKey(nil))
+}
+
+func TestLoadDefinitionsErrs(t *testing.T) {
+	data := `# a comment
+	unterm unterm {
+	nofunc
+
+	`
+	kb := stdlib.NewStdKeyBuilder()
+	funcs, err := LoadDefinitions(kb, strings.NewReader(data))
+	assert.NotNil(t, err)
+	assert.Len(t, funcs, 0)
+}
+
+func TestLoadFile(t *testing.T) {
+	kb := stdlib.NewStdKeyBuilder()
+	funcs, err := LoadDefinitionsFile(kb, "example.funcfile")
+	assert.NoError(t, err)
+	assert.Len(t, funcs, 3)
+
+	_, err = LoadDefinitionsFile(kb, "notfile.funcfile")
+	assert.Error(t, err)
+}

pkg/expressions/funcfile/stage.go

@@ -0,0 +1,40 @@
+package funcfile
+
+import (
+	"rare/pkg/expressions"
+	"rare/pkg/slicepool"
+)
+
+type lazySubContext struct {
+	args []expressions.KeyBuilderStage
+	sub  expressions.KeyBuilderContext
+}
+
+func (s *lazySubContext) GetMatch(idx int) string {
+	if idx < 0 || idx >= len(s.args) {
+		return ""
+	}
+	return s.args[idx](s.sub)
+}
+
+func (s *lazySubContext) GetKey(name string) string {
+	return s.sub.GetKey(name)
+}
+
+func keyBuilderToFunction(stage *expressions.CompiledKeyBuilder) expressions.KeyBuilderFunction {
+	return func(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) {
+		ctxPool := slicepool.NewObjectPoolEx(5, func() *lazySubContext {
+			return &lazySubContext{
+				args: args,
+			}
+		})
+
+		return func(kbc expressions.KeyBuilderContext) string {
+			subCtx := ctxPool.Get()
+			defer ctxPool.Return(subCtx)
+			subCtx.sub = kbc
+
+			return stage.BuildKey(subCtx)
+		}, nil
+	}
+}