From 52d672c829280879fd94f8e799ef6a6474993d19 Mon Sep 17 00:00:00 2001 From: Chris LaPointe Date: Tue, 4 Jun 2024 22:04:43 -0400 Subject: [PATCH] Funcfile Support (#104) Implement funcfile, `len` and `@len` --- cmd/expressions.go | 4 +- cmd/reduce.go | 4 +- docs/cli-help.md | Bin 15566 -> 15726 bytes docs/usage/expressions.md | 20 ++++- docs/usage/funcsfile.md | 48 ++++++++++ main.go | 13 +++ mkdocs.yml | 1 + pkg/expressions/funcfile/example.funcfile | 17 ++++ pkg/expressions/funcfile/loader.go | 92 ++++++++++++++++++++ pkg/expressions/funcfile/loader_test.go | 55 ++++++++++++ pkg/expressions/funcfile/stage.go | 40 +++++++++ pkg/expressions/funcfile/stage_test.go | 75 ++++++++++++++++ pkg/expressions/funclib/builder.go | 13 +++ pkg/expressions/funclib/builder_test.go | 11 +++ pkg/expressions/funclib/funcs.go | 38 ++++++++ pkg/expressions/funclib/funcs_test.go | 29 ++++++ pkg/expressions/keyBuilder.go | 7 +- pkg/expressions/stdlib/funcs.go | 2 + pkg/expressions/stdlib/funcsRange.go | 17 ++++ pkg/expressions/stdlib/funcsRange_test.go | 7 ++ pkg/expressions/stdlib/funcsStrings.go | 10 +++ pkg/expressions/stdlib/funcsStrings_test.go | 7 ++ pkg/extractor/extractor.go | 7 +- pkg/extractor/ignoreset.go | 4 +- pkg/markdowncli/replacers.go | 2 +- pkg/slicepool/objpool.go | 7 +- pkg/slicepool/objpool_test.go | 1 + 27 files changed, 518 insertions(+), 13 deletions(-) create mode 100644 docs/usage/funcsfile.md create mode 100644 pkg/expressions/funcfile/example.funcfile create mode 100644 pkg/expressions/funcfile/loader.go create mode 100644 pkg/expressions/funcfile/loader_test.go create mode 100644 pkg/expressions/funcfile/stage.go create mode 100644 pkg/expressions/funcfile/stage_test.go create mode 100644 pkg/expressions/funclib/builder.go create mode 100644 pkg/expressions/funclib/builder_test.go create mode 100644 pkg/expressions/funclib/funcs.go create mode 100644 pkg/expressions/funclib/funcs_test.go diff --git a/cmd/expressions.go b/cmd/expressions.go index 023b0bcc..cbc676be 100644 --- a/cmd/expressions.go +++ b/cmd/expressions.go @@ -8,7 +8,7 @@ import ( "rare/pkg/color" "rare/pkg/expressions" "rare/pkg/expressions/exprofiler" - "rare/pkg/expressions/stdlib" + "rare/pkg/expressions/funclib" "rare/pkg/humanize" "rare/pkg/minijson" "strconv" @@ -45,7 +45,7 @@ func expressionFunction(c *cli.Context) error { expString = string(b) } - builder := stdlib.NewStdKeyBuilderEx(!noOptimize) + builder := funclib.NewKeyBuilderEx(!noOptimize) compiled, err := builder.Compile(expString) if err != nil { diff --git a/cmd/reduce.go b/cmd/reduce.go index 07efac3b..29a54aa1 100644 --- a/cmd/reduce.go +++ b/cmd/reduce.go @@ -7,7 +7,7 @@ import ( "rare/pkg/aggregation/sorting" "rare/pkg/color" "rare/pkg/csv" - "rare/pkg/expressions/stdlib" + "rare/pkg/expressions/funclib" "rare/pkg/logger" "rare/pkg/multiterm/termrenderers" "strings" @@ -31,7 +31,7 @@ func reduceFunction(c *cli.Context) error { batcher := helpers.BuildBatcherFromArguments(c) extractor := helpers.BuildExtractorFromArguments(c, batcher) - aggr := aggregation.NewAccumulatingGroup(stdlib.NewStdKeyBuilder()) + aggr := aggregation.NewAccumulatingGroup(funclib.NewKeyBuilder()) // Set up groups for _, group := range groupExpr { diff --git a/docs/cli-help.md b/docs/cli-help.md index 3119c2f40633c0b95cb5ca08e432ee1a959e0cdd..a836708597bce8df954c4ab0d94bf91476bc87c5 100644 GIT binary patch delta 160 zcmX?C`L1e07H?WtQ3L^ zQj;^&DizW)b5ipXb5n~IO7ayz$`w*83W`#Ti!<}{iWSm|@^dFgFf~|#WONijCTVF| mDY#@7Cnn{j0+p7e7Ud=8fK37G&df_!$jn1FWAg>3f0h9MEIc9r delta 22 ecmaD?b*^$k*2Egaje8F>PTs>*wYh@%n}` + +Returns the length of an array. Empty "" returns 0, a literal will be 1. + +**Note:** This is a linear-time operation. + #### @in -Syntax: `{@in array}` or `{@in {@ val0 val1 val2 ...}}` +Syntax: `{@in }` or `{@in {@ val0 val1 val2 ...}}` Returns truthy if a given `val` is contained within the array. diff --git a/docs/usage/funcsfile.md b/docs/usage/funcsfile.md new file mode 100644 index 00000000..9e7097d9 --- /dev/null +++ b/docs/usage/funcsfile.md @@ -0,0 +1,48 @@ +# Expression Functions File + +A *functions file* allows you to specify additional expression +helpers that can be loaded and used in *rare*. + +## Example + +For example, if you frequently need to double a number you +could create a function: + +```funcfile title="custom.funcs" +double {multi {0} 2} +``` + +And use it in rare with argument `--funcs`: +```sh +rare --funcs custom.funcs filter -m '(\d+)' -e '{double {1}}' access.log +``` + +Or via environment variable `RARE_FUNC_FILES`: +```sh +export RARE_FUNC_FILES=/path/to/custom.funcs +rare filter -m '(\d+)' -e '{double {1}}' access.log +``` + +You can load multiple files by providing `--funcs` argument multiple +times, or providing a comma-separated list to `$RARE_FUNC_FILES` + +## Format + +A functions file is key-value pairs of `name` to `expression`. Lines +starting with `#`, or any characters after `#`, are considered comments. + +*Expressions* can be multi-line by ending the previous line with a `\`. + +```funcsfile +# Allows comments that start with '#' +name-of-func {sumi {0} {1}} # comments can also go here + +# Multi-line ends with '\' +classifylen {switch \ + # short string + {lt {len {0}} 5} short \ + # long string + {gt {len {0}} 15} long \ + medium \ # else, medium +} +``` diff --git a/main.go b/main.go index 67a360cc..f2c71744 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,8 @@ import ( "rare/cmd" "rare/cmd/helpers" "rare/pkg/color" + "rare/pkg/expressions/funcfile" + "rare/pkg/expressions/funclib" "rare/pkg/expressions/stdlib" "rare/pkg/fastregex" "rare/pkg/humanize" @@ -64,6 +66,11 @@ func buildApp() *cli.App { Aliases: []string{"nl"}, Usage: "Disable external file loading in expressions", }, + &cli.StringSliceFlag{ + Name: "funcs", + EnvVars: []string{"RARE_FUNC_FILES"}, + Usage: "Specify filenames to load expressions from", + }, &cli.BoolFlag{ Name: "color", Usage: "Force-enable color output", @@ -111,6 +118,12 @@ func buildApp() *cli.App { if c.Bool("noload") { stdlib.DisableLoad = true } + if funcs := c.StringSlice("funcs"); len(funcs) > 0 { + cmplr := funclib.NewKeyBuilder() + for _, ff := range funcs { + funclib.TryAddFunctions(funcfile.LoadDefinitionsFile(cmplr, ff)) + } + } return nil }) diff --git a/mkdocs.yml b/mkdocs.yml index 1e708911..a77cf8c2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,6 +17,7 @@ nav: - Examples: usage/examples.md - Advanced: - JSON: usage/json.md + - Funcs File: usage/funcsfile.md - Regular Expressions: usage/regexp.md - CLI Docs: cli-help.md - Benchmarks: benchmarks.md diff --git a/pkg/expressions/funcfile/example.funcfile b/pkg/expressions/funcfile/example.funcfile new file mode 100644 index 00000000..8caa987c --- /dev/null +++ b/pkg/expressions/funcfile/example.funcfile @@ -0,0 +1,17 @@ +# test func +double {sumi {0} {0}} + +# Blank space above +multipline {switch \ + {eq {0} a} isa \ + {eq {0} b} isb \ +} + +multipline2 {switch \ + {eq {0} a} isa \ #inline + + #Blank above + {eq {0} b} isb \ + # else + b \ +} diff --git a/pkg/expressions/funcfile/loader.go b/pkg/expressions/funcfile/loader.go new file mode 100644 index 00000000..8df5edd6 --- /dev/null +++ b/pkg/expressions/funcfile/loader.go @@ -0,0 +1,92 @@ +package funcfile + +import ( + "bufio" + "fmt" + "io" + "os" + "rare/pkg/expressions" + "rare/pkg/logger" + "strings" +) + +func LoadDefinitionsFile(compiler *expressions.KeyBuilder, filename string) (map[string]expressions.KeyBuilderFunction, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + return LoadDefinitions(compiler, f) +} + +func LoadDefinitions(compiler *expressions.KeyBuilder, r io.Reader) (map[string]expressions.KeyBuilderFunction, error) { + scanner := bufio.NewScanner(r) + ret := make(map[string]expressions.KeyBuilderFunction) + + errors := 0 + linenum := 0 + for { + // read possible multiline + var sb strings.Builder + for scanner.Scan() { + linenum++ + + // Get line and split out comments + line := strings.TrimSpace(trimAfter(scanner.Text(), '#')) + if line == "" { + continue + } + + if strings.HasSuffix(line, "\\") { // multiline + sb.WriteString(strings.TrimSuffix(line, "\\")) + } else { + sb.WriteString(line) + break + } + } + if sb.Len() == 0 { + break + } + phrase := sb.String() + + // Split arguments + args := strings.SplitN(phrase, " ", 2) + if len(args) != 2 { + continue + } + + // Compile and save + fnc, err := createAndAddFunc(compiler, args[0], args[1]) + if err != nil { + logger.Printf("Error creating function '%s', line %d: %s", args[0], linenum, err) + errors++ + } else { + ret[args[0]] = fnc + } + } + + if errors > 0 { + return ret, fmt.Errorf("%d compile errors while loading func spec", errors) + } + return ret, nil +} + +func trimAfter(s string, r rune) string { + idx := strings.IndexRune(s, r) + if idx < 0 { + return s + } + return s[:idx] +} + +func createAndAddFunc(compiler *expressions.KeyBuilder, name, expr string) (expressions.KeyBuilderFunction, error) { + kb, err := compiler.Compile(expr) + if err != nil { + return nil, err + } + + fnc := keyBuilderToFunction(kb) + compiler.Func(name, fnc) + return fnc, nil +} diff --git a/pkg/expressions/funcfile/loader_test.go b/pkg/expressions/funcfile/loader_test.go new file mode 100644 index 00000000..0205be76 --- /dev/null +++ b/pkg/expressions/funcfile/loader_test.go @@ -0,0 +1,55 @@ +package funcfile + +import ( + "rare/pkg/expressions/stdlib" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadDefinitions(t *testing.T) { + data := `# a comment + double {sumi {0} {0}} + + # Another comment + quad this equals: \ + {multi \ + {0} \ + {0} \ + #Mixed comment + {0} {0}} + ` + kb := stdlib.NewStdKeyBuilder() + funcs, err := LoadDefinitions(kb, strings.NewReader(data)) + assert.NoError(t, err) + assert.Len(t, funcs, 2) + assert.Contains(t, funcs, "quad") + assert.Contains(t, funcs, "double") + + val, err := kb.Compile("{quad 5}") + assert.Nil(t, err) + assert.Equal(t, "this equals: 625", val.BuildKey(nil)) +} + +func TestLoadDefinitionsErrs(t *testing.T) { + data := `# a comment + unterm unterm { + nofunc + + ` + kb := stdlib.NewStdKeyBuilder() + funcs, err := LoadDefinitions(kb, strings.NewReader(data)) + assert.NotNil(t, err) + assert.Len(t, funcs, 0) +} + +func TestLoadFile(t *testing.T) { + kb := stdlib.NewStdKeyBuilder() + funcs, err := LoadDefinitionsFile(kb, "example.funcfile") + assert.NoError(t, err) + assert.Len(t, funcs, 3) + + _, err = LoadDefinitionsFile(kb, "notfile.funcfile") + assert.Error(t, err) +} diff --git a/pkg/expressions/funcfile/stage.go b/pkg/expressions/funcfile/stage.go new file mode 100644 index 00000000..8acfb40a --- /dev/null +++ b/pkg/expressions/funcfile/stage.go @@ -0,0 +1,40 @@ +package funcfile + +import ( + "rare/pkg/expressions" + "rare/pkg/slicepool" +) + +type lazySubContext struct { + args []expressions.KeyBuilderStage + sub expressions.KeyBuilderContext +} + +func (s *lazySubContext) GetMatch(idx int) string { + if idx < 0 || idx >= len(s.args) { + return "" + } + return s.args[idx](s.sub) +} + +func (s *lazySubContext) GetKey(name string) string { + return s.sub.GetKey(name) +} + +func keyBuilderToFunction(stage *expressions.CompiledKeyBuilder) expressions.KeyBuilderFunction { + return func(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + ctxPool := slicepool.NewObjectPoolEx(5, func() *lazySubContext { + return &lazySubContext{ + args: args, + } + }) + + return func(kbc expressions.KeyBuilderContext) string { + subCtx := ctxPool.Get() + defer ctxPool.Return(subCtx) + subCtx.sub = kbc + + return stage.BuildKey(subCtx) + }, nil + } +} diff --git a/pkg/expressions/funcfile/stage_test.go b/pkg/expressions/funcfile/stage_test.go new file mode 100644 index 00000000..ebd92b0e --- /dev/null +++ b/pkg/expressions/funcfile/stage_test.go @@ -0,0 +1,75 @@ +package funcfile + +import ( + "rare/pkg/expressions" + "rare/pkg/expressions/stdlib" + "testing" + + "github.com/stretchr/testify/assert" +) + +var testContext = expressions.KeyBuilderContextArray{ + Elements: []string{"ab", "cd", "123"}, + Keys: map[string]string{ + "test": "testval", + }, +} + +func TestCustomFunc(t *testing.T) { + k := stdlib.NewStdKeyBuilder() + _, err := createAndAddFunc(k, "double", "{sumi {0} {0} 4}") + assert.NoError(t, err) + + kb, err := k.Compile("kd: {double {2}}") + assert.Nil(t, err) + val := kb.BuildKey(&testContext) + assert.Equal(t, val, "kd: 250") +} + +func TestCustomEdgeCases(t *testing.T) { + k := stdlib.NewStdKeyBuilder() + _, err := createAndAddFunc(k, "err", "{unclosed func") + assert.Error(t, err) + + _, err = createAndAddFunc(k, "doublesrc", "{test} {sumi {0} {0}} missing: {1}") + assert.NoError(t, err) + kb, _ := k.Compile("{doublesrc 5}") + assert.Equal(t, "testval 10 missing: ", kb.BuildKey(&testContext)) +} + +// BenchmarkCustomFunc-4 7563214 160.4 ns/op 3 B/op 1 allocs/op +func BenchmarkCustomFunc(b *testing.B) { + k := stdlib.NewStdKeyBuilder() + createAndAddFunc(k, "double", "{sumi {0} {0} 4}") + + kb, err := k.Compile("{double {2}}") + if err != nil { + b.Error(err) + } + + for i := 0; i < b.N; i++ { + kb.BuildKey(&testContext) + } +} + +func TestDeferredResolve(t *testing.T) { + k := stdlib.NewStdKeyBuilderEx(false) // disable optimization because static analysis will trigger panic + k.Func("panic", func(kbs []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + return func(kbc expressions.KeyBuilderContext) string { + panic("not supposed to get here") + }, nil + }) + + _, ferr := createAndAddFunc(k, "panicmissing", "{coalesce {0} {panic {0}}}") + assert.NoError(t, ferr) + + kb, err := k.Compile("{panicmissing {1}}") + assert.Nil(t, err) + assert.Equal(t, "cd", kb.BuildKey(&testContext)) + + assert.PanicsWithValue(t, "not supposed to get here", func() { + kb.BuildKey(&expressions.KeyBuilderContextArray{ + Elements: []string{}, + }) + }) +} diff --git a/pkg/expressions/funclib/builder.go b/pkg/expressions/funclib/builder.go new file mode 100644 index 00000000..7c7de310 --- /dev/null +++ b/pkg/expressions/funclib/builder.go @@ -0,0 +1,13 @@ +package funclib + +import "rare/pkg/expressions" + +func NewKeyBuilderEx(autoOptimize bool) *expressions.KeyBuilder { + kb := expressions.NewKeyBuilderEx(autoOptimize) + kb.Funcs(Functions) + return kb +} + +func NewKeyBuilder() *expressions.KeyBuilder { + return NewKeyBuilderEx(true) +} diff --git a/pkg/expressions/funclib/builder_test.go b/pkg/expressions/funclib/builder_test.go new file mode 100644 index 00000000..0f0cc8d8 --- /dev/null +++ b/pkg/expressions/funclib/builder_test.go @@ -0,0 +1,11 @@ +package funclib + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewKeyBuilder(t *testing.T) { + assert.NotNil(t, NewKeyBuilder()) +} diff --git a/pkg/expressions/funclib/funcs.go b/pkg/expressions/funclib/funcs.go new file mode 100644 index 00000000..5b8f1917 --- /dev/null +++ b/pkg/expressions/funclib/funcs.go @@ -0,0 +1,38 @@ +package funclib + +import ( + "rare/pkg/expressions" + "rare/pkg/expressions/stdlib" + "rare/pkg/logger" +) + +type FunctionSet map[string]expressions.KeyBuilderFunction + +var Functions FunctionSet = mapMerge( + stdlib.StandardFunctions) + +func AddFunctions(funcs FunctionSet) { + for name, fnc := range funcs { + Functions[name] = fnc + } +} + +func TryAddFunctions(funcs FunctionSet, err error) error { + if err != nil { + logger.Printf("Error adding functions: %s", err) + } + if funcs != nil { + AddFunctions(funcs) + } + return nil +} + +func mapMerge[T comparable, Q any](maps ...map[T]Q) (ret map[T]Q) { + ret = make(map[T]Q) + for _, m := range maps { + for k, v := range m { + ret[k] = v + } + } + return ret +} diff --git a/pkg/expressions/funclib/funcs_test.go b/pkg/expressions/funclib/funcs_test.go new file mode 100644 index 00000000..c1df5c21 --- /dev/null +++ b/pkg/expressions/funclib/funcs_test.go @@ -0,0 +1,29 @@ +package funclib + +import ( + "errors" + "rare/pkg/expressions" + "testing" + + "github.com/stretchr/testify/assert" +) + +func voidFunc(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + return nil, nil +} + +func TestFunctionSet(t *testing.T) { + assert.NotZero(t, Functions) +} + +func TestAddFunction(t *testing.T) { + AddFunctions(FunctionSet{ + "_test": voidFunc, + }) + TryAddFunctions(FunctionSet{ + "_test": voidFunc, + }, nil) + TryAddFunctions(FunctionSet{ + "_test": voidFunc, + }, errors.New("nope")) +} diff --git a/pkg/expressions/keyBuilder.go b/pkg/expressions/keyBuilder.go index aa8b1bc4..a2db8bb2 100644 --- a/pkg/expressions/keyBuilder.go +++ b/pkg/expressions/keyBuilder.go @@ -33,10 +33,15 @@ func NewKeyBuilder() *KeyBuilder { // Funcs appends a map of functions to be used by the parser func (s *KeyBuilder) Funcs(funcs map[string]KeyBuilderFunction) { for k, f := range funcs { - s.functions[k] = f + s.Func(k, f) } } +// Funcs adds a single function used by the parser +func (s *KeyBuilder) Func(name string, f KeyBuilderFunction) { + s.functions[name] = f +} + // Compile builds a new key-builder, returning error(s) on build issues // if the CompiledKeyBuilder is not nil, then something is still useable (albeit may have problems) func (s *KeyBuilder) Compile(template string) (*CompiledKeyBuilder, *CompilerErrors) { diff --git a/pkg/expressions/stdlib/funcs.go b/pkg/expressions/stdlib/funcs.go index 0ec4e80b..b19704c6 100644 --- a/pkg/expressions/stdlib/funcs.go +++ b/pkg/expressions/stdlib/funcs.go @@ -66,6 +66,7 @@ var StandardFunctions = map[string]KeyBuilderFunction{ "or": KeyBuilderFunction(kfOr), // Strings + "len": KeyBuilderFunction(kfLen), "like": KeyBuilderFunction(kfLike), "prefix": KeyBuilderFunction(kfPrefix), "suffix": KeyBuilderFunction(kfSuffix), @@ -81,6 +82,7 @@ var StandardFunctions = map[string]KeyBuilderFunction{ // Ranges "@": kfJoin(ArraySeparator), + "@len": kfArrayLen, "@map": kfArrayMap, "@split": kfArraySplit, "@select": kfArraySelect, diff --git a/pkg/expressions/stdlib/funcsRange.go b/pkg/expressions/stdlib/funcsRange.go index d5ac0b7a..4973ba38 100644 --- a/pkg/expressions/stdlib/funcsRange.go +++ b/pkg/expressions/stdlib/funcsRange.go @@ -4,6 +4,7 @@ import ( . "rare/pkg/expressions" //lint:ignore ST1001 Legacy "rare/pkg/slicepool" "rare/pkg/stringSplitter" + "strconv" "strings" ) @@ -34,6 +35,22 @@ func (s *subContext) Eval(stage KeyBuilderStage, v0, v1 string) string { return stage(s) } +// {@len } +func kfArrayLen(args []KeyBuilderStage) (KeyBuilderStage, error) { + if len(args) != 1 { + return stageErrArgCount(args, 1) + } + return func(context KeyBuilderContext) string { + val := args[0](context) + if val == "" { + return "0" + } + + count := strings.Count(val, ArraySeparatorString) + 1 + return strconv.Itoa(count) + }, nil +} + // {@split "delim"} func kfArraySplit(args []KeyBuilderStage) (KeyBuilderStage, error) { if !isArgCountBetween(args, 1, 2) { diff --git a/pkg/expressions/stdlib/funcsRange_test.go b/pkg/expressions/stdlib/funcsRange_test.go index 3ceee36e..fe955d0a 100644 --- a/pkg/expressions/stdlib/funcsRange_test.go +++ b/pkg/expressions/stdlib/funcsRange_test.go @@ -11,6 +11,13 @@ func TestArray(t *testing.T) { testExpression(t, mockContext("q"), "{$ {0}}", "q") } +func TestArrayLen(t *testing.T) { + testExpression(t, mockContext("abc"), "{@len {0}}", "1") + testExpression(t, mockContext(expressions.MakeArray("a", "bc", "c")), "{@len {0}}", "3") + testExpression(t, mockContext(""), "{@len {0}}", "0") + testExpressionErr(t, mockContext(), "{@len a b}", "", ErrArgCount) +} + func TestArraySplit(t *testing.T) { testExpression( t, diff --git a/pkg/expressions/stdlib/funcsStrings.go b/pkg/expressions/stdlib/funcsStrings.go index ed6d36a6..aff5310b 100644 --- a/pkg/expressions/stdlib/funcsStrings.go +++ b/pkg/expressions/stdlib/funcsStrings.go @@ -9,6 +9,16 @@ import ( . "rare/pkg/expressions" //lint:ignore ST1001 Legacy ) +// {len string} +func kfLen(args []KeyBuilderStage) (KeyBuilderStage, error) { + if len(args) != 1 { + return stageErrArgCount(args, 1) + } + return func(context KeyBuilderContext) string { + return strconv.Itoa(len(args[0](context))) + }, nil +} + // {prefix string prefix} func kfPrefix(args []KeyBuilderStage) (KeyBuilderStage, error) { if len(args) != 2 { diff --git a/pkg/expressions/stdlib/funcsStrings_test.go b/pkg/expressions/stdlib/funcsStrings_test.go index c429e817..ea76b745 100644 --- a/pkg/expressions/stdlib/funcsStrings_test.go +++ b/pkg/expressions/stdlib/funcsStrings_test.go @@ -8,6 +8,13 @@ import ( "github.com/stretchr/testify/assert" ) +func TestLen(t *testing.T) { + testExpression(t, mockContext("hello"), "{len {0}}", "5") + testExpression(t, mockContext("hello"), "{len \"\"}", "0") + testExpression(t, mockContext("hello"), "{len hi}", "2") + testExpressionErr(t, mockContext("hello"), "{len {0} there}", "", ErrArgCount) +} + func TestUpperLower(t *testing.T) { testExpressionErr(t, mockContext("aBc"), "{upper {0}} {upper a b}", "ABC ", ErrArgCount) testExpressionErr(t, mockContext("aBc"), "{lower {0}} {lower a b}", "abc ", ErrArgCount) diff --git a/pkg/extractor/extractor.go b/pkg/extractor/extractor.go index a0f4dc84..5c7db8b1 100644 --- a/pkg/extractor/extractor.go +++ b/pkg/extractor/extractor.go @@ -2,7 +2,7 @@ package extractor import ( "rare/pkg/expressions" - "rare/pkg/expressions/stdlib" + "rare/pkg/expressions/funclib" "rare/pkg/fastregex" "sync" "sync/atomic" @@ -39,7 +39,8 @@ type Config struct { } // Extractor is the representation of the reader -// Expects someone to consume its ReadChan() +// +// Expects someone to consume its ReadChan() type Extractor struct { readChan chan []Match compiledRegexp fastregex.CompiledRegexp @@ -151,7 +152,7 @@ func (s *Extractor) asyncWorker(wg *sync.WaitGroup, inputBatch <-chan InputBatch // New an extractor from an input channel func New(inputBatch <-chan InputBatch, config *Config) (*Extractor, error) { - compiledExpression, compErr := stdlib.NewStdKeyBuilder().Compile(config.Extract) + compiledExpression, compErr := funclib.NewKeyBuilder().Compile(config.Extract) if compErr != nil { return nil, compErr } diff --git a/pkg/extractor/ignoreset.go b/pkg/extractor/ignoreset.go index 4dfe7812..61353e5f 100644 --- a/pkg/extractor/ignoreset.go +++ b/pkg/extractor/ignoreset.go @@ -2,7 +2,7 @@ package extractor import ( "rare/pkg/expressions" - "rare/pkg/expressions/stdlib" + "rare/pkg/expressions/funclib" ) type IgnoreSet interface { @@ -22,7 +22,7 @@ func NewIgnoreExpressions(expSet ...string) (IgnoreSet, error) { } for _, exp := range expSet { - compiled, err := stdlib.NewStdKeyBuilder().Compile(exp) + compiled, err := funclib.NewKeyBuilder().Compile(exp) if err != nil { return nil, err } diff --git a/pkg/markdowncli/replacers.go b/pkg/markdowncli/replacers.go index ff7d322a..ceb0ef88 100644 --- a/pkg/markdowncli/replacers.go +++ b/pkg/markdowncli/replacers.go @@ -18,7 +18,7 @@ func replaceWithColor(clr color.ColorCode) regexReplacerFunc { }) } -var localLinkRegexp = regexp.MustCompile(`\[\w+\]\((\w+).md\)`) +var localLinkRegexp = regexp.MustCompile(`\[[\w\s]+\]\((\w+).md\)`) func replaceLocalLinkWithDocsCommand(match string) string { parts := localLinkRegexp.FindStringSubmatch(match) diff --git a/pkg/slicepool/objpool.go b/pkg/slicepool/objpool.go index 19f52085..d0566ecb 100644 --- a/pkg/slicepool/objpool.go +++ b/pkg/slicepool/objpool.go @@ -12,11 +12,16 @@ type ObjectPool[T any] struct { // Create an object pool of an initial size. May grow later func NewObjectPool[T any](size int) *ObjectPool[T] { + return NewObjectPoolEx(size, func() *T { return new(T) }) +} + +// Create an object pool with a custom object initializer +func NewObjectPoolEx[T any](size int, newer func() *T) *ObjectPool[T] { ret := &ObjectPool[T]{ pool: make([]*T, size), } for i := 0; i < size; i++ { - ret.pool[i] = new(T) + ret.pool[i] = newer() } return ret } diff --git a/pkg/slicepool/objpool_test.go b/pkg/slicepool/objpool_test.go index 0bbf375b..b89f0529 100644 --- a/pkg/slicepool/objpool_test.go +++ b/pkg/slicepool/objpool_test.go @@ -30,6 +30,7 @@ func TestZeroAllocs(t *testing.T) { assert.Zero(t, res.AllocsPerOp()) } +// BenchmarkObjPool-4 24965836 44.38 ns/op 0 B/op 0 allocs/op func BenchmarkObjPool(b *testing.B) { type testObj struct{}