From 0192d220345d0d1c1c4088e18b03a2b2c0a93567 Mon Sep 17 00:00:00 2001 From: Chris LaPointe Date: Sun, 19 May 2024 09:49:58 -0400 Subject: [PATCH] File load, key lookup, and switch statement (#103) * File load, key lookup, and switch statement * Flag to disable expressions external loading --- cmd/testdata/lookup.txt | 6 + docs/usage/expressions.md | 43 +++++++ main.go | 9 ++ pkg/expressions/stdlib/errors.go | 8 ++ pkg/expressions/stdlib/funcs.go | 6 + pkg/expressions/stdlib/funcsComparators.go | 19 +++ .../stdlib/funcsComparators_test.go | 9 ++ pkg/expressions/stdlib/funcsLookups.go | 111 ++++++++++++++++++ pkg/expressions/stdlib/funcsLookups_test.go | 42 +++++++ pkg/expressions/stdlib/testutil_test.go | 18 +++ pkg/expressions/truthy.go | 7 ++ pkg/expressions/truthy_test.go | 18 +++ 12 files changed, 296 insertions(+) create mode 100644 cmd/testdata/lookup.txt create mode 100644 pkg/expressions/stdlib/funcsLookups.go create mode 100644 pkg/expressions/stdlib/funcsLookups_test.go create mode 100644 pkg/expressions/truthy_test.go diff --git a/cmd/testdata/lookup.txt b/cmd/testdata/lookup.txt new file mode 100644 index 00000000..96d07711 --- /dev/null +++ b/cmd/testdata/lookup.txt @@ -0,0 +1,6 @@ +#test 22 +bob 33 +jill 44 + +cat 55 +key \ No newline at end of file diff --git a/docs/usage/expressions.md b/docs/usage/expressions.md index 07227488..6c99258e 100644 --- a/docs/usage/expressions.md +++ b/docs/usage/expressions.md @@ -158,6 +158,14 @@ Syntax: `{if val ifTrue ifFalse}`, `{if val ifTrue}`, `{unless val ifFalse}` If `val` is truthy, then return `ifTrue` else optionally return `ifFalse` +#### Switch + +Syntax: `{switch ifTrue val ifTrue val ... [ifFalseVal]}` + +In pairs, if a given value is truthy, return the value immediately after. If +there is an odd number of arguments, the last value is used as the "else" result. +Otherwise, empty string is returned. + #### Equals, NotEquals, Not Syntax: `{eq a b}`, `{neq a b}`, `{not a}` @@ -259,6 +267,41 @@ to form arrays that have meaning for a given aggregator. Specifying multiple expressions is equivalent, eg. `{$ a b}` is the same as `-e a -e b` +### File Loading and Lookup Tables + +Load external static content as either raw string, or to be used to lookup +a value given a key. + +#### Load + +Syntax: `{load "filename"}` + +Loads a given filename as text. + +To globally disable file loading in expressions for security reasons, specify +`--noload` as global argument. + +#### Lookup, HasKey + +Syntax: `{lookup key "kv-pairs" ["commentPrefix"]}`, `{haskey key "kv-pairs" ["commentPrefix"]}` + +Given a set of kv-pairs (eg. from a loaded file), lookup a key. For `lookup` return a value +and for `haskey` return truthy or falsey. + +If a `commentPrefix` is provided, lines in lookup text are ignored if they start with the prefix. + +Example kv-pairs text. Keys and values are separated by any whitespace. + +``` +key1 val1 +key2 val2 +#comment if '#' set as prefix +key3 val3 + +#blank lines are ignored +too many values are also ignored +``` + ### Ranges (Arrays) Range functions provide the ability to work with arrays in expressions. You diff --git a/main.go b/main.go index ebbc8792..67a360cc 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "rare/cmd" "rare/cmd/helpers" "rare/pkg/color" + "rare/pkg/expressions/stdlib" "rare/pkg/fastregex" "rare/pkg/humanize" "rare/pkg/logger" @@ -58,6 +59,11 @@ func buildApp() *cli.App { Aliases: []string{"nu"}, Usage: "Disable usage of unicode characters", }, + &cli.BoolFlag{ + Name: "noload", + Aliases: []string{"nl"}, + Usage: "Disable external file loading in expressions", + }, &cli.BoolFlag{ Name: "color", Usage: "Force-enable color output", @@ -102,6 +108,9 @@ func buildApp() *cli.App { if c.Bool("nounicode") { termunicode.UnicodeEnabled = false } + if c.Bool("noload") { + stdlib.DisableLoad = true + } return nil }) diff --git a/pkg/expressions/stdlib/errors.go b/pkg/expressions/stdlib/errors.go index fa2d1696..28b92208 100644 --- a/pkg/expressions/stdlib/errors.go +++ b/pkg/expressions/stdlib/errors.go @@ -24,6 +24,7 @@ const ( ErrorEnum = "" // A given value is not contained within a set ErrorArgName = "" // A variable accessed by a given name does not exist ErrorEmpty = "" // A value was expected, but was empty + ErrorFile = "" ) // Compilation errors @@ -33,6 +34,7 @@ var ( ErrConst = newFuncErr(ErrorConst, "expected const") ErrEnum = newFuncErr(ErrorEnum, "unable to find value in set") ErrEmpty = newFuncErr(ErrorEmpty, "invalid empty value") + ErrFile = newFuncErr(ErrorFile, "unable to read file") ) var ( @@ -45,6 +47,12 @@ func stageError(err funcError) (KeyBuilderStage, error) { }, err.err } +func stageErrorf(err funcError, msg string) (KeyBuilderStage, error) { + return func(ctx KeyBuilderContext) string { + return err.expr + }, fmt.Errorf("%s, %w", msg, err.err) +} + func stageArgError(err funcError, argIndex int) (KeyBuilderStage, error) { return func(ctx KeyBuilderContext) string { return err.expr diff --git a/pkg/expressions/stdlib/funcs.go b/pkg/expressions/stdlib/funcs.go index 6c02b194..0ec4e80b 100644 --- a/pkg/expressions/stdlib/funcs.go +++ b/pkg/expressions/stdlib/funcs.go @@ -43,6 +43,7 @@ var StandardFunctions = map[string]KeyBuilderFunction{ // Comparisons "if": KeyBuilderFunction(kfIf), + "switch": kfSwitch, "unless": KeyBuilderFunction(kfUnless), "eq": stringComparator(func(a, b string) string { if a == b { @@ -94,6 +95,11 @@ var StandardFunctions = map[string]KeyBuilderFunction{ "dirname": kfPathDir, "extname": kfPathExt, + // File operations + "load": kfLoadFile, + "lookup": kfLookupKey, + "haskey": kfHasKey, + // Formatting "hi": KeyBuilderFunction(kfHumanizeInt), "hf": KeyBuilderFunction(kfHumanizeFloat), diff --git a/pkg/expressions/stdlib/funcsComparators.go b/pkg/expressions/stdlib/funcsComparators.go index 51ac2cd8..5e93f1d7 100644 --- a/pkg/expressions/stdlib/funcsComparators.go +++ b/pkg/expressions/stdlib/funcsComparators.go @@ -115,6 +115,25 @@ func kfIf(args []KeyBuilderStage) (KeyBuilderStage, error) { }), nil } +// {switch ifTrue val ifTrue val ... [ifFalseVal]} +func kfSwitch(args []KeyBuilderStage) (KeyBuilderStage, error) { + if len(args) <= 1 { + return stageErrArgRange(args, "2+") + } + + return func(context KeyBuilderContext) string { + for i := 0; i+1 < len(args); i += 2 { + if Truthy(args[i](context)) { + return args[i+1](context) + } + } + if len(args)%2 == 1 { + return args[len(args)-1](context) + } + return "" + }, nil +} + func kfUnless(args []KeyBuilderStage) (KeyBuilderStage, error) { if len(args) != 2 { return stageErrArgCount(args, 2) diff --git a/pkg/expressions/stdlib/funcsComparators_test.go b/pkg/expressions/stdlib/funcsComparators_test.go index ff9f1adf..7bf54500 100644 --- a/pkg/expressions/stdlib/funcsComparators_test.go +++ b/pkg/expressions/stdlib/funcsComparators_test.go @@ -17,6 +17,15 @@ func TestIfStatement(t *testing.T) { testExpression(t, mockContext(), `{if {neq "" ""} true false}`, "false") } +func TestSwitch(t *testing.T) { + testExpression(t, mockContext("a"), "{switch {eq {0} a} isa {eq {0} b} isb 1 null}", "isa") + testExpression(t, mockContext("b"), "{switch {eq {0} a} isa {eq {0} b} isb 1 null}", "isb") + testExpression(t, mockContext("c"), "{switch {eq {0} a} isa {eq {0} b} isb 1 null}", "null") + testExpression(t, mockContext("c"), "{switch {eq {0} a} isa {eq {0} b} isb null}", "null") + testExpression(t, mockContext("a"), "{switch {eq {0} a} isa {eq {0} b} isb 1}", "isa") + testExpressionErr(t, mockContext("a"), "{switch {eq {0} a}}", "", ErrArgCount) +} + func TestUnlessStatement(t *testing.T) { testExpression(t, mockContext("abc"), `{unless {1} {0}} {unless abc efg} {unless "" bob}`, "abc bob") testExpressionErr(t, mockContext("abc"), `{unless joe}`, "", ErrArgCount) diff --git a/pkg/expressions/stdlib/funcsLookups.go b/pkg/expressions/stdlib/funcsLookups.go new file mode 100644 index 00000000..5d646b72 --- /dev/null +++ b/pkg/expressions/stdlib/funcsLookups.go @@ -0,0 +1,111 @@ +package stdlib + +import ( + "bufio" + "io" + "os" + "rare/pkg/expressions" + "strings" +) + +var DisableLoad = false + +// {load "filename"} +// loads static file as string +func kfLoadFile(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + if DisableLoad { + return stageErrorf(ErrFile, "loading disabled") + } + + if len(args) != 1 { + return stageErrArgCount(args, 1) + } + + filename, ok := expressions.EvalStaticStage(args[0]) + if !ok { + return stageError(ErrConst) + } + + f, err := os.Open(filename) + if err != nil { + return stageErrorf(ErrFile, "Unable to open file: "+filename) + } + defer f.Close() + + content, err := io.ReadAll(f) + if err != nil { + return stageErrorf(ErrFile, "Error reading file: "+filename) + } + + sContent := string(content) + + return func(context expressions.KeyBuilderContext) string { + return sContent + }, nil +} + +func buildLookupTable(content string, commentPrefix string) map[string]string { + lookup := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(content)) + + for scanner.Scan() { + line := scanner.Text() + + if commentPrefix != "" && strings.HasPrefix(line, commentPrefix) { + continue + } + + parts := strings.Fields(line) + switch len(parts) { + case 0: //noop + case 1: + lookup[parts[0]] = "" + case 2: + lookup[parts[0]] = parts[1] + } + } + return lookup +} + +// {lookup key "table" [commentPrefix]} +func kfLookupKey(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + if !isArgCountBetween(args, 2, 3) { + return stageErrArgRange(args, "2-3") + } + + content, ok := expressions.EvalStaticStage(args[1]) + if !ok { + return stageArgError(ErrConst, 1) + } + + commentPrefix := expressions.EvalStageIndexOrDefault(args, 2, "") + + lookup := buildLookupTable(content, commentPrefix) + + return func(context expressions.KeyBuilderContext) string { + key := args[0](context) + return lookup[key] + }, nil +} + +// {haskey key "table" [commentprefix]} +func kfHasKey(args []expressions.KeyBuilderStage) (expressions.KeyBuilderStage, error) { + if !isArgCountBetween(args, 2, 3) { + return stageErrArgRange(args, "2-3") + } + + content, ok := expressions.EvalStaticStage(args[1]) + if !ok { + return stageArgError(ErrConst, 1) + } + + commentPrefix := expressions.EvalStageIndexOrDefault(args, 2, "") + + lookup := buildLookupTable(content, commentPrefix) + + return func(context expressions.KeyBuilderContext) string { + key := args[0](context) + _, has := lookup[key] + return expressions.TruthyStr(has) + }, nil +} diff --git a/pkg/expressions/stdlib/funcsLookups_test.go b/pkg/expressions/stdlib/funcsLookups_test.go new file mode 100644 index 00000000..eb82082d --- /dev/null +++ b/pkg/expressions/stdlib/funcsLookups_test.go @@ -0,0 +1,42 @@ +package stdlib + +import ( + "rare/pkg/testutil" + "testing" +) + +func TestLoadFile(t *testing.T) { + testExpression(t, mockContext(), "{load ../../../cmd/testdata/graph.txt}", "bob 22\njack 93\njill 3\nmaria 19") + testExpressionErr(t, mockContext(), "{load {0}}", "", ErrConst) + testExpressionErr(t, mockContext(), "{load a b}", "", ErrArgCount) + testExpressionErr(t, mockContext(), "{load notarealfile.txt}", "", ErrFile) + + testutil.SwitchGlobal(&DisableLoad, true) + defer testutil.RestoreGlobals() + testExpressionErr(t, mockContext(), "{load ../../../cmd/testdata/graph.txt}", "", ErrFile) +} + +func TestLookup(t *testing.T) { + testExpression(t, mockContext("bob"), "{lookup {0} {load ../../../cmd/testdata/lookup.txt}}", "33") + testExpression(t, mockContext("bobert"), "{lookup {0} {load ../../../cmd/testdata/lookup.txt}}", "") + testExpression(t, mockContext("#test"), "{lookup {0} {load ../../../cmd/testdata/lookup.txt}}", "22") + testExpression(t, mockContext("#test"), "{lookup {0} {load ../../../cmd/testdata/lookup.txt} \"#\"}", "") + testExpressionErr(t, mockContext(), "{lookup a}", "", ErrArgCount) + testExpressionErr(t, mockContext(), "{lookup a b c d}", "", ErrArgCount) + testExpressionErr(t, mockContext("fn"), "{lookup fn {0}}", "", ErrConst) +} + +func TestHasKey(t *testing.T) { + testExpression(t, mockContext("bob"), "{haskey {0} {load ../../../cmd/testdata/lookup.txt}}", "1") + testExpression(t, mockContext("nop"), "{haskey {0} {load ../../../cmd/testdata/lookup.txt}}", "") + testExpression(t, mockContext("key"), "{haskey {0} {load ../../../cmd/testdata/lookup.txt}}", "1") + testExpression(t, mockContext("#test"), "{haskey {0} {load ../../../cmd/testdata/lookup.txt}}", "1") + testExpression(t, mockContext("#test"), "{haskey {0} {load ../../../cmd/testdata/lookup.txt} #}", "") + testExpressionErr(t, mockContext(), "{haskey a}", "", ErrArgCount) + testExpressionErr(t, mockContext(), "{haskey a b c d}", "", ErrArgCount) + testExpressionErr(t, mockContext("fn"), "{haskey fn {0}}", "", ErrConst) +} + +func BenchmarkLoadFile(b *testing.B) { + benchmarkExpression(b, mockContext(), "{load ../../../cmd/testdata/graph.txt}", "bob 22\njack 93\njill 3\nmaria 19") +} diff --git a/pkg/expressions/stdlib/testutil_test.go b/pkg/expressions/stdlib/testutil_test.go index 6fa9d51c..abe669af 100644 --- a/pkg/expressions/stdlib/testutil_test.go +++ b/pkg/expressions/stdlib/testutil_test.go @@ -54,3 +54,21 @@ func testExpressionErr(t *testing.T, context KeyBuilderContext, expression strin } } } + +// benchmark an expreession, as a sub-benchmark. Checks value before running test +func benchmarkExpression(b *testing.B, context KeyBuilderContext, expression, expected string) { + kb, err := NewStdKeyBuilderEx(false).Compile(expression) + if err != nil { + b.Fatal(err) + } + + if s := kb.BuildKey(context); s != expected { + b.Fatalf("%s != %s", s, expected) + } + + b.Run(expression, func(b *testing.B) { + for i := 0; i < b.N; i++ { + kb.BuildKey(context) + } + }) +} diff --git a/pkg/expressions/truthy.go b/pkg/expressions/truthy.go index 5028d737..566d893c 100644 --- a/pkg/expressions/truthy.go +++ b/pkg/expressions/truthy.go @@ -8,3 +8,10 @@ const FalsyVal = "" func Truthy(s string) bool { return strings.TrimSpace(s) != FalsyVal } + +func TruthyStr(is bool) string { + if is { + return TruthyVal + } + return FalsyVal +} diff --git a/pkg/expressions/truthy_test.go b/pkg/expressions/truthy_test.go new file mode 100644 index 00000000..d94045ed --- /dev/null +++ b/pkg/expressions/truthy_test.go @@ -0,0 +1,18 @@ +package expressions + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTruthy(t *testing.T) { + assert.True(t, Truthy(TruthyVal)) + assert.False(t, Truthy(FalsyVal)) + assert.False(t, Truthy(" ")) +} + +func TestTruthyStr(t *testing.T) { + assert.Equal(t, TruthyVal, TruthyStr(true)) + assert.Equal(t, FalsyVal, TruthyStr(false)) +}