diff --git a/exec.go b/exec.go index 1863656..a5d22fc 100644 --- a/exec.go +++ b/exec.go @@ -1,7 +1,9 @@ package gptscript import ( + "bytes" "context" + "encoding/json" "fmt" "io" "os" @@ -38,7 +40,7 @@ func (o Opts) toArgs() []string { // Version will return the output of `gptscript --version` func Version(ctx context.Context) (string, error) { out, err := exec.CommandContext(ctx, getCommand(), "--version").CombinedOutput() - return string(out), err + return string(bytes.TrimSpace(out)), err } // ListTools will list all the available tools. @@ -242,6 +244,57 @@ func StreamExecFileWithEvents(ctx context.Context, toolPath, input string, opts return stdout, stderr, eventsRead, wait } +// Parse will parse the given file into an array of Nodes. +func Parse(ctx context.Context, fileName string, opts Opts) ([]Node, error) { + output, err := exec.CommandContext(ctx, getCommand(), append(opts.toArgs(), "parse", fileName)...).CombinedOutput() + if err != nil { + return nil, err + } + + var doc Document + if err = json.Unmarshal(output, &doc); err != nil { + return nil, err + } + + return doc.Nodes, nil +} + +// ParseTool will parse the given string into a tool. +func ParseTool(ctx context.Context, input string) ([]Node, error) { + c := exec.CommandContext(ctx, getCommand(), "parse", "-") + c.Stdin = strings.NewReader(input) + + output, err := c.CombinedOutput() + if err != nil { + return nil, err + } + + var doc Document + if err = json.Unmarshal(output, &doc); err != nil { + return nil, err + } + + return doc.Nodes, nil +} + +// Fmt will format the given nodes into a string. +func Fmt(ctx context.Context, nodes []Node) (string, error) { + b, err := json.Marshal(Document{Nodes: nodes}) + if err != nil { + return "", fmt.Errorf("failed to marshal nodes: %w", err) + } + + c := exec.CommandContext(ctx, getCommand(), "fmt", "-") + c.Stdin = bytes.NewReader(b) + + output, err := c.CombinedOutput() + if err != nil { + return "", err + } + + return string(output), nil +} + func concatTools(tools []fmt.Stringer) string { var sb strings.Builder for i, tool := range tools { diff --git a/exec_test.go b/exec_test.go index 008862b..32ff278 100644 --- a/exec_test.go +++ b/exec_test.go @@ -8,6 +8,8 @@ import ( "runtime" "strings" "testing" + + "github.com/getkin/kin-openapi/openapi3" ) func TestMain(m *testing.M) { @@ -77,7 +79,7 @@ func TestExecFileChdir(t *testing.T) { } func TestExecComplexTool(t *testing.T) { - tool := &Tool{ + tool := &SimpleTool{ JSONResponse: true, Instructions: ` Create three short graphic artist descriptions and their muses. @@ -110,11 +112,11 @@ func TestExecWithToolList(t *testing.T) { shebang = "#!/usr/bin/env powershell.exe" } tools := []fmt.Stringer{ - &Tool{ + &SimpleTool{ Tools: []string{"echo"}, Instructions: "echo hello there", }, - &Tool{ + &SimpleTool{ Name: "echo", Tools: []string{"sys.exec"}, Description: "Echoes the input", @@ -141,16 +143,16 @@ func TestExecWithToolListAndSubTool(t *testing.T) { shebang = "#!/usr/bin/env powershell.exe" } tools := []fmt.Stringer{ - &Tool{ + &SimpleTool{ Tools: []string{"echo"}, Instructions: "echo hello there", }, - &Tool{ + &SimpleTool{ Name: "other", Tools: []string{"echo"}, Instructions: "echo hello somewhere else", }, - &Tool{ + &SimpleTool{ Name: "echo", Tools: []string{"sys.exec"}, Description: "Echoes the input", @@ -296,3 +298,205 @@ func TestStreamExecFileWithEvents(t *testing.T) { t.Error("No events output") } } + +func TestParseSimpleFile(t *testing.T) { + tools, err := Parse(context.Background(), "./test/test.gpt", Opts{}) + if err != nil { + t.Errorf("Error parsing file: %v", err) + } + + if len(tools) != 1 { + t.Errorf("Unexpected number of tools: %d", len(tools)) + } + + if tools[0].ToolNode == nil { + t.Error("No tool node found") + } + + if tools[0].ToolNode.Tool.Instructions != "Respond with a hello, in a random language. Also include the language in the response." { + t.Errorf("Unexpected instructions: %s", tools[0].ToolNode.Tool.Instructions) + } +} + +func TestParseSimpleFileWithChdir(t *testing.T) { + tools, err := Parse(context.Background(), "./test.gpt", Opts{Chdir: "./test"}) + if err != nil { + t.Errorf("Error parsing file: %v", err) + } + + if len(tools) != 1 { + t.Errorf("Unexpected number of tools: %d", len(tools)) + } + + if tools[0].ToolNode == nil { + t.Error("No tool node found") + } + + if tools[0].ToolNode.Tool.Instructions != "Respond with a hello, in a random language. Also include the language in the response." { + t.Errorf("Unexpected instructions: %s", tools[0].ToolNode.Tool.Instructions) + } +} + +func TestParseTool(t *testing.T) { + tools, err := ParseTool(context.Background(), "echo hello") + if err != nil { + t.Errorf("Error parsing tool: %v", err) + } + + if len(tools) != 1 { + t.Errorf("Unexpected number of tools: %d", len(tools)) + } + + if tools[0].ToolNode == nil { + t.Error("No tool node found") + } + + if tools[0].ToolNode.Tool.Instructions != "echo hello" { + t.Errorf("Unexpected instructions: %s", tools[0].ToolNode.Tool.Instructions) + } +} + +func TestParseToolWithTextNode(t *testing.T) { + tools, err := ParseTool(context.Background(), "echo hello\n---\n!markdown\nhello") + if err != nil { + t.Errorf("Error parsing tool: %v", err) + } + + if len(tools) != 2 { + t.Errorf("Unexpected number of tools: %d", len(tools)) + } + + if tools[0].ToolNode == nil { + t.Error("No tool node found") + } + + if tools[0].ToolNode.Tool.Instructions != "echo hello" { + t.Errorf("Unexpected instructions: %s", tools[0].ToolNode.Tool.Instructions) + } + + if tools[1].TextNode == nil { + t.Error("No text node found") + } + + if tools[1].TextNode.Text != "!markdown\nhello\n" { + t.Errorf("Unexpected text: %s", tools[1].TextNode.Text) + } +} + +func TestFmt(t *testing.T) { + nodes := []Node{ + { + ToolNode: &ToolNode{ + Tool: Tool{ + Parameters: Parameters{ + Tools: []string{"echo"}, + }, + Instructions: "echo hello there", + }, + }, + }, + { + ToolNode: &ToolNode{ + Tool: Tool{ + Parameters: Parameters{ + Name: "echo", + Arguments: &openapi3.Schema{ + Type: "object", + Properties: map[string]*openapi3.SchemaRef{ + "input": { + Value: &openapi3.Schema{ + Description: "The string input to echo", + Type: "string", + }, + }, + }, + }, + }, + Instructions: "#!/bin/bash\necho hello there", + }, + }, + }, + } + + out, err := Fmt(context.Background(), nodes) + if err != nil { + t.Errorf("Error formatting nodes: %v", err) + } + + if out != `Tools: echo + +echo hello there + +--- +Name: echo +Args: input: The string input to echo + +#!/bin/bash +echo hello there +` { + t.Errorf("Unexpected output: %s", out) + } +} + +func TestFmtWithTextNode(t *testing.T) { + nodes := []Node{ + { + ToolNode: &ToolNode{ + Tool: Tool{ + Parameters: Parameters{ + Tools: []string{"echo"}, + }, + Instructions: "echo hello there", + }, + }, + }, + { + TextNode: &TextNode{ + Text: "!markdown\nWe now echo hello there\n", + }, + }, + { + ToolNode: &ToolNode{ + Tool: Tool{ + Parameters: Parameters{ + Name: "echo", + Arguments: &openapi3.Schema{ + Type: "object", + Properties: map[string]*openapi3.SchemaRef{ + "input": { + Value: &openapi3.Schema{ + Description: "The string input to echo", + Type: "string", + }, + }, + }, + }, + }, + Instructions: "#!/bin/bash\necho hello there", + }, + }, + }, + } + + out, err := Fmt(context.Background(), nodes) + if err != nil { + t.Errorf("Error formatting nodes: %v", err) + } + + if out != `Tools: echo + +echo hello there + +--- +!markdown +We now echo hello there +--- +Name: echo +Args: input: The string input to echo + +#!/bin/bash +echo hello there +` { + t.Errorf("Unexpected output: %s", out) + } +} diff --git a/go.mod b/go.mod index e7b343c..5a82240 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ module github.com/gptscript-ai/go-gptscript go 1.22.2 + +require github.com/getkin/kin-openapi v0.123.0 + +require ( + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/swag v0.22.8 // indirect + github.com/invopop/yaml v0.2.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index e69de29..41f1c59 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8= +github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= +github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tool.go b/tool.go index 39c1186..de4f21c 100644 --- a/tool.go +++ b/tool.go @@ -3,10 +3,12 @@ package gptscript import ( "fmt" "strings" + + "github.com/getkin/kin-openapi/openapi3" ) -// Tool struct represents a tool with various configurations. -type Tool struct { +// SimpleTool struct represents a tool with various configurations. +type SimpleTool struct { Name string Description string Tools []string @@ -20,9 +22,9 @@ type Tool struct { JSONResponse bool } -// NewTool is a constructor for Tool struct. -func NewTool(name, description string, tools []string, maxTokens *int, model string, cache bool, temperature *float64, args map[string]string, internalPrompt bool, instructions string, jsonResponse bool) *Tool { - return &Tool{ +// NewSimpleTool is a constructor for SimpleTool struct. +func NewSimpleTool(name, description string, tools []string, maxTokens *int, model string, cache bool, temperature *float64, args map[string]string, internalPrompt bool, instructions string, jsonResponse bool) *SimpleTool { + return &SimpleTool{ Name: name, Description: description, Tools: tools, @@ -37,8 +39,8 @@ func NewTool(name, description string, tools []string, maxTokens *int, model str } } -// String method returns the string representation of Tool. -func (t *Tool) String() string { +// String method returns the string representation of SimpleTool. +func (t *SimpleTool) String() string { var sb strings.Builder if t.Name != "" { @@ -95,7 +97,7 @@ func (f *FreeForm) String() string { return f.Content } -type Tools []Tool +type Tools []SimpleTool func (t Tools) String() string { resp := make([]string, 0, len(t)) @@ -104,3 +106,67 @@ func (t Tools) String() string { } return strings.Join(resp, "\n---\n") } + +type Document struct { + Nodes []Node `json:"nodes,omitempty"` +} + +type Node struct { + TextNode *TextNode `json:"textNode,omitempty"` + ToolNode *ToolNode `json:"toolNode,omitempty"` +} + +type TextNode struct { + Text string `json:"text,omitempty"` +} + +type ToolNode struct { + Tool Tool `json:"tool,omitempty"` +} + +type Tool struct { + Parameters `json:",inline"` + Instructions string `json:"instructions,omitempty"` + + ID string `json:"id,omitempty"` + ToolMapping map[string]string `json:"toolMapping,omitempty"` + LocalTools map[string]string `json:"localTools,omitempty"` + Source ToolSource `json:"source,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` +} + +type ToolSource struct { + Location string `json:"location,omitempty"` + LineNo int `json:"lineNo,omitempty"` + Repo *Repo `json:"repo,omitempty"` +} + +type Repo struct { + VCS string + Root string + Path string + Name string + Revision string +} + +type Parameters struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + MaxTokens int `json:"maxTokens,omitempty"` + ModelName string `json:"modelName,omitempty"` + ModelProvider bool `json:"modelProvider,omitempty"` + JSONResponse bool `json:"jsonResponse,omitempty"` + Chat bool `json:"chat,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + Cache *bool `json:"cache,omitempty"` + InternalPrompt *bool `json:"internalPrompt"` + Arguments *openapi3.Schema `json:"arguments,omitempty"` + Tools []string `json:"tools,omitempty"` + GlobalTools []string `json:"globalTools,omitempty"` + GlobalModelName string `json:"globalModelName,omitempty"` + Context []string `json:"context,omitempty"` + ExportContext []string `json:"exportContext,omitempty"` + Export []string `json:"export,omitempty"` + Credentials []string `json:"credentials,omitempty"` + Blocking bool `json:"-"` +}