Firecrawl Tool (#481)

Author: sanjay920

firecrawl/Makefile

@@ -0,0 +1,3 @@
+.PHONY: build
+build:
+	go build -o bin/gptscript-go-tool .

firecrawl/cmd/scrape.go

@@ -0,0 +1,37 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/mendableai/firecrawl-go"
+)
+
+func Scrape(ctx context.Context, apiKey, url string) (string, error) {
+	// Parameters are already validated in main.go, so we can proceed directly
+	
+	// Initialize the FirecrawlApp
+	apiUrl := "https://api.firecrawl.dev"
+	app, err := firecrawl.NewFirecrawlApp(apiKey, apiUrl)
+	if err != nil {
+		return "", fmt.Errorf("failed to initialize FirecrawlApp: %w", err)
+	}
+
+	// Scrape with hardcoded format
+	params := &firecrawl.ScrapeParams{
+		Formats: []string{"markdown"},
+	}
+
+	scrapeResult, err := app.ScrapeURL(url, params)
+	if err != nil {
+		return "", fmt.Errorf("failed to scrape URL: %w", err)
+	}
+
+	resultJSON, err := json.MarshalIndent(scrapeResult, "", "  ")
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal scrape result: %w", err)
+	}
+
+	return string(resultJSON), nil
+}

firecrawl/credential/tool.gpt

@@ -0,0 +1,25 @@
+Name: Firecrawl Credential
+Description: Credentials for the Firecrawl API
+Share Credential: firecrawl-cred as firecrawl
+Type: credential
+
+---
+Name: firecrawl-cred
+Tools: ../../generic-credential
+
+#!sys.call ../../generic-credential
+
+{
+	"promptInfo": {
+		"fields" : [
+			{
+				"name": "Firecrawl API key",
+				"description": "Your Firecrawl API key",
+				"env": "FIRECRAWL_API_KEY",
+				"sensitive": true
+			}
+		],
+		"message": "Enter your Firecrawl API key."
+	}
+}
+

firecrawl/go.mod

@@ -0,0 +1,5 @@
+module github.com/obot-platform/tools/firecrawl
+
+go 1.23.4
+
+require github.com/mendableai/firecrawl-go v1.0.0 // indirect

firecrawl/go.sum

@@ -0,0 +1,2 @@
+github.com/mendableai/firecrawl-go v1.0.0 h1:nABWG1eaYtthPAwu8dmUNXz3DcSnV28EdvtHgA5ES+I=
+github.com/mendableai/firecrawl-go v1.0.0/go.mod h1:mTGbJ37fy43aaqonp/tdpzCH516jHFw/XVvfFi4QXHo=

firecrawl/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/obot-platform/tools/firecrawl/cmd"
+)
+
+func validateAPIKey() string {
+	apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
+	if apiKey == "" {
+		exitWithError("API key is required")
+	}
+	return apiKey
+}
+
+func validateRequiredParam(value, name string) string {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		exitWithError(fmt.Sprintf("%s is required", name))
+	}
+	return value
+}
+
+func exitWithError(msg string) {
+	fmt.Println(msg)
+	os.Exit(1)
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Println("Usage: gptscript-go-tool <command>")
+		os.Exit(1)
+	}
+	command := os.Args[1]
+
+	var (
+		result string
+		err    error
+		ctx    = context.Background()
+	)
+
+	apiKey := validateAPIKey()
+
+	switch command {
+	case "scrapeUrl":
+		url := validateRequiredParam(os.Getenv("URL"), "URL")
+		result, err = cmd.Scrape(ctx, apiKey, url)
+		
+	default:
+		exitWithError(fmt.Sprintf("unknown command: %s", command))
+	}
+
+	if err != nil {
+		exitWithError(err.Error())
+	}
+
+	fmt.Print(result)
+}

firecrawl/tool.gpt

@@ -0,0 +1,37 @@
+---
+Name: Firecrawl
+Description: Tools for interacting with Firecrawl.
+Metadata: bundle: false
+Share Tools: Scrape URL
+
+---
+Name: Scrape URL
+Description: Scrape a URL and convert it to markdown using Firecrawl.
+Share Context: Firecrawl Context
+Credentials: ./credential
+Param: url: The URL to scrape.
+
+#!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool scrapeUrl
+
+---
+Name: Firecrawl Context
+Type: context
+
+#!sys.echo
+
+# START INSTRUCTIONS: Scrape URL tool
+
+The Scrape URL tool allows you to scrape websites using Firecrawl.
+
+When calling the Scrape URL tool:
+- The `url` parameter is required.
+- The tool will automatically convert the webpage content to markdown format.
+- Always validate the URL before scraping.
+- The tool returns full scrape results including markdown content and metadata.
+
+# END INSTRUCTIONS: Scrape URL tool
+
+---
+!metadata:*:icon
+https://firecrawl.dev/favicon.ico
+

index.yaml

@@ -75,7 +75,9 @@ tools:
     reference: ./smartthings
   gemini-image-generator:
     reference: ./gemini/image-generator.gpt
-
+  firecrawl:
+    reference: ./firecrawl
+    
 knowledgeDataSources:
   notion-data-source:
     reference: ./knowledge/data-sources/notion