Skip to content

Commit 5938a1d

Browse files
PeriniMPeriniM
committed
feat: add logging
1 parent a157c32 commit 5938a1d

File tree

2 files changed

+122
-28
lines changed

2 files changed

+122
-28
lines changed

Diff for: ‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
44
[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3100/)
55
[![smithery badge](https://smithery.ai/badge/@ScrapeGraphAI/scrapegraph-mcp)](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
6+
67
A [Model Context Protocol](https://modelcontextprotocol.io/introduction) (MCP) server that provides access to the [ScapeGraph AI](https://scrapegraphai.com) API. It allows language models to use AI-powered web scraping capabilities.
78

89
## Available Tools

Diff for: ‎src/scrapegraph_mcp/server.py

+121-28
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,26 @@
88
"""
99

1010
import os
11+
import sys
12+
import logging
13+
import traceback
1114
from typing import Any, Dict
1215

1316
import httpx
1417
from mcp.server.fastmcp import FastMCP
1518

1619

20+
# Configure logging
21+
logging.basicConfig(
22+
level=logging.INFO,
23+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
24+
handlers=[
25+
logging.StreamHandler(sys.stderr)
26+
]
27+
)
28+
logger = logging.getLogger("scrapegraph_mcp")
29+
30+
1731
class ScapeGraphClient:
1832
"""Client for interacting with the ScapeGraph API."""
1933

@@ -26,12 +40,14 @@ def __init__(self, api_key: str):
2640
Args:
2741
api_key: API key for ScapeGraph API
2842
"""
43+
logger.info("Initializing ScapeGraphClient")
2944
self.api_key = api_key
3045
self.headers = {
3146
"SGAI-APIKEY": api_key,
3247
"Content-Type": "application/json"
3348
}
3449
self.client = httpx.Client(timeout=60.0)
50+
logger.info("ScapeGraphClient initialized successfully")
3551

3652
def markdownify(self, website_url: str) -> Dict[str, Any]:
3753
"""
@@ -43,18 +59,26 @@ def markdownify(self, website_url: str) -> Dict[str, Any]:
4359
Returns:
4460
Dictionary containing the markdown result
4561
"""
62+
logger.info(f"Calling markdownify for URL: {website_url}")
4663
url = f"{self.BASE_URL}/markdownify"
4764
data = {
4865
"website_url": website_url
4966
}
5067

51-
response = self.client.post(url, headers=self.headers, json=data)
52-
53-
if response.status_code != 200:
54-
error_msg = f"Error {response.status_code}: {response.text}"
55-
raise Exception(error_msg)
56-
57-
return response.json()
68+
try:
69+
logger.debug(f"Making POST request to {url}")
70+
response = self.client.post(url, headers=self.headers, json=data)
71+
72+
if response.status_code != 200:
73+
error_msg = f"Error {response.status_code}: {response.text}"
74+
logger.error(f"API request failed: {error_msg}")
75+
raise Exception(error_msg)
76+
77+
logger.info("markdownify request successful")
78+
return response.json()
79+
except Exception as e:
80+
logger.error(f"Exception in markdownify: {str(e)}")
81+
raise
5882

5983
def smartscraper(self, user_prompt: str, website_url: str) -> Dict[str, Any]:
6084
"""
@@ -67,19 +91,27 @@ def smartscraper(self, user_prompt: str, website_url: str) -> Dict[str, Any]:
6791
Returns:
6892
Dictionary containing the extracted data
6993
"""
94+
logger.info(f"Calling smartscraper for URL: {website_url} with prompt: {user_prompt}")
7095
url = f"{self.BASE_URL}/smartscraper"
7196
data = {
7297
"user_prompt": user_prompt,
7398
"website_url": website_url
7499
}
75100

76-
response = self.client.post(url, headers=self.headers, json=data)
77-
78-
if response.status_code != 200:
79-
error_msg = f"Error {response.status_code}: {response.text}"
80-
raise Exception(error_msg)
81-
82-
return response.json()
101+
try:
102+
logger.debug(f"Making POST request to {url}")
103+
response = self.client.post(url, headers=self.headers, json=data)
104+
105+
if response.status_code != 200:
106+
error_msg = f"Error {response.status_code}: {response.text}"
107+
logger.error(f"API request failed: {error_msg}")
108+
raise Exception(error_msg)
109+
110+
logger.info("smartscraper request successful")
111+
return response.json()
112+
except Exception as e:
113+
logger.error(f"Exception in smartscraper: {str(e)}")
114+
raise
83115

84116
def searchscraper(self, user_prompt: str) -> Dict[str, Any]:
85117
"""
@@ -91,30 +123,57 @@ def searchscraper(self, user_prompt: str) -> Dict[str, Any]:
91123
Returns:
92124
Dictionary containing search results and reference URLs
93125
"""
126+
logger.info(f"Calling searchscraper with prompt: {user_prompt}")
94127
url = f"{self.BASE_URL}/searchscraper"
95128
data = {
96129
"user_prompt": user_prompt
97130
}
98131

99-
response = self.client.post(url, headers=self.headers, json=data)
100-
101-
if response.status_code != 200:
102-
error_msg = f"Error {response.status_code}: {response.text}"
103-
raise Exception(error_msg)
104-
105-
return response.json()
132+
try:
133+
logger.debug(f"Making POST request to {url}")
134+
response = self.client.post(url, headers=self.headers, json=data)
135+
136+
if response.status_code != 200:
137+
error_msg = f"Error {response.status_code}: {response.text}"
138+
logger.error(f"API request failed: {error_msg}")
139+
raise Exception(error_msg)
140+
141+
logger.info("searchscraper request successful")
142+
return response.json()
143+
except Exception as e:
144+
logger.error(f"Exception in searchscraper: {str(e)}")
145+
raise
106146

107147
def close(self) -> None:
108148
"""Close the HTTP client."""
149+
logger.info("Closing ScapeGraphClient")
109150
self.client.close()
151+
logger.info("ScapeGraphClient closed")
152+
110153

154+
# Log environment information
155+
logger.info(f"Python version: {sys.version}")
156+
logger.info(f"Current working directory: {os.getcwd()}")
157+
logger.info(f"PATH environment variable: {os.environ.get('PATH', 'Not set')}")
111158

112159
# Create MCP server
160+
logger.info("Creating MCP server")
113161
mcp = FastMCP("ScapeGraph API MCP Server")
162+
logger.info("MCP server created")
114163

115164
# Default API key (will be overridden in main or by direct assignment)
116165
default_api_key = os.environ.get("SGAI_API_KEY")
117-
scrapegraph_client = ScapeGraphClient(default_api_key) if default_api_key else None
166+
logger.info(f"SGAI_API_KEY environment variable is {'set' if default_api_key else 'not set'}")
167+
168+
scrapegraph_client = None
169+
if default_api_key:
170+
try:
171+
logger.info("Initializing ScapeGraphClient with default API key")
172+
scrapegraph_client = ScapeGraphClient(default_api_key)
173+
logger.info("ScapeGraphClient initialized successfully")
174+
except Exception as e:
175+
logger.error(f"Failed to initialize ScapeGraphClient: {str(e)}")
176+
logger.error(traceback.format_exc())
118177

119178

120179
# Add tool for markdownify
@@ -129,12 +188,19 @@ def markdownify(website_url: str) -> Dict[str, Any]:
129188
Returns:
130189
Dictionary containing the markdown result
131190
"""
191+
logger.info(f"Tool markdownify called with URL: {website_url}")
192+
132193
if scrapegraph_client is None:
194+
logger.warning("ScapeGraph client not initialized")
133195
return {"error": "ScapeGraph client not initialized. Please provide an API key."}
134196

135197
try:
136-
return scrapegraph_client.markdownify(website_url)
198+
result = scrapegraph_client.markdownify(website_url)
199+
logger.info("markdownify tool call successful")
200+
return result
137201
except Exception as e:
202+
logger.error(f"Error in markdownify tool: {str(e)}")
203+
logger.error(traceback.format_exc())
138204
return {"error": str(e)}
139205

140206

@@ -154,12 +220,19 @@ def smartscraper(
154220
Returns:
155221
Dictionary containing the extracted data
156222
"""
223+
logger.info(f"Tool smartscraper called with URL: {website_url} and prompt: {user_prompt}")
224+
157225
if scrapegraph_client is None:
226+
logger.warning("ScapeGraph client not initialized")
158227
return {"error": "ScapeGraph client not initialized. Please provide an API key."}
159228

160229
try:
161-
return scrapegraph_client.smartscraper(user_prompt, website_url)
230+
result = scrapegraph_client.smartscraper(user_prompt, website_url)
231+
logger.info("smartscraper tool call successful")
232+
return result
162233
except Exception as e:
234+
logger.error(f"Error in smartscraper tool: {str(e)}")
235+
logger.error(traceback.format_exc())
163236
return {"error": str(e)}
164237

165238

@@ -177,20 +250,40 @@ def searchscraper(
177250
Returns:
178251
Dictionary containing search results and reference URLs
179252
"""
253+
logger.info(f"Tool searchscraper called with prompt: {user_prompt}")
254+
180255
if scrapegraph_client is None:
256+
logger.warning("ScapeGraph client not initialized")
181257
return {"error": "ScapeGraph client not initialized. Please provide an API key."}
182258

183259
try:
184-
return scrapegraph_client.searchscraper(user_prompt)
260+
result = scrapegraph_client.searchscraper(user_prompt)
261+
logger.info("searchscraper tool call successful")
262+
return result
185263
except Exception as e:
264+
logger.error(f"Error in searchscraper tool: {str(e)}")
265+
logger.error(traceback.format_exc())
186266
return {"error": str(e)}
187267

188268

189269
def main() -> None:
190270
"""Run the ScapeGraph MCP server."""
191-
print("Starting ScapeGraph MCP server!")
192-
# Run the server
193-
mcp.run(transport="stdio")
271+
try:
272+
logger.info("Starting ScapeGraph MCP server!")
273+
print("Starting ScapeGraph MCP server!", file=sys.stderr)
274+
275+
# Log system information
276+
logger.info(f"Python executable: {sys.executable}")
277+
logger.info(f"Arguments: {sys.argv}")
278+
279+
# Run the server
280+
logger.info("Running MCP server with stdio transport")
281+
mcp.run(transport="stdio")
282+
except Exception as e:
283+
logger.critical(f"Fatal error in main: {str(e)}")
284+
logger.critical(traceback.format_exc())
285+
print(f"Fatal error: {str(e)}", file=sys.stderr)
286+
sys.exit(1)
194287

195288

196289
if __name__ == "__main__":

0 commit comments

Comments
 (0)