Fix proxyaddress feature for twitter #18
Conversation
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
packages/adapter-qdrant/src/index.ts
Outdated
| const processedContent = content | ||
| .replace(/```[\s\S]*?```/g, "") | ||
| .replace(/`.*?`/g, "") | ||
| .replace(/#{1,6}\s*(.*)/g, "$1") | ||
| .replace(/!\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/(https?:\/\/)?(www\.)?([^\s]+\.[^\s]+)/g, "$3") | ||
| .replace(/<@[!&]?\d+>/g, "") | ||
| .replace(/<[^>]*>/g, "") |
Check failure
Code scanning / CodeQL
Incomplete multi-character sanitization High
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the issue, we will modify the preprocess method to apply the regular expression replacements repeatedly until no more replacements can be performed. This ensures that all instances of the targeted patterns are removed, effectively sanitizing the input string. Additionally, we will use a well-tested sanitization library, sanitize-html, to handle HTML tags and other potentially unsafe content.
| @@ -40,19 +40,26 @@ | ||
| } | ||
| const processedContent = content | ||
| .replace(/```[\s\S]*?```/g, "") | ||
| .replace(/`.*?`/g, "") | ||
| .replace(/#{1,6}\s*(.*)/g, "$1") | ||
| .replace(/!\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/(https?:\/\/)?(www\.)?([^\s]+\.[^\s]+)/g, "$3") | ||
| .replace(/<@[!&]?\d+>/g, "") | ||
| .replace(/<[^>]*>/g, "") | ||
| .replace(/^\s*[-*_]{3,}\s*$/gm, "") | ||
| .replace(/\/\*[\s\S]*?\*\//g, "") | ||
| .replace(/\/\/.*/g, "") | ||
| .replace(/\s+/g, " ") | ||
| .replace(/\n{3,}/g, "\n\n") | ||
| .replace(/[^a-zA-Z0-9\s\-_./:?=&]/g, "") | ||
| .trim() | ||
| return processedContent | ||
| const sanitizeHtml = require("sanitize-html"); | ||
| let previous; | ||
| let processedContent = content; | ||
| do { | ||
| previous = processedContent; | ||
| processedContent = processedContent | ||
| .replace(/```[\s\S]*?```/g, "") | ||
| .replace(/`.*?`/g, "") | ||
| .replace(/#{1,6}\s*(.*)/g, "$1") | ||
| .replace(/!\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/(https?:\/\/)?(www\.)?([^\s]+\.[^\s]+)/g, "$3") | ||
| .replace(/<@[!&]?\d+>/g, "") | ||
| .replace(/<[^>]*>/g, "") | ||
| .replace(/^\s*[-*_]{3,}\s*$/gm, "") | ||
| .replace(/\/\*[\s\S]*?\*\//g, "") | ||
| .replace(/\/\/.*/g, "") | ||
| .replace(/\s+/g, " ") | ||
| .replace(/\n{3,}/g, "\n\n") | ||
| .replace(/[^a-zA-Z0-9\s\-_./:?=&]/g, "") | ||
| .trim(); | ||
| } while (processedContent !== previous); | ||
| processedContent = sanitizeHtml(processedContent); | ||
| return processedContent; | ||
| } |
| @@ -19,3 +19,4 @@ | ||
| "@elizaos/core": "workspace:*", | ||
| "@qdrant/js-client-rest": "^1.12.0" | ||
| "@qdrant/js-client-rest": "^1.12.0", | ||
| "sanitize-html": "^2.14.0" | ||
| }, |
| Package | Version | Security advisories |
| sanitize-html (npm) | 2.14.0 | None |
packages/adapter-sqlite/src/index.ts
Outdated
| // Execute the transaction and ensure it's called with () | ||
| await this.db.transaction(() => { | ||
| if (id.includes("*")) { | ||
| const pattern = id.replace("*", "%"); |
Check failure
Code scanning / CodeQL
Incomplete string escaping or encoding High
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the problem, we need to ensure that all occurrences of the asterisk (*) in the id are replaced with a percent sign (%). This can be achieved by using a regular expression with the global flag (g). This change will ensure that all instances of the asterisk are replaced, preventing any potential issues with the SQL query.
| @@ -977,3 +977,3 @@ | ||
| if (id.includes("*")) { | ||
| const pattern = id.replace("*", "%"); | ||
| const pattern = id.replace(/\*/g, "%"); | ||
| const sql = "DELETE FROM knowledge WHERE id LIKE ?"; |
packages/client-direct/src/api.ts
Outdated
| router.get('/storage', async (req, res) => { | ||
| try { | ||
| const uploadDir = path.join(process.cwd(), "data", "characters"); | ||
| const files = await fs.promises.readdir(uploadDir); | ||
| res.json({ files }); | ||
| } catch (error) { | ||
| res.status(500).json({ error: error.message }); | ||
| } | ||
| }); |
Check failure
Code scanning / CodeQL
Missing rate limiting High
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the problem, we will introduce rate limiting to the Express application using the express-rate-limit package. This will ensure that the number of requests to the route handler performing file system access is limited, thereby mitigating the risk of denial-of-service attacks.
- Install the
express-rate-limitpackage if it is not already installed. - Import the
express-rate-limitpackage in the file. - Set up a rate limiter with appropriate configuration (e.g., maximum of 100 requests per 15 minutes).
- Apply the rate limiter to the specific route handler that performs file system access.
| @@ -6,2 +6,8 @@ | ||
| import fs from "fs"; | ||
| import rateLimit from "express-rate-limit"; | ||
|
|
||
| const limiter = rateLimit({ | ||
| windowMs: 15 * 60 * 1000, // 15 minutes | ||
| max: 100, // limit each IP to 100 requests per windowMs | ||
| }); | ||
|
|
||
| @@ -88,3 +94,3 @@ | ||
|
|
||
| router.get('/storage', async (req, res) => { | ||
| router.get('/storage', limiter, async (req, res) => { | ||
| try { |
| @@ -34,3 +34,4 @@ | ||
| "multer": "1.4.5-lts.1", | ||
| "openai": "4.73.0" | ||
| "openai": "4.73.0", | ||
| "express-rate-limit": "^7.5.0" | ||
| }, |
| Package | Version | Security advisories |
| express-rate-limit (npm) | 7.5.0 | None |
packages/client-direct/src/api.ts
Outdated
| router.post("/agents/:agentId/set", async (req, res) => { | ||
| const { agentId } = validateUUIDParams(req.params, res) ?? { | ||
| agentId: null, | ||
| }; | ||
| if (!agentId) return; | ||
|
|
||
| let agent: AgentRuntime = agents.get(agentId); | ||
| console.log("ip", req.connection.remoteAddress); | ||
|
|
||
| // update character | ||
| if (agent) { | ||
| // stop agent | ||
| agent.stop(); | ||
| directClient.unregisterAgent(agent); | ||
| // if it has a different name, the agentId will change | ||
| } | ||
|
|
||
| // stores the json data before it is modified with added data | ||
| const characterJson = { ...req.body }; | ||
|
|
||
| // load character from body | ||
| const character = req.body; | ||
| console.log("character", character); | ||
| try { | ||
| validateCharacterConfig(character); | ||
| } catch (e) { | ||
| elizaLogger.error(`Error parsing character: ${e}`); | ||
| res.status(400).json({ | ||
| success: false, | ||
| message: e.message, | ||
| }); | ||
| return; | ||
| } | ||
|
|
||
| // start it up (and register it) | ||
| agent = await directClient.startAgent(character); | ||
| try { | ||
| agent = await directClient.startAgent(character); | ||
| elizaLogger.log(`${character.name} started`); | ||
| } catch (e) { | ||
| elizaLogger.error(`Error starting agent: ${e}`); | ||
| res.status(500).json({ | ||
| success: false, | ||
| message: e.message, | ||
| }); | ||
| return; | ||
| } | ||
|
|
||
| if (process.env.USE_CHARACTER_STORAGE === "true") { | ||
| try { | ||
| const filename = `${agent.agentId}.json`; | ||
| const uploadDir = path.join( | ||
| process.cwd(), | ||
| "data", | ||
| "characters" | ||
| ); | ||
| const filepath = path.join(uploadDir, filename); | ||
| await fs.promises.mkdir(uploadDir, { recursive: true }); | ||
| await fs.promises.writeFile( | ||
| filepath, | ||
| JSON.stringify( | ||
| { ...characterJson, id: agent.agentId }, | ||
| null, | ||
| 2 | ||
| ) | ||
| ); | ||
| elizaLogger.info( | ||
| `Character stored successfully at ${filepath}` | ||
| ); | ||
| } catch (error) { | ||
| elizaLogger.error( | ||
| `Failed to store character: ${error.message}` | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| res.json({ | ||
| id: character.id, | ||
| character: character, | ||
| }); | ||
| }); |
Check failure
Code scanning / CodeQL
Missing rate limiting High
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the problem, we need to introduce rate limiting to the Express application to prevent denial-of-service attacks. The best way to do this is by using the express-rate-limit package, which allows us to set a maximum number of requests per window of time. We will apply this rate limiter to the specific route handler that performs file system operations.
- Install the
express-rate-limitpackage. - Import the
express-rate-limitpackage in the file. - Set up a rate limiter with appropriate configuration.
- Apply the rate limiter to the route handler that performs file system operations.
| @@ -6,2 +6,3 @@ | ||
| import fs from "fs"; | ||
| import RateLimit from "express-rate-limit"; | ||
|
|
||
| @@ -162,3 +163,8 @@ | ||
|
|
||
| router.post("/agents/:agentId/set", async (req, res) => { | ||
| const agentRateLimiter = RateLimit({ | ||
| windowMs: 15 * 60 * 1000, // 15 minutes | ||
| max: 100, // max 100 requests per windowMs | ||
| }); | ||
|
|
||
| router.post("/agents/:agentId/set", agentRateLimiter, async (req, res) => { | ||
| const { agentId } = validateUUIDParams(req.params, res) ?? { |
| @@ -34,3 +34,4 @@ | ||
| "multer": "1.4.5-lts.1", | ||
| "openai": "4.73.0" | ||
| "openai": "4.73.0", | ||
| "express-rate-limit": "^7.5.0" | ||
| }, |
| Package | Version | Security advisories |
| express-rate-limit (npm) | 7.5.0 | None |
| 'https://api.smartthings.com/v1/devices', | ||
| expect.objectContaining({ | ||
| headers: expect.objectContaining({ | ||
| 'Authorization': 'Bearer mock-token', |
Check failure
Code scanning / CodeQL
Hard-coded credentials Critical test
packages/core/src/ragknowledge.ts
Outdated
| content | ||
| .replace(/```[\s\S]*?```/g, "") | ||
| .replace(/`.*?`/g, "") | ||
| .replace(/#{1,6}\s*(.*)/g, "$1") | ||
| .replace(/!\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/\[(.*?)\]\(.*?\)/g, "$1") | ||
| .replace(/(https?:\/\/)?(www\.)?([^\s]+\.[^\s]+)/g, "$3") | ||
| .replace(/<@[!&]?\d+>/g, "") | ||
| .replace(/<[^>]*>/g, "") |
Check failure
Code scanning / CodeQL
Incomplete multi-character sanitization High
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the problem, we need to ensure that the regular expression replacements are applied repeatedly until no more matches are found. This will ensure that all instances of the targeted patterns are removed from the input content. We will modify the preprocess method to repeatedly apply the replacements in a loop until the content no longer changes.
| @@ -126,4 +126,6 @@ | ||
|
|
||
| return ( | ||
| content | ||
| let previous; | ||
| do { | ||
| previous = content; | ||
| content = content | ||
| .replace(/```[\s\S]*?```/g, "") | ||
| @@ -143,4 +145,6 @@ | ||
| .trim() | ||
| .toLowerCase() | ||
| ); | ||
| .toLowerCase(); | ||
| } while (content !== previous); | ||
|
|
||
| return content; | ||
| } |
| }, | ||
| { | ||
| headers: { | ||
| Authorization: 'Bearer mock-jwt-token', |
Check failure
Code scanning / CodeQL
Hard-coded credentials Critical test
| logGranular("Initiating asset upload authorization", { | ||
| headers: { | ||
| ...headers, | ||
| "Authorization": "Bearer [REDACTED]" |
Check failure
Code scanning / CodeQL
Hard-coded credentials Critical
| url: invoke_url, | ||
| headers: { | ||
| ...inferHeaders, | ||
| "Authorization": "Bearer [REDACTED]" |
Check failure
Code scanning / CodeQL
Hard-coded credentials Critical
scripts/update-package-access.js
Outdated
|
|
||
| // Format the file using npx Prettier | ||
| try { | ||
| execSync(`npx prettier --write ${packageJsonPath}`, { stdio: 'inherit' }); |
Check warning
Code scanning / CodeQL
Shell command built from environment values Medium
Show autofix suggestion
Hide autofix suggestion
Copilot Autofix AI 3 days ago
To fix the problem, we should avoid constructing the shell command dynamically with packageJsonPath directly. Instead, we can use the execFileSync function, which allows us to pass the command and its arguments separately. This approach ensures that the arguments are not interpreted by the shell, thus preventing command injection vulnerabilities.
We need to:
- Replace the
execSynccall withexecFileSync. - Pass the command (
npx) and its arguments (prettier --write ${packageJsonPath}) separately.
| @@ -3,3 +3,3 @@ | ||
| const glob = require('glob'); | ||
| const { execSync } = require('node:child_process'); | ||
| const { execFileSync } = require('node:child_process'); | ||
|
|
||
| @@ -18,3 +18,3 @@ | ||
| try { | ||
| execSync(`npx prettier --write ${packageJsonPath}`, { stdio: 'inherit' }); | ||
| execFileSync('npx', ['prettier', '--write', packageJsonPath], { stdio: 'inherit' }); | ||
| console.log(`Formatted: ${packageJsonPath}`); |
Relates to
Risks
Background
What does this PR do?
What kind of change is this?
Documentation changes needed?
Testing
Where should a reviewer start?
Detailed testing steps