Skip to content

Commit 4c80f97

Browse files
jeeshumittal7jeeshumittal7
committed
First commit
0 parents  commit 4c80f97

12 files changed

+425
-0
lines changed

‎.env

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
PORT=3001
2+
MONGODB_URI=mongodb://localhost:27017/codewhisperer
3+
OPENAI_API_KEY={your_openai_api_key}

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
node_modules/
2+
.gpt-pilot/
3+
package-lock.json

‎README.md

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# CodeWhisperer
2+
3+
CodeWhisperer is a Node.js application leveraging Express and MongoDB, with a Bootstrap-based UI. It facilitates conversation with code repositories by summarizing files and projects using OpenAI's gpt-3.5-turbo-16k and gpt-4-turbo-preview models.
4+
5+
## Overview
6+
7+
The app utilizes Express.js for server-side operations and MongoDB as a data storage solution. It uses the Bootstrap framework to create a responsive UI. The application operates by accepting GitHub repository URLs and email addresses, cloning repositories, extracting and summarizing textual content, and storing summaries in MongoDB. Email notifications are sent through Sendgrid upon task completion.
8+
9+
## Features
10+
11+
- Accept GitHub repository URLs and emails on the main page
12+
- Clone, summarize, and delete repositories upon processing
13+
- Interact with OpenAI's API to generate code and project summaries
14+
- Email users with links to interact with their repository summaries
15+
- Display project summary and offer a chat-like interface for Q&A powered by OpenAI
16+
17+
## Getting started
18+
19+
### Requirements
20+
21+
- Node.js
22+
- MongoDB
23+
- An OpenAI API key
24+
25+
### Quickstart
26+
27+
1. Clone the repository to your local machine.
28+
2. Install dependencies with `npm install`.
29+
3. Set up your `.env` file with the necessary environment variables (PORT, MONGODB_URI, and OPENAI_API_KEY).
30+
4. Run the server using `npm start` or `node server.js`.
31+
32+
### License
33+
34+
Copyright (c) 2024.

‎database.js

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
const mongoose = require('mongoose');
2+
3+
const mongoURI = process.env.MONGODB_URI;
4+
mongoose.connect(mongoURI, { useNewUrlParser: true, useUnifiedTopology: true });
5+
6+
const db = mongoose.connection;
7+
8+
db.on('error', (error) => {
9+
console.error('MongoDB connection error:', error);
10+
});
11+
12+
db.once('open', function() {
13+
console.log('MongoDB connected successfully!');
14+
});
15+
16+
module.exports = db;

‎gitHandler.js

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
const simpleGit = require('simple-git');
2+
const fs = require('fs-extra');
3+
const tmp = require('tmp');
4+
const path = require('path');
5+
6+
const maxFileSize = 32 * 1024; // 32kb in bytes
7+
8+
const cloneAndProcessRepo = async (repoUrl) => {
9+
const tempDir = tmp.dirSync({ unsafeCleanup: true });
10+
console.log(`Temporary directory created at: ${tempDir.name}`); // gpt_pilot_debugging_log
11+
try {
12+
console.log(`Cloning the repository: ${repoUrl}`);
13+
const git = simpleGit();
14+
await git.clone(repoUrl, tempDir.name);
15+
console.log('Repository cloned.');
16+
const files = await getAllFiles(tempDir.name);
17+
const processedFiles = await filterAndCheckFiles(files);
18+
console.log('Files have been checked.');
19+
console.log(`Temporary directory will be kept at: ${tempDir.name} for file processing.`); // gpt_pilot_debugging_log
20+
return { processedFiles: processedFiles, tempDirPath: tempDir.name };
21+
} catch (error) {
22+
console.error('Error occurred in cloneAndProcessRepo:', error.message, error.stack); // gpt_pilot_debugging_log
23+
tempDir.removeCallback(); // Ensure cleanup even in case of error
24+
throw error;
25+
}
26+
};
27+
28+
const getAllFiles = async (dirPath, arrayOfFiles = []) => {
29+
try {
30+
const files = await fs.readdir(dirPath);
31+
for (const file of files) {
32+
const fullPath = path.join(dirPath, file);
33+
console.log(`Full path of file: ${fullPath}`); // gpt_pilot_debugging_log
34+
const stat = await fs.stat(fullPath);
35+
if (stat.isDirectory()) {
36+
await getAllFiles(fullPath, arrayOfFiles);
37+
} else {
38+
arrayOfFiles.push(fullPath);
39+
}
40+
}
41+
return arrayOfFiles;
42+
} catch (error) {
43+
console.error('Error occurred while getting all files:', error.message, error.stack); // gpt_pilot_debugging_log
44+
throw error;
45+
}
46+
};
47+
48+
const filterAndCheckFiles = async (files) => {
49+
try {
50+
const processedFiles = [];
51+
for (const file of files) {
52+
const stat = await fs.stat(file);
53+
if (stat.size <= maxFileSize && isText(file)) {
54+
processedFiles.push(file);
55+
}
56+
}
57+
console.log(`Text files smaller than 32kb:`, processedFiles); // gpt_pilot_debugging_log
58+
return processedFiles;
59+
} catch (error) {
60+
console.error('Error occurred while filtering and checking files:', error.message, error.stack); // gpt_pilot_debugging_log
61+
throw error;
62+
}
63+
};
64+
65+
const isText = (filename) => {
66+
const textFileExtensions = /\.txt$/i;
67+
const isText = textFileExtensions.test(path.extname(filename));
68+
console.log(`Checking if ${filename} is a text file: ${isText}`); // gpt_pilot_debugging_log
69+
return isText;
70+
};
71+
72+
module.exports = {
73+
cloneAndProcessRepo
74+
};

‎openaiService.js

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
const OpenAI = require('openai');
2+
require('dotenv').config();
3+
4+
const openai = new OpenAI(process.env.OPENAI_API_KEY);
5+
6+
console.log('OpenAI API Key provided:', process.env.OPENAI_API_KEY ? 'Yes' : 'No');
7+
8+
async function generateSummary(textContent) {
9+
console.log('Generating summary for content length:', textContent.length);
10+
11+
try {
12+
const response = await openai.chat.completions.create({
13+
model: 'gpt-3.5-turbo-16k',
14+
messages: [{ role: "system", content: "Summarize the following content." }, { role: "user", content: textContent }],
15+
max_tokens: 1024,
16+
temperature: 0.5
17+
});
18+
console.log('Summary generation successful.');
19+
20+
// gpt_pilot_debugging_log
21+
console.log('OpenAI response object:', response);
22+
if (!response.choices || response.choices.length === 0 || !response.choices[0].message) {
23+
console.error('Received an invalid response from OpenAI:', response); // gpt_pilot_debugging_log
24+
throw new Error('Invalid response from OpenAI');
25+
}
26+
27+
// gpt_pilot_debugging_log
28+
console.log('OpenAI response choices:', response.choices);
29+
return response.choices[0].message.content.trim();
30+
} catch (error) {
31+
// Ensuring that the full error stack is logged
32+
console.error('Error generating summary with OpenAI:', error.message, error.stack);
33+
throw error;
34+
}
35+
}
36+
37+
module.exports = {
38+
generateSummary
39+
};

‎package.json

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"name": "codewhisperer",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "ISC",
12+
"dependencies": {
13+
"axios": "^1.6.7",
14+
"body-parser": "^1.20.2",
15+
"bootstrap": "^5.3.2",
16+
"dotenv": "^16.4.1",
17+
"express": "^4.18.2",
18+
"fs-extra": "^11.2.0",
19+
"mongoose": "^8.1.1",
20+
"nodemailer": "^6.9.9",
21+
"nodemailer-sendgrid-transport": "^0.2.0",
22+
"openai": "^4.26.1",
23+
"simple-git": "^3.22.0",
24+
"tmp": "^0.2.1",
25+
"uuid": "^9.0.1"
26+
}
27+
}

‎processRepository.js

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
const Repository = require('./repository');
2+
const { cloneAndProcessRepo } = require('./gitHandler');
3+
const { generateSummary } = require('./openaiService');
4+
const fs = require('fs-extra');
5+
const OpenAI = require('openai');
6+
require('dotenv').config();
7+
8+
const openai = new OpenAI(process.env.OPENAI_API_KEY);
9+
10+
async function processRepository(githubUrl, email) {
11+
// Start processing asynchronously
12+
processRepoInBackground(githubUrl, email).catch(error => {
13+
console.error('Asynchronous processing error:', error.message, error.stack); // gpt_pilot_debugging_log
14+
// Save the error state to the database
15+
Repository.findOneAndUpdate({ githubUrl, email }, { isProcessed: true, processingError: error.message }, { new: true }).catch(err => {
16+
console.error('Failed to update repository with error state:', err.message, err.stack); // gpt_pilot_debugging_log
17+
});
18+
});
19+
20+
// Return immediately for the server to send the response
21+
console.log(`Processing started for repository: ${githubUrl}`); // This log confirms the asynchronous start
22+
}
23+
24+
async function processRepoInBackground(githubUrl, email) {
25+
let tempDirPath;
26+
try {
27+
const { processedFiles, tempDirPath: dirPath } = await cloneAndProcessRepo(githubUrl);
28+
tempDirPath = dirPath;
29+
let allSummaries = [];
30+
31+
for (const file of processedFiles) {
32+
try {
33+
const content = await fs.readFile(file, 'utf8');
34+
const summary = await generateSummary(content);
35+
allSummaries.push(summary);
36+
} catch (fileReadError) {
37+
console.error('Error reading file:', fileReadError.message, fileReadError.stack); // gpt_pilot_debugging_log
38+
}
39+
}
40+
41+
const combinedSummaries = allSummaries.join(' ');
42+
const projectSummaryResponse = await openai.chat.completions.create({
43+
model: 'gpt-4-turbo-preview',
44+
messages: [{ role: "system", content: "Summarize this project based on the individual file summaries." }, { role: "user", content: combinedSummaries }],
45+
max_tokens: 1024,
46+
temperature: 0.5
47+
});
48+
49+
const projectSummary = projectSummaryResponse.choices[0].message.content.trim();
50+
await Repository.findOneAndUpdate({ githubUrl, email }, { summary: projectSummary, isProcessed: true }, { new: true });
51+
52+
console.log(`Repository has been processed: ${githubUrl}`); // gpt_pilot_debugging_log
53+
} catch (error) {
54+
console.error('Error during repository background processing:', error.message, error.stack); // gpt_pilot_debugging_log
55+
throw error; // This preserves the original behavior of rethrowing the error after logging.
56+
} finally {
57+
if (tempDirPath) {
58+
try {
59+
await fs.remove(tempDirPath);
60+
console.log('Temporary directory has been removed.'); // gpt_pilot_debugging_log
61+
} catch (tempDirError) {
62+
console.error('Error removing temporary directory:', tempDirError.message, tempDirError.stack); // gpt_pilot_debugging_log
63+
}
64+
} else {
65+
console.log('Temporary directory path is undefined, so skipping removal.'); // gpt_pilot_debugging_log
66+
}
67+
}
68+
}
69+
70+
module.exports = {
71+
processRepository
72+
};

‎public/index.html

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<title>CodeWhisperer</title>
6+
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css">
7+
</head>
8+
<body>
9+
<div class="container">
10+
<h1>Submit a GitHub Repository</h1>
11+
<form method="POST" action="/submit">
12+
<div class="form-group">
13+
<label for="githubUrl">GitHub URL</label>
14+
<input type="url" id="githubUrl" name="githubUrl" class="form-control" required>
15+
</div>
16+
<div class="form-group">
17+
<label for="email">Email address</label>
18+
<input type="email" id="email" name="email" class="form-control" required>
19+
</div>
20+
<button type="submit" class="btn btn-primary">Submit</button>
21+
</form>
22+
</div>
23+
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
24+
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"></script>
25+
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"></script>
26+
</body>
27+
</html>

‎repository.js

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
const mongoose = require('mongoose');
2+
const uuid = require('uuid');
3+
4+
const repositorySchema = new mongoose.Schema({
5+
githubUrl: { type: String, required: true },
6+
uuid: { type: String, default: function genUUID() { return uuid.v4(); }, unique: true },
7+
email: { type: String, required: true },
8+
summary: { type: String, default: '' },
9+
isProcessed: { type: Boolean, default: false }
10+
}, { timestamps: true });
11+
12+
repositorySchema.post('save', function(error, doc, next) {
13+
if (error.name === 'MongoError' && error.code === 11000) {
14+
next(new Error('There was a duplicate key error'));
15+
} else {
16+
next(error);
17+
}
18+
});
19+
20+
repositorySchema.post('save', function(error, doc, next) {
21+
console.log('Repository saved:', doc);
22+
next();
23+
});
24+
25+
repositorySchema.post('validate', function(error, doc, next) {
26+
console.error('Validation error:', error);
27+
next(error);
28+
});
29+
30+
const Repository = mongoose.model('Repository', repositorySchema);
31+
32+
module.exports = Repository;

0 commit comments

Comments
 (0)