Added improvements to detect and handle non-text files effectively

Author: jeeshumittal7

Diff for: README.md

@@ -1,34 +1,35 @@
 # CodeWhisperer
 
-CodeWhisperer is an innovative Node.js application that allows users to have conversations with their code repositories. By leveraging the power of OpenAI's AI models, it can summarize text files and entire code repositories to facilitate a chat-like interaction.
+CodeWhisperer is a web-based application designed to facilitate insightful interactions between users and their code repositories. By leveraging cutting-edge AI technologies, CodeWhisperer provides intelligent summaries and detailed explanations, engaging users in a conversational interface that demystifies complex codebases.
 
 ## Overview
 
-The application is built using Express for server management, MongoDB for data persistence, and Bootstrap for the front-end interface. It primarily interacts with OpenAI's API to create summaries of repositories and engage users in conversations about their code. The app processes GitHub repositories by cloning them, generating summaries through the gpt-3.5-turbo-16k and gpt-4-turbo-preview AI models, and then communicates the results via email using Sendgrid.
+The app integrates the Express framework within a Node.js environment, utilizing MongoDB for data persistence. For UI elegance and interactivity, Bootstrap is employed. The core functionality is built around OpenAI's powerful language models, enabling the app to generate concise text summaries and interactive Q&A sessions about a user's code repository.
 
 ## Features
 
-- Accepts GitHub repository URLs and user email addresses for processing
-- Clones repositories and summarizes text files and overall project contents
-- Communicates with OpenAI's API to generate summaries and answer user queries
-- Sends email notifications with links to interact with the repository summary
-- Provides a chat interface for dynamic interaction with the project summary
+- Submits GitHub repository URLs and email addresses for processing
+- Clones GitHub repositories and communicates with OpenAI API to generate summaries
+- Provides interactive explanations of code via a unique link, offering insights into repositories
+- Utilizes Sendgrid for email notifications upon the completion of repository analysis
+- Proffers a unique and engaging way to understand and engage with one's programming projects
 
 ## Getting started
 
 ### Requirements
 
-- Node.js environment
-- MongoDB instance
-- OpenAI API key
-- Sendgrid credentials for email notifications
+- Node.js
+- MongoDB
+- A Sendgrid account for email services
+- An OpenAI API key for natural language processing
 
 ### Quickstart
 
-1. Clone the repository to your machine.
-2. Install necessary Node.js packages with `npm install`.
-3. Configure the required environment variables within an `.env` file.
-4. Run the application using `npm start` or `node server.js`.
+1. Ensure that MongoDB is running on your system.
+2. Clone the repository to your local machine.
+3. Install node modules by running `npm install`.
+4. Set up the necessary environment variables in an `.env` file.
+5. Start the application with `npm start`, and navigate to `localhost:3001` on your web browser.
 
 ### License
 

Diff for: gitHandler.js

@@ -7,40 +7,41 @@ const maxFileSize = 32 * 1024; // 32kb in bytes
 
 const cloneAndProcessRepo = async (repoUrl) => {
   const tempDir = tmp.dirSync({ unsafeCleanup: true });
-  console.log(`Temporary directory created at: ${tempDir.name}`); // gpt_pilot_debugging_log
+  console.log(`Temporary directory created at: ${tempDir.name}`);
   try {
     console.log(`Cloning the repository: ${repoUrl}`);
     const git = simpleGit();
     await git.clone(repoUrl, tempDir.name);
     console.log('Repository cloned.');
-    const files = await getAllFiles(tempDir.name);
-    const processedFiles = await filterAndCheckFiles(files);
-    console.log('Files have been checked.');
-    console.log(`Temporary directory will be kept at: ${tempDir.name} for file processing.`); // gpt_pilot_debugging_log
-    return { processedFiles: processedFiles, tempDirPath: tempDir.name };
+    const { allFiles, textFiles } = await getAllFiles(tempDir.name);
+    const processedFiles = await filterAndCheckFiles(textFiles);
+    console.log('Files have been checked and filtered.');
+    return { processedFiles: processedFiles, allFiles: allFiles, tempDirPath: tempDir.name };
   } catch (error) {
-    console.error('Error occurred in cloneAndProcessRepo:', error.message, error.stack); // gpt_pilot_debugging_log
-    tempDir.removeCallback(); // Ensure cleanup even in case of error
+    console.error('Error occurred in cloneAndProcessRepo:', error.message, error.stack);
+    tempDir.removeCallback();
     throw error;
   }
 };
 
-const getAllFiles = async (dirPath, arrayOfFiles = []) => {
+const getAllFiles = async (dirPath, allFiles = [], textFiles = []) => {
   try {
     const files = await fs.readdir(dirPath);
     for (const file of files) {
       const fullPath = path.join(dirPath, file);
-      console.log(`Full path of file: ${fullPath}`); // gpt_pilot_debugging_log
       const stat = await fs.stat(fullPath);
+      allFiles.push(fullPath);
       if (stat.isDirectory()) {
-        await getAllFiles(fullPath, arrayOfFiles);
+        await getAllFiles(fullPath, allFiles, textFiles);
       } else {
-        arrayOfFiles.push(fullPath);
+        if (isText(fullPath)) {
+          textFiles.push(fullPath);
+        }
       }
     }
-    return arrayOfFiles;
+    return { allFiles, textFiles };
   } catch (error) {
-    console.error('Error occurred while getting all files:', error.message, error.stack); // gpt_pilot_debugging_log
+    console.error('Error occurred while getting all files:', error.message, error.stack);
     throw error;
   }
 };
@@ -50,25 +51,34 @@ const filterAndCheckFiles = async (files) => {
     const processedFiles = [];
     for (const file of files) {
       const stat = await fs.stat(file);
-      if (stat.size <= maxFileSize && isText(file)) {
-        processedFiles.push(file);
+      if (stat.size <= maxFileSize) {
+        const isTextFile = await isText(file);
+        console.log(`Checked if file is text: ${file}, Result: ${isTextFile}`);
+        if (isTextFile) {
+          processedFiles.push(file);
+        }
       }
     }
-    console.log(`Text files smaller than 32kb:`, processedFiles); // gpt_pilot_debugging_log
+    console.log(`Text files smaller than 32kb:`, processedFiles);
     return processedFiles;
   } catch (error) {
-    console.error('Error occurred while filtering and checking files:', error.message, error.stack); // gpt_pilot_debugging_log
+    console.error('Error occurred while filtering and checking files:', error.message, error.stack);
     throw error;
   }
 };
 
-const isText = (filename) => {
-  const textFileExtensions = /\.txt$/i;
-  const isText = textFileExtensions.test(path.extname(filename));
-  console.log(`Checking if ${filename} is a text file: ${isText}`); // gpt_pilot_debugging_log
-  return isText;
+const isText = async (filename) => {
+  try {
+    await fs.readFile(filename, 'utf8');
+    console.log(`The file ${filename} is a text file.`);
+    return true;
+  } catch (error) {
+    console.error('Error when checking if file is a text file:', error.message, error.stack);
+    return false;
+  }
 };
 
 module.exports = {
-  cloneAndProcessRepo
+  cloneAndProcessRepo,
+  isText
 };

Diff for: package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "jest"
   },
   "keywords": [],
   "author": "",
@@ -23,5 +23,8 @@
     "simple-git": "^3.22.0",
     "tmp": "^0.2.1",
     "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "jest": "^29.7.0"
   }
 }

Diff for: processRepository.js

@@ -11,21 +11,21 @@ const openai = new OpenAI(process.env.OPENAI_API_KEY);
 async function processRepository(githubUrl, email) {
   // Start processing asynchronously
   processRepoInBackground(githubUrl, email).catch(error => {
-    console.error('Asynchronous processing error:', error.message, error.stack); // gpt_pilot_debugging_log
+    console.error('Asynchronous processing error:', error.message, error.stack);
     // Save the error state to the database
     Repository.findOneAndUpdate({ githubUrl, email }, { isProcessed: true, processingError: error.message }, { new: true }).catch(err => {
-      console.error('Failed to update repository with error state:', err.message, err.stack); // gpt_pilot_debugging_log
+      console.error('Failed to update repository with error state:', err.message, err.stack);
     });
   });
 
   // Return immediately for the server to send the response
-  console.log(`Processing started for repository: ${githubUrl}`); // This log confirms the asynchronous start
+  console.log(`Processing started for repository: ${githubUrl}`);
 }
 
 async function processRepoInBackground(githubUrl, email) {
   let tempDirPath;
   try {
-    const { processedFiles, tempDirPath: dirPath } = await cloneAndProcessRepo(githubUrl);
+    const { processedFiles, allFiles, tempDirPath: dirPath } = await cloneAndProcessRepo(githubUrl);
     tempDirPath = dirPath;
     let fileSummariesObject = {};
 
@@ -35,21 +35,34 @@ async function processRepoInBackground(githubUrl, email) {
         const summary = await generateSummary(content);
         fileSummariesObject[file] = summary; // Store summary associated with file name
       } catch (fileReadError) {
-        console.error('Error reading file:', fileReadError.message, fileReadError.stack); // gpt_pilot_debugging_log
+        console.error('Error reading or summarizing file:', file, fileReadError.message, fileReadError.stack);
       }
     }
 
     const fileSummariesArray = Object.values(fileSummariesObject); // Convert summaries object to array
-    console.log('File summaries:', fileSummariesArray); // gpt_pilot_debugging_log
-    
+    console.log('File summaries:', fileSummariesArray);
+
     const combinedSummaries = fileSummariesArray.join(' ');
+
+    // Convert allFiles array to string, where each file path is separated by a new line
+    const allFilesString = allFiles.join('\n');
+    console.log(`All files as string: ${allFilesString}`); // gpt_pilot_debugging_log
+    
+    // Now, combine the individual file summaries and the allFilesString
+    const combinedSummariesWithPaths = `${combinedSummaries}\n\n${allFilesString}`;
+
+    // Update the projectSummaryResponse OpenAI call:
     const projectSummaryResponse = await openai.chat.completions.create({
       model: 'gpt-4-turbo-preview',
-      messages: [{ role: "system", content: "Summarize this project based on the individual file summaries." }, { role: "user", content: combinedSummaries }],
-      max_tokens: 1024,
+      messages: [{ role: "system", content: "Summarize this project based on the individual file summaries and the list of all file paths." }, { role: "user", content: combinedSummariesWithPaths }],
+      max_tokens: 2048,
       temperature: 0.5
+    }).catch(error => {
+      console.error('Error during OpenAI project summary call with all file paths:', error.message, error.stack); // gpt_pilot_debugging_log
+      throw error;
     });
-
+    console.log(`Project summary with all file paths has been generated.`); // gpt_pilot_debugging_log
+    
     const projectSummary = projectSummaryResponse.choices[0].message.content.trim();
     const updatedRepository = await Repository.findOneAndUpdate(
       { githubUrl, email }, 

Diff for: routes.js

@@ -21,33 +21,32 @@ router.post('/submit', async (req, res) => {
   }
   try {
     const repoResponse = await axios.get(githubUrl.replace('https://github.com', 'https://api.github.com/repos'));
-    if (!repoResponse.data.private && repoResponse.data.size <= 500) {
-      const existingRepo = await Repository.findOne({ githubUrl });
-      if (existingRepo) {
-        if (existingRepo.isProcessed) {
-          return res.redirect(`/explain/${existingRepo.uuid}`);
-        } else {
-          return res.status(200).send('Repository is currently being processed.');
-        }
-      }
-      const newRepo = new Repository({ githubUrl, email });
-      await newRepo.save();
-      console.log(`New repository saved with URL: ${githubUrl}`);
-      
-      try {
-        await processRepository(newRepo.githubUrl, newRepo.email);
-        console.log(`Processing started for repository: ${newRepo.githubUrl}`);
-      } catch (error) {
-        console.error(`Error during the processing of the repository: ${error.message}`, error.stack);
-        return res.status(500).send(`Error during the processing of the repository: ${error.message}`);
-      }
-
-      res.status(201).send('Repository processing started. You will receive an email when it is complete.');
+    if (repoResponse.data.private) {
+      console.error('Repository is private:', githubUrl);
+      return res.status(400).send('Repository is private or does not exist.');
+    } else if (repoResponse.data.size === 0) {
+      console.log(`Repository ${githubUrl} is empty and cannot be processed.`);
+      return res.status(400).send('The repository is empty and cannot be processed.');
     } else if (repoResponse.data.size > 500) {
-      res.status(400).send('Repository has more than 500 files and cannot be processed.');
-    } else {
-      res.status(400).send('Repository is private or does not exist.');
+      console.error('Repository has more than 500 files:', githubUrl);
+      return res.status(400).send('Repository has more than 500 files and cannot be processed.');
+    }
+    const existingRepo = await Repository.findOne({ githubUrl });
+    if (existingRepo) {
+      if (existingRepo.isProcessed) {
+        console.log(`Redirecting to existing processed repository: ${existingRepo.uuid}`);
+        return res.redirect(`/explain/${existingRepo.uuid}`);
+      } else {
+        console.log(`Repository is currently being processed: ${githubUrl}`);
+        return res.status(200).send('Repository is currently being processed.');
+      }
     }
+    const newRepo = new Repository({ githubUrl, email });
+    await newRepo.save();
+    console.log(`New repository saved with URL: ${githubUrl}`);
+    await processRepository(newRepo.githubUrl, newRepo.email);
+    console.log(`Processing started for repository: ${newRepo.githubUrl}`);
+    res.status(201).send('Repository processing started. You will receive an email when it is complete.');
   } catch (error) {
     console.error('Error handling POST /submit:', error.message, error.stack);
     res.status(400).send('Error checking repository. Make sure the URL is correct and the repository is public.');

Diff for: tests/gitHandler.test.js

@@ -0,0 +1,47 @@
+const fs = require('fs-extra');
+const { isText } = require('../gitHandler'); // Adjust the import statement to destructure isText directly
+
+describe('isText function', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('should consider a file with text contents as a text file', async () => {
+    jest.spyOn(fs, 'readFile').mockResolvedValue('This is a text file');
+    const filename = 'test.txt';
+    // try-catch is unnecessary here because jest will catch rejections for us
+    const result = await isText(filename);
+    expect(result).toBe(true);
+  });
+
+  it('should not consider a binary file as a text file', async () => {
+    jest.spyOn(fs, 'readFile').mockRejectedValue(new Error('Binary file'));
+    const filename = 'binaryfile.bin';
+    // try-catch is unnecessary here because jest will catch rejections for us
+    const result = await isText(filename);
+    expect(result).toBe(false);
+  });
+
+  it('should properly read files regardless of extension', async () => {
+    jest.spyOn(fs, 'readFile')
+      .mockResolvedValueOnce('Normal text')
+      .mockRejectedValueOnce(new Error('Binary content'));
+    const textFilename = 'file.with.unknownext';
+    const binaryFilename = 'image.jpg';
+    // try-catch is unnecessary here because jest will catch rejections for us
+    const textResult = await isText(textFilename);
+    const binaryResult = await isText(binaryFilename);
+    expect(textResult).toBe(true);
+    expect(binaryResult).toBe(false);
+  });
+
+  it('should handle encoding-related read errors gracefully', async () => {
+    const error = new Error('EncodingError');
+    error.code = 'ENOENT';
+    jest.spyOn(fs, 'readFile').mockRejectedValue(error);
+    const filename = 'fileWithEncodingIssues.txt';
+    // try-catch is unnecessary here because jest will catch rejections for us
+    const result = await isText(filename);
+    expect(result).toBe(false);
+  });
+});