implement data extraction and insertion to MongoDB with error handling

Author: AnimeshBasak-14

dataset/_init_.py renamed to logs/01_25_2025_21_22_07.log/01_25_2025_21_22_07.log

@@ -1,14 +1,64 @@
+import os
+import sys
+import json
 
-from pymongo.mongo_client import MongoClient
+from dotenv import load_dotenv
 
-uri = "mongodb+srv://basakanimesh16:[email protected]/?retryWrites=true&w=majority&appName=Cluster0"
+load_dotenv()
 
-# Create a new client and connect to the server
-client = MongoClient(uri)
+MONGO_DB_URI = os.getenv("MONGO_DB_URI")    
 
-# Send a ping to confirm a successful connection
-try:
-    client.admin.command('ping')
-    print("Pinged your deployment. You successfully connected to MongoDB!")
-except Exception as e:
-    print(e)
+import certifi
+
+ca = certifi.where()
+
+import pandas as pd
+import numpy as np
+import pymongo
+from src.exception.exception import NetworkSecurityException
+from src.logging.logger import logging
+
+class NetworkDataExtract:
+    def __init__(self):
+        try:
+            pass
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+    
+    def cv_to_json_converter(self, file_path):
+        try:
+            data = pd.read_csv(file_path)
+            data.reset_index(drop=True,inplace=True)
+            records = list(json.loads(data.T.to_json()).values())
+            return records
+            
+        except Exception as e:
+            logging.error(e)
+            raise NetworkSecurityException(e, sys)
+        
+    def insert_data_mongodb(self, records, database, collection):
+        try:
+            self.database = database
+            self.collection = collection
+            self.records = records
+            
+            self.mongo_client = pymongo.MongoClient(MONGO_DB_URI, tlsCAFile=ca)
+            
+            self.database = self.mongo_client[self.database]
+            self.collection = self.database[self.collection]
+            self.collection.insert_many(self.records)
+            return(len(self.records))
+        
+        except Exception as e:
+            logging.error(e)
+            raise NetworkSecurityException(e, sys)
+        
+        
+if __name__ == "__main__":
+    FILE_PATH = "dataset\phisingData.csv"
+    DATABASE = "ANIMESHDB"
+    COLLECTION = "NETWORKDATA"
+    networkobj = NetworkDataExtract()
+    RECORDS = networkobj.cv_to_json_converter(file_path=FILE_PATH)
+    no_of_records = networkobj.insert_data_mongodb(RECORDS, DATABASE, COLLECTION)
+    print(f"Total {no_of_records} records inserted in MongoDB")

logs/01_25_2025_21_33_44.log/01_25_2025_21_33_44.log

@@ -0,0 +1,21 @@
+
+from pymongo.mongo_client import MongoClient
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+MONGO_DB_URI = os.getenv("MONGO_DB_URI")    
+
+
+uri = MONGO_DB_URI
+
+# Create a new client and connect to the server
+client = MongoClient(uri)
+
+# Send a ping to confirm a successful connection
+try:
+    client.admin.command('ping')
+    print("Pinged your deployment. You successfully connected to MongoDB!")
+except Exception as e:
+    print(e)