push_data.py

import os
import sys
import json

from dotenv import load_dotenv

load_dotenv()

MONGO_DB_URI = os.getenv("MONGO_DB_URI")    

import certifi

ca = certifi.where()

import pandas as pd
import numpy as np
import pymongo
from src.exception.exception import NetworkSecurityException
from src.logging.logger import logging

class NetworkDataExtract:
    def __init__(self):
        try:
            pass
        except Exception as e:
            raise NetworkSecurityException(e, sys)
    
    def cv_to_json_converter(self, file_path):
        try:
            data = pd.read_csv(file_path)
            data.reset_index(drop=True,inplace=True)
            records = list(json.loads(data.T.to_json()).values())
            return records
            
        except Exception as e:
            logging.error(e)
            raise NetworkSecurityException(e, sys)
        
    def insert_data_mongodb(self, records, database, collection):
        try:
            self.database = database
            self.collection = collection
            self.records = records
            
            self.mongo_client = pymongo.MongoClient(MONGO_DB_URI, tlsCAFile=ca)
            
            self.database = self.mongo_client[self.database]
            self.collection = self.database[self.collection]
            self.collection.insert_many(self.records)
            return(len(self.records))
        
        except Exception as e:
            logging.error(e)
            raise NetworkSecurityException(e, sys)
        
        
if __name__ == "__main__":
    FILE_PATH = "dataset\phisingData.csv"
    DATABASE = "ANIMESHDB"
    COLLECTION = "NETWORKDATA"
    networkobj = NetworkDataExtract()
    RECORDS = networkobj.cv_to_json_converter(file_path=FILE_PATH)
    no_of_records = networkobj.insert_data_mongodb(RECORDS, DATABASE, COLLECTION)
    print(f"Total {no_of_records} records inserted in MongoDB")