data validation added

Author: “vijayg15”

main.py

@@ -1,6 +1,13 @@
 import os
 from mlProject import logger
 from mlProject.pipeline.stage_01_data_ingestion import DataIngestionTrainingPipeline
+from mlProject.pipeline.stage_02_data_validation import DataValidationTrainingPipeline
+
+
+
+
+
+
 
 
 
@@ -10,6 +17,18 @@
    data_ingestion = DataIngestionTrainingPipeline()
    data_ingestion.main()
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e
+
+
+
+STAGE_NAME = "Data Validation stage"
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   data_ingestion = DataValidationTrainingPipeline()
+   data_ingestion.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
         raise e

schema.yaml

@@ -1 +1,19 @@
-key: value
+COLUMNS:
+  RowNumber: int64
+  CustomerId: int64
+  Surname: str
+  CreditScore: int64
+  Geography: str
+  Gender: str
+  Age: float64
+  Tenure: int64
+  Balance: float64
+  NumOfProducts: int64
+  HasCrCardl: float64
+  IsActiveMember: float64
+  EstimatedSalary: float64
+  Exited: int64
+
+
+TARGET_COLUMN:
+  name: Exited

src/mlProject/components/data_validation.py

@@ -0,0 +1,36 @@
+import os
+from mlProject import logger
+from mlProject.entity.config_entity import DataValidationConfig
+import pandas as pd
+
+
+class DataValiadtion:
+    def __init__(self, config: DataValidationConfig):
+        self.config = config
+
+
+    def validate_all_columns(self)-> bool:
+        try:
+            validation_status = None
+
+            data = pd.read_csv(self.config.unzip_data_dir)
+            all_cols = list(data.columns)
+
+            all_schema = self.config.all_schema.keys()
+
+            
+            for col in all_cols:
+                if col not in all_schema:
+                    validation_status = False
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+                else:
+                    validation_status = True
+                    with open(self.config.STATUS_FILE, 'w') as f:
+                        f.write(f"Validation status: {validation_status}")
+
+            return validation_status
+        
+        except Exception as e:
+            raise e
+

src/mlProject/config/configuration.py

@@ -1,8 +1,11 @@
 from mlProject.constants import *
 from mlProject.utils.common import read_yaml, create_directories
-from mlProject.entity.config_entity import (DataIngestionConfig, 
+from mlProject.entity.config_entity import (DataIngestionConfig,
+                                            DataValidationConfig 
                                             )
 
+
+
 class ConfigurationManager:
     def __init__(
         self,
@@ -30,4 +33,20 @@ def get_data_ingestion_config(self) -> DataIngestionConfig:
             unzip_dir=config.unzip_dir 
         )
 
-        return data_ingestion_config
+        return data_ingestion_config
+    
+
+    def get_data_validation_config(self) -> DataValidationConfig:
+        config = self.config.data_validation
+        schema = self.schema.COLUMNS
+
+        create_directories([config.root_dir])
+
+        data_validation_config = DataValidationConfig(
+            root_dir=config.root_dir,
+            STATUS_FILE=config.STATUS_FILE,
+            unzip_data_dir = config.unzip_data_dir,
+            all_schema=schema,
+            )
+
+        return data_validation_config

src/mlProject/entity/config_entity.py

@@ -7,4 +7,13 @@ class DataIngestionConfig:
     root_dir: Path
     source_URL: str
     local_data_file: Path
-    unzip_dir: Path
+    unzip_dir: Path
+
+
+
+@dataclass(frozen=True)
+class DataValidationConfig:
+    root_dir: Path
+    STATUS_FILE: str
+    unzip_data_dir: Path
+    all_schema: dict

src/mlProject/pipeline/stage_02_data_validation.py

@@ -0,0 +1,31 @@
+from mlProject.config.configuration import ConfigurationManager
+from mlProject.components.data_validation import DataValiadtion
+from mlProject import logger
+
+
+STAGE_NAME = "Data Validation stage"
+
+class DataValidationTrainingPipeline:
+    def __init__(self):
+        pass
+
+    def main(self):
+        config = ConfigurationManager()
+        data_validation_config = config.get_data_validation_config()
+        data_validation = DataValiadtion(config=data_validation_config)
+        data_validation.validate_all_columns()
+
+
+
+
+
+if __name__ == '__main__':
+    try:
+        logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<")
+        obj = DataValidationTrainingPipeline()
+        obj.main()
+        logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+    except Exception as e:
+        logger.exception(e)
+        raise e
+