remove undersampling, transform copy of dataframe

Author: watsang

Makefile

@@ -8,6 +8,7 @@
 NO_OF_TEST_FILES := $(words $(wildcard tests/test_*.py))
 NO_OF_REPORT_FILES := $(words $(wildcard reports/))
 NO_OF_REPORT_FILES := $(words $(filter-out reports/.gitkeep, $(SRC_FILES)))
+TEST_CSV := ./data/transformed/test_balanced_creditcard.csv
 
 ###############################################################
 # COMMANDS                                                    #
@@ -39,10 +40,12 @@ linting:
 
 test-package:
 	@echo ">>> running coverage pytest"
-	coverage run -m pytest ./tests/
+	coverage run -m pytest ./tests/test_data.py ./tests/test_generate_data.py ./tests/test_train.py ./tests/test_predict.py
 	coverage report -m --include=./tests/*
 
 test: generate-dataset train prediction clean test-package ## run extensive tests
 
 help: ## show help on available commands
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+

ml_skeleton_py/etl/generate_dataset.py

@@ -31,12 +31,13 @@ def remove_outliers(df: pd.DataFrame, params: dict) -> pd.DataFrame:
     Return:
         df (pd.DataFrame): dataframe with removed outliers
     """
+    df_dropped = df.copy(deep=True)
     for variable in ["V10", "V12", "V14"]:
-        upper_outliers = df[variable] > params[f"{variable}_upper"]
-        lower_outliers = df[variable] < params[f"{variable}_lower"]
-        df = df.drop(df[upper_outliers | lower_outliers].index)
-    logger.info(f"Number of Instances after outliers removal: {len(df)}")
-    return df
+        upper_outliers = df_dropped[variable] > params[f"{variable}_upper"]
+        lower_outliers = df_dropped[variable] < params[f"{variable}_lower"]
+        df_dropped = df_dropped.drop(df_dropped[upper_outliers | lower_outliers].index)
+    logger.info(f"Number of Instances after outliers removal: {len(df_dropped)}")
+    return df_dropped
 
 
 def generate(dataset: str) -> Optional[pd.DataFrame]:

ml_skeleton_py/model/predict.py

@@ -37,7 +37,7 @@ def load_model(model_name: str) -> BaseEstimator:
         model = pickle.load(handle)["model"]
     return model
 
-
+# @dploy endpoint predict
 def predict(observation: np.array, model_name: str = "lr.p") -> float:
     """
     Predict one single observation.

ml_skeleton_py/model/train.py

@@ -20,7 +20,7 @@
 logging.getLogger().setLevel(logging.INFO)
 
 
-def train(model_name: str, dataset: str) -> None:
+def train(dataset: str, model_name: str = "lr") -> None:
     """
     Train models using X_train and y_train with a specific classifier.
 
@@ -46,25 +46,24 @@ def train(model_name: str, dataset: str) -> None:
 
     # preprocessing
     scaler = RobustScaler()
-    X = scaler.fit_transform(X)
-    rus = RandomUnderSampler(replacement=False)
-    X, y = rus.fit_resample(X, y)
 
     # In this specific example logistic regression was chosen as
     # the most optimal model after running several experiments.
     classifier = LogisticRegression(max_iter=4000, penalty="l2", C=0.01)
 
+    # create pipeline
+    predict_pipeline = make_pipeline(scaler, classifier)
+
     # training
-    classifier.fit(X, y)
-    training_score = cross_val_score(classifier, X, y, cv=5, scoring="roc_auc")
-    logger.info(f"Classifier: {classifier.__class__.__name__}")
+    predict_pipeline.fit(X, y)
+    training_score = cross_val_score(predict_pipeline, X, y, cv=5, scoring="roc_auc")
+    logger.info(f"Classifier: {predict_pipeline.__class__.__name__}")
     logger.info(
         "Has a training score "
         + f"of {round(training_score.mean(), 2) * 100} % roc_auc"
     )
 
     # saving
-    predict_pipeline = make_pipeline(scaler, classifier)
     pred_result = {
         "clf": model_name,
         "training score roc_auc": training_score.mean(),

scripts/train.py

@@ -7,21 +7,21 @@
 @click.command()
 @click.option("--model_name", default="lr")
 @click.option("--dataset", default="creditcard.csv")
-def train(model_name: str, dataset: str) -> None:
+def train(dataset: str, model_name: str) -> None:
     """
     Train a model on a dataset and store the model and its results.
 
     Parameters:
+        dataset (str): the dataset on which you want to train
+
         model_name (str): the model_name that you want to use as a save
                      default:
                         "lr": logistic regression
 
-        dataset (str): the dataset on which you want to train
-
     Returns:
         None
     """
-    model.train(model_name, dataset)
+    model.train(dataset, model_name)
 
 
 if __name__ == "__main__":

tests/test_predict.py

@@ -13,7 +13,7 @@ def test_predict_1() -> None:
     """
     Test whether an observation makes a prediction.
     """
-    train(MODEL_NAME, DATASET)
+    train(DATASET, MODEL_NAME)
     model_name = "lr_test.p"
     observation = [
         -0.51056756,

tests/test_train.py

@@ -12,7 +12,7 @@ def test_train_lr() -> None:
     """
     Test whether logistic regression is trained and can be loaded.
     """
-    train(MODEL_NAME, DATASET)
+    train(DATASET, MODEL_NAME)
     with open(os.path.join(s.MODEL_DIR, MODEL_NAME) + ".p", "rb") as handle:
         pred_result = pickle.load(handle)
     classifier = is_classifier(pred_result["model"])