New Objective Metrics to Version 1.3-1

requirements.txt

@@ -8,7 +8,7 @@ gunicorn==19.10.0
 itsdangerous==2.0.1
 matplotlib==3.4.1
 multi-model-server==1.1.2
-numpy==1.21.0
+numpy==1.21.6
 pandas==1.2.4
 protobuf==3.20.1
 psutil==5.6.7  # sagemaker-containers requires psutil 5.6.7

src/sagemaker_xgboost_container/metrics/custom_metrics.py

@@ -12,7 +12,8 @@
 # language governing permissions and limitations under the License.
 import numpy as np
 from sklearn.metrics import f1_score, mean_squared_error, accuracy_score, \
-    precision_score, r2_score, recall_score, balanced_accuracy_score
+    precision_score, r2_score, \
+    recall_score, balanced_accuracy_score, mean_absolute_error
 
 
 # From 1.2, custom evaluation metric receives raw prediction.
@@ -93,6 +94,17 @@ def f1_macro(preds, dtrain):
                                                              f1_score(x, y, average='macro'), preds, dtrain)
 
 
+def mae(preds, dtrain):
+    """Compute mean absolute error.
+    For more information see: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html
+    :param preds: Prediction values
+    :param dtrain: Training data with labels
+    :return: Metric name, mean absolute error
+    """
+    labels = dtrain.get_label()
+    return 'mae', mean_absolute_error(labels, preds)
+
+
 def mse(preds, dtrain):
     """Compute mean squared error.
 
@@ -105,6 +117,17 @@ def mse(preds, dtrain):
     return 'mse', mean_squared_error(labels, preds)
 
 
+def rmse(preds, dtrain):
+    """Compute mean squared error.
+    For more information see: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html
+    :param preds: Prediction values
+    :param dtrain: Training data with labels
+    :return: Metric name, root mean squared error
+    """
+    labels = dtrain.get_label()
+    return 'rmse', mean_squared_error(labels, preds, squared=False)
+
+
 def precision(preds, dtrain):
     """Compute precision.
 
@@ -203,6 +226,8 @@ def compute_multiclass_and_binary_metrics(metricfunc, preds, dtrain):
     "f1_binary": f1_binary,
     "f1_macro": f1_macro,
     "mse": mse,
+    "rmse": rmse,
+    "mae": mae,
     "precision": precision,
     "precision_macro": precision_macro,
     "precision_micro": precision_micro,

test/resources/versions/train.py

@@ -15,7 +15,7 @@
 gunicorn==19.10.0
 matplotlib==3.4.1
 multi-model-server==1.1.2
-numpy==1.21.0
+numpy==1.21.6
 pandas==1.2.4
 psutil==5.6.7
 pyarrow==1.0.1

test/unit/algorithm_mode/test_custom_metrics.py

@@ -12,9 +12,10 @@
 # language governing permissions and limitations under the License.
 import numpy as np
 import xgboost as xgb
-from math import log
+from math import log, sqrt
 from sagemaker_xgboost_container.metrics.custom_metrics import accuracy, f1, mse, r2, f1_binary, f1_macro, \
-    precision_macro, precision_micro, recall_macro, recall_micro
+    precision_macro, precision_micro, recall_macro, recall_micro, mae, rmse, balanced_accuracy, \
+    precision, recall
 
 
 binary_train_data = np.random.rand(10, 2)
@@ -31,6 +32,12 @@ def test_binary_accuracy():
     assert accuracy_result == .5
 
 
+def test_binary_balanced_accuracy():
+    bal_accuracy_name, bal_accuracy_result = balanced_accuracy(binary_preds, binary_dtrain)
+    assert bal_accuracy_name == 'balanced_accuracy'
+    assert bal_accuracy_result == .5
+
+
 def test_binary_accuracy_logistic():
     accuracy_name, accuracy_result = accuracy(binary_preds_logistic, binary_dtrain)
     assert accuracy_name == 'accuracy'
@@ -61,6 +68,18 @@ def test_binary_f1_binary_logistic():
     assert f1_score_result == 2/3
 
 
+def test_binary_precision():
+    precision_score_name, precision_score_result = precision(binary_preds, binary_dtrain)
+    assert precision_score_name == 'precision'
+    assert precision_score_result == .5
+
+
+def test_binary_recall():
+    recall_score_name, recall_score_result = recall(binary_preds, binary_dtrain)
+    assert recall_score_name == 'recall'
+    assert recall_score_result == 1
+
+
 multiclass_train_data = np.random.rand(10, 2)
 multiclass_train_label = np.array([0, 0, 1, 1, 1, 1, 1, 2, 2, 2])
 multiclass_dtrain = xgb.DMatrix(multiclass_train_data, label=multiclass_train_label)
@@ -83,6 +102,12 @@ def test_multiclass_accuracy():
     assert accuracy_result == .5
 
 
+def test_multiclass_balanced_accuracy():
+    bal_accuracy_name, bal_accuracy_result = balanced_accuracy(multiclass_preds, multiclass_dtrain)
+    assert bal_accuracy_name == 'balanced_accuracy'
+    assert bal_accuracy_result == 1/3
+
+
 def test_multiclass_accuracy_softprob():
     accuracy_name, accuracy_result = accuracy(multiclass_preds_softprob, multiclass_dtrain)
     assert accuracy_name == 'accuracy'
@@ -153,3 +178,15 @@ def test_r2():
     r2_score_name, r2_score_result = r2(regression_preds, regression_dtrain)
     assert r2_score_name == 'r2'
     assert r2_score_result == -1
+
+
+def test_rmse():
+    rmse_score_name, rmse_score_result = rmse(regression_preds, regression_dtrain)
+    assert rmse_score_name == 'rmse'
+    assert rmse_score_result == sqrt(0.5)
+
+
+def test_mae():
+    mae_score_name, mae_score_result = mae(regression_preds, regression_dtrain)
+    assert mae_score_name == 'mae'
+    assert mae_score_result == .5

test/unit/algorithm_mode/test_train_utils.py

@@ -17,7 +17,7 @@
 import os
 import tempfile
 import shutil
-
+import math
 
 from sagemaker_xgboost_container.algorithm_mode import train_utils
 
@@ -39,16 +39,21 @@ def test_get_eval_metrics_and_feval():
     test_eval_metrics, test_configured_eval, tuning_metric = train_utils.get_eval_metrics_and_feval(test_objective,
                                                                                                     test_evals)
 
-    assert len(test_eval_metrics) == 2
+    assert len(test_eval_metrics) == 1
     for metric in test_eval_metrics:
-        assert metric in ['logloss', 'rmse']
+        assert metric in ['logloss']
 
     binary_train_data = np.random.rand(10, 2)
     binary_train_label = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
     binary_dtrain = xgb.DMatrix(binary_train_data, label=binary_train_label)
     binary_preds = np.ones(10)
 
-    assert ('accuracy', .5) == test_configured_eval(binary_preds, binary_dtrain)[0]
+    custom_metric_results = test_configured_eval(binary_preds, binary_dtrain)
+    custom_metric_results.sort()
+
+    assert 2 == len(custom_metric_results)
+    assert ('accuracy', .5) == custom_metric_results[0]
+    assert ('rmse', math.sqrt(0.5)) == custom_metric_results[1]
 
 
 def test_cleanup_dir():