optimize code

Author: zhaoyichanghong

README.md

@@ -150,7 +150,7 @@
     regularizer algorithm including L1, L2, elastic-net
 
 ## [metrics.py](metrics.py)
-    scores including accuracy, precision, recall, f-score, R2 score, confusion matrix, pr curve, roc curve, auc, silhouette coefficient, 2d feature scatter, learning curve
+    scores including accuracy, precision, recall, f-score, R2 score, confusion matrix, pr curve, roc curve, auc, silhouette coefficient, 2d feature scatter, learning curve, information value
 
 ## [optimizer.py](optimizer.py)
     optimizer algorithm including following components

metrics.py

@@ -266,4 +266,46 @@ def learning_curve(train_X, train_y, train_ratios, test_X, test_y, fit, accuracy
 
     plt.plot(accuracy_train, 'r')
     plt.plot(accuracy_test, 'b')
-    plt.show()
+    plt.show()
+
+def information_value(X, y):
+    '''
+    Parameters
+    ----------
+    X : shape (data_number, feature_number)
+              Training data
+    y : shape (data_number, 1)
+              Target label
+
+    Returns
+    -------
+    information values of each feature
+    '''
+    feature_number = X.shape[1]
+
+    n_positive_total = sum(y == 1)
+    n_negative_total = sum(y == 0)
+
+    ivs = []
+    for i in range(feature_number):
+        iv = 0
+        feature_labels = np.unique(X[:, i])
+        for feature_label in feature_labels:
+            indexes = np.flatnonzero(X[:, i] == feature_label)
+
+            n_positive = sum(y[indexes] == 1)
+            n_negative = sum(y[indexes] == 0)
+
+            p_positive = n_positive / n_positive_total
+            p_negative = n_negative / n_negative_total
+
+            iv += (p_positive - p_negative) * np.log(p_positive + 1e-8 / p_negative + 1e-8)
+        
+        ivs.append(iv)
+
+    return ivs
+
+            
+
+    
+    

softmax_regression.py

@@ -79,4 +79,5 @@ def score(self, X):
         y : shape (data_number, class_number)
             Predicted score of class per sample.
         '''
-        return scipy.special.softmax(X.dot(self.__W) + self.__b, axis=1)
+        out = X.dot(self.__W) + self.__b
+        return scipy.special.softmax(out - np.max(out), axis=1)