optimize code

Author: zhaoyichanghong

decision_tree_id3.py

@@ -1,79 +1,79 @@
-import numpy as np
-import treelib
-import scipy.stats
-
-class ID3():
-    def __init__(self):
-        self.__tree = treelib.Tree()
-
-    def __get_entropy(self, y):
-        _, counts = np.unique(y, return_counts=True)
-        prob_classes = counts / np.sum(counts)
-        return scipy.stats.entropy(prob_classes)
-
-    def __create_tree(self, parent, X, y):
-        data_number, feature_number = X.shape
-
-        if data_number == 0:
-            return
-
-        if len(np.unique(y)) == 1 or (X == X[0]).all():
-            self.__tree.update_node(parent.identifier, data=max(set(y), key=y.tolist().count))
-            return
-
-        info_gain_max = -np.inf
-        for i in range(feature_number):
-            if len(np.unique(X[:, i])) == 1:
-                continue
-
-            y_subs = [y[np.flatnonzero(X[:, i] == feature_label)] for feature_label in np.unique(X[:, i])]
-
-            info_gain = self.__get_info_gain(y_subs, y)
-
-            if info_gain > info_gain_max:
-                info_gain_max = info_gain
-                feature_split = i
-
-        self.__tree.update_node(parent.identifier, data=feature_split)
-        for feature_label in np.unique(X[:, feature_split]):
-            node = self.__tree.create_node(feature_label, parent=parent)
-            self.__create_tree(node, X[np.flatnonzero(X[:, feature_split] == feature_label)], y[np.flatnonzero(X[:, feature_split] == feature_label)])
-
-    def __get_info_gain(self, y_subs, y):
-        return self.__get_entropy(y) - sum([self.__get_entropy(y_sub) * len(y_sub) for y_sub in y_subs]) / len(y)
-
-    def fit(self, X, y):
-        '''
-        Parameters
-        ----------
-        X : shape (data_number, feature_number)
-            Training data
-        y : shape (data_number)
-            Target values, discrete value
-        '''
-        root = self.__tree.create_node('root')
-        self.__create_tree(root, X, y)
-        self.__tree.show()
-
-    def __query(self, x, node):
-        if node.is_leaf():
-            return node.data
-
-        feature_split = node.data
-        for child in self.__tree.children(node.identifier):
-            if x[feature_split] == child.tag:
-                return self.__query(x, child)
-
-    def predict(self, X):
-        '''
-        Parameters
-        ----------
-        X : shape (data_number, feature_number)
-            Predicting data
-
-        Returns
-        -------
-        y : shape (data_number,)
-            Predicted class label per sample
-        '''
+import numpy as np
+import treelib
+import scipy.stats
+
+class ID3():
+    def __init__(self):
+        self.__tree = treelib.Tree()
+
+    def __get_entropy(self, y):
+        _, counts = np.unique(y, return_counts=True)
+        prob_classes = counts / np.sum(counts)
+        return scipy.stats.entropy(prob_classes)
+
+    def __create_tree(self, parent, X, y):
+        data_number, feature_number = X.shape
+
+        if data_number == 0:
+            return
+
+        if len(np.unique(y)) == 1 or (X == X[0]).all():
+            self.__tree.update_node(parent.identifier, data=max(set(y), key=y.tolist().count))
+            return
+
+        info_gain_max = -np.inf
+        for i in range(feature_number):
+            if len(np.unique(X[:, i])) == 1:
+                continue
+
+            y_subs = [y[np.flatnonzero(X[:, i] == feature_label)] for feature_label in np.unique(X[:, i])]
+
+            info_gain = self.__get_info_gain(y_subs, y)
+
+            if info_gain > info_gain_max:
+                info_gain_max = info_gain
+                feature_split = i
+
+        self.__tree.update_node(parent.identifier, data=feature_split)
+        for feature_label in np.unique(X[:, feature_split]):
+            node = self.__tree.create_node(feature_label, parent=parent)
+            self.__create_tree(node, X[np.flatnonzero(X[:, feature_split] == feature_label)], y[np.flatnonzero(X[:, feature_split] == feature_label)])
+
+    def __get_info_gain(self, y_subs, y):
+        return self.__get_entropy(y) - sum([self.__get_entropy(y_sub) * len(y_sub) for y_sub in y_subs]) / len(y)
+
+    def fit(self, X, y):
+        '''
+        Parameters
+        ----------
+        X : shape (data_number, feature_number)
+            Training data, must be discrete value
+        y : shape (data_number)
+            Target values
+        '''
+        root = self.__tree.create_node('root')
+        self.__create_tree(root, X, y)
+        self.__tree.show()
+
+    def __query(self, x, node):
+        if node.is_leaf():
+            return node.data
+
+        feature_split = node.data
+        for child in self.__tree.children(node.identifier):
+            if x[feature_split] == child.tag:
+                return self.__query(x, child)
+
+    def predict(self, X):
+        '''
+        Parameters
+        ----------
+        X : shape (data_number, feature_number)
+            Predicting data, must be discrete value
+
+        Returns
+        -------
+        y : shape (data_number,)
+            Predicted class label per sample
+        '''
         return np.apply_along_axis(self.__query, 1, X, self.__tree.get_node(self.__tree.root))

metrics.py

@@ -177,9 +177,9 @@ def r2_score(y_true, y_pred):
     Parameters
     ----------
     y_true : shape (data_number, 1)
-             True label
+             True value
     y_pred : shape (data_number, 1)
-             Predicting label
+             Predicting value
     
     Returns
     -------

preprocess.py

@@ -62,7 +62,7 @@ def transform(self, X):
         X : shape (data_number, feature_number)
             The Predicting data standard scaler encoded.
         '''
-        return (X - self.__mean) / self.__std
+        return (X - self.__mean) / (self.__std + 1e-8)
 
 class OneHot:
     @property

random_forest.py

@@ -10,15 +10,25 @@ def __init__(self, mode='classification', debug=True):
         self.__mode = mode
         self.__debug = debug
 
-    def fit(self, X, y, trees_number, pick_feature_number):
+    def fit(self, X, y, n_trees, pick_feature_number):
+        '''
+        Parameters
+        ----------
+        X : shape (data_number, feature_number)
+            Training data
+        y : shape (data_number, 1)
+            Target values, 1 or 0
+        n_trees : The number of trees in the forest.
+        pick_feature_number : The number of features picked randomly
+        '''
         data_number, feature_number = X.shape
 
-        self.__indexs, self.__indexs_oob = preprocess.bagging(data_number, trees_number)
+        self.__indexs, self.__indexs_oob = preprocess.bagging(data_number, n_trees)
 
         if self.__debug:
             accuracy = []
 
-        for i in range(trees_number):
+        for i in range(n_trees):
             features = np.random.choice(feature_number, pick_feature_number, replace=False)
 
             X_bag = X[self.__indexs[i]][:, features]
@@ -38,10 +48,10 @@ def fit(self, X, y, trees_number, pick_feature_number):
 
     def __oob_verification(self, X, y):
         data_number = X.shape[0]
-        trees_number = len(self.__trees)
+        n_trees = len(self.__trees)
 
-        results = np.full((data_number, trees_number), None)
-        for i in range(trees_number):
+        results = np.full((data_number, n_trees), None)
+        for i in range(n_trees):
             tree = self.__trees[i]['model']
             features = self.__trees[i]['features']
             X_bag_oob = X[self.__indexs_oob[i]][:, features]
@@ -63,6 +73,17 @@ def __oob_verification(self, X, y):
             return metrics.accuracy(y, y_pred)
 
     def predict(self, X):
+        '''
+        Parameters
+        ----------
+        X : shape (data_number, feature_number)
+            Predicting data
+
+        Returns
+        -------
+        y : shape (data_number, 1)
+            Predicted value per sample
+        '''
         data_number = X.shape[0]
 
         results = np.empty((data_number, 0))