optimize code

Author: zhaoyichanghong

README.md

@@ -72,6 +72,9 @@
 ### [random_forest.py](random_forest.py)
     random_forest algorithm including bagging, random features, oob verification, feature selection
 
+### [gbdt.py](gbdt.py)
+    gradient boost decision tree algorithm
+
 ## Dimensionality Reduction
 
 ### [linear_discriminant_analysis.py](linear_discriminant_analysis.py)

gbdt.py

@@ -1,49 +1,54 @@
-import numpy as np
-import decision_tree
-
-class GBDT:
-    def __sigmoid(self, x):
-        return 1 / (1 + np.exp(-x))
-
-    def __softmax(self, x):
-        return np.exp(x) / np.sum(np.exp(x), axis=1, keepdims=True)
-
-    def __init__(self, loss):
-        self.__models = []
-        self.__alpha = []
-        self.__loss = loss
-
-    def fit(self, X, y, epochs, learning_rate):
-        self.__learning_rate = learning_rate
-
-        residual = y
-        for _ in range(epochs):
-            model = decision_tree.Cart('regression')
-            model.fit(X, residual)
-            self.__models.append(model)
-
-            alpha = np.mean(residual / (model.predict(X) + 1e-8), axis=0)
-            self.__alpha.append(alpha)
-
-            residual = y - self.score(X)
-
-    def predict(self, X, classes=None):
-        if self.__loss == 'mse':
-            return self.score(X)
-        elif self.__loss == 'binary_crossentropy':
-            return np.around(self.score(X))
-        elif self.__loss == 'categorical_crossentropy':
-            return classes[np.argmax(self.score(X), axis=1)].reshape((-1, 1))
-
-    def score(self, X):
-        h = 0
-        for alpha, model in zip(self.__alpha, self.__models):
-            h += self.__learning_rate * model.predict(X) * alpha
-
-        if self.__loss == 'mse':
-            return h
-        elif self.__loss == 'binary_crossentropy':
-            return self.__sigmoid(h)
-        elif self.__loss == 'categorical_crossentropy':
-            return self.__softmax(h)
+import numpy as np
+import decision_tree_cart
+import scipy
+
+class GBDT:
+    def __init__(self, loss):
+        self.__models = []
+        self.__alpha = []
+        self.__loss = loss
+
+    def fit(self, X, y, epochs, learning_rate):
+        '''
+        Parameters
+        ----------
+        X : shape (n_samples, n_features)
+            Training data
+        y : shape (n_samples,)
+            Target values 
+        epochs : The number of epochs
+        learning_rate : Learning rate
+        '''
+        self.__learning_rate = learning_rate
+
+        residual = y
+        for _ in range(epochs):
+            model = decision_tree_cart.CART('regression')
+            model.fit(X, residual)
+            self.__models.append(model)
+
+            alpha = np.mean(residual / (model.predict(X) + 1e-8), axis=0)
+            self.__alpha.append(alpha)
+
+            residual = y - self.score(X)
+
+    def predict(self, X):
+        '''
+        Parameters
+        ----------
+        X : shape (n_samples, n_features)
+            Predicting data
+
+        Returns
+        -------
+        y : shape (n_samples,)
+            Predicted value per sample.
+        '''
+        if self.__loss == 'regression':
+            return self.score(X)
+        elif self.__loss == 'classification':
+            return np.around(self.score(X))
+
+    def score(self, X):
+        return self.__learning_rate * sum([model.predict(X) * alpha for alpha, model in zip(self.__alpha, self.__models)])
 

support_vector_data_description.py

@@ -34,7 +34,10 @@ def __qp(self, X, kernel, C):
         self.__a_support = alpha[support_items]
 
         free_items = np.flatnonzero(self.__a_support < C)
-        self.__X_free = self.__X_support[free_items]
+        X_free = self.__X_support[free_items]
+        
+        self.__center = self.__a_support.dot(self.__X_support)
+        self.__radius = np.mean(distance.euclidean_distance(self.__center, X_free))
 
     def fit(self, X, kernel_func, C, sigma=1):
         '''
@@ -51,8 +54,6 @@ def fit(self, X, kernel_func, C, sigma=1):
 
         kernel = self.__kernel_func(X, X, self.__sigma)
         self.__qp(X, kernel, C)
-        self.__center = self.__a_support.dot(self.__X_support)
-        self.__radius = np.mean(distance.euclidean_distance(self.__center, self.__X_free))
 
     def predict(self, X):
         '''

svm.py

@@ -26,7 +26,6 @@ def __qp(self, X, y, kernel, C):
         self.__a_support = alpha[support_items]
 
         free_items = np.flatnonzero(self.__a_support < C)
-        X_free = X[free_items]
         y_free = y[free_items]
 
         self.__bias = y_free[0] - (self.__a_support * self.__y_support).T.dot(kernel[support_items, free_items[0]])