python 3

Author: lazyprogrammer

supervised_class2/adaboost.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.tree import DecisionTreeClassifier
@@ -17,7 +23,7 @@ def fit(self, X, Y):
     N, _ = X.shape
     W = np.ones(N) / N
 
-    for m in xrange(self.M):
+    for m in range(self.M):
       tree = DecisionTreeClassifier(max_depth=1)
       tree.fit(X, Y, sample_weight=W)
       P = tree.predict(X)
@@ -60,14 +66,14 @@ def score(self, X, Y):
   train_errors = np.empty(T)
   test_losses = np.empty(T)
   test_errors = np.empty(T)
-  for num_trees in xrange(T):
+  for num_trees in range(T):
     if num_trees == 0:
       train_errors[num_trees] = None
       test_errors[num_trees] = None
       test_losses[num_trees] = None
       continue
     if num_trees % 20 == 0:
-      print num_trees
+      print(num_trees)
 
     model = AdaBoost(num_trees)
     model.fit(Xtrain, Ytrain)
@@ -78,8 +84,8 @@ def score(self, X, Y):
     test_losses[num_trees] = loss
 
     if num_trees == T - 1:
-      print "final train error:", 1 - acc_train
-      print "final test error:", 1 - acc
+      print("final train error:", 1 - acc_train)
+      print("final test error:", 1 - acc)
 
   plt.plot(test_errors, label='test errors')
   plt.plot(test_losses, label='test losses')

supervised_class2/bagging_classification.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.tree import DecisionTreeClassifier
@@ -34,7 +40,7 @@
 # lone decision tree
 model = DecisionTreeClassifier()
 model.fit(X, Y)
-print "score for 1 tree:", model.score(X, Y)
+print("score for 1 tree:", model.score(X, Y))
 
 # plot data with boundary
 plt.scatter(X[:,0], X[:,1], s=100, c=Y, alpha=0.5)
@@ -50,7 +56,7 @@ def __init__(self, B):
   def fit(self, X, Y):
     N = len(X)
     self.models = []
-    for b in xrange(self.B):
+    for b in range(self.B):
       idx = np.random.choice(N, size=N, replace=True)
       Xb = X[idx]
       Yb = Y[idx]
@@ -74,7 +80,7 @@ def score(self, X, Y):
 model = BaggedTreeClassifier(200)
 model.fit(X, Y)
 
-print "score for bagged model:", model.score(X, Y)
+print("score for bagged model:", model.score(X, Y))
 
 # plot data with boundary
 plt.scatter(X[:,0], X[:,1], s=100, c=Y, alpha=0.5)

supervised_class2/bagging_regression.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.tree import DecisionTreeRegressor
@@ -21,7 +27,7 @@
 model = DecisionTreeRegressor()
 model.fit(Xtrain, Ytrain)
 prediction = model.predict(x_axis.reshape(T, 1))
-print "score for 1 tree:", model.score(x_axis.reshape(T, 1), y_axis)
+print("score for 1 tree:", model.score(x_axis.reshape(T, 1), y_axis))
 
 # plot the lone decision tree's predictions
 plt.plot(x_axis, prediction)
@@ -36,7 +42,7 @@ def __init__(self, B):
   def fit(self, X, Y):
     N = len(X)
     self.models = []
-    for b in xrange(self.B):
+    for b in range(self.B):
       idx = np.random.choice(N, size=N, replace=True)
       Xb = X[idx]
       Yb = Y[idx]
@@ -59,7 +65,7 @@ def score(self, X, Y):
 
 model = BaggedTreeRegressor(200)
 model.fit(Xtrain, Ytrain)
-print "score for bagged tree:", model.score(x_axis.reshape(T, 1), y_axis)
+print("score for bagged tree:", model.score(x_axis.reshape(T, 1), y_axis))
 prediction = model.predict(x_axis.reshape(T, 1))
 
 # plot the bagged regressor's predictions

supervised_class2/bias_variance_demo.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.linear_model import LinearRegression
@@ -17,7 +23,7 @@
 def make_poly(x, D):
   N = len(x)
   X = np.empty((N, D+1))
-  for d in xrange(D+1):
+  for d in range(D+1):
     X[:,d] = x**d
     if d > 1:
       X[:,d] = (X[:,d] - X[:,d].mean()) / X[:,d].std()
@@ -53,7 +59,7 @@ def f(X):
 # create the model
 model = LinearRegression()
 
-for k in xrange(NUM_DATASETS):
+for k in range(NUM_DATASETS):
   Y = f_X + np.random.randn(N)*NOISE_VARIANCE
 
   Xtrain = Xpoly[:Ntrain]
@@ -62,7 +68,7 @@ def f(X):
   Xtest = Xpoly[Ntrain:]
   Ytest = Y[Ntrain:]
 
-  for d in xrange(MAX_POLY):
+  for d in range(MAX_POLY):
     model.fit(Xtrain[:,:d+2], Ytrain)
     predictions = model.predict(Xpoly[:,:d+2])
 
@@ -87,8 +93,8 @@ def f(X):
 
 # show all prediction curves for each polynomial degree
 # along with the mean curve
-for d in xrange(MAX_POLY):
-  for k in xrange(NUM_DATASETS):
+for d in range(MAX_POLY):
+  for k in range(NUM_DATASETS):
     plt.plot(x_axis, prediction_curves[:,k,d], color='green', alpha=0.5)
   plt.plot(x_axis, prediction_curves[:,:,d].mean(axis=1), color='blue', linewidth=2.0)
   plt.title("All curves for degree = %d" % (d+1))
@@ -98,15 +104,15 @@ def f(X):
 avg_train_prediction = np.zeros((Ntrain, MAX_POLY))
 squared_bias = np.zeros(MAX_POLY)
 f_Xtrain = f_X[:Ntrain]
-for d in xrange(MAX_POLY):
-  for i in xrange(Ntrain):
+for d in range(MAX_POLY):
+  for i in range(Ntrain):
     avg_train_prediction[i,d] = train_predictions[i,:,d].mean()
   squared_bias[d] = ((avg_train_prediction[:,d] - f_Xtrain)**2).mean()
 
 # calculate the variance
 variances = np.zeros((Ntrain, MAX_POLY))
-for d in xrange(MAX_POLY):
-  for i in xrange(Ntrain):
+for d in range(MAX_POLY):
+  for i in range(Ntrain):
     delta = train_predictions[i,:,d] - avg_train_prediction[i,d]
     variances[i,d] = delta.dot(delta) / N
 variance = variances.mean(axis=0)

supervised_class2/bootstrap.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from scipy.stats import norm, t
@@ -8,10 +14,10 @@
 N = 20
 X = np.random.randn(N)
 
-print "sample mean of X:", X.mean()
+print("sample mean of X:", X.mean())
 
 individual_estimates = np.empty(B)
-for b in xrange(B):
+for b in range(B):
   sample = np.random.choice(X, size=N)
   individual_estimates[b] = sample.mean()
 
@@ -25,7 +31,7 @@
 lower2 = X.mean() + norm.ppf(0.025)*X.std()/np.sqrt(N)
 upper2 = X.mean() + norm.ppf(0.975)*X.std()/np.sqrt(N)
 
-print "bootstrap mean of X:", bmean
+print("bootstrap mean of X:", bmean)
 
 plt.hist(individual_estimates, bins=20)
 plt.axvline(x=lower, linestyle='--', color='g', label="lower bound for 95%% CI (bootstrap)")

supervised_class2/knn_dt_demo.py

@@ -1,5 +1,11 @@
 # https://deeplearningcourses.com/c/machine-learning-in-python-random-forest-adaboost
 # https://www.udemy.com/machine-learning-in-python-random-forest-adaboost
+from __future__ import print_function, division
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
@@ -27,6 +33,7 @@
 
 plt.scatter(Xtrain, Ytrain, s=50, alpha=0.7, c='blue')
 plt.scatter(Xtrain, model.predict(Xtrain.reshape(Ntrain, 1)), s=50, alpha=0.7, c='green')
+plt.title("decision tree - low bias, high variance")
 # plt.show()
 
 # plt.scatter(X, Y)
@@ -46,6 +53,7 @@
 plt.scatter(Xtrain, model.predict(Xtrain.reshape(Ntrain, 1)), s=50, alpha=0.7, c='green')
 plt.plot(Xaxis, Yaxis)
 plt.plot(Xaxis, model.predict(Xaxis.reshape(T, 1)))
+plt.title("decision tree - high bias, low variance")
 plt.show()
 
 
@@ -57,6 +65,7 @@
 plt.scatter(Xtrain, model.predict(Xtrain.reshape(Ntrain, 1)), s=50, alpha=0.7, c='green')
 plt.plot(Xaxis, Yaxis)
 plt.plot(Xaxis, model.predict(Xaxis.reshape(T, 1)))
+plt.title("knn - low bias, high variance")
 plt.show()
 
 # knn - high bias, low variance
@@ -67,6 +76,7 @@
 plt.scatter(Xtrain, model.predict(Xtrain.reshape(Ntrain, 1)), s=50, alpha=0.7, c='green')
 plt.plot(Xaxis, Yaxis)
 plt.plot(Xaxis, model.predict(Xaxis.reshape(T, 1)))
+plt.title("knn - high bias, low variance")
 plt.show()
 
 
@@ -76,10 +86,10 @@
 N = 100
 D = 2
 X = np.random.randn(N, D)
-X[:N/2] += np.array([1, 1]) # center it at (1,1)
-X[N/2:] += np.array([-1, -1]) # center it at (-1, -1)
+X[:N//2] += np.array([1, 1]) # center it at (1,1)
+X[N//2:] += np.array([-1, -1]) # center it at (-1, -1)
 
-Y = np.array([0]*(N/2) + [1]*(N/2))
+Y = np.array([0]*(N//2) + [1]*(N//2))
 
 
 def plot_decision_boundary(X, model):
@@ -110,6 +120,7 @@ def plot_decision_boundary(X, model):
 
 plt.scatter(X[:,0], X[:,1], s=50, c=Y, alpha=0.7)
 plot_decision_boundary(X, model)
+plt.title("dt - low bias, high variance")
 plt.show()
 
 # dt - high bias, low variance
@@ -118,6 +129,7 @@ def plot_decision_boundary(X, model):
 
 plt.scatter(X[:,0], X[:,1], s=50, c=Y, alpha=0.7)
 plot_decision_boundary(X, model)
+plt.title("dt - high bias, low variance")
 plt.show()
 
 
@@ -127,6 +139,7 @@ def plot_decision_boundary(X, model):
 
 plt.scatter(X[:,0], X[:,1], s=50, c=Y, alpha=0.7)
 plot_decision_boundary(X, model)
+plt.title("knn - low bias, high variance")
 plt.show()
 
 # knn - high bias, low variance
@@ -135,4 +148,5 @@ def plot_decision_boundary(X, model):
 
 plt.scatter(X[:,0], X[:,1], s=50, c=Y, alpha=0.7)
 plot_decision_boundary(X, model)
+plt.title("knn - high bias, low variance")
 plt.show()

supervised_class2/rf_classification.py

@@ -3,6 +3,13 @@
 # mushroom data from:
 # https://archive.ics.uci.edu/ml/datasets/Mushroom
 # put all files in the folder ../large_files/
+from __future__ import print_function, division
+from future.utils import iteritems
+from builtins import range, input
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
@@ -39,19 +46,19 @@ def fit(self, df):
 
     # find dimensionality
     self.D = len(NUMERICAL_COLS)
-    for col, encoder in self.labelEncoders.iteritems():
+    for col, encoder in iteritems(self.labelEncoders):
       self.D += len(encoder.classes_)
-    print "dimensionality:", self.D
+    print("dimensionality:", self.D)
 
   def transform(self, df):
     N, _ = df.shape
     X = np.zeros((N, self.D))
     i = 0
-    for col, scaler in self.scalers.iteritems():
+    for col, scaler in iteritems(self.scalers):
       X[:,i] = scaler.transform(df[col].as_matrix().reshape(-1, 1)).flatten()
       i += 1
 
-    for col, encoder in self.labelEncoders.iteritems():
+    for col, encoder in iteritems(self.labelEncoders):
       # print "transforming col:", col
       K = len(encoder.classes_)
       X[np.arange(N), encoder.transform(df[col]) + i] = 1
@@ -73,7 +80,7 @@ def replace_missing(df):
   # set a special value = 'missing'
   for col in CATEGORICAL_COLS:
     if np.any(df[col].isnull()):
-      print col
+      print(col)
       df.loc[ df[col].isnull(), col ] = 'missing'
 
 
@@ -100,11 +107,11 @@ def get_data():
 
   # do a quick baseline test
   baseline = LogisticRegression()
-  print "CV baseline:", cross_val_score(baseline, X, Y, cv=8).mean()
+  print("CV baseline:", cross_val_score(baseline, X, Y, cv=8).mean())
 
   # single tree
   tree = DecisionTreeClassifier()
-  print "CV one tree:", cross_val_score(tree, X, Y, cv=8).mean()
+  print("CV one tree:", cross_val_score(tree, X, Y, cv=8).mean())
 
   model = RandomForestClassifier(n_estimators=20) # try 10, 20, 50, 100, 200
-  print "CV forest:", cross_val_score(model, X, Y, cv=8).mean()
+  print("CV forest:", cross_val_score(model, X, Y, cv=8).mean())