Add files via upload

Author: liuweitb

distillation_analysis.py

@@ -0,0 +1,117 @@
+"""Analyzes, visualizes knowledge distillation"""
+
+import argparse
+import logging
+import os
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch.autograd import Variable
+import utils
+import model.net as net
+import model.resnet as resnet
+import model.data_loader as data_loader
+from torchnet.meter import ConfusionMeter
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--model_dir', default='experiments/base_model', help="Directory of params.json")
+parser.add_argument('--restore_file', default='best', help="name of the file in --model_dir \
+                     containing weights to load")
+parser.add_argument('--dataset', default='dev', help="dataset to analze the model on")
+parser.add_argument('--temperature', type=float, default=1.0, \
+                    help="temperature used for softmax output")
+
+
+def model_analysis(model, dataloader, params, temperature=1., num_classes=10):
+    """
+        Generate Confusion Matrix on evaluation set
+    """
+    model.eval()
+    confusion_matrix = ConfusionMeter(num_classes)
+    softmax_scores = []
+    predict_correct = []
+
+    with tqdm(total=len(dataloader)) as t:
+        for idx, (data_batch, labels_batch) in enumerate(dataloader):
+
+            if params.cuda:
+                data_batch, labels_batch = data_batch.cuda(async=True), \
+                                           labels_batch.cuda(async=True)
+            data_batch, labels_batch = Variable(data_batch), Variable(labels_batch)
+
+            output_batch = model(data_batch)
+
+            confusion_matrix.add(output_batch.data, labels_batch.data)
+
+            softmax_scores_batch = F.softmax(output_batch/temperature, dim=1)
+            softmax_scores_batch = softmax_scores_batch.data.cpu().numpy()
+            softmax_scores.append(softmax_scores_batch)
+
+            # extract data from torch Variable, move to cpu, convert to numpy arrays
+            output_batch = output_batch.data.cpu().numpy()
+            labels_batch = labels_batch.data.cpu().numpy()
+
+            predict_correct_batch = (np.argmax(output_batch, axis=1) == labels_batch).astype(int)
+            predict_correct.append(np.reshape(predict_correct_batch, (labels_batch.size, 1)))
+
+            t.update()
+
+    softmax_scores = np.vstack(softmax_scores)
+    predict_correct = np.vstack(predict_correct)
+
+    return softmax_scores, predict_correct, confusion_matrix.value().astype(int)
+
+
+if __name__ == '__main__':
+    """
+        Evaluate the model on the test set.
+    """
+    # Load the parameters
+    args = parser.parse_args()
+    json_path = os.path.join(args.model_dir, 'params.json')
+    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
+    params = utils.Params(json_path)
+
+    # use GPU if available
+    params.cuda = torch.cuda.is_available()     # use GPU is available
+
+    # Set the random seed for reproducible experiments
+    torch.manual_seed(230)
+    if params.cuda: torch.cuda.manual_seed(230)
+        
+    # Get the logger
+    utils.set_logger(os.path.join(args.model_dir, 'analysis.log'))
+
+    # Create the input data pipeline
+    logging.info("Loading the dataset...")
+
+    # fetch dataloaders
+    # train_dl = data_loader.fetch_dataloader('train', params)
+    # dev_dl = data_loader.fetch_dataloader('dev', params)
+    dataloader = data_loader.fetch_dataloader(args.dataset, params)
+
+    logging.info("- done.")
+
+    # Define the model graph
+    model = resnet.ResNet18().cuda() if params.cuda else resnet.ResNet18()
+
+    # fetch loss function and metrics
+    metrics = resnet.metrics
+    
+    logging.info("Starting analysis...")
+
+    # Reload weights from the saved file
+    utils.load_checkpoint(os.path.join(args.model_dir, args.restore_file + '.pth.tar'), model)
+
+    # Evaluate and analyze
+    softmax_scores, predict_correct, confusion_matrix = model_analysis(model, dataloader, params,
+                                                                       args.temperature)
+
+    results = {'softmax_scores': softmax_scores, 'predict_correct': predict_correct,
+               'confusion_matrix': confusion_matrix}
+
+    for k, v in results.items():
+        filename = args.dataset + '_temp' + str(args.temperature) + '_' + k + '.txt'
+        save_path = os.path.join(args.model_dir, filename)
+        np.savetxt(save_path, v)

requirements.txt

@@ -0,0 +1,9 @@
+scipy==1.0.0
+numpy==1.14.0
+Pillow==8.1.1
+tabulate==0.8.2
+tensorflow==1.7.0rc0
+torch==0.3.0.post4
+torchvision==0.2.0
+tqdm==4.19.8
+torchnet

search_hyperparams.py

@@ -0,0 +1,91 @@
+"""
+   Peform hyperparemeters search
+
+   A brief definition/clarification of 'params.json' files:
+
+   "model_version": "resnet18", # "base" models or "modelname"_distill models
+   "subset_percent": 1.0,       # use full (1.0) train set or partial (<1.0) train set
+   "augmentation": "yes",       # whether to use data augmentation in data_loader
+   "teacher": "densenet",       # no need to specify this for "base" cnn/resnet18
+   "alpha": 0.0,                # only used for experiments involving distillation
+   "temperature": 1,            # only used for experiments involving distillation
+   "learning_rate": 1e-1,       # as the name suggests
+   "batch_size": 128,           # for both train/eval
+   "num_epochs": 200,           # as the name suggests
+   "dropout_rate": 0.5,         # only valid for "cnn"-related models, not in resnet18
+   "num_channels": 32,          # only valid for "cnn"-related models, not in resnet18
+   "save_summary_steps": 100,
+   "num_workers": 4
+
+"""
+
+
+import argparse
+import os
+from subprocess import check_call
+import sys
+import utils
+import logging
+
+
+PYTHON = sys.executable
+parser = argparse.ArgumentParser()
+parser.add_argument('--parent_dir', default='experiments/learning_rate',
+                    help='Directory containing params.json')
+
+def launch_training_job(parent_dir, job_name, params):
+    """Launch training of the model with a set of hyperparameters in parent_dir/job_name
+
+    Args:
+        model_dir: (string) directory containing config, weights and log
+        data_dir: (string) directory containing the dataset
+        params: (dict) containing hyperparameters
+    """
+    # Create a new folder in parent_dir with unique_name "job_name"
+    model_dir = os.path.join(parent_dir, job_name)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+
+    # Write parameters in json file
+    json_path = os.path.join(model_dir, 'params.json')
+    params.save(json_path)
+
+    # Launch training with this config
+    cmd = "{python} train.py --model_dir={model_dir}".format(python=PYTHON,
+                                                             model_dir=model_dir)
+    print(cmd)
+    check_call(cmd, shell=True)
+
+
+if __name__ == "__main__":
+    # Load the "reference" parameters from parent_dir json file
+    args = parser.parse_args()
+    json_path = os.path.join(args.parent_dir, 'params.json')
+    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
+    params = utils.Params(json_path)
+
+    # Set the logger
+    utils.set_logger(os.path.join(args.parent_dir, 'search_hyperparameters.log'))
+
+    '''
+    Temperature and alpha search for KD on CNN (teacher model picked in params.json)
+    Perform hypersearch (empirical grid): distilling 'temperature', loss weight 'alpha'
+    '''
+
+    # hyperparameters for KD
+    alphas = [0.99, 0.95, 0.5, 0.1, 0.05]
+    temperatures = [20., 10., 8., 6., 4.5, 3., 2., 1.5]
+
+    logging.info("Searching hyperparameters...")
+    logging.info("alphas: {}".format(alphas))
+    logging.info("temperatures: {}".format(temperatures))
+
+    for alpha in alphas:
+        for temperature in temperatures:
+            # [Modify] the relevant parameter in params (others remain unchanged)
+            params.alpha = alpha
+            params.temperature = temperature
+
+            # Launch job (name has to be unique)
+            job_name = "alpha_{}_Temp_{}".format(alpha, temperature)
+            launch_training_job(args.parent_dir, job_name, params)