PyTorch Versions of TC1 and P1B2 Benchmarks

Pilot1/P1B2/p1b2_baseline_pytorch.py

@@ -0,0 +1,144 @@
+from __future__ import print_function
+
+import numpy as np
+import os
+import sys
+
+import torch
+
+
+if True:
+    print("Restricting #of GPUs to 1")
+    os.environ["CUDA_VISIBLE_DEVICES"]="0"
+    #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7"
+    #os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"
+
+file_path = os.path.dirname(os.path.realpath(__file__))
+lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
+sys.path.append(lib_path2)
+os.chdir(file_path)
+
+import p1b2 as bmk
+import candle
+from torch_deps.p1b2_pytorch_model import P1B2Model
+from torch_deps.random_seeding import seed_random_state
+
+np.set_printoptions(precision=4)
+
+def initialize_parameters(default_model = 'p1b2_default_model_pytorch.txt'):
+
+    # Build benchmark object
+    p1b2Bmk = bmk.BenchmarkP1B2(bmk.file_path, default_model, 'pytorch',
+    prog='p1b2_baseline', desc='Train Classifier - Pilot 1 Benchmark 2')
+
+    print("Created P1B2 benchmark")
+
+    # Initialize parameters
+    gParameters = candle.finalize_parameters(p1b2Bmk)
+    #benchmark.logger.info('Params: {}'.format(gParameters))
+    print("Parameters initialized")
+
+    return gParameters
+
+
+def run(params):
+
+    args = candle.ArgumentStruct(**params)
+    args.no_cuda = args.no_cuda if hasattr(args,'no_cuda') else False
+    args.multi_gpu = args.multi_gpu if hasattr(args,'multi_gpu') else True
+    args.max_num_batches = args.max_num_batches if hasattr(args,'max_num_batches') else 1000
+    args.dry_run = args.dry_run if hasattr(args,'dry_run') else False
+    args.log_interval = args.log_interval if hasattr(args,'log_interval') else 10
+
+
+    seed = args.rng_seed
+    candle.set_seed(seed)
+    # Setting up random seed for reproducible and deterministic results
+    seed_random_state(args.rng_seed)
+
+    args.keras_defaults = candle.keras_default_config()
+
+    # Construct extension to save validation results
+    ext = bmk.extension_from_parameters(params, '.pytorch')
+
+    candle.verify_path(params['save_path'])
+    prefix = '{}{}'.format(params['save_path'], ext)
+    logfile = params['logfile'] if params['logfile'] else prefix+'.log'
+    candle.set_up_logger(logfile, bmk.logger, params['verbose'])
+    bmk.logger.info('Params: {}'.format(params))
+
+    args.tensorboard_dir = "tb/{}".format(ext)
+    args.logger = bmk.logger 
+
+    #Autosave model 
+    model_name = params['model_name']
+    args_filename = "{}.model.args".format(params['save_path'])
+    args.model_autosave_filename = "{}.autosave.model.pth".format(params['save_path'])
+    # CSV logging
+    args.csv_filename = '{}{}_training.log'.format(params['save_path'], ext)
+
+    # Computation device config (cuda or cpu)
+    use_cuda = not args.no_cuda and torch.cuda.is_available()
+    device = torch.device('cuda' if use_cuda else 'cpu')
+
+    # save args to file    
+    import pickle
+    args_file = open(args_filename, 'wb')
+    pickle.dump(args, args_file)
+    args_file.close()
+
+    modelP1B2 = P1B2Model(args, use_cuda, device)
+
+    #model.summary()
+    #print(modelP1B2.p1b2_net)  # Model summary
+    bmk.logger.info('Model summary: {}'.format(modelP1B2.p1b2_net))  # Model summary
+
+    modelP1B2.train()
+    modelP1B2.print_final_stats()
+
+    #Save model 
+    model_name = params['model_name']
+    model_filename = "{}.model_state_dict.pth".format(params['save_path'])
+    if hasattr(modelP1B2.p1b2_net,'module'):
+        # Saving the DataParallel model
+        torch.save(modelP1B2.p1b2_net.module.state_dict(), model_filename)
+    else:
+        torch.save(modelP1B2.p1b2_net.state_dict(), model_filename)
+
+    #reload args from file
+    args_file = open(args_filename, 'rb')
+    loaded_args = pickle.load(args_file)
+    args_file.close()
+
+    # load weights into new model
+    loaded_modelP1B2 = P1B2Model(loaded_args)
+    loaded_modelP1B2.p1b2_net.load_state_dict(torch.load(model_filename, map_location=torch.device('cpu')))
+    print("Loaded torch model from disk")
+
+    # evaluate loaded model on test data
+    loaded_modelP1B2.p1b2_net.eval()
+    val_acc,val_loss = loaded_modelP1B2.validation(0)
+
+    print("Model State Dict Validation loss: %5.2f"  % (val_loss))
+    print("Model State Dict Validation accuracy: %5.2f%%" %(val_acc))
+
+    print('Test data: ')
+    test_acc,test_loss = loaded_modelP1B2.test()
+
+    print("Model State Dict Test loss: %5.2f"  % (test_loss))
+    print("Model State Dict Test accuracy: %5.2f%%" %(test_acc))
+
+
+def main():
+
+    gParameters = initialize_parameters()
+    run(gParameters)
+
+if __name__ == '__main__':
+    main()
+    try:
+        tmp = 1
+    except AttributeError:      # theano does not have this function
+        pass
+
+

Pilot1/P1B2/p1b2_default_model_pytorch.txt

@@ -0,0 +1,28 @@
+[Global_Params]
+data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B2/'
+train_data = 'P1B2.train.csv'
+test_data = 'P1B2.test.csv'
+model_name='p1b2'
+dense=[1024, 512, 256]
+batch_size=60
+epochs=20
+activation='sigmoid'
+out_activation='log_softmax'
+loss='nll'
+optimizer='rmsprop'
+learning_rate=0.001
+scaling='minmax'
+dropout=0.
+classes=10
+feature_subsample=0
+reg_l2=0.00001
+val_split=0.1
+rng_seed=2017
+initialization='glorot_uniform'
+save_path='save'
+shuffle = True
+
+# Miscellaneous settings ##################################
+# multi_gpu=True
+# no_cuda=True
+# rng_seed=0

Pilot1/P1B2/pytorch_utils.py

@@ -0,0 +1,140 @@
+from __future__ import absolute_import
+
+import torch
+import torch.nn
+import torch.nn.init
+import torch.optim
+import torch.nn.functional as F
+
+from default_utils import set_seed as set_seed_defaultUtils
+
+def set_parallelism_threads(): # for compatibility
+    pass
+
+def set_seed(seed):
+    """ Set the random number seed to the desired value
+
+        Parameters
+        ----------
+        seed : integer
+            Random number seed.
+    """
+
+    set_seed_defaultUtils(seed)
+    torch.manual_seed(seed)
+
+
+def get_function(name):
+    mapping = {}
+
+    # loss
+    mapping['mse'] = torch.nn.MSELoss()
+    mapping['binary_crossentropy'] = torch.nn.BCELoss()
+    mapping['categorical_crossentropy'] = torch.nn.CrossEntropyLoss()
+    mapping['smoothL1'] = torch.nn.SmoothL1Loss()
+
+    mapped = mapping.get(name)
+    if not mapped:
+        raise Exception('No pytorch function found for "{}"'.format(name))
+
+    return mapped
+
+
+def build_activation(type, dim=1):
+
+    # activation
+
+    if type=='relu':
+         return torch.nn.ReLU()
+    elif type=='sigmoid':
+         return torch.nn.Sigmoid()
+    elif type=='tanh':
+         return torch.nn.Tanh()
+    elif type=='softmax':
+         return torch.nn.Softmax(dim)    
+    elif type=='log_softmax':
+         return torch.nn.LogSoftmax(dim)
+
+def build_optimizer(model, type, lr, kerasDefaults, trainable_only=True):
+
+    if trainable_only:
+        params = filter(lambda p: p.requires_grad, model.parameters())
+    else:
+        params = model.parameters()
+
+    #schedule = optimizers.optimizer.Schedule() # constant lr (equivalent to default keras setting)
+
+    if type == 'sgd':
+        return torch.optim.SGD(params,
+                              lr=lr,
+                              momentum=kerasDefaults['momentum_sgd'],
+                              nesterov=kerasDefaults['nesterov_sgd'])
+                              #schedule=schedule)
+
+    elif type == 'rmsprop':
+        return torch.optim.RMSprop(model.parameters(),
+                                  lr=lr,
+                                  alpha=kerasDefaults['rho'],
+                                  eps=kerasDefaults['epsilon'],
+                                  weight_decay=kerasDefaults['weight_decay'])
+                                  #schedule=schedule)
+
+    elif type == 'adagrad':
+        return torch.optim.Adagrad(model.parameters(),
+                              lr=lr,
+                              eps=kerasDefaults['epsilon'])
+
+    elif type == 'adadelta':
+        return torch.optim.Adadelta(params,
+                              eps=kerasDefaults['epsilon'],
+                              rho=kerasDefaults['rho'])
+
+    elif type == 'adam':
+        return torch.optim.Adam(params,
+                               lr=lr,
+                               betas=[kerasDefaults['beta_1'], kerasDefaults['beta_2']],
+                               eps=kerasDefaults['epsilon'])
+
+
+
+def initialize(weights, type, kerasDefaults, seed=None, constant=0.):
+
+    if type == 'constant':
+        return torch.nn.init.constant_(weights,
+                                    val=constant)
+
+    elif type == 'uniform':
+        return torch.nn.init.uniform(weights,
+                                  a=kerasDefaults['minval_uniform'],
+                                  b=kerasDefaults['maxval_uniform'])
+
+    elif type == 'normal':
+        return torch.nn.init.normal(weights,
+                                  mean=kerasDefaults['mean_normal'],
+                                  std=kerasDefaults['stddev_normal'])
+
+    elif type == 'glorot_normal': # not quite Xavier
+        return torch.nn.init.xavier_normal(weights)
+
+    elif type == 'glorot_uniform':
+        return torch.nn.init.xavier_uniform_(weights)
+
+    elif type == 'he_normal':
+        return torch.nn.init.kaiming_uniform(weights)
+
+
+def xent(y_true, y_pred):
+    return F.cross_entropy(y_pred, y_true)
+
+
+def mse(y_true, y_pred):
+    return F.mse_loss(y_pred, y_true)
+
+def build_loss(type, y_pred, y_true):
+
+    if type=='categorical_crossentropy':
+         return xent(y_true, y_pred)
+    elif type=='mse':
+         return mse(y_true, y_pred)
+    elif type=='nll':
+         return F.nll_loss(y_pred, y_true)