From cabd80df418f375a541eb3c970d7ba4939c22e69 Mon Sep 17 00:00:00 2001 From: shashikg Date: Sun, 26 Jan 2020 18:03:09 +0530 Subject: [PATCH 1/7] added option to provide input size to base models --- .gitignore | 6 +- candidate_models/base_models/__init__.py | 328 +++++++++++++++-------- 2 files changed, 223 insertions(+), 111 deletions(-) mode change 100644 => 100755 .gitignore diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 7849784..c95a93f --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -*.pyc -*.DS_Store +__pycache__/ +.ipynb_checkpoints +build/* +dist/* diff --git a/candidate_models/base_models/__init__.py b/candidate_models/base_models/__init__.py index 8b96d2b..fe38391 100644 --- a/candidate_models/base_models/__init__.py +++ b/candidate_models/base_models/__init__.py @@ -334,118 +334,228 @@ class BaseModelPool(UniqueKeyDict): Each entry maps from `name` to an activations extractor. """ - def __init__(self): + def __init__(self, input_size=None): super(BaseModelPool, self).__init__() self._accessed_base_models = set() - _key_functions = { - 'alexnet': lambda: pytorch_model('alexnet', image_size=224), - 'squeezenet1_0': lambda: pytorch_model('squeezenet1_0', image_size=224), - 'squeezenet1_1': lambda: pytorch_model('squeezenet1_1', image_size=224), - 'resnet-18': lambda: pytorch_model('resnet18', image_size=224), - 'resnet-34': lambda: pytorch_model('resnet34', image_size=224), - 'resnet-50-pytorch': lambda: pytorch_model('resnet50', image_size=224), - 'resnet-50-robust': lambda: robust_model('resnet50', image_size=224), - - 'vgg-16': lambda: keras_model('vgg16', 'VGG16', image_size=224), - 'vgg-19': lambda: keras_model('vgg19', 'VGG19', image_size=224), - 'vggface': vggface, - 'xception': lambda: keras_model('xception', 'Xception', image_size=299), - 'densenet-121': lambda: keras_model('densenet', 'DenseNet121', image_size=224), - 'densenet-169': lambda: keras_model('densenet', 'DenseNet169', image_size=224), - 'densenet-201': lambda: keras_model('densenet', 'DenseNet201', image_size=224), - - 'inception_v1': lambda: TFSlimModel.init('inception_v1', preprocessing_type='inception', image_size=224), - 'inception_v2': lambda: TFSlimModel.init('inception_v2', preprocessing_type='inception', image_size=224), - 'inception_v3': lambda: TFSlimModel.init('inception_v3', preprocessing_type='inception', image_size=299), - 'inception_v4': lambda: TFSlimModel.init('inception_v4', preprocessing_type='inception', image_size=299), - 'inception_resnet_v2': lambda: TFSlimModel.init('inception_resnet_v2', preprocessing_type='inception', - image_size=299), - 'resnet-50_v1': lambda: TFSlimModel.init('resnet-50_v1', net_name='resnet_v1_50', preprocessing_type='vgg', - image_size=224, labels_offset=0), - 'resnet-101_v1': lambda: TFSlimModel.init('resnet-101_v1', net_name='resnet_v1_101', - preprocessing_type='vgg', - image_size=224, labels_offset=0), - 'resnet-152_v1': lambda: TFSlimModel.init('resnet-152_v1', net_name='resnet_v1_152', - preprocessing_type='vgg', - image_size=224, labels_offset=0), - # image_size is 299 for resnet-v2, this is a bug in tf-slim. - # see https://github.com/tensorflow/models/tree/8b18491b26e4b8271db757a3245008882ea112b3/research/slim: - # "ResNet V2 models use Inception pre-processing and input image size of 299" - 'resnet-50_v2': lambda: TFSlimModel.init('resnet-50_v2', net_name='resnet_v2_50', - preprocessing_type='inception', - image_size=299), - 'resnet-101_v2': lambda: TFSlimModel.init('resnet-101_v2', net_name='resnet_v2_101', - preprocessing_type='inception', - image_size=299), - 'resnet-152_v2': lambda: TFSlimModel.init('resnet-152_v2', net_name='resnet_v2_152', - preprocessing_type='inception', - image_size=299), - 'nasnet_mobile': lambda: TFSlimModel.init('nasnet_mobile', preprocessing_type='inception', image_size=331), - 'nasnet_large': lambda: TFSlimModel.init('nasnet_large', preprocessing_type='inception', image_size=331), - 'pnasnet_large': lambda: TFSlimModel.init('pnasnet_large', preprocessing_type='inception', image_size=331), - 'bagnet9': lambda: bagnet("bagnet9"), - 'bagnet17': lambda: bagnet("bagnet17"), - 'bagnet33': lambda: bagnet("bagnet33"), - # CORnets. Note that these are only here for the base_model_pool, their commitment works separately - # from the models here due to anatomical alignment. - 'CORnet-Z': lambda: cornet('CORnet-Z'), - 'CORnet-R': lambda: cornet('CORnet-R'), - 'CORnet-S': lambda: cornet('CORnet-S'), - - 'resnet50-SIN': lambda: texture_vs_shape(model_identifier='resnet50-SIN', - model_name='resnet50_trained_on_SIN'), - 'resnet50-SIN_IN': lambda: texture_vs_shape(model_identifier='resnet50-SIN_IN', - model_name='resnet50_trained_on_SIN_and_IN'), - 'resnet50-SIN_IN_IN': lambda: texture_vs_shape( - model_identifier='resnet50-SIN_IN_IN', - model_name='resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN'), - - 'resnext101_32x8d_wsl': lambda: wsl(8), - 'resnext101_32x16d_wsl': lambda: wsl(16), - 'resnext101_32x32d_wsl': lambda: wsl(32), - 'resnext101_32x48d_wsl': lambda: wsl(48), - - 'fixres_resnext101_32x48d_wsl': lambda: fixres( - 'resnext101_32x48d_wsl', - 'https://dl.fbaipublicfiles.com/FixRes_data/FixRes_Pretrained_Models/ResNeXt_101_32x48d.pth'), - - 'dcgan': lambda: dcgan("get_discriminator"), - - 'convrnn_224': lambda: TFUtilsModel.init(load_median_model, 'convrnn_224', tnn_model=True, - preprocessing_type='convrnn', image_size=224, image_resize=None), - } - # MobileNets - for version, multiplier, image_size in [ - # v1 - (1, 1.0, 224), (1, 1.0, 192), (1, 1.0, 160), (1, 1.0, 128), - (1, 0.75, 224), (1, 0.75, 192), (1, 0.75, 160), (1, 0.75, 128), - (1, 0.5, 224), (1, 0.5, 192), (1, 0.5, 160), (1, 0.5, 128), - (1, 0.25, 224), (1, 0.25, 192), (1, 0.25, 160), (1, 0.25, 128), - # v2 - (2, 1.4, 224), - (2, 1.3, 224), - (2, 1.0, 224), (2, 1.0, 192), (2, 1.0, 160), (2, 1.0, 128), (2, 1.0, 96), - (2, 0.75, 224), (2, 0.75, 192), (2, 0.75, 160), (2, 0.75, 128), (2, 0.75, 96), - (2, 0.5, 224), (2, 0.5, 192), (2, 0.5, 160), (2, 0.5, 128), (2, 0.5, 96), - (2, 0.35, 224), (2, 0.35, 192), (2, 0.35, 160), (2, 0.35, 128), (2, 0.35, 96), - ]: - identifier = f"mobilenet_v{version}_{multiplier}_{image_size}" - if (version == 1 and multiplier in [.75, .5, .25]) or (version == 2 and multiplier == 1.4): - net_name = f"mobilenet_v{version}_{multiplier * 100:03.0f}" - else: - net_name = f"mobilenet_v{version}" - # arg=arg default value enforces closure: - # https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result - _key_functions[identifier] = \ - lambda identifier=identifier, image_size=image_size, net_name=net_name, \ - multiplier=multiplier: TFSlimModel.init( - identifier, preprocessing_type='inception', image_size=image_size, net_name=net_name, - model_ctr_kwargs={'depth_multiplier': multiplier}) - - # instantiate models with LazyLoad wrapper - for identifier, function in _key_functions.items(): - self[identifier] = LazyLoad(function) + if input_size==None: + _key_functions = { + 'alexnet': lambda: pytorch_model('alexnet', image_size=224), + 'squeezenet1_0': lambda: pytorch_model('squeezenet1_0', image_size=224), + 'squeezenet1_1': lambda: pytorch_model('squeezenet1_1', image_size=224), + 'resnet-18': lambda: pytorch_model('resnet18', image_size=224), + 'resnet-34': lambda: pytorch_model('resnet34', image_size=224), + 'resnet-50-pytorch': lambda: pytorch_model('resnet50', image_size=224), + 'resnet-50-robust': lambda: robust_model('resnet50', image_size=224), + + 'vgg-16': lambda: keras_model('vgg16', 'VGG16', image_size=224), + 'vgg-19': lambda: keras_model('vgg19', 'VGG19', image_size=224), + 'vggface': vggface, + 'xception': lambda: keras_model('xception', 'Xception', image_size=299), + 'densenet-121': lambda: keras_model('densenet', 'DenseNet121', image_size=224), + 'densenet-169': lambda: keras_model('densenet', 'DenseNet169', image_size=224), + 'densenet-201': lambda: keras_model('densenet', 'DenseNet201', image_size=224), + + 'inception_v1': lambda: TFSlimModel.init('inception_v1', preprocessing_type='inception', image_size=224), + 'inception_v2': lambda: TFSlimModel.init('inception_v2', preprocessing_type='inception', image_size=224), + 'inception_v3': lambda: TFSlimModel.init('inception_v3', preprocessing_type='inception', image_size=299), + 'inception_v4': lambda: TFSlimModel.init('inception_v4', preprocessing_type='inception', image_size=299), + 'inception_resnet_v2': lambda: TFSlimModel.init('inception_resnet_v2', preprocessing_type='inception', + image_size=299), + 'resnet-50_v1': lambda: TFSlimModel.init('resnet-50_v1', net_name='resnet_v1_50', preprocessing_type='vgg', + image_size=224, labels_offset=0), + 'resnet-101_v1': lambda: TFSlimModel.init('resnet-101_v1', net_name='resnet_v1_101', + preprocessing_type='vgg', + image_size=224, labels_offset=0), + 'resnet-152_v1': lambda: TFSlimModel.init('resnet-152_v1', net_name='resnet_v1_152', + preprocessing_type='vgg', + image_size=224, labels_offset=0), + # image_size is 299 for resnet-v2, this is a bug in tf-slim. + # see https://github.com/tensorflow/models/tree/8b18491b26e4b8271db757a3245008882ea112b3/research/slim: + # "ResNet V2 models use Inception pre-processing and input image size of 299" + 'resnet-50_v2': lambda: TFSlimModel.init('resnet-50_v2', net_name='resnet_v2_50', + preprocessing_type='inception', + image_size=299), + 'resnet-101_v2': lambda: TFSlimModel.init('resnet-101_v2', net_name='resnet_v2_101', + preprocessing_type='inception', + image_size=299), + 'resnet-152_v2': lambda: TFSlimModel.init('resnet-152_v2', net_name='resnet_v2_152', + preprocessing_type='inception', + image_size=299), + 'nasnet_mobile': lambda: TFSlimModel.init('nasnet_mobile', preprocessing_type='inception', image_size=331), + 'nasnet_large': lambda: TFSlimModel.init('nasnet_large', preprocessing_type='inception', image_size=331), + 'pnasnet_large': lambda: TFSlimModel.init('pnasnet_large', preprocessing_type='inception', image_size=331), + 'bagnet9': lambda: bagnet("bagnet9"), + 'bagnet17': lambda: bagnet("bagnet17"), + 'bagnet33': lambda: bagnet("bagnet33"), + # CORnets. Note that these are only here for the base_model_pool, their commitment works separately + # from the models here due to anatomical alignment. + 'CORnet-Z': lambda: cornet('CORnet-Z'), + 'CORnet-R': lambda: cornet('CORnet-R'), + 'CORnet-S': lambda: cornet('CORnet-S'), + + 'resnet50-SIN': lambda: texture_vs_shape(model_identifier='resnet50-SIN', + model_name='resnet50_trained_on_SIN'), + 'resnet50-SIN_IN': lambda: texture_vs_shape(model_identifier='resnet50-SIN_IN', + model_name='resnet50_trained_on_SIN_and_IN'), + 'resnet50-SIN_IN_IN': lambda: texture_vs_shape( + model_identifier='resnet50-SIN_IN_IN', + model_name='resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN'), + + 'resnext101_32x8d_wsl': lambda: wsl(8), + 'resnext101_32x16d_wsl': lambda: wsl(16), + 'resnext101_32x32d_wsl': lambda: wsl(32), + 'resnext101_32x48d_wsl': lambda: wsl(48), + + 'fixres_resnext101_32x48d_wsl': lambda: fixres( + 'resnext101_32x48d_wsl', + 'https://dl.fbaipublicfiles.com/FixRes_data/FixRes_Pretrained_Models/ResNeXt_101_32x48d.pth'), + + 'dcgan': lambda: dcgan("get_discriminator"), + + 'convrnn_224': lambda: TFUtilsModel.init(load_median_model, 'convrnn_224', tnn_model=True, + preprocessing_type='convrnn', image_size=224, image_resize=None), + } + # MobileNets + for version, multiplier, image_size in [ + # v1 + (1, 1.0, 224), (1, 1.0, 192), (1, 1.0, 160), (1, 1.0, 128), + (1, 0.75, 224), (1, 0.75, 192), (1, 0.75, 160), (1, 0.75, 128), + (1, 0.5, 224), (1, 0.5, 192), (1, 0.5, 160), (1, 0.5, 128), + (1, 0.25, 224), (1, 0.25, 192), (1, 0.25, 160), (1, 0.25, 128), + # v2 + (2, 1.4, 224), + (2, 1.3, 224), + (2, 1.0, 224), (2, 1.0, 192), (2, 1.0, 160), (2, 1.0, 128), (2, 1.0, 96), + (2, 0.75, 224), (2, 0.75, 192), (2, 0.75, 160), (2, 0.75, 128), (2, 0.75, 96), + (2, 0.5, 224), (2, 0.5, 192), (2, 0.5, 160), (2, 0.5, 128), (2, 0.5, 96), + (2, 0.35, 224), (2, 0.35, 192), (2, 0.35, 160), (2, 0.35, 128), (2, 0.35, 96), + ]: + identifier = f"mobilenet_v{version}_{multiplier}_{image_size}" + if (version == 1 and multiplier in [.75, .5, .25]) or (version == 2 and multiplier == 1.4): + net_name = f"mobilenet_v{version}_{multiplier * 100:03.0f}" + else: + net_name = f"mobilenet_v{version}" + # arg=arg default value enforces closure: + # https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result + _key_functions[identifier] = \ + lambda identifier=identifier, image_size=image_size, net_name=net_name, \ + multiplier=multiplier: TFSlimModel.init( + identifier, preprocessing_type='inception', image_size=image_size, net_name=net_name, + model_ctr_kwargs={'depth_multiplier': multiplier}) + + # instantiate models with LazyLoad wrapper + for identifier, function in _key_functions.items(): + self[identifier] = LazyLoad(function) + else: + _key_functions = { + 'alexnet': lambda: pytorch_model('alexnet', image_size=input_size), + 'squeezenet1_0': lambda: pytorch_model('squeezenet1_0', image_size=input_size), + 'squeezenet1_1': lambda: pytorch_model('squeezenet1_1', image_size=input_size), + 'resnet-18': lambda: pytorch_model('resnet18', image_size=input_size), + 'resnet-34': lambda: pytorch_model('resnet34', image_size=input_size), + 'resnet-50-pytorch': lambda: pytorch_model('resnet50', image_size=input_size), + 'resnet-50-robust': lambda: robust_model('resnet50', image_size=input_size), + + 'vgg-16': lambda: keras_model('vgg16', 'VGG16', image_size=input_size), + 'vgg-19': lambda: keras_model('vgg19', 'VGG19', image_size=input_size), + 'vggface': vggface, + 'xception': lambda: keras_model('xception', 'Xception', image_size=input_size), + 'densenet-121': lambda: keras_model('densenet', 'DenseNet121', image_size=input_size), + 'densenet-169': lambda: keras_model('densenet', 'DenseNet169', image_size=input_size), + 'densenet-201': lambda: keras_model('densenet', 'DenseNet201', image_size=input_size), + + 'inception_v1': lambda: TFSlimModel.init('inception_v1', preprocessing_type='inception', image_size=input_size), + 'inception_v2': lambda: TFSlimModel.init('inception_v2', preprocessing_type='inception', image_size=input_size), + 'inception_v3': lambda: TFSlimModel.init('inception_v3', preprocessing_type='inception', image_size=input_size), + 'inception_v4': lambda: TFSlimModel.init('inception_v4', preprocessing_type='inception', image_size=input_size), + 'inception_resnet_v2': lambda: TFSlimModel.init('inception_resnet_v2', preprocessing_type='inception', + image_size=input_size), + 'resnet-50_v1': lambda: TFSlimModel.init('resnet-50_v1', net_name='resnet_v1_50', preprocessing_type='vgg', + image_size=input_size, labels_offset=0), + 'resnet-101_v1': lambda: TFSlimModel.init('resnet-101_v1', net_name='resnet_v1_101', + preprocessing_type='vgg', + image_size=input_size, labels_offset=0), + 'resnet-152_v1': lambda: TFSlimModel.init('resnet-152_v1', net_name='resnet_v1_152', + preprocessing_type='vgg', + image_size=input_size, labels_offset=0), + # image_size is 299 for resnet-v2, this is a bug in tf-slim. + # see https://github.com/tensorflow/models/tree/8b18491b26e4b8271db757a3245008882ea112b3/research/slim: + # "ResNet V2 models use Inception pre-processing and input image size of 299" + 'resnet-50_v2': lambda: TFSlimModel.init('resnet-50_v2', net_name='resnet_v2_50', + preprocessing_type='inception', + image_size=input_size), + 'resnet-101_v2': lambda: TFSlimModel.init('resnet-101_v2', net_name='resnet_v2_101', + preprocessing_type='inception', + image_size=input_size), + 'resnet-152_v2': lambda: TFSlimModel.init('resnet-152_v2', net_name='resnet_v2_152', + preprocessing_type='inception', + image_size=input_size), + 'nasnet_mobile': lambda: TFSlimModel.init('nasnet_mobile', preprocessing_type='inception', image_size=input_size), + 'nasnet_large': lambda: TFSlimModel.init('nasnet_large', preprocessing_type='inception', image_size=input_size), + 'pnasnet_large': lambda: TFSlimModel.init('pnasnet_large', preprocessing_type='inception', image_size=input_size), + 'bagnet9': lambda: bagnet("bagnet9"), + 'bagnet17': lambda: bagnet("bagnet17"), + 'bagnet33': lambda: bagnet("bagnet33"), + # CORnets. Note that these are only here for the base_model_pool, their commitment works separately + # from the models here due to anatomical alignment. + 'CORnet-Z': lambda: cornet('CORnet-Z'), + 'CORnet-R': lambda: cornet('CORnet-R'), + 'CORnet-S': lambda: cornet('CORnet-S'), + + 'resnet50-SIN': lambda: texture_vs_shape(model_identifier='resnet50-SIN', + model_name='resnet50_trained_on_SIN'), + 'resnet50-SIN_IN': lambda: texture_vs_shape(model_identifier='resnet50-SIN_IN', + model_name='resnet50_trained_on_SIN_and_IN'), + 'resnet50-SIN_IN_IN': lambda: texture_vs_shape( + model_identifier='resnet50-SIN_IN_IN', + model_name='resnet50_trained_on_SIN_and_IN_then_finetuned_on_IN'), + + 'resnext101_32x8d_wsl': lambda: wsl(8), + 'resnext101_32x16d_wsl': lambda: wsl(16), + 'resnext101_32x32d_wsl': lambda: wsl(32), + 'resnext101_32x48d_wsl': lambda: wsl(48), + + 'fixres_resnext101_32x48d_wsl': lambda: fixres( + 'resnext101_32x48d_wsl', + 'https://dl.fbaipublicfiles.com/FixRes_data/FixRes_Pretrained_Models/ResNeXt_101_32x48d.pth'), + + 'dcgan': lambda: dcgan("get_discriminator"), + + 'convrnn_224': lambda: TFUtilsModel.init(load_median_model, 'convrnn_224', tnn_model=True, + preprocessing_type='convrnn', image_size=224, image_resize=None), + } + # MobileNets + for version, multiplier, image_size in [ + # v1 + (1, 1.0, 224), (1, 1.0, 192), (1, 1.0, 160), (1, 1.0, 128), + (1, 0.75, 224), (1, 0.75, 192), (1, 0.75, 160), (1, 0.75, 128), + (1, 0.5, 224), (1, 0.5, 192), (1, 0.5, 160), (1, 0.5, 128), + (1, 0.25, 224), (1, 0.25, 192), (1, 0.25, 160), (1, 0.25, 128), + # v2 + (2, 1.4, 224), + (2, 1.3, 224), + (2, 1.0, 224), (2, 1.0, 192), (2, 1.0, 160), (2, 1.0, 128), (2, 1.0, 96), + (2, 0.75, 224), (2, 0.75, 192), (2, 0.75, 160), (2, 0.75, 128), (2, 0.75, 96), + (2, 0.5, 224), (2, 0.5, 192), (2, 0.5, 160), (2, 0.5, 128), (2, 0.5, 96), + (2, 0.35, 224), (2, 0.35, 192), (2, 0.35, 160), (2, 0.35, 128), (2, 0.35, 96), + ]: + identifier = f"mobilenet_v{version}_{multiplier}_{image_size}" + if (version == 1 and multiplier in [.75, .5, .25]) or (version == 2 and multiplier == 1.4): + net_name = f"mobilenet_v{version}_{multiplier * 100:03.0f}" + else: + net_name = f"mobilenet_v{version}" + # arg=arg default value enforces closure: + # https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result + _key_functions[identifier] = \ + lambda identifier=identifier, image_size=input_size, net_name=net_name, \ + multiplier=multiplier: TFSlimModel.init( + identifier, preprocessing_type='inception', image_size=input_size, net_name=net_name, + model_ctr_kwargs={'depth_multiplier': multiplier}) + + # instantiate models with LazyLoad wrapper + for identifier, function in _key_functions.items(): + self[identifier] = LazyLoad(function) def __getitem__(self, basemodel_identifier): if basemodel_identifier in self._accessed_base_models: From 4b382c3c043c0c84670422937627a33302575861 Mon Sep 17 00:00:00 2001 From: shashikg Date: Sun, 26 Jan 2020 18:04:16 +0530 Subject: [PATCH 2/7] activation layers for visual search model --- .../model_commitments/vs_layer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 candidate_models/model_commitments/vs_layer.py diff --git a/candidate_models/model_commitments/vs_layer.py b/candidate_models/model_commitments/vs_layer.py new file mode 100644 index 0000000..e5fd987 --- /dev/null +++ b/candidate_models/model_commitments/vs_layer.py @@ -0,0 +1,26 @@ +import warnings + +import itertools + +from candidate_models.utils import UniqueKeyDict + +class VisualSearchLayers(UniqueKeyDict): + def __init__(self): + super(VisualSearchLayers, self).__init__() + layers = { + 'vgg-16': [f'block{i + 1}_pool' for i in range(3,5)], + } + for basemodel_identifier, default_layers in layers.items(): + self[basemodel_identifier] = default_layers + + @staticmethod + def _item(item): + return item + + def __getitem__(self, item): + return super(VisualSearchLayers, self).__getitem__(self._item(item)) + + def __contains__(self, item): + return super(VisualSearchLayers, self).__contains__(self._item(item)) + +visual_search_layer = VisualSearchLayers() From de2ddd34576c7d94ee9c84165578732d7eb0a036 Mon Sep 17 00:00:00 2001 From: shashikg Date: Sun, 26 Jan 2020 23:08:32 +0530 Subject: [PATCH 3/7] add test_search --- tests/test_search.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test_search.py diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..bd9703d --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,13 @@ +import pytest +from pytest import approx + +from candidate_models.model_commitments import brain_translated_pool +from brainscore.benchmarks import benchmark_pool +import numpy as np +import matplotlib.pyplot as plt +import brainscore + +def test_search(): + model = brain_translated_pool['vgg-16'] + score = score_model(model_identifier='vgg-16', model=model, benchmark_identifier='klab.Zhang2018-ObjArray') + assert score.raw.sel(aggregation='center') == approx(0.407328, abs=.005) From ba40323eb77bad62f0ae7c86ec637bdf6d0612d7 Mon Sep 17 00:00:00 2001 From: shashikg Date: Sun, 26 Jan 2020 23:42:06 +0530 Subject: [PATCH 4/7] example for visual search benchmark --- .gitignore | 1 + examples/visual-search-score-model.ipynb | 140 +++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 examples/visual-search-score-model.ipynb diff --git a/.gitignore b/.gitignore index c95a93f..d9c7be6 100755 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ .ipynb_checkpoints build/* dist/* +candidate_models.egg-info/* diff --git a/examples/visual-search-score-model.ipynb b/examples/visual-search-score-model.ipynb new file mode 100644 index 0000000..b9dd3cc --- /dev/null +++ b/examples/visual-search-score-model.ipynb @@ -0,0 +1,140 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.simplefilter(\"ignore\")\n", + "\n", + "try:\n", + " from tensorflow.python.util import module_wrapper as deprecation\n", + "except ImportError:\n", + " from tensorflow.python.util import deprecation_wrapper as deprecation\n", + "deprecation._PER_MODULE_WARNING_LIMIT = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:\n", + "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", + "For more information, please see:\n", + " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", + " * https://github.com/tensorflow/addons\n", + " * https://github.com/tensorflow/io (for I/O related ops)\n", + "If you depend on functionality not listed there, please file an issue.\n", + "\n" + ] + } + ], + "source": [ + "from candidate_models import score_model\n", + "from candidate_models.model_commitments import brain_translated_pool\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "## Starting visual search task...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n", + "activations: 100%|██████████| 320/320 [00:01<00:00, 246.05it/s]\n", + "layer packaging: 100%|██████████| 1/1 [00:00<00:00, 98.36it/s]\n", + "activations: 100%|██████████| 320/320 [00:58<00:00, 5.47it/s]\n", + "layer packaging: 100%|██████████| 1/1 [00:00<00:00, 2.16it/s]\n", + "visual search stimuli: 100%|██████████| 300/300 [00:59<00:00, 5.08it/s]\n", + "comparing with human data: 0%| | 0/15 [00:00\n", + "array([0.923438, 0.005427])\n", + "Coordinates:\n", + " * aggregation (aggregation) \\narray([0.407328, ...\n", + " ceiling: \\narray([0.4411, ...\n", + " model_identifier: vgg-16\n", + " benchmark_identifier: klab.Zhang2018-ObjArray\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "identifier = 'vgg-16'\n", + "model = brain_translated_pool[identifier]\n", + "score = score_model(model_identifier=identifier, model=model, benchmark_identifier='klab.Zhang2018-ObjArray')\n", + "print(score)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 4f6f270f777a3d766aec3952ea4b981994169e8d Mon Sep 17 00:00:00 2001 From: shashikg Date: Sun, 26 Jan 2020 23:51:33 +0530 Subject: [PATCH 5/7] changes to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d9c7be6..ee34ce1 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.pyc +*.DS_Store __pycache__/ .ipynb_checkpoints build/* From 6edada729bcf3c2eddce130bf07e8a3df0a83045 Mon Sep 17 00:00:00 2001 From: shashikg Date: Tue, 18 Feb 2020 18:40:00 +0530 Subject: [PATCH 6/7] added visual search models inside ml_pool --- candidate_models/model_commitments/ml_pool.py | 32 +++++++++++-- examples/score-model.ipynb | 2 +- examples/visual-search-score-model.ipynb | 48 ++++++------------- tests/test_search.py | 2 +- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/candidate_models/model_commitments/ml_pool.py b/candidate_models/model_commitments/ml_pool.py index a274701..b6d63e2 100644 --- a/candidate_models/model_commitments/ml_pool.py +++ b/candidate_models/model_commitments/ml_pool.py @@ -7,6 +7,8 @@ from candidate_models.utils import UniqueKeyDict from model_tools.activations.pca import LayerPCA from model_tools.brain_transformation import ModelCommitment, PixelsToDegrees +from candidate_models.base_models import BaseModelPool +from candidate_models.model_commitments.vs_layer import visual_search_layer class Hooks: @@ -262,10 +264,14 @@ def __init__(self): class MLBrainPool(UniqueKeyDict): - def __init__(self): + def __init__(self, target_img_size=28, search_image_size=224): super(MLBrainPool, self).__init__() - for basemodel_identifier, activations_model in base_model_pool.items(): + # for visual search + target_model_pool = BaseModelPool(input_size=target_img_size) + stimuli_model_pool = BaseModelPool(input_size=search_image_size) + + for (basemodel_identifier, activations_model), (target_model_identifier, target_model), (stimuli_model_identifier, stimuli_model) in zip(base_model_pool.items(), target_model_pool.items(), stimuli_model_pool.items()): if basemodel_identifier not in model_layers: continue layers = model_layers[basemodel_identifier] @@ -274,10 +280,28 @@ def __init__(self): if identifier in self: # already pre-defined continue + # adding extra attributes for visual search model + search_target_model_param = {} + search_stimuli_model_param = {} + if basemodel_identifier == 'vgg-16': #as vs_layer is implemented only for vgg-16 as of now + search_target_model_param['target_model'] = target_model + search_stimuli_model_param['stimuli_model'] = stimuli_model + search_target_model_param['target_layer'] = visual_search_layer[basemodel_identifier][0] + search_stimuli_model_param['stimuli_layer'] = visual_search_layer[basemodel_identifier][0] + search_target_model_param['target_img_size'] = target_img_size + search_stimuli_model_param['search_image_size'] = search_image_size + else: + search_target_model_param['target_model'] = None + search_stimuli_model_param['stimuli_model'] = None + search_target_model_param['target_layer'] = None + search_stimuli_model_param['stimuli_layer'] = None + search_target_model_param['target_img_size'] = None + search_stimuli_model_param['search_image_size'] = None + # enforce early parameter binding: https://stackoverflow.com/a/3431699/2225200 self[identifier] = LazyLoad( - lambda identifier=identifier, activations_model=activations_model, layers=layers: - ModelCommitment(identifier=identifier, activations_model=activations_model, layers=layers)) + lambda identifier=identifier, activations_model=activations_model, layers=layers, search_target_model_param=search_target_model_param, search_stimuli_model_param=search_stimuli_model_param: + ModelCommitment(identifier=identifier, activations_model=activations_model, layers=layers, search_target_model_param=search_target_model_param, search_stimuli_model_param=search_stimuli_model_param)) ml_brain_pool = MLBrainPool() diff --git a/examples/score-model.ipynb b/examples/score-model.ipynb index 8ad577c..a3871c9 100644 --- a/examples/score-model.ipynb +++ b/examples/score-model.ipynb @@ -358,7 +358,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.7.6" } }, "nbformat": 4, diff --git a/examples/visual-search-score-model.ipynb b/examples/visual-search-score-model.ipynb index b9dd3cc..e1ffcd1 100644 --- a/examples/visual-search-score-model.ipynb +++ b/examples/visual-search-score-model.ipynb @@ -47,48 +47,23 @@ "execution_count": 3, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "## Starting visual search task...\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n", - "activations: 100%|██████████| 320/320 [00:01<00:00, 246.05it/s]\n", - "layer packaging: 100%|██████████| 1/1 [00:00<00:00, 98.36it/s]\n", - "activations: 100%|██████████| 320/320 [00:58<00:00, 5.47it/s]\n", - "layer packaging: 100%|██████████| 1/1 [00:00<00:00, 2.16it/s]\n", - "visual search stimuli: 100%|██████████| 300/300 [00:59<00:00, 5.08it/s]\n", - "comparing with human data: 0%| | 0/15 [00:00\n", "array([0.923438, 0.005427])\n", "Coordinates:\n", @@ -97,7 +72,7 @@ " raw: \\narray([0.407328, ...\n", " ceiling: \\narray([0.4411, ...\n", " model_identifier: vgg-16\n", - " benchmark_identifier: klab.Zhang2018-ObjArray\n" + " benchmark_identifier: klab.Zhang2018-object_search\n" ] }, { @@ -111,9 +86,16 @@ "source": [ "identifier = 'vgg-16'\n", "model = brain_translated_pool[identifier]\n", - "score = score_model(model_identifier=identifier, model=model, benchmark_identifier='klab.Zhang2018-ObjArray')\n", + "score = score_model(model_identifier=identifier, model=model, benchmark_identifier='klab.Zhang2018-object_search')\n", "print(score)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tests/test_search.py b/tests/test_search.py index bd9703d..367b3de 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -9,5 +9,5 @@ def test_search(): model = brain_translated_pool['vgg-16'] - score = score_model(model_identifier='vgg-16', model=model, benchmark_identifier='klab.Zhang2018-ObjArray') + score = score_model(model_identifier='vgg-16', model=model, benchmark_identifier='klab.Zhang2018-object_search') assert score.raw.sel(aggregation='center') == approx(0.407328, abs=.005) From 76ef12e52cc078562be51c716cca402a3a2a5093 Mon Sep 17 00:00:00 2001 From: shashikg Date: Wed, 22 Jul 2020 03:43:19 -0400 Subject: [PATCH 7/7] visual search - waldo and natural design --- candidate_models/model_commitments/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/candidate_models/model_commitments/__init__.py b/candidate_models/model_commitments/__init__.py index 4c3946f..1a2a8b6 100644 --- a/candidate_models/model_commitments/__init__.py +++ b/candidate_models/model_commitments/__init__.py @@ -27,7 +27,7 @@ def MLSearchPool(target_img_size=28, search_image_size=224): vs_model_param['stim_pool'] = stimuli_model_pool vs_model_param['model_layers'] = visual_search_layer vs_model_param['tar_size'] = target_img_size - vs_model_param['stim_size'] = stimuli_img_size + vs_model_param['stim_size'] = search_image_size ml_search_pool = MLBrainPool(base_model_pool, model_layers, vs_model_param=vs_model_param)