Fix Inf Values in PYBNN Benchmark (#114)

* Fix a bug in the pybnn benchmarks.
  Limit the variance before computing the nll. (inf otherwise)

Author: PhMueller

changelog.md

@@ -8,8 +8,12 @@
     * We also change the configuration file. The container does not read the yaml file anymore. Instead we bind the 
       cache dir, data dir and socket dir into the container and let the container use them directly. We also remove the 
       global data directory and use only the data dir from now onwards.
-  * Add the surrogate SVM on MNIST benchmark from the BOHB paper. 
-
+  * Add the surrogate SVM on MNIST benchmark from the BOHB paper.
+  * Fix an error in PyBnn Benchmark:
+    We introduce the benchmark version 0.0.4.  
+    In this new version, we prevent infinity values in the nll loss when the predicted variance
+    is 0. We set the predicted variance before computing the log to max(var, 1e-10)
+         
 # 0.0.7
   * Fix an error in the NASBench1shot1 Benchmark (SearchSpace3).
   * Improve the behavior when a benchmark container is shut down.

ci_scripts/install_singularity.sh

@@ -32,4 +32,4 @@ make -C builddir && \
 sudo make -C builddir install
 
 cd ..
-pip install .[singulaity]
+pip install .

hpobench/benchmarks/ml/pybnn.py

@@ -10,6 +10,9 @@
 
 Changelog:
 ==========
+0.0.4
+* Limit variance in NLL loss to prevent very large values.
+
 0.0.3
 * New container release due to a general change in the communication between container and HPOBench.
   Works with HPOBench >= v0.0.8
@@ -51,7 +54,7 @@
 from sgmcmc.bnn.lasagne_layers import AppendLayer  # noqa: E402
 
 
-__version__ = '0.0.3'
+__version__ = '0.0.4'
 logger = logging.getLogger('PyBnnBenchmark')
 
 
@@ -220,6 +223,7 @@ def objective_function_test(self, configuration: Union[Dict, CS.Configuration],
     @staticmethod
     def _neg_log_likelihood(targets: np.ndarray, mean_pred: np.ndarray, var_pred: np.ndarray) -> np.ndarray:
         """ Compute the negative log likelihood for normal distributions. """
+        var_pred = np.clip(var_pred, a_min=1e-10, a_max=np.inf)
         nll = [stats.norm.logpdf(targets[i], loc=mean_pred[i], scale=np.sqrt(var_pred[i]))
                for i in range(targets.shape[0])]
         return -np.mean(nll)

hpobench/container/benchmarks/ml/pybnn.py

@@ -10,29 +10,29 @@ class BNNOnToyFunction(AbstractBenchmarkClient):
     def __init__(self, **kwargs):
         kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnToyFunction')
         kwargs['container_name'] = kwargs.get('container_name', 'pybnn')
-        kwargs['latest'] = kwargs.get('container_tag', '0.0.3')
+        kwargs['latest'] = kwargs.get('container_tag', '0.0.4')
         super(BNNOnToyFunction, self).__init__(**kwargs)
 
 
 class BNNOnBostonHousing(AbstractBenchmarkClient):
     def __init__(self, **kwargs):
         kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnBostonHousing')
         kwargs['container_name'] = kwargs.get('container_name', 'pybnn')
-        kwargs['latest'] = kwargs.get('container_tag', '0.0.3')
+        kwargs['latest'] = kwargs.get('container_tag', '0.0.4')
         super(BNNOnBostonHousing, self).__init__(**kwargs)
 
 
 class BNNOnProteinStructure(AbstractBenchmarkClient):
     def __init__(self, **kwargs):
         kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnProteinStructure')
         kwargs['container_name'] = kwargs.get('container_name', 'pybnn')
-        kwargs['latest'] = kwargs.get('container_tag', '0.0.3')
+        kwargs['latest'] = kwargs.get('container_tag', '0.0.4')
         super(BNNOnProteinStructure, self).__init__(**kwargs)
 
 
 class BNNOnYearPrediction(AbstractBenchmarkClient):
     def __init__(self, **kwargs):
         kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnYearPrediction')
         kwargs['container_name'] = kwargs.get('container_name', 'pybnn')
-        kwargs['latest'] = kwargs.get('container_tag', '0.0.3')
+        kwargs['latest'] = kwargs.get('container_tag', '0.0.4')
         super(BNNOnYearPrediction, self).__init__(**kwargs)