diff --git a/docs/changelog.rst b/docs/changelog.rst
index 697e483..54e6bba 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -1,6 +1,13 @@
 Changelog
 =========
 
+2.0.3
+-----
+Bug fixes
+~~~~~~~~~
+- Fix an error that happened when the histogram widget was open, but a layer that doesn't support
+  histogramming (e.g., a labels layer) was selected.
+
 2.0.2
 -----
 Dependencies
diff --git a/src/napari_matplotlib/base.py b/src/napari_matplotlib/base.py
index c455335..720333e 100644
--- a/src/napari_matplotlib/base.py
+++ b/src/napari_matplotlib/base.py
@@ -224,6 +224,15 @@ def _setup_callbacks(self) -> None:
             self._update_layers
         )
 
+    @property
+    def _valid_layer_selection(self) -> bool:
+        """
+        Return `True` if layer selection is valid.
+        """
+        return self.n_selected_layers in self.n_layers_input and all(
+            isinstance(layer, self.input_layer_types) for layer in self.layers
+        )
+
     def _update_layers(self, event: napari.utils.events.Event) -> None:
         """
         Update the ``layers`` attribute with currently selected layers and re-draw.
@@ -231,7 +240,8 @@ def _update_layers(self, event: napari.utils.events.Event) -> None:
         self.layers = list(self.viewer.layers.selection)
         self.layers = sorted(self.layers, key=lambda layer: layer.name)
         self.on_update_layers()
-        self._draw()
+        if self._valid_layer_selection:
+            self._draw()
 
     def _draw(self) -> None:
         """
@@ -243,10 +253,7 @@ def _draw(self) -> None:
         with mplstyle.context(self.napari_theme_style_sheet):
             # everything should be done in the style context
             self.clear()
-            if self.n_selected_layers in self.n_layers_input and all(
-                isinstance(layer, self.input_layer_types)
-                for layer in self.layers
-            ):
+            if self._valid_layer_selection:
                 self.draw()
             self.canvas.draw()  # type: ignore[no-untyped-call]
 
diff --git a/src/napari_matplotlib/histogram.py b/src/napari_matplotlib/histogram.py
index adbbae6..2881cf7 100644
--- a/src/napari_matplotlib/histogram.py
+++ b/src/napari_matplotlib/histogram.py
@@ -55,8 +55,10 @@ def on_update_layers(self) -> None:
         Called when the selected layers are updated.
         """
         super().on_update_layers()
-        for layer in self.viewer.layers:
-            layer.events.contrast_limits.connect(self._update_contrast_lims)
+        if self._valid_layer_selection:
+            self.layers[0].events.contrast_limits.connect(
+                self._update_contrast_lims
+            )
 
     def _update_contrast_lims(self) -> None:
         for lim, line in zip(
@@ -209,10 +211,12 @@ def draw(self) -> None:
         # get the colormap from the layer depending on its type
         if isinstance(self.layers[0], napari.layers.Points):
             colormap = self.layers[0].face_colormap
-            self.layers[0].face_color = self.x_axis_key
+            if self.x_axis_key:
+                self.layers[0].face_color = self.x_axis_key
         elif isinstance(self.layers[0], napari.layers.Vectors):
             colormap = self.layers[0].edge_colormap
-            self.layers[0].edge_color = self.x_axis_key
+            if self.x_axis_key:
+                self.layers[0].edge_color = self.x_axis_key
         else:
             colormap = None