diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 47c04ee323..4cbb2a6c61 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -25,6 +25,7 @@ It also brings some dreadfully awaited fixes, so be sure to go through the chang
 - The notebook gallery has been moved to https://github.com/pymc-devs/pymc-examples (see [#4348](https://github.com/pymc-devs/pymc3/pull/4348)).
 - `math.logsumexp` now matches `scipy.special.logsumexp` when arrays contain infinite values (see [#4360](https://github.com/pymc-devs/pymc3/pull/4360)).
 - Fixed mathematical formulation in `MvStudentT` random method. (see [#4359](https://github.com/pymc-devs/pymc3/pull/4359))
+- Fix issue in `logp` method of `HyperGeometric`. It now returns `-inf` for invalid parameters (see [4367](https://github.com/pymc-devs/pymc3/pull/4367))
 
 ## PyMC3 3.10.0 (7 December 2020)
 
diff --git a/pymc3/distributions/discrete.py b/pymc3/distributions/discrete.py
index c835e2dc44..e639ba6684 100644
--- a/pymc3/distributions/discrete.py
+++ b/pymc3/distributions/discrete.py
@@ -930,7 +930,10 @@ def logp(self, value):
             - betaln(n - value + 1, bad - n + value + 1)
             - betaln(tot + 1, 1)
         )
-        return result
+        # value in [max(0, n - N + k), min(k, n)]
+        lower = tt.switch(tt.gt(n - N + k, 0), n - N + k, 0)
+        upper = tt.switch(tt.lt(k, n), k, n)
+        return bound(result, lower <= value, value <= upper)
 
 
 class DiscreteUniform(Discrete):
diff --git a/pymc3/tests/test_distributions.py b/pymc3/tests/test_distributions.py
index daffdbe8e1..16ed273488 100644
--- a/pymc3/tests/test_distributions.py
+++ b/pymc3/tests/test_distributions.py
@@ -805,11 +805,15 @@ def test_geometric(self):
         )
 
     def test_hypergeometric(self):
+        def modified_scipy_hypergeom_logpmf(value, N, k, n):
+            original_res = sp.hypergeom.logpmf(value, N, k, n)
+            return original_res if not np.isnan(original_res) else -np.inf
+
         self.pymc3_matches_scipy(
             HyperGeometric,
             Nat,
             {"N": NatSmall, "k": NatSmall, "n": NatSmall},
-            lambda value, N, k, n: sp.hypergeom.logpmf(value, N, k, n),
+            lambda value, N, k, n: modified_scipy_hypergeom_logpmf(value, N, k, n),
         )
 
     def test_negative_binomial(self):