From 03e094f5b7cd664fb42756a5c3ee8c050d52f14c Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 19:10:42 +0200 Subject: [PATCH 1/6] Run test_distributions in its own job because it's so slow --- .github/workflows/pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5e0807bc53..8c821de9d3 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -57,6 +57,8 @@ jobs: --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_idata_conversion.py + - pymc3/tests/test_distributions.py + - | pymc3/tests/test_modelcontext.py pymc3/tests/test_dist_math.py @@ -67,7 +69,6 @@ jobs: - | pymc3/tests/test_idata_conversion.py - pymc3/tests/test_distributions.py pymc3/tests/test_distributions_random.py pymc3/tests/test_examples.py pymc3/tests/test_gp.py From 554d7f559d9c19109486524356a27e5520b8cbea Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 19:11:57 +0200 Subject: [PATCH 2/6] Reenable timeseries tests --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8c821de9d3..5219c07ab3 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -70,6 +70,7 @@ jobs: - | pymc3/tests/test_idata_conversion.py pymc3/tests/test_distributions_random.py + pymc3/tests/test_distributions_timeseries.py pymc3/tests/test_examples.py pymc3/tests/test_gp.py pymc3/tests/test_model.py @@ -141,6 +142,7 @@ jobs: test-subset: - | pymc3/tests/test_distributions_random.py + pymc3/tests/test_distributions_timeseries.py - | pymc3/tests/test_sampling.py pymc3/tests/test_shared.py From 7bb5a3349fc516dd2fb10d0b88da6171b0859604 Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 19:39:00 +0200 Subject: [PATCH 3/6] Reenable test_parallel_sampling and test_profile A few XFAILs were added because DensityDist is not yet refactored and a shape issues causes the CompoundStep to fail. --- .github/workflows/pytest.yml | 8 ++++ pymc3/tests/test_parallel_sampling.py | 68 +++++++++++++++++---------- pymc3/tests/test_profile.py | 5 +- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5219c07ab3..aef9832d21 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -34,6 +34,7 @@ jobs: --ignore=pymc3/tests/test_model_graph.py --ignore=pymc3/tests/test_modelcontext.py --ignore=pymc3/tests/test_parallel_sampling.py + --ignore=pymc3/tests/test_sampling.py --ignore=pymc3/tests/test_profile.py --ignore=pymc3/tests/test_step.py --ignore=pymc3/tests/test_tuning.py @@ -67,6 +68,10 @@ jobs: pymc3/tests/test_plots.py pymc3/tests/test_updates.py + - | + pymc3/tests/test_parallel_sampling.py + pymc3/tests/test_sampling.py + - | pymc3/tests/test_idata_conversion.py pymc3/tests/test_distributions_random.py @@ -78,6 +83,7 @@ jobs: pymc3/tests/test_model_graph.py pymc3/tests/test_ode.py pymc3/tests/test_posdef_sym.py + pymc3/tests/test_profile.py pymc3/tests/test_quadpotential.py pymc3/tests/test_shape_handling.py pymc3/tests/test_step.py @@ -144,6 +150,7 @@ jobs: pymc3/tests/test_distributions_random.py pymc3/tests/test_distributions_timeseries.py - | + pymc3/tests/test_parallel_sampling.py pymc3/tests/test_sampling.py pymc3/tests/test_shared.py - | @@ -155,6 +162,7 @@ jobs: pymc3/tests/test_modelcontext.py pymc3/tests/test_model_graph.py pymc3/tests/test_pickling.py + pymc3/tests/test_profile.py fail-fast: false runs-on: ${{ matrix.os }} diff --git a/pymc3/tests/test_parallel_sampling.py b/pymc3/tests/test_parallel_sampling.py index d8cc36620c..5348cd72ad 100644 --- a/pymc3/tests/test_parallel_sampling.py +++ b/pymc3/tests/test_parallel_sampling.py @@ -89,39 +89,51 @@ def test_remote_pipe_closed(): pm.sample(step=step, mp_ctx="spawn", tune=2, draws=2, cores=2, chains=2) +@pytest.mark.xfail( + reason="Possibly the same issue described in https://github.com/pymc-devs/pymc3/pull/4701" +) def test_abort(): with pm.Model() as model: a = pm.Normal("a", shape=1) pm.HalfNormal("b") step1 = pm.NUTS([a]) - step2 = pm.Metropolis([model.b_log__]) + step2 = pm.Metropolis([model["b_log__"]]) step = pm.CompoundStep([step1, step2]) - ctx = multiprocessing.get_context() - proc = ps.ProcessAdapter( - 10, - 10, - step, - chain=3, - seed=1, - mp_ctx=ctx, - start={"a": 1.0, "b_log__": 2.0}, - step_method_pickled=None, - pickle_backend="pickle", - ) - proc.start() - proc.write_next() - proc.abort() - proc.join() - - + for abort in [False, True]: + ctx = multiprocessing.get_context() + proc = ps.ProcessAdapter( + 10, + 10, + step, + chain=3, + seed=1, + mp_ctx=ctx, + start={"a": np.array([1.0]), "b_log__": np.array(2.0)}, + step_method_pickled=None, + pickle_backend="pickle", + ) + proc.start() + while True: + proc.write_next() + out = ps.ProcessAdapter.recv_draw([proc]) + if out[1]: + break + if abort: + proc.abort() + proc.join() + + +@pytest.mark.xfail( + reason="Possibly the same issue described in https://github.com/pymc-devs/pymc3/pull/4701" +) def test_explicit_sample(): with pm.Model() as model: a = pm.Normal("a", shape=1) pm.HalfNormal("b") step1 = pm.NUTS([a]) - step2 = pm.Metropolis([model.b_log__]) + step2 = pm.Metropolis([model["b_log__"]]) step = pm.CompoundStep([step1, step2]) @@ -133,7 +145,7 @@ def test_explicit_sample(): chain=3, seed=1, mp_ctx=ctx, - start={"a": 1.0, "b_log__": 2.0}, + start={"a": np.array([1.0]), "b_log__": np.array(2.0)}, step_method_pickled=None, pickle_backend="pickle", ) @@ -149,22 +161,26 @@ def test_explicit_sample(): proc.join() +@pytest.mark.xfail( + reason="Possibly the same issue described in https://github.com/pymc-devs/pymc3/pull/4701" +) def test_iterator(): with pm.Model() as model: a = pm.Normal("a", shape=1) pm.HalfNormal("b") step1 = pm.NUTS([a]) - step2 = pm.Metropolis([model.b_log__]) + step2 = pm.Metropolis([model["b_log__"]]) step = pm.CompoundStep([step1, step2]) - start = {"a": 1.0, "b_log__": 2.0} + start = {"a": np.array([1.0]), "b_log__": np.array(2.0)} sampler = ps.ParallelSampler(10, 10, 3, 2, [2, 3, 4], [start] * 3, step, 0, False) with sampler: for draw in sampler: pass +@pytest.mark.xfail(reason="DensityDist was not yet refactored for v4") def test_spawn_densitydist_function(): with pm.Model() as model: mu = pm.Normal("mu", 0, 1) @@ -176,16 +192,19 @@ def func(x): pm.sample(draws=10, tune=10, step=pm.Metropolis(), cores=2, mp_ctx="spawn") +@pytest.mark.xfail(reason="DensityDist was not yet refactored for v4") def test_spawn_densitydist_bound_method(): with pm.Model() as model: mu = pm.Normal("mu", 0, 1) normal_dist = pm.Normal.dist(mu, 1) - obs = pm.DensityDist("density_dist", normal_dist.logp, observed=np.random.randn(100)) + logp = lambda x: pm.logp(normal_dist, x, transformed=False) + obs = pm.DensityDist("density_dist", logp, observed=np.random.randn(100)) msg = "logp for DensityDist is a bound method, leading to RecursionError while serializing" with pytest.raises(ValueError, match=msg): pm.sample(draws=10, tune=10, step=pm.Metropolis(), cores=2, mp_ctx="spawn") +@pytest.mark.xfail(reason="DensityDist was not yet refactored for v4") def test_spawn_densitydist_syswarning(monkeypatch): monkeypatch.setattr("pymc3.distributions.distribution.PLATFORM", "win32") with pm.Model() as model: @@ -195,6 +214,7 @@ def test_spawn_densitydist_syswarning(monkeypatch): obs = pm.DensityDist("density_dist", normal_dist.logp, observed=np.random.randn(100)) +@pytest.mark.xfail(reason="DensityDist was not yet refactored for v4") def test_spawn_densitydist_mpctxwarning(monkeypatch): ctx = multiprocessing.get_context("spawn") monkeypatch.setattr(multiprocessing, "get_context", lambda: ctx) diff --git a/pymc3/tests/test_profile.py b/pymc3/tests/test_profile.py index e7a7d5af2a..39d34a075e 100644 --- a/pymc3/tests/test_profile.py +++ b/pymc3/tests/test_profile.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pymc3 as pm + from pymc3.tests.models import simple_model @@ -23,7 +25,8 @@ def test_profile_model(self): assert self.model.profile(self.model.logpt).fct_call_time > 0 def test_profile_variable(self): - assert self.model.profile(self.model.value_vars[0].logpt).fct_call_time > 0 + rv = self.model.basic_RVs[0] + assert self.model.profile(pm.logpt(rv, self.model.rvs_to_values[rv])).fct_call_time def test_profile_count(self): count = 1005 From c4aca95007802c7fe56392181edda46dc7be961c Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 21:00:50 +0200 Subject: [PATCH 4/6] Reenable test_tuning --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index aef9832d21..ff6a0edc81 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -71,6 +71,7 @@ jobs: - | pymc3/tests/test_parallel_sampling.py pymc3/tests/test_sampling.py + pymc3/tests/test_tuning.py - | pymc3/tests/test_idata_conversion.py @@ -152,6 +153,7 @@ jobs: - | pymc3/tests/test_parallel_sampling.py pymc3/tests/test_sampling.py + pymc3/tests/test_tuning.py pymc3/tests/test_shared.py - | pymc3/tests/test_gp.py From 45ae2443199d3cd06ea46e6fd190a0ee23c09534 Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 21:13:19 +0200 Subject: [PATCH 5/6] Skip numerically unstable tests instead of marking them as XFAIL This way they don't XPASS sometimes. Also it doesn't waste compute resources on tests whoose outcome has no relevance. --- pymc3/tests/test_distributions.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pymc3/tests/test_distributions.py b/pymc3/tests/test_distributions.py index bdc2bf991a..742b0ee3cc 100644 --- a/pymc3/tests/test_distributions.py +++ b/pymc3/tests/test_distributions.py @@ -1056,7 +1056,7 @@ def test_chisquared_logp(self): lambda value, nu: sp.chi2.logpdf(value, df=nu), ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to numerical issues", ) @@ -1423,7 +1423,7 @@ def test_fun(value, mu, sigma): test_fun, ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to numerical issues", ) @@ -1449,7 +1449,7 @@ def test_inverse_gamma_logp(self): # pymc-devs/aesara#224: skip_paramdomain_outside_edge_test has to be set # True to avoid triggering a C-level assertion in the Aesara GammaQ function - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to numerical issues", ) @@ -1465,7 +1465,7 @@ def test_inverse_gamma_logcdf(self): skip_paramdomain_outside_edge_test=True, ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to scaling issues", ) @@ -1496,7 +1496,7 @@ def test_pareto(self): lambda value, alpha, m: sp.pareto.logcdf(value, alpha, scale=m), ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to numerical issues", ) @@ -1508,7 +1508,7 @@ def test_weibull_logp(self): lambda value, alpha, beta: sp.exponweib.logpdf(value, 1, alpha, scale=beta), ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to inf issues", ) @@ -1560,7 +1560,7 @@ def test_binomial(self): ) @pytest.mark.xfail(reason="checkd tests has not been refactored") - @pytest.mark.xfail(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") + @pytest.mark.skipif(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") def test_beta_binomial_distribution(self): self.checkd( BetaBinomial, @@ -1681,7 +1681,7 @@ def test_constantdist(self): self.check_logp(Constant, I, {"c": I}, lambda value, c: np.log(c == value)) @pytest.mark.xfail(reason="Test has not been refactored") - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to inf issues", ) @@ -1723,7 +1723,7 @@ def logcdf_fn(value, psi, theta): ) @pytest.mark.xfail(reason="Test not refactored yet") - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to inf issues", ) @@ -1860,7 +1860,7 @@ def test_mvnormal(self, n): extra_args={"lower": False}, ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Fails on float32 due to inf issues", ) @@ -2521,7 +2521,7 @@ def test_ex_gaussian_cdf_outside_edges(self): skip_paramdomain_inside_edge_test=True, # Valid values are tested above ) - @pytest.mark.xfail(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") + @pytest.mark.skipif(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") def test_vonmises(self): self.check_logp( VonMises, @@ -2571,7 +2571,7 @@ def test_logitnormal(self): decimal=select_by_precision(float64=6, float32=1), ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Some combinations underflow to -inf in float32 in pymc version", ) @@ -2603,7 +2603,7 @@ def test_moyal_logp(self): lambda value, mu, sigma: floatX(sp.moyal.logpdf(value, mu, sigma)), ) - @pytest.mark.xfail( + @pytest.mark.skipif( condition=(aesara.config.floatX == "float32"), reason="Pymc3 underflows earlier than scipy on float32", ) @@ -2617,7 +2617,7 @@ def test_moyal_logcdf(self): if aesara.config.floatX == "float32": raise Exception("Flaky test: It passed this time, but XPASS is not allowed.") - @pytest.mark.xfail(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") + @pytest.mark.skipif(condition=(aesara.config.floatX == "float32"), reason="Fails on float32") def test_interpolated(self): for mu in R.vals: for sigma in Rplus.vals: From d0ec6fb964013705fa9a2de386c657e99cfd97bd Mon Sep 17 00:00:00 2001 From: Michael Osthege Date: Thu, 1 Jul 2021 22:32:05 +0200 Subject: [PATCH 6/6] Redistribute slow tests from the longest-running job --- .github/workflows/pytest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ff6a0edc81..5b591c7772 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -34,10 +34,12 @@ jobs: --ignore=pymc3/tests/test_model_graph.py --ignore=pymc3/tests/test_modelcontext.py --ignore=pymc3/tests/test_parallel_sampling.py + --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py --ignore=pymc3/tests/test_profile.py --ignore=pymc3/tests/test_step.py --ignore=pymc3/tests/test_tuning.py + --ignore=pymc3/tests/test_transforms.py --ignore=pymc3/tests/test_types.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_sampling_jax.py @@ -67,11 +69,13 @@ jobs: pymc3/tests/test_pickling.py pymc3/tests/test_plots.py pymc3/tests/test_updates.py + pymc3/tests/test_transforms.py - | pymc3/tests/test_parallel_sampling.py pymc3/tests/test_sampling.py pymc3/tests/test_tuning.py + pymc3/tests/test_posteriors.py - | pymc3/tests/test_idata_conversion.py