Merge pull request statsmodels#8825 from bashtage/pre-deprecations

bashtage · web-flow · commit f745e5bc72ed · 2023-04-23T12:38:01.000+01:00
MAINT: Remove casts from array to scalar
diff --git a/setup.cfg b/setup.cfg
@@ -77,6 +77,7 @@ filterwarnings =
     error:The probit link alias is deprecated:FutureWarning:
     error:Parsing dates in:UserWarning
     error:A value is trying to be set on a copy::
+    error:Conversion of an array with ndim:DeprecationWarning:statsmodels
 markers =
     example: mark a test that runs example code
     matplotlib: mark a test that requires matplotlib
diff --git a/statsmodels/base/model.py b/statsmodels/base/model.py
@@ -1912,7 +1912,7 @@ def wald_test(self, r_matrix, cov_p=None, invcov=None,
             )
             scalar = False
         if scalar and F.size == 1:
-            F = float(F)
+            F = float(np.squeeze(F))
         if use_f:
             F /= J
             return ContrastResults(F=F, df_denom=df_resid,
diff --git a/statsmodels/conftest.py b/statsmodels/conftest.py
@@ -16,11 +16,16 @@
 
 logger = logging.getLogger(__name__)
 
+cow = False
 try:
     cow = bool(os.environ.get("SM_TEST_COPY_ON_WRITE", False))
     pd.options.mode.copy_on_write = cow
-    logger.critical("Copy on Write Enabled!")
 except AttributeError:
+    pass
+
+if cow:
+    logger.critical("Copy on Write Enabled!")
+else:
     logger.critical("Copy on Write disabled")
 
 
diff --git a/statsmodels/discrete/discrete_model.py b/statsmodels/discrete/discrete_model.py
@@ -2080,7 +2080,7 @@ def hessian(self, params):
 
         for i in range(dim):
             for j in range(i + 1):
-                hess_arr[i,j] = np.sum(mu * exog[:,i,None] * exog[:,j,None] *
+                hess_val = np.sum(mu * exog[:,i,None] * exog[:,j,None] *
                     (mu * (a3 * a4 / a1**2 -
                            2 * a3**2 * a2 / a1**3 +
                            2 * a3 * (a4 + 1) / a1**2 -
@@ -2091,6 +2091,7 @@ def hessian(self, params):
                            a4 * (p - 1) / (a1 * mu)) +
                      ((y - 1) * (1 + a4) / a2 -
                       (1 + a4) / a1)), axis=0)
+                hess_arr[i, j] = np.squeeze(hess_val)
         tri_idx = np.triu_indices(dim, k=1)
         hess_arr[tri_idx] = hess_arr.T[tri_idx]
 
@@ -3447,8 +3448,11 @@ def _hessian_geom(self, params):
             for j in range(dim):
                 if j > i:
                     continue
-                hess_arr[i,j] = np.sum(-exog[:,i,None] * exog[:,j,None] *
-                                       const_arr, axis=0)
+                hess_arr[i,j] = np.squeeze(
+                    np.sum(-exog[:,i,None] * exog[:,j,None] * const_arr,
+                           axis=0
+                           )
+                )
         tri_idx = np.triu_indices(dim, k=1)
         hess_arr[tri_idx] = hess_arr.T[tri_idx]
         return hess_arr
@@ -3488,9 +3492,13 @@ def _hessian_nb1(self, params):
             for j in range(dim):
                 if j > i:
                     continue
-                hess_arr[i,j] = np.sum(dparams[:,i,None] * dmudb[:,j,None] +
-                                 xmu_alpha[:,i,None] * xmu_alpha[:,j,None] *
-                                 trigamma, axis=0)
+                hess_arr[i,j] = np.squeeze(
+                    np.sum(
+                        dparams[:,i,None] * dmudb[:,j,None] +
+                        xmu_alpha[:,i,None] * xmu_alpha[:,j,None] * trigamma,
+                        axis=0
+                    )
+                )
         tri_idx = np.triu_indices(dim, k=1)
         hess_arr[tri_idx] = hess_arr.T[tri_idx]
 
diff --git a/statsmodels/discrete/tests/test_truncated_model.py b/statsmodels/discrete/tests/test_truncated_model.py
@@ -359,7 +359,7 @@ def test_predict(self):
                         rtol=5e-4, atol=5e-4)
 
         prob_main = res1.results_count.predict(ex, which="prob")[0] * prob_nz
-        prob_main[0] = prob_zero
+        prob_main[0] = np.squeeze(prob_zero)
         assert_allclose(prob_main[:4], res2.predict_prob, rtol=5e-4, atol=5e-4)
 
         assert_allclose(mean_main * prob_nz, res2.predict_mean,
diff --git a/statsmodels/emplike/elanova.py b/statsmodels/emplike/elanova.py
@@ -113,7 +113,7 @@ def compute_ANOVA(self, mu=None, mu_start=0, return_weights=0):
             res = optimize.fmin_powell(self._opt_common_mu, mu_start,
                                        full_output=1, disp=False)
             llr = res[1]
-            mu_common = float(res[0])
+            mu_common = float(np.squeeze(res[0]))
             pval = 1 - chi2.cdf(llr, self.num_groups - 1)
             if return_weights:
                 return llr, pval, mu_common, self.new_weights
diff --git a/statsmodels/genmod/generalized_estimating_equations.py b/statsmodels/genmod/generalized_estimating_equations.py
@@ -2390,7 +2390,7 @@ def setup_ordinal(self, endog, exog, groups, time, offset):
             for thresh_ix, thresh in enumerate(endog_cuts):
 
                 exog_out[jrow, :] = exog_row
-                endog_out[jrow] = (int(endog_value > thresh))
+                endog_out[jrow] = int(np.squeeze(endog_value > thresh))
                 intercepts[jrow, thresh_ix] = 1
                 groups_out[jrow] = group_value
                 time_out[jrow] = time_value
diff --git a/statsmodels/graphics/gofplots.py b/statsmodels/graphics/gofplots.py
@@ -985,7 +985,7 @@ def _fmt_probplot_axis(ax, dist, nobs):
     axis_qntls = dist.ppf(axis_probs)
     ax.set_xticks(axis_qntls)
     ax.set_xticklabels(
-        axis_probs * 100,
+        [str(lbl) for lbl in (axis_probs * 100)],
         rotation=45,
         rotation_mode="anchor",
         horizontalalignment="right",
diff --git a/statsmodels/graphics/mosaicplot.py b/statsmodels/graphics/mosaicplot.py
@@ -326,7 +326,7 @@ def _normalize_dataframe(dataframe, index):
     #groupby the given keys, extract the same columns and count the element
     # then collapse them with a mean
     data = dataframe[index].dropna()
-    grouped = data.groupby(index, sort=False)
+    grouped = data.groupby(index, sort=False, observed=False)
     counted = grouped[index].count()
     averaged = counted.mean(axis=1)
     # Fill empty missing with 0, see GH5639
diff --git a/statsmodels/iolib/summary.py b/statsmodels/iolib/summary.py
@@ -18,6 +18,7 @@
 
 
 def forg(x, prec=3):
+    x = np.squeeze(x)
     if prec == 3:
         # for 3 decimals
         if (abs(x) >= 1e4) or (abs(x) < 1e-4):
diff --git a/statsmodels/iolib/summary2.py b/statsmodels/iolib/summary2.py
@@ -595,7 +595,10 @@ def _df_to_simpletable(df, align='r', float_format="%.4f", header=True,
                        index=True, table_dec_above='-', table_dec_below=None,
                        header_dec_below='-', pad_col=0, pad_index=0):
     dat = df.copy()
-    dat = dat.applymap(lambda x: _formatter(x, float_format))
+    try:
+        dat = dat.map(lambda x: _formatter(x, float_format))
+    except AttributeError:
+        dat = dat.applymap(lambda x: _formatter(x, float_format))
     if header:
         headers = [str(x) for x in dat.columns.tolist()]
     else:
diff --git a/statsmodels/nonparametric/kde.py b/statsmodels/nonparametric/kde.py
@@ -215,7 +215,7 @@ def cdf(self):
             a, b = kern.domain
 
         def func(x, s):
-            return kern.density(s, x)
+            return np.squeeze(kern.density(s, x))
 
         support = self.support
         support = np.r_[a, support]
diff --git a/statsmodels/nonparametric/kernel_regression.py b/statsmodels/nonparametric/kernel_regression.py
@@ -389,7 +389,7 @@ def fit(self, data_predict=None):
         for i in range(N_data_predict):
             mean_mfx = func(self.bw, self.endog, self.exog,
                             data_predict=data_predict[i, :])
-            mean[i] = mean_mfx[0]
+            mean[i] = np.squeeze(mean_mfx[0])
             mfx_c = np.squeeze(mean_mfx[1])
             mfx[i, :] = mfx_c
 
@@ -688,7 +688,7 @@ def fit(self, data_predict=None):
             mean_mfx = func(self.bw, self.endog, self.exog,
                             data_predict=data_predict[i, :],
                             W=self.W_in)
-            mean[i] = mean_mfx[0]
+            mean[i] = np.squeeze(mean_mfx[0])
             mfx_c = np.squeeze(mean_mfx[1])
             mfx[i, :] = mfx_c
 
diff --git a/statsmodels/regression/mixed_linear_model.py b/statsmodels/regression/mixed_linear_model.py
@@ -1979,9 +1979,9 @@ def hessian(self, params):
                         vt += 2 * _dot(_multi_dot_three(
                             vir[None, :], matr2, le), vir[:, None])
 
-                    D[jj1, jj2] += vt
+                    D[jj1, jj2] += np.squeeze(vt)
                     if jj1 != jj2:
-                        D[jj2, jj1] += vt
+                        D[jj2, jj1] += np.squeeze(vt)
 
                     rt = _dotsum(vsl2, re.T) / 2
                     if not sym2:
diff --git a/statsmodels/regression/rolling.py b/statsmodels/regression/rolling.py
@@ -673,7 +673,7 @@ def fvalue(self):
                 rp = p[i : i + 1] @ r.T
                 denom = rp.shape[1]
                 inv_cov = np.linalg.inv(rvcvr[i])
-                stat[i] = (rp @ inv_cov @ rp.T) / denom
+                stat[i] = np.squeeze(rp @ inv_cov @ rp.T) / denom
             return stat
 
     @cache_readonly
diff --git a/statsmodels/robust/norms.py b/statsmodels/robust/norms.py
@@ -1059,7 +1059,7 @@ def estimate_location(a, scale, norm=None, axis=0, initial=None,
     for _ in range(maxiter):
         W = norm.weights((a-mu)/scale)
         nmu = np.sum(W*a, axis) / np.sum(W, axis)
-        if np.alltrue(np.less(np.abs(mu - nmu), scale * tol)):
+        if np.all(np.less(np.abs(mu - nmu), scale * tol)):
             return nmu
         else:
             mu = nmu
diff --git a/statsmodels/sandbox/bspline.py b/statsmodels/sandbox/bspline.py
@@ -482,7 +482,7 @@ def fit(self, y, x=None, weights=None, pen=0.):
 
         # throw out rows with zeros (this happens at boundary points!)
 
-        mask = np.flatnonzero(1 - np.alltrue(np.equal(bt, 0), axis=0))
+        mask = np.flatnonzero(1 - np.all(np.equal(bt, 0), axis=0))
 
         bt = bt[:,mask]
         y = y[mask]
diff --git a/statsmodels/sandbox/distributions/extras.py b/statsmodels/sandbox/distributions/extras.py
@@ -817,7 +817,10 @@ def _sf(self, x, *args, **kwargs):
             return 1.0 - self._cdf(x, *args, **kwargs)
 
     def _munp(self, n, *args, **kwargs):
-        return self._mom0_sc(n, *args)
+        out = np.squeeze(self._mom0_sc(n, *args))
+        if np.isscalar(out):
+            return float(out)
+        return out
 
 
 # ppf might not be possible in general case?
diff --git a/statsmodels/sandbox/distributions/tests/test_transf.py b/statsmodels/sandbox/distributions/tests/test_transf.py
@@ -8,7 +8,7 @@
 
 todo:
 change moment calculation, (currently uses default _ppf method - I think)
->>> lognormalg.moment(4)
+# >>> lognormalg.moment(4)
 Warning: The algorithm does not converge.  Roundoff error is detected
   in the extrapolation table.  It is assumed that the requested tolerance
   cannot be achieved, and that the returned result (if full_output = 1) is
diff --git a/statsmodels/stats/diagnostic.py b/statsmodels/stats/diagnostic.py
@@ -1480,9 +1480,9 @@ def recursive_olsresiduals(res, skip=None, lamda=0.0, alpha=0.95,
 
         # get prediction error with previous beta
         yipred = np.dot(xi, beta)
-        rypred[i] = yipred
+        rypred[i] = np.squeeze(yipred)
         residi = yi - yipred
-        rresid[i] = residi
+        rresid[i] = np.squeeze(residi)
 
         # update beta and inverse(X'X)
         tmp = np.dot(xtxi, xi.T)
@@ -1492,7 +1492,7 @@ def recursive_olsresiduals(res, skip=None, lamda=0.0, alpha=0.95,
 
         beta = beta + (tmp * residi / ft).ravel()  # BigJudge equ 5.5.14
         rparams[i] = beta
-        rvarraw[i] = ft
+        rvarraw[i] = np.squeeze(ft)
 
     rresid_scaled = rresid / np.sqrt(rvarraw)  # N(0,sigma2) distributed
     nrr = nobs - skip
diff --git a/statsmodels/stats/libqsturng/qsturng_.py b/statsmodels/stats/libqsturng/qsturng_.py
@@ -839,7 +839,10 @@ def opt_func(p, r, v):
         soln = 1. - fminbound(opt_func, .1, .999, args=(r,v))
         return np.atleast_1d(soln)
 
-_vpsturng = np.vectorize(_psturng)
+def _psturng_scalar(q, r, v):
+    return np.squeeze(_psturng(q, r, v))
+
+_vpsturng = np.vectorize(_psturng_scalar)
 _vpsturng.__doc__ = """vector version of psturng"""
 
 def psturng(q, r, v):
diff --git a/statsmodels/stats/tests/test_multi.py b/statsmodels/stats/tests/test_multi.py
@@ -322,7 +322,8 @@ def test_fdr_bky():
     # also alpha_star is the same as theirs for TST
 
     # alpha_star for stage 2
-    res_tst = fdrcorrection_twostage(pvals, alpha=0.05, iter=False)
+    with pytest.warns(FutureWarning, match="iter keyword"):
+        res_tst = fdrcorrection_twostage(pvals, alpha=0.05, iter=False)
     assert_almost_equal([0.047619, 0.0649], res_tst[-1][:2], 3)
     assert_equal(8, res_tst[0].sum())
 
diff --git a/statsmodels/tools/numdiff.py b/statsmodels/tools/numdiff.py
@@ -336,9 +336,11 @@ def approx_hess_cs(x, f, epsilon=None, args=(), kwargs={}):
 
     for i in range(n):
         for j in range(i, n):
-            hess[i, j] = (f(*((x + 1j*ee[i, :] + ee[j, :],) + args), **kwargs)
+            hess[i, j] = np.squeeze(
+                (f(*((x + 1j*ee[i, :] + ee[j, :],) + args), **kwargs)
                           - f(*((x + 1j*ee[i, :] - ee[j, :],)+args),
                               **kwargs)).imag/2./hess[i, j]
+            )
             hess[j, i] = hess[i, j]
 
     return hess
@@ -447,11 +449,13 @@ def approx_hess3(x, f, epsilon=None, args=(), kwargs={}):
 
     for i in range(n):
         for j in range(i, n):
-            hess[i, j] = (f(*((x + ee[i, :] + ee[j, :],) + args), **kwargs)
-                          - f(*((x + ee[i, :] - ee[j, :],) + args), **kwargs)
-                          - (f(*((x - ee[i, :] + ee[j, :],) + args), **kwargs)
-                          - f(*((x - ee[i, :] - ee[j, :],) + args), **kwargs))
-                          )/(4.*hess[i, j])
+            hess[i, j] = np.squeeze(
+                (f(*((x + ee[i, :] + ee[j, :],) + args), **kwargs)
+                 - f(*((x + ee[i, :] - ee[j, :],) + args), **kwargs)
+                 - (f(*((x - ee[i, :] + ee[j, :],) + args), **kwargs)
+                    - f(*((x - ee[i, :] - ee[j, :],) + args), **kwargs))
+                 )/(4.*hess[i, j])
+            )
             hess[j, i] = hess[i, j]
     return hess
 
diff --git a/statsmodels/tools/transform_model.py b/statsmodels/tools/transform_model.py
@@ -51,7 +51,7 @@ def __init__(self, data, ddof=1, const_idx=None, demean=True):
             if len(const_idx) == 0:
                 const_idx = 'n'
             else:
-                const_idx = int(const_idx)
+                const_idx = int(np.squeeze(const_idx))
 
         if const_idx != 'n':
             self.mean[const_idx] = 0
diff --git a/statsmodels/tsa/ar_model.py b/statsmodels/tsa/ar_model.py
@@ -666,7 +666,7 @@ def _dynamic_predict(
                     # If before the start of the forecasts, use actual values
                     val = self.endog[fcast_loc + start]
                 _reg[h, det_col_idx + j] = val
-            forecasts[h] = _reg[h : h + 1] @ params
+            forecasts[h] = np.squeeze(_reg[h : h + 1] @ params)
         return self._wrap_prediction(forecasts, start, end + 1 + num_oos, adj)
 
     def _static_oos_predict(
@@ -682,8 +682,8 @@ def _static_oos_predict(
             for j, lag in enumerate(self._lags):
                 loc = i - lag
                 val = self._y[loc] if loc < 0 else forecasts[loc]
-                new_x[i, ar_offset + j] = val
-            forecasts[i] = new_x[i : i + 1] @ params
+                new_x[i, ar_offset + j] = np.squeeze(val)
+            forecasts[i] = np.squeeze(new_x[i : i + 1] @ params)
         return forecasts
 
     def _static_predict(
diff --git a/statsmodels/tsa/regime_switching/tests/test_markov_regression.py b/statsmodels/tsa/regime_switching/tests/test_markov_regression.py
@@ -1284,10 +1284,12 @@ def test_exog_tvtp():
 
     params = np.r_[0.98209618, 0.05036498, 3.70877542, 9.55676298, 4.44181911]
     params_tvtp = params.copy()
-    params_tvtp[0] = mod2._untransform_logistic(
-        np.r_[0.], np.r_[1 - params[0]])
-    params_tvtp[1] = mod2._untransform_logistic(
-        np.r_[0.], np.r_[1 - params[1]])
+    params_tvtp[0] = np.squeeze(
+        mod2._untransform_logistic(np.r_[0.], np.r_[1 - params[0]])
+    )
+    params_tvtp[1] = np.squeeze(
+        mod2._untransform_logistic(np.r_[0.], np.r_[1 - params[1]])
+    )
 
     res1 = mod1.smooth(params)
     res2 = mod2.smooth(params_tvtp)
diff --git a/statsmodels/tsa/statespace/dynamic_factor_mq.py b/statsmodels/tsa/statespace/dynamic_factor_mq.py
diff --git a/statsmodels/tsa/statespace/kalman_filter.py b/statsmodels/tsa/statespace/kalman_filter.py
diff --git a/statsmodels/tsa/statespace/kalman_smoother.py b/statsmodels/tsa/statespace/kalman_smoother.py
diff --git a/statsmodels/tsa/stattools.py b/statsmodels/tsa/stattools.py
diff --git a/statsmodels/tsa/vector_ar/var_model.py b/statsmodels/tsa/vector_ar/var_model.py

Original file line number	Diff line number	Diff line change
`@@ -1912,7 +1912,7 @@ def wald_test(self, r_matrix, cov_p=None, invcov=None,`
`1912`	`1912`	`)`
`1913`	`1913`	`scalar = False`
`1914`	`1914`	`if scalar and F.size == 1:`
`1915`		`- F = float(F)`
	`1915`	`+ F = float(np.squeeze(F))`
`1916`	`1916`	`if use_f:`
`1917`	`1917`	`F /= J`
`1918`	`1918`	`return ContrastResults(F=F, df_denom=df_resid,`