From 9083e5d2287181c35455ab82ac34be6465c05dfd Mon Sep 17 00:00:00 2001 From: Deborah Date: Mon, 8 Jun 2020 14:35:41 -0400 Subject: [PATCH 01/13] #34640: CLN: remove 'private_key' and 'verbose' from gbq --- pandas/io/gbq.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 9b46f970afc66..d60e352d5a8dd 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -30,8 +30,6 @@ def read_gbq( configuration: Optional[Dict[str, Any]] = None, credentials=None, use_bqstorage_api: Optional[bool] = None, - private_key=None, - verbose=None, progress_bar_type: Optional[str] = None, ) -> "DataFrame": """ @@ -198,8 +196,6 @@ def to_gbq( location: Optional[str] = None, progress_bar: bool = True, credentials=None, - verbose=None, - private_key=None, ) -> None: pandas_gbq = _try_import() pandas_gbq.to_gbq( @@ -214,6 +210,4 @@ def to_gbq( location=location, progress_bar=progress_bar, credentials=credentials, - verbose=verbose, - private_key=private_key, ) From d60a3608ae93617c630f43a293b425349882c7b9 Mon Sep 17 00:00:00 2001 From: Deborah Date: Mon, 15 Jun 2020 14:28:15 -0400 Subject: [PATCH 02/13] close GH 34640 (updated) --- doc/source/whatsnew/v1.1.0.rst | 2 +- pandas/io/parsers.py | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index b2b55b7b503ec..13d7621c05e19 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -379,7 +379,7 @@ Other API changes - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`) - Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max``, ``median``, ``skew``, ``cov``, ``corr`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`) - Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations. -- +- Removed ``private_key`` and ``verbose`` from gbq (:issue:`34654`) Backwards incompatible API changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index c54e264faedd2..3a328ac3c9c32 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -215,8 +215,6 @@ Detect missing value markers (empty strings and the value of na_values). In data without any NAs, passing na_filter=False can improve the performance of reading a large file. -verbose : bool, default False - Indicate number of NA values placed in non-numeric columns. skip_blank_lines : bool, default True If True, skip over blank lines rather than interpreting as NaN values. parse_dates : bool or list of int or names or list of lists or dict, \ @@ -498,7 +496,6 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): "usecols": None, # 'iterator': False, "chunksize": None, - "verbose": False, "encoding": None, "squeeze": False, "compression": None, @@ -554,7 +551,6 @@ def parser_f( na_values=None, keep_default_na=True, na_filter=True, - verbose=False, skip_blank_lines=True, # Datetime Handling parse_dates=False, @@ -658,7 +654,6 @@ def parser_f( converters=converters, dtype=dtype, usecols=usecols, - verbose=verbose, encoding=encoding, squeeze=squeeze, memory_map=memory_map, @@ -1710,7 +1705,7 @@ def _agg_index(self, index, try_parse_dates=True): return index def _convert_to_ndarrays( - self, dct, na_values, na_fvalues, verbose=False, converters=None, dtypes=None + self, dct, na_values, na_fvalues, converters=None, dtypes=None ): result = {} for c, values in dct.items(): @@ -1780,7 +1775,7 @@ def _convert_to_ndarrays( cvals = self._cast_types(cvals, cast_type, c) result[c] = cvals - if verbose and na_count: + if na_count: print(f"Filled {na_count} NA values in column {c!s}") return result @@ -2303,7 +2298,6 @@ def __init__(self, f, **kwds): if "has_index_names" in kwds: self.has_index_names = kwds["has_index_names"] - self.verbose = kwds["verbose"] self.converters = kwds["converters"] self.dtype = kwds["dtype"] @@ -2588,7 +2582,6 @@ def _clean_mapping(mapping): data, clean_na_values, clean_na_fvalues, - self.verbose, clean_conv, clean_dtypes, ) From 07293e60608ba0809a41a93d4e416ce4e39fd8d6 Mon Sep 17 00:00:00 2001 From: Deborah Date: Wed, 17 Jun 2020 11:51:22 -0400 Subject: [PATCH 03/13] close GH 34640 (update doc) --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 13d7621c05e19..9a229d4df531d 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -379,7 +379,7 @@ Other API changes - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`) - Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max``, ``median``, ``skew``, ``cov``, ``corr`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`) - Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations. -- Removed ``private_key`` and ``verbose`` from gbq (:issue:`34654`) +- Removed ``private_key`` and ``verbose`` from gbq (:issue:`34654` :issue: `30200`) Backwards incompatible API changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 8b8427a79ecd7676786df48ef1d0276c686e91cb Mon Sep 17 00:00:00 2001 From: Deborah Date: Wed, 17 Jun 2020 11:57:10 -0400 Subject: [PATCH 04/13] close GH 34640 (update doc) --- doc/source/user_guide/io.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index df6b44ac654ce..38f8c4aac1617 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -232,8 +232,6 @@ na_filter : boolean, default ``True`` Detect missing value markers (empty strings and the value of na_values). In data without any NAs, passing ``na_filter=False`` can improve the performance of reading a large file. -verbose : boolean, default ``False`` - Indicate number of NA values placed in non-numeric columns. skip_blank_lines : boolean, default ``True`` If ``True``, skip over blank lines rather than interpreting as NaN values. From 182d453596d6ac27acf25043647e0ce251f3c04d Mon Sep 17 00:00:00 2001 From: Deborah Date: Thu, 25 Jun 2020 15:35:17 -0400 Subject: [PATCH 05/13] close GH 34640 edits --- doc/source/user_guide/io.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index 38f8c4aac1617..df6b44ac654ce 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -232,6 +232,8 @@ na_filter : boolean, default ``True`` Detect missing value markers (empty strings and the value of na_values). In data without any NAs, passing ``na_filter=False`` can improve the performance of reading a large file. +verbose : boolean, default ``False`` + Indicate number of NA values placed in non-numeric columns. skip_blank_lines : boolean, default ``True`` If ``True``, skip over blank lines rather than interpreting as NaN values. From 522e9cc6f4c58543341b4f75bd920da153ba6d97 Mon Sep 17 00:00:00 2001 From: parkdj1 <59840783+parkdj1@users.noreply.github.com> Date: Sun, 28 Jun 2020 20:35:15 -0400 Subject: [PATCH 06/13] Update parsers.py --- pandas/io/parsers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 1d7e456c884a0..15dfdeda189da 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -527,7 +527,6 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): _deprecated_args: Set[str] = set() - def _make_parser_function(name, default_sep=","): def parser_f( filepath_or_buffer: FilePathOrBuffer, @@ -706,8 +705,6 @@ def read_csv( # Alias sep -> delimiter. if delimiter is None: delimiter = sep - - kwds.update( delimiter=delimiter, engine=engine, From 7a7351a58b05fcdf09d2267ccd06e5c8d7391a97 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 1 Sep 2020 14:49:59 -0700 Subject: [PATCH 07/13] fixed whatsnew note --- doc/source/whatsnew/v1.2.0.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 1617bf66c4f04..26b572c90cbbb 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -281,7 +281,8 @@ ExtensionArray Other ^^^^^ -- + +- Removed ``private_key`` and ``verbose`` from :func:`pandas.read_gbq` (:issue:`34654` :issue: `30200`) - .. --------------------------------------------------------------------------- From 72c5f481c507b64ffc88f04a2fd683c4c793dcac Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 1 Sep 2020 14:51:32 -0700 Subject: [PATCH 08/13] fixed merge --- pandas/io/parsers.py | 137 ------------------------------------------- 1 file changed, 137 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 9956215e911d3..8b37f487d3b39 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -528,94 +528,6 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): _deprecated_args: Set[str] = set() -def _make_parser_function(name, default_sep=","): - def parser_f( - filepath_or_buffer: FilePathOrBuffer, - sep=default_sep, - delimiter=None, - # Column and Index Locations and Names - header="infer", - names=None, - index_col=None, - usecols=None, - squeeze=False, - prefix=None, - mangle_dupe_cols=True, - # General Parsing Configuration - dtype=None, - engine=None, - converters=None, - true_values=None, - false_values=None, - skipinitialspace=False, - skiprows=None, - skipfooter=0, - nrows=None, - # NA and Missing Data Handling - na_values=None, - keep_default_na=True, - na_filter=True, - skip_blank_lines=True, - # Datetime Handling - parse_dates=False, - infer_datetime_format=False, - keep_date_col=False, - date_parser=None, - dayfirst=False, - cache_dates=True, - # Iteration - iterator=False, - chunksize=None, - # Quoting, Compression, and File Format - compression="infer", - thousands=None, - decimal: str = ".", - lineterminator=None, - quotechar='"', - quoting=csv.QUOTE_MINIMAL, - doublequote=True, - escapechar=None, - comment=None, - encoding=None, - dialect=None, - # Error Handling - error_bad_lines=True, - warn_bad_lines=True, - # Internal - delim_whitespace=False, - low_memory=_c_parser_defaults["low_memory"], - memory_map=False, - float_precision=None, - ): - - # gh-23761 - # - # When a dialect is passed, it overrides any of the overlapping - # parameters passed in directly. We don't want to warn if the - # default parameters were passed in (since it probably means - # that the user didn't pass them in explicitly in the first place). - # - # "delimiter" is the annoying corner case because we alias it to - # "sep" before doing comparison to the dialect values later on. - # Thus, we need a flag to indicate that we need to "override" - # the comparison to dialect values by checking if default values - # for BOTH "delimiter" and "sep" were provided. - if dialect is not None: - sep_override = delimiter is None and sep == default_sep - kwds = dict(sep_override=sep_override) - else: - kwds = dict() - - # Alias sep -> delimiter. - if delimiter is None: - delimiter = sep - - if delim_whitespace and delimiter != default_sep: - raise ValueError( - "Specified a delimiter with both sep and " - "delim_whitespace=True; you can only specify one." - ) - @Appender( _doc_read_csv_and_table.format( func_name="read_csv", @@ -707,60 +619,11 @@ def read_csv( # Alias sep -> delimiter. if delimiter is None: delimiter = sep - kwds.update( - delimiter=delimiter, - engine=engine, - dialect=dialect, - compression=compression, - engine_specified=engine_specified, - doublequote=doublequote, - escapechar=escapechar, - quotechar=quotechar, - quoting=quoting, - skipinitialspace=skipinitialspace, - lineterminator=lineterminator, - header=header, - index_col=index_col, - names=names, - prefix=prefix, - skiprows=skiprows, - skipfooter=skipfooter, - na_values=na_values, - true_values=true_values, - false_values=false_values, - keep_default_na=keep_default_na, - thousands=thousands, - comment=comment, - decimal=decimal, - parse_dates=parse_dates, - keep_date_col=keep_date_col, - dayfirst=dayfirst, - date_parser=date_parser, - cache_dates=cache_dates, - nrows=nrows, - iterator=iterator, - chunksize=chunksize, - converters=converters, - dtype=dtype, - usecols=usecols, - encoding=encoding, - squeeze=squeeze, - memory_map=memory_map, - float_precision=float_precision, - na_filter=na_filter, - delim_whitespace=delim_whitespace, - warn_bad_lines=warn_bad_lines, - error_bad_lines=error_bad_lines, - low_memory=low_memory, - mangle_dupe_cols=mangle_dupe_cols, - infer_datetime_format=infer_datetime_format, - skip_blank_lines=skip_blank_lines, if delim_whitespace and delimiter != default_sep: raise ValueError( "Specified a delimiter with both sep and " "delim_whitespace=True; you can only specify one." - ) if engine is not None: From a7a404375b3de3cd46f89ba327138a586b91155a Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 1 Sep 2020 14:54:30 -0700 Subject: [PATCH 09/13] extra space --- pandas/io/parsers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 8b37f487d3b39..36ddd30472fd9 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -615,7 +615,6 @@ def read_csv( else: kwds = dict() - # Alias sep -> delimiter. if delimiter is None: delimiter = sep From 5810047b43fff4d1011e068ceb2b9b309589f96b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 1 Sep 2020 15:26:18 -0700 Subject: [PATCH 10/13] reverted parser changes --- pandas/io/parsers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 36ddd30472fd9..9ad527684120e 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -215,6 +215,8 @@ Detect missing value markers (empty strings and the value of na_values). In data without any NAs, passing na_filter=False can improve the performance of reading a large file. +verbose : bool, default False + Indicate number of NA values placed in non-numeric columns. skip_blank_lines : bool, default True If True, skip over blank lines rather than interpreting as NaN values. parse_dates : bool or list of int or names or list of lists or dict, \ @@ -500,6 +502,7 @@ def _read(filepath_or_buffer: FilePathOrBuffer, kwds): "usecols": None, # 'iterator': False, "chunksize": None, + "verbose": False, "encoding": None, "squeeze": False, "compression": None, @@ -1766,7 +1769,7 @@ def _agg_index(self, index, try_parse_dates=True): return index def _convert_to_ndarrays( - self, dct, na_values, na_fvalues, converters=None, dtypes=None + self, dct, na_values, na_fvalues, verbose=False, converters=None, dtypes=None ): result = {} for c, values in dct.items(): @@ -1836,7 +1839,7 @@ def _convert_to_ndarrays( cvals = self._cast_types(cvals, cast_type, c) result[c] = cvals - if na_count: + if verbose and na_count: print(f"Filled {na_count} NA values in column {c!s}") return result @@ -1964,10 +1967,6 @@ def _do_date_conversions(self, names, data): class CParserWrapper(ParserBase): - """ - - """ - def __init__(self, src, **kwds): self.kwds = kwds kwds = kwds.copy() @@ -2357,6 +2356,7 @@ def __init__(self, f, **kwds): if "has_index_names" in kwds: self.has_index_names = kwds["has_index_names"] + self.verbose = kwds["verbose"] self.converters = kwds["converters"] self.dtype = kwds["dtype"] @@ -2641,6 +2641,7 @@ def _clean_mapping(mapping): data, clean_na_values, clean_na_fvalues, + self.verbose, clean_conv, clean_dtypes, ) From 7c0bf576a928dfb4633fe0d2ddd650061a0dc003 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 10 Sep 2020 14:38:51 -0700 Subject: [PATCH 11/13] func fixup --- doc/source/whatsnew/v1.2.0.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 9de0ea0c22801..ec08095a090c3 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -339,7 +339,8 @@ ExtensionArray Other ^^^^^ -- Removed ``private_key`` and ``verbose`` from :func:`pandas.read_gbq` (:issue:`34654` :issue: `30200`) + +- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` (:issue:`34654` :issue: `30200`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`) - From 4cadb0f832ba4dab8c0131102e660ea4b140a4bf Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 11 Sep 2020 09:45:48 -0700 Subject: [PATCH 12/13] removed space --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 8ba71c3c69d14..bf9b984f2c6cb 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -341,7 +341,7 @@ ExtensionArray Other ^^^^^ -- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` (:issue:`34654` :issue: `30200`) +- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` (:issue:`34654` :issue:`30200`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`) - From a09851f04dfee9447fbc722ed14bc31b5c9b1cc3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 21 Sep 2020 14:49:27 -0700 Subject: [PATCH 13/13] whatsnew updates --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 354bdf9dd2364..0ea8048a97963 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -326,6 +326,7 @@ I/O - :meth:`to_picke` and :meth:`read_pickle` did not support compression for file-objects (:issue:`26237`, :issue:`29054`, and :issue:`29570`) - Bug in :func:`LongTableBuilder.middle_separator` was duplicating LaTeX longtable entires in the List of Tables of a LaTeX document (:issue:`34360`) - Bug in :meth:`read_csv` with `engine='python'` truncating data if multiple items present in first row and first element started with BOM (:issue:`36343`) +- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` as they are no longer supported in `pandas-gbq` (:issue:`34654` :issue:`30200`) Plotting ^^^^^^^^ @@ -373,7 +374,6 @@ ExtensionArray Other ^^^^^ -- Removed ``private_key`` and ``verbose`` from :func:`read_gbq` (:issue:`34654` :issue:`30200`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` incorrectly raising ``AssertionError`` instead of ``ValueError`` when invalid parameter combinations are passed (:issue:`36045`) - Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` with numeric values and string ``to_replace`` (:issue:`34789`) - Bug in :meth:`Series.transform` would give incorrect results or raise when the argument ``func`` was dictionary (:issue:`35811`)