implimentating suggested changes

Author: aysim319

_delphi_utils_python/delphi_utils/covidcast_wrapper.py

@@ -8,22 +8,29 @@
 from epiweeks import Week
 
 
-def date_generator(startdate: date, enddate: date) -> Iterable[date]:
+def date_generator(startdate: date, enddate: date, time_type: str) -> Iterable[date]:
     """
     Take start date and end date and generates date string.
 
     Parameters
     ----------
     startdate: date
     enddate: date
+    time_type: str
 
     Returns
     -------
     generator of str
     """
-    while startdate <= enddate:
-        yield startdate.strftime("%Y-%m-%d")
-        startdate = startdate + timedelta(days=1)
+    if time_type.lower() == "day":
+        while startdate <= enddate:
+            yield startdate.strftime("%Y-%m-%d")
+            startdate = startdate + timedelta(days=1)
+    elif time_type.lower() == "week":
+        while startdate <= enddate:
+            epiweek = Week.fromdate(startdate)
+            yield epiweek
+            startdate = startdate + timedelta(days=7)
 
 
 def _parse_datetimes(date_int: int, time_type: str, date_format: str = "%Y%m%d") -> Union[pd.Timestamp, None]:
@@ -34,8 +41,11 @@ def _parse_datetimes(date_int: int, time_type: str, date_format: str = "%Y%m%d")
 
     Epiweeks use the CDC format.
 
-    :param date_int: Int representation of date.
-    :param date_format: String of the date format to parse.
+    date_int: Int representation of date.
+    time_type: The temporal resolution to request this data. Most signals
+      are available at the "day" resolution (the default); some are only
+      available at the "week" resolution, representing an MMWR week ("epiweek").
+    date_format: String of the date format to parse.
     :returns: Timestamp.
     """
     date_str = str(date_int)
@@ -55,8 +65,7 @@ def metadata() -> Union[pd.DataFrame, None]:
     -------
     pd.DataFrame of covidcast metadata.
     """
-    # pylint: disable=W0212
-    response = Epidata._request("covidcast_meta")
+    response = Epidata.covidcast_meta()
 
     if response["result"] != 1:
         # Something failed in the API and we did not get real metadata
@@ -80,145 +89,43 @@ def signal(
     lag: int = None,
     time_type: str = "day",
 ) -> Union[pd.DataFrame, None]:
-    """Download a Pandas data frame for one signal.
-
-    Obtains data for selected date ranges for all geographic regions of the
-    United States. Available data sources and signals are documented in the
-    `COVIDcast signal documentation
-    <https://cmu-delphi.github.io/delphi-epidata/api/covidcast_signals.html>`_.
-    Most (but not all) data sources are available at the county level, but the
-    API can also return data aggregated to metropolitan statistical areas,
-    hospital referral regions, or states, as desired, by using the ``geo_type``
-    argument.
-
-    The COVIDcast API tracks updates and changes to its underlying data, and
-    records the first date each observation became available. For example, a
-    data source may report its estimate for a specific state on June 3rd on June
-    5th, once records become available. This data is considered "issued" on June
-    5th. Later, the data source may update its estimate for June 3rd based on
-    revised data, creating a new issue on June 8th. By default, ``signal()``
-    returns the most recent issue available for every observation. The
-    ``as_of``, ``issues``, and ``lag`` parameters allow the user to select
-    specific issues instead, or to see all updates to observations. These
-    options are mutually exclusive; if you specify more than one, ``as_of`` will
-    take priority over ``issues``, which will take priority over ``lag``.
-
-    Note that the API only tracks the initial value of an estimate and *changes*
-    to that value. If a value was first issued on June 5th and never updated,
-    asking for data issued on June 6th (using ``issues`` or ``lag``) would *not*
-    return that value, though asking for data ``as_of`` June 6th would.
-
-    Note also that the API enforces a maximum result row limit; results beyond
-    the maximum limit are truncated. This limit is sufficient to fetch
-    observations in all counties in the United States on one day. This client
-    automatically splits queries for multiple days across multiple API calls.
-    However, if data for one day has been issued many times, using the
-    ``issues`` argument may return more results than the query limit. A warning
-    will be issued in this case. To see all results, split your query across
-    multiple calls with different ``issues`` arguments.
-
-    See the `COVIDcast API documentation
-    <https://cmu-delphi.github.io/delphi-epidata/api/covidcast.html>`_ for more
-    information on available geography types, signals, and data formats, and
-    further discussion of issue dates and data versioning.
-
-    :param data_source: String identifying the data source to query, such as
+    """
+    Makes covidcast signal api call.
+
+    data_source: String identifying the data source to query, such as
       ``"fb-survey"``.
-    :param signal: String identifying the signal from that source to query,
+    signal: String identifying the signal from that source to query,
       such as ``"smoothed_cli"``.
-    :param start_day: Query data beginning on this date. Provided as a
+    start_day: Query data beginning on this date. Provided as a
       ``datetime.date`` object. If ``start_day`` is ``None``, defaults to the
       first day data is available for this signal. If ``time_type == "week"``, then
       this is rounded to the epiweek containing the day (i.e. the previous Sunday).
-    :param end_day: Query data up to this date, inclusive. Provided as a
+    end_day: Query data up to this date, inclusive. Provided as a
       ``datetime.date`` object. If ``end_day`` is ``None``, defaults to the most
       recent day data is available for this signal. If ``time_type == "week"``, then
       this is rounded to the epiweek containing the day (i.e. the previous Sunday).
-    :param geo_type: The geography type for which to request this data, such as
+    geo_type: The geography type for which to request this data, such as
       ``"county"`` or ``"state"``. Available types are described in the
       COVIDcast signal documentation. Defaults to ``"county"``.
-    :param geo_values: The geographies to fetch data for. The default, ``"*"``,
+    geo_values: The geographies to fetch data for. The default, ``"*"``,
       fetches all geographies. To fetch one geography, specify its ID as a
       string; multiple geographies can be provided as an iterable (list, tuple,
       ...) of strings.
-    :param as_of: Fetch only data that was available on or before this date,
+    as_of: Fetch only data that was available on or before this date,
       provided as a ``datetime.date`` object. If ``None``, the default, return
       the most recent available data. If ``time_type == "week"``, then
       this is rounded to the epiweek containing the day (i.e. the previous Sunday).
-    :param issues: Fetch only data that was published or updated ("issued") on
-      these dates. Provided as either a single ``datetime.date`` object,
-      indicating a single date to fetch data issued on, or a tuple or list
-      specifying (start, end) dates. In this case, return all data issued in
-      this range. There may be multiple rows for each observation, indicating
-      several updates to its value. If ``None``, the default, return the most
-      recently issued data. If ``time_type == "week"``, then these are rounded to
-      the epiweek containing the day (i.e. the previous Sunday).
-    :param lag: Integer. If, for example, ``lag=3``, fetch only data that was
+    lag: Integer. If, for example, ``lag=3``, fetch only data that was
       published or updated exactly 3 days after the date. For example, a row
       with ``time_value`` of June 3 will only be included in the results if its
       data was issued or updated on June 6. If ``None``, the default, return the
       most recently issued data regardless of its lag.
-    :param time_type: The temporal resolution to request this data. Most signals
+    time_type: The temporal resolution to request this data. Most signals
       are available at the "day" resolution (the default); some are only
       available at the "week" resolution, representing an MMWR week ("epiweek").
     :returns: A Pandas data frame with matching data, or ``None`` if no data is
       returned. Each row is one observation on one day in one geographic location.
       Contains the following columns:
-
-      ``geo_value``
-        Identifies the location, such as a state name or county FIPS code. The
-        geographic coding used by COVIDcast is described in the `API
-        documentation here
-        <https://cmu-delphi.github.io/delphi-epidata/api/covidcast_geography.html>`_.
-
-      ``signal``
-        Name of the signal, same as the value of the ``signal`` input argument. Used for
-        downstream functions to recognize where this signal is from.
-
-      ``time_value``
-        Contains a `pandas Timestamp object
-        <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timestamp.html>`_
-        identifying the date this estimate is for. For data with ``time_type = "week"``, this
-        is the first day of the corresponding epiweek.
-
-      ``issue``
-        Contains a `pandas Timestamp object
-        <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timestamp.html>`_
-        identifying the date this estimate was issued. For example, an estimate
-        with a ``time_value`` of June 3 might have been issued on June 5, after
-        the data for June 3rd was collected and ingested into the API.
-
-      ``lag``
-        Integer giving the difference between ``issue`` and ``time_value``,
-        in days.
-
-      ``value``
-        The signal quantity requested. For example, in a query for the
-        ``confirmed_cumulative_num`` signal from the ``usa-facts`` source,
-        this would be the cumulative number of confirmed cases in the area, as
-        of the ``time_value``.
-
-      ``stderr``
-        The value's standard error, if available.
-
-      ``sample_size``
-        Indicates the sample size available in that geography on that day;
-        sample size may not be available for all signals, due to privacy or
-        other constraints.
-
-      ``geo_type``
-        Geography type for the signal, same as the value of the ``geo_type`` input argument.
-        Used for downstream functions to parse ``geo_value`` correctly
-
-      ``data_source``
-        Name of the signal source, same as the value of the ``data_source`` input argument. Used for
-        downstream functions to recognize where this signal is from.
-
-    Consult the `signal documentation
-    <https://cmu-delphi.github.io/delphi-epidata/api/covidcast_signals.html>`_
-    for more details on how values and standard errors are calculated for
-    specific signals.
-
     """
     if start_day > end_day:
         raise ValueError(
@@ -239,7 +146,7 @@ def signal(
     )
     if response["result"] != 1:
         # Something failed in the API and we did not get real metadata
-        raise RuntimeError("Error when fetching metadata from the API", response["message"])
+        raise RuntimeError("Error when fetching signal data from the API", response["message"])
 
     api_df = pd.DataFrame.from_dict(response["epidata"])
     api_df["issue"] = pd.to_datetime(api_df["issue"], format="%Y%m%d")

_delphi_utils_python/delphi_utils/validator/dynamic.py

@@ -80,8 +80,6 @@ def validate(self, all_frames, report):
         # Get 14 days prior to the earliest list date
         outlier_lookbehind = timedelta(days=14)
 
-        # Authenticate API
-        # Epidata.auth = ("epidata", api)
 
         # Get all expected combinations of geo_type and signal.
         geo_signal_combos = get_geo_signal_combos(self.params.data_source,

_delphi_utils_python/tests/test_covidcast_wrapper.py

@@ -4,19 +4,22 @@
 from delphi_utils import covidcast_wrapper
 import covidcast
 from freezegun import freeze_time
+from delphi_epidata import Epidata
 from pandas.testing import assert_frame_equal
 
 TEST_DIR = Path(__file__).parent
 class TestCovidcastWrapper:
+    Epidata.debug = True
     def test_metadata(self):
         expected_df = covidcast.metadata()
         df = covidcast_wrapper.metadata()
         assert_frame_equal(expected_df, df)
 
-    @freeze_time("2024-07-29")
+    @freeze_time("2022-01-29")
     def test_signal(self):
-        meta_df = covidcast_wrapper.metadata()
-        data_filter = ((meta_df["max_time"] >= datetime(year=2024, month=6, day=1)) & (meta_df["time_type"] == "day"))
+        meta_df = pd.read_pickle(f"{TEST_DIR}/test_data/covidcast_metadata.pkl")
+
+        data_filter = (meta_df["max_time"] >= datetime(year=2024, month=6, day=1))
         signal_df = meta_df[data_filter].groupby("data_source")["signal"].agg(['unique'])
         enddate = datetime.today()
         startdate = enddate - timedelta(days=15)