feat: add message about mirror when nvd is down

Signed-off-by: Aryan Bakliwal <[email protected]>

Author: AryanBakliwal

cve_bin_tool/data_sources/nvd_source.py

@@ -409,19 +409,30 @@ async def fetch_cves(self):
         tasks = []
         LOGGER.info("Getting NVD CVE data...")
         if self.nvd_type == "api2":
+            self.LOGGER.info("Using mirror, with nvd_type %s and feed %s", self.nvd_type, self.feed)
             self.all_cve_entries = await asyncio.create_task(
                 self.nist_fetch_using_api(),
             )
         else:
-            nvd_metadata = await asyncio.create_task(
-                self.nist_scrape(self.session),
-            )
+            self.LOGGER.info("Using NVD, with nvd_type %s and feed %s", self.nvd_type, self.feed)
+            try:
+                nvd_metadata = await asyncio.create_task(
+                    self.nist_scrape(self.session),
+                )
+
+                tasks = [
+                    self.cache_update(self.session, url, meta["sha256"])
+                    for url, meta in nvd_metadata.items()
+                    if meta is not None
+                ]
+            except Exception as e:
+                self.LOGGER.info(f"nvd_source.py 429 fetch_cves() {e}")
+                self.nvd_type = "api2"
+                self.LOGGER.info("Falling back to mirror, with nvd_type %s and feed %s", self.nvd_type, self.feed)
+                self.all_cve_entries = await asyncio.create_task(
+                    self.nist_fetch_using_api(),
+                )
 
-            tasks = [
-                self.cache_update(self.session, url, meta["sha256"])
-                for url, meta in nvd_metadata.items()
-                if meta is not None
-            ]
 
         total_tasks = len(tasks)
 
@@ -464,14 +475,17 @@ async def nist_fetch_using_api(self) -> list:
             api_key=self.nvd_api_key,
             api_version=api_version,
         )
-        if self.incremental_update:
-            await nvd_api.get_nvd_params(
-                time_of_last_update=datetime.datetime.fromtimestamp(
-                    db.get_db_update_date()
+        try:
+            if self.incremental_update:
+                await nvd_api.get_nvd_params(
+                    time_of_last_update=datetime.datetime.fromtimestamp(
+                        db.get_db_update_date()
+                    )
                 )
-            )
-        else:
-            await nvd_api.get_nvd_params()
+            else:
+                await nvd_api.get_nvd_params()
+        except Exception as e:
+            self.LOGGER.info(f"nvd_source.py 488 nist_fetch_using_api() {e}")
         await nvd_api.get()
         await nvd_api.session.close()
         nvd_api.session = None
@@ -490,18 +504,21 @@ async def getmeta(
         Returns:
             tuple: A tuple containing the URL for the JSON data and a dictionary of metadata.
         """
-        async with await session.get(meta_url) as response:
-            response.raise_for_status()
-            return (
-                meta_url.replace(".meta", ".json.gz"),
-                dict(
-                    [
-                        line.split(":", maxsplit=1)
-                        for line in (await response.text()).splitlines()
-                        if ":" in line
-                    ]
-                ),
-            )
+        try:
+            async with await session.get(meta_url) as response:
+                response.raise_for_status()
+                return (
+                    meta_url.replace(".meta", ".json.gz"),
+                    dict(
+                        [
+                            line.split(":", maxsplit=1)
+                            for line in (await response.text()).splitlines()
+                            if ":" in line
+                        ]
+                    ),
+                )
+        except Exception as e:
+            self.LOGGER.info(f"nvd_source.py 521 getmeta() {e}")
 
     async def nist_scrape(self, session: RateLimiter):
         """
@@ -513,23 +530,26 @@ async def nist_scrape(self, session: RateLimiter):
         Returns:
             dict: A dictionary containing metadata links and their corresponding SHA values.
         """
-        async with await session.get(self.feed) as response:
-            response.raise_for_status()
-            page = await response.text()
-            if self.nvd_type == "json-nvd":
-                json_meta_links = self.META_REGEX_NVD.findall(page)
-                meta_host = self.META_LINK_NVD
-            else:
-                json_meta_links = self.META_REGEX_MIRROR.findall(page)
-                meta_host = self.META_LINK_MIRROR
-            return dict(
-                await asyncio.gather(
-                    *(
-                        self.getmeta(session, f"{meta_host}/{meta_url}")
-                        for meta_url in json_meta_links
+        try:
+            async with await session.get(self.feed) as response:
+                response.raise_for_status()
+                page = await response.text()
+                if self.nvd_type == "json-nvd":
+                    json_meta_links = self.META_REGEX_NVD.findall(page)
+                    meta_host = self.META_LINK_NVD
+                else:
+                    json_meta_links = self.META_REGEX_MIRROR.findall(page)
+                    meta_host = self.META_LINK_MIRROR
+                return dict(
+                    await asyncio.gather(
+                        *(
+                            self.getmeta(session, f"{meta_host}/{meta_url}")
+                            for meta_url in json_meta_links
+                        )
                     )
                 )
-            )
+        except Exception as e:
+            self.LOGGER.info(f"nvd_source.py 552 nist_scrape() {e}")
 
     async def cache_update(
         self,

cve_bin_tool/nvd_api.py

@@ -163,7 +163,11 @@ async def get_nvd_params(
             )
 
         self.logger.info("Fetching metadata from NVD...")
-        cve_count = await self.nvd_count_metadata(self.session)
+        try:
+            cve_count = await self.nvd_count_metadata(self.session)
+        except Exception as e:
+            self.logger.info(f"nvd_api.py 169 get_nvd_params() {e}")
+            raise e
         self.logger.debug(f"NVD metadata {cve_count}")
 
         await self.validate_nvd_api()
@@ -316,6 +320,7 @@ async def load_nvd_request(self, start_index):
                 else:
                     self.logger.debug(f"Pausing requests for {self.interval} seconds")
                     time.sleep(self.interval)
+                raise error
 
     async def get(self):
         """Calls load_nvd_request() multiple times to fetch all NVD feeds"""