implement exponential retry per rom

Cargo.toml

@@ -10,6 +10,7 @@ ctrlc = { version = "3.4.4", features = ["termination"] }
 indicatif = "0.17.8"
 once_cell = "1.19.0"
 reqwest = { version = "0.12.5", features = ["blocking"] }
+retry = "2.0.0"
 roxmltree = "0.20.0"
 select = "0.6.0"
 

src/main.rs

@@ -122,35 +122,25 @@ fn main() {
         );
     }
 
-    let mut roms_with_errors: Vec<myrient::Rom> = Vec::new();
-
-    let download = |roms: Vec<myrient::Rom>, on_error: &mut dyn FnMut(myrient::Rom)| {
-        download_roms(roms, &output_dir, &catalog_url, &collection_url, on_error);
-    };
-
     if !args.list {
-        download(wanted_roms, &mut |rom| {
-            println!("{}", format!("Error with  {}", rom.name).red());
-            roms_with_errors.push(rom);
-        });
-
-        let mut final_roms_with_errors: Vec<myrient::Rom> = Vec::new();
-
-        if roms_with_errors.len() > 0 {
-            println!("{}", "Retrying failed downloads...".yellow());
-            download(roms_with_errors, &mut |rom| {
-                println!("{}", format!("Error with  {}", rom.name).red());
-                final_roms_with_errors.push(rom);
-            });
-        }
+        match myrient::download_roms(&wanted_roms, &output_dir, &catalog_url, &collection_url) {
+            Ok(_) => {
+                println!("{}", "All downloads successful!".green());
+            }
+            Err(err) => {
+                println!(
+                    "{}",
+                    format!(
+                        "Following {} ROMs failed to download:",
+                        err.failed_roms.len()
+                    )
+                    .red()
+                );
 
-        if final_roms_with_errors.len() > 0 {
-            println!("{}", "Failed downloads:".red());
-            for rom in final_roms_with_errors.iter() {
-                println!("{}", rom.name);
+                for rom in err.failed_roms {
+                    println!("{}", rom.name.red());
+                }
             }
-        } else {
-            println!("{}", "All downloads successful!".green());
         }
     }
 
@@ -336,29 +326,3 @@ fn get_collection_url(catalog_url: &String, system_name: &String, select_system:
 
     collection_url.unwrap()
 }
-
-fn download_roms<F>(
-    roms: Vec<myrient::Rom>,
-    output_dir: &String,
-    catalog_url: &String,
-    collection_url: &String,
-    mut on_error: F,
-) where
-    F: FnMut(myrient::Rom),
-{
-    for index in 0..roms.len() {
-        let rom = roms.get(index).unwrap();
-        let file = myrient::download_rom(
-            output_dir,
-            &format!("{}{}{}", &catalog_url, &collection_url, rom.url),
-            &rom,
-            &(index + 1),
-            &roms.len(),
-        );
-
-        if file.is_err() {
-            println!("{}", format!("Error with  {}", rom.name).red());
-            on_error(rom.clone());
-        }
-    }
-}

src/myrient.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::fmt;
 use std::fs::File;
 use std::io::{self, Read};
 use std::path::Path;
@@ -9,13 +10,17 @@ use once_cell::sync::Lazy;
 use reqwest::blocking::Client;
 
 use reqwest::header;
+use retry::delay::Exponential;
+use retry::retry;
 use select::document::Document;
 use select::predicate::{Attr, Name, Predicate};
 
 use crate::constants;
 
 const HTTP_CLIENT: Lazy<Client> = Lazy::new(|| Client::new());
 
+const MAX_RETRIES: usize = 3;
+
 const PROGRESS_PERCENT_PART: &str = "percent:>2";
 const PROGESS_TEMPLATE: &str =
     " | {decimal_bytes:>9} / {decimal_total_bytes} | {bar} | ETA: {eta:>3} | {decimal_bytes_per_sec:>11}";
@@ -29,7 +34,7 @@ pub struct Collection {
     pub title: String,
     pub url: String,
 }
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct Rom {
     pub name: String,
     pub file: String,
@@ -73,7 +78,7 @@ impl<R: Read> Read for DownloadProgress<R> {
         self.inner.read(buf).map(|n| {
             self.progress_bar.set_style(
                 ProgressStyle::with_template(build_progress_template(&self.progress_bar).as_str())
-                .unwrap(),
+                    .unwrap(),
             );
             self.progress_bar.inc(n as u64);
             n
@@ -332,3 +337,54 @@ pub fn download_rom(
 
     Ok(File::open(local_path).unwrap())
 }
+
+// custom error type that includes a vector of the failed Rom objects
+#[derive(Debug, Clone)]
+pub struct BulkDownloadError {
+    pub failed_roms: Vec<Rom>,
+}
+
+impl fmt::Display for BulkDownloadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Failed to download the following ROMs: ")?;
+        for rom in &self.failed_roms {
+            write!(f, "{} ", rom.name)?;
+        }
+        Ok(())
+    }
+}
+
+pub fn download_roms(
+    roms: &Vec<Rom>,
+    output_dir: &String,
+    catalog_url: &String,
+    collection_url: &String,
+) -> Result<(), BulkDownloadError> {
+    let mut roms_with_errors: Vec<Rom> = Vec::new();
+
+    for index in 0..roms.len() {
+        let rom = roms.get(index).unwrap();
+        let result = retry(Exponential::from_millis(100).take(MAX_RETRIES), || {
+            download_rom(
+                output_dir,
+                &format!("{}{}{}", catalog_url, collection_url, rom.url),
+                rom,
+                &(index + 1),
+                &roms.len(),
+            )
+        });
+
+        if result.is_err() {
+            println!("{}", format!("Error with  {}", rom.name).red());
+            roms_with_errors.push(rom.clone());
+        }
+    }
+
+    if roms_with_errors.len() > 0 {
+        return Err(BulkDownloadError {
+            failed_roms: roms_with_errors,
+        });
+    }
+
+    Ok(())
+}