diff --git a/Cargo.lock b/Cargo.lock index 44769d2..7fdbe66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,6 +83,7 @@ dependencies = [ "once_cell", "openssl", "reqwest", + "retry", "roxmltree", "select", ] @@ -1069,6 +1070,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "retry" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9166d72162de3575f950507683fac47e30f6f2c3836b71b7fbc61aa517c9c5f4" +dependencies = [ + "rand", +] + [[package]] name = "ring" version = "0.17.8" diff --git a/Cargo.toml b/Cargo.toml index cf85329..0be7e4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ ctrlc = { version = "3.4.4", features = ["termination"] } indicatif = "0.17.8" once_cell = "1.19.0" reqwest = { version = "0.12.5", features = ["blocking"] } +retry = "2.0.0" roxmltree = "0.20.0" select = "0.6.0" diff --git a/src/main.rs b/src/main.rs index 4d70f6a..10b34a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -122,35 +122,25 @@ fn main() { ); } - let mut roms_with_errors: Vec = Vec::new(); - - let download = |roms: Vec, on_error: &mut dyn FnMut(myrient::Rom)| { - download_roms(roms, &output_dir, &catalog_url, &collection_url, on_error); - }; - if !args.list { - download(wanted_roms, &mut |rom| { - println!("{}", format!("Error with {}", rom.name).red()); - roms_with_errors.push(rom); - }); - - let mut final_roms_with_errors: Vec = Vec::new(); - - if roms_with_errors.len() > 0 { - println!("{}", "Retrying failed downloads...".yellow()); - download(roms_with_errors, &mut |rom| { - println!("{}", format!("Error with {}", rom.name).red()); - final_roms_with_errors.push(rom); - }); - } + match myrient::download_roms(&wanted_roms, &output_dir, &catalog_url, &collection_url) { + Ok(_) => { + println!("{}", "All downloads successful!".green()); + } + Err(err) => { + println!( + "{}", + format!( + "Following {} ROMs failed to download:", + err.failed_roms.len() + ) + .red() + ); - if final_roms_with_errors.len() > 0 { - println!("{}", "Failed downloads:".red()); - for rom in final_roms_with_errors.iter() { - println!("{}", rom.name); + for rom in err.failed_roms { + println!("{}", rom.name.red()); + } } - } else { - println!("{}", "All downloads successful!".green()); } } @@ -336,29 +326,3 @@ fn get_collection_url(catalog_url: &String, system_name: &String, select_system: collection_url.unwrap() } - -fn download_roms( - roms: Vec, - output_dir: &String, - catalog_url: &String, - collection_url: &String, - mut on_error: F, -) where - F: FnMut(myrient::Rom), -{ - for index in 0..roms.len() { - let rom = roms.get(index).unwrap(); - let file = myrient::download_rom( - output_dir, - &format!("{}{}{}", &catalog_url, &collection_url, rom.url), - &rom, - &(index + 1), - &roms.len(), - ); - - if file.is_err() { - println!("{}", format!("Error with {}", rom.name).red()); - on_error(rom.clone()); - } - } -} diff --git a/src/myrient.rs b/src/myrient.rs index 81faf1e..db4a0f3 100644 --- a/src/myrient.rs +++ b/src/myrient.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::fmt; use std::fs::File; use std::io::{self, Read}; use std::path::Path; @@ -9,6 +10,8 @@ use once_cell::sync::Lazy; use reqwest::blocking::Client; use reqwest::header; +use retry::delay::Exponential; +use retry::retry; use select::document::Document; use select::predicate::{Attr, Name, Predicate}; @@ -16,6 +19,8 @@ use crate::constants; const HTTP_CLIENT: Lazy = Lazy::new(|| Client::new()); +const MAX_RETRIES: usize = 3; + const PROGRESS_PERCENT_PART: &str = "percent:>2"; const PROGESS_TEMPLATE: &str = " | {decimal_bytes:>9} / {decimal_total_bytes} | {bar} | ETA: {eta:>3} | {decimal_bytes_per_sec:>11}"; @@ -29,7 +34,7 @@ pub struct Collection { pub title: String, pub url: String, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct Rom { pub name: String, pub file: String, @@ -73,7 +78,7 @@ impl Read for DownloadProgress { self.inner.read(buf).map(|n| { self.progress_bar.set_style( ProgressStyle::with_template(build_progress_template(&self.progress_bar).as_str()) - .unwrap(), + .unwrap(), ); self.progress_bar.inc(n as u64); n @@ -332,3 +337,54 @@ pub fn download_rom( Ok(File::open(local_path).unwrap()) } + +// custom error type that includes a vector of the failed Rom objects +#[derive(Debug, Clone)] +pub struct BulkDownloadError { + pub failed_roms: Vec, +} + +impl fmt::Display for BulkDownloadError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Failed to download the following ROMs: ")?; + for rom in &self.failed_roms { + write!(f, "{} ", rom.name)?; + } + Ok(()) + } +} + +pub fn download_roms( + roms: &Vec, + output_dir: &String, + catalog_url: &String, + collection_url: &String, +) -> Result<(), BulkDownloadError> { + let mut roms_with_errors: Vec = Vec::new(); + + for index in 0..roms.len() { + let rom = roms.get(index).unwrap(); + let result = retry(Exponential::from_millis(100).take(MAX_RETRIES), || { + download_rom( + output_dir, + &format!("{}{}{}", catalog_url, collection_url, rom.url), + rom, + &(index + 1), + &roms.len(), + ) + }); + + if result.is_err() { + println!("{}", format!("Error with {}", rom.name).red()); + roms_with_errors.push(rom.clone()); + } + } + + if roms_with_errors.len() > 0 { + return Err(BulkDownloadError { + failed_roms: roms_with_errors, + }); + } + + Ok(()) +}