From 9bb49ae7d07ca558f03ada892f1fb6ec35740180 Mon Sep 17 00:00:00 2001 From: Audrow Nash Date: Wed, 7 Feb 2024 22:11:23 -0600 Subject: [PATCH] Make Github API calls more robust + add issue matching tools --- Cargo.lock | 13 ++ src/yatm_v2/Cargo.toml | 2 + src/yatm_v2/src/main.rs | 39 ++--- src/yatm_v2/src/utils/github.rs | 89 ++++++++--- src/yatm_v2/src/utils/github_utils.rs | 185 +++++++++++++++++++++++ src/yatm_v2/src/utils/make_test_cases.rs | 103 +++++++++++-- src/yatm_v2/src/utils/mod.rs | 1 + src/yatm_v2/src/utils/template.rs | 6 +- 8 files changed, 385 insertions(+), 53 deletions(-) create mode 100644 src/yatm_v2/src/utils/github_utils.rs diff --git a/Cargo.lock b/Cargo.lock index e7ad0d6..1500ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,6 +136,17 @@ dependencies = [ "nom", ] +[[package]] +name = "async-recursion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-trait" version = "0.1.77" @@ -1762,6 +1773,7 @@ version = "0.1.0" dependencies = [ "anyhow", "askama", + "async-recursion", "chrono", "clap", "common", @@ -1772,6 +1784,7 @@ dependencies = [ "serde", "serde_yaml", "tokio", + "url", ] [[package]] diff --git a/src/yatm_v2/Cargo.toml b/src/yatm_v2/Cargo.toml index 9df4b4d..35b2340 100644 --- a/src/yatm_v2/Cargo.toml +++ b/src/yatm_v2/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [dependencies] anyhow = "1.0.79" askama = "0.12.1" +async-recursion = "1.0.5" chrono = "0.4.33" clap = { version = "4.4.18", features = ["derive"] } common = { path = "../common" } @@ -19,3 +20,4 @@ percent-encoding = "2.3.1" serde = { version = "1.0.196", features = ["derive"] } serde_yaml = "0.9.31" tokio = { version = "1.36.0", features = ["full"] } +url = "2.5.0" diff --git a/src/yatm_v2/src/main.rs b/src/yatm_v2/src/main.rs index cacfb2f..b68c901 100644 --- a/src/yatm_v2/src/main.rs +++ b/src/yatm_v2/src/main.rs @@ -101,26 +101,29 @@ fn demo_template() { async fn main() -> Result<()> { let gh = Github::new("paudrow".to_string(), "test-yatm-v2".to_string())?; - let issue_title = format!( - "My issue {}", - chrono::Local::now().format("%Y-%m-%d %H:%M:%S") - ); - gh.create_issue( - issue_title, - "My issue body".to_string(), - vec![String::from("label")], - ) - .await?; - // gh.close_all_issues().await?; - - let issues = gh.get_issues().await?; - for issue in issues { - println!( - "{}, {}", - issue.title, - issue.body.unwrap_or("No body".into()) + for _ in 0..300 { + let issue_title = format!( + "My issue {}", + chrono::Local::now().format("%Y-%m-%d %H:%M:%S") ); + gh.create_issue( + issue_title.clone(), + "My issue body".to_string(), + vec![String::from("label")], + ) + .await?; } + // let issues = gh.get_issues().await?; + // for issue in &issues { + // println!( + // "{}, {}", + // issue.title, + // issue.body.clone().unwrap_or("No body".into()) + // ); + // } + // println!("Total issues: {}", &issues.len()); + // gh.close_all_issues().await?; + Ok(()) } diff --git a/src/yatm_v2/src/utils/github.rs b/src/yatm_v2/src/utils/github.rs index 4395a8d..4ddd579 100644 --- a/src/yatm_v2/src/utils/github.rs +++ b/src/yatm_v2/src/utils/github.rs @@ -1,6 +1,11 @@ -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; +use async_recursion::async_recursion; use dotenv::dotenv; +use octocrab::models::issues::Issue; +use octocrab::models::IssueState; +use octocrab::params::State; use octocrab::Octocrab; +use tokio::time::{sleep, Duration}; pub struct Github { octocrab: Octocrab, @@ -60,54 +65,92 @@ impl Github { Ok(()) } - pub async fn get_issues(&self) -> Result> { - let issues = self + pub async fn get_issues(&self, state: Option) -> Result> { + let mut page: u32 = 1; + let mut issues: Vec = Vec::new(); + loop { + let page_issues = self.get_issues_helper(page, state).await?; + if page_issues.is_empty() { + break; + } + issues.extend(page_issues); + page += 1; + } + Ok(issues) + } + + async fn get_issues_helper(&self, page: u32, state: Option) -> Result> { + let page = self .octocrab .issues(&self.owner, &self.repo) .list() - .state(octocrab::params::State::Open) + .per_page(100) + .state(state.unwrap_or(State::Open)) + .page(page) .send() .await .context("Failed to list issues")?; - Ok(issues) + Ok(page.items) } + #[async_recursion] pub async fn create_issue( &self, title: String, body: String, labels: Vec, ) -> Result<()> { - self.octocrab + let result = self + .octocrab .issues(&self.owner, &self.repo) .create(&title) .body(&body) - .labels(labels) + .labels(labels.clone()) .send() - .await - .context(format!("Failed to create issue: {}", &title))?; - Ok(()) + .await; + + if result.is_ok() { + return Ok(()); + } + + let error = result.unwrap_err(); + if let octocrab::Error::GitHub { source, .. } = &error { + if source + .message + .contains("You have exceeded a secondary rate limit") + { + println!("Secondary rate limit exceeded, waiting 60 seconds and retrying"); + sleep(Duration::from_secs(60)).await; + // Retry the request by calling the function recursively. + return self.create_issue(title, body, labels).await; + } + } + Err(anyhow!(error)) } pub async fn close_all_issues(&self) -> Result<()> { let issues = self - .octocrab - .issues(&self.owner, &self.repo) - .list() - .state(octocrab::params::State::Open) - .send() + .get_issues(Some(State::Open)) .await - .context("Failed to list issues")?; + .context("Failed to get issues")?; for issue in issues { - self.octocrab - .issues(&self.owner, &self.repo) - .update(issue.number) - .state(octocrab::models::IssueState::Closed) - .send() - .await - .context(format!("Failed to delete issue: {}", issue.number))?; + self.close_issue(issue).await?; } Ok(()) } + + pub async fn close_issue(&self, issue: Issue) -> Result<()> { + self.octocrab + .issues(&self.owner, &self.repo) + .update(issue.number) + .state(IssueState::Closed) + .send() + .await + .context(format!( + "Failed to delete issue: #{} {}", + issue.number, issue.title + ))?; + Ok(()) + } } diff --git a/src/yatm_v2/src/utils/github_utils.rs b/src/yatm_v2/src/utils/github_utils.rs new file mode 100644 index 0000000..8b93a6a --- /dev/null +++ b/src/yatm_v2/src/utils/github_utils.rs @@ -0,0 +1,185 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use crate::utils::template::LocalIssue; +use octocrab::models::issues::Issue as GithubIssue; +use url::Url; + +struct GithubIssueHelper { + pub title: String, + pub labels: Vec, +} + +pub fn get_local_issues_without_matches( + local_issues: Vec, + github_issues: &Vec, +) -> Vec { + let github_issues: Vec = github_issues + .iter() + .map(|issue| GithubIssueHelper { + title: issue.title.clone(), + labels: issue + .labels + .clone() + .iter() + .map(|label| label.name.clone()) + .collect(), + }) + .collect(); + get_local_issues_without_matches_helper(local_issues, &github_issues) +} + +fn get_local_issues_without_matches_helper( + local_issues: Vec, + github_issues: &Vec, +) -> Vec { + let github_issue_hashes: Vec<_> = github_issues + .iter() + .map(|issue| get_issue_hash(issue.title.clone(), issue.labels.clone())) + .collect(); + local_issues + .iter() + .filter(|local_issue| { + let local_labels = local_issue.labels.clone(); + let local_hash = get_issue_hash(local_issue.title.clone(), local_labels); + !github_issue_hashes.contains(&local_hash) + }) + .cloned() + .collect() +} + +#[cfg(test)] +mod test_get_local_issues_without_matches { + use super::get_local_issues_without_matches_helper; + use crate::utils::{github_utils::GithubIssueHelper, template::LocalIssue}; + + #[test] + fn no_matches() { + let local_issues = vec![ + LocalIssue { + labels: vec!["label1".to_string()], + title: "title".to_string(), + text_body: "text".to_string(), + }, + LocalIssue { + labels: vec!["label2".to_string()], + title: "title2".to_string(), + text_body: "text2".to_string(), + }, + ]; + let github_issues = vec![GithubIssueHelper { + labels: vec![], + title: "title3".to_string(), + }]; + let result = get_local_issues_without_matches_helper(local_issues, &github_issues); + assert_eq!(result.len(), 2, "All local issues should be returned"); + } + + #[test] + fn one_match() { + let local_issues = vec![ + LocalIssue { + labels: vec!["label1".to_string(), "label2".to_string()], + title: "title".to_string(), + text_body: "text".to_string(), + }, + LocalIssue { + labels: vec!["label2".to_string()], + title: "title2".to_string(), + text_body: "text2".to_string(), + }, + ]; + let github_issues = vec![GithubIssueHelper { + labels: vec!["label1".to_string(), "label2".to_string()], + title: "title".to_string(), + }]; + let result = get_local_issues_without_matches_helper(local_issues, &github_issues); + assert_eq!(result.len(), 1, "One local issue should be returned"); + assert_eq!( + result[0].title, "title2", + "The second local issue should be returned" + ); + } + + #[test] + fn all_matches() { + let local_issues = vec![ + LocalIssue { + labels: vec!["label1".to_string(), "label2".to_string()], + title: "title".to_string(), + text_body: "text".to_string(), + }, + LocalIssue { + labels: vec!["label2".to_string()], + title: "title2".to_string(), + text_body: "text2".to_string(), + }, + ]; + let github_issues = vec![ + GithubIssueHelper { + labels: vec!["label1".to_string(), "label2".to_string()], + title: "title".to_string(), + }, + GithubIssueHelper { + labels: vec!["label2".to_string()], + title: "title2".to_string(), + }, + ]; + let result = get_local_issues_without_matches_helper(local_issues, &github_issues); + assert_eq!(result.len(), 0, "No local issues should be returned"); + } +} + +pub fn get_issue_hash(title: String, labels: Vec) -> u64 { + let mut hasher = DefaultHasher::new(); + title.hash(&mut hasher); + + let mut labels = labels.clone(); + labels.sort(); + for label in labels { + label.hash(&mut hasher); + } + hasher.finish() +} + +#[cfg(test)] +mod test_get_issue_hash { + use super::get_issue_hash; + + #[test] + fn different_order_labels_should_still_match() { + let hash1 = get_issue_hash( + "title".to_string(), + vec!["label1".to_string(), "label2".to_string()], + ); + let hash2 = get_issue_hash( + "title".to_string(), + vec!["label2".to_string(), "label1".to_string()], + ); + assert_eq!(hash1, hash2, "Hashes should be equal"); + } + + #[test] + fn different_shouldnt_match() { + let hash1 = get_issue_hash( + "title".to_string(), + vec!["label1".to_string(), "label2".to_string()], + ); + let hash2 = get_issue_hash( + "title".to_string(), + vec!["label2".to_string(), "label3".to_string()], + ); + let hash3 = get_issue_hash( + "title - new".to_string(), + vec!["label1".to_string(), "label2".to_string()], + ); + let hash4 = get_issue_hash( + "TITLE".to_string(), + vec!["label1".to_string(), "label2".to_string()], + ); + + assert_ne!(hash1, hash2, "Labels don't match"); + assert_ne!(hash1, hash3, "Title text doesn't match"); + assert_ne!(hash1, hash4, "Title casing doesn't match"); + } +} diff --git a/src/yatm_v2/src/utils/make_test_cases.rs b/src/yatm_v2/src/utils/make_test_cases.rs index 65c2f2d..d2b7ab6 100644 --- a/src/yatm_v2/src/utils/make_test_cases.rs +++ b/src/yatm_v2/src/utils/make_test_cases.rs @@ -10,14 +10,20 @@ pub fn make_test_cases( ) -> Vec { let permutations = get_cartesian_product(test_cases_builder.permutations.clone()); let requirements = get_selected_requirements(requirements, test_cases_builder); - permutations - .into_iter() - .map(|permutation| TestCase { - requirement: requirements[0].clone(), - builder_used: test_cases_builder.clone(), - selected_permutation: permutation, - }) - .collect() + + let mut test_cases = Vec::new(); + + for requirement in requirements.iter() { + for permutation in &permutations { + test_cases.push(TestCase { + requirement: requirement.clone(), + builder_used: test_cases_builder.clone(), + selected_permutation: permutation.clone(), + }); + } + } + + test_cases } #[cfg(test)] @@ -44,7 +50,86 @@ mod test_make_test_cases { } #[test] - fn test_make_test_cases() { + fn make_test_cases_two_matches() { + let requirements = vec![ + Requirement { + name: "name1".to_string(), + description: "description".to_string(), + labels: Some(vec!["label1".to_string()]), + links: None, + steps: vec![], + }, + Requirement { + name: "name2".to_string(), + description: "description".to_string(), + labels: Some(vec!["label2".to_string()]), + links: None, + steps: vec![], + }, + ]; + let test_cases_builder = TestCasesBuilder { + name: "Test test case".to_string(), + description: "My description".to_string(), + labels: Some(vec!["label1".to_string(), "label2".to_string()]), + set: vec![SetSteps::Include(Filter { + all_labels: None, + any_names: Some(vec!["name1".to_string(), "name2".to_string()]), + negate: false, + })], + permutations: { + let mut m = std::collections::HashMap::new(); + m.insert( + "key1".to_string(), + vec!["value1".to_string(), "value2".to_string()], + ); + m + }, + version: 1, + }; + let result = make_test_cases(&test_cases_builder, &requirements); + let expected = vec![ + TestCase { + requirement: requirements[0].clone(), + builder_used: test_cases_builder.clone(), + selected_permutation: { + let mut m = std::collections::HashMap::new(); + m.insert("key1".to_string(), "value1".to_string()); + m + }, + }, + TestCase { + requirement: requirements[0].clone(), + builder_used: test_cases_builder.clone(), + selected_permutation: { + let mut m = std::collections::HashMap::new(); + m.insert("key1".to_string(), "value2".to_string()); + m + }, + }, + TestCase { + requirement: requirements[1].clone(), + builder_used: test_cases_builder.clone(), + selected_permutation: { + let mut m = std::collections::HashMap::new(); + m.insert("key1".to_string(), "value1".to_string()); + m + }, + }, + TestCase { + requirement: requirements[1].clone(), + builder_used: test_cases_builder.clone(), + selected_permutation: { + let mut m = std::collections::HashMap::new(); + m.insert("key1".to_string(), "value2".to_string()); + m + }, + }, + ]; + assert!(is_match_test_cases(&result, &expected)); + } + + #[test] + fn test_make_test_cases_one_match() { let requirements = vec![ Requirement { name: "name1".to_string(), diff --git a/src/yatm_v2/src/utils/mod.rs b/src/yatm_v2/src/utils/mod.rs index 5656247..822f347 100644 --- a/src/yatm_v2/src/utils/mod.rs +++ b/src/yatm_v2/src/utils/mod.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod config; pub mod github; +pub mod github_utils; pub mod make_test_cases; pub mod template; diff --git a/src/yatm_v2/src/utils/template.rs b/src/yatm_v2/src/utils/template.rs index e5f8d73..1ac0b6b 100644 --- a/src/yatm_v2/src/utils/template.rs +++ b/src/yatm_v2/src/utils/template.rs @@ -14,13 +14,13 @@ struct GithubIssueTemplate { } #[derive(Debug, Serialize, Deserialize, Clone)] -pub struct GithubIssueContent { +pub struct LocalIssue { pub labels: Vec, pub title: String, pub text_body: String, } -pub fn get_github_issue_content(test_case: TestCase) -> Result { +pub fn get_github_issue_content(test_case: TestCase) -> Result { let labels = get_labels(&test_case); let template = GithubIssueTemplate { @@ -31,7 +31,7 @@ pub fn get_github_issue_content(test_case: TestCase) -> Result