From 068a2838c7b92413f19706a676df430042901e3f Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 15:33:16 +0900 Subject: [PATCH 01/13] add url_generator --- Cargo.lock | 17 +++++++++++++++++ Cargo.toml | 1 + src/main.rs | 1 + src/url_generator.rs | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 src/url_generator.rs diff --git a/Cargo.lock b/Cargo.lock index 0fdd2cc7..f4947644 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1015,6 +1015,7 @@ dependencies = [ "libc", "native-tls", "rand", + "rand_regex", "ratatui", "rlimit", "rustls", @@ -1234,6 +1235,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_regex" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a9fe2d7d9eeaf3279d1780452a5bbd26b31b27938787ef1c3e930d1e9cfbd" +dependencies = [ + "rand", + "regex-syntax", +] + [[package]] name = "ratatui" version = "0.21.0" @@ -1262,6 +1273,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "resolv-conf" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index f3932808..cf07a492 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ base64 = "0.21.0" lazy_static = "1.4.0" rand = "0.8" trust-dns-resolver = "0.22.0" +rand_regex = "0.15.1" [target.'cfg(unix)'.dependencies] rlimit = "0.9.0" diff --git a/src/main.rs b/src/main.rs index 73a908ef..4d298433 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ mod histogram; mod monitor; mod printer; mod timescale; +mod url_generator; #[cfg(all(target_env = "musl", target_pointer_width = "64"))] #[global_allocator] diff --git a/src/url_generator.rs b/src/url_generator.rs new file mode 100644 index 00000000..b6c98202 --- /dev/null +++ b/src/url_generator.rs @@ -0,0 +1,38 @@ +use std::{borrow::Cow, str::FromStr, string::FromUtf8Error}; + +use http::{uri::InvalidUri, Uri}; +use rand::prelude::*; +use rand_regex::Regex; +use thiserror::Error; + +pub enum UrlGenerator { + Static(Uri), + Dynamic(Regex), +} + +#[derive(Error, Debug)] +pub enum UrlGeneratorError { + #[error(transparent)] + InvalidUri(#[from] InvalidUri), + #[error(transparent)] + FromUtf8Error(#[from] FromUtf8Error), +} + +impl UrlGenerator { + pub fn new_static(url: Uri) -> Self { + Self::Static(url) + } + + pub fn new_dynamic(url: Regex) -> Self { + Self::Dynamic(url) + } + + pub fn generate(&self, rng: &mut R) -> Result, UrlGeneratorError> { + match self { + Self::Static(url) => Ok(Cow::Borrowed(&url)), + Self::Dynamic(regex) => Ok(Cow::Owned(Uri::from_str( + Distribution::>::sample(regex, rng)?.as_str(), + )?)), + } + } +} From 76b1884d2b39976a87ac8b7681c9ab63e7f54cd6 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:07:34 +0900 Subject: [PATCH 02/13] wip --- src/client.rs | 79 +++++++++++++++++++++++--------------------- src/main.rs | 21 ++++++++++-- src/url_generator.rs | 5 +-- 3 files changed, 62 insertions(+), 43 deletions(-) diff --git a/src/client.rs b/src/client.rs index 30207f65..a19df24a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,6 +4,7 @@ use rand::prelude::*; use std::sync::Arc; use thiserror::Error; +use crate::url_generator::{UrlGenerator, UrlGeneratorError}; use crate::ConnectToEntry; #[derive(Debug, Clone)] @@ -39,8 +40,6 @@ impl RequestResult { #[allow(clippy::upper_case_acronyms)] struct DNS { - // To pick a random address from DNS. - rng: rand::rngs::StdRng, connect_to: Arc>, resolver: Arc< trust_dns_resolver::AsyncResolver< @@ -54,7 +53,11 @@ struct DNS { impl DNS { /// Perform a DNS lookup for a given url and returns (ip_addr, port) - async fn lookup(&mut self, url: &http::Uri) -> Result<(std::net::IpAddr, u16), ClientError> { + async fn lookup( + &self, + url: &http::Uri, + rng: &mut R, + ) -> Result<(std::net::IpAddr, u16), ClientError> { let host = url.host().ok_or(ClientError::HostNotFound)?; let port = get_http_port(url).ok_or(ClientError::PortNotFound)?; @@ -87,9 +90,7 @@ impl DNS { .iter() .collect::>(); - let addr = *addrs - .choose(&mut self.rng) - .ok_or(ClientError::DNSNoRecord)?; + let addr = *addrs.choose(rng).ok_or(ClientError::DNSNoRecord)?; Ok((addr, port)) } @@ -97,7 +98,7 @@ impl DNS { pub struct ClientBuilder { pub http_version: http::Version, - pub url: http::Uri, + pub url_generator: UrlGenerator, pub method: http::Method, pub headers: http::header::HeaderMap, pub body: Option<&'static [u8]>, @@ -122,15 +123,15 @@ pub struct ClientBuilder { impl ClientBuilder { pub fn build(&self) -> Client { Client { - url: self.url.clone(), + url_generator: self.url_generator.clone(), method: self.method.clone(), headers: self.headers.clone(), body: self.body, dns: DNS { resolver: self.resolver.clone(), connect_to: self.connect_to.clone(), - rng: rand::rngs::StdRng::from_entropy(), }, + rng: StdRng::from_entropy(), client: None, timeout: self.timeout, http_version: self.http_version, @@ -191,15 +192,18 @@ pub enum ClientError { InvalidUri(#[from] http::uri::InvalidUri), #[error("timeout")] Timeout, + #[error(transparent)] + UrlGeneratorError(#[from] UrlGeneratorError), } pub struct Client { http_version: http::Version, - url: http::Uri, + url_generator: UrlGenerator, method: http::Method, headers: http::header::HeaderMap, body: Option<&'static [u8]>, dns: DNS, + rng: rand::rngs::StdRng, client: Option>, timeout: Option, redirect_limit: usize, @@ -212,11 +216,12 @@ pub struct Client { impl Client { #[cfg(unix)] async fn client( - &mut self, + &self, addr: (std::net::IpAddr, u16), + url: &http::Uri, ) -> Result, ClientError> { - if self.url.scheme() == Some(&http::uri::Scheme::HTTPS) { - self.tls_client(addr).await + if url.scheme() == Some(&http::uri::Scheme::HTTPS) { + self.tls_client(addr, url).await } else if let Some(socket_path) = &self.unix_socket { let stream = tokio::net::UnixStream::connect(socket_path).await?; let (send, conn) = hyper::client::conn::handshake(stream).await?; @@ -234,11 +239,12 @@ impl Client { #[cfg(not(unix))] async fn client( - &mut self, + &self, addr: (std::net::IpAddr, u16), + url: &http::Uri, ) -> Result, ClientError> { - if self.url.scheme() == Some(&http::uri::Scheme::HTTPS) { - self.tls_client(addr).await + if url.scheme() == Some(&http::uri::Scheme::HTTPS) { + self.tls_client(addr, url).await } else { let stream = tokio::net::TcpStream::connect(addr).await?; stream.set_nodelay(true)?; @@ -251,8 +257,9 @@ impl Client { #[cfg(all(feature = "native-tls", not(feature = "rustls")))] async fn tls_client( - &mut self, + &self, addr: (std::net::IpAddr, u16), + url: &http::Uri, ) -> Result, ClientError> { let stream = tokio::net::TcpStream::connect(addr).await?; stream.set_nodelay(true)?; @@ -267,7 +274,7 @@ impl Client { }; let connector = tokio_native_tls::TlsConnector::from(connector); let stream = connector - .connect(self.url.host().ok_or(ClientError::HostNotFound)?, stream) + .connect(url.host().ok_or(ClientError::HostNotFound)?, stream) .await?; let (send, conn) = hyper::client::conn::handshake(stream).await?; @@ -277,8 +284,9 @@ impl Client { #[cfg(feature = "rustls")] async fn tls_client( - &mut self, + &self, addr: (std::net::IpAddr, u16), + url: &http::Uri, ) -> Result, ClientError> { let stream = tokio::net::TcpStream::connect(addr).await?; stream.set_nodelay(true)?; @@ -297,8 +305,7 @@ impl Client { .set_certificate_verifier(Arc::new(AcceptAnyServerCert)); } let connector = tokio_rustls::TlsConnector::from(Arc::new(config)); - let domain = - rustls::ServerName::try_from(self.url.host().ok_or(ClientError::HostNotFound)?)?; + let domain = rustls::ServerName::try_from(url.host().ok_or(ClientError::HostNotFound)?)?; let stream = connector.connect(domain, stream).await?; let (send, conn) = hyper::client::conn::handshake(stream).await?; @@ -336,15 +343,16 @@ impl Client { }; let do_req = async { + let url = self.url_generator.generate(&mut self.rng)?; let mut start = std::time::Instant::now(); let mut connection_time: Option = None; let mut send_request = if let Some(send_request) = self.client.take() { send_request } else { - let addr = self.dns.lookup(&self.url).await?; + let addr = self.dns.lookup(&url, &mut self.rng).await?; let dns_lookup = std::time::Instant::now(); - let send_request = self.client(addr).await?; + let send_request = self.client(addr, &url).await?; let dialup = std::time::Instant::now(); connection_time = Some(ConnectionTime { dns_lookup, dialup }); @@ -355,13 +363,13 @@ impl Client { .is_err() { start = std::time::Instant::now(); - let addr = self.dns.lookup(&self.url).await?; + let addr = self.dns.lookup(&url, &mut self.rng).await?; let dns_lookup = std::time::Instant::now(); - send_request = self.client(addr).await?; + send_request = self.client(addr, &url).await?; let dialup = std::time::Instant::now(); connection_time = Some(ConnectionTime { dns_lookup, dialup }); } - let request = self.request(&self.url)?; + let request = self.request(&url)?; match send_request.send_request(request).await { Ok(res) => { let (parts, mut stream) = res.into_parts(); @@ -375,12 +383,7 @@ impl Client { if self.redirect_limit != 0 { if let Some(location) = parts.headers.get("Location") { let (send_request_redirect, new_status, len) = self - .redirect( - send_request, - &self.url.clone(), - location, - self.redirect_limit, - ) + .redirect(send_request, &url, location, self.redirect_limit) .await?; send_request = send_request_redirect; @@ -425,7 +428,7 @@ impl Client { #[allow(clippy::type_complexity)] fn redirect<'a>( - &'a mut self, + &'a self, send_request: hyper::client::conn::SendRequest, base_url: &'a http::Uri, location: &'a http::header::HeaderValue, @@ -460,20 +463,20 @@ impl Client { // reuse connection (send_request, None) } else { - let addr = self.dns.lookup(&url).await?; - (self.client(addr).await?, Some(send_request)) + let addr = self.dns.lookup(&url, &mut self.rng).await?; + (self.client(addr, &url).await?, Some(send_request)) }; while futures::future::poll_fn(|ctx| send_request.poll_ready(ctx)) .await .is_err() { - let addr = self.dns.lookup(&url).await?; - send_request = self.client(addr).await?; + let addr = self.dns.lookup(&url, &mut self.rng).await?; + send_request = self.client(addr, &url).await?; } let mut request = self.request(&url)?; - if url.authority() != self.url.authority() { + if url.authority() != base_url.authority() { request.headers_mut().insert( http::header::HOST, http::HeaderValue::from_str( diff --git a/src/main.rs b/src/main.rs index 4d298433..0b8b57cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,9 +3,13 @@ use clap::Parser; use crossterm::tty::IsTty; use futures::prelude::*; use http::header::{HeaderName, HeaderValue}; +use http::Uri; use printer::PrintMode; +use rand::prelude::*; +use rand_regex::Regex; use std::sync::Arc; use std::{io::Read, str::FromStr}; +use url_generator::UrlGenerator; mod client; mod histogram; @@ -24,7 +28,7 @@ use client::{ClientError, RequestResult}; #[clap(author, about, version, override_usage = "oha [FLAGS] [OPTIONS] ")] struct Opts { #[clap(help = "Target URL.")] - url: http::Uri, + url: String, #[structopt( help = "Number of requests to run.", short = 'n', @@ -49,6 +53,9 @@ Examples: -z 10s -z 3m.", help = "Correct latency to avoid coordinated omission problem. It's ignored if -q is not set.", long = "latency-correction" )] + rand_regex_url: bool, + #[clap(default_value = "4")] + max_repeat: u32, latency_correction: bool, #[clap(help = "No realtime tui", long = "no-tui")] no_tui: bool, @@ -180,6 +187,14 @@ async fn main() -> anyhow::Result<()> { http::Version::HTTP_11 }; + let url_generator = if opts.rand_regex_url { + UrlGenerator::new_dynamic(Regex::compile(&opts.url, opts.max_repeat)?) + } else { + UrlGenerator::new_static(Uri::from_str(&opts.url)?) + }; + + let url = url_generator.generate(&mut thread_rng())?; + let headers = { let mut headers: http::header::HeaderMap = Default::default(); @@ -200,7 +215,7 @@ async fn main() -> anyhow::Result<()> { headers.insert( http::header::HOST, http::header::HeaderValue::from_str( - opts.url.authority().context("get authority")?.as_str(), + url.authority().context("get authority")?.as_str(), )?, ); @@ -367,7 +382,7 @@ async fn main() -> anyhow::Result<()> { // client_builder builds client for each workers let client_builder = client::ClientBuilder { http_version, - url: opts.url, + url_generator, method: opts.method, headers, body, diff --git a/src/url_generator.rs b/src/url_generator.rs index b6c98202..f327dad8 100644 --- a/src/url_generator.rs +++ b/src/url_generator.rs @@ -5,6 +5,7 @@ use rand::prelude::*; use rand_regex::Regex; use thiserror::Error; +#[derive(Clone, Debug)] pub enum UrlGenerator { Static(Uri), Dynamic(Regex), @@ -23,8 +24,8 @@ impl UrlGenerator { Self::Static(url) } - pub fn new_dynamic(url: Regex) -> Self { - Self::Dynamic(url) + pub fn new_dynamic(regex: Regex) -> Self { + Self::Dynamic(regex) } pub fn generate(&self, rng: &mut R) -> Result, UrlGeneratorError> { From 396f72bf3d691e1eff4cdfc72146018d60da689c Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:12:26 +0900 Subject: [PATCH 03/13] it compiles --- src/client.rs | 39 ++++++++++++++++++++++----------------- src/url_generator.rs | 2 +- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/client.rs b/src/client.rs index a19df24a..99f7bd9a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -131,7 +131,6 @@ impl ClientBuilder { resolver: self.resolver.clone(), connect_to: self.connect_to.clone(), }, - rng: StdRng::from_entropy(), client: None, timeout: self.timeout, http_version: self.http_version, @@ -203,7 +202,6 @@ pub struct Client { headers: http::header::HeaderMap, body: Option<&'static [u8]>, dns: DNS, - rng: rand::rngs::StdRng, client: Option>, timeout: Option, redirect_limit: usize, @@ -335,7 +333,7 @@ impl Client { } } - pub async fn work(&mut self) -> Result { + pub async fn work(&mut self, rng: &mut R) -> Result { let timeout = if let Some(timeout) = self.timeout { tokio::time::sleep(timeout).boxed() } else { @@ -343,14 +341,14 @@ impl Client { }; let do_req = async { - let url = self.url_generator.generate(&mut self.rng)?; + let url = self.url_generator.generate(rng)?; let mut start = std::time::Instant::now(); let mut connection_time: Option = None; let mut send_request = if let Some(send_request) = self.client.take() { send_request } else { - let addr = self.dns.lookup(&url, &mut self.rng).await?; + let addr = self.dns.lookup(&url, rng).await?; let dns_lookup = std::time::Instant::now(); let send_request = self.client(addr, &url).await?; let dialup = std::time::Instant::now(); @@ -363,7 +361,7 @@ impl Client { .is_err() { start = std::time::Instant::now(); - let addr = self.dns.lookup(&url, &mut self.rng).await?; + let addr = self.dns.lookup(&url, rng).await?; let dns_lookup = std::time::Instant::now(); send_request = self.client(addr, &url).await?; let dialup = std::time::Instant::now(); @@ -383,7 +381,7 @@ impl Client { if self.redirect_limit != 0 { if let Some(location) = parts.headers.get("Location") { let (send_request_redirect, new_status, len) = self - .redirect(send_request, &url, location, self.redirect_limit) + .redirect(send_request, &url, location, self.redirect_limit, rng) .await?; send_request = send_request_redirect; @@ -427,12 +425,13 @@ impl Client { } #[allow(clippy::type_complexity)] - fn redirect<'a>( + fn redirect<'a, R: Rng + Send>( &'a self, send_request: hyper::client::conn::SendRequest, base_url: &'a http::Uri, location: &'a http::header::HeaderValue, limit: usize, + rng: &'a mut R, ) -> futures::future::BoxFuture< 'a, Result< @@ -463,7 +462,7 @@ impl Client { // reuse connection (send_request, None) } else { - let addr = self.dns.lookup(&url, &mut self.rng).await?; + let addr = self.dns.lookup(&url, rng).await?; (self.client(addr, &url).await?, Some(send_request)) }; @@ -471,7 +470,7 @@ impl Client { .await .is_err() { - let addr = self.dns.lookup(&url, &mut self.rng).await?; + let addr = self.dns.lookup(&url, rng).await?; send_request = self.client(addr, &url).await?; } @@ -497,7 +496,7 @@ impl Client { if let Some(location) = parts.headers.get("Location") { let (send_request_redirect, new_status, len) = self - .redirect(send_request, &url, location, limit - 1) + .redirect(send_request, &url, location, limit - 1, rng) .await?; send_request = send_request_redirect; status = new_status; @@ -587,11 +586,12 @@ pub async fn work( let futures = (0..n_workers) .map(|_| { let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); let report_tx = report_tx.clone(); let counter = counter.clone(); tokio::spawn(async move { while counter.fetch_add(1, Ordering::Relaxed) < n_tasks { - let res = w.work().await; + let res = w.work(&mut rng).await; let is_cancel = is_too_many_open_files(&res); report_tx.send_async(res).await.unwrap(); if is_cancel { @@ -632,11 +632,12 @@ pub async fn work_with_qps( let futures = (0..n_workers) .map(|_| { let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); let report_tx = report_tx.clone(); let rx = rx.clone(); tokio::spawn(async move { while let Ok(()) = rx.recv_async().await { - let res = w.work().await; + let res = w.work(&mut rng).await; let is_cancel = is_too_many_open_files(&res); report_tx.send_async(res).await.unwrap(); if is_cancel { @@ -677,11 +678,12 @@ pub async fn work_with_qps_latency_correction( let futures = (0..n_workers) .map(|_| { let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); let report_tx = report_tx.clone(); let rx = rx.clone(); tokio::spawn(async move { while let Ok(start) = rx.recv_async().await { - let mut res = w.work().await; + let mut res = w.work(&mut rng).await; if let Ok(request_result) = &mut res { request_result.start_latency_correction = Some(start); @@ -713,9 +715,10 @@ pub async fn work_until( .map(|_| { let report_tx = report_tx.clone(); let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); tokio::spawn(async move { loop { - let res = w.work().await; + let res = w.work(&mut rng).await; let is_cancel = is_too_many_open_files(&res); report_tx.send_async(res).await.unwrap(); if is_cancel { @@ -762,11 +765,12 @@ pub async fn work_until_with_qps( let futures = (0..n_workers) .map(|_| { let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); let report_tx = report_tx.clone(); let rx = rx.clone(); tokio::spawn(async move { while let Ok(()) = rx.recv_async().await { - let res = w.work().await; + let res = w.work(&mut rng).await; let is_cancel = is_too_many_open_files(&res); report_tx.send_async(res).await.unwrap(); if is_cancel { @@ -814,11 +818,12 @@ pub async fn work_until_with_qps_latency_correction( let futures = (0..n_workers) .map(|_| { let mut w = client_builder.build(); + let mut rng = StdRng::from_entropy(); let report_tx = report_tx.clone(); let rx = rx.clone(); tokio::spawn(async move { while let Ok(start) = rx.recv_async().await { - let mut res = w.work().await; + let mut res = w.work(&mut rng).await; if let Ok(request_result) = &mut res { request_result.start_latency_correction = Some(start); diff --git a/src/url_generator.rs b/src/url_generator.rs index f327dad8..4a55c5cc 100644 --- a/src/url_generator.rs +++ b/src/url_generator.rs @@ -30,7 +30,7 @@ impl UrlGenerator { pub fn generate(&self, rng: &mut R) -> Result, UrlGeneratorError> { match self { - Self::Static(url) => Ok(Cow::Borrowed(&url)), + Self::Static(url) => Ok(Cow::Borrowed(url)), Self::Dynamic(regex) => Ok(Cow::Owned(Uri::from_str( Distribution::>::sample(regex, rng)?.as_str(), )?)), From 053c82f75f259b3c1447de408a35357a7d7b3293 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:17:37 +0900 Subject: [PATCH 04/13] fix --- src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 0b8b57cf..120e8001 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,13 +49,14 @@ Examples: -z 10s -z 3m.", duration: Option, #[clap(help = "Rate limit for all, in queries per second (QPS)", short = 'q')] query_per_second: Option, + #[clap(default_value = "false", long)] + rand_regex_url: bool, + #[clap(default_value = "4", long)] + max_repeat: u32, #[clap( help = "Correct latency to avoid coordinated omission problem. It's ignored if -q is not set.", long = "latency-correction" )] - rand_regex_url: bool, - #[clap(default_value = "4")] - max_repeat: u32, latency_correction: bool, #[clap(help = "No realtime tui", long = "no-tui")] no_tui: bool, From 99a1e0f51fc054e16afabaa187e9df487ad9e4dd Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:31:59 +0900 Subject: [PATCH 05/13] Add help message --- src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 120e8001..210e31ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,9 +49,17 @@ Examples: -z 10s -z 3m.", duration: Option, #[clap(help = "Rate limit for all, in queries per second (QPS)", short = 'q')] query_per_second: Option, - #[clap(default_value = "false", long)] + #[clap( + help = "Generate URL by rand_regex crate for each query. Currently dynamic scheme, host and port are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", + default_value = "false", + long + )] rand_regex_url: bool, - #[clap(default_value = "4", long)] + #[clap( + help = "A parameter for the '--query_per_second'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become.", + default_value = "4", + long + )] max_repeat: u32, #[clap( help = "Correct latency to avoid coordinated omission problem. It's ignored if -q is not set.", From c0f14d0a06843a7ab18eef340e8fad050e4b9000 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:42:40 +0900 Subject: [PATCH 06/13] Update txts --- CHANGELOG.md | 2 ++ README.md | 2 ++ src/main.rs | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1217e8bd..cf2562e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- Support randomly generated URL using rand_regex crate + # 0.5.9 (2023-06-12) - Fix -H Header parser diff --git a/README.md b/README.md index d62229b6..d54410e4 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ Options: -z Duration of application to send requests. If duration is specified, n is ignored. Examples: -z 10s -z 3m. -q Rate limit for all, in queries per second (QPS) + --rand-regex-url Generate URL by rand_regex crate for each query e.g. http://localhost/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. + --max-repeat A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become. [default: 4] --latency-correction Correct latency to avoid coordinated omission problem. It's ignored if -q is not set. --no-tui No realtime tui -j, --json Print results as JSON diff --git a/src/main.rs b/src/main.rs index 210e31ab..8a14e7ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,13 +50,13 @@ Examples: -z 10s -z 3m.", #[clap(help = "Rate limit for all, in queries per second (QPS)", short = 'q')] query_per_second: Option, #[clap( - help = "Generate URL by rand_regex crate for each query. Currently dynamic scheme, host and port are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", + help = "Generate URL by rand_regex crate for each query e.g. http://localhost/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", default_value = "false", long )] rand_regex_url: bool, #[clap( - help = "A parameter for the '--query_per_second'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become.", + help = "A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become.", default_value = "4", long )] From dd03a8087688625b7d9000263641de9373c5f191 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:58:09 +0900 Subject: [PATCH 07/13] test --- tests/tests.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/tests.rs b/tests/tests.rs index 29d00005..c8186e86 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -101,6 +101,33 @@ async fn get_query(p: &'static str) -> String { rx.try_recv().unwrap() } +async fn get_query_rand_regex(p: &'static str) -> String { + let (tx, rx) = flume::unbounded(); + let report_headers = warp::path!(String).map(move |path: String| { + tx.send(path).unwrap(); + "Hello World" + }); + + let _guard = PORT_LOCK.lock().await; + let port = get_port::tcp::TcpPort::any("127.0.0.1").unwrap(); + tokio::spawn(warp::serve(report_headers).run(([127, 0, 0, 1], port))); + // It's not guaranteed that the port is used here. + // So we can't drop guard here. + + tokio::task::spawn_blocking(move || { + Command::cargo_bin("oha") + .unwrap() + .args(["-n", "1", "--no-tui", "--rand-regex-url"]) + .arg(format!(r"http://127\.0\.0\.1:{port}/{p}")) + .assert() + .success(); + }) + .await + .unwrap(); + + rx.try_recv().unwrap() +} + async fn redirect(n: usize, is_relative: bool, limit: usize) -> bool { let (tx, rx) = flume::unbounded(); let _guard = PORT_LOCK.lock().await; @@ -387,6 +414,16 @@ async fn test_query() { ); } +#[tokio::test] +async fn test_query_rand_regex() { + let query = get_query_rand_regex("[a-z][0-9][a-z]").await; + let chars = query.chars().collect::>(); + assert_eq!(chars.len(), 3); + assert!(chars[0].is_ascii_lowercase()); + assert!(chars[1].is_ascii_digit()); + assert!(chars[2].is_ascii_lowercase()); +} + #[tokio::test] async fn test_connect_to() { assert_eq!( From 43d2615e912d53077d8dc5197cf378d52be718a8 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 16:58:48 +0900 Subject: [PATCH 08/13] test --- tests/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.rs b/tests/tests.rs index c8186e86..4a1311da 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -101,7 +101,7 @@ async fn get_query(p: &'static str) -> String { rx.try_recv().unwrap() } -async fn get_query_rand_regex(p: &'static str) -> String { +async fn get_path_rand_regex(p: &'static str) -> String { let (tx, rx) = flume::unbounded(); let report_headers = warp::path!(String).map(move |path: String| { tx.send(path).unwrap(); @@ -416,7 +416,7 @@ async fn test_query() { #[tokio::test] async fn test_query_rand_regex() { - let query = get_query_rand_regex("[a-z][0-9][a-z]").await; + let query = get_path_rand_regex("[a-z][0-9][a-z]").await; let chars = query.chars().collect::>(); assert_eq!(chars.len(), 3); assert!(chars[0].is_ascii_lowercase()); From 0d3c3024e99872662abf2d6578a196169a9b0201 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 17 Jun 2023 17:06:35 +0900 Subject: [PATCH 09/13] CI: test and check also rustls --- .github/workflows/CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ee354588..3a744392 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -10,6 +10,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] rust: [stable, beta, nightly] + additional_args: ["", "--no-default-features --features rustls"] steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -20,6 +21,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: check + args: ${{ matrix.additional_args }} test: name: Test Suite @@ -28,6 +30,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] rust: [stable, beta, nightly] + additional_args: ["", "--no-default-features --features rustls"] steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -38,6 +41,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: test + args: ${{ matrix.additional_args }} fmt: name: Rustfmt From d036c136b56fd77d52254363fd1b491852681579 Mon Sep 17 00:00:00 2001 From: hatoo Date: Mon, 19 Jun 2023 17:01:44 +0900 Subject: [PATCH 10/13] Better error message --- src/url_generator.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/url_generator.rs b/src/url_generator.rs index 4a55c5cc..0eb38cfe 100644 --- a/src/url_generator.rs +++ b/src/url_generator.rs @@ -13,8 +13,8 @@ pub enum UrlGenerator { #[derive(Error, Debug)] pub enum UrlGeneratorError { - #[error(transparent)] - InvalidUri(#[from] InvalidUri), + #[error("{0}, generated url: {1}")] + InvalidUri(InvalidUri, String), #[error(transparent)] FromUtf8Error(#[from] FromUtf8Error), } @@ -31,9 +31,12 @@ impl UrlGenerator { pub fn generate(&self, rng: &mut R) -> Result, UrlGeneratorError> { match self { Self::Static(url) => Ok(Cow::Borrowed(url)), - Self::Dynamic(regex) => Ok(Cow::Owned(Uri::from_str( - Distribution::>::sample(regex, rng)?.as_str(), - )?)), + Self::Dynamic(regex) => { + let generated = Distribution::>::sample(regex, rng)?; + Ok(Cow::Owned(Uri::from_str(generated.as_str()).map_err( + |e| UrlGeneratorError::InvalidUri(e, generated), + )?)) + } } } } From 00229000673356fabe4cef34db974a1d83b543ed Mon Sep 17 00:00:00 2001 From: hatoo Date: Mon, 19 Jun 2023 17:37:21 +0900 Subject: [PATCH 11/13] disable dot in rand_regex --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 16 ++++++++++++++-- tests/tests.rs | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4947644..cc494eed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1017,6 +1017,7 @@ dependencies = [ "rand", "rand_regex", "ratatui", + "regex-syntax", "rlimit", "rustls", "rustls-native-certs", diff --git a/Cargo.toml b/Cargo.toml index cf07a492..543b311e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ lazy_static = "1.4.0" rand = "0.8" trust-dns-resolver = "0.22.0" rand_regex = "0.15.1" +regex-syntax = "0.6.22" [target.'cfg(unix)'.dependencies] rlimit = "0.9.0" diff --git a/src/main.rs b/src/main.rs index 8a14e7ab..564f8236 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,7 +50,7 @@ Examples: -z 10s -z 3m.", #[clap(help = "Rate limit for all, in queries per second (QPS)", short = 'q')] query_per_second: Option, #[clap( - help = "Generate URL by rand_regex crate for each query e.g. http://localhost/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", + help = "Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", default_value = "false", long )] @@ -197,7 +197,19 @@ async fn main() -> anyhow::Result<()> { }; let url_generator = if opts.rand_regex_url { - UrlGenerator::new_dynamic(Regex::compile(&opts.url, opts.max_repeat)?) + // Almost URL has dot in domain, so disable dot in regex for convenience. + let dot_disabled: String = opts + .url + .chars() + .map(|c| { + if c == '.' { + regex_syntax::escape(".") + } else { + c.to_string() + } + }) + .collect(); + UrlGenerator::new_dynamic(Regex::compile(&dot_disabled, opts.max_repeat)?) } else { UrlGenerator::new_static(Uri::from_str(&opts.url)?) }; diff --git a/tests/tests.rs b/tests/tests.rs index 4a1311da..addaceca 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -118,7 +118,7 @@ async fn get_path_rand_regex(p: &'static str) -> String { Command::cargo_bin("oha") .unwrap() .args(["-n", "1", "--no-tui", "--rand-regex-url"]) - .arg(format!(r"http://127\.0\.0\.1:{port}/{p}")) + .arg(format!(r"http://127.0.0.1:{port}/{p}")) .assert() .success(); }) From 96d59601e929c3984ca4ee326ace4f2b501fdb92 Mon Sep 17 00:00:00 2001 From: hatoo Date: Mon, 19 Jun 2023 18:15:55 +0900 Subject: [PATCH 12/13] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d54410e4..13bc679c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Options: -z Duration of application to send requests. If duration is specified, n is ignored. Examples: -z 10s -z 3m. -q Rate limit for all, in queries per second (QPS) - --rand-regex-url Generate URL by rand_regex crate for each query e.g. http://localhost/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. + --rand-regex-url Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. --max-repeat A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become. [default: 4] --latency-correction Correct latency to avoid coordinated omission problem. It's ignored if -q is not set. --no-tui No realtime tui From c0b9b65566c89938ae76a3eba55f53a3913dba53 Mon Sep 17 00:00:00 2001 From: hatoo Date: Sat, 24 Jun 2023 15:18:37 +0900 Subject: [PATCH 13/13] doc doc doc --- README.md | 16 +++++++++++++++- src/main.rs | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 13bc679c..81a0fb0f 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Options: -z Duration of application to send requests. If duration is specified, n is ignored. Examples: -z 10s -z 3m. -q Rate limit for all, in queries per second (QPS) - --rand-regex-url Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. + --rand-regex-url Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. --max-repeat A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become. [default: 4] --latency-correction Correct latency to avoid coordinated omission problem. It's ignored if -q is not set. --no-tui No realtime tui @@ -109,6 +109,20 @@ oha <-z or -n> -c -q --la You can avoid `Coordinated Omission Problem` by using `--latency-correction`. +## Dynamic url feature + +You can use `--rand-regex-url` option to generate random url for each connection. + +```sh +oha --rand-regex-url http://127.0.0.1/[a-z][a-z][0-9] +``` + +Each Urls are generated by [rand_regex](https://github.com/kennytm/rand_regex) crate but regex's dot is disabled since it's not useful for this purpose and it's very incovenient if url's dots are interpreted as regex's dot. + +Optionaly you can set `--max-repeat` option to limit max repeat count for each regex. e.g http://127.0.0.1/[a-z]* with `--max-repeat 4` will generate url like http://127.0.0.1/[a-z]{0,4} + +If keep-alive is enabled, path and query are generated for each request but shceme, host and port are fixed during connection. + # Contribution Feel free to help us! diff --git a/src/main.rs b/src/main.rs index 564f8236..71e3c166 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,7 +50,7 @@ Examples: -z 10s -z 3m.", #[clap(help = "Rate limit for all, in queries per second (QPS)", short = 'q')] query_per_second: Option, #[clap( - help = "Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive are not works well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", + help = "Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax.", default_value = "false", long )]