Download rest specs from tarball

This commit updates howrest specs are
downloaded by streaming the tarball for
a specific branch, and extracting rest specs
from the tarball. This is much faster
than downloading separate files,

Author: russcam

api_generator/Cargo.toml

@@ -11,19 +11,21 @@ publish = false
 array_tool = "1.0.3"
 dialoguer = "0.3.0"
 failure = "0.1.5"
+flate2 = "~1"
+globset = "~0.4"
 Inflector = "0.11.4"
 indicatif = "0.12.0"
 lazy_static = "1.4.0"
 quote = "~0.3"
 reduce = "0.1.2"
 regex = "1.3.1"
-reqwest = { version = "~0.10", features = ["blocking", "json", "gzip"] }
+reqwest = "~0.9"
 rustfmt-nightly = "1.4.17"
 semver = "0.9.0"
 serde = "~1"
 serde_json = "~1"
 serde_derive = "~1"
 syn = { version = "~0.11", features = ["full"] }
-rayon = "1.2.0"
+tar = "~0.4"
 url = "2.1.1"
 void = "1.0.2"

api_generator/src/bin/run.rs

@@ -69,6 +69,7 @@ fn main() -> Result<(), failure::Error> {
             .unwrap();
 
         fs::remove_dir_all(&download_dir)?;
+        fs::create_dir_all(&download_dir)?;
         rest_spec::download_specs(&branch, &download_dir)?;
         File::create(&last_downloaded_version)?.write_all(branch.as_bytes())?;
     }

api_generator/src/rest_spec/mod.rs

@@ -17,78 +17,58 @@
  * under the License.
  */
 extern crate reqwest;
+use self::reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
+use flate2::read::GzDecoder;
+use globset::Glob;
+use reqwest::Response;
+use std::{fs::File, io, path::PathBuf};
+use tar::{Archive, Entry};
 
-mod parallel_downloading;
-
-use parallel_downloading::download_specs_to_dir;
-use serde::Deserialize;
-use std::{fs, path::PathBuf};
-
-struct GitHubSpec {
-    dir: String,
-    branch: String,
-    url: String,
-}
-
-#[derive(Deserialize, Debug)]
-struct Links {
-    #[serde(rename = "self")]
-    self_link: String,
-    git: String,
-    html: String,
-}
+pub fn download_specs(branch: &str, download_dir: &PathBuf) -> Result<(), failure::Error> {
+    let url = format!(
+        "https://api.github.com/repos/elastic/elasticsearch/tarball/{}",
+        branch
+    );
 
-#[derive(Deserialize, Debug)]
-struct RestApiSpec {
-    name: String,
-    path: String,
-    sha: String,
-    size: i32,
-    url: String,
-    html_url: String,
-    git_url: String,
-    download_url: String,
-    #[serde(rename = "type")]
-    ty: String,
-    #[serde(rename = "_links")]
-    links: Links,
-}
+    let mut headers = HeaderMap::new();
+    headers.append(
+        USER_AGENT,
+        HeaderValue::from_str(&format!("elasticsearch-rs/{}", env!("CARGO_PKG_NAME")))?,
+    );
+    let client = reqwest::ClientBuilder::new()
+        .default_headers(headers)
+        .build()
+        .unwrap();
 
-pub fn download_specs(branch: &str, download_dir: &PathBuf) -> Result<(), failure::Error> {
-    let spec_urls = [
-        ("core".to_string(), "https://api.github.com/repos/elastic/elasticsearch/contents/rest-api-spec/src/main/resources/rest-api-spec/api".to_string()),
-        ("xpack".to_string(), "https://api.github.com/repos/elastic/elasticsearch/contents/x-pack/plugin/src/test/resources/rest-api-spec/api".to_string())];
+    let response = client.get(&url).send()?;
+    let tar = GzDecoder::new(response);
+    let mut archive = Archive::new(tar);
 
-    let specs: Vec<GitHubSpec> = spec_urls
-        .iter()
-        .map(|(dir, template_url)| {
-            let url = format!("{}?ref={}", template_url, branch);
-            GitHubSpec {
-                dir: dir.to_string(),
-                branch: branch.to_string(),
-                url,
-            }
-        })
-        .collect();
+    let oss_spec = Glob::new("**/rest-api-spec/src/main/resources/rest-api-spec/api/*.json")?
+        .compile_matcher();
+    let xpack_spec = Glob::new("**/x-pack/plugin/src/test/resources/rest-api-spec/api/*.json")?
+        .compile_matcher();
 
-    fs::create_dir_all(download_dir)?;
-    for spec in specs {
-        download_endpoints(&spec, &download_dir)?;
+    for entry in archive.entries()? {
+        let file = entry?;
+        let path = file.path()?;
+        if oss_spec.is_match(&path) || xpack_spec.is_match(&path) {
+            write_spec_file(download_dir, file)?;
+        }
     }
 
     Ok(())
 }
 
-fn download_endpoints(spec: &GitHubSpec, download_dir: &PathBuf) -> Result<(), failure::Error> {
-    let client = reqwest::blocking::ClientBuilder::new()
-        .user_agent(concat!("RustApiGenerator/", env!("CARGO_PKG_VERSION")))
-        .build()
-        .unwrap();
+fn write_spec_file(
+    download_dir: &PathBuf,
+    mut entry: Entry<GzDecoder<Response>>,
+) -> Result<(), failure::Error> {
+    let path = entry.path()?;
+    let mut dir = download_dir.clone();
+    dir.push(path.file_name().unwrap());
+    let mut file = File::create(&dir)?;
+    io::copy(&mut entry, &mut file)?;
 
-    let response = client.get(&spec.url).send().unwrap();
-    let rest_api_specs: Vec<RestApiSpec> = response.json().unwrap();
-    println!("Downloading {} specs from {}", spec.dir, spec.branch);
-    download_specs_to_dir(client, rest_api_specs.as_slice(), download_dir).unwrap();
-    println!("Done downloading {} specs from {}", spec.dir, spec.branch);
     Ok(())
 }

api_generator/src/rest_spec/parallel_downloading.rs

@@ -47,7 +47,7 @@ pub fn download_test_suites(branch: &str, download_dir: &PathBuf) -> Result<(),
     let mut headers = HeaderMap::new();
     headers.append(
         USER_AGENT,
-        HeaderValue::from_str("elasticsearch-rs/yaml_test_runner")?,
+        HeaderValue::from_str(&format!("elasticsearch-rs/{}", env!("CARGO_PKG_NAME")))?,
     );
     let client = reqwest::ClientBuilder::new()
         .default_headers(headers)