Download yaml tests from tarball

This commit updates how yaml tests are
downloaded by streaming the tarball for
a specific branch, and extracting yaml tests
from the tarball. Not only is this much faster
than downloading separate files, it removes
the need for a GitHub access token as
anonymous access limits are likely sufficient.

Don't download rest specs when running
yaml_test_runner as the checked in rest specs
will be those that have been used to generate
the client, and therefor should be the ones
used when generating tests using the client.

Author: russcam

.ci/run-repository.sh

@@ -5,15 +5,12 @@
 # STACK_VERSION -- version e.g Major.Minor.Patch(-Prelease)
 # TEST_SUITE -- which test suite to run: oss or xpack
 # ELASTICSEARCH_URL -- The url at which elasticsearch is reachable, a default is composed based on STACK_VERSION and TEST_SUITE
-# BRANCH -- Elasticsearch branch from which to pull the YAML test files
 # RUST_TOOLCHAIN -- Rust toolchain version to compile and run tests
-# TOKEN -- GitHub access token used to download the YAML files from the GitHub API
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
 RUST_TOOLCHAIN=${RUST_TOOLCHAIN-nightly-2020-06-09}
-TOKEN=${TOKEN-}
 ELASTICSEARCH_URL=${ELASTICSEARCH_URL-"$elasticsearch_url"}
 elasticsearch_container=${elasticsearch_container-}
 
@@ -31,16 +28,16 @@ echo -e "\033[1m>>>>> Run [elastic/elasticsearch-rs container] >>>>>>>>>>>>>>>>>
 
 repo=$(realpath $(dirname $(realpath -s $0))/../)
 
+# ES_TEST_SERVER en var is needed for cargo tests
 docker run \
   --network=${network_name} \
   --env "ES_TEST_SERVER=${ELASTICSEARCH_URL}" \
-  --env "TOKEN=${TOKEN}" \
   --name test-runner \
   --volume ${repo}/test_results:/usr/src/elasticsearch-rs/test_results \
   --rm \
   elastic/elasticsearch-rs \
   /bin/bash -c \
-  "cargo run -p yaml_test_runner -- -u \"${ELASTICSEARCH_URL}\" -p \"api_generator/rest_specs\"; \\
+  "cargo run -p yaml_test_runner -- -u \"${ELASTICSEARCH_URL}\"; \\
    mkdir -p test_results; \\
    cargo test -p yaml_test_runner -- --test-threads=1 -Z unstable-options --format json | tee test_results/results.json; \\
    cat test_results/results.json | cargo2junit > test_results/junit.xml"

yaml_test_runner/Cargo.toml

@@ -31,6 +31,9 @@ syn = { version = "~0.11", features = ["full"] }
 sysinfo = "0.9.6"
 url = "2.1.1"
 yaml-rust = "0.4.3"
+tar = "~0.4"
+flate2 = "~1"
+globset = "~0.4"
 
 [dev-dependencies]
 tokio = { version = "0.2.0", default-features = false, features = ["macros", "tcp", "time"] }

yaml_test_runner/src/github.rs

@@ -16,93 +16,64 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+use flate2::read::GzDecoder;
+use globset::Glob;
 use io::Write;
-use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION};
-use serde::Deserialize;
-use std::{error::Error as StdError, fmt::Formatter, fs, fs::File, io, path::PathBuf};
-
-struct YamlTestSuite {
-    dir: String,
-    branch: String,
-    url: String,
-}
-
-#[derive(Deserialize, Debug)]
-struct Links {
-    #[serde(rename = "self")]
-    self_link: String,
-    git: String,
-    html: String,
-}
-
-#[derive(Deserialize, Debug)]
-struct GitHubContent {
-    name: String,
-    path: String,
-    sha: String,
-    size: i32,
-    url: String,
-    html_url: String,
-    git_url: String,
-    download_url: Option<String>,
-    #[serde(rename = "type")]
-    ty: String,
-    #[serde(rename = "_links")]
-    links: Links,
-}
+use reqwest::{
+    header::{HeaderMap, HeaderValue, USER_AGENT},
+    Response,
+};
+use std::{fs, fs::File, io, path::PathBuf};
+use tar::{Archive, Entry};
 
 /// Downloads the yaml tests if not already downloaded
-pub fn download_test_suites(
-    token: &str,
-    branch: &str,
-    download_dir: &PathBuf,
-) -> Result<(), failure::Error> {
+pub fn download_test_suites(branch: &str, download_dir: &PathBuf) -> Result<(), failure::Error> {
     let mut last_downloaded_version = download_dir.clone();
     last_downloaded_version.push("last_downloaded_version");
     if last_downloaded_version.exists() {
         let version = fs::read_to_string(&last_downloaded_version)
             .expect("Unable to read last_downloaded_version of yaml tests");
         if version == branch {
-            info!("yaml tests for branch {} already downloaded", branch);
+            info!("Already downloaded yaml tests from {}", branch);
             return Ok(());
         }
     }
 
-    let test_suite_map = [
-        ("oss".to_string(), "https://api.github.com/repos/elastic/elasticsearch/contents/rest-api-spec/src/main/resources/rest-api-spec/test".to_string()),
-        ("xpack".to_string(), "https://api.github.com/repos/elastic/elasticsearch/contents/x-pack/plugin/src/test/resources/rest-api-spec/test".to_string())];
-
-    let test_suites: Vec<YamlTestSuite> = test_suite_map
-        .iter()
-        .map(|(dir, template_url)| {
-            let url = format!("{}?ref={}", template_url, branch);
-            YamlTestSuite {
-                dir: dir.to_string(),
-                branch: branch.to_string(),
-                url,
-            }
-        })
-        .collect();
-
+    info!("Downloading yaml tests from {}", branch);
+    let url = format!(
+        "https://api.github.com/repos/elastic/elasticsearch/tarball/{}",
+        branch
+    );
     let mut headers = HeaderMap::new();
-    let token_value = format!("token {}", token);
-    headers.append(AUTHORIZATION, HeaderValue::from_str(&token_value)?);
+    headers.append(
+        USER_AGENT,
+        HeaderValue::from_str("elasticsearch-rs/yaml_test_runner")?,
+    );
     let client = reqwest::ClientBuilder::new()
         .default_headers(headers)
         .build()
         .unwrap();
 
-    // delete existing yaml tests
-    if download_dir.exists() {
-        fs::remove_dir_all(&download_dir)?;
-    }
-
-    fs::create_dir_all(download_dir)?;
-
-    for suite in test_suites {
-        download_tests(&client, &suite, &download_dir)?;
+    let response = client.get(&url).send()?;
+    let tar = GzDecoder::new(response);
+    let mut archive = Archive::new(tar);
+
+    let oss_test = Glob::new("**/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.yml")?
+        .compile_matcher();
+    let xpack_test = Glob::new("**/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml")?
+        .compile_matcher();
+
+    for entry in archive.entries()? {
+        let file = entry?;
+        let path = file.path()?;
+        if oss_test.is_match(&path) {
+            write_test_file(download_dir, "oss", file)?;
+        } else if xpack_test.is_match(&path) {
+            write_test_file(download_dir, "xpack", file)?;
+        }
     }
 
+    info!("Downloaded yaml tests from {}", &branch);
     File::create(last_downloaded_version)
         .expect("failed to create last_downloaded_version file")
         .write_all(branch.as_bytes())
@@ -111,115 +82,25 @@ pub fn download_test_suites(
     Ok(())
 }
 
-fn download_tests(
-    client: &reqwest::Client,
-    suite: &YamlTestSuite,
+fn write_test_file(
     download_dir: &PathBuf,
-) -> Result<(), DownloadError> {
-    let suite_dir = {
-        let mut d = download_dir.clone();
-        d.push(&suite.dir);
-        d
+    suite_dir: &str,
+    mut entry: Entry<GzDecoder<Response>>,
+) -> Result<(), failure::Error> {
+    let path = entry.path()?;
+
+    let mut dir = {
+        let mut dir = download_dir.clone();
+        dir.push(suite_dir);
+        let parent = path.parent().unwrap().file_name().unwrap();
+        dir.push(parent);
+        dir
     };
 
-    fs::create_dir_all(&suite_dir)?;
-    info!("Downloading {} tests from {}", &suite.dir, &suite.branch);
-    download(client, &suite.url, &suite_dir)?;
-    info!(
-        "Done downloading {} tests from {}",
-        &suite.dir, &suite.branch
-    );
+    fs::create_dir_all(&dir)?;
+    dir.push(path.file_name().unwrap());
+    let mut file = File::create(&dir)?;
+    io::copy(&mut entry, &mut file)?;
 
     Ok(())
 }
-
-fn download(
-    client: &reqwest::Client,
-    url: &str,
-    download_dir: &PathBuf,
-) -> Result<(), DownloadError> {
-    let mut response = client.get(url).send()?;
-
-    let remaining_rate_limit: i32 = response
-        .headers()
-        .get("X-RateLimit-Remaining")
-        .unwrap()
-        .to_str()
-        .unwrap()
-        .parse()
-        .unwrap();
-
-    if remaining_rate_limit < 10 {
-        warn!("Remaining rate limit: {}", remaining_rate_limit);
-    }
-
-    let contents: Vec<GitHubContent> = response.json()?;
-    for content in contents {
-        let content_path = {
-            let mut d = download_dir.clone();
-            d.push(&content.name);
-            d
-        };
-
-        match content.ty.as_str() {
-            "file" => {
-                let mut file = File::create(content_path)?;
-                // no need to send the token for downloading content
-                let mut file_response = reqwest::get(&content.download_url.unwrap())?;
-                io::copy(&mut file_response, &mut file)?;
-            }
-            "dir" => {
-                fs::create_dir_all(&content_path)?;
-                download(client, &content.url, &content_path)?;
-            }
-            t => {
-                return Err(DownloadError::InvalidType(format!(
-                    "Unexpected GitHub content type: {}",
-                    t
-                )))
-            }
-        }
-    }
-
-    Ok(())
-}
-
-#[derive(Debug)]
-pub enum DownloadError {
-    IoErr(io::Error),
-    HttpError(reqwest::Error),
-    InvalidType(String),
-}
-
-impl std::fmt::Display for DownloadError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
-        match self {
-            DownloadError::IoErr(err) => write!(f, "IoErr {}", err),
-            DownloadError::HttpError(err) => write!(f, "HttpError {}", err),
-            DownloadError::InvalidType(s) => write!(f, "InvalidType {}", s),
-        }
-    }
-}
-
-impl StdError for DownloadError {
-    #[allow(warnings)]
-    fn description(&self) -> &str {
-        match self {
-            DownloadError::IoErr(err) => err.description(),
-            DownloadError::HttpError(err) => err.description(),
-            DownloadError::InvalidType(s) => s.as_ref(),
-        }
-    }
-}
-
-impl From<io::Error> for DownloadError {
-    fn from(e: io::Error) -> Self {
-        DownloadError::IoErr(e)
-    }
-}
-
-impl From<reqwest::Error> for DownloadError {
-    fn from(e: reqwest::Error) -> Self {
-        DownloadError::HttpError(e)
-    }
-}