Emit cargo test json output

This commit uses cargo test flags to emit test results in
JSON format, then use cargo2junit tooling to convert
JSON output into junit xml file. The cargo test flags are
unstable and work only on nightly, but api_generator and
yaml_test_runner require nightly channel because rustfmt-nightly
is used.

Determine branch/commit, test suite and version
from running Elasticsearch instance.

Author: russcam

.ci/DockerFile

@@ -36,6 +36,7 @@ RUN set -eux; \
     rustup --version; \
     cargo --version; \
     rustc --version; \
+    cargo install cargo2junit; \
     \
     apt-get remove -y --auto-remove \
         wget \

.ci/run-repository.sh

@@ -13,7 +13,6 @@ source $script_path/functions/imports.sh
 set -euo pipefail
 
 RUST_TOOLCHAIN=${RUST_TOOLCHAIN-nightly-2020-06-09}
-BRANCH=${BRANCH-master}
 TOKEN=${TOKEN-}
 ELASTICSEARCH_URL=${ELASTICSEARCH_URL-"$elasticsearch_url"}
 elasticsearch_container=${elasticsearch_container-}
@@ -22,7 +21,6 @@ echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m CONTAINER ${elasticsearch_container}\033[0m"
-echo -e "\033[34;1mINFO:\033[0m BRANCH ${BRANCH}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m RUST_TOOLCHAIN ${RUST_TOOLCHAIN}\033[0m"
 
 echo -e "\033[1m>>>>> Build [elastic/elasticsearch-rs container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
@@ -36,12 +34,14 @@ repo=$(realpath $(dirname $(realpath -s $0))/../)
 docker run \
   --network=${network_name} \
   --env "ES_TEST_SERVER=${ELASTICSEARCH_URL}" \
-  --env "ELASTICSEARCH_VERSION=${elasticsearch_container}" \
   --env "TOKEN=${TOKEN}" \
   --name test-runner \
   --volume ${repo}/test_results:/usr/src/elasticsearch-rs/test_results \
   --rm \
   elastic/elasticsearch-rs \
   /bin/bash -c \
-  "cargo run -p yaml_test_runner -- --branch \"${BRANCH}\" --path \"/usr/src/elasticsearch-rs/api_generator/rest_specs\"; cargo test -p yaml_test_runner -- --test-threads=1"
+  "cargo run -p yaml_test_runner -- -u \"${ELASTICSEARCH_URL}\" -p \"api_generator/rest_specs\"; \\
+   mkdir -p test_results; \\
+   cargo test -p yaml_test_runner -- --test-threads=1 -Z unstable-options --format json | tee test_results/results.json; \\
+   cat test_results/results.json | cargo2junit > test_results/junit.xml"
 

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto
+
+*.sh text eol=lf

.gitignore

@@ -9,3 +9,4 @@ yaml_test_runner/yaml/
 yaml_test_runner/tests/oss
 yaml_test_runner/tests/xpack
 yaml_test_runner/tests/mod.rs
+test_results/

yaml_test_runner/src/main.rs

@@ -28,6 +28,7 @@ extern crate simple_logger;
 
 use clap::{App, Arg};
 use log::Level;
+use serde_json::Value;
 use std::{fs, path::PathBuf, process::exit};
 
 mod generator;
@@ -42,21 +43,20 @@ fn main() -> Result<(), failure::Error> {
 
     let matches = App::new(env!("CARGO_PKG_NAME"))
         .about(env!("CARGO_PKG_DESCRIPTION"))
-        .arg(Arg::with_name("branch")
-            .short("b")
-            .long("branch")
-            .value_name("BRANCH")
-            .help("The git branch in the Elasticsearch repository from which to download yaml tests")
-            .required(true)
-            .default_value("master")
-            .takes_value(true))
         .arg(Arg::with_name("token")
             .short("t")
             .long("token")
             .value_name("TOKEN")
             .help("The GitHub access token. Increases the rate limit to be able to download all yaml tests. Must be specified if not passed through environment variable")
             .required(false)
             .takes_value(true))
+        .arg(Arg::with_name("url")
+            .short("u")
+            .long("url")
+            .value_name("ELASTICSEARCH_URL")
+            .help("The url of a running Elasticsearch cluster. Used to determine the version, test suite and branch to use to compile tests")
+            .required(true)
+            .takes_value(true))
         .arg(Arg::with_name("path")
             .short("p")
             .long("path")
@@ -66,39 +66,8 @@ fn main() -> Result<(), failure::Error> {
             .takes_value(true))
         .get_matches();
 
-    // Get the version from ELASTICSEARCH_VERSION environment variable, if set.
-    // any prerelease part needs to be trimmed because the semver crate only allows
-    // a version with a prerelease to match against predicates, if at least one predicate
-    // has a prerelease. See
-    // https://github.com/steveklabnik/semver/blob/afa5fc853cb4d6d2b1329579e5528f86f3b550f9/src/version_req.rs#L319-L331
-    let (suite, version) = match std::env::var("ELASTICSEARCH_VERSION") {
-        Ok(v) => {
-            let suite = if v.contains("oss") {
-                TestSuite::Oss
-            } else {
-                TestSuite::XPack
-            };
-
-            let v = v
-                .split(':')
-                .next_back()
-                .unwrap()
-                .trim_end_matches(|c: char| c.is_alphabetic() || c == '-');
-
-            (suite, semver::Version::parse(v)?)
-        }
-        Err(_) => {
-            error!("ELASTICSEARCH_VERSION environment variable must be set to compile tests");
-            exit(1);
-        }
-    };
-
-    info!("Using version {:?} to compile tests", &version);
-
-    let branch = matches
-        .value_of("branch")
-        .expect("missing 'branch' argument");
-
+    let url = matches.value_of("url").expect("missing 'url' argument");
+    let path = matches.value_of("path").expect("missing 'path' argument");
     let token = match std::env::var("TOKEN") {
         Ok(v) => v,
         Err(_) => match matches.value_of("token") {
@@ -109,12 +78,27 @@ fn main() -> Result<(), failure::Error> {
             }
         },
     };
-    let path = matches.value_of("path").expect("missing 'path' argument");
+
+    let (branch, suite, version) = match branch_suite_and_version_from_elasticsearch(url) {
+        Ok(v) => v,
+        Err(e) => {
+            error!(
+                "Problem getting values from Elasticsearch at {}. {:?}",
+                url, e
+            );
+            exit(1);
+        }
+    };
+
+    info!("Using version {}", &version.to_string());
+    info!("Using branch {}", &branch);
+    info!("Using test_suite {:?}", &suite);
+
     let rest_specs_dir = PathBuf::from(path);
     let download_dir = PathBuf::from(format!("./{}/yaml", env!("CARGO_PKG_NAME")));
     let generated_dir = PathBuf::from(format!("./{}/tests", env!("CARGO_PKG_NAME")));
 
-    github::download_test_suites(&token, branch, &download_dir)?;
+    github::download_test_suites(&token, &branch, &download_dir)?;
 
     let mut last_downloaded_rest_spec_branch = rest_specs_dir.clone();
     last_downloaded_rest_spec_branch.push("last_downloaded_version");
@@ -134,10 +118,10 @@ fn main() -> Result<(), failure::Error> {
     }
 
     if download_rest_specs {
-        api_generator::rest_spec::download_specs(branch, &rest_specs_dir)?;
+        api_generator::rest_spec::download_specs(&branch, &rest_specs_dir)?;
     }
 
-    let api = api_generator::generator::read_api(branch, &rest_specs_dir)?;
+    let api = api_generator::generator::read_api(&branch, &rest_specs_dir)?;
 
     // delete everything under the generated_dir except common dir
     if generated_dir.exists() {
@@ -168,3 +152,26 @@ fn main() -> Result<(), failure::Error> {
 
     Ok(())
 }
+
+fn branch_suite_and_version_from_elasticsearch(
+    url: &str,
+) -> Result<(String, TestSuite, semver::Version), failure::Error> {
+    let mut response = reqwest::get(url)?;
+    let json: Value = response.json()?;
+    let branch = json["version"]["build_hash"].as_str().unwrap().to_string();
+    let suite = match json["version"]["build_flavor"].as_str().unwrap() {
+        "oss" => TestSuite::Oss,
+        _ => TestSuite::XPack,
+    };
+
+    // any prerelease part needs to be trimmed because the semver crate only allows
+    // a version with a prerelease to match against predicates, if at least one predicate
+    // has a prerelease. See
+    // https://github.com/steveklabnik/semver/blob/afa5fc853cb4d6d2b1329579e5528f86f3b550f9/src/version_req.rs#L319-L331
+    let version = json["version"]["number"]
+        .as_str()
+        .unwrap()
+        .trim_end_matches(|c: char| c.is_alphabetic() || c == '-');
+
+    Ok((branch, suite, semver::Version::parse(version)?))
+}