Skip to content

Commit 37b8066

Browse files
committed
Update build: download API spec artifacts, security on by default
1 parent 66b781d commit 37b8066

24 files changed

+489
-317
lines changed

.cargo/config.toml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[alias]
2+
xtask = "run -q --package xtask --"

.ci/DockerFile

+1
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,6 @@ COPY elasticsearch/Cargo.toml ./elasticsearch/Cargo.toml
2727
COPY elasticsearch/src ./elasticsearch/src
2828
COPY elasticsearch/build.rs ./elasticsearch/build.rs
2929
COPY yaml_test_runner ./yaml_test_runner
30+
COPY xtask ./xtask
3031

3132
RUN cargo build --tests

.ci/functions/imports.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@ if [[ -z $es_node_name ]]; then
2626
export es_node_name=instance
2727
export elastic_password=changeme
2828
export elasticsearch_image=elasticsearch
29-
export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
29+
export elasticsearch_scheme="https"
3030
if [[ $TEST_SUITE != "platinum" ]]; then
31-
export elasticsearch_url=http://${es_node_name}:9200
31+
export elasticsearch_scheme="http"
3232
fi
33+
export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200
3334
export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
3435
export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
3536

.ci/run-elasticsearch.sh

+22-3
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,18 @@
77
# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
88
# Export the NUMBER_OF_NODES variable to start more than 1 node
99

10-
# Version 1.1.0
10+
# Version 1.5.0
1111
# - Initial version of the run-elasticsearch.sh script
1212
# - Deleting the volume should not dependent on the container still running
1313
# - Fixed `ES_JAVA_OPTS` config
1414
# - Moved to STACK_VERSION and TEST_VERSION
1515
# - Refactored into functions and imports
1616
# - Support NUMBER_OF_NODES
17+
# - Added 5 retries on docker pull for fixing transient network errors
18+
# - Added flags to make local CCR configurations work
19+
# - Added action.destructive_requires_name=false as the default will be true in v8
20+
# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing
21+
# - Moved ELASTIC_PASSWORD to the base arguments for "Security On by default"
1722

1823
script_path=$(dirname $(realpath -s $0))
1924
source $script_path/functions/imports.sh
@@ -27,6 +32,7 @@ cluster_name=${moniker}${suffix}
2732

2833
declare -a volumes
2934
environment=($(cat <<-END
35+
--env ELASTIC_PASSWORD=$elastic_password
3036
--env node.name=$es_node_name
3137
--env cluster.name=$cluster_name
3238
--env cluster.initial_master_nodes=$master_node_name
@@ -36,6 +42,8 @@ environment=($(cat <<-END
3642
--env node.attr.testattr=test
3743
--env path.repo=/tmp
3844
--env repositories.url.allowed_urls=http://snapshot.test*
45+
--env action.destructive_requires_name=false
46+
--env ingest.geoip.downloader.enabled=false
3947
END
4048
))
4149
if [[ "$TEST_SUITE" == "platinum" ]]; then
@@ -49,6 +57,7 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
4957
--env xpack.security.http.ssl.certificate=certs/testnode.crt
5058
--env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
5159
--env xpack.security.transport.ssl.enabled=true
60+
--env xpack.security.transport.ssl.verification_mode=certificate
5261
--env xpack.security.transport.ssl.key=certs/testnode.key
5362
--env xpack.security.transport.ssl.certificate=certs/testnode.crt
5463
--env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
@@ -67,6 +76,17 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
6776
cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
6877
fi
6978

79+
# Pull the container, retry on failures up to 5 times with
80+
# short delays between each attempt. Fixes most transient network errors.
81+
docker_pull_attempts=0
82+
until [ "$docker_pull_attempts" -ge 5 ]
83+
do
84+
docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
85+
docker_pull_attempts=$((docker_pull_attempts+1))
86+
echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
87+
sleep 10
88+
done
89+
7090
NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
7191
http_port=9200
7292
for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
@@ -92,7 +112,7 @@ END
92112
docker run \
93113
--name "$node_name" \
94114
--network "$network_name" \
95-
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
115+
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \
96116
"${environment[@]}" \
97117
"${volumes[@]}" \
98118
--publish "$http_port":9200 \
@@ -112,4 +132,3 @@ END
112132
fi
113133

114134
done
115-

.gitignore

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ Cargo.lock
66
*.iml
77
.vscode/
88
*.log
9-
yaml_test_runner/yaml/
10-
yaml_test_runner/tests/oss
9+
yaml_test_runner/tests/free
1110
yaml_test_runner/tests/xpack
1211
yaml_test_runner/tests/mod.rs
1312
test_results/
13+
checkout/

Cargo.toml

+8-2
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,11 @@
22
members = [
33
"api_generator",
44
"elasticsearch",
5-
"yaml_test_runner"
6-
]
5+
"yaml_test_runner",
6+
"xtask"
7+
]
8+
9+
# See https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/
10+
# and https://github.com/rust-lang/cargo/pull/9298
11+
[profile.dev]
12+
split-debuginfo = "unpacked"

Makefile.toml

+13-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ category = "Elasticsearch"
1616
description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when free test suite used"
1717
private = true
1818
condition = { env = { "TEST_SUITE" = "free" }, env_not_set = ["ELASTICSEARCH_URL"] }
19-
env = { "ELASTICSEARCH_URL" = "http://localhost:9200" }
19+
env = { "ELASTICSEARCH_URL" = "http://elastic:changeme@localhost:9200" }
2020

2121
[tasks.set-platinum-env]
2222
category = "Elasticsearch"
@@ -25,6 +25,16 @@ private = true
2525
condition = { env = { "TEST_SUITE" = "platinum" }, env_not_set = ["ELASTICSEARCH_URL"] }
2626
env = { "ELASTICSEARCH_URL" = "https://elastic:changeme@localhost:9200" }
2727

28+
[tasks.download-specs]
29+
category = "Elasticsearch"
30+
description = '''Donwload Rest API specs and YAML tests'''
31+
private = true
32+
command = "cargo"
33+
# cargo-make insists on installing cargo-xtask and ignores .cargo/config.toml
34+
#args = ["xtask", "download-specs", "--url", "${ELASTICSEARCH_URL}"]
35+
args = ["run", "-q", "-p", "xtask", "--", "download-specs", "--url", "${ELASTICSEARCH_URL}"]
36+
dependencies = ["start-elasticsearch"]
37+
2838
[tasks.run-yaml-test-runner]
2939
category = "Elasticsearch"
3040
description = '''
@@ -34,7 +44,7 @@ The commit to use is retrieved from the running Elasticsearch instance
3444
private = true
3545
command = "cargo"
3646
args = ["run", "-p", "yaml_test_runner", "--", "-u", "${ELASTICSEARCH_URL}"]
37-
dependencies = ["start-elasticsearch"]
47+
dependencies = ["download-specs"]
3848

3949
[tasks.test-yaml-test-runner]
4050
category = "Elasticsearch"
@@ -64,6 +74,7 @@ category = "Elasticsearch"
6474
private = true
6575
command = "cargo"
6676
args = ["run", "-p", "api_generator"]
77+
dependencies = ["download-specs"]
6778

6879
[tasks.create-test-results-dir]
6980
category = "Elasticsearch"

api_generator/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ edition = "2018"
99
license = "Apache-2.0"
1010

1111
[dependencies]
12+
anyhow = "1.0"
1213
array_tool = "1.0.3"
1314
dialoguer = "0.3.0"
14-
failure = "0.1.5"
1515
flate2 = "~1"
1616
globset = "~0.4"
1717
Inflector = "0.11.4"

api_generator/src/bin/run.rs

+23-87
Original file line numberDiff line numberDiff line change
@@ -19,107 +19,43 @@
1919
extern crate api_generator;
2020
extern crate dialoguer;
2121

22-
use api_generator::{generator, rest_spec};
23-
use dialoguer::Input;
24-
use std::{
25-
fs::{self, File},
26-
io::Write,
27-
path::PathBuf,
28-
};
22+
use anyhow::{bail, Context};
23+
use api_generator::generator;
24+
use std::{fs, path::PathBuf};
2925

30-
fn main() -> Result<(), failure::Error> {
26+
fn main() -> anyhow::Result<()> {
3127
simple_logger::SimpleLogger::new()
3228
.with_level(log::LevelFilter::Info)
3329
.init()
3430
.unwrap();
3531

36-
// This must be run from the repo root directory, with cargo make generate-api
37-
let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
38-
let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;
39-
let last_downloaded_version =
40-
PathBuf::from("./api_generator/rest_specs/last_downloaded_version");
41-
42-
let mut download_specs = false;
43-
let mut answer = String::new();
44-
let default_branch = if last_downloaded_version.exists() {
45-
fs::read_to_string(&last_downloaded_version)?
46-
} else {
47-
String::from("master")
48-
};
49-
let mut branch = default_branch.clone();
32+
let stack_version = std::env::var("STACK_VERSION").context("Missing STACK_VERSION env var")?;
5033

51-
while answer != "y" && answer != "n" {
52-
answer = Input::new()
53-
.default(String::from("n"))
54-
.show_default(false)
55-
.with_prompt("Download rest specifications [y/N]")
56-
.interact()
57-
.unwrap()
58-
.to_lowercase();
59-
download_specs = answer == "y";
60-
}
61-
62-
if download_specs {
63-
branch = Input::new()
64-
.default(default_branch.clone())
65-
.show_default(false)
66-
.with_prompt(
67-
format!(
68-
"Branch to download specification from [default {}]",
69-
default_branch
70-
)
71-
.as_str(),
72-
)
73-
.interact()
74-
.unwrap();
34+
// This must be run from the repo root directory, with cargo make generate-api
35+
let download_dir = PathBuf::from(&format!("./checkout/{}/rest-api-spec/api", stack_version));
7536

76-
fs::remove_dir_all(&download_dir)?;
77-
fs::create_dir_all(&download_dir)?;
78-
rest_spec::download_specs(&branch, &download_dir)?;
79-
File::create(&last_downloaded_version)?.write_all(branch.as_bytes())?;
37+
if !download_dir.is_dir() {
38+
bail!("No specs found at {:?}", download_dir);
8039
}
8140

82-
// only offer to generate if there are downloaded specs
83-
if download_dir
84-
.read_dir()
85-
.map(|mut r| r.next().is_some())
86-
.unwrap_or(false)
87-
{
88-
let mut generate_code = true;
89-
answer = String::new();
90-
while answer != "y" && answer != "n" {
91-
answer = Input::new()
92-
.default(String::from("y"))
93-
.show_default(false)
94-
.with_prompt(
95-
format!("Generate code from rest specifications {} [Y/n]", branch).as_str(),
96-
)
97-
.interact()
98-
.unwrap()
99-
.to_lowercase();
100-
generate_code = answer == "y";
101-
}
102-
103-
if generate_code {
104-
// Delete previously generated files
105-
let mut generated = generated_dir.clone();
106-
generated.push(generator::GENERATED_TOML);
107-
108-
if generated.exists() {
109-
let files =
110-
toml::from_str::<generator::GeneratedFiles>(&fs::read_to_string(generated)?)?;
41+
// let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
42+
let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;
11143

112-
for f in files.written {
113-
let mut generated_file = generated_dir.clone();
114-
generated_file.push(f);
115-
let _ = fs::remove_file(generated_file); // ignore missing files
116-
}
117-
}
44+
// Delete previously generated files
45+
let mut generated = generated_dir.clone();
46+
generated.push(generator::GENERATED_TOML);
47+
if generated.exists() {
48+
let files = toml::from_str::<generator::GeneratedFiles>(&fs::read_to_string(generated)?)?;
11849

119-
// and generate!
120-
generator::generate(&branch, &download_dir, &generated_dir)?;
50+
for f in files.written {
51+
let mut generated_file = generated_dir.clone();
52+
generated_file.push(f);
53+
let _ = fs::remove_file(generated_file); // ignore missing files
12154
}
12255
}
12356

57+
// and generate!
58+
generator::generate(&download_dir, &generated_dir)?;
59+
12460
Ok(())
12561
}

api_generator/src/generator/code_gen/namespace_clients.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use quote::Tokens;
2525
use std::path::PathBuf;
2626

2727
/// Generates the source code for a namespaced client
28-
pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<Vec<(String, String)>, failure::Error> {
28+
pub fn generate(api: &Api, docs_dir: &PathBuf) -> anyhow::Result<Vec<(String, String)>> {
2929
let mut output = Vec::new();
3030

3131
for (namespace_name, namespace) in &api.namespaces {

api_generator/src/generator/code_gen/params.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use inflector::Inflector;
2222
use quote::Tokens;
2323
use regex::Regex;
2424

25-
pub fn generate(api: &Api) -> Result<String, failure::Error> {
25+
pub fn generate(api: &Api) -> anyhow::Result<String> {
2626
let mut tokens = quote!(
2727
use serde::{Serialize, Deserialize};
2828
);

api_generator/src/generator/code_gen/root.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use quote::Tokens;
2525
use std::path::PathBuf;
2626

2727
/// Generates the source code for the methods on the root of Elasticsearch
28-
pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<String, failure::Error> {
28+
pub fn generate(api: &Api, docs_dir: &PathBuf) -> anyhow::Result<String> {
2929
let mut tokens = Tokens::new();
3030
tokens.append(use_declarations());
3131

0 commit comments

Comments
 (0)