Skip to content

Commit b49d8b4

Browse files
JoshMockJoshMock
authored
Merge branch 'main' into 410-gone
2 parents b83163b + 24a4583 commit b49d8b4

13 files changed

+521
-2
lines changed

‎.buildkite/Dockerfile

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
ARG PYTHON_VERSION=3.8
2+
FROM python:${PYTHON_VERSION}
3+
4+
# Default UID/GID to 1000
5+
# it can be overridden at build time
6+
ARG BUILDER_UID=1000
7+
ARG BUILDER_GID=1000
8+
ENV BUILDER_USER elastic
9+
ENV BUILDER_GROUP elastic
10+
ENV PATH="${PATH}:/var/lib/elastic/.local/bin"
11+
12+
# Create user
13+
RUN groupadd --system -g ${BUILDER_GID} ${BUILDER_GROUP} \
14+
&& useradd --system --shell /bin/bash -u ${BUILDER_UID} -g ${BUILDER_GROUP} -d /var/lib/elastic -m elastic 1>/dev/null 2>/dev/null \
15+
&& mkdir -p /code/elasticsearch-py && mkdir /code/elasticsearch-py/build \
16+
&& chown -R ${BUILDER_USER}:${BUILDER_GROUP} /code/
17+
WORKDIR /code/elasticsearch-py
18+
USER ${BUILDER_USER}:${BUILDER_GROUP}
19+
RUN python -m pip install --disable-pip-version-check nox
20+
COPY --chown=$BUILDER_USER:$BUILDER_GROUP . .

‎.buildkite/certs/ca.crt

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL
3+
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
4+
cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD
5+
VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
6+
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/
7+
40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y
8+
RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq
9+
Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n
10+
gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT
11+
8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C
12+
AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY
13+
MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
14+
hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX
15+
dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb
16+
GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0
17+
hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj
18+
cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx
19+
EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI=
20+
-----END CERTIFICATE-----

‎.buildkite/certs/ca.pem

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL
3+
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
4+
cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD
5+
VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
6+
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/
7+
40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y
8+
RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq
9+
Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n
10+
gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT
11+
8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C
12+
AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY
13+
MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
14+
hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX
15+
dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb
16+
GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0
17+
hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj
18+
cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx
19+
EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI=
20+
-----END CERTIFICATE-----

‎.buildkite/certs/testnode.crt

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDODCCAiCgAwIBAgIVAKLWEcNzTd4B0NqnrJL0xAKaS8DWMA0GCSqGSIb3DQEB
3+
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
4+
ZXJhdGVkIENBMB4XDTIzMDgyMTE3MjcwMloXDTI2MDgyMDE3MjcwMlowEzERMA8G
5+
A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8
6+
eLXL3ZX5v8JlHcfg+96Bpq24EeiqV+7RPPKbcH80ODjkETqYUpam+TcOl2gt23p/
7+
rpiPSSpOX8pFdmY78wTmxo2GCQZ/db2h0gZOOYpb8HQku+hJ4bAmtzizrqWW76Wz
8+
csen3DSUkT0bKkJTjUMmwVhRaMpfv8EIcUbrHAwc3VCj7grnFL0kdAuQa6iyBH4I
9+
lTUYOIOVyEJ8zZ7R4BJO3QU+TRuJ5+w/QiZMeDqxtrdDL37vYQHPW7L/XISCCOMp
10+
sA3avzFphoQXBQ8mjdB8Txkd4sH7mJTqnRp5ILhRzVpcPPgQYFeIB567B+kFeSau
11+
aJJmc0EVgOcK5aSMtOH3AgMBAAGjYjBgMB0GA1UdDgQWBBQsZbZDudZ63h52FlU5
12+
N2g3pznkETAfBgNVHSMEGDAWgBR1YUax123eUBRIJN0RZ+GOC2KPTzATBgNVHREE
13+
DDAKgghpbnN0YW5jZTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAyv0Cw
14+
OrvZn7FHHS8TJI5vTi1F43R/eSNMNL/+q/nK93KaxWJH1T4zrJhrJ9KpzkFcalXP
15+
bu02oTh28b3o3QpS2wdwMv/Q3NLoMBEmQlG2UrELFvV43nS8LCiwCX3o11L1HZP3
16+
1Z/rclwxbA4OQ/ZkPcol++TDZQTM/8WkIdZmTL4UDb/ppDjX24nTOitkMRZlYAOY
17+
mid9GGExhKrUJ0I9/A3w1hWRA1Hwc+1TFDcPphl2x2uQ9HJFBueAvuFXmIjDki1x
18+
qrvnFZ+mneI9kR4m82MX900WF15KS35GzmMui0tsf0wbfy3Jh+WnpMlIIa2OQXw7
19+
prbkg9tScQSsvhC8
20+
-----END CERTIFICATE-----

‎.buildkite/certs/testnode.key

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEowIBAAKCAQEAvHi1y92V+b/CZR3H4PvegaatuBHoqlfu0Tzym3B/NDg45BE6
3+
mFKWpvk3DpdoLdt6f66Yj0kqTl/KRXZmO/ME5saNhgkGf3W9odIGTjmKW/B0JLvo
4+
SeGwJrc4s66llu+ls3LHp9w0lJE9GypCU41DJsFYUWjKX7/BCHFG6xwMHN1Qo+4K
5+
5xS9JHQLkGuosgR+CJU1GDiDlchCfM2e0eASTt0FPk0biefsP0ImTHg6sba3Qy9+
6+
72EBz1uy/1yEggjjKbAN2r8xaYaEFwUPJo3QfE8ZHeLB+5iU6p0aeSC4Uc1aXDz4
7+
EGBXiAeeuwfpBXkmrmiSZnNBFYDnCuWkjLTh9wIDAQABAoIBAAU0iEDTI9s78pB8
8+
XBLYofKOuemFhRl/SDc7KbAlUT4N93RFDYs7bLG73Eto3xW1JBL2rXv3l1WGy71T
9+
YctyEMaW4T28bhODGvOnK0lpyWp0n6CMGARCWW0YTlaYEjay866bEuyN5l3cDQX9
10+
Csvn8NzXJitJa51tXFVxW3YO1j7Nyc/M59oyBZ1ARYYmQqFYLEu6lvJOW0cKDFkZ
11+
AcMVlOIxZQL/Mf+RO72aQGVuYNjqxlLIXLuE9zFR2gDFM2+l3FMUWDGHGBDFyjKU
12+
iMk4+sSlOTFXqO9VQzua6FLFMsQT6m5PFD4uPY92KR6CPfH/NrWqwqr+jpjaU+gs
13+
3U9GN+ECgYEA58qX7tKPk7CWdk3kyk5NsNcs/qib+heXWEubfhoU8LmSnbBQhOAz
14+
wi//r/xm0OHGj84y66+G3T347iudrLjhr07oGM1QfjYT3kb90efLjwAfCECtyVYL
15+
EQrWO5UeoTnmrhlB1mGL3sWaVAsVqNLz8i2H5c7sj0hxHsvM62159r8CgYEA0Cff
16+
opJqmUpMpHm3sgjMWctylVrHBuQe5cl5Ad80pbd6mvtt4TvGXbUGNdzURfyve9DS
17+
x1CVlj4Sz8VuelFQgYL+7/qUqZoms1aSgJpxWv8ou+wUHmlF3kVO8VKt3BNHV+8J
18+
euSB6NG91BGguBoHgnOoVcjbDGdhJGRTojCNWskCgYEA1jE3nwDCnrbTA3XNk0ky
19+
r9TXhmgm4r+EIpqTkL7nVOAXZVJ1xaQtECgsveKe3C2WwHLKSVMFbFMFQonZha+/
20+
FbHz9l9cH5U3XPL7QEpTp8xz4LtsHJ4/UbtS5vJQwKnxyjYaydGQYAb4KuunUz/F
21+
H6kFaM6DeZB2v/+SWIfs6Z8CgYARUdAEyeP+vzTjVpFXSe4e5pOxI619wEtl2T6t
22+
TjImO78C2DrLS9r0fxR2NNqgvCapybVQCj94EdAk44uOt+dX71thAluORRpFP8XO
23+
14rpBGQSRtFhumaq2N95quR2dFAyW9xREmRQx+rgk1rpFplbXF48TQsU3CE0Evj2
24+
fM22KQKBgDhob7M9sWvXecxoyy3J17jUTcFqmqKcqGnx3ZJ7Q9CgAfjYqNNQws27
25+
wTuaJB0PEuCOu4t+lUHEBMIjGkBfo1bHd4EZaW04Xgbfn2j8MK2e+9GlRtedxxFg
26+
c1JdRb5+eTgPwLcDsmMWIW357PDW7RDEI07G1ZB4SqxGTKkU7JOW
27+
-----END RSA PRIVATE KEY-----

‎.buildkite/functions/cleanup.sh

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Shared cleanup routines between different steps
4+
#
5+
# Please source .ci/functions/imports.sh as a whole not just this file
6+
#
7+
# Version 1.0.0
8+
# - Initial version after refactor
9+
10+
function cleanup_volume {
11+
if [[ "$(docker volume ls -q -f name=$1)" ]]; then
12+
echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
13+
(docker volume rm "$1") || true
14+
fi
15+
}
16+
function container_running {
17+
if [[ "$(docker ps -q -f name=$1)" ]]; then
18+
return 0;
19+
else return 1;
20+
fi
21+
}
22+
function cleanup_node {
23+
if container_running "$1"; then
24+
echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
25+
(docker container rm --force --volumes "$1") || true
26+
fi
27+
if [[ -n "$1" ]]; then
28+
echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
29+
cleanup_volume "$1-${suffix}-data"
30+
fi
31+
}
32+
function cleanup_network {
33+
if [[ "$(docker network ls -q -f name=$1)" ]]; then
34+
echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
35+
(docker network rm "$1") || true
36+
fi
37+
}
38+
39+
function cleanup_trap {
40+
status=$?
41+
set +x
42+
if [[ "$DETACH" != "true" ]]; then
43+
echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
44+
cleanup_all_in_network "$1"
45+
fi
46+
# status is 0 or SIGINT
47+
if [[ "$status" == "0" || "$status" == "130" ]]; then
48+
echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
49+
exit 0
50+
else
51+
echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
52+
exit ${status}
53+
fi
54+
};
55+
function cleanup_all_in_network {
56+
57+
if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
58+
echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
59+
return 0
60+
fi
61+
containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
62+
while read -r container; do
63+
cleanup_node "$container"
64+
done <<< "$containers"
65+
cleanup_network $1
66+
echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
67+
};

‎.buildkite/functions/imports.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Sets up all the common variables and imports relevant functions
4+
#
5+
# Version 1.0.1
6+
# - Initial version after refactor
7+
# - Validate STACK_VERSION asap
8+
9+
function require_stack_version() {
10+
if [[ -z $STACK_VERSION ]]; then
11+
echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
12+
exit 1
13+
fi
14+
}
15+
16+
require_stack_version
17+
18+
if [[ -z $es_node_name ]]; then
19+
# only set these once
20+
set -euo pipefail
21+
export TEST_SUITE=${TEST_SUITE-platinum}
22+
export RUNSCRIPTS=${RUNSCRIPTS-}
23+
export DETACH=${DETACH-false}
24+
export CLEANUP=${CLEANUP-false}
25+
26+
export es_node_name=instance
27+
export elastic_password=changeme
28+
export elasticsearch_image=elasticsearch
29+
export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
30+
if [[ $TEST_SUITE != "platinum" ]]; then
31+
export elasticsearch_url=http://${es_node_name}:9200
32+
fi
33+
export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
34+
export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
35+
36+
export suffix=rest-test
37+
export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
38+
export network_name=${moniker}${suffix}
39+
40+
export ssl_cert="${script_path}/certs/testnode.crt"
41+
export ssl_key="${script_path}/certs/testnode.key"
42+
export ssl_ca="${script_path}/certs/ca.crt"
43+
44+
fi
45+
46+
export script_path=$(dirname $(realpath -s $0))
47+
source $script_path/functions/cleanup.sh
48+
source $script_path/functions/wait-for-container.sh
49+
trap "cleanup_trap ${network_name}" EXIT
50+
51+
52+
if [[ "$CLEANUP" == "true" ]]; then
53+
cleanup_all_in_network $network_name
54+
exit 0
55+
fi
56+
57+
echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
58+
docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
59+

‎.buildkite/functions/wait-for-container.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Exposes a routine scripts can call to wait for a container if that container set up a health command
4+
#
5+
# Please source .ci/functions/imports.sh as a whole not just this file
6+
#
7+
# Version 1.0.1
8+
# - Initial version after refactor
9+
# - Make sure wait_for_contiainer is silent
10+
11+
function wait_for_container {
12+
set +x
13+
until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
14+
echo ""
15+
docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
16+
echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
17+
sleep 2;
18+
done;
19+
20+
# Always show logs if the container is running, this is very useful both on CI as well as while developing
21+
if container_running $1; then
22+
docker logs $1
23+
fi
24+
25+
if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
26+
cleanup_all_in_network $2
27+
echo
28+
echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
29+
echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
30+
return 1
31+
else
32+
echo
33+
echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
34+
return 0
35+
fi
36+
}

‎.buildkite/pipeline.yml

+20-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,21 @@
11
steps:
2-
- label: ":python: Greetings"
3-
command: "echo 'Hello, world!'"
2+
- label: ":elasticsearch: :python: ES Python ({{ matrix.python }})"
3+
agents:
4+
provider: "gcp"
5+
env:
6+
PYTHON_VERSION: "{{ matrix.python }}"
7+
TEST_SUITE: "platinum"
8+
STACK_VERSION: 8.10.0-SNAPSHOT
9+
PYTHON_CONNECTION_CLASS: "{{ matrix.connection }}"
10+
matrix:
11+
setup:
12+
python:
13+
- "3.7"
14+
- "3.8"
15+
- "3.9"
16+
- "3.10"
17+
- "3.11"
18+
connection:
19+
- "urllib3"
20+
- "requests"
21+
command: ./.buildkite/run-tests

0 commit comments

Comments
 (0)