Merge branch 'main' into 410-gone

Author: pquentin

Diff for: .buildkite/Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.13
 FROM python:${PYTHON_VERSION}
 
 # Default UID/GID to 1000

Diff for: .buildkite/certs/README.md

@@ -0,0 +1,26 @@
+# CI certificates
+
+This directory contains certificates that can be used to test against Elasticsearch in CI
+
+## Generating new certificates using the Certificate Authority cert and key
+
+Before adding support for Python 3.13, we generated certificates with
+[`elasticsearch-certutil`](https://www.elastic.co/guide/en/elasticsearch/reference/current/certutil.html).
+However, those certificates are not compliant with RFC 5280, and Python now
+enforces compliance by enabling the VERIFY_X509_STRICT flag by default.
+
+If you need to generate new certificates, you can do so with
+[trustme](https://trustme.readthedocs.io/en/latest/) as follows:
+
+```
+```bash
+pip install trustme
+python -m trustme --identities instance
+# Use the filenames expected by our tests
+mv client.pem ca.crt
+mv server.pem testnode.crt
+mv server.key testnode.key
+```
+
+For more control over the generated certificates, trustme also offers a Python
+API, but we have not needed it so far.

Diff for: .buildkite/certs/ca.crt

@@ -1,20 +1,12 @@
 -----BEGIN CERTIFICATE-----
-MIIDSTCCAjGgAwIBAgIUHTeTPPuZIX3wdyudMsllXa9yZ1kwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMjMwODIxMTcyNTMyWhcNMjYwODIwMTcyNTMyWjA0MTIwMAYD
-VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMZs7DXbV7ovFvQ/CiqvHHZ/
-40rLyAcBQMhVBke2VVCQk3hIOPpHYt3xZgb61Oyrf14lFxny483beXaUqGThZ67Y
-RsxzSOS8NUi21OLZ3xaE+p+Yx9Xe6lTMQJM4RpD/A5V35uikji1K4+F0ooJghELq
-Fndmark/7SQFh6Bg8/aaf6Hpyar3WOWdQjHXgszNAv1Ez7+pPlfnCS8XNjYB5Y2n
-gAayb1krMRW/3E6hRVZAig3I2H8mezL5tF8iS5aJW1WLpw4oYnbH0DdS+gpCK1lT
-8GZd8Dk0QbNGpXNTu67BravVhgEoprBVMz6G1C4MiuVcBy7gA671/f46S4Tgb10C
-AwEAAaNTMFEwHQYDVR0OBBYEFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMB8GA1UdIwQY
-MBaAFHVhRrHXbd5QFEgk3RFn4Y4LYo9PMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
-hvcNAQELBQADggEBACoGVPqeYE3IPRdSAtihIq071BfGA8vgfJWv0qiI0T+gYetX
-dnebmQc5EccdEIrxD0bLKPgzd5c3ILwQy5+uo03ua7PrplwPVdeNXnU1LodAQ0Zb
-GmTixXqgj8AMcvRsA7qARjXvf6w3Yyb7GO3FXRIGtqk12Vb1qnJg894CSIWrHiw0
-hRO5b7eJyrOy2s6QA6FucM/scM1Z/8D9tHfgwmrKM875VGerJORwfHCaCvF1YvBj
-cIpYNnw2vFzDvRevh63sSQbZ9q3nbtD27AZSN9LKEbipSEOoBZMKG2zgDTT/Olzx
-EQJ2t+Z487UuFX6+WaLZMteL2F4eh9OFWIYM3EI=
+MIIByTCCAW+gAwIBAgIUIYClYWXiTsB8aMrEEMrzdrk5rOswCgYIKoZIzj0EAwIw
+QDEXMBUGA1UECgwOdHJ1c3RtZSB2MS4yLjAxJTAjBgNVBAsMHFRlc3RpbmcgQ0Eg
+I2JpdzFXYzEwbHBxQ0ZRTDUwIBcNMDAwMTAxMDAwMDAwWhgPMzAwMDAxMDEwMDAw
+MDBaMEAxFzAVBgNVBAoMDnRydXN0bWUgdjEuMi4wMSUwIwYDVQQLDBxUZXN0aW5n
+IENBICNiaXcxV2MxMGxwcUNGUUw1MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE
+SN7++A76LmOR0tKKra1M6VVzGUljjL9fVPxOEIblOOJJhA7mKLQguNzEHjucNV23
+LcDzMX/M/oUBGdYZBbAv4qNFMEMwHQYDVR0OBBYEFCrGGcO9v0UAWSsD93P/x2MT
+NiJbMBIGA1UdEwEB/wQIMAYBAf8CAQkwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49
+BAMCA0gAMEUCIQDGyO21zIAwmARtoc2atVmmqZdPVkegHkCKCFY4P+KeEAIgKMCz
+aU8LPCVyA+ZF9K+tcqkNK5h/5s7wlQ5DSeKSuE8=
 -----END CERTIFICATE-----

Diff for: .buildkite/certs/ca.pem

@@ -1,20 +1,14 @@
 -----BEGIN CERTIFICATE-----
-MIIDODCCAiCgAwIBAgIVAKLWEcNzTd4B0NqnrJL0xAKaS8DWMA0GCSqGSIb3DQEB
-CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
-ZXJhdGVkIENBMB4XDTIzMDgyMTE3MjcwMloXDTI2MDgyMDE3MjcwMlowEzERMA8G
-A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC8
-eLXL3ZX5v8JlHcfg+96Bpq24EeiqV+7RPPKbcH80ODjkETqYUpam+TcOl2gt23p/
-rpiPSSpOX8pFdmY78wTmxo2GCQZ/db2h0gZOOYpb8HQku+hJ4bAmtzizrqWW76Wz
-csen3DSUkT0bKkJTjUMmwVhRaMpfv8EIcUbrHAwc3VCj7grnFL0kdAuQa6iyBH4I
-lTUYOIOVyEJ8zZ7R4BJO3QU+TRuJ5+w/QiZMeDqxtrdDL37vYQHPW7L/XISCCOMp
-sA3avzFphoQXBQ8mjdB8Txkd4sH7mJTqnRp5ILhRzVpcPPgQYFeIB567B+kFeSau
-aJJmc0EVgOcK5aSMtOH3AgMBAAGjYjBgMB0GA1UdDgQWBBQsZbZDudZ63h52FlU5
-N2g3pznkETAfBgNVHSMEGDAWgBR1YUax123eUBRIJN0RZ+GOC2KPTzATBgNVHREE
-DDAKgghpbnN0YW5jZTAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAyv0Cw
-OrvZn7FHHS8TJI5vTi1F43R/eSNMNL/+q/nK93KaxWJH1T4zrJhrJ9KpzkFcalXP
-bu02oTh28b3o3QpS2wdwMv/Q3NLoMBEmQlG2UrELFvV43nS8LCiwCX3o11L1HZP3
-1Z/rclwxbA4OQ/ZkPcol++TDZQTM/8WkIdZmTL4UDb/ppDjX24nTOitkMRZlYAOY
-mid9GGExhKrUJ0I9/A3w1hWRA1Hwc+1TFDcPphl2x2uQ9HJFBueAvuFXmIjDki1x
-qrvnFZ+mneI9kR4m82MX900WF15KS35GzmMui0tsf0wbfy3Jh+WnpMlIIa2OQXw7
-prbkg9tScQSsvhC8
+MIICKzCCAdKgAwIBAgIUZeLIKR7XTP5Gx/moiuzcWcfHaSswCgYIKoZIzj0EAwIw
+QDEXMBUGA1UECgwOdHJ1c3RtZSB2MS4yLjAxJTAjBgNVBAsMHFRlc3RpbmcgQ0Eg
+I2JpdzFXYzEwbHBxQ0ZRTDUwIBcNMDAwMTAxMDAwMDAwWhgPMzAwMDAxMDEwMDAw
+MDBaMEIxFzAVBgNVBAoMDnRydXN0bWUgdjEuMi4wMScwJQYDVQQLDB5UZXN0aW5n
+IGNlcnQgIzNPWkpxTWh0WmxrNGlDMm0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+AASp6UadRZ0ZP3F2KeEkIUOf0B8GOTX55B91RO/PLUQb26wZcWmHGPOJ0HAy9F2E
+Y+rJ1zDUnfB5msowei/iuoaMo4GlMIGiMB0GA1UdDgQWBBSP5z3h8b13ul407YOd
+kyjKNcf/vTAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFCrGGcO9v0UAWSsD93P/
+x2MTNiJbMBYGA1UdEQEB/wQMMAqCCGluc3RhbmNlMA4GA1UdDwEB/wQEAwIFoDAq
+BgNVHSUBAf8EIDAeBggrBgEFBQcDAgYIKwYBBQUHAwEGCCsGAQUFBwMDMAoGCCqG
+SM49BAMCA0cAMEQCIHPP7chQolK+N+GZ+rJ49euoTSzb2YIU5vnCY/bFEWO+AiBC
+OTFYhR9Mw/e+WdJVZO78XZYKy5uA28JwsZuu7E0kZA==
 -----END CERTIFICATE-----

Diff for: .buildkite/certs/testnode.crt

@@ -1,27 +1,5 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEAvHi1y92V+b/CZR3H4PvegaatuBHoqlfu0Tzym3B/NDg45BE6
-mFKWpvk3DpdoLdt6f66Yj0kqTl/KRXZmO/ME5saNhgkGf3W9odIGTjmKW/B0JLvo
-SeGwJrc4s66llu+ls3LHp9w0lJE9GypCU41DJsFYUWjKX7/BCHFG6xwMHN1Qo+4K
-5xS9JHQLkGuosgR+CJU1GDiDlchCfM2e0eASTt0FPk0biefsP0ImTHg6sba3Qy9+
-72EBz1uy/1yEggjjKbAN2r8xaYaEFwUPJo3QfE8ZHeLB+5iU6p0aeSC4Uc1aXDz4
-EGBXiAeeuwfpBXkmrmiSZnNBFYDnCuWkjLTh9wIDAQABAoIBAAU0iEDTI9s78pB8
-XBLYofKOuemFhRl/SDc7KbAlUT4N93RFDYs7bLG73Eto3xW1JBL2rXv3l1WGy71T
-YctyEMaW4T28bhODGvOnK0lpyWp0n6CMGARCWW0YTlaYEjay866bEuyN5l3cDQX9
-Csvn8NzXJitJa51tXFVxW3YO1j7Nyc/M59oyBZ1ARYYmQqFYLEu6lvJOW0cKDFkZ
-AcMVlOIxZQL/Mf+RO72aQGVuYNjqxlLIXLuE9zFR2gDFM2+l3FMUWDGHGBDFyjKU
-iMk4+sSlOTFXqO9VQzua6FLFMsQT6m5PFD4uPY92KR6CPfH/NrWqwqr+jpjaU+gs
-3U9GN+ECgYEA58qX7tKPk7CWdk3kyk5NsNcs/qib+heXWEubfhoU8LmSnbBQhOAz
-wi//r/xm0OHGj84y66+G3T347iudrLjhr07oGM1QfjYT3kb90efLjwAfCECtyVYL
-EQrWO5UeoTnmrhlB1mGL3sWaVAsVqNLz8i2H5c7sj0hxHsvM62159r8CgYEA0Cff
-opJqmUpMpHm3sgjMWctylVrHBuQe5cl5Ad80pbd6mvtt4TvGXbUGNdzURfyve9DS
-x1CVlj4Sz8VuelFQgYL+7/qUqZoms1aSgJpxWv8ou+wUHmlF3kVO8VKt3BNHV+8J
-euSB6NG91BGguBoHgnOoVcjbDGdhJGRTojCNWskCgYEA1jE3nwDCnrbTA3XNk0ky
-r9TXhmgm4r+EIpqTkL7nVOAXZVJ1xaQtECgsveKe3C2WwHLKSVMFbFMFQonZha+/
-FbHz9l9cH5U3XPL7QEpTp8xz4LtsHJ4/UbtS5vJQwKnxyjYaydGQYAb4KuunUz/F
-H6kFaM6DeZB2v/+SWIfs6Z8CgYARUdAEyeP+vzTjVpFXSe4e5pOxI619wEtl2T6t
-TjImO78C2DrLS9r0fxR2NNqgvCapybVQCj94EdAk44uOt+dX71thAluORRpFP8XO
-14rpBGQSRtFhumaq2N95quR2dFAyW9xREmRQx+rgk1rpFplbXF48TQsU3CE0Evj2
-fM22KQKBgDhob7M9sWvXecxoyy3J17jUTcFqmqKcqGnx3ZJ7Q9CgAfjYqNNQws27
-wTuaJB0PEuCOu4t+lUHEBMIjGkBfo1bHd4EZaW04Xgbfn2j8MK2e+9GlRtedxxFg
-c1JdRb5+eTgPwLcDsmMWIW357PDW7RDEI07G1ZB4SqxGTKkU7JOW
------END RSA PRIVATE KEY-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIN+K8+F47YchiH+7gA8KBG8u35PWcOJN+Fszv8TPEEpdoAoGCCqGSM49
+AwEHoUQDQgAEqelGnUWdGT9xdinhJCFDn9AfBjk1+eQfdUTvzy1EG9usGXFphxjz
+idBwMvRdhGPqydcw1J3weZrKMHov4rqGjA==
+-----END EC PRIVATE KEY-----

Diff for: .buildkite/certs/testnode.key

@@ -2,7 +2,7 @@
 #
 # Shared cleanup routines between different steps
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.0
 # - Initial version after refactor

Diff for: .buildkite/functions/cleanup.sh

@@ -2,7 +2,7 @@
 #
 # Exposes a routine scripts can call to wait for a container if that container set up a health command
 #
-# Please source .ci/functions/imports.sh as a whole not just this file
+# Please source .buildkite/functions/imports.sh as a whole not just this file
 #
 # Version 1.0.1
 # - Initial version after refactor

Diff for: .buildkite/functions/wait-for-container.sh

@@ -1,21 +1,34 @@
 steps:
-  - label: ":elasticsearch: :python: ES Python ({{ matrix.python }})"
+  - label: ":elasticsearch: :python: ES Python {{ matrix.python }} {{ matrix.nox_session }} ({{ matrix.connection }})"
     agents:
       provider: "gcp"
     env:
       PYTHON_VERSION: "{{ matrix.python }}"
       TEST_SUITE: "platinum"
-      STACK_VERSION: 8.10.0-SNAPSHOT
+      STACK_VERSION: "8.16.0-SNAPSHOT"
       PYTHON_CONNECTION_CLASS: "{{ matrix.connection }}"
+      NOX_SESSION: "{{ matrix.nox_session }}"
     matrix:
       setup:
         python:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
+          - "3.13"
         connection:
           - "urllib3"
           - "requests"
+        nox_session:
+          - "test"
+      adjustments:
+        - with:
+            python: "3.8"
+            connection: "urllib3"
+            nox_session: "test_otel"
+        - with:
+            python: "3.13"
+            connection: "urllib3"
+            nox_session: "test_otel"
     command: ./.buildkite/run-tests

Diff for: .buildkite/pipeline.yml

@@ -0,0 +1,14 @@
+{
+  "jobs": [
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-py-integration-tests",
+      "allow_org_users": true
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "docs-build-pr",
+      "allow_org_users": true
+    }
+  ]
+}

Diff for: .buildkite/pull-requests.json

@@ -18,6 +18,7 @@ echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m NOX_SESSION ${NOX_SESSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLASS}\033[0m"
 
@@ -42,7 +43,8 @@ docker run \
   --env "TEST_SUITE=${TEST_SUITE}" \
   --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \
   --env "TEST_TYPE=server" \
+  --env "FORCE_COLOR=1" \
   --name elasticsearch-py \
   --rm \
   elastic/elasticsearch-py \
-  nox -s test-${PYTHON_VERSION}
+  nox -s ${NOX_SESSION}-${PYTHON_VERSION}

Diff for: .buildkite/run-repository.sh

@@ -7,7 +7,7 @@
 # Default environment variables
 export STACK_VERSION="${STACK_VERSION:=8.0.0-SNAPSHOT}"
 export TEST_SUITE="${TEST_SUITE:=platinum}"
-export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
+export PYTHON_VERSION="${PYTHON_VERSION:=3.13}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
 
 script_path=$(dirname $(realpath -s $0))
@@ -18,7 +18,7 @@ echo "--- :elasticsearch: Starting Elasticsearch"
 DETACH=true bash $script_path/run-elasticsearch.sh
 
 if [[ -n "$RUNSCRIPTS" ]]; then
-  for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
+  for RUNSCRIPT in ${RUNSCRIPTS//,/ }; do
     echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
     CONTAINER_NAME=${RUNSCRIPT} \
       DETACH=true \