H2 settings (#648)

Author: TingDaoK

awscrt/http.py

@@ -1,14 +1,16 @@
 [build-system]
 requires = [
   "setuptools>=75.3.1",
-  "wheel>=0.45.1", # used by our setup.py
+  "wheel>=0.45.1",      # used by our setup.py
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "awscrt"
-license = {text = "Apache-2.0", files = ["LICENSE"]}
-authors = [{name = "Amazon Web Services, Inc", email = "[email protected]"}]
+license = { text = "Apache-2.0", files = ["LICENSE"] }
+authors = [
+  { name = "Amazon Web Services, Inc", email = "[email protected]" },
+]
 description = "A common runtime for AWS Python projects"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -19,9 +21,7 @@ classifiers = [
   "Operating System :: Unix",
   "Operating System :: MacOS",
 ]
-dynamic = [
-  "version",
-]
+dynamic = ["version"]
 
 [project.urls]
 github = "https://github.com/awslabs/aws-crt-python"
@@ -31,9 +31,10 @@ releasenotes = "https://github.com/awslabs/aws-crt-python/releases"
 
 [project.optional-dependencies]
 dev = [
-  "autopep8>=2.3.1", # for code formatting
-  "build>=1.2.2", # for building wheels
+  "autopep8>=2.3.1",                             # for code formatting
+  "build>=1.2.2",                                # for building wheels
   "sphinx>=7.2.6,<7.3; python_version >= '3.9'", # for building docs
-  "websockets>=13.1", # for tests
-  "h2", # for tests
+  "websockets>=13.1",                            # for tests
+  # for tests, restrict to exact version for test_http_client/TestClientMockServer/_on_remote_settings_changed that relies on the implementation details. Also, this is no needs to update this package.
+  "h2==4.1.0",
 ]

crt/aws-c-common

@@ -138,6 +138,85 @@ static void s_on_client_connection_setup(
     PyGILState_Release(state);
 }
 
+/* Function to convert Python list of Http2Setting to C array of aws_http2_setting */
+static int s_convert_http2_settings(
+    PyObject *initial_settings_py,
+    struct aws_allocator *allocator,
+    struct aws_http2_setting *out_settings[],
+    size_t *out_size) {
+    Py_ssize_t py_list_size = PyList_Size(initial_settings_py);
+    if (py_list_size == 0) {
+        *out_size = 0;
+        *out_settings = NULL;
+        return AWS_OP_SUCCESS;
+    }
+
+    *out_settings = aws_mem_calloc(allocator, py_list_size, sizeof(struct aws_http2_setting));
+
+    for (Py_ssize_t i = 0; i < py_list_size; i++) {
+        PyObject *setting_py = PyList_GetItem(initial_settings_py, i);
+
+        /* Get id attribute */
+        enum aws_http2_settings_id id = PyObject_GetAttrAsIntEnum(setting_py, "Http2Setting", "id");
+        if (PyErr_Occurred()) {
+            goto error;
+        }
+
+        /* Get value attribute */
+        uint32_t value = PyObject_GetAttrAsUint32(setting_py, "Http2Setting", "value");
+        if (PyErr_Occurred()) {
+            goto error;
+        }
+        (*out_settings)[i].id = id;
+        (*out_settings)[i].value = value;
+    }
+
+    *out_size = (size_t)py_list_size;
+    return AWS_OP_SUCCESS;
+error:
+    *out_size = 0;
+    aws_mem_release(allocator, *out_settings);
+    *out_settings = NULL;
+    return AWS_OP_ERR;
+}
+
+static void s_http2_on_remote_settings_change(
+    struct aws_http_connection *http2_connection,
+    const struct aws_http2_setting *settings_array,
+    size_t num_settings,
+    void *user_data) {
+    (void)http2_connection;
+    struct http_connection_binding *connection = user_data;
+
+    PyGILState_STATE state;
+    if (aws_py_gilstate_ensure(&state)) {
+        return; /* Python has shut down. Nothing matters anymore, but don't crash */
+    }
+    // Create a new list to hold tuples
+    PyObject *py_settings_list = PyList_New(num_settings);
+    if (!py_settings_list) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        return;
+    }
+    for (size_t i = 0; i < num_settings; i++) {
+        PyObject *tuple = Py_BuildValue("(iI)", settings_array[i].id, settings_array[i].value);
+        if (!tuple) {
+            PyErr_WriteUnraisable(PyErr_Occurred());
+            goto done;
+        }
+        PyList_SetItem(py_settings_list, i, tuple); /* steals reference to tuple */
+    }
+    PyObject *result = PyObject_CallMethod(connection->py_core, "_on_remote_settings_changed", "(O)", py_settings_list);
+    if (!result) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        goto done;
+    }
+    Py_DECREF(result);
+done:
+    Py_XDECREF(py_settings_list);
+    PyGILState_Release(state);
+}
+
 PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
     (void)self;
 
@@ -150,18 +229,23 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
     PyObject *socket_options_py;
     PyObject *tls_options_py;
     PyObject *proxy_options_py;
+    PyObject *initial_settings_py;
+    PyObject *on_remote_settings_changed_py;
     PyObject *py_core;
+    bool success = false;
 
     if (!PyArg_ParseTuple(
             args,
-            "Os#IOOOO",
+            "Os#IOOOOOO",
             &bootstrap_py,
             &host_name,
             &host_name_len,
             &port_number,
             &socket_options_py,
             &tls_options_py,
             &proxy_options_py,
+            &initial_settings_py,
+            &on_remote_settings_changed_py,
             &py_core)) {
         return NULL;
     }
@@ -172,23 +256,34 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
     }
 
     struct http_connection_binding *connection = aws_mem_calloc(allocator, 1, sizeof(struct http_connection_binding));
-    if (!connection) {
-        return PyErr_AwsLastError();
-    }
-
     /* From hereon, we need to clean up if errors occur */
+    struct aws_http2_setting *http2_settings = NULL;
+    size_t http2_settings_count = 0;
+    struct aws_http2_connection_options http2_options = {0};
 
     struct aws_tls_connection_options *tls_options = NULL;
     if (tls_options_py != Py_None) {
         tls_options = aws_py_get_tls_connection_options(tls_options_py);
         if (!tls_options) {
-            goto error;
+            goto done;
         }
     }
 
     struct aws_socket_options socket_options;
     if (!aws_py_socket_options_init(&socket_options, socket_options_py)) {
-        goto error;
+        goto done;
+    }
+
+    if (initial_settings_py != Py_None) {
+        /* Get the array from the pylist */
+        if (s_convert_http2_settings(initial_settings_py, allocator, &http2_settings, &http2_settings_count)) {
+            goto done;
+        }
+        http2_options.initial_settings_array = http2_settings;
+        http2_options.num_initial_settings = http2_settings_count;
+    }
+    if (on_remote_settings_changed_py != Py_None) {
+        http2_options.on_remote_settings_change = s_http2_on_remote_settings_change;
     }
 
     /* proxy options are optional */
@@ -197,7 +292,7 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
     if (proxy_options_py != Py_None) {
         proxy_options = &proxy_options_storage;
         if (!aws_py_http_proxy_options_init(proxy_options, proxy_options_py)) {
-            goto error;
+            goto done;
         }
     }
 
@@ -214,21 +309,27 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args) {
         .socket_options = &socket_options,
         .on_setup = s_on_client_connection_setup,
         .on_shutdown = s_on_connection_shutdown,
+        .http2_options = &http2_options,
     };
 
     connection->py_core = py_core;
     Py_INCREF(connection->py_core);
 
     if (aws_http_client_connect(&http_options)) {
         PyErr_SetAwsLastError();
-        goto error;
+        goto done;
     }
+    success = true;
 
+done:
+    if (http2_settings) {
+        aws_mem_release(allocator, http2_settings);
+    }
+    if (!success) {
+        s_connection_destroy(connection);
+        return NULL;
+    }
     Py_RETURN_NONE;
-
-error:
-    s_connection_destroy(connection);
-    return NULL;
 }
 
 PyObject *aws_py_http_connection_close(PyObject *self, PyObject *args) {

crt/aws-c-http

@@ -1,22 +1,22 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-import awscrt.exceptions
-from awscrt.http import HttpClientConnection, HttpClientStream, HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion, Http2ClientConnection
-from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions, TlsCipherPref
-from concurrent.futures import Future, thread
-from http.server import HTTPServer, SimpleHTTPRequestHandler
-from io import BytesIO
-import os
-import ssl
-from test import NativeResourceTest
-import threading
-import unittest
-from urllib.parse import urlparse
-import subprocess
-import sys
-import socket
 import time
+import socket
+import sys
+import subprocess
+from urllib.parse import urlparse
+import unittest
+import threading
+from test import NativeResourceTest
+import ssl
+import os
+from io import BytesIO
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from concurrent.futures import Future, thread
+from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions, TlsCipherPref
+from awscrt.http import HttpClientConnection, HttpClientStream, HttpHeaders, HttpProxyOptions, HttpRequest, HttpVersion, Http2ClientConnection, Http2Setting, Http2SettingID
+import awscrt.exceptions
 
 
 class Response:
@@ -449,7 +449,24 @@ def tearDown(self):
             self.p_server.kill()
         super().tearDown()
 
-    def _new_mock_connection(self):
+    def _on_remote_settings_changed(self, settings):
+        # The mock server has the default settings with
+        # ENABLE_PUSH = 0
+        # MAX_CONCURRENT_STREAMS = 100
+        # MAX_HEADER_LIST_SIZE = 2**16
+        # using [email protected], code can be found in
+        # https://github.com/python-hyper/h2/blob/191ac06e0949fcfe3367b06eeb101a5a1a335964/src/h2/connection.py#L340-L359
+        # Check the settings here
+        self.assertEqual(len(settings), 3)
+        for i in settings:
+            if i.id == Http2SettingID.ENABLE_PUSH:
+                self.assertEqual(i.value, 0)
+            if i.id == Http2SettingID.MAX_CONCURRENT_STREAMS:
+                self.assertEqual(i.value, 100)
+            if i.id == Http2SettingID.MAX_HEADER_LIST_SIZE:
+                self.assertEqual(i.value, 2**16)
+
+    def _new_mock_connection(self, initial_settings=None):
 
         event_loop_group = EventLoopGroup()
         host_resolver = DefaultHostResolver(event_loop_group)
@@ -465,11 +482,15 @@ def _new_mock_connection(self):
         tls_conn_opt = tls_ctx.new_connection_options()
         tls_conn_opt.set_server_name(self.mock_server_url.hostname)
         tls_conn_opt.set_alpn_list(["h2"])
+        if initial_settings is None:
+            initial_settings = [Http2Setting(Http2SettingID.ENABLE_PUSH, 0)]
 
         connection_future = Http2ClientConnection.new(host_name=self.mock_server_url.hostname,
                                                       port=port,
                                                       bootstrap=bootstrap,
-                                                      tls_connection_options=tls_conn_opt)
+                                                      tls_connection_options=tls_conn_opt,
+                                                      initial_settings=initial_settings,
+                                                      on_remote_settings_changed=self._on_remote_settings_changed)
         return connection_future.result(self.timeout)
 
     def test_h2_mock_server_manual_write(self):
@@ -571,6 +592,37 @@ def test_h2_mock_server_manual_write_lifetime(self):
         stream.completion_future.result()
         self.assertEqual(None, connection.close().exception(self.timeout))
 
+    def test_h2_mock_server_settings(self):
+        exception = None
+        try:
+            # invalid settings, should throw an exception
+            initial_settings = [100]
+            connection = self._new_mock_connection(initial_settings)
+        except Exception as e:
+            exception = e
+        self.assertIsNotNone(exception)
+
+        connection = self._new_mock_connection()
+        # check we set an h2 connection
+        self.assertEqual(connection.version, HttpVersion.Http2)
+
+        request = HttpRequest('POST', self.mock_server_url.path)
+        request.headers.add('host', self.mock_server_url.hostname)
+        response = Response()
+        stream = connection.request(request, response.on_response, response.on_body, manual_write=True)
+        stream.activate()
+        exception = None
+        stream.write_data(BytesIO(b'hello'), True)
+
+        self.assertIsNone(exception)
+        stream_completion_result = stream.completion_future.result(80)
+        # check result
+        self.assertEqual(200, response.status_code)
+        self.assertEqual(200, stream_completion_result)
+        print(response.body)
+
+        self.assertEqual(None, connection.close().exception(self.timeout))
+
 
 if __name__ == '__main__':
     unittest.main()