extmod/mbedtls: Try GC before failing to setup socket on esp32, unix.

On mbedTLS ports with non-baremetal configs (mostly esp32, technically also
unix port), mbedTLS memory is allocated from the libc heap. This means an
old SSL socket may be holding large SSL buffers and preventing a new SSL
socket from being allocated.

As a workaround, trigger a GC pass and retry before failing outright.

This was originally implemented as a global mbedTLS calloc function, but
there is complexity around the possibility of C user modules calling into
mbedTLS without holding the GIL. It would be interesting to try making a
generic version for any malloc which fails, but this would require checking
for a Python thread and probably making the GIL recursive.

This work was funded through GitHub Sponsors.

Signed-off-by: Angus Gratton <[email protected]>

Author: projectgus

extmod/modtls_mbedtls.c

@@ -37,6 +37,7 @@
 #include "py/stream.h"
 #include "py/objstr.h"
 #include "py/reader.h"
+#include "py/gc.h"
 #include "extmod/vfs.h"
 
 // mbedtls_time_t
@@ -58,6 +59,10 @@
 #include "mbedtls/asn1.h"
 #endif
 
+#ifndef MICROPY_MBEDTLS_CONFIG_BARE_METAL
+#define MICROPY_MBEDTLS_CONFIG_BARE_METAL (0)
+#endif
+
 #define MP_STREAM_POLL_RDWR (MP_STREAM_POLL_RD | MP_STREAM_POLL_WR)
 
 // This corresponds to an SSLContext object.
@@ -545,6 +550,16 @@ static mp_obj_t ssl_socket_make_new(mp_obj_ssl_context_t *ssl_context, mp_obj_t
     mbedtls_ssl_init(&o->ssl);
 
     ret = mbedtls_ssl_setup(&o->ssl, &ssl_context->conf);
+    #if !MICROPY_MBEDTLS_CONFIG_BARE_METAL
+    if (ret == MBEDTLS_ERR_SSL_ALLOC_FAILED) {
+        // If mbedTLS relies on platform libc heap for buffers (i.e. esp32
+        // port), then run a GC pass and then try again. This is useful because
+        // it may free a Python object (like an old SSL socket) whose finaliser
+        // frees some platform-level heap.
+        gc_collect();
+        ret = mbedtls_ssl_setup(&o->ssl, &ssl_context->conf);
+    }
+    #endif
     if (ret != 0) {
         goto cleanup;
     }