Skip to content

Commit 68b2c08

Browse files
TingDaoKgraebm
TingDaoK
and
graebm
authored
Fork workaround (#628)
Co-authored-by: Michael Graeb <[email protected]>
1 parent ba5898e commit 68b2c08

File tree

12 files changed

+229
-62
lines changed

12 files changed

+229
-62
lines changed

‎.github/workflows/ci.yml

+51-16
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ jobs:
4949
image:
5050
- x64
5151
- x86
52-
- aarch64
5352
python:
5453
- cp38-cp38
5554
- cp39-cp39
@@ -65,23 +64,43 @@ jobs:
6564
with:
6665
role-to-assume: ${{ env.CRT_CI_ROLE }}
6766
aws-region: ${{ env.AWS_DEFAULT_REGION }}
68-
# Only aarch64 needs this, but it doesn't hurt anything
69-
- name: Install qemu/docker
70-
run: docker run --privileged --rm tonistiigi/binfmt --install aarch64
71-
7267
- name: Build ${{ env.PACKAGE_NAME }}
7368
run: |
7469
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
7570
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
7671
72+
manylinux2014-arm64:
73+
runs-on: ubuntu-24.04-arm
74+
strategy:
75+
fail-fast: false
76+
matrix:
77+
python:
78+
- cp38-cp38
79+
- cp39-cp39
80+
- cp310-cp310
81+
- cp311-cp311
82+
- cp312-cp312
83+
- cp313-cp313
84+
permissions:
85+
id-token: write # This is required for requesting the JWT
86+
steps:
87+
- name: configure AWS credentials (containers)
88+
uses: aws-actions/configure-aws-credentials@v4
89+
with:
90+
role-to-assume: ${{ env.CRT_CI_ROLE }}
91+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
92+
- name: Build ${{ env.PACKAGE_NAME }}
93+
run: |
94+
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
95+
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-aarch64 build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
96+
7797
musllinux-1-1:
7898
runs-on: ubuntu-24.04 # latest
7999
strategy:
80100
fail-fast: false
81101
matrix:
82102
image:
83103
- x64
84-
- aarch64
85104
python:
86105
- cp38-cp38
87106
- cp39-cp39
@@ -97,18 +116,38 @@ jobs:
97116
with:
98117
role-to-assume: ${{ env.CRT_CI_ROLE }}
99118
aws-region: ${{ env.AWS_DEFAULT_REGION }}
100-
101-
# Only aarch64 needs this, but it doesn't hurt anything
102-
- name: Install qemu/docker
103-
run: docker run --privileged --rm tonistiigi/binfmt --install aarch64
104-
105119
- name: Build ${{ env.PACKAGE_NAME }}
106120
run: |
107121
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
108122
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
109123
124+
musllinux-1-1-arm64:
125+
runs-on: ubuntu-24.04-arm
126+
strategy:
127+
fail-fast: false
128+
matrix:
129+
python:
130+
- cp38-cp38
131+
- cp39-cp39
132+
- cp310-cp310
133+
- cp311-cp311
134+
- cp312-cp312
135+
- cp313-cp313
136+
permissions:
137+
id-token: write # This is required for requesting the JWT
138+
steps:
139+
- name: configure AWS credentials (containers)
140+
uses: aws-actions/configure-aws-credentials@v4
141+
with:
142+
role-to-assume: ${{ env.CRT_CI_ROLE }}
143+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
144+
- name: Build ${{ env.PACKAGE_NAME }}
145+
run: |
146+
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
147+
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-aarch64 build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
148+
110149
raspberry:
111-
runs-on: ubuntu-24.04 # latest
150+
runs-on: ubuntu-24.04-arm
112151
strategy:
113152
fail-fast: false
114153
matrix:
@@ -123,10 +162,6 @@ jobs:
123162
role-to-assume: ${{ env.CRT_CI_ROLE }}
124163
aws-region: ${{ env.AWS_DEFAULT_REGION }}
125164

126-
# set arm arch
127-
- name: Install qemu/docker
128-
run: docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
129-
130165
- name: Build ${{ env.PACKAGE_NAME }}
131166
run: |
132167
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh

‎README.md

+36-15
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,36 @@ git submodule update --init
3232
python3 -m pip install .
3333
```
3434

35-
To use from your Python application, declare `awscrt` as a dependency.
35+
See [Advanced Build Options](#advanced-build-options) for more info about building from source.
36+
37+
## Fork and Multiprocessing
38+
39+
aws-crt-python uses background threads. This makes [os.fork()](https://docs.python.org/3/library/os.html#os.fork) unsafe. In a forked child process, all background threads vanish. The child will hang or crash when it tries to communicate with any of these (vanished) threads.
40+
41+
Unfortunately, Python's [multiprocessing](https://docs.python.org/3/library/multiprocessing.html) module defaults to using fork when it creates child processes (on POSIX systems except macOS, in Python versions 3.13 and earlier). `multiprocessing` is used under the hood by many tools that do work in parallel, including [concurrent.futures.ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor), and [pytorch.multiprocessing](https://pytorch.org/docs/stable/multiprocessing.html).
42+
43+
If you need to use `multiprocessing` with aws-crt-python, set it to use "spawn" or "forkserver" instead of "fork" ([see docs](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods)). The Python community agrees, and `multiprocessing` will changes its default from "fork" to "spawn" in 3.14. It already uses "spawn" by default on macOS (because system libraries may start threads) and on Windows (because fork does not exist).
44+
45+
If you **must** use fork with aws-crt-python, you may be able to avoid hangs and crashes if you manage your threads very carefully:
46+
47+
1. Release all CRT resources with background threads (e.g. clean up any `io.EventLoopGroup` instances).
48+
2. Join all CRT threads before forking (use `common.join_all_native_threads()` ).
49+
50+
For an example, see `test.test_s3.py.S3RequestTest.test_fork_workaround` .
51+
52+
## Mac-Only TLS Behavior
53+
54+
Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level:
55+
56+
```
57+
static: certificate has an existing certificate-key pair that was previously imported into the Keychain. Using key from Keychain instead of the one provided.
58+
```
59+
60+
## Crash Handler
61+
62+
You can enable the crash handler by setting the environment variable `AWS_CRT_CRASH_HANDLER=1` . This will print the callstack to `stderr` in the event of a fatal error.
63+
64+
## Advanced Build Options
3665

3766
### OpenSSL and LibCrypto
3867

@@ -54,20 +83,23 @@ set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1` while building f
5483
```sh
5584
AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1 python3 -m pip install --no-binary :all: --verbose awscrt
5685
```
86+
5787
( `--no-binary :all:` ensures you do not use the precompiled wheel from PyPI)
5888

59-
aws-crt-python also exposes a number of cryptographic primitives.
89+
aws-crt-python also exposes a number of cryptographic primitives.
6090
On Unix, those depend on libcrypto as described above.
6191
On Apple and Windows OS level crypto libraries are used whenever possible.
62-
One exception to above statement is that for ED25519 keygen on Windows and Apple,
92+
One exception to above statement is that for ED25519 keygen on Windows and Apple,
6393
libcrypto is used as no viable OS level alternative exists. In that case Unix level notes
64-
about libcrypto apply to Apple and Windows as well. Libcrypto usage for ED25519 support is
94+
about libcrypto apply to Apple and Windows as well. Libcrypto usage for ED25519 support is
6595
enabled on Windows and Apple by default and can be disabled by setting environment variable
6696
`AWS_CRT_BUILD_DISABLE_LIBCRYPTO_USE_FOR_ED25519_EVERYWHERE` as follows:
6797
(Note: ED25519 keygen functions will start returning not supported error in this case)
98+
6899
```sh
69100
AWS_CRT_BUILD_DISABLE_LIBCRYPTO_USE_FOR_ED25519_EVERYWHERE=1 python3 -m pip install --no-binary :all: --verbose awscrt
70101
```
102+
71103
( `--no-binary :all:` ensures you do not use the precompiled wheel from PyPI)
72104

73105
### AWS_CRT_BUILD_USE_SYSTEM_LIBS ###
@@ -84,14 +116,3 @@ AWS_CRT_BUILD_USE_SYSTEM_LIBS=1 python3 -m pip install .
84116
```
85117

86118
If these dependencies are available as both static and shared libs, you can force the static ones to be used by setting: `AWS_CRT_BUILD_FORCE_STATIC_LIBS=1`
87-
88-
## Mac-Only TLS Behavior
89-
90-
Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level:
91-
92-
```
93-
static: certificate has an existing certificate-key pair that was previously imported into the Keychain. Using key from Keychain instead of the one provided.
94-
```
95-
96-
## Crash Handler
97-
You can enable the crash handler by setting the environment variable `AWS_CRT_CRASH_HANDLER=1`. This will print the callstack to `stderr` in the event of a fatal error.

‎awscrt/_test.py

+1-17
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"""
44
import _awscrt
55
from awscrt import NativeResource
6+
from awscrt.common import join_all_native_threads
67
import gc
78
import inspect
89
import os
@@ -47,23 +48,6 @@ def dump_native_memory():
4748
return _awscrt.native_memory_dump()
4849

4950

50-
def join_all_native_threads(*, timeout_sec: float = -1.0) -> bool:
51-
"""
52-
Waits for all native threads to complete their join call.
53-
54-
This can only be safely called from the main thread.
55-
This call may be required for native memory usage to reach zero.
56-
57-
Args:
58-
timeout_sec (float): Number of seconds to wait before a timeout exception is raised.
59-
By default the wait is unbounded.
60-
61-
Returns:
62-
bool: Returns whether threads could be joined before the timeout.
63-
"""
64-
return _awscrt.thread_join_all_managed(timeout_sec)
65-
66-
6751
def check_for_leaks(*, timeout_sec=10.0):
6852
"""
6953
Checks that all awscrt resources have been freed after a test.

‎awscrt/common.py

+17
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,20 @@ def get_cpu_count_for_group(group_idx: int) -> int:
1818
Returns number of processors in a given group.
1919
"""
2020
return _awscrt.get_cpu_count_for_group(group_idx)
21+
22+
23+
def join_all_native_threads(*, timeout_sec: float = -1.0) -> bool:
24+
"""
25+
Waits for all native threads to complete their join call.
26+
27+
This can only be safely called from the main thread.
28+
This call may be required for native memory usage to reach zero.
29+
30+
Args:
31+
timeout_sec (float): Number of seconds to wait before a timeout exception is raised.
32+
By default the wait is unbounded.
33+
34+
Returns:
35+
bool: Returns whether threads could be joined before the timeout.
36+
"""
37+
return _awscrt.thread_join_all_managed(timeout_sec)

‎awscrt/s3.py

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def acquire(self):
4242

4343
def __enter__(self):
4444
self.acquire()
45+
return self
4546

4647
def release(self):
4748
_awscrt.s3_cross_process_lock_release(self._binding)

‎crt/aws-c-auth

‎crt/aws-c-common

‎crt/aws-c-http

‎crt/aws-c-s3

‎crt/aws-lc

‎source/common.c

+11-4
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,17 @@ PyObject *aws_py_thread_join_all_managed(PyObject *self, PyObject *args) {
4646
}
4747
}
4848
aws_thread_set_managed_join_timeout_ns(timeout_ns);
49-
50-
if (aws_thread_join_all_managed()) {
49+
int res = AWS_OP_SUCCESS;
50+
/* clang-format off */
51+
Py_BEGIN_ALLOW_THREADS
52+
/* Drop GIL to allow other threads callback into python to finish joining. */
53+
res = aws_thread_join_all_managed();
54+
Py_END_ALLOW_THREADS
55+
56+
if(res == AWS_OP_SUCCESS) {
57+
Py_RETURN_TRUE;
58+
} else {
5159
Py_RETURN_FALSE;
5260
}
53-
54-
Py_RETURN_TRUE;
61+
/* clang-format on */
5562
}

0 commit comments

Comments
 (0)