Skip to content

Commit 57bdc6c

Browse files
corona10corona10
authored
gh-111968: Introduce _PyFreeListState and _PyFreeListState_GET API (gh-113584)
1 parent cdca0ce commit 57bdc6c

17 files changed

+171
-50
lines changed

‎Include/internal/pycore_freelist.h

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#ifndef Py_INTERNAL_FREELIST_H
2+
#define Py_INTERNAL_FREELIST_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
#ifndef WITH_FREELISTS
12+
// without freelists
13+
# define PyList_MAXFREELIST 0
14+
#endif
15+
16+
/* Empty list reuse scheme to save calls to malloc and free */
17+
#ifndef PyList_MAXFREELIST
18+
# define PyList_MAXFREELIST 80
19+
#endif
20+
21+
struct _Py_list_state {
22+
#if PyList_MAXFREELIST > 0
23+
PyListObject *free_list[PyList_MAXFREELIST];
24+
int numfree;
25+
#endif
26+
};
27+
28+
typedef struct _Py_freelist_state {
29+
struct _Py_list_state list;
30+
} _PyFreeListState;
31+
32+
#ifdef __cplusplus
33+
}
34+
#endif
35+
#endif /* !Py_INTERNAL_FREELIST_H */

‎Include/internal/pycore_gc.h

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_freelist.h" // _PyFreeListState
12+
1113
/* GC information is stored BEFORE the object structure. */
1214
typedef struct {
1315
// Pointer to next object in the list.
@@ -238,9 +240,11 @@ extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generat
238240
extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
239241

240242
// Functions to clear types free lists
243+
extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
244+
extern void _Py_ClearFreeLists(_PyFreeListState *state, int is_finalization);
241245
extern void _PyTuple_ClearFreeList(PyInterpreterState *interp);
242246
extern void _PyFloat_ClearFreeList(PyInterpreterState *interp);
243-
extern void _PyList_ClearFreeList(PyInterpreterState *interp);
247+
extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
244248
extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
245249
extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
246250
extern void _PyContext_ClearFreeList(PyInterpreterState *interp);

‎Include/internal/pycore_interp.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ struct _is {
179179
// One bit is set for each non-NULL entry in code_watchers
180180
uint8_t active_code_watchers;
181181

182+
#if !defined(Py_GIL_DISABLED)
183+
struct _Py_freelist_state freelist_state;
184+
#endif
182185
struct _py_object_state object_state;
183186
struct _Py_unicode_state unicode;
184187
struct _Py_float_state float_state;
@@ -190,7 +193,6 @@ struct _is {
190193
PySliceObject *slice_cache;
191194

192195
struct _Py_tuple_state tuple;
193-
struct _Py_list_state list;
194196
struct _Py_dict_state dict_state;
195197
struct _Py_async_gen_state async_gen;
196198
struct _Py_context_state context;

‎Include/internal/pycore_list.h

+2-20
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,17 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_freelist.h" // _PyFreeListState
1112

1213
extern PyObject* _PyList_Extend(PyListObject *, PyObject *);
1314
extern void _PyList_DebugMallocStats(FILE *out);
1415

1516

1617
/* runtime lifecycle */
1718

18-
extern void _PyList_Fini(PyInterpreterState *);
19+
extern void _PyList_Fini(_PyFreeListState *);
1920

2021

21-
/* other API */
22-
23-
#ifndef WITH_FREELISTS
24-
// without freelists
25-
# define PyList_MAXFREELIST 0
26-
#endif
27-
28-
/* Empty list reuse scheme to save calls to malloc and free */
29-
#ifndef PyList_MAXFREELIST
30-
# define PyList_MAXFREELIST 80
31-
#endif
32-
33-
struct _Py_list_state {
34-
#if PyList_MAXFREELIST > 0
35-
PyListObject *free_list[PyList_MAXFREELIST];
36-
int numfree;
37-
#endif
38-
};
39-
4022
#define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)
4123

4224
extern int

‎Include/internal/pycore_pystate.h

+16
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_freelist.h" // _PyFreeListState
1112
#include "pycore_runtime.h" // _PyRuntime
13+
#include "pycore_tstate.h" // _PyThreadStateImpl
1214

1315

1416
// Values for PyThreadState.state. A thread must be in the "attached" state
@@ -239,6 +241,20 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
239241
// See also PyInterpreterState_Get() and _PyInterpreterState_GET().
240242
extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
241243

244+
static inline _PyFreeListState* _PyFreeListState_GET(void)
245+
{
246+
PyThreadState *tstate = _PyThreadState_GET();
247+
#ifdef Py_DEBUG
248+
_Py_EnsureTstateNotNULL(tstate);
249+
#endif
250+
251+
#ifdef Py_GIL_DISABLED
252+
return &((_PyThreadStateImpl*)tstate)->freelist_state;
253+
#else
254+
return &tstate->interp->freelist_state;
255+
#endif
256+
}
257+
242258
#ifdef __cplusplus
243259
}
244260
#endif

‎Include/internal/pycore_tstate.h

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_freelist.h" // struct _Py_freelist_state
1112
#include "pycore_mimalloc.h" // struct _mimalloc_thread_state
1213

1314

@@ -20,6 +21,7 @@ typedef struct _PyThreadStateImpl {
2021

2122
#ifdef Py_GIL_DISABLED
2223
struct _mimalloc_thread_state mimalloc;
24+
struct _Py_freelist_state freelist_state;
2325
#endif
2426

2527
} _PyThreadStateImpl;

‎Makefile.pre.in

+3
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,8 @@ PYTHON_OBJS= \
418418
Python/frozenmain.o \
419419
Python/future.o \
420420
Python/gc.o \
421+
Python/gc_free_threading.o \
422+
Python/gc_gil.o \
421423
Python/getargs.o \
422424
Python/getcompiler.o \
423425
Python/getcopyright.o \
@@ -1828,6 +1830,7 @@ PYTHON_HEADERS= \
18281830
$(srcdir)/Include/internal/pycore_floatobject.h \
18291831
$(srcdir)/Include/internal/pycore_format.h \
18301832
$(srcdir)/Include/internal/pycore_frame.h \
1833+
$(srcdir)/Include/internal/pycore_freelist.h \
18311834
$(srcdir)/Include/internal/pycore_function.h \
18321835
$(srcdir)/Include/internal/pycore_genobject.h \
18331836
$(srcdir)/Include/internal/pycore_getopt.h \

‎Objects/listobject.c

+11-11
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ _Py_DECLARE_STR(list_err, "list index out of range");
2424
static struct _Py_list_state *
2525
get_list_state(void)
2626
{
27-
PyInterpreterState *interp = _PyInterpreterState_GET();
28-
return &interp->list;
27+
_PyFreeListState *state = _PyFreeListState_GET();
28+
assert(state != NULL);
29+
return &state->list;
2930
}
3031
#endif
3132

@@ -120,26 +121,25 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
120121
}
121122

122123
void
123-
_PyList_ClearFreeList(PyInterpreterState *interp)
124+
_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
124125
{
125126
#if PyList_MAXFREELIST > 0
126-
struct _Py_list_state *state = &interp->list;
127-
while (state->numfree) {
127+
struct _Py_list_state *state = &freelist_state->list;
128+
while (state->numfree > 0) {
128129
PyListObject *op = state->free_list[--state->numfree];
129130
assert(PyList_CheckExact(op));
130131
PyObject_GC_Del(op);
131132
}
133+
if (is_finalization) {
134+
state->numfree = -1;
135+
}
132136
#endif
133137
}
134138

135139
void
136-
_PyList_Fini(PyInterpreterState *interp)
140+
_PyList_Fini(_PyFreeListState *state)
137141
{
138-
_PyList_ClearFreeList(interp);
139-
#if defined(Py_DEBUG) && PyList_MAXFREELIST > 0
140-
struct _Py_list_state *state = &interp->list;
141-
state->numfree = -1;
142-
#endif
142+
_PyList_ClearFreeList(state, 1);
143143
}
144144

145145
/* Print summary info about the state of the optimized allocator */

‎PCbuild/_freeze_module.vcxproj

+2
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@
208208
<ClCompile Include="..\Python\frame.c" />
209209
<ClCompile Include="..\Python\future.c" />
210210
<ClCompile Include="..\Python\gc.c" />
211+
<ClCompile Include="..\Python\gc_gil.c" />
212+
<ClCompile Include="..\Python\gc_free_threading.c" />
211213
<ClCompile Include="..\Python\getargs.c" />
212214
<ClCompile Include="..\Python\getcompiler.c" />
213215
<ClCompile Include="..\Python\getcopyright.c" />

‎PCbuild/_freeze_module.vcxproj.filters

+6
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,12 @@
169169
<ClCompile Include="..\Python\gc.c">
170170
<Filter>Source Files</Filter>
171171
</ClCompile>
172+
<ClCompile Include="..\Python\gc_free_threading.c">
173+
<Filter>Source Files</Filter>
174+
</ClCompile>
175+
<ClCompile Include="..\Python\gc_gil.c">
176+
<Filter>Source Files</Filter>
177+
</ClCompile>
172178
<ClCompile Include="..\Modules\gcmodule.c">
173179
<Filter>Source Files</Filter>
174180
</ClCompile>

‎PCbuild/pythoncore.vcxproj

+3
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@
231231
<ClInclude Include="..\Include\internal\pycore_floatobject.h" />
232232
<ClInclude Include="..\Include\internal\pycore_format.h" />
233233
<ClInclude Include="..\Include\internal\pycore_frame.h" />
234+
<ClInclude Include="..\Include\internal\pycore_freelist.h" />
234235
<ClInclude Include="..\Include\internal\pycore_function.h" />
235236
<ClInclude Include="..\Include\internal\pycore_gc.h" />
236237
<ClInclude Include="..\Include\internal\pycore_genobject.h" />
@@ -568,6 +569,8 @@
568569
</ClCompile>
569570
<ClCompile Include="..\Python\future.c" />
570571
<ClCompile Include="..\Python\gc.c" />
572+
<ClCompile Include="..\Python\gc_free_threading.c" />
573+
<ClCompile Include="..\Python\gc_gil.c" />
571574
<ClCompile Include="..\Python\getargs.c" />
572575
<ClCompile Include="..\Python\getcompiler.c" />
573576
<ClCompile Include="..\Python\getcopyright.c" />

‎PCbuild/pythoncore.vcxproj.filters

+12
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,12 @@
618618
<ClInclude Include="..\Include\internal\pycore_format.h">
619619
<Filter>Include\internal</Filter>
620620
</ClInclude>
621+
<ClInclude Include="..\Include\internal\pycore_frame.h">
622+
<Filter>Include\internal</Filter>
623+
</ClInclude>
624+
<ClInclude Include="..\Include\internal\pycore_freelist.h">
625+
<Filter>Include\internal</Filter>
626+
</ClInclude>
621627
<ClInclude Include="..\Include\internal\pycore_function.h">
622628
<Filter>Include\internal</Filter>
623629
</ClInclude>
@@ -1286,6 +1292,12 @@
12861292
<ClCompile Include="..\Python\gc.c">
12871293
<Filter>Python</Filter>
12881294
</ClCompile>
1295+
<ClCompile Include="..\Python\gc_free_threading.c">
1296+
<Filter>Python</Filter>
1297+
</ClCompile>
1298+
<ClCompile Include="..\Python\gc_gil.c">
1299+
<Filter>Python</Filter>
1300+
</ClCompile>
12891301
<ClCompile Include="..\Python\getargs.c">
12901302
<Filter>Python</Filter>
12911303
</ClCompile>

‎Python/gc.c

+1-16
Original file line numberDiff line numberDiff line change
@@ -1019,21 +1019,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate,
10191019
}
10201020
}
10211021

1022-
/* Clear all free lists
1023-
* All free lists are cleared during the collection of the highest generation.
1024-
* Allocated items in the free list may keep a pymalloc arena occupied.
1025-
* Clearing the free lists may give back memory to the OS earlier.
1026-
*/
1027-
static void
1028-
clear_freelists(PyInterpreterState *interp)
1029-
{
1030-
_PyTuple_ClearFreeList(interp);
1031-
_PyFloat_ClearFreeList(interp);
1032-
_PyList_ClearFreeList(interp);
1033-
_PyDict_ClearFreeList(interp);
1034-
_PyAsyncGen_ClearFreeLists(interp);
1035-
_PyContext_ClearFreeList(interp);
1036-
}
10371022

10381023
// Show stats for objects in each generations
10391024
static void
@@ -1449,7 +1434,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
14491434
/* Clear free list only during the collection of the highest
14501435
* generation */
14511436
if (generation == NUM_GENERATIONS-1) {
1452-
clear_freelists(tstate->interp);
1437+
_PyGC_ClearAllFreeLists(tstate->interp);
14531438
}
14541439

14551440
if (_PyErr_Occurred(tstate)) {

‎Python/gc_free_threading.c

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#include "Python.h"
2+
#include "pycore_pystate.h" // _PyFreeListState_GET()
3+
#include "pycore_tstate.h" // _PyThreadStateImpl
4+
5+
#ifdef Py_GIL_DISABLED
6+
7+
/* Clear all free lists
8+
* All free lists are cleared during the collection of the highest generation.
9+
* Allocated items in the free list may keep a pymalloc arena occupied.
10+
* Clearing the free lists may give back memory to the OS earlier.
11+
* Free-threading version: Since freelists are managed per thread,
12+
* GC should clear all freelists by traversing all threads.
13+
*/
14+
void
15+
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
16+
{
17+
_PyTuple_ClearFreeList(interp);
18+
_PyFloat_ClearFreeList(interp);
19+
_PyDict_ClearFreeList(interp);
20+
_PyAsyncGen_ClearFreeLists(interp);
21+
_PyContext_ClearFreeList(interp);
22+
23+
HEAD_LOCK(&_PyRuntime);
24+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;
25+
while (tstate != NULL) {
26+
_Py_ClearFreeLists(&tstate->freelist_state, 0);
27+
tstate = (_PyThreadStateImpl *)tstate->base.next;
28+
}
29+
HEAD_UNLOCK(&_PyRuntime);
30+
}
31+
32+
#endif

‎Python/gc_gil.c

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#include "Python.h"
2+
#include "pycore_pystate.h" // _Py_ClearFreeLists()
3+
4+
#ifndef Py_GIL_DISABLED
5+
6+
/* Clear all free lists
7+
* All free lists are cleared during the collection of the highest generation.
8+
* Allocated items in the free list may keep a pymalloc arena occupied.
9+
* Clearing the free lists may give back memory to the OS earlier.
10+
*/
11+
void
12+
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
13+
{
14+
_PyTuple_ClearFreeList(interp);
15+
_PyFloat_ClearFreeList(interp);
16+
_PyDict_ClearFreeList(interp);
17+
_PyAsyncGen_ClearFreeLists(interp);
18+
_PyContext_ClearFreeList(interp);
19+
20+
_Py_ClearFreeLists(&interp->freelist_state, 0);
21+
}
22+
23+
#endif

‎Python/pylifecycle.c

+4-1
Original file line numberDiff line numberDiff line change
@@ -1752,13 +1752,16 @@ finalize_interp_types(PyInterpreterState *interp)
17521752
_PyUnicode_ClearInterned(interp);
17531753

17541754
_PyDict_Fini(interp);
1755-
_PyList_Fini(interp);
17561755
_PyTuple_Fini(interp);
17571756

17581757
_PySlice_Fini(interp);
17591758

17601759
_PyUnicode_Fini(interp);
17611760
_PyFloat_Fini(interp);
1761+
1762+
_PyFreeListState *state = _PyFreeListState_GET();
1763+
_PyList_Fini(state);
1764+
17621765
#ifdef Py_DEBUG
17631766
_PyStaticObjects_CheckRefcnt(interp);
17641767
#endif

0 commit comments

Comments
 (0)