Skip to content

Commit b482538

Browse files
colesburycolesbury
authored
gh-124218: Refactor per-thread reference counting (#124844)
Currently, we only use per-thread reference counting for heap type objects and the naming reflects that. We will extend it to a few additional types in an upcoming change to avoid scaling bottlenecks when creating nested functions. Rename some of the files and functions in preparation for this change.
1 parent 5aa91c5 commit b482538

15 files changed

+168
-167
lines changed

‎Include/internal/pycore_interp.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ extern "C" {
3535
#include "pycore_qsbr.h" // struct _qsbr_state
3636
#include "pycore_tstate.h" // _PyThreadStateImpl
3737
#include "pycore_tuple.h" // struct _Py_tuple_state
38-
#include "pycore_typeid.h" // struct _Py_type_id_pool
38+
#include "pycore_uniqueid.h" // struct _Py_unique_id_pool
3939
#include "pycore_typeobject.h" // struct types_state
4040
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
4141
#include "pycore_warnings.h" // struct _warnings_runtime_state
@@ -221,7 +221,7 @@ struct _is {
221221
#if defined(Py_GIL_DISABLED)
222222
struct _mimalloc_interp_state mimalloc;
223223
struct _brc_state brc; // biased reference counting state
224-
struct _Py_type_id_pool type_ids;
224+
struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts
225225
PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
226226
#endif
227227

‎Include/internal/pycore_object.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ extern "C" {
1414
#include "pycore_interp.h" // PyInterpreterState.gc
1515
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
1616
#include "pycore_pystate.h" // _PyInterpreterState_GET()
17-
#include "pycore_typeid.h" // _PyType_IncrefSlow
17+
#include "pycore_uniqueid.h" // _PyType_IncrefSlow
1818

1919

2020
#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
@@ -335,12 +335,12 @@ _Py_INCREF_TYPE(PyTypeObject *type)
335335
// Unsigned comparison so that `unique_id=-1`, which indicates that
336336
// per-thread refcounting has been disabled on this type, is handled by
337337
// the "else".
338-
if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
338+
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
339339
# ifdef Py_REF_DEBUG
340340
_Py_INCREF_IncRefTotal();
341341
# endif
342342
_Py_INCREF_STAT_INC();
343-
tstate->types.refcounts[ht->unique_id]++;
343+
tstate->refcounts.values[ht->unique_id]++;
344344
}
345345
else {
346346
// The slow path resizes the thread-local refcount array if necessary.
@@ -368,12 +368,12 @@ _Py_DECREF_TYPE(PyTypeObject *type)
368368
// Unsigned comparison so that `unique_id=-1`, which indicates that
369369
// per-thread refcounting has been disabled on this type, is handled by
370370
// the "else".
371-
if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
371+
if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
372372
# ifdef Py_REF_DEBUG
373373
_Py_DECREF_DecRefTotal();
374374
# endif
375375
_Py_DECREF_STAT_INC();
376-
tstate->types.refcounts[ht->unique_id]--;
376+
tstate->refcounts.values[ht->unique_id]--;
377377
}
378378
else {
379379
// Directly decref the type if the type id is not assigned or if

‎Include/internal/pycore_tstate.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl {
3232
struct _Py_freelists freelists;
3333
struct _brc_thread_state brc;
3434
struct {
35-
// The thread-local refcounts for heap type objects
36-
Py_ssize_t *refcounts;
35+
// The per-thread refcounts
36+
Py_ssize_t *values;
3737

3838
// Size of the refcounts array.
3939
Py_ssize_t size;
4040

41-
// If set, don't use thread-local refcounts
41+
// If set, don't use per-thread refcounts
4242
int is_finalized;
43-
} types;
43+
} refcounts;
4444
#endif
4545

4646
#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)

‎Include/internal/pycore_typeid.h

Lines changed: 0 additions & 75 deletions
This file was deleted.

‎Include/internal/pycore_uniqueid.h

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#ifndef Py_INTERNAL_UNIQUEID_H
2+
#define Py_INTERNAL_UNIQUEID_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
#ifdef Py_GIL_DISABLED
12+
13+
// This contains code for allocating unique ids to objects for per-thread
14+
// reference counting.
15+
//
16+
// Per-thread reference counting is used along with deferred reference
17+
// counting to avoid scaling bottlenecks due to reference count contention.
18+
//
19+
// An id of -1 is used to indicate that an object doesn't use per-thread
20+
// refcounting. This value is used when the object is finalized by the GC
21+
// and during interpreter shutdown to allow the object to be
22+
// deallocated promptly when the object's refcount reaches zero.
23+
//
24+
// Each entry implicitly represents a unique id based on its offset in the
25+
// table. Non-allocated entries form a free-list via the 'next' pointer.
26+
// Allocated entries store the corresponding PyObject.
27+
typedef union _Py_unique_id_entry {
28+
// Points to the next free type id, when part of the freelist
29+
union _Py_unique_id_entry *next;
30+
31+
// Stores the object when the id is assigned
32+
PyObject *obj;
33+
} _Py_unique_id_entry;
34+
35+
struct _Py_unique_id_pool {
36+
PyMutex mutex;
37+
38+
// combined table of object with allocated unique ids and unallocated ids.
39+
_Py_unique_id_entry *table;
40+
41+
// Next entry to allocate inside 'table' or NULL
42+
_Py_unique_id_entry *freelist;
43+
44+
// size of 'table'
45+
Py_ssize_t size;
46+
};
47+
48+
// Assigns the next id from the pool of ids.
49+
extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
50+
51+
// Releases the allocated id back to the pool.
52+
extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id);
53+
54+
// Merges the per-thread reference counts into the corresponding objects.
55+
extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate);
56+
57+
// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread
58+
// array of refcounts.
59+
extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate);
60+
61+
// Frees the interpreter's pool of type ids.
62+
extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp);
63+
64+
// Increfs the type, resizing the per-thread refcount array if necessary.
65+
PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
66+
67+
#endif /* Py_GIL_DISABLED */
68+
69+
#ifdef __cplusplus
70+
}
71+
#endif
72+
#endif /* !Py_INTERNAL_UNIQUEID_H */

‎Makefile.pre.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@ PYTHON_OBJS= \
490490
Python/thread.o \
491491
Python/traceback.o \
492492
Python/tracemalloc.o \
493-
Python/typeid.o \
493+
Python/uniqueid.o \
494494
Python/getopt.o \
495495
Python/pystrcmp.o \
496496
Python/pystrtod.o \
@@ -1279,7 +1279,7 @@ PYTHON_HEADERS= \
12791279
$(srcdir)/Include/internal/pycore_tracemalloc.h \
12801280
$(srcdir)/Include/internal/pycore_tstate.h \
12811281
$(srcdir)/Include/internal/pycore_tuple.h \
1282-
$(srcdir)/Include/internal/pycore_typeid.h \
1282+
$(srcdir)/Include/internal/pycore_uniqueid.h \
12831283
$(srcdir)/Include/internal/pycore_typeobject.h \
12841284
$(srcdir)/Include/internal/pycore_typevarobject.h \
12851285
$(srcdir)/Include/internal/pycore_ucnhash.h \

‎Objects/typeobject.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3932,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx)
39323932
et->ht_token = NULL;
39333933

39343934
#ifdef Py_GIL_DISABLED
3935-
_PyType_AssignId(et);
3935+
et->unique_id = _PyObject_AssignUniqueId((PyObject *)et);
39363936
#endif
39373937

39383938
return type;
@@ -5026,7 +5026,7 @@ PyType_FromMetaclass(
50265026

50275027
#ifdef Py_GIL_DISABLED
50285028
// Assign a type id to enable thread-local refcounting
5029-
_PyType_AssignId(res);
5029+
res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
50305030
#endif
50315031

50325032
/* Ready the type (which includes inheritance).
@@ -6080,7 +6080,7 @@ type_dealloc(PyObject *self)
60806080
Py_XDECREF(et->ht_module);
60816081
PyMem_Free(et->_ht_tpname);
60826082
#ifdef Py_GIL_DISABLED
6083-
_PyType_ReleaseId(et);
6083+
_PyObject_ReleaseUniqueId(et->unique_id);
60846084
#endif
60856085
et->ht_token = NULL;
60866086
Py_TYPE(type)->tp_free((PyObject *)type);

‎PCbuild/_freeze_module.vcxproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@
268268
<ClCompile Include="..\Python\thread.c" />
269269
<ClCompile Include="..\Python\traceback.c" />
270270
<ClCompile Include="..\Python\tracemalloc.c" />
271-
<ClCompile Include="..\Python\typeid.c" />
271+
<ClCompile Include="..\Python\uniqueid.c" />
272272
</ItemGroup>
273273
<ItemGroup>
274274
<ClInclude Include="..\PC\pyconfig.h.in" />

‎PCbuild/_freeze_module.vcxproj.filters

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@
467467
<ClCompile Include="..\Python\tracemalloc.c">
468468
<Filter>Source Files</Filter>
469469
</ClCompile>
470-
<ClCompile Include="..\Python\typeid.c">
470+
<ClCompile Include="..\Python\uniqueid.c">
471471
<Filter>Source Files</Filter>
472472
</ClCompile>
473473
<ClCompile Include="..\Objects\tupleobject.c">

‎PCbuild/pythoncore.vcxproj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,13 +304,13 @@
304304
<ClInclude Include="..\Include\internal\pycore_tracemalloc.h" />
305305
<ClInclude Include="..\Include\internal\pycore_tstate.h" />
306306
<ClInclude Include="..\Include\internal\pycore_tuple.h" />
307-
<ClInclude Include="..\Include\internal\pycore_typeid.h" />
308307
<ClInclude Include="..\Include\internal\pycore_typeobject.h" />
309308
<ClInclude Include="..\Include\internal\pycore_typevarobject.h" />
310309
<ClInclude Include="..\Include\internal\pycore_ucnhash.h" />
311310
<ClInclude Include="..\Include\internal\pycore_unionobject.h" />
312311
<ClInclude Include="..\Include\internal\pycore_unicodeobject.h" />
313312
<ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
313+
<ClInclude Include="..\Include\internal\pycore_uniqueid.h" />
314314
<ClInclude Include="..\Include\internal\pycore_warnings.h" />
315315
<ClInclude Include="..\Include\internal\pycore_weakref.h" />
316316
<ClInclude Include="..\Include\intrcheck.h" />
@@ -657,7 +657,7 @@
657657
<ClCompile Include="..\Python\thread.c" />
658658
<ClCompile Include="..\Python\traceback.c" />
659659
<ClCompile Include="..\Python\tracemalloc.c" />
660-
<ClCompile Include="..\Python\typeid.c" />
660+
<ClCompile Include="..\Python\uniqueid.c" />
661661
</ItemGroup>
662662
<ItemGroup Condition="$(IncludeExternals)">
663663
<ClCompile Include="..\Modules\zlibmodule.c" />

‎PCbuild/pythoncore.vcxproj.filters

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -831,9 +831,6 @@
831831
<ClInclude Include="..\Include\internal\pycore_tuple.h">
832832
<Filter>Include\internal</Filter>
833833
</ClInclude>
834-
<ClInclude Include="..\Include\internal\pycore_typeid.h">
835-
<Filter>Include\internal</Filter>
836-
</ClInclude>
837834
<ClInclude Include="..\Include\internal\pycore_typeobject.h">
838835
<Filter>Include\internal</Filter>
839836
</ClInclude>
@@ -846,6 +843,9 @@
846843
<ClInclude Include="..\Include\internal\pycore_unionobject.h">
847844
<Filter>Include\internal</Filter>
848845
</ClInclude>
846+
<ClInclude Include="..\Include\internal\pycore_uniqueid.h">
847+
<Filter>Include\internal</Filter>
848+
</ClInclude>
849849
<ClInclude Include="..\Include\internal\mimalloc\mimalloc.h">
850850
<Filter>Include\internal\mimalloc</Filter>
851851
</ClInclude>
@@ -1499,7 +1499,7 @@
14991499
<ClCompile Include="..\Python\tracemalloc.c">
15001500
<Filter>Python</Filter>
15011501
</ClCompile>
1502-
<ClCompile Include="..\Python\typeid.c">
1502+
<ClCompile Include="..\Python\uniqueid.c">
15031503
<Filter>Python</Filter>
15041504
</ClCompile>
15051505
<ClCompile Include="..\Python\bootstrap_hash.c">

‎Python/gc_free_threading.c

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
#include "pycore_tstate.h" // _PyThreadStateImpl
1616
#include "pycore_weakref.h" // _PyWeakref_ClearRef()
1717
#include "pydtrace.h"
18-
#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts
18+
#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts
1919

2020
#ifdef Py_GIL_DISABLED
2121

@@ -217,12 +217,12 @@ disable_deferred_refcounting(PyObject *op)
217217
merge_refcount(op, 0);
218218
}
219219

220-
// Heap types also use thread-local refcounting -- disable it here.
220+
// Heap types also use per-thread refcounting -- disable it here.
221221
if (PyType_Check(op)) {
222-
// Disable thread-local refcounting for heap types
223-
PyTypeObject *type = (PyTypeObject *)op;
224-
if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
225-
_PyType_ReleaseId((PyHeapTypeObject *)op);
222+
if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
223+
PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
224+
_PyObject_ReleaseUniqueId(ht->unique_id);
225+
ht->unique_id = -1;
226226
}
227227
}
228228

@@ -1221,7 +1221,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
12211221
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p;
12221222

12231223
// merge per-thread refcount for types into the type's actual refcount
1224-
_PyType_MergeThreadLocalRefcounts(tstate);
1224+
_PyObject_MergePerThreadRefcounts(tstate);
12251225

12261226
// merge refcounts for all queued objects
12271227
merge_queued_objects(tstate, state);

‎Python/pylifecycle.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
#include "pycore_sliceobject.h" // _PySlice_Fini()
2929
#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
3030
#include "pycore_traceback.h" // _Py_DumpTracebackThreads()
31-
#include "pycore_typeid.h" // _PyType_FinalizeIdPool()
31+
#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool()
3232
#include "pycore_typeobject.h" // _PyTypes_InitTypes()
3333
#include "pycore_typevarobject.h" // _Py_clear_generic_types()
3434
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
@@ -1834,7 +1834,7 @@ finalize_interp_types(PyInterpreterState *interp)
18341834

18351835
_PyTypes_Fini(interp);
18361836
#ifdef Py_GIL_DISABLED
1837-
_PyType_FinalizeIdPool(interp);
1837+
_PyObject_FinalizeUniqueIdPool(interp);
18381838
#endif
18391839

18401840
_PyCode_Fini(interp);

0 commit comments

Comments
 (0)