Skip to content

Commit 5fc6bb2

Browse files
eendebakpteendebakpt
authored
gh-126868: Add freelist for compact int objects (GH-126865)
1 parent 9b4bbf4 commit 5fc6bb2

File tree

8 files changed

+102
-55
lines changed

8 files changed

+102
-55
lines changed

‎Include/internal/pycore_freelist_state.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ extern "C" {
1414
# define Py_dicts_MAXFREELIST 80
1515
# define Py_dictkeys_MAXFREELIST 80
1616
# define Py_floats_MAXFREELIST 100
17+
# define Py_ints_MAXFREELIST 100
1718
# define Py_slices_MAXFREELIST 1
1819
# define Py_contexts_MAXFREELIST 255
1920
# define Py_async_gens_MAXFREELIST 80
@@ -35,6 +36,7 @@ struct _Py_freelist {
3536

3637
struct _Py_freelists {
3738
struct _Py_freelist floats;
39+
struct _Py_freelist ints;
3840
struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
3941
struct _Py_freelist lists;
4042
struct _Py_freelist dicts;

‎Include/internal/pycore_long.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp);
5555

5656
/* other API */
5757

58+
PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self);
59+
5860
#define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints)
5961

6062
// _PyLong_GetZero() and _PyLong_GetOne() must always be available

‎Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Increase performance of :class:`int` by adding a freelist for compact ints.

‎Objects/longobject.c

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "pycore_bitutils.h" // _Py_popcount32()
77
#include "pycore_initconfig.h" // _PyStatus_OK()
88
#include "pycore_call.h" // _PyObject_MakeTpCall
9+
#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP
910
#include "pycore_long.h" // _Py_SmallInts
1011
#include "pycore_object.h" // _PyObject_Init()
1112
#include "pycore_runtime.h" // _PY_NSMALLPOSINTS
@@ -42,7 +43,7 @@ static inline void
4243
_Py_DECREF_INT(PyLongObject *op)
4344
{
4445
assert(PyLong_CheckExact(op));
45-
_Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free);
46+
_Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc);
4647
}
4748

4849
static inline int
@@ -220,15 +221,18 @@ _PyLong_FromMedium(sdigit x)
220221
{
221222
assert(!IS_SMALL_INT(x));
222223
assert(is_medium_int(x));
223-
/* We could use a freelist here */
224-
PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
224+
225+
PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints);
225226
if (v == NULL) {
226-
PyErr_NoMemory();
227-
return NULL;
227+
v = PyObject_Malloc(sizeof(PyLongObject));
228+
if (v == NULL) {
229+
PyErr_NoMemory();
230+
return NULL;
231+
}
232+
_PyObject_Init((PyObject*)v, &PyLong_Type);
228233
}
229234
digit abs_x = x < 0 ? -x : x;
230235
_PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
231-
_PyObject_Init((PyObject*)v, &PyLong_Type);
232236
v->long_value.ob_digit[0] = abs_x;
233237
return (PyObject*)v;
234238
}
@@ -3611,24 +3615,60 @@ long_richcompare(PyObject *self, PyObject *other, int op)
36113615
Py_RETURN_RICHCOMPARE(result, 0, op);
36123616
}
36133617

3618+
static inline int
3619+
compact_int_is_small(PyObject *self)
3620+
{
3621+
PyLongObject *pylong = (PyLongObject *)self;
3622+
assert(_PyLong_IsCompact(pylong));
3623+
stwodigits ival = medium_value(pylong);
3624+
if (IS_SMALL_INT(ival)) {
3625+
PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
3626+
if (pylong == small_pylong) {
3627+
return 1;
3628+
}
3629+
}
3630+
return 0;
3631+
}
3632+
3633+
void
3634+
_PyLong_ExactDealloc(PyObject *self)
3635+
{
3636+
assert(PyLong_CheckExact(self));
3637+
if (_PyLong_IsCompact((PyLongObject *)self)) {
3638+
#ifndef Py_GIL_DISABLED
3639+
if (compact_int_is_small(self)) {
3640+
// See PEP 683, section Accidental De-Immortalizing for details
3641+
_Py_SetImmortal(self);
3642+
return;
3643+
}
3644+
#endif
3645+
_Py_FREELIST_FREE(ints, self, PyObject_Free);
3646+
return;
3647+
}
3648+
PyObject_Free(self);
3649+
}
3650+
36143651
static void
36153652
long_dealloc(PyObject *self)
36163653
{
3617-
/* This should never get called, but we also don't want to SEGV if
3618-
* we accidentally decref small Ints out of existence. Instead,
3619-
* since small Ints are immortal, re-set the reference count.
3620-
*/
3621-
PyLongObject *pylong = (PyLongObject*)self;
3622-
if (pylong && _PyLong_IsCompact(pylong)) {
3623-
stwodigits ival = medium_value(pylong);
3624-
if (IS_SMALL_INT(ival)) {
3625-
PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
3626-
if (pylong == small_pylong) {
3627-
_Py_SetImmortal(self);
3628-
return;
3629-
}
3654+
assert(self);
3655+
if (_PyLong_IsCompact((PyLongObject *)self)) {
3656+
if (compact_int_is_small(self)) {
3657+
/* This should never get called, but we also don't want to SEGV if
3658+
* we accidentally decref small Ints out of existence. Instead,
3659+
* since small Ints are immortal, re-set the reference count.
3660+
*
3661+
* See PEP 683, section Accidental De-Immortalizing for details
3662+
*/
3663+
_Py_SetImmortal(self);
3664+
return;
3665+
}
3666+
if (PyLong_CheckExact(self)) {
3667+
_Py_FREELIST_FREE(ints, self, PyObject_Free);
3668+
return;
36303669
}
36313670
}
3671+
36323672
Py_TYPE(self)->tp_free(self);
36333673
}
36343674

‎Objects/object.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
936936
clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
937937
}
938938
clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free);
939+
clear_freelist(&freelists->ints, is_finalization, free_object);
939940
}
940941

941942
/*

‎Python/bytecodes.c

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
2727
#include "pycore_pystate.h" // _PyInterpreterState_GET()
2828
#include "pycore_range.h" // _PyRangeIterObject
29+
#include "pycore_long.h" // _PyLong_ExactDealloc()
2930
#include "pycore_setobject.h" // _PySet_NextEntry()
3031
#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
3132
#include "pycore_tuple.h" // _PyTuple_ITEMS()
@@ -514,8 +515,8 @@ dummy_func(
514515

515516
STAT_INC(BINARY_OP, hit);
516517
PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
517-
PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
518-
PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
518+
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
519+
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
519520
INPUTS_DEAD();
520521
ERROR_IF(res_o == NULL, error);
521522
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -527,8 +528,8 @@ dummy_func(
527528

528529
STAT_INC(BINARY_OP, hit);
529530
PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
530-
PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
531-
PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
531+
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
532+
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
532533
INPUTS_DEAD();
533534
ERROR_IF(res_o == NULL, error);
534535
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -540,8 +541,8 @@ dummy_func(
540541

541542
STAT_INC(BINARY_OP, hit);
542543
PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
543-
PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
544-
PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
544+
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
545+
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
545546
INPUTS_DEAD();
546547
ERROR_IF(res_o == NULL, error);
547548
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -801,7 +802,7 @@ dummy_func(
801802
assert(res_o != NULL);
802803
Py_INCREF(res_o);
803804
#endif
804-
PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
805+
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
805806
DEAD(sub_st);
806807
PyStackRef_CLOSE(list_st);
807808
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -821,7 +822,7 @@ dummy_func(
821822
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
822823
STAT_INC(BINARY_SUBSCR, hit);
823824
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
824-
PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
825+
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
825826
DEAD(sub_st);
826827
PyStackRef_CLOSE(str_st);
827828
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -842,7 +843,7 @@ dummy_func(
842843
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
843844
assert(res_o != NULL);
844845
Py_INCREF(res_o);
845-
PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
846+
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
846847
DEAD(sub_st);
847848
PyStackRef_CLOSE(tuple_st);
848849
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -959,7 +960,7 @@ dummy_func(
959960
assert(old_value != NULL);
960961
UNLOCK_OBJECT(list); // unlock before decrefs!
961962
Py_DECREF(old_value);
962-
PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
963+
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
963964
DEAD(sub_st);
964965
PyStackRef_CLOSE(list_st);
965966
}
@@ -2476,9 +2477,9 @@ dummy_func(
24762477
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
24772478
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
24782479
int sign_ish = COMPARISON_BIT(ileft, iright);
2479-
PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
2480+
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
24802481
DEAD(left);
2481-
PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
2482+
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
24822483
DEAD(right);
24832484
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
24842485
// It's always a bool, so we don't care about oparg & 16.

‎Python/executor_cases.c.h

Lines changed: 12 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/generated_cases.c.h

Lines changed: 12 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)