Skip to content

Commit da6c785

Browse files
sweeneydesweeneyde
authored
gh-90667: Add specializations of Py_DECREF when types are known (GH-30872)
1 parent ab0d35d commit da6c785

13 files changed

+146
-84
lines changed

‎Include/internal/pycore_floatobject.h

+2
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ struct _Py_float_state {
3838
#endif
3939
};
4040

41+
void _PyFloat_ExactDealloc(PyObject *op);
42+
4143

4244
PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out);
4345

‎Include/internal/pycore_object.h

+36-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ extern "C" {
1414
#include "pycore_pystate.h" // _PyInterpreterState_GET()
1515
#include "pycore_runtime.h" // _PyRuntime
1616

17-
1817
#define _PyObject_IMMORTAL_INIT(type) \
1918
{ \
2019
.ob_refcnt = 999999999, \
@@ -26,6 +25,42 @@ extern "C" {
2625
.ob_size = size, \
2726
}
2827

28+
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
29+
const char *func,
30+
const char *message);
31+
32+
#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message)
33+
34+
static inline void
35+
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
36+
{
37+
#ifdef Py_REF_DEBUG
38+
_Py_RefTotal--;
39+
#endif
40+
if (--op->ob_refcnt != 0) {
41+
assert(op->ob_refcnt > 0);
42+
}
43+
else {
44+
#ifdef Py_TRACE_REFS
45+
_Py_ForgetReference(op);
46+
#endif
47+
destruct(op);
48+
}
49+
}
50+
51+
static inline void
52+
_Py_DECREF_NO_DEALLOC(PyObject *op)
53+
{
54+
#ifdef Py_REF_DEBUG
55+
_Py_RefTotal--;
56+
#endif
57+
op->ob_refcnt--;
58+
#ifdef Py_DEBUG
59+
if (op->ob_refcnt <= 0) {
60+
_Py_FatalRefcountError("Expected a positive remaining refcount");
61+
}
62+
#endif
63+
}
2964

3065
PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
3166
PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);

‎Include/internal/pycore_pyerrors.h

-7
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,6 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception);
100100
PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b,
101101
Py_ssize_t max_cost);
102102

103-
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
104-
const char *func,
105-
const char *message);
106-
107-
#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message)
108-
109-
110103
#ifdef __cplusplus
111104
}
112105
#endif

‎Include/internal/pycore_unicodeobject.h

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ extern "C" {
1010

1111
#include "pycore_fileutils.h" // _Py_error_handler
1212

13+
void _PyUnicode_ExactDealloc(PyObject *op);
1314

1415
/* runtime lifecycle */
1516

‎Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time.

‎Objects/boolobject.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
/* Boolean type, a subtype of int */
22

33
#include "Python.h"
4+
#include "pycore_object.h" // _Py_FatalRefcountError()
45
#include "pycore_runtime.h" // _Py_ID()
5-
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
66

77
/* We define bool_repr to return "False" or "True" */
88

‎Objects/floatobject.c

+27-14
Original file line numberDiff line numberDiff line change
@@ -238,28 +238,41 @@ PyFloat_FromString(PyObject *v)
238238
return result;
239239
}
240240

241-
static void
242-
float_dealloc(PyFloatObject *op)
241+
void
242+
_PyFloat_ExactDealloc(PyObject *obj)
243243
{
244+
assert(PyFloat_CheckExact(obj));
245+
PyFloatObject *op = (PyFloatObject *)obj;
244246
#if PyFloat_MAXFREELIST > 0
245-
if (PyFloat_CheckExact(op)) {
246-
struct _Py_float_state *state = get_float_state();
247+
struct _Py_float_state *state = get_float_state();
247248
#ifdef Py_DEBUG
248-
// float_dealloc() must not be called after _PyFloat_Fini()
249-
assert(state->numfree != -1);
249+
// float_dealloc() must not be called after _PyFloat_Fini()
250+
assert(state->numfree != -1);
250251
#endif
251-
if (state->numfree >= PyFloat_MAXFREELIST) {
252-
PyObject_Free(op);
253-
return;
254-
}
255-
state->numfree++;
256-
Py_SET_TYPE(op, (PyTypeObject *)state->free_list);
257-
state->free_list = op;
252+
if (state->numfree >= PyFloat_MAXFREELIST) {
253+
PyObject_Free(op);
254+
return;
255+
}
256+
state->numfree++;
257+
Py_SET_TYPE(op, (PyTypeObject *)state->free_list);
258+
state->free_list = op;
259+
#else
260+
PyObject_Free(op);
261+
#endif
262+
}
263+
264+
static void
265+
float_dealloc(PyObject *op)
266+
{
267+
assert(PyFloat_Check(op));
268+
#if PyFloat_MAXFREELIST > 0
269+
if (PyFloat_CheckExact(op)) {
270+
_PyFloat_ExactDealloc(op);
258271
}
259272
else
260273
#endif
261274
{
262-
Py_TYPE(op)->tp_free((PyObject *)op);
275+
Py_TYPE(op)->tp_free(op);
263276
}
264277
}
265278

‎Objects/longobject.c

+23-15
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,15 @@ medium_value(PyLongObject *x)
3636
#define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS)
3737
#define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS)
3838

39-
static inline int is_medium_int(stwodigits x)
39+
static inline void
40+
_Py_DECREF_INT(PyLongObject *op)
41+
{
42+
assert(PyLong_CheckExact(op));
43+
_Py_DECREF_SPECIALIZED((PyObject *)op, PyObject_Free);
44+
}
45+
46+
static inline int
47+
is_medium_int(stwodigits x)
4048
{
4149
/* Take care that we are comparing unsigned values. */
4250
twodigits x_plus_mask = ((twodigits)x) + PyLong_MASK;
@@ -58,7 +66,7 @@ maybe_small_long(PyLongObject *v)
5866
if (v && IS_MEDIUM_VALUE(v)) {
5967
stwodigits ival = medium_value(v);
6068
if (IS_SMALL_INT(ival)) {
61-
Py_DECREF(v);
69+
_Py_DECREF_INT(v);
6270
return (PyLongObject *)get_small_int((sdigit)ival);
6371
}
6472
}
@@ -1856,7 +1864,7 @@ long_to_decimal_string_internal(PyObject *aa,
18561864
#undef WRITE_DIGITS
18571865
#undef WRITE_UNICODE_DIGITS
18581866

1859-
Py_DECREF(scratch);
1867+
_Py_DECREF_INT(scratch);
18601868
if (writer) {
18611869
writer->pos += strlen;
18621870
}
@@ -3561,15 +3569,15 @@ k_mul(PyLongObject *a, PyLongObject *b)
35613569
*/
35623570
i = Py_SIZE(ret) - shift; /* # digits after shift */
35633571
(void)v_isub(ret->ob_digit + shift, i, t2->ob_digit, Py_SIZE(t2));
3564-
Py_DECREF(t2);
3572+
_Py_DECREF_INT(t2);
35653573

35663574
(void)v_isub(ret->ob_digit + shift, i, t1->ob_digit, Py_SIZE(t1));
3567-
Py_DECREF(t1);
3575+
_Py_DECREF_INT(t1);
35683576

35693577
/* 6. t3 <- (ah+al)(bh+bl), and add into result. */
35703578
if ((t1 = x_add(ah, al)) == NULL) goto fail;
3571-
Py_DECREF(ah);
3572-
Py_DECREF(al);
3579+
_Py_DECREF_INT(ah);
3580+
_Py_DECREF_INT(al);
35733581
ah = al = NULL;
35743582

35753583
if (a == b) {
@@ -3580,21 +3588,21 @@ k_mul(PyLongObject *a, PyLongObject *b)
35803588
Py_DECREF(t1);
35813589
goto fail;
35823590
}
3583-
Py_DECREF(bh);
3584-
Py_DECREF(bl);
3591+
_Py_DECREF_INT(bh);
3592+
_Py_DECREF_INT(bl);
35853593
bh = bl = NULL;
35863594

35873595
t3 = k_mul(t1, t2);
3588-
Py_DECREF(t1);
3589-
Py_DECREF(t2);
3596+
_Py_DECREF_INT(t1);
3597+
_Py_DECREF_INT(t2);
35903598
if (t3 == NULL) goto fail;
35913599
assert(Py_SIZE(t3) >= 0);
35923600

35933601
/* Add t3. It's not obvious why we can't run out of room here.
35943602
* See the (*) comment after this function.
35953603
*/
35963604
(void)v_iadd(ret->ob_digit + shift, i, t3->ob_digit, Py_SIZE(t3));
3597-
Py_DECREF(t3);
3605+
_Py_DECREF_INT(t3);
35983606

35993607
return long_normalize(ret);
36003608

@@ -3699,13 +3707,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b)
36993707
/* Add into result. */
37003708
(void)v_iadd(ret->ob_digit + nbdone, Py_SIZE(ret) - nbdone,
37013709
product->ob_digit, Py_SIZE(product));
3702-
Py_DECREF(product);
3710+
_Py_DECREF_INT(product);
37033711

37043712
bsize -= nbtouse;
37053713
nbdone += nbtouse;
37063714
}
37073715

3708-
Py_DECREF(bslice);
3716+
_Py_DECREF_INT(bslice);
37093717
return long_normalize(ret);
37103718

37113719
fail:
@@ -5993,7 +6001,7 @@ PyTypeObject PyLong_Type = {
59936001
0, /* tp_init */
59946002
0, /* tp_alloc */
59956003
long_new, /* tp_new */
5996-
PyObject_Del, /* tp_free */
6004+
PyObject_Free, /* tp_free */
59976005
};
59986006

59996007
static PyTypeObject Int_InfoType;

‎Objects/object.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
1010
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
1111
#include "pycore_namespace.h" // _PyNamespace_Type
12-
#include "pycore_object.h" // _PyType_CheckConsistency()
12+
#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError()
1313
#include "pycore_pyerrors.h" // _PyErr_Occurred()
1414
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
1515
#include "pycore_pystate.h" // _PyThreadState_GET()

‎Objects/tupleobject.c

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@
55
#include "pycore_abstract.h" // _PyIndex_Check()
66
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
77
#include "pycore_initconfig.h" // _PyStatus_OK()
8-
#include "pycore_object.h" // _PyObject_GC_TRACK()
9-
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
8+
#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError()
109

1110
/*[clinic input]
1211
class tuple "PyTupleObject *" "&PyTuple_Type"

‎Objects/unicodeobject.c

+8-2
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,8 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
4848
#include "pycore_initconfig.h" // _PyStatus_OK()
4949
#include "pycore_interp.h" // PyInterpreterState.fs_codec
5050
#include "pycore_long.h" // _PyLong_FormatWriter()
51-
#include "pycore_object.h" // _PyObject_GC_TRACK()
51+
#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError()
5252
#include "pycore_pathconfig.h" // _Py_DumpPathConfig()
53-
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
5453
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
5554
#include "pycore_pystate.h" // _PyInterpreterState_GET()
5655
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
@@ -15369,6 +15368,13 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode)
1536915368
return NULL;
1537015369
}
1537115370

15371+
void
15372+
_PyUnicode_ExactDealloc(PyObject *op)
15373+
{
15374+
assert(PyUnicode_CheckExact(op));
15375+
unicode_dealloc(op);
15376+
}
15377+
1537215378
PyDoc_STRVAR(unicode_doc,
1537315379
"str(object='') -> str\n\
1537415380
str(bytes_or_buffer[, encoding[, errors]]) -> str\n\

‎Python/bltinmodule.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -2511,7 +2511,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start)
25112511
}
25122512
if (PyFloat_CheckExact(item)) {
25132513
f_result += PyFloat_AS_DOUBLE(item);
2514-
Py_DECREF(item);
2514+
_Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc);
25152515
continue;
25162516
}
25172517
if (PyLong_Check(item)) {

0 commit comments

Comments
 (0)