From 0496190ddab7e2579c1deb042b363bfb5753f0d9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 21 May 2021 18:17:59 +0100 Subject: [PATCH 01/10] Store log2(size) instead of size in dict-keys. --- Objects/dict-common.h | 2 +- Objects/dictobject.c | 129 +++++++++++++++++++++-------------------- Objects/odictobject.c | 4 +- Tools/gdb/libpython.py | 2 +- 4 files changed, 69 insertions(+), 68 deletions(-) diff --git a/Objects/dict-common.h b/Objects/dict-common.h index 71d6b0274420b8..28f540c2f007d9 100644 --- a/Objects/dict-common.h +++ b/Objects/dict-common.h @@ -23,7 +23,7 @@ struct _dictkeysobject { Py_ssize_t dk_refcnt; /* Size of the hash table (dk_indices). It must be a power of 2. */ - Py_ssize_t dk_size; + uint8_t dk_log2_size; /* Function to lookup in the hash table (dk_indices): diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 24973c07b1d488..72f23fb39ab14e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -108,6 +108,7 @@ converting the dict to the combined table. * Making this 8, rather than 4 reduces the number of resizes for most * dictionaries, without any significant extra memory use. */ +#define PyDict_LOG_MINSIZE 3 #define PyDict_MINSIZE 8 #include "Python.h" @@ -237,7 +238,7 @@ lookdict_unicode_nodummy(PyDictObject *mp, PyObject *key, static Py_ssize_t lookdict_split(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); -static int dictresize(PyDictObject *mp, Py_ssize_t newsize); +static int dictresize(PyDictObject *mp, uint8_t log_newsize); static PyObject* dict_iter(PyDictObject *dict); @@ -295,24 +296,25 @@ _PyDict_DebugMallocStats(FILE *out) state->numfree, sizeof(PyDictObject)); } - -#define DK_SIZE(dk) ((dk)->dk_size) +#define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 -#define DK_IXSIZE(dk) \ - (DK_SIZE(dk) <= 0xff ? \ - 1 : DK_SIZE(dk) <= 0xffff ? \ - 2 : DK_SIZE(dk) <= 0xffffffff ? \ +#define DK_SIZE(dk) (((int64_t)1)<dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)])) -#define DK_MASK(dk) (((dk)->dk_size)-1) +#define DK_MASK(dk) (DK_SIZE(dk)-1) #define IS_POWER_OF_2(x) (((x) & (x-1)) == 0) static void free_keys_object(PyDictKeysObject *keys); @@ -413,25 +415,25 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) #define USABLE_FRACTION(n) (((n) << 1)/3) /* Find the smallest dk_size >= minsize. */ -static inline Py_ssize_t -calculate_keysize(Py_ssize_t minsize) +static inline uint8_t +calculate_log2_keysize(Py_ssize_t minsize) { #if SIZEOF_LONG == SIZEOF_SIZE_T minsize = (minsize | PyDict_MINSIZE) - 1; - return 1LL << _Py_bit_length(minsize | (PyDict_MINSIZE-1)); + return _Py_bit_length(minsize | (PyDict_MINSIZE-1)); #elif defined(_MSC_VER) // On 64bit Windows, sizeof(long) == 4. minsize = (minsize | PyDict_MINSIZE) - 1; unsigned long msb; _BitScanReverse64(&msb, (uint64_t)minsize); - return 1LL << (msb + 1); + return msb + 1; #else - Py_ssize_t size; - for (size = PyDict_MINSIZE; - size < minsize && size > 0; - size <<= 1) + uint8_t log2_size; + for (log2_size = PyDict_LOG_MINSIZE; + (((Py_ssize_t)1) << log2_size) < minsize; + log2_size++) ; - return size; + return log2_size; #endif } @@ -440,10 +442,10 @@ calculate_keysize(Py_ssize_t minsize) * This can be used to reserve enough size to insert n entries without * resizing. */ -static inline Py_ssize_t -estimate_keysize(Py_ssize_t n) +static inline uint8_t +estimate_log2_keysize(Py_ssize_t n) { - return calculate_keysize((n*3 + 1) / 2); + return calculate_log2_keysize((n*3 + 1) / 2); } @@ -469,7 +471,7 @@ estimate_keysize(Py_ssize_t n) */ static PyDictKeysObject empty_keys_struct = { 1, /* dk_refcnt */ - 1, /* dk_size */ + 0, /* dk_log2_size */ lookdict_split, /* dk_lookup */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ @@ -503,10 +505,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) PyDictKeysObject *keys = mp->ma_keys; int splitted = _PyDict_HasSplitTable(mp); - Py_ssize_t usable = USABLE_FRACTION(keys->dk_size); + Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(IS_POWER_OF_2(keys->dk_size)); + CHECK(IS_POWER_OF_2(DK_SIZE(keys))); CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); CHECK(keys->dk_usable + keys->dk_nentries <= usable); @@ -520,7 +522,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) PyDictKeyEntry *entries = DK_ENTRIES(keys); Py_ssize_t i; - for (i=0; i < keys->dk_size; i++) { + for (i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); } @@ -563,23 +565,22 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(Py_ssize_t size) +new_keys_object(uint8_t log2_size) { PyDictKeysObject *dk; Py_ssize_t es, usable; - assert(size >= PyDict_MINSIZE); - assert(IS_POWER_OF_2(size)); + assert(log2_size >= PyDict_LOG_MINSIZE); - usable = USABLE_FRACTION(size); - if (size <= 0xff) { + usable = USABLE_FRACTION(1< 4 - else if (size <= 0xffffffff) { + else if (log2_size <= 31) { es = 4; } #endif @@ -592,13 +593,13 @@ new_keys_object(Py_ssize_t size) // new_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); #endif - if (size == PyDict_MINSIZE && state->keys_numfree > 0) { + if (log2_size == PyDict_LOG_MINSIZE && state->keys_numfree > 0) { dk = state->keys_free_list[--state->keys_numfree]; } else { dk = PyObject_Malloc(sizeof(PyDictKeysObject) - + es * size + + (es<dk_refcnt = 1; - dk->dk_size = size; + dk->dk_log2_size = log2_size; dk->dk_usable = usable; dk->dk_lookup = lookdict_unicode_nodummy; dk->dk_nentries = 0; - memset(&dk->dk_indices[0], 0xff, es * size); + memset(&dk->dk_indices[0], 0xff, es * (1<keys_numfree != -1); #endif - if (keys->dk_size == PyDict_MINSIZE && state->keys_numfree < PyDict_MAXFREELIST) { + if (DK_SIZE(keys) == PyDict_MINSIZE && state->keys_numfree < PyDict_MAXFREELIST) { state->keys_free_list[state->keys_numfree++] = keys; return; } @@ -1057,7 +1058,7 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) static int insertion_resize(PyDictObject *mp) { - return dictresize(mp, calculate_keysize(GROWTH_RATE(mp))); + return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp))); } /* @@ -1160,7 +1161,7 @@ insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, { assert(mp->ma_keys == Py_EMPTY_KEYS); - PyDictKeysObject *newkeys = new_keys_object(PyDict_MINSIZE); + PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE); if (newkeys == NULL) { return -1; } @@ -1217,19 +1218,18 @@ After resizing a table is always combined, but can be resplit by make_keys_shared(). */ static int -dictresize(PyDictObject *mp, Py_ssize_t newsize) +dictresize(PyDictObject *mp, uint8_t log2_newsize) { Py_ssize_t numentries; PyDictKeysObject *oldkeys; PyObject **oldvalues; PyDictKeyEntry *oldentries, *newentries; - if (newsize <= 0) { + if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; } - assert(IS_POWER_OF_2(newsize)); - assert(newsize >= PyDict_MINSIZE); + assert(log2_newsize >= PyDict_LOG_MINSIZE); oldkeys = mp->ma_keys; @@ -1239,7 +1239,7 @@ dictresize(PyDictObject *mp, Py_ssize_t newsize) */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(newsize); + mp->ma_keys = new_keys_object(log2_newsize); if (mp->ma_keys == NULL) { mp->ma_keys = oldkeys; return -1; @@ -1297,7 +1297,7 @@ dictresize(PyDictObject *mp, Py_ssize_t newsize) // dictresize() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); #endif - if (oldkeys->dk_size == PyDict_MINSIZE && + if (DK_SIZE(oldkeys) == PyDict_MINSIZE && state->keys_numfree < PyDict_MAXFREELIST) { state->keys_free_list[state->keys_numfree++] = oldkeys; @@ -1333,7 +1333,7 @@ make_keys_shared(PyObject *op) } else if (mp->ma_keys->dk_lookup == lookdict_unicode) { /* Remove dummy keys */ - if (dictresize(mp, DK_SIZE(mp->ma_keys))) + if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) return NULL; } assert(mp->ma_keys->dk_lookup == lookdict_unicode_nodummy); @@ -1360,8 +1360,9 @@ make_keys_shared(PyObject *op) PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - const Py_ssize_t max_presize = 128 * 1024; - Py_ssize_t newsize; + const uint8_t log2_max_presize = 17; + const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; + uint8_t log2_newsize; PyDictKeysObject *new_keys; if (minused <= USABLE_FRACTION(PyDict_MINSIZE)) { @@ -1372,13 +1373,13 @@ _PyDict_NewPresized(Py_ssize_t minused) * large dict or MemoryError. */ if (minused > USABLE_FRACTION(max_presize)) { - newsize = max_presize; + log2_newsize = log2_max_presize; } else { - newsize = estimate_keysize(minused); + log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(newsize); + new_keys = new_keys_object(log2_newsize); if (new_keys == NULL) return NULL; return new_dict(new_keys, NULL); @@ -1709,7 +1710,7 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) // Split table doesn't allow deletion. Combine it. if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_SIZE(mp->ma_keys))) { + if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return -1; } ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); @@ -1752,7 +1753,7 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, // Split table doesn't allow deletion. Combine it. if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_SIZE(mp->ma_keys))) { + if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return -1; } ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); @@ -1916,7 +1917,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d // Split table doesn't allow deletion. Combine it. if (_PyDict_HasSplitTable(mp)) { - if (dictresize(mp, DK_SIZE(mp->ma_keys))) { + if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return NULL; } ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); @@ -1983,7 +1984,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_keysize(PyDict_GET_SIZE(iterable)))) { + if (dictresize(mp, estimate_log2_keysize(PyDict_GET_SIZE(iterable)))) { Py_DECREF(d); return NULL; } @@ -2002,7 +2003,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_keysize(PySet_GET_SIZE(iterable)))) { + if (dictresize(mp, estimate_log2_keysize(PySet_GET_SIZE(iterable)))) { Py_DECREF(d); return NULL; } @@ -2578,8 +2579,8 @@ dict_merge(PyObject *a, PyObject *b, int override) // If other is clean, combined, and just allocated, just clone it. if (other->ma_values == NULL && other->ma_used == okeys->dk_nentries && - (okeys->dk_size == PyDict_MINSIZE || - USABLE_FRACTION(okeys->dk_size/2) < other->ma_used)) { + (DK_SIZE(okeys) == PyDict_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; @@ -2610,8 +2611,8 @@ dict_merge(PyObject *a, PyObject *b, int override) * incrementally resizing as we insert new items. Expect * that there will be no (or few) overlapping keys. */ - if (USABLE_FRACTION(mp->ma_keys->dk_size) < other->ma_used) { - if (dictresize(mp, estimate_keysize(mp->ma_used + other->ma_used))) { + if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { + if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used))) { return -1; } } @@ -3195,7 +3196,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (self->ma_keys->dk_lookup == lookdict_split) { - if (dictresize(self, DK_SIZE(self->ma_keys))) { + if (dictresize(self, DK_LOG_SIZE(self->ma_keys))) { Py_DECREF(res); return NULL; } @@ -4974,7 +4975,7 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(PyDict_MINSIZE); + PyDictKeysObject *keys = new_keys_object(PyDict_LOG_MINSIZE); if (keys == NULL) { PyErr_Clear(); } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 6c7f1175cd652c..22f00200e1bb3c 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -553,7 +553,7 @@ _odict_resize(PyODictObject *od) _ODictNode **fast_nodes, *node; /* Initialize a new "fast nodes" table. */ - size = ((PyDictObject *)od)->ma_keys->dk_size; + size = 1 << (((PyDictObject *)od)->ma_keys->dk_log2_size); fast_nodes = PyMem_NEW(_ODictNode *, size); if (fast_nodes == NULL) { PyErr_NoMemory(); @@ -592,7 +592,7 @@ _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) /* Ensure od_fast_nodes and dk_entries are in sync. */ if (od->od_resize_sentinel != keys || - od->od_fast_nodes_size != keys->dk_size) { + od->od_fast_nodes_size != (1 << (keys->dk_log2_size))) { int resize_res = _odict_resize(od); if (resize_res < 0) return -1; diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index b726b353b77ab8..c1d2cd8ced68c4 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -730,7 +730,7 @@ def write_repr(self, out, visited): def _get_entries(self, keys): dk_nentries = int(keys['dk_nentries']) - dk_size = int(keys['dk_size']) + dk_size = 1< Date: Mon, 24 May 2021 11:11:23 +0100 Subject: [PATCH 02/10] Use enum instead of function pointer to record kind of keys. --- Lib/test/test_ordered_dict.py | 2 +- Lib/test/test_sys.py | 8 +-- Objects/dict-common.h | 29 ++++------ Objects/dictobject.c | 102 ++++++++++++++++++++-------------- Objects/odictobject.c | 6 +- 5 files changed, 79 insertions(+), 68 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index eb404463e92550..b183953d4ee8ae 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -752,7 +752,7 @@ def test_sizeof_exact(self): check = self.check_sizeof basicsize = size('nQ2P' + '3PnPn2P') - keysize = calcsize('2nP2n') + keysize = calcsize('n2B2n') entrysize = calcsize('n2P') p = calcsize('P') diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 6574c4f9b70273..a14c85d8b31772 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1229,9 +1229,9 @@ def inner(): # empty dict check({}, size('nQ2P')) # dict - check({"a": 1}, size('nQ2P') + calcsize('2nP2n') + 8 + (8*2//3)*calcsize('n2P')) + check({"a": 1}, size('nQ2P') + calcsize('n2B2n') + 8 + (8*2//3)*calcsize('n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size('nQ2P') + calcsize('2nP2n') + 16 + (16*2//3)*calcsize('n2P')) + check(longdict, size('nQ2P') + calcsize('n2B2n') + 16 + (16*2//3)*calcsize('n2P')) # dictionary-keyview check({}.keys(), size('P')) # dictionary-valueview @@ -1385,13 +1385,13 @@ def delx(self): del self.__x '5P') class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries - check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P")) + check(newstyleclass, s + calcsize("n2B2n0P") + 8 + 5*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 5*self.P) o = newstyleclass() o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1 # Separate block for PyDictKeysObject with 16 keys and 10 entries - check(newstyleclass, s + calcsize("2nP2n0P") + 16 + 10*calcsize("n2P")) + check(newstyleclass, s + calcsize("n2B2n0P") + 16 + 10*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 10*self.P) # unicode diff --git a/Objects/dict-common.h b/Objects/dict-common.h index 28f540c2f007d9..1234d63400ae58 100644 --- a/Objects/dict-common.h +++ b/Objects/dict-common.h @@ -8,16 +8,23 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictKeyEntry; -/* dict_lookup_func() returns index of entry which can be used like DK_ENTRIES(dk)[index]. +/* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. */ -typedef Py_ssize_t (*dict_lookup_func) - (PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); + #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ #define DKIX_ERROR (-3) +typedef enum { + DICT_KEYS_GENERAL = 0, + DICT_KEYS_UNICODE = 1, + DICT_KEYS_UNICODE_NO_DUMMY = 2, + DICT_KEYS_SPLIT = 3 +} DictKeysKind; + /* See dictobject.c for actual layout of DictKeysObject */ struct _dictkeysobject { Py_ssize_t dk_refcnt; @@ -25,20 +32,8 @@ struct _dictkeysobject { /* Size of the hash table (dk_indices). It must be a power of 2. */ uint8_t dk_log2_size; - /* Function to lookup in the hash table (dk_indices): - - - lookdict(): general-purpose, and may return DKIX_ERROR if (and - only if) a comparison raises an exception. - - - lookdict_unicode(): specialized to Unicode string keys, comparison of - which can never raise an exception; that function can never return - DKIX_ERROR. - - - lookdict_unicode_nodummy(): similar to lookdict_unicode() but further - specialized for Unicode string keys that cannot be the value. - - - lookdict_split(): Version of lookdict() for split tables. */ - dict_lookup_func dk_lookup; + /* Kind of keys */ + uint8_t dk_kind; /* Number of usable entries in dk_entries. */ Py_ssize_t dk_usable; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 72f23fb39ab14e..7cbafe3e435b06 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -18,8 +18,8 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: +---------------+ | dk_refcnt | -| dk_size | -| dk_lookup | +| dk_log2_size | +| dk_kind | | dk_usable | | dk_nentries | +---------------+ @@ -462,8 +462,8 @@ estimate_log2_keysize(Py_ssize_t n) #define GROWTH_RATE(d) ((d)->ma_used*3) #define ENSURE_ALLOWS_DELETIONS(d) \ - if ((d)->ma_keys->dk_lookup == lookdict_unicode_nodummy) { \ - (d)->ma_keys->dk_lookup = lookdict_unicode; \ + if ((d)->ma_keys->dk_kind == DICT_KEYS_UNICODE_NO_DUMMY) { \ + (d)->ma_keys->dk_kind = DICT_KEYS_UNICODE; \ } /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() @@ -472,7 +472,7 @@ estimate_log2_keysize(Py_ssize_t n) static PyDictKeysObject empty_keys_struct = { 1, /* dk_refcnt */ 0, /* dk_log2_size */ - lookdict_split, /* dk_lookup */ + DICT_KEYS_SPLIT, /* dk_kind */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, @@ -612,7 +612,7 @@ new_keys_object(uint8_t log2_size) dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; dk->dk_usable = usable; - dk->dk_lookup = lookdict_unicode_nodummy; + dk->dk_kind = DICT_KEYS_UNICODE_NO_DUMMY; dk->dk_nentries = 0; memset(&dk->dk_indices[0], 0xff, es * (1<ma_keys->dk_kind; + switch (kind) { + case DICT_KEYS_GENERAL: + return lookdict(mp, key, hash, value_addr); + case DICT_KEYS_UNICODE: + return lookdict_unicode(mp, key, hash, value_addr); + case DICT_KEYS_UNICODE_NO_DUMMY: + return lookdict_unicode_nodummy(mp, key, hash, value_addr); + case DICT_KEYS_SPLIT: + return lookdict_split(mp, key, hash, value_addr); + }; +} + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -981,7 +997,7 @@ _PyDict_HasOnlyStringKeys(PyObject *dict) PyObject *key, *value; assert(PyDict_Check(dict)); /* Shortcut */ - if (((PyDictObject *)dict)->ma_keys->dk_lookup != lookdict) + if (((PyDictObject *)dict)->ma_keys->dk_kind != DICT_KEYS_GENERAL) return 1; while (PyDict_Next(dict, &pos, &key, &value)) if (!PyUnicode_Check(key)) @@ -1079,7 +1095,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) goto Fail; } - Py_ssize_t ix = mp->ma_keys->dk_lookup(mp, key, hash, &old_value); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) goto Fail; @@ -1104,8 +1120,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (insertion_resize(mp) < 0) goto Fail; } - if (!PyUnicode_CheckExact(key) && mp->ma_keys->dk_lookup != lookdict) { - mp->ma_keys->dk_lookup = lookdict; + if (!PyUnicode_CheckExact(key) && mp->ma_keys->dk_kind != DICT_KEYS_GENERAL) { + mp->ma_keys->dk_kind = DICT_KEYS_GENERAL; } Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; @@ -1166,7 +1182,7 @@ insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, return -1; } if (!PyUnicode_CheckExact(key)) { - newkeys->dk_lookup = lookdict; + newkeys->dk_kind = DICT_KEYS_GENERAL; } dictkeys_decref(Py_EMPTY_KEYS); mp->ma_keys = newkeys; @@ -1246,8 +1262,8 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) } // New table must be large enough. assert(mp->ma_keys->dk_usable >= mp->ma_used); - if (oldkeys->dk_lookup == lookdict) - mp->ma_keys->dk_lookup = lookdict; + if (oldkeys->dk_kind == DICT_KEYS_GENERAL) + mp->ma_keys->dk_kind = DICT_KEYS_GENERAL; numentries = mp->ma_used; oldentries = DK_ENTRIES(oldkeys); @@ -1287,7 +1303,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize) } } - assert(oldkeys->dk_lookup != lookdict_split); + assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #ifdef Py_REF_DEBUG _Py_RefTotal--; @@ -1328,15 +1344,15 @@ make_keys_shared(PyObject *op) PyDictKeyEntry *ep0; PyObject **values; assert(mp->ma_keys->dk_refcnt == 1); - if (mp->ma_keys->dk_lookup == lookdict) { + if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { return NULL; } - else if (mp->ma_keys->dk_lookup == lookdict_unicode) { + else if (mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { /* Remove dummy keys */ if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) return NULL; } - assert(mp->ma_keys->dk_lookup == lookdict_unicode_nodummy); + assert(mp->ma_keys->dk_kind == DICT_KEYS_UNICODE_NO_DUMMY); /* Copy values into a new array */ ep0 = DK_ENTRIES(mp->ma_keys); size = USABLE_FRACTION(DK_SIZE(mp->ma_keys)); @@ -1350,7 +1366,7 @@ make_keys_shared(PyObject *op) values[i] = ep0[i].me_value; ep0[i].me_value = NULL; } - mp->ma_keys->dk_lookup = lookdict_split; + mp->ma_keys->dk_kind = DICT_KEYS_SPLIT; mp->ma_values = values; } dictkeys_incref(mp->ma_keys); @@ -1427,7 +1443,7 @@ PyDict_GetItem(PyObject *op, PyObject *key) Py_ssize_t ix; _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); /* Ignore any exception raised by the lookup */ _PyErr_Restore(tstate, exc_type, exc_value, exc_tb); @@ -1451,7 +1467,7 @@ _PyDict_GetItemHint(PyDictObject *mp, PyObject *key, PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys) + (size_t)hint; if (ep->me_key == key) { - if (mp->ma_keys->dk_lookup == lookdict_split) { + if (mp->ma_keys->dk_kind == DICT_KEYS_SPLIT) { assert(mp->ma_values != NULL); res = mp->ma_values[(size_t)hint]; } @@ -1473,7 +1489,7 @@ _PyDict_GetItemHint(PyDictObject *mp, PyObject *key, } } - return (mp->ma_keys->dk_lookup)(mp, key, hash, value); + return _Py_dict_lookup(mp, key, hash, value); } /* Same as PyDict_GetItemWithError() but with hash supplied by caller. @@ -1492,7 +1508,7 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix < 0) { return NULL; } @@ -1524,7 +1540,7 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix < 0) return NULL; return value; @@ -1578,14 +1594,14 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) } /* namespace 1: globals */ - ix = globals->ma_keys->dk_lookup(globals, key, hash, &value); + ix = _Py_dict_lookup(globals, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix != DKIX_EMPTY && value != NULL) return value; /* namespace 2: builtins */ - ix = builtins->ma_keys->dk_lookup(builtins, key, hash, &value); + ix = _Py_dict_lookup(builtins, key, hash, &value); if (ix < 0) return NULL; return value; @@ -1700,7 +1716,7 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) assert(key); assert(hash != -1); mp = (PyDictObject *)op; - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) return -1; if (ix == DKIX_EMPTY || old_value == NULL) { @@ -1713,7 +1729,7 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return -1; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); assert(ix >= 0); } @@ -1743,7 +1759,7 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, if (hash == -1) return -1; mp = (PyDictObject *)op; - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) return -1; if (ix == DKIX_EMPTY || old_value == NULL) { @@ -1756,7 +1772,7 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return -1; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); assert(ix >= 0); } @@ -1903,7 +1919,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d _PyErr_SetKeyError(key); return NULL; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || old_value == NULL) { @@ -1920,7 +1936,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) { return NULL; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &old_value); + ix = _Py_dict_lookup(mp, key, hash, &old_value); assert(ix >= 0); } @@ -2193,7 +2209,7 @@ dict_subscript(PyDictObject *mp, PyObject *key) if (hash == -1) return NULL; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2909,7 +2925,7 @@ dict_equal(PyDictObject *a, PyDictObject *b) /* ditto for key */ Py_INCREF(key); /* reuse the known hash value */ - b->ma_keys->dk_lookup(b, key, ep->me_hash, &bval); + _Py_dict_lookup(b, key, ep->me_hash, &bval); if (bval == NULL) { Py_DECREF(key); Py_DECREF(aval); @@ -2976,7 +2992,7 @@ dict___contains__(PyDictObject *self, PyObject *key) if (hash == -1) return NULL; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) @@ -3008,7 +3024,7 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) if (hash == -1) return NULL; } - ix = (self->ma_keys->dk_lookup) (self, key, hash, &val); + ix = _Py_dict_lookup(self, key, hash, &val); if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || val == NULL) { @@ -3048,7 +3064,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; } - Py_ssize_t ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) return NULL; @@ -3069,8 +3085,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; } } - if (!PyUnicode_CheckExact(key) && mp->ma_keys->dk_lookup != lookdict) { - mp->ma_keys->dk_lookup = lookdict; + if (!PyUnicode_CheckExact(key) && mp->ma_keys->dk_kind != DICT_KEYS_GENERAL) { + mp->ma_keys->dk_kind = DICT_KEYS_GENERAL; } Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); ep0 = DK_ENTRIES(mp->ma_keys); @@ -3195,7 +3211,7 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_lookup == lookdict_split) { + if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { if (dictresize(self, DK_LOG_SIZE(self->ma_keys))) { Py_DECREF(res); return NULL; @@ -3237,7 +3253,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) PyDictKeyEntry *entries = DK_ENTRIES(keys); Py_ssize_t i, n = keys->dk_nentries; - if (keys->dk_lookup == lookdict) { + if (keys->dk_kind == DICT_KEYS_GENERAL) { for (i = 0; i < n; i++) { if (entries[i].me_value != NULL) { Py_VISIT(entries[i].me_value); @@ -3402,7 +3418,7 @@ PyDict_Contains(PyObject *op, PyObject *key) if (hash == -1) return -1; } - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) return -1; return (ix != DKIX_EMPTY && value != NULL); @@ -3416,7 +3432,7 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; - ix = (mp->ma_keys->dk_lookup)(mp, key, hash, &value); + ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) return -1; return (ix != DKIX_EMPTY && value != NULL); @@ -4980,7 +4996,7 @@ _PyDict_NewKeysForClass(void) PyErr_Clear(); } else { - keys->dk_lookup = lookdict_split; + keys->dk_kind = DICT_KEYS_SPLIT; } return keys; } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 22f00200e1bb3c..fe6e4b2a12a50c 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -40,8 +40,8 @@ we've considered: The approach with the least performance impact (time and space) is #2, mirroring the key order of dict's dk_entries with an array of node pointers. -While lookdict() and friends (dk_lookup) don't give us the index into the -array, we make use of pointer arithmetic to get that index. An alternative +While _Py_dict_lookup() does not give us the index into the array, +we make use of pointer arithmetic to get that index. An alternative would be to refactor lookdict() to provide the index, explicitly exposing the implementation detail. We could even just use a custom lookup function for OrderedDict that facilitates our need. However, both approaches are @@ -535,7 +535,7 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; - ix = (keys->dk_lookup)((PyDictObject *)od, key, hash, &value); + ix = _Py_dict_lookup((PyDictObject *)od, key, hash, &value); if (ix == DKIX_EMPTY) { return keys->dk_nentries; /* index of new entry */ } From 6ad4bb72000f40e8987c2265fbfbbcdc1a0ce5b1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 24 May 2021 12:14:33 +0100 Subject: [PATCH 03/10] Add version number to dict keys. --- Include/cpython/dictobject.h | 4 ++++ Objects/dict-common.h | 3 +++ Objects/dictobject.c | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 6822a65cad95e8..de3c1160b489ce 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -82,3 +82,7 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); + +/* Gets a version number unique to the current state of the keys of dict, if possible. + * Returns the version number, or zero if it was not possible to get a version number. */ +uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictObject *dict); diff --git a/Objects/dict-common.h b/Objects/dict-common.h index 1234d63400ae58..436f1ee2b230cc 100644 --- a/Objects/dict-common.h +++ b/Objects/dict-common.h @@ -35,6 +35,9 @@ struct _dictkeysobject { /* Kind of keys */ uint8_t dk_kind; + /* Version number -- Reset to 0 by any modification to keys */ + uint32_t dk_version; + /* Number of usable entries in dk_entries. */ Py_ssize_t dk_usable; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 7cbafe3e435b06..a86861bc23b0eb 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -376,6 +376,7 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) Py_ssize_t s = DK_SIZE(keys); assert(ix >= DKIX_DUMMY); + assert(keys->dk_version == 0); if (s <= 0xff) { int8_t *indices = (int8_t*)(keys->dk_indices); @@ -464,6 +465,7 @@ estimate_log2_keysize(Py_ssize_t n) #define ENSURE_ALLOWS_DELETIONS(d) \ if ((d)->ma_keys->dk_kind == DICT_KEYS_UNICODE_NO_DUMMY) { \ (d)->ma_keys->dk_kind = DICT_KEYS_UNICODE; \ + (d)->ma_keys->dk_version = 0; \ } /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() @@ -473,6 +475,7 @@ static PyDictKeysObject empty_keys_struct = { 1, /* dk_refcnt */ 0, /* dk_log2_size */ DICT_KEYS_SPLIT, /* dk_kind */ + 1, /* dk_version */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ {DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, DKIX_EMPTY, @@ -614,6 +617,7 @@ new_keys_object(uint8_t log2_size) dk->dk_usable = usable; dk->dk_kind = DICT_KEYS_UNICODE_NO_DUMMY; dk->dk_nentries = 0; + dk->dk_version = 0; memset(&dk->dk_indices[0], 0xff, es * (1<ma_keys->dk_version = 0; assert(old_value == NULL); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ @@ -3078,6 +3084,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (ix == DKIX_EMPTY) { + mp->ma_keys->dk_version = 0; PyDictKeyEntry *ep, *ep0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { @@ -5108,3 +5115,18 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) { dictkeys_decref(keys); } + +static uint32_t next_dict_keys_version = 2; + +uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictObject *dict) +{ + if (dict->ma_keys->dk_version != 0) { + return dict->ma_keys->dk_version; + } + if (next_dict_keys_version == 0) { + return 0; + } + uint32_t v = next_dict_keys_version++; + dict->ma_keys->dk_version = v; + return v; +} From 730113de41f9ca6d70c18dc0e8f7160f12c0912f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 24 May 2021 12:25:00 +0100 Subject: [PATCH 04/10] Merge lookdict_unicode and lookdict_unicode_nodummy. Make sure dk_version is 0 before modification of a key. --- Objects/dictobject.c | 65 ++++---------------------------------------- 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a86861bc23b0eb..8f6a01b2ffdd12 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -232,9 +232,6 @@ static Py_ssize_t lookdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); static Py_ssize_t lookdict_unicode(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); -static Py_ssize_t -lookdict_unicode_nodummy(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr); static Py_ssize_t lookdict_split(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); @@ -462,12 +459,6 @@ estimate_log2_keysize(Py_ssize_t n) */ #define GROWTH_RATE(d) ((d)->ma_used*3) -#define ENSURE_ALLOWS_DELETIONS(d) \ - if ((d)->ma_keys->dk_kind == DICT_KEYS_UNICODE_NO_DUMMY) { \ - (d)->ma_keys->dk_kind = DICT_KEYS_UNICODE; \ - (d)->ma_keys->dk_version = 0; \ - } - /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() * (which cannot fail and thus can do no allocation). */ @@ -792,9 +783,7 @@ comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can never raise an exception; that function can never return DKIX_ERROR when key is string. Otherwise, it falls back to lookdict(). -lookdict_unicode_nodummy is further specialized for string keys that cannot be -the value. -For both, when the key isn't found a DKIX_EMPTY is returned. +When the key isn't found a DKIX_EMPTY is returned. */ static Py_ssize_t _Py_HOT_FUNCTION lookdict(PyDictObject *mp, PyObject *key, @@ -892,51 +881,10 @@ lookdict_unicode(PyDictObject *mp, PyObject *key, Py_UNREACHABLE(); } -/* Faster version of lookdict_unicode when it is known that no keys - * will be present. */ -static Py_ssize_t _Py_HOT_FUNCTION -lookdict_unicode_nodummy(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr) -{ - assert(mp->ma_values == NULL); - /* Make sure this function doesn't have to handle non-unicode keys, - including subclasses of str; e.g., one reason to subclass - unicodes is to override __eq__, and for speed we don't cater to - that here. */ - if (!PyUnicode_CheckExact(key)) { - return lookdict(mp, key, hash, value_addr); - } - - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - size_t mask = DK_MASK(mp->ma_keys); - size_t perturb = (size_t)hash; - size_t i = (size_t)hash & mask; - - for (;;) { - Py_ssize_t ix = dictkeys_get_index(mp->ma_keys, i); - assert (ix != DKIX_DUMMY); - if (ix == DKIX_EMPTY) { - *value_addr = NULL; - return DKIX_EMPTY; - } - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { - *value_addr = ep->me_value; - return ix; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} - /* Version of lookdict for split tables. * All split tables and only split tables use this lookup function. * Split tables only contain unicode keys and no dummy keys, - * so algorithm is the same as lookdict_unicode_nodummy. + * so algorithm is the same as lookdict_unicode, but doesn't check for dummy keys. */ static Py_ssize_t _Py_HOT_FUNCTION lookdict_split(PyDictObject *mp, PyObject *key, @@ -986,9 +934,8 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu case DICT_KEYS_GENERAL: return lookdict(mp, key, hash, value_addr); case DICT_KEYS_UNICODE: - return lookdict_unicode(mp, key, hash, value_addr); case DICT_KEYS_UNICODE_NO_DUMMY: - return lookdict_unicode_nodummy(mp, key, hash, value_addr); + return lookdict_unicode(mp, key, hash, value_addr); case DICT_KEYS_SPLIT: return lookdict_split(mp, key, hash, value_addr); }; @@ -1682,7 +1629,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, mp->ma_version_tag = DICT_NEXT_VERSION(); ep = &DK_ENTRIES(mp->ma_keys)[ix]; dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); - ENSURE_ALLOWS_DELETIONS(mp); + mp->ma_keys->dk_version = 0; old_key = ep->me_key; ep->me_key = NULL; ep->me_value = NULL; @@ -1953,7 +1900,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d mp->ma_version_tag = DICT_NEXT_VERSION(); dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); ep = &DK_ENTRIES(mp->ma_keys)[ix]; - ENSURE_ALLOWS_DELETIONS(mp); + mp->ma_keys->dk_version = 0; old_key = ep->me_key; ep->me_key = NULL; ep->me_value = NULL; @@ -3224,7 +3171,7 @@ dict_popitem_impl(PyDictObject *self) return NULL; } } - ENSURE_ALLOWS_DELETIONS(self); + self->ma_keys->dk_version = 0; /* Pop last item */ ep0 = DK_ENTRIES(self->ma_keys); From 00883710be2b505d88de4919b80209d9adaac2d8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 24 May 2021 13:31:46 +0100 Subject: [PATCH 05/10] Cleanup old dict lookup functions. --- Objects/dictobject.c | 176 ++++++++++-------------------------------- Objects/odictobject.c | 4 +- 2 files changed, 44 insertions(+), 136 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 8f6a01b2ffdd12..6221711f268400 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -227,14 +227,6 @@ equally good collision statistics, needed less code & used less memory. */ -/* forward declarations */ -static Py_ssize_t lookdict(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr); -static Py_ssize_t lookdict_unicode(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr); -static Py_ssize_t lookdict_split(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr); - static int dictresize(PyDictObject *mp, uint8_t log_newsize); static PyObject* dict_iter(PyDictObject *dict); @@ -774,34 +766,46 @@ probe indices are computed as explained earlier. All arithmetic on hash should ignore overflow. -The details in this version are due to Tim Peters, building on many past -contributions by Reimer Behrends, Jyrki Alakuijala, Vladimir Marangozov and -Christian Tismer. - -lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a +_Py_dict_lookup() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. -lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR when key -is string. Otherwise, it falls back to lookdict(). When the key isn't found a DKIX_EMPTY is returned. */ -static Py_ssize_t _Py_HOT_FUNCTION -lookdict(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr) +Py_ssize_t _Py_HOT_FUNCTION +_Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) { - size_t i, mask, perturb; PyDictKeysObject *dk; - PyDictKeyEntry *ep0; - -top: +start: dk = mp->ma_keys; - ep0 = DK_ENTRIES(dk); - mask = DK_MASK(dk); - perturb = hash; - i = (size_t)hash & mask; - + DictKeysKind kind = dk->dk_kind; + PyDictKeyEntry *ep0 = DK_ENTRIES(dk); + size_t mask = DK_MASK(dk); + size_t perturb = hash; + size_t i = (size_t)hash & mask; + Py_ssize_t ix; + if (PyUnicode_CheckExact(key) && kind != DICT_KEYS_GENERAL) { + /* Strings only */ + for (;;) { + ix = dictkeys_get_index(mp->ma_keys, i); + if (ix == DKIX_EMPTY) { + *value_addr = NULL; + return DKIX_EMPTY; + } + if (ix >= 0) { + PyDictKeyEntry *ep = &ep0[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + goto found; + } + } + perturb >>= PERTURB_SHIFT; + i = mask & (i*5 + perturb + 1); + } + Py_UNREACHABLE(); + } for (;;) { - Py_ssize_t ix = dictkeys_get_index(dk, i); + ix = dictkeys_get_index(dk, i); if (ix == DKIX_EMPTY) { *value_addr = NULL; return ix; @@ -810,8 +814,7 @@ lookdict(PyDictObject *mp, PyObject *key, PyDictKeyEntry *ep = &ep0[ix]; assert(ep->me_key != NULL); if (ep->me_key == key) { - *value_addr = ep->me_value; - return ix; + goto found; } if (ep->me_hash == hash) { PyObject *startkey = ep->me_key; @@ -824,13 +827,12 @@ lookdict(PyDictObject *mp, PyObject *key, } if (dk == mp->ma_keys && ep->me_key == startkey) { if (cmp > 0) { - *value_addr = ep->me_value; - return ix; + goto found; } } else { /* The dict was mutated, restart */ - goto top; + goto start; } } } @@ -838,108 +840,14 @@ lookdict(PyDictObject *mp, PyObject *key, i = (i*5 + perturb + 1) & mask; } Py_UNREACHABLE(); -} - -/* Specialized version for string-only keys */ -static Py_ssize_t _Py_HOT_FUNCTION -lookdict_unicode(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr) -{ - assert(mp->ma_values == NULL); - /* Make sure this function doesn't have to handle non-unicode keys, - including subclasses of str; e.g., one reason to subclass - unicodes is to override __eq__, and for speed we don't cater to - that here. */ - if (!PyUnicode_CheckExact(key)) { - return lookdict(mp, key, hash, value_addr); - } - - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - size_t mask = DK_MASK(mp->ma_keys); - size_t perturb = (size_t)hash; - size_t i = (size_t)hash & mask; - - for (;;) { - Py_ssize_t ix = dictkeys_get_index(mp->ma_keys, i); - if (ix == DKIX_EMPTY) { - *value_addr = NULL; - return DKIX_EMPTY; - } - if (ix >= 0) { - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { - *value_addr = ep->me_value; - return ix; - } - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); - } - Py_UNREACHABLE(); -} - -/* Version of lookdict for split tables. - * All split tables and only split tables use this lookup function. - * Split tables only contain unicode keys and no dummy keys, - * so algorithm is the same as lookdict_unicode, but doesn't check for dummy keys. - */ -static Py_ssize_t _Py_HOT_FUNCTION -lookdict_split(PyDictObject *mp, PyObject *key, - Py_hash_t hash, PyObject **value_addr) -{ - /* mp must split table */ - assert(mp->ma_values != NULL); - if (!PyUnicode_CheckExact(key)) { - Py_ssize_t ix = lookdict(mp, key, hash, value_addr); - if (ix >= 0) { - *value_addr = mp->ma_values[ix]; - } - return ix; +found: + if (dk->dk_kind == DICT_KEYS_SPLIT) { + *value_addr = mp->ma_values[ix]; } - - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - size_t mask = DK_MASK(mp->ma_keys); - size_t perturb = (size_t)hash; - size_t i = (size_t)hash & mask; - - for (;;) { - Py_ssize_t ix = dictkeys_get_index(mp->ma_keys, i); - assert (ix != DKIX_DUMMY); - if (ix == DKIX_EMPTY) { - *value_addr = NULL; - return DKIX_EMPTY; - } - PyDictKeyEntry *ep = &ep0[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { - *value_addr = mp->ma_values[ix]; - return ix; - } - perturb >>= PERTURB_SHIFT; - i = mask & (i*5 + perturb + 1); + else { + *value_addr = ep0[ix].me_value; } - Py_UNREACHABLE(); -} - -Py_ssize_t -_Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) -{ - DictKeysKind kind = mp->ma_keys->dk_kind; - switch (kind) { - case DICT_KEYS_GENERAL: - return lookdict(mp, key, hash, value_addr); - case DICT_KEYS_UNICODE: - case DICT_KEYS_UNICODE_NO_DUMMY: - return lookdict_unicode(mp, key, hash, value_addr); - case DICT_KEYS_SPLIT: - return lookdict_split(mp, key, hash, value_addr); - }; - Py_UNREACHABLE(); + return ix; } int diff --git a/Objects/odictobject.c b/Objects/odictobject.c index fe6e4b2a12a50c..470322f866eea2 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -41,8 +41,8 @@ we've considered: The approach with the least performance impact (time and space) is #2, mirroring the key order of dict's dk_entries with an array of node pointers. While _Py_dict_lookup() does not give us the index into the array, -we make use of pointer arithmetic to get that index. An alternative -would be to refactor lookdict() to provide the index, explicitly exposing +we make use of pointer arithmetic to get that index. An alternative would +be to refactor _Py_dict_lookup() to provide the index, explicitly exposing the implementation detail. We could even just use a custom lookup function for OrderedDict that facilitates our need. However, both approaches are significantly more complicated than just using pointer arithmetic. From 52170f88f7fbf6cfe6fe6d2e848aa71421bebc02 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 25 May 2021 10:11:41 +0100 Subject: [PATCH 06/10] Fix struct format for dict keys in tests. --- Lib/test/test_ordered_dict.py | 2 +- Lib/test/test_sys.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index b183953d4ee8ae..d48edb5881a14d 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -752,7 +752,7 @@ def test_sizeof_exact(self): check = self.check_sizeof basicsize = size('nQ2P' + '3PnPn2P') - keysize = calcsize('n2B2n') + keysize = calcsize('n2BI2n') entrysize = calcsize('n2P') p = calcsize('P') diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index a14c85d8b31772..40fb721f3fa595 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -22,6 +22,7 @@ # strings to intern in test_intern() INTERN_NUMRUNS = 0 +DICT_KEY_STRUCT_FORMAT = 'n2BI2n' class DisplayHookTest(unittest.TestCase): @@ -1229,9 +1230,9 @@ def inner(): # empty dict check({}, size('nQ2P')) # dict - check({"a": 1}, size('nQ2P') + calcsize('n2B2n') + 8 + (8*2//3)*calcsize('n2P')) + check({"a": 1}, size('nQ2P') + calcsize(DICT_KEY_STRUCT_FORMAT) + 8 + (8*2//3)*calcsize('n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size('nQ2P') + calcsize('n2B2n') + 16 + (16*2//3)*calcsize('n2P')) + check(longdict, size('nQ2P') + calcsize(DICT_KEY_STRUCT_FORMAT) + 16 + (16*2//3)*calcsize('n2P')) # dictionary-keyview check({}.keys(), size('P')) # dictionary-valueview @@ -1385,13 +1386,13 @@ def delx(self): del self.__x '5P') class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries - check(newstyleclass, s + calcsize("n2B2n0P") + 8 + 5*calcsize("n2P")) + check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 8 + 5*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 5*self.P) o = newstyleclass() o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1 # Separate block for PyDictKeysObject with 16 keys and 10 entries - check(newstyleclass, s + calcsize("n2B2n0P") + 16 + 10*calcsize("n2P")) + check(newstyleclass, s + calcsize(DICT_KEY_STRUCT_FORMAT) + 16 + 10*calcsize("n2P")) # dict with shared keys check(newstyleclass().__dict__, size('nQ2P') + 10*self.P) # unicode From e1ebb89ac4e9763e84e871b5f8ae75125131253d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 25 May 2021 10:12:00 +0100 Subject: [PATCH 07/10] Remove redundant assert --- Objects/dictobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6221711f268400..01fb5725aaac57 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -494,7 +494,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(IS_POWER_OF_2(DK_SIZE(keys))); CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); CHECK(keys->dk_usable + keys->dk_nentries <= usable); From e67d414882a7a56d3eb32795dcd654a396dec4a5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 25 May 2021 17:33:46 +0100 Subject: [PATCH 08/10] Reorder tests are unroll once dict in _Py_dict_lookup. --- Objects/dictobject.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 01fb5725aaac57..15d18e829f3667 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -785,10 +785,22 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu /* Strings only */ for (;;) { ix = dictkeys_get_index(mp->ma_keys, i); - if (ix == DKIX_EMPTY) { + if (ix >= 0) { + PyDictKeyEntry *ep = &ep0[ix]; + assert(ep->me_key != NULL); + assert(PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + goto found; + } + } + else if (ix == DKIX_EMPTY) { *value_addr = NULL; return DKIX_EMPTY; } + perturb >>= PERTURB_SHIFT; + i = mask & (i*5 + perturb + 1); + ix = dictkeys_get_index(mp->ma_keys, i); if (ix >= 0) { PyDictKeyEntry *ep = &ep0[ix]; assert(ep->me_key != NULL); @@ -798,6 +810,10 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu goto found; } } + else if (ix == DKIX_EMPTY) { + *value_addr = NULL; + return DKIX_EMPTY; + } perturb >>= PERTURB_SHIFT; i = mask & (i*5 + perturb + 1); } From a316f34dd8dceb34269e176a10b0fbc8dea55980 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 26 May 2021 09:53:10 +0100 Subject: [PATCH 09/10] Remove some redundant checks. --- Objects/dictobject.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 15d18e829f3667..bf518c1e6fd911 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1324,9 +1324,8 @@ PyDict_GetItem(PyObject *op, PyObject *key) /* Ignore any exception raised by the lookup */ _PyErr_Restore(tstate, exc_type, exc_value, exc_tb); - if (ix < 0) { - return NULL; - } + + assert(ix >= 0 || value == NULL); return value; } @@ -1385,9 +1384,7 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) } ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix < 0) { - return NULL; - } + assert(ix >= 0 || value == NULL); return value; } @@ -1417,8 +1414,7 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix < 0) - return NULL; + assert(ix >= 0 || value == NULL); return value; } @@ -1478,8 +1474,7 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) /* namespace 2: builtins */ ix = _Py_dict_lookup(builtins, key, hash, &value); - if (ix < 0) - return NULL; + assert(ix >= 0 || value == NULL); return value; } From 057c5a6fabf0d28b58a0ebe831eea6e8bad70888 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 27 May 2021 10:08:28 +0100 Subject: [PATCH 10/10] Remove NODUMMY kind. --- Objects/dict-common.h | 3 +-- Objects/dictobject.c | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Objects/dict-common.h b/Objects/dict-common.h index 436f1ee2b230cc..a6f518f301885a 100644 --- a/Objects/dict-common.h +++ b/Objects/dict-common.h @@ -21,8 +21,7 @@ Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb typedef enum { DICT_KEYS_GENERAL = 0, DICT_KEYS_UNICODE = 1, - DICT_KEYS_UNICODE_NO_DUMMY = 2, - DICT_KEYS_SPLIT = 3 + DICT_KEYS_SPLIT = 2 } DictKeysKind; /* See dictobject.c for actual layout of DictKeysObject */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index bf518c1e6fd911..47b69c8f286c54 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -597,7 +597,7 @@ new_keys_object(uint8_t log2_size) dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; dk->dk_usable = usable; - dk->dk_kind = DICT_KEYS_UNICODE_NO_DUMMY; + dk->dk_kind = DICT_KEYS_UNICODE; dk->dk_nentries = 0; dk->dk_version = 0; memset(&dk->dk_indices[0], 0xff, es * (1<ma_keys->dk_kind == DICT_KEYS_GENERAL) { return NULL; } - else if (mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { + else if (mp->ma_used > mp->ma_keys->dk_nentries) { /* Remove dummy keys */ if (dictresize(mp, DK_LOG_SIZE(mp->ma_keys))) return NULL; } - assert(mp->ma_keys->dk_kind == DICT_KEYS_UNICODE_NO_DUMMY); + assert(mp->ma_used == mp->ma_keys->dk_nentries); /* Copy values into a new array */ ep0 = DK_ENTRIES(mp->ma_keys); size = USABLE_FRACTION(DK_SIZE(mp->ma_keys));