BUG: Attributes skipped when serialising plain Python objects to JSON (pandas-dev#42768)

Author: joelgibson

doc/source/whatsnew/v1.3.2.rst

@@ -36,6 +36,7 @@ Bug fixes
 - :meth:`.Styler.hide_columns` now hides the index name header row as well as column headers (:issue:`42101`)
 - :meth:`.Styler.set_sticky` has amended CSS to control the column/index names and ensure the correct sticky positions (:issue:`42537`)
 - Bug in de-serializing datetime indexes in PYTHONOPTIMIZED mode (:issue:`42866`)
+- Some attributes were skipped when serialising plain Python objects to JSON (:issue:`42768`, :issue:`33043`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_libs/src/ujson/python/objToJSON.c

@@ -882,37 +882,33 @@ void Dir_iterEnd(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {
 
 int Dir_iterNext(JSOBJ _obj, JSONTypeContext *tc) {
     PyObject *obj = (PyObject *)_obj;
-    PyObject *itemValue = GET_TC(tc)->itemValue;
-    PyObject *itemName = GET_TC(tc)->itemName;
-    PyObject *attr;
-    PyObject *attrName;
-    char *attrStr;
 
     if (PyErr_Occurred() || ((JSONObjectEncoder *)tc->encoder)->errorMsg) {
         return 0;
     }
 
-    if (itemValue) {
+    if (GET_TC(tc)->itemValue) {
         Py_DECREF(GET_TC(tc)->itemValue);
-        GET_TC(tc)->itemValue = itemValue = NULL;
+        GET_TC(tc)->itemValue = NULL;
     }
 
-    if (itemName) {
+    if (GET_TC(tc)->itemName) {
         Py_DECREF(GET_TC(tc)->itemName);
-        GET_TC(tc)->itemName = itemName = NULL;
+        GET_TC(tc)->itemName = NULL;
     }
 
     for (; GET_TC(tc)->index < GET_TC(tc)->size; GET_TC(tc)->index++) {
-        attrName = PyList_GET_ITEM(GET_TC(tc)->attrList, GET_TC(tc)->index);
-        attr = PyUnicode_AsUTF8String(attrName);
-        attrStr = PyBytes_AS_STRING(attr);
+        PyObject *attrName =
+            PyList_GET_ITEM(GET_TC(tc)->attrList, GET_TC(tc)->index);
+        PyObject *attr = PyUnicode_AsUTF8String(attrName);
+        char *attrStr = PyBytes_AS_STRING(attr);
 
         if (attrStr[0] == '_') {
             Py_DECREF(attr);
             continue;
         }
 
-        itemValue = PyObject_GetAttr(obj, attrName);
+        PyObject *itemValue = PyObject_GetAttr(obj, attrName);
         if (itemValue == NULL) {
             PyErr_Clear();
             Py_DECREF(attr);
@@ -925,25 +921,15 @@ int Dir_iterNext(JSOBJ _obj, JSONTypeContext *tc) {
             continue;
         }
 
-        GET_TC(tc)->itemName = itemName;
+        GET_TC(tc)->itemName = attr;
         GET_TC(tc)->itemValue = itemValue;
         GET_TC(tc)->index++;
-
-        itemName = attr;
-        break;
-    }
-
-    if (itemName == NULL) {
-        GET_TC(tc)->index = GET_TC(tc)->size;
-        GET_TC(tc)->itemValue = NULL;
-        return 0;
+        return 1;
     }
 
-    GET_TC(tc)->itemName = itemName;
-    GET_TC(tc)->itemValue = itemValue;
-    GET_TC(tc)->index++;
-
-    return 1;
+    GET_TC(tc)->index = GET_TC(tc)->size;
+    GET_TC(tc)->itemValue = NULL;
+    return 0;
 }
 
 JSOBJ Dir_iterGetValue(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {

pandas/tests/io/json/test_ujson.py

@@ -720,6 +720,21 @@ def my_obj_handler(_):
             ujson.encode(obj_list, default_handler=str)
         )
 
+    def test_encode_object(self):
+        # Keys should be all non-callable non-underscore attributes, see GH-42768
+        class _TestObject:
+            def __init__(self, a, b, _c, d):
+                self.a = a
+                self.b = b
+                self._c = _c
+                self.d = d
+
+            def e(self):
+                return 5
+
+        test_object = _TestObject(a=1, b=2, _c=3, d=4)
+        assert ujson.decode(ujson.encode(test_object)) == {"a": 1, "b": 2, "d": 4}
+
 
 class TestNumpyJSONTests:
     @pytest.mark.parametrize("bool_input", [True, False])