add whatsnew and other minor changes (pandas-dev#41876)

Author: neelmraman

doc/source/whatsnew/v1.4.0.rst

@@ -172,7 +172,7 @@ MultiIndex
 
 I/O
 ^^^
--
+- Bug in :func:`json_normalize` where ``errors=Ignore`` could fail to ignore missing values of ``meta`` when ``record_path`` has a length greater than one (:issue:`41876`)
 -
 
 Period

pandas/io/json/_normalize.py

@@ -380,14 +380,32 @@ def _json_normalize(
     Returns normalized data with columns prefixed with the given string.
     """
 
-    def _pull_field(js: dict[str, Any], spec: list | str) -> Scalar | Iterable:
+    def _pull_field(
+        js: dict[str, Any], spec: list | str, extract_record: bool = False
+    ) -> Scalar | Iterable:
         """Internal function to pull field"""
         result = js
-        if isinstance(spec, list):
-            for field in spec:
-                result = result[field]
-        else:
-            result = result[spec]
+        try:
+            if isinstance(spec, list):
+                for field in spec:
+                    result = result[field]
+            else:
+                result = result[spec]
+        except KeyError as e:
+            if extract_record:
+                raise KeyError(
+                    f"Key {e} not found. If specifying a record_path, all elements of "
+                    f"data should have the path."
+                ) from e
+            else:
+                if errors == "ignore":
+                    result = np.nan
+                else:
+                    raise KeyError(
+                        f"Key {e} not found. To replace missing values of {e} with "
+                        f"np.nan, pass in errors='ignore'"
+                    ) from e
+
         return result
 
     def _pull_records(js: dict[str, Any], spec: list | str) -> list:
@@ -396,7 +414,7 @@ def _pull_records(js: dict[str, Any], spec: list | str) -> list:
         _pull_field, but require to return list. And will raise error
         if has non iterable value.
         """
-        result = _pull_field(js, spec)
+        result = _pull_field(js, spec, extract_record=True)
 
         # GH 31507 GH 30145, GH 26284 if result is not list, raise TypeError if not
         # null, otherwise return an empty list
@@ -469,16 +487,7 @@ def _recursive_extract(data, path, seen_meta, level=0):
             for obj in data:
                 for val, key in zip(_meta, meta_keys):
                     if level + 1 == len(val):
-                        try:
-                            seen_meta[key] = _pull_field(obj, val[-1])
-                        except KeyError as e:
-                            if errors == "ignore":
-                                seen_meta[key] = np.nan
-                            else:
-                                raise KeyError(
-                                    "Try running with errors='ignore' as key "
-                                    f"{e} is not always present"
-                                ) from e
+                        seen_meta[key] = _pull_field(obj, val[-1])
 
                 _recursive_extract(obj[path[0]], path[1:], seen_meta, level=level + 1)
         else:
@@ -497,16 +506,7 @@ def _recursive_extract(data, path, seen_meta, level=0):
                     if level + 1 > len(val):
                         meta_val = seen_meta[key]
                     else:
-                        try:
-                            meta_val = _pull_field(obj, val[level:])
-                        except KeyError as e:
-                            if errors == "ignore":
-                                meta_val = np.nan
-                            else:
-                                raise KeyError(
-                                    "Try running with errors='ignore' as key "
-                                    f"{e} is not always present"
-                                ) from e
+                        meta_val = _pull_field(obj, val[level:])
                     meta_vals[key].append(meta_val)
                 records.extend(recs)
 

pandas/tests/io/json/test_normalize.py

@@ -600,7 +600,10 @@ def test_json_normalize_errors(self, missing_metadata):
         # If meta keys are not always present a new option to set
         # errors='ignore' has been implemented
 
-        msg = "Try running with errors='ignore' as key 'name' is not always present"
+        msg = (
+            "Key 'name' not found. To replace missing values of "
+            "'name' with np.nan, pass in errors='ignore'"
+        )
         with pytest.raises(KeyError, match=msg):
             json_normalize(
                 data=missing_metadata,
@@ -628,7 +631,10 @@ def test_missing_meta_multilevel_record_path_errors_raise(self, missing_metadata
         # GH41876
         # Ensure errors='raise' works as intended even when a record_path of length
         # greater than one is passed in
-        msg = "Try running with errors='ignore' as key 'name' is not always present"
+        msg = (
+            "Key 'name' not found. To replace missing values of "
+            "'name' with np.nan, pass in errors='ignore'"
+        )
         with pytest.raises(KeyError, match=msg):
             json_normalize(
                 data=missing_metadata,