diff --git a/pandas/core/generic.py b/pandas/core/generic.py
index e3e59639de56b..50126cabbacf3 100644
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -3353,9 +3353,13 @@ def _maybe_update_cacher(self, clear=False, verify_is_copy=True):
             if ref is None:
                 del self._cacher
             else:
+                # Note: we need to call ref._maybe_cache_changed even in the
+                #  case where it will raise.  (Uh, not clear why)
                 try:
                     ref._maybe_cache_changed(cacher[0], self)
-                except Exception:
+                except AssertionError:
+                    # ref._data.setitem can raise
+                    #  AssertionError because of shape mismatch
                     pass
 
         if verify_is_copy: