markdown: Remove most of Twitter integration.

Twitter removed their v1 API. We take care to keep the existing cached
results around for now, and to not poison that cache, since we might
be able replace this with something that can still use the existing
cache.

Author: timabbott

zerver/lib/markdown/__init__.py

@@ -65,7 +65,7 @@
 from zerver.lib.subdomains import is_static_or_current_realm_url
 from zerver.lib.tex import render_tex
 from zerver.lib.thumbnail import user_uploads_or_external
-from zerver.lib.timeout import TimeoutExpiredError, timeout
+from zerver.lib.timeout import timeout
 from zerver.lib.timezone import common_timezones
 from zerver.lib.types import LinkifierDict
 from zerver.lib.url_encoding import encode_stream, hash_util_encode
@@ -420,68 +420,13 @@ def has_blockquote_ancestor(element_pair: Optional[ElementPair]) -> bool:
 
 @cache_with_key(lambda tweet_id: tweet_id, cache_name="database")
 def fetch_tweet_data(tweet_id: str) -> Optional[Dict[str, Any]]:
-    if settings.TEST_SUITE:
-        from . import testing_mocks
-
-        res = testing_mocks.twitter(tweet_id)
-    else:
-        creds = {
-            "consumer_key": settings.TWITTER_CONSUMER_KEY,
-            "consumer_secret": settings.TWITTER_CONSUMER_SECRET,
-            "access_token_key": settings.TWITTER_ACCESS_TOKEN_KEY,
-            "access_token_secret": settings.TWITTER_ACCESS_TOKEN_SECRET,
-        }
-        if not all(creds.values()):
-            return None
-
-        # We lazily import twitter here because its import process is
-        # surprisingly slow, and doing so has a significant impact on
-        # the startup performance of `manage.py` commands.
-        import twitter
-
-        api = twitter.Api(tweet_mode="extended", **creds)
-
-        try:
-            # Sometimes Twitter hangs on responses.  Timing out here
-            # will cause the Tweet to go through as-is with no inline
-            # preview, rather than having the message be rejected
-            # entirely. This timeout needs to be less than our overall
-            # formatting timeout.
-            tweet = timeout(3, lambda: api.GetStatus(tweet_id))
-            res = tweet.AsDict()
-        except TimeoutExpiredError:
-            # We'd like to try again later and not cache the bad result,
-            # so we need to re-raise the exception (just as though
-            # we were being rate-limited)
-            raise
-        except twitter.TwitterError as e:
-            t = e.args[0]
-            if len(t) == 1 and ("code" in t[0]):
-                # https://developer.twitter.com/en/docs/basics/response-codes
-                code = t[0]["code"]
-                if code in [34, 144, 421, 422]:
-                    # All these "correspond with HTTP 404," and mean
-                    # that the message doesn't exist; return None so
-                    # that we will cache the error.
-                    return None
-                elif code in [63, 179]:
-                    # 63 is that the account is suspended, 179 is that
-                    # it is now locked; cache the None.
-                    return None
-                elif code in [88, 130, 131]:
-                    # Code 88 means that we were rate-limited, 130
-                    # means Twitter is having capacity issues, and 131
-                    # is other 400-equivalent; in these cases, raise
-                    # the error so we don't cache None and will try
-                    # again later.
-                    raise
-            # It's not clear what to do in cases of other errors,
-            # but for now it seems reasonable to log at error
-            # level (so that we get notified), but then cache the
-            # failure to proceed with our usual work
-            markdown_logger.exception("Unknown error fetching tweet data", stack_info=True)
-            return None
-    return res
+    # Twitter removed support for the v1 API that this integration
+    # used. Given that, there's no point wasting time trying to make
+    # network requests to Twitter. But we leave this function, because
+    # existing cached renderings for Tweets is useful. We throw an
+    # exception rather than returning `None` to avoid caching that the
+    # link doesn't exist.
+    raise NotImplementedError("Twitter desupported their v1 API")
 
 
 class OpenGraphSession(OutgoingSession):
@@ -1049,6 +994,8 @@ def twitter_link(self, url: str) -> Optional[Element]:
                 img.set("src", media_url)
 
             return tweet
+        except NotImplementedError:
+            return None
         except Exception:
             # We put this in its own try-except because it requires external
             # connectivity. If Twitter flakes out, we don't want to not-render