From c24c1e2a476d1a69a0665e7f997f5846c3c12717 Mon Sep 17 00:00:00 2001 From: MaximSmolskiy Date: Sun, 21 Apr 2024 20:27:42 +0300 Subject: [PATCH 1/2] Enable ruff S113 rule --- machine_learning/linear_regression.py | 3 ++- pyproject.toml | 1 - scripts/validate_solutions.py | 2 +- web_programming/co2_emission.py | 4 ++-- web_programming/covid_stats_via_xpath.py | 2 +- web_programming/crawl_google_results.py | 2 +- web_programming/crawl_google_scholar_citation.py | 2 +- web_programming/currency_converter.py | 2 +- web_programming/current_stock_price.py | 2 +- web_programming/current_weather.py | 4 ++-- web_programming/daily_horoscope.py | 2 +- web_programming/download_images_from_google_query.py | 2 +- web_programming/emails_from_url.py | 4 ++-- web_programming/fetch_anime_and_play.py | 6 +++--- web_programming/fetch_bbc_news.py | 2 +- web_programming/fetch_github_info.py | 2 +- web_programming/fetch_jobs.py | 2 +- web_programming/fetch_quotes.py | 4 ++-- web_programming/fetch_well_rx_price.py | 2 +- web_programming/get_amazon_product_data.py | 2 +- web_programming/get_imdb_top_250_movies_csv.py | 2 +- web_programming/get_ip_geolocation.py | 2 +- web_programming/get_top_billionaires.py | 2 +- web_programming/get_top_hn_posts.py | 4 ++-- web_programming/giphy.py | 2 +- web_programming/instagram_crawler.py | 2 +- web_programming/instagram_pic.py | 4 ++-- web_programming/instagram_video.py | 4 ++-- web_programming/nasa_data.py | 6 +++--- web_programming/open_google_results.py | 1 + web_programming/random_anime_character.py | 4 ++-- web_programming/recaptcha_verification.py | 2 +- web_programming/reddit.py | 1 + web_programming/search_books_by_isbn.py | 2 +- web_programming/slack_message.py | 2 +- web_programming/world_covid19_stats.py | 2 +- 36 files changed, 48 insertions(+), 46 deletions(-) diff --git a/machine_learning/linear_regression.py b/machine_learning/linear_regression.py index 39bee5712c16..d4bfe1df2ab8 100644 --- a/machine_learning/linear_regression.py +++ b/machine_learning/linear_regression.py @@ -19,7 +19,8 @@ def collect_dataset(): """ response = requests.get( "https://raw.githubusercontent.com/yashLadha/The_Math_of_Intelligence/" - "master/Week1/ADRvsRating.csv" + "master/Week1/ADRvsRating.csv", + timeout=10 ) lines = response.text.splitlines() data = [] diff --git a/pyproject.toml b/pyproject.toml index e46293a8d526..ff22fba81c8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ lint.ignore = [ # `ruff rule S101` for a description of that rule "RUF002", # Docstring contains ambiguous {}. Did you mean {}? "RUF003", # Comment contains ambiguous {}. Did you mean {}? "S101", # Use of `assert` detected -- DO NOT FIX - "S113", # Probable use of requests call without timeout -- FIX ME "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes -- FIX ME "SLF001", # Private member accessed: `_Iterator` -- FIX ME "UP038", # Use `X | Y` in `{}` call instead of `(X, Y)` -- DO NOT FIX diff --git a/scripts/validate_solutions.py b/scripts/validate_solutions.py index 68dcd68b3947..325c245e0d77 100755 --- a/scripts/validate_solutions.py +++ b/scripts/validate_solutions.py @@ -57,7 +57,7 @@ def added_solution_file_path() -> list[pathlib.Path]: "Accept": "application/vnd.github.v3+json", "Authorization": "token " + os.environ["GITHUB_TOKEN"], } - files = requests.get(get_files_url(), headers=headers).json() + files = requests.get(get_files_url(), headers=headers, timeout=10).json() for file in files: filepath = pathlib.Path.cwd().joinpath(file["filename"]) if ( diff --git a/web_programming/co2_emission.py b/web_programming/co2_emission.py index 88a426cb976d..19af70489d1d 100644 --- a/web_programming/co2_emission.py +++ b/web_programming/co2_emission.py @@ -11,13 +11,13 @@ # Emission in the last half hour def fetch_last_half_hour() -> str: - last_half_hour = requests.get(BASE_URL).json()["data"][0] + last_half_hour = requests.get(BASE_URL, timeout=10).json()["data"][0] return last_half_hour["intensity"]["actual"] # Emissions in a specific date range def fetch_from_to(start, end) -> list: - return requests.get(f"{BASE_URL}/{start}/{end}").json()["data"] + return requests.get(f"{BASE_URL}/{start}/{end}", timeout=10).json()["data"] if __name__ == "__main__": diff --git a/web_programming/covid_stats_via_xpath.py b/web_programming/covid_stats_via_xpath.py index 7011a02bffa8..97a92c356bd9 100644 --- a/web_programming/covid_stats_via_xpath.py +++ b/web_programming/covid_stats_via_xpath.py @@ -18,7 +18,7 @@ class CovidData(NamedTuple): def covid_stats(url: str = "https://www.worldometers.info/coronavirus/") -> CovidData: xpath_str = '//div[@class = "maincounter-number"]/span/text()' - return CovidData(*html.fromstring(requests.get(url).content).xpath(xpath_str)) + return CovidData(*html.fromstring(requests.get(url, timeout=10).content).xpath(xpath_str)) fmt = """Total COVID-19 cases in the world: {} diff --git a/web_programming/crawl_google_results.py b/web_programming/crawl_google_results.py index 1f5e6d31992b..cb75d450ff82 100644 --- a/web_programming/crawl_google_results.py +++ b/web_programming/crawl_google_results.py @@ -8,7 +8,7 @@ if __name__ == "__main__": print("Googling.....") url = "https://www.google.com/search?q=" + " ".join(sys.argv[1:]) - res = requests.get(url, headers={"UserAgent": UserAgent().random}) + res = requests.get(url, headers={"UserAgent": UserAgent().random}, timeout=10) # res.raise_for_status() with open("project1a.html", "wb") as out_file: # only for knowing the class for data in res.iter_content(10000): diff --git a/web_programming/crawl_google_scholar_citation.py b/web_programming/crawl_google_scholar_citation.py index f92a3d139520..cf37a25e80ea 100644 --- a/web_programming/crawl_google_scholar_citation.py +++ b/web_programming/crawl_google_scholar_citation.py @@ -11,7 +11,7 @@ def get_citation(base_url: str, params: dict) -> str: """ Return the citation number. """ - soup = BeautifulSoup(requests.get(base_url, params=params).content, "html.parser") + soup = BeautifulSoup(requests.get(base_url, params=params, timeout=10).content, "html.parser") div = soup.find("div", attrs={"class": "gs_ri"}) anchors = div.find("div", attrs={"class": "gs_fl"}).find_all("a") return anchors[2].get_text() diff --git a/web_programming/currency_converter.py b/web_programming/currency_converter.py index 3bbcafa8f89b..9623504b89ea 100644 --- a/web_programming/currency_converter.py +++ b/web_programming/currency_converter.py @@ -176,7 +176,7 @@ def convert_currency( params = locals() # from is a reserved keyword params["from"] = params.pop("from_") - res = requests.get(URL_BASE, params=params).json() + res = requests.get(URL_BASE, params=params, timeout=10).json() return str(res["amount"]) if res["error"] == 0 else res["error_message"] diff --git a/web_programming/current_stock_price.py b/web_programming/current_stock_price.py index 0c06354d8998..7487b0302218 100644 --- a/web_programming/current_stock_price.py +++ b/web_programming/current_stock_price.py @@ -4,7 +4,7 @@ def stock_price(symbol: str = "AAPL") -> str: url = f"https://finance.yahoo.com/quote/{symbol}?p={symbol}" - yahoo_finance_source = requests.get(url, headers={"USER-AGENT": "Mozilla/5.0"}).text + yahoo_finance_source = requests.get(url, headers={"USER-AGENT": "Mozilla/5.0"}, timeout=10).text soup = BeautifulSoup(yahoo_finance_source, "html.parser") specific_fin_streamer_tag = soup.find("fin-streamer", {"data-test": "qsp-price"}) diff --git a/web_programming/current_weather.py b/web_programming/current_weather.py index 3b6cd177cdfb..4a8fa5e3c845 100644 --- a/web_programming/current_weather.py +++ b/web_programming/current_weather.py @@ -20,13 +20,13 @@ def current_weather(location: str) -> list[dict]: if OPENWEATHERMAP_API_KEY: params_openweathermap = {"q": location, "appid": OPENWEATHERMAP_API_KEY} response_openweathermap = requests.get( - OPENWEATHERMAP_URL_BASE, params=params_openweathermap + OPENWEATHERMAP_URL_BASE, params=params_openweathermap, timeout=10 ) weather_data.append({"OpenWeatherMap": response_openweathermap.json()}) if WEATHERSTACK_API_KEY: params_weatherstack = {"query": location, "access_key": WEATHERSTACK_API_KEY} response_weatherstack = requests.get( - WEATHERSTACK_URL_BASE, params=params_weatherstack + WEATHERSTACK_URL_BASE, params=params_weatherstack, timeout=10 ) weather_data.append({"Weatherstack": response_weatherstack.json()}) if not weather_data: diff --git a/web_programming/daily_horoscope.py b/web_programming/daily_horoscope.py index b0dd1cd65924..75e637d8e52c 100644 --- a/web_programming/daily_horoscope.py +++ b/web_programming/daily_horoscope.py @@ -7,7 +7,7 @@ def horoscope(zodiac_sign: int, day: str) -> str: "https://www.horoscope.com/us/horoscopes/general/" f"horoscope-general-daily-{day}.aspx?sign={zodiac_sign}" ) - soup = BeautifulSoup(requests.get(url).content, "html.parser") + soup = BeautifulSoup(requests.get(url, timeout=10).content, "html.parser") return soup.find("div", class_="main-horoscope").p.text diff --git a/web_programming/download_images_from_google_query.py b/web_programming/download_images_from_google_query.py index 441347459f8e..5f00a746f388 100644 --- a/web_programming/download_images_from_google_query.py +++ b/web_programming/download_images_from_google_query.py @@ -39,7 +39,7 @@ def download_images_from_google_query(query: str = "dhaka", max_images: int = 5) "ijn": "0", } - html = requests.get("https://www.google.com/search", params=params, headers=headers) + html = requests.get("https://www.google.com/search", params=params, headers=headers, timeout=10) soup = BeautifulSoup(html.text, "html.parser") matched_images_data = "".join( re.findall(r"AF_initDataCallback\(([^<]+)\);", str(soup.select("script"))) diff --git a/web_programming/emails_from_url.py b/web_programming/emails_from_url.py index 26c88e1b13a5..43fd78dcf5a4 100644 --- a/web_programming/emails_from_url.py +++ b/web_programming/emails_from_url.py @@ -77,7 +77,7 @@ def emails_from_url(url: str = "https://github.com") -> list[str]: try: # Open URL - r = requests.get(url) + r = requests.get(url, timeout=10) # pass the raw HTML to the parser to get links parser.feed(r.text) @@ -88,7 +88,7 @@ def emails_from_url(url: str = "https://github.com") -> list[str]: # open URL. # read = requests.get(link) try: - read = requests.get(link) + read = requests.get(link, timeout=10) # Get the valid email. emails = re.findall("[a-zA-Z0-9]+@" + domain, read.text) # If not in list then append it. diff --git a/web_programming/fetch_anime_and_play.py b/web_programming/fetch_anime_and_play.py index 366807785e85..ef5e9ad5e260 100644 --- a/web_programming/fetch_anime_and_play.py +++ b/web_programming/fetch_anime_and_play.py @@ -28,7 +28,7 @@ def search_scraper(anime_name: str) -> list: search_url = f"{BASE_URL}/search/{anime_name}" response = requests.get( - search_url, headers={"UserAgent": UserAgent().chrome} + search_url, headers={"UserAgent": UserAgent().chrome}, timeout=10 ) # request the url. # Is the response ok? @@ -82,7 +82,7 @@ def search_anime_episode_list(episode_endpoint: str) -> list: request_url = f"{BASE_URL}{episode_endpoint}" - response = requests.get(url=request_url, headers={"UserAgent": UserAgent().chrome}) + response = requests.get(url=request_url, headers={"UserAgent": UserAgent().chrome}, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") @@ -132,7 +132,7 @@ def get_anime_episode(episode_endpoint: str) -> list: episode_page_url = f"{BASE_URL}{episode_endpoint}" response = requests.get( - url=episode_page_url, headers={"User-Agent": UserAgent().chrome} + url=episode_page_url, headers={"User-Agent": UserAgent().chrome}, timeout=10 ) response.raise_for_status() diff --git a/web_programming/fetch_bbc_news.py b/web_programming/fetch_bbc_news.py index 7f8bc57b69f5..e5cd864a9d83 100644 --- a/web_programming/fetch_bbc_news.py +++ b/web_programming/fetch_bbc_news.py @@ -7,7 +7,7 @@ def fetch_bbc_news(bbc_news_api_key: str) -> None: # fetching a list of articles in json format - bbc_news_page = requests.get(_NEWS_API + bbc_news_api_key).json() + bbc_news_page = requests.get(_NEWS_API + bbc_news_api_key, timeout=10).json() # each article in the list is a dict for i, article in enumerate(bbc_news_page["articles"], 1): print(f"{i}.) {article['title']}") diff --git a/web_programming/fetch_github_info.py b/web_programming/fetch_github_info.py index 7a4985b68841..25d44245bb58 100644 --- a/web_programming/fetch_github_info.py +++ b/web_programming/fetch_github_info.py @@ -42,7 +42,7 @@ def fetch_github_info(auth_token: str) -> dict[Any, Any]: "Authorization": f"token {auth_token}", "Accept": "application/vnd.github.v3+json", } - return requests.get(AUTHENTICATED_USER_ENDPOINT, headers=headers).json() + return requests.get(AUTHENTICATED_USER_ENDPOINT, headers=headers, timeout=10).json() if __name__ == "__main__": # pragma: no cover diff --git a/web_programming/fetch_jobs.py b/web_programming/fetch_jobs.py index 49abd3c88eec..77cb59d265cf 100644 --- a/web_programming/fetch_jobs.py +++ b/web_programming/fetch_jobs.py @@ -13,7 +13,7 @@ def fetch_jobs(location: str = "mumbai") -> Generator[tuple[str, str], None, None]: - soup = BeautifulSoup(requests.get(url + location).content, "html.parser") + soup = BeautifulSoup(requests.get(url + location, timeout=10).content, "html.parser") # This attribute finds out all the specifics listed in a job for job in soup.find_all("div", attrs={"data-tn-component": "organicJob"}): job_title = job.find("a", attrs={"data-tn-element": "jobTitle"}).text.strip() diff --git a/web_programming/fetch_quotes.py b/web_programming/fetch_quotes.py index d557e2d95e74..cf0add43f002 100644 --- a/web_programming/fetch_quotes.py +++ b/web_programming/fetch_quotes.py @@ -14,11 +14,11 @@ def quote_of_the_day() -> list: - return requests.get(API_ENDPOINT_URL + "/today").json() + return requests.get(API_ENDPOINT_URL + "/today", timeout=10).json() def random_quotes() -> list: - return requests.get(API_ENDPOINT_URL + "/random").json() + return requests.get(API_ENDPOINT_URL + "/random", timeout=10).json() if __name__ == "__main__": diff --git a/web_programming/fetch_well_rx_price.py b/web_programming/fetch_well_rx_price.py index ee51b9a5051b..93be2a9235d9 100644 --- a/web_programming/fetch_well_rx_price.py +++ b/web_programming/fetch_well_rx_price.py @@ -42,7 +42,7 @@ def fetch_pharmacy_and_price_list(drug_name: str, zip_code: str) -> list | None: return None request_url = BASE_URL.format(drug_name, zip_code) - response = get(request_url) + response = get(request_url, timeout=10) # Is the response ok? response.raise_for_status() diff --git a/web_programming/get_amazon_product_data.py b/web_programming/get_amazon_product_data.py index c2f2ac5ab291..538c2c126d7c 100644 --- a/web_programming/get_amazon_product_data.py +++ b/web_programming/get_amazon_product_data.py @@ -24,7 +24,7 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame: ), "Accept-Language": "en-US, en;q=0.5", } - soup = BeautifulSoup(requests.get(url, headers=header).text, features="lxml") + soup = BeautifulSoup(requests.get(url, headers=header, timeout=10).text, features="lxml") # Initialize a Pandas dataframe with the column titles data_frame = DataFrame( columns=[ diff --git a/web_programming/get_imdb_top_250_movies_csv.py b/web_programming/get_imdb_top_250_movies_csv.py index e54b076ebd94..c914b29cb3b3 100644 --- a/web_programming/get_imdb_top_250_movies_csv.py +++ b/web_programming/get_imdb_top_250_movies_csv.py @@ -8,7 +8,7 @@ def get_imdb_top_250_movies(url: str = "") -> dict[str, float]: url = url or "https://www.imdb.com/chart/top/?ref_=nv_mv_250" - soup = BeautifulSoup(requests.get(url).text, "html.parser") + soup = BeautifulSoup(requests.get(url, timeout=10).text, "html.parser") titles = soup.find_all("td", attrs="titleColumn") ratings = soup.find_all("td", class_="ratingColumn imdbRating") return { diff --git a/web_programming/get_ip_geolocation.py b/web_programming/get_ip_geolocation.py index 62eaeafceb7e..574d287f0db1 100644 --- a/web_programming/get_ip_geolocation.py +++ b/web_programming/get_ip_geolocation.py @@ -8,7 +8,7 @@ def get_ip_geolocation(ip_address: str) -> str: url = f"https://ipinfo.io/{ip_address}/json" # Send a GET request to the API - response = requests.get(url) + response = requests.get(url, timeout=10) # Check if the HTTP request was successful response.raise_for_status() diff --git a/web_programming/get_top_billionaires.py b/web_programming/get_top_billionaires.py index 703b635eef82..24828b6d787c 100644 --- a/web_programming/get_top_billionaires.py +++ b/web_programming/get_top_billionaires.py @@ -57,7 +57,7 @@ def get_forbes_real_time_billionaires() -> list[dict[str, int | str]]: Returns: List of top 10 realtime billionaires data. """ - response_json = requests.get(API_URL).json() + response_json = requests.get(API_URL, timeout=10).json() return [ { "Name": person["personName"], diff --git a/web_programming/get_top_hn_posts.py b/web_programming/get_top_hn_posts.py index fbb7c051a88e..f5d4f874c6c6 100644 --- a/web_programming/get_top_hn_posts.py +++ b/web_programming/get_top_hn_posts.py @@ -5,7 +5,7 @@ def get_hackernews_story(story_id: str) -> dict: url = f"https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty" - return requests.get(url).json() + return requests.get(url, timeout=10).json() def hackernews_top_stories(max_stories: int = 10) -> list[dict]: @@ -13,7 +13,7 @@ def hackernews_top_stories(max_stories: int = 10) -> list[dict]: Get the top max_stories posts from HackerNews - https://news.ycombinator.com/ """ url = "https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty" - story_ids = requests.get(url).json()[:max_stories] + story_ids = requests.get(url, timeout=10).json()[:max_stories] return [get_hackernews_story(story_id) for story_id in story_ids] diff --git a/web_programming/giphy.py b/web_programming/giphy.py index a5c3f8f7493e..2bf3e3ea9c0b 100644 --- a/web_programming/giphy.py +++ b/web_programming/giphy.py @@ -11,7 +11,7 @@ def get_gifs(query: str, api_key: str = giphy_api_key) -> list: """ formatted_query = "+".join(query.split()) url = f"https://api.giphy.com/v1/gifs/search?q={formatted_query}&api_key={api_key}" - gifs = requests.get(url).json()["data"] + gifs = requests.get(url, timeout=10).json()["data"] return [gif["url"] for gif in gifs] diff --git a/web_programming/instagram_crawler.py b/web_programming/instagram_crawler.py index 0816cd181051..df62735fb328 100644 --- a/web_programming/instagram_crawler.py +++ b/web_programming/instagram_crawler.py @@ -39,7 +39,7 @@ def get_json(self) -> dict: """ Return a dict of user information """ - html = requests.get(self.url, headers=headers).text + html = requests.get(self.url, headers=headers, timeout=10).text scripts = BeautifulSoup(html, "html.parser").find_all("script") try: return extract_user_profile(scripts[4]) diff --git a/web_programming/instagram_pic.py b/web_programming/instagram_pic.py index 2d987c1766dc..292cacc16c04 100644 --- a/web_programming/instagram_pic.py +++ b/web_programming/instagram_pic.py @@ -15,7 +15,7 @@ def download_image(url: str) -> str: A message indicating the result of the operation. """ try: - response = requests.get(url) + response = requests.get(url, timeout=10) response.raise_for_status() except requests.exceptions.RequestException as e: return f"An error occurred during the HTTP request to {url}: {e!r}" @@ -30,7 +30,7 @@ def download_image(url: str) -> str: return f"Image URL not found in meta tag {image_meta_tag}." try: - image_data = requests.get(image_url).content + image_data = requests.get(image_url, timeout=10).content except requests.exceptions.RequestException as e: return f"An error occurred during the HTTP request to {image_url}: {e!r}" if not image_data: diff --git a/web_programming/instagram_video.py b/web_programming/instagram_video.py index 1f1b0e297034..a4cddce25138 100644 --- a/web_programming/instagram_video.py +++ b/web_programming/instagram_video.py @@ -5,8 +5,8 @@ def download_video(url: str) -> bytes: base_url = "https://downloadgram.net/wp-json/wppress/video-downloader/video?url=" - video_url = requests.get(base_url + url).json()[0]["urls"][0]["src"] - return requests.get(video_url).content + video_url = requests.get(base_url + url, timeout=10).json()[0]["urls"][0]["src"] + return requests.get(video_url, timeout=10).content if __name__ == "__main__": diff --git a/web_programming/nasa_data.py b/web_programming/nasa_data.py index 81125e0a4f05..33a6406c52a6 100644 --- a/web_programming/nasa_data.py +++ b/web_programming/nasa_data.py @@ -9,14 +9,14 @@ def get_apod_data(api_key: str) -> dict: Get your API Key from: https://api.nasa.gov/ """ url = "https://api.nasa.gov/planetary/apod" - return requests.get(url, params={"api_key": api_key}).json() + return requests.get(url, params={"api_key": api_key}, timeout=10).json() def save_apod(api_key: str, path: str = ".") -> dict: apod_data = get_apod_data(api_key) img_url = apod_data["url"] img_name = img_url.split("/")[-1] - response = requests.get(img_url, stream=True) + response = requests.get(img_url, stream=True, timeout=10) with open(f"{path}/{img_name}", "wb+") as img_file: shutil.copyfileobj(response.raw, img_file) @@ -29,7 +29,7 @@ def get_archive_data(query: str) -> dict: Get the data of a particular query from NASA archives """ url = "https://images-api.nasa.gov/search" - return requests.get(url, params={"q": query}).json() + return requests.get(url, params={"q": query}, timeout=10).json() if __name__ == "__main__": diff --git a/web_programming/open_google_results.py b/web_programming/open_google_results.py index f61e3666dd7e..52dd37d7b91a 100644 --- a/web_programming/open_google_results.py +++ b/web_programming/open_google_results.py @@ -16,6 +16,7 @@ res = requests.get( url, headers={"User-Agent": str(UserAgent().random)}, + timeout=10, ) try: diff --git a/web_programming/random_anime_character.py b/web_programming/random_anime_character.py index f15a9c05d9e5..28509b7d8635 100644 --- a/web_programming/random_anime_character.py +++ b/web_programming/random_anime_character.py @@ -12,7 +12,7 @@ def save_image(image_url: str, image_title: str) -> None: """ Saves the image of anime character """ - image = requests.get(image_url, headers=headers) + image = requests.get(image_url, headers=headers, timeout=10) with open(image_title, "wb") as file: file.write(image.content) @@ -21,7 +21,7 @@ def random_anime_character() -> tuple[str, str, str]: """ Returns the Title, Description, and Image Title of a random anime character . """ - soup = BeautifulSoup(requests.get(URL, headers=headers).text, "html.parser") + soup = BeautifulSoup(requests.get(URL, headers=headers, timeout=10).text, "html.parser") title = soup.find("meta", attrs={"property": "og:title"}).attrs["content"] image_url = soup.find("meta", attrs={"property": "og:image"}).attrs["content"] description = soup.find("p", id="description").get_text() diff --git a/web_programming/recaptcha_verification.py b/web_programming/recaptcha_verification.py index c9b691b28a8b..d33608dd843d 100644 --- a/web_programming/recaptcha_verification.py +++ b/web_programming/recaptcha_verification.py @@ -56,7 +56,7 @@ def login_using_recaptcha(request): client_key = request.POST.get("g-recaptcha-response") # post recaptcha response to Google's recaptcha api - response = requests.post(url, data={"secret": secret_key, "response": client_key}) + response = requests.post(url, data={"secret": secret_key, "response": client_key}, timeout=10) # if the recaptcha api verified our keys if response.json().get("success", False): # authenticate the user diff --git a/web_programming/reddit.py b/web_programming/reddit.py index 1c165ecc49ec..6cc1a6b62009 100644 --- a/web_programming/reddit.py +++ b/web_programming/reddit.py @@ -31,6 +31,7 @@ def get_subreddit_data( response = requests.get( f"https://reddit.com/r/{subreddit}/{age}.json?limit={limit}", headers={"User-agent": "A random string"}, + timeout=10, ) if response.status_code == 429: raise requests.HTTPError(response=response) diff --git a/web_programming/search_books_by_isbn.py b/web_programming/search_books_by_isbn.py index 07429e9a9678..6b69018e6639 100644 --- a/web_programming/search_books_by_isbn.py +++ b/web_programming/search_books_by_isbn.py @@ -25,7 +25,7 @@ def get_openlibrary_data(olid: str = "isbn/0140328726") -> dict: if new_olid.count("/") != 1: msg = f"{olid} is not a valid Open Library olid" raise ValueError(msg) - return requests.get(f"https://openlibrary.org/{new_olid}.json").json() + return requests.get(f"https://openlibrary.org/{new_olid}.json", timeout=10).json() def summarize_book(ol_book_data: dict) -> dict: diff --git a/web_programming/slack_message.py b/web_programming/slack_message.py index 5e97d6b64c75..85a5e36c476c 100644 --- a/web_programming/slack_message.py +++ b/web_programming/slack_message.py @@ -5,7 +5,7 @@ def send_slack_message(message_body: str, slack_url: str) -> None: headers = {"Content-Type": "application/json"} - response = requests.post(slack_url, json={"text": message_body}, headers=headers) + response = requests.post(slack_url, json={"text": message_body}, headers=headers, timeout=10) if response.status_code != 200: msg = ( "Request to slack returned an error " diff --git a/web_programming/world_covid19_stats.py b/web_programming/world_covid19_stats.py index ca81abdc4ce9..4948d8cfd43c 100644 --- a/web_programming/world_covid19_stats.py +++ b/web_programming/world_covid19_stats.py @@ -13,7 +13,7 @@ def world_covid19_stats(url: str = "https://www.worldometers.info/coronavirus") """ Return a dict of current worldwide COVID-19 statistics """ - soup = BeautifulSoup(requests.get(url).text, "html.parser") + soup = BeautifulSoup(requests.get(url, timeout=10).text, "html.parser") keys = soup.findAll("h1") values = soup.findAll("div", {"class": "maincounter-number"}) keys += soup.findAll("span", {"class": "panel-title"}) From d63654f6382a6583a783e0b75b44de35e98a0638 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 17:28:53 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- machine_learning/linear_regression.py | 2 +- web_programming/covid_stats_via_xpath.py | 4 +++- web_programming/crawl_google_scholar_citation.py | 4 +++- web_programming/current_stock_price.py | 4 +++- web_programming/download_images_from_google_query.py | 4 +++- web_programming/fetch_anime_and_play.py | 4 +++- web_programming/fetch_jobs.py | 4 +++- web_programming/get_amazon_product_data.py | 4 +++- web_programming/random_anime_character.py | 4 +++- web_programming/recaptcha_verification.py | 4 +++- web_programming/slack_message.py | 4 +++- 11 files changed, 31 insertions(+), 11 deletions(-) diff --git a/machine_learning/linear_regression.py b/machine_learning/linear_regression.py index d4bfe1df2ab8..839a5366d1cc 100644 --- a/machine_learning/linear_regression.py +++ b/machine_learning/linear_regression.py @@ -20,7 +20,7 @@ def collect_dataset(): response = requests.get( "https://raw.githubusercontent.com/yashLadha/The_Math_of_Intelligence/" "master/Week1/ADRvsRating.csv", - timeout=10 + timeout=10, ) lines = response.text.splitlines() data = [] diff --git a/web_programming/covid_stats_via_xpath.py b/web_programming/covid_stats_via_xpath.py index 97a92c356bd9..c27a5d12bb3f 100644 --- a/web_programming/covid_stats_via_xpath.py +++ b/web_programming/covid_stats_via_xpath.py @@ -18,7 +18,9 @@ class CovidData(NamedTuple): def covid_stats(url: str = "https://www.worldometers.info/coronavirus/") -> CovidData: xpath_str = '//div[@class = "maincounter-number"]/span/text()' - return CovidData(*html.fromstring(requests.get(url, timeout=10).content).xpath(xpath_str)) + return CovidData( + *html.fromstring(requests.get(url, timeout=10).content).xpath(xpath_str) + ) fmt = """Total COVID-19 cases in the world: {} diff --git a/web_programming/crawl_google_scholar_citation.py b/web_programming/crawl_google_scholar_citation.py index cf37a25e80ea..5f2ccad5f414 100644 --- a/web_programming/crawl_google_scholar_citation.py +++ b/web_programming/crawl_google_scholar_citation.py @@ -11,7 +11,9 @@ def get_citation(base_url: str, params: dict) -> str: """ Return the citation number. """ - soup = BeautifulSoup(requests.get(base_url, params=params, timeout=10).content, "html.parser") + soup = BeautifulSoup( + requests.get(base_url, params=params, timeout=10).content, "html.parser" + ) div = soup.find("div", attrs={"class": "gs_ri"}) anchors = div.find("div", attrs={"class": "gs_fl"}).find_all("a") return anchors[2].get_text() diff --git a/web_programming/current_stock_price.py b/web_programming/current_stock_price.py index 7487b0302218..9567c05b0558 100644 --- a/web_programming/current_stock_price.py +++ b/web_programming/current_stock_price.py @@ -4,7 +4,9 @@ def stock_price(symbol: str = "AAPL") -> str: url = f"https://finance.yahoo.com/quote/{symbol}?p={symbol}" - yahoo_finance_source = requests.get(url, headers={"USER-AGENT": "Mozilla/5.0"}, timeout=10).text + yahoo_finance_source = requests.get( + url, headers={"USER-AGENT": "Mozilla/5.0"}, timeout=10 + ).text soup = BeautifulSoup(yahoo_finance_source, "html.parser") specific_fin_streamer_tag = soup.find("fin-streamer", {"data-test": "qsp-price"}) diff --git a/web_programming/download_images_from_google_query.py b/web_programming/download_images_from_google_query.py index 5f00a746f388..235cd35763ef 100644 --- a/web_programming/download_images_from_google_query.py +++ b/web_programming/download_images_from_google_query.py @@ -39,7 +39,9 @@ def download_images_from_google_query(query: str = "dhaka", max_images: int = 5) "ijn": "0", } - html = requests.get("https://www.google.com/search", params=params, headers=headers, timeout=10) + html = requests.get( + "https://www.google.com/search", params=params, headers=headers, timeout=10 + ) soup = BeautifulSoup(html.text, "html.parser") matched_images_data = "".join( re.findall(r"AF_initDataCallback\(([^<]+)\);", str(soup.select("script"))) diff --git a/web_programming/fetch_anime_and_play.py b/web_programming/fetch_anime_and_play.py index ef5e9ad5e260..fd7c3a3a7381 100644 --- a/web_programming/fetch_anime_and_play.py +++ b/web_programming/fetch_anime_and_play.py @@ -82,7 +82,9 @@ def search_anime_episode_list(episode_endpoint: str) -> list: request_url = f"{BASE_URL}{episode_endpoint}" - response = requests.get(url=request_url, headers={"UserAgent": UserAgent().chrome}, timeout=10) + response = requests.get( + url=request_url, headers={"UserAgent": UserAgent().chrome}, timeout=10 + ) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") diff --git a/web_programming/fetch_jobs.py b/web_programming/fetch_jobs.py index 77cb59d265cf..0d89bf45de57 100644 --- a/web_programming/fetch_jobs.py +++ b/web_programming/fetch_jobs.py @@ -13,7 +13,9 @@ def fetch_jobs(location: str = "mumbai") -> Generator[tuple[str, str], None, None]: - soup = BeautifulSoup(requests.get(url + location, timeout=10).content, "html.parser") + soup = BeautifulSoup( + requests.get(url + location, timeout=10).content, "html.parser" + ) # This attribute finds out all the specifics listed in a job for job in soup.find_all("div", attrs={"data-tn-component": "organicJob"}): job_title = job.find("a", attrs={"data-tn-element": "jobTitle"}).text.strip() diff --git a/web_programming/get_amazon_product_data.py b/web_programming/get_amazon_product_data.py index 538c2c126d7c..b98ff2c030af 100644 --- a/web_programming/get_amazon_product_data.py +++ b/web_programming/get_amazon_product_data.py @@ -24,7 +24,9 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame: ), "Accept-Language": "en-US, en;q=0.5", } - soup = BeautifulSoup(requests.get(url, headers=header, timeout=10).text, features="lxml") + soup = BeautifulSoup( + requests.get(url, headers=header, timeout=10).text, features="lxml" + ) # Initialize a Pandas dataframe with the column titles data_frame = DataFrame( columns=[ diff --git a/web_programming/random_anime_character.py b/web_programming/random_anime_character.py index 28509b7d8635..aed932866258 100644 --- a/web_programming/random_anime_character.py +++ b/web_programming/random_anime_character.py @@ -21,7 +21,9 @@ def random_anime_character() -> tuple[str, str, str]: """ Returns the Title, Description, and Image Title of a random anime character . """ - soup = BeautifulSoup(requests.get(URL, headers=headers, timeout=10).text, "html.parser") + soup = BeautifulSoup( + requests.get(URL, headers=headers, timeout=10).text, "html.parser" + ) title = soup.find("meta", attrs={"property": "og:title"}).attrs["content"] image_url = soup.find("meta", attrs={"property": "og:image"}).attrs["content"] description = soup.find("p", id="description").get_text() diff --git a/web_programming/recaptcha_verification.py b/web_programming/recaptcha_verification.py index d33608dd843d..168862204fa9 100644 --- a/web_programming/recaptcha_verification.py +++ b/web_programming/recaptcha_verification.py @@ -56,7 +56,9 @@ def login_using_recaptcha(request): client_key = request.POST.get("g-recaptcha-response") # post recaptcha response to Google's recaptcha api - response = requests.post(url, data={"secret": secret_key, "response": client_key}, timeout=10) + response = requests.post( + url, data={"secret": secret_key, "response": client_key}, timeout=10 + ) # if the recaptcha api verified our keys if response.json().get("success", False): # authenticate the user diff --git a/web_programming/slack_message.py b/web_programming/slack_message.py index 85a5e36c476c..d4d5658898ac 100644 --- a/web_programming/slack_message.py +++ b/web_programming/slack_message.py @@ -5,7 +5,9 @@ def send_slack_message(message_body: str, slack_url: str) -> None: headers = {"Content-Type": "application/json"} - response = requests.post(slack_url, json={"text": message_body}, headers=headers, timeout=10) + response = requests.post( + slack_url, json={"text": message_body}, headers=headers, timeout=10 + ) if response.status_code != 200: msg = ( "Request to slack returned an error "