diff --git a/cron/update_questions.py b/cron/update_questions.py index 8c05222b..02135316 100644 --- a/cron/update_questions.py +++ b/cron/update_questions.py @@ -1,9 +1,11 @@ import os import json +import uuid import leetcode import leetcode.auth from datetime import datetime from leetcode.rest import ApiException +from concurrent.futures import ThreadPoolExecutor, as_completed def create_leetcode_api(): @@ -26,6 +28,7 @@ def get_question_metadata(api, title_slug): query='''query questionData($titleSlug: String!) { question(titleSlug: $titleSlug) { title + titleSlug difficulty companyTagStats isPaidOnly @@ -38,13 +41,12 @@ def get_question_metadata(api, title_slug): try: response = api.graphql_post(body=graphql_request) + return response except ApiException as e: print( f'Exception occurred when contacting the Leetcode GraphQL API: ${e}') exit() - return response - def construct_company_tag_list(company_tags_json, sections): companies = [] @@ -60,13 +62,24 @@ def construct_company_tag_list(company_tags_json, sections): return sorted(companies, key=lambda d: d['frequency'], reverse=True) -def update_question_metadata(question, title, difficulty, companies, is_premium): - print(f"🔄 Updating question metadata for {title}") +def update_question_metadata(question, response): + question_title = response.data.question.title + question_difficulty = response.data.question.difficulty + question_company_tags = json.loads( + response.data.question.company_tag_stats) + question_is_premium = response.data.question.is_paid_only + + # Retrieve companies who have asked this question within the following two + # company_tag_stat sections: + # 1. 0-6 months + # 2. 6 months to 1 year + companies = construct_company_tag_list( + question_company_tags, ["1", "2"]) - question["title"] = title - question["difficulty"] = difficulty + question["title"] = question_title + question["difficulty"] = question_difficulty question["companies"] = companies - question["premium"] = is_premium + question["premium"] = question_is_premium def read_questions(file_name): @@ -97,30 +110,28 @@ def write_questions(file_name, questions): exit() -def main(file_name): - api = create_leetcode_api() - questions = read_questions(file_name) +def runners(api, question_list): + print(f"📡 Retrieving question metadata from Leetcode") - for question in questions["data"]: - title_slug = question["slug"] + threads = [] - response = get_question_metadata(api, title_slug) + with ThreadPoolExecutor(max_workers=3) as executor: + for question in question_list: + title_slug = question["slug"] + threads.append(executor.submit( + get_question_metadata, api, title_slug)) - question_title = response.data.question.title - question_difficulty = response.data.question.difficulty - question_company_tags = json.loads( - response.data.question.company_tag_stats) - question_is_premium = response.data.question.is_paid_only + for task in as_completed(threads): + update_question_metadata(question, task.result()) - # Retrieve companies who have asked this question within the following two - # company_tag_stat sections: - # 1. 0-6 months - # 2. 6 months to 1 year - companies = construct_company_tag_list( - question_company_tags, ["1", "2"]) + print(f"✅ Finished retrieving question metadata from Leetcode") + + +def main(file_name): + api = create_leetcode_api() + questions = read_questions(file_name) - update_question_metadata(question, question_title, question_difficulty, - companies, question_is_premium) + runners(api, questions["data"]) write_questions(file_name, questions) diff --git a/src/data/questions.json b/src/data/questions.json index 7ebb9eb1..06a21259 100644 --- a/src/data/questions.json +++ b/src/data/questions.json @@ -1,5 +1,5 @@ { - "updated": "2022-08-07T17:24:54.842630", + "updated": "2022-08-10T23:49:33.616820", "data": [ { "id": 0,