|
| 1 | +import requests |
| 2 | +from requests.exceptions import HTTPError |
| 3 | +from sqlalchemy import create_engine |
| 4 | +from sqlalchemy.exc import SQLAlchemyError |
| 5 | + |
| 6 | +def get_all_slugs(): |
| 7 | + url = "https://leetcode.com/api/problems/all/" |
| 8 | + slugs = [] |
| 9 | + try: |
| 10 | + response = requests.get(url) |
| 11 | + response.raise_for_status() |
| 12 | + except HTTPError as http_err: |
| 13 | + print(f'HTTP error occurred: {http_err}') # Python 3.6 |
| 14 | + except Exception as err: |
| 15 | + print(f'Other error occurred: {err}') # Python 3.6 |
| 16 | + else: |
| 17 | + r_json = response.json() |
| 18 | + for slug in r_json["stat_status_pairs"]: |
| 19 | + slugs.append(slug["stat"]["question__title_slug"]) |
| 20 | + return slugs |
| 21 | + |
| 22 | +def get_quest_info(slug): |
| 23 | + query = """ |
| 24 | + query questionData($titleSlug: String!) {\n question(titleSlug: $titleSlug) {\n questionId\n questionFrontendId\n boundTopicId\n title\n titleSlug\n content\n translatedTitle\n translatedContent\n isPaidOnly\n difficulty\n likes\n dislikes\n isLiked\n similarQuestions\n contributors {\n username\n profileUrl\n avatarUrl\n __typename\n }\n langToValidPlayground\n topicTags {\n name\n slug\n translatedName\n __typename\n }\n companyTagStats\n codeSnippets {\n lang\n langSlug\n code\n __typename\n }\n stats\n hints\n solution {\n id\n canSeeDetail\n __typename\n }\n status\n sampleTestCase\n metaData\n judgerAvailable\n judgeType\n mysqlSchemas\n enableRunCode\n enableTestMode\n envInfo\n libraryUrl\n __typename\n }\n}\n |
| 25 | + """ |
| 26 | + body = {"operationName":"questionData", |
| 27 | + "variables":{"titleSlug":slug}, |
| 28 | + "query":query} |
| 29 | + |
| 30 | + url = "https://leetcode.com/graphql" |
| 31 | + try: |
| 32 | + response = requests.post(url, json=body) |
| 33 | + response.raise_for_status() |
| 34 | + except HTTPError as http_err: |
| 35 | + print(f'HTTP error occurred: {http_err}') # Python 3.6 |
| 36 | + except Exception as err: |
| 37 | + print(f'Other error occurred: {err}') # Python 3.6 |
| 38 | + else: |
| 39 | + r_json = response.json() |
| 40 | + return r_json["data"]["question"] |
| 41 | + |
| 42 | +def createRawTable(dbName='leetcode.db'): |
| 43 | + createTableSQL = "CREATE TABLE questions( " \ |
| 44 | + "questionId INT, questionFrontendId INT, title TEXT, titleSlug TEXT, content TEXT, " \ |
| 45 | + "isPaidOnly INT, difficulty TEXT, likes INT, dislikes INT);" |
| 46 | + db_connect = create_engine('sqlite:///{}'.format(dbName)) |
| 47 | + conn = db_connect.connect() |
| 48 | + try: |
| 49 | + conn.execute(createTableSQL) |
| 50 | + print("created table: {}".format(dbName)) |
| 51 | + except SQLAlchemyError as e: |
| 52 | + print("SQL error: {}".format(e)) |
| 53 | + raise SQLAlchemyError |
| 54 | + |
| 55 | +def insertQuestInfo(quest_json, dbName='leetcode.db'): |
| 56 | + questionId = quest_json.get('questionId', -1) |
| 57 | + questionFrontendId = quest_json.get('questionFrontendId', -1) |
| 58 | + title = quest_json.get('title', "").replace("'", " ").replace('"', " ").replace(";", " ") |
| 59 | + titleSlug = quest_json.get('titleSlug', "").replace("'", " ").replace('"', " ").replace(";", " ") |
| 60 | + # content = quest_json.get('content', "") # TODO: store question content |
| 61 | + content = "" |
| 62 | + isPaidOnly = quest_json.get('isPaidOnly', -1) |
| 63 | + difficulty = quest_json.get('difficulty', "").replace("'", " ").replace('"', " ").replace(";", " ") |
| 64 | + likes = quest_json.get('likes', -1) |
| 65 | + dislikes = quest_json.get('dislikes', -1) |
| 66 | + # print(questionId, questionFrontendId, title, titleSlug, content, isPaidOnly, difficulty, likes, dislikes) |
| 67 | + |
| 68 | + insertSQL = "INSERT INTO questions (questionId, questionFrontendId, title, titleSlug, content, isPaidOnly, difficulty, likes, dislikes) " \ |
| 69 | + "VALUES ( {}, {}, '{}', '{}', '{}', {}, '{}', {}, {});".format(questionId, questionFrontendId, title, titleSlug, content, isPaidOnly, difficulty, likes, dislikes) |
| 70 | + |
| 71 | + db_connect = create_engine('sqlite:///{}'.format(dbName)) |
| 72 | + conn = db_connect.connect() |
| 73 | + try: |
| 74 | + conn.execute(insertSQL) |
| 75 | + except SQLAlchemyError as e: |
| 76 | + print("SQL error: {}".format(e)) |
| 77 | + raise SQLAlchemyError |
| 78 | + |
| 79 | +def insert_all_quest(): |
| 80 | + slugs = get_all_slugs() |
| 81 | + inserted = 0 |
| 82 | + for slug in slugs: |
| 83 | + quest_json = get_quest_info(slug) |
| 84 | + if quest_json: |
| 85 | + insertQuestInfo(quest_json) |
| 86 | + else: |
| 87 | + print("json invalid in {}".format(slug)) |
| 88 | + inserted += 1 |
| 89 | + print("inserted {}/{}".format(inserted, len(slugs))) |
| 90 | + |
| 91 | + |
| 92 | +if __name__ == "__main__": |
| 93 | + createRawTable() |
| 94 | + insert_all_quest() |
0 commit comments