|
10 | 10 | import org.openqa.selenium.edge.EdgeDriver; |
11 | 11 |
|
12 | 12 | import java.io.FileWriter; |
13 | | -import java.io.IOException; |
14 | 13 | import java.nio.file.Files; |
15 | 14 | import java.nio.file.Path; |
16 | 15 | import java.nio.file.Paths; |
@@ -50,16 +49,26 @@ public void setup() throws InterruptedException { |
50 | 49 | companyURLs.add(link); |
51 | 50 | } |
52 | 51 |
|
53 | | - for (String companyURL : companyURLs) { |
| 52 | + for (int i = 0; i < companyURLs.size(); i++) { |
| 53 | + String companyURL = companyURLs.get(i); |
54 | 54 | String companyName = extractCompanyNameWithRegex(companyURL); |
55 | 55 | for (String recency : new String[]{"thirty-days", "three-months", "six-months", "more-than-six-months", "all"}) { |
56 | 56 | visitCompanies(String.format("https://leetcode.com/company/%s/?favoriteSlug=%s-%s", companyName, companyName, recency), driver, recency); |
57 | 57 | } |
| 58 | + log.info("Pending {} companies", companyURLs.size() - i - 1); |
58 | 59 | } |
59 | 60 | } |
60 | 61 |
|
61 | 62 | private void visitCompanies(String companyURL, WebDriver driver, String recency) throws InterruptedException { |
62 | 63 | String companyName = extractCompanyNameWithRegex(companyURL); |
| 64 | + |
| 65 | + // Create directory structure: companyName/recency/ |
| 66 | + Path outputDir = Paths.get(companyName, String.format("%s.csv", recency)); |
| 67 | + if (Files.exists(outputDir)) { |
| 68 | + log.info("File already exists: {}", outputDir.toAbsolutePath()); |
| 69 | + return; |
| 70 | + } |
| 71 | + |
63 | 72 | log.info("Visiting {} with recency {}", companyName, recency); |
64 | 73 | driver.get(companyURL); |
65 | 74 | Thread.sleep(QUESTIONS_PAGE_WAIT_MILLIS); |
@@ -89,38 +98,25 @@ private String extractCompanyNameWithRegex(String companyURL) { |
89 | 98 | private static void loadAllProblems(WebDriver driver) { |
90 | 99 | JavascriptExecutor js = (JavascriptExecutor) driver; |
91 | 100 | int maxScrolls = 30; // Maximum number of scrolls |
92 | | - int consecutiveNoChange = 0; // Counter for consecutive scrolls with no new content |
93 | | - int maxConsecutiveNoChange = 3; // Stop after 3 consecutive scrolls with no new content |
94 | 101 |
|
95 | 102 | log.info("Loading all problems by scrolling..."); |
96 | 103 |
|
97 | 104 | for (int i = 0; i < maxScrolls; i++) { |
98 | 105 | int currentCount = driver.findElements(By.cssSelector("a[href*='/problems/'][id]")).size(); |
99 | | - |
100 | 106 | boolean scrolled = performScroll(driver, js); |
101 | | - |
102 | 107 | if (!scrolled) { |
103 | | - log.info("Scroll " + (i + 1) + ": Unable to scroll further, stopping."); |
| 108 | + log.info("Scroll {}: Unable to scroll further, stopping.", i + 1); |
104 | 109 | break; |
105 | 110 | } |
106 | 111 |
|
107 | 112 | // Check if new problems were loaded |
108 | 113 | int newCount = driver.findElements(By.cssSelector("a[href*='/problems/'][id]")).size(); |
109 | | - |
110 | 114 | if (newCount > currentCount) { |
111 | 115 | // New content loaded |
112 | | - consecutiveNoChange = 0; |
113 | | - log.info("Scroll " + (i + 1) + ": Found " + newCount + " problems (+" + (newCount - currentCount) + " new)"); |
| 116 | + log.info("Scroll {}: Found {} problems (+{} new)", i + 1, newCount, newCount - currentCount); |
114 | 117 | } else { |
115 | 118 | // No new content |
116 | | - consecutiveNoChange++; |
117 | | - log.info("Scroll " + (i + 1) + ": No new problems loaded (" + consecutiveNoChange + "/" + maxConsecutiveNoChange + ")"); |
118 | | - |
119 | | - // If we've had consecutive scrolls with no new content, assume we've reached the end |
120 | | - if (consecutiveNoChange >= maxConsecutiveNoChange) { |
121 | | - log.info("No new content loaded after " + maxConsecutiveNoChange + " consecutive scrolls. Assuming all problems are loaded."); |
122 | | - break; |
123 | | - } |
| 119 | + break; |
124 | 120 | } |
125 | 121 | } |
126 | 122 | log.info("Finished loading problems."); |
|
0 commit comments