|
1 | 1 | import requests |
2 | 2 | from bs4 import BeautifulSoup |
3 | 3 |
|
4 | | -base_url = "https://www.yelp.com/search?find_desc=Restaurants&find_loc={}" |
| 4 | +base_url = "https://www.yelp.com/search?find_desc=Restaurants&find_loc={}&start={}" |
5 | 5 | city = "los+angeles" |
6 | | -start = 30 |
7 | | - |
8 | | -url = base_url.format(city) |
9 | | - |
10 | | -third_page = url + '&start=' + str(start) |
11 | | - |
12 | | -response = requests.get(third_page) |
13 | | -print(f"STATUS CODE: {response.status_code} FOR {response.url}") |
14 | | - |
| 6 | +start = 0 |
15 | 7 | file_path = f'yelp-{city}.txt' |
16 | 8 |
|
17 | | -with open(file_path, 'w') as textFile: |
| 9 | +while start < 990: |
| 10 | + print(start) |
| 11 | + url = base_url.format(city, start) |
| 12 | + response = requests.get(url) |
| 13 | + print(f"STATUS CODE: {response.status_code} FOR {response.url}") |
18 | 14 | soup = BeautifulSoup(response.text, 'html.parser') |
19 | 15 | businesses = soup.findAll('div', {'class': 'biz-listing-large'}) |
20 | 16 | count = 0 |
| 17 | + |
21 | 18 | for biz in businesses: |
22 | | - title = biz.find('a', {'class': 'biz-name'}).text |
23 | | - address = biz.find('address').text |
24 | | - phone = biz.find('span', {'class': 'biz-phone'}).text |
25 | | - detail = f"{title}\n{address}\n{phone}" |
26 | | - textFile.write(str(detail) + '\n\n') |
| 19 | + try: |
| 20 | + title = biz.find('a', {'class': 'biz-name'}).text |
| 21 | + address = biz.find('address').text |
| 22 | + phone = biz.find('span', {'class': 'biz-phone'}).text |
| 23 | + detail = f"{title}\n{address}\n{phone}" |
| 24 | + print(detail) |
| 25 | + count += 1 |
| 26 | + except Exception as e: |
| 27 | + print(e) |
| 28 | + start += 30 |
| 29 | + |
| 30 | + |
| 31 | + |
| 32 | +# with open(file_path, 'w') as textFile: |
| 33 | +# soup = BeautifulSoup(response.text, 'html.parser') |
| 34 | +# businesses = soup.findAll('div', {'class': 'biz-listing-large'}) |
| 35 | +# count = 0 |
| 36 | +# for biz in businesses: |
| 37 | +# title = biz.find('a', {'class': 'biz-name'}).text |
| 38 | +# address = biz.find('address').text |
| 39 | +# phone = biz.find('span', {'class': 'biz-phone'}).text |
| 40 | +# detail = f"{title}\n{address}\n{phone}" |
| 41 | +# textFile.write(str(detail) + '\n\n') |
0 commit comments