Scraping Web Info

Question

I am trying to scrape information from a website and save it into CSV format. However, even though I am getting a response from the website I am not able to retrieve any data into excel. all I am getting back are my headers with empty columns. What could be wrong with my code? Any help is appreciated.

from csv import writer
from bs4 import BeautifulSoup
import requests
url = "https://www.sephora.com/beauty/new-beauty-products"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, 
like Gecko) Chrome/102.0.5005.63 Safari/537.36'}
page = requests.get(url, headers=headers)
print(page.status_code)
soup = BeautifulSoup(page.content, 'html.parser')
lists = soup.find_all('div', class_="s-card-container")
with open('Competition Pricing Sepho.csv', 'w+', encoding='utf8', newline='') as f:
 thewriter = writer(f)
 header = ['Title', 'Price']
 thewriter.writerow(header)
 for list in lists:
 Title = list.find('span', class_="css-12vkztw").text
 Price = list.find('span', class_="css-0").text
 info = [Title, Price]
 thewriter.writerow(info)

• Alexander: where are you getting s-card-container from? I don't see that class anywhere.

Answer 1

Page is dynamic and not in the static html.

import requests
import pandas as pd
import re
import json
url = 'https://www.res-x.com/ws/r2/Resonance.aspx'
payload = {
'appid': 'sephora01',
'tk': '283272623958663',
'ss': '985647104696884',
'sg': '1',
'pg': 'res22061315349337742234660',
'vr': '5.10x',
'bx': 'true',
'sc': 'content1_rr',
'ev': '',
'ei': '',
'no': '20',
'language': 'ENGLISH',
'categoryid': '',
'page': '1',
'ccb': 'Sephora.certona',
'ur': 'https://www.sephora.com/beauty/new-beauty-products',
'plk': '',
'rf': ''}
rows = []
for page in range(1,11):
 payload.update({'page':f'{page}'})
 response = requests.get(url, params=payload).text.strip('Sephora.certona(').rsplit(');',1)[0]
 jsonData = json.loads(response)
 
 items = jsonData['resonance']['schemes'][0]['items']
 rows += items
 print(f'Page: {page}')
 
df = pd.DataFrame(rows)

Output:

print(df.head(5).to_string())
 id display_name variation_type product_type product_url brand_name default_sku_id reviews rating heroImageAltText certona_algorithm_id certona_experience_id certona_audience_id certona_strategy_id skus
0 P500452 SoftSculpt® Transforming Skin Enhancer Color standard /product/softsculpt-transforming-skin-enhancer-P500452 MAKEUP BY MARIO 2578011 101 4.485 vn 4264 651 81501 [{'sku_number': '2578011', 'sku_size': '0.18 oz / 5 g', 'sku_type': 'Standard', 'list_price': 30.0, 'primary_product_id': 'P500452', 'additional_sku_desc': 'SoftSculpt® Transforming Skin Enhancer Light Medium', 'grid_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2578011-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'Light Medium', 'deep_link': 'https://sephora.app.link/FZ6NdCyKjqb?fallbackurl=/product/softsculpt-transforming-skin-enhancer-P500452?skuId=2578011', 'badgeAltText': '', 'highlights': ['Light Coverage', 'Vegan', 'Radiant Finish', 'Cream Formula', 'Fragrance Free', 'Cruelty-Free']}]
1 P500423 Multi-Peptide Lash and Brow Serum None standard /product/multi-peptide-lash-brow-serum-P500423 The Ordinary 2532588 20 4.700 vn 4264 651 81501 [{'sku_number': '2532588', 'sku_size': '.16 oz / 5 mL', 'sku_type': 'Standard', 'list_price': 14.5, 'primary_product_id': 'P500423', 'additional_sku_desc': 'Multi-Peptide Lash and Brow Serum', 'grid_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2532588-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': '', 'deep_link': 'https://sephora.app.link/HgVj5WH5hqb?fallbackurl=/product/multi-peptide-lash-brow-serum-P500423?skuId=2532588', 'badgeAltText': '', 'highlights': ['Vegan', 'Good for: Hair Thinning', 'Fragrance Free', 'Cruelty-Free']}]
2 P500062 Y Eau de Toilette Size + Concentration + Formulation standard /product/y-eau-de-toilette-P500062 Yves Saint Laurent 2546216 1072 4.738 vn 4264 651 81501 [{'sku_number': '2546216', 'sku_size': '3.4 oz / 100 mL', 'sku_type': 'Standard', 'list_price': 100.0, 'primary_product_id': 'P500062', 'additional_sku_desc': 'Y Eau de Toilette 3.4 oz / 100 mL eau de toilette spray', 'grid_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2546216-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': False, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'size:3.4 oz / 100 mL concentration:eau de toilette formulation:spray', 'deep_link': 'https://sephora.app.link/ppre1pZZlpb?fallbackurl=/product/y-eau-de-toilette-P500062?skuId=2546216', 'badgeAltText': '', 'highlights': ['Layerable Scent', 'Woody & Earthy Scent']}]
3 P500403 SoftSculpt® Transforming Skin Perfector Color standard /product/softsculpt-transforming-skin-perfector-P500403 MAKEUP BY MARIO 2577963 37 4.216 14 4264 651 81470 [{'sku_number': '2577963', 'sku_size': '0.31 oz / 8.8 g', 'sku_type': 'Standard', 'list_price': 34.0, 'primary_product_id': 'P500403', 'additional_sku_desc': 'SoftSculpt® Transforming Skin Perfector Light', 'grid_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2577963-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': True, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': 'Light', 'deep_link': 'https://sephora.app.link/urAEdzxKjqb?fallbackurl=/product/softsculpt-transforming-skin-perfector-P500403?skuId=2577963', 'badgeAltText': '', 'highlights': ['Light Coverage', 'Vegan', 'Radiant Finish', 'Fragrance Free', 'Cruelty-Free']}]
4 P500285 Beautiful Skin Sun-Kissed Glow Cream Bronzer Color standard /product/beautiful-skin-sun-kissed-glow-cream-bronzer-P500285 Charlotte Tilbury 2569366 47 4.319 14 4264 651 81470 [{'sku_number': '2569366', 'sku_size': '0.74 oz / 21 g', 'sku_type': 'Standard', 'list_price': 56.0, 'primary_product_id': 'P500285', 'additional_sku_desc': 'Beautiful Skin Sun-Kissed Glow Cream Bronzer 1 Fair', 'grid_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=135', 'hero_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=270', 'large_hero_images': '/productimages/sku/s2569366-main-zoom.jpg?imwidth=450', 'is_sephora_exclusive': False, 'is_online_only': False, 'is_limited_edition': False, 'is_app_exclusive': False, 'is_first_access': False, 'is_limited_time_offer': False, 'is_new': True, 'variation_value': '1 Fair', 'deep_link': 'https://sephora.app.link/j1Xzw9D5hqb?fallbackurl=/product/beautiful-skin-sun-kissed-glow-cream-bronzer-P500285?skuId=2569366', 'badgeAltText': '', 'highlights': ['Hyaluronic Acid', 'Long-wearing', 'Best for Dry, Combo, Normal Skin', 'Cruelty-Free']}]
....
[131 rows x 15 columns]