- Using BeautifulSoup, a python library useful for web scraping, this script helps to scrape a desired subreddit to obtain all relevant data regarding its posts.

- In the `fetch_reddit.py` , we take user input for the subreddit name, tags and the maximum count of posts to be scraped, we fetch and store all this information in a database file.

- In the `display_reddit.py` , we display the desired results from the database to the user.

- The requirements can be installed as follows:

$ pip install -r requirements.txt



[Rohini Rao](www.github.com/RohiniRG)

import sqlite3

import os

def sql_connection():

"""

path = os.path.abspath('SubredditDatabase.db')

con = sqlite3.connect(path)

return con

def sql_fetcher(con):

"""

subreddit = input("\nEnter subreddit to search: r/")

count = 0

cur = con.cursor()

cur.execute('SELECT * FROM posts') # SQL search query

rows = cur.fetchall()

for r in rows:

if subreddit in r:

count += 1

print(f'\nTAG: {r[1]}\nPOST TITLE: {r[2]}\nAUTHOR: {r[3]}\n'

f'TIME STAMP: {r[4]}\nUPVOTES: {r[5]}\nCOMMENTS: {r[6]}'

f'\nURL: {r[7]}\n')

if count:

print(f'{count} posts fetched from database\n')

else:

print('\nNo posts stored for this subreddit\n')

con = sql_connection()

while 1:

sql_fetcher(con)

ans = input('\nPress (y) to continue or any other key to exit: ').lower()

if ans == 'y':

continue

else:

print('\nExiting..\n')

break

import requests

import csv

import time

import sqlite3

from bs4 import BeautifulSoup

def sql_connection():

"""

con = sqlite3.connect('SubredditDatabase.db')

return con

def sql_table(con):

"""

cur = con.cursor()

cur.execute("CREATE TABLE IF NOT EXISTS posts(SUBREDDIT text, TAG text, "

" TITLE text, AUTHOR text, TIMESTAMP text, UPVOTES int, "

" COMMENTS text, URL text)")

con.commit()

def sql_insert_table(con, entities):

"""

cur = con.cursor()

cur.execute('INSERT INTO posts(SUBREDDIT, TAG, TITLE, AUTHOR, '

'TIMESTAMP, UPVOTES, COMMENTS, URL) '

'VALUES(?, ?, ?, ?, ?, ?, ?, ?)', entities)

con.commit()

def scraper():

"""

con = sql_connection()

sql_table(con)

while 1:

subreddit = input('\n\nEnter the name of the subreddit: r/').lower()

max_count = int(input('Enter the maximum number of entries to collect: '))

select = int(input('Select tags to add for the search: \n1. hot\n2. new'

'\n3. rising\n4. controversial\n5. top\nMake your choice: '))

if select == 1:

tag = 'hot'

tag_url = '/'

elif select == 2:

tag = 'new'

tag_url = '/new/'

elif select == 3:

tag = 'rising'

tag_url = '/rising/'

elif select == 4:

tag = 'controversial'

tag_url = '/controversial/'

elif select == 5:

tag = 'top'

tag_url = '/top/'

# URL for the desired subreddit

url = 'https://old.reddit.com/r/' + subreddit

# Using a user-agent to mimic browser activity

headers = {'User-Agent': 'Mozilla/5.0'}

req = requests.get(url, headers=headers)

if req.status_code == 200:

soup = BeautifulSoup(req.text, 'html.parser')

print(f'\nCOLLECTING INFORMATION FOR r/{subreddit}....')

attrs = {'class': 'thing'}

counter = 1

full = 0

reddit_info = []

while 1:

for post in soup.find_all('div', attrs=attrs):

try:

# To obtain the post title

title = post.find('a', class_='title').text

# To get the username of the post author

author = post.find('a', class_='author').text

# To obtain the time of the post

time_stamp = post.time.attrs['title']

# To obtain the number of comments on the post

comments = post.find('a', class_='comments').text.split()[0]

if comments == 'comment':

comments = 0

# To get the number of comments on the post

upvotes = post.find('div', class_='score likes').text

if upvotes == '•':

upvotes = "None"

# To get the URL of the post

link = post.find('a', class_='title')['href']

link = 'www.reddit.com' + link

# Entering all the collected information into our database

entities = (subreddit, tag, title, author, time_stamp, upvotes,

comments, link)

sql_insert_table(con, entities)

if counter == max_count:

full = 1

break

counter += 1

except AttributeError:

continue

if full:

break

try:

# To go to the next page

next_button = soup.find('span', class_='next-button')

next_page_link = next_button.find('a').attrs['href']

time.sleep(2)

req = requests.get(next_page_link, headers=headers)

soup = BeautifulSoup(req.text, 'html.parser')

except:

break

print('DONE!\n')

ans = input('Press (y) to continue or any other key to exit: ').lower()

if ans == 'y':

continue

else:

print('Exiting..')

break

else:

print('Error fetching results.. Try again!')

if __name__ == '__main__':

scraper()