A web crawler in Python

Question

The crawler crawls for a set of keywords and saves the count in a database:

import re
import time
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
import os
import httplib2
#import Links
#import Keywords
import MySQLdb
import peewee
from peewee import *
from datetime import datetime
import argparse
import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('crawler.log')
fh.setLevel(logging.DEBUG)
#ch = logging.StreamHandler()
#ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
#ch.setFormatter(formatter)
#logger.addHandler(ch)
fh.setFormatter(formatter)
logger.addHandler(fh)
parser = argparse.ArgumentParser()
parser.add_argument('-l', '--url', help="The base link to be crawled", required=True)
parser.add_argument('-k', '--keywords', help="Keywords to search", required=True)
args = parser.parse_args()
keywords = (args.keywords).split(',')
mapping = dict()
mapping[args.url] = keywords
logger.info(mapping)
db = MySQLDatabase('WebSpider', user='ruut', passwd='ruut')
parsed = set()
class DATA(peewee.Model):
 parent_link = peewee.CharField()
 sub_link = peewee.CharField()
 keyword = peewee.CharField()
 count = peewee.IntegerField()
 class Meta:
 database = db
 db_table = 'DATA'
def make_soup(s):
 match=re.compile('https://|http://')
 if re.search(match,s):
 try:
 http = httplib2.Http()
 status, response = http.request(s)
 page = BeautifulSoup(response,'lxml')
 return page
 except:
 return None
 else:
 return None
def get_list_of_urls(url):
 match = re.compile('(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9]\.[^\s]{2,})')
 soup = make_soup(url)
 l = set()
 try:
 for a in soup.find_all('a'):
 try:
 if '?' not in a['href'] and re.search(match,a['href']) and re.search(re.compile(url),a['href']) and a['href']!=url:
 l.add(str(a['href']))
 except Exception as e:
 logger.info('Exception ' + str(a)+' has no href')
 logger.info(e)
 continue
 except Exception as e:
 logger.info('Exception ' + url+' has no links')
 logger.info(e)
 pass
 return l
def get_all_the_urls(base,list_of_urls,depth):
 logger.info(depth)
 if depth == 10:
 return 
 else: 
 depth = depth + 1
 for i in list_of_urls: #scan the list of urls 
 s = get_list_of_urls(i)
 get_all_the_urls(base,s,depth)
 for j in s: #scan the sublinks
 try:
 if j in parsed:
 continue
 soup = make_soup(j)
 logger.info('url is '+ j)
 for k in mapping[base]: #look for keys on the webpage
 key_count = len(soup(text=re.compile(k, re.IGNORECASE)))
 logger.info('Key count is '+str(key_count))
 if(key_count>0):
 record = DATA(parent_link = base,sub_link = j ,keyword = k ,count = key_count) #i,j,k,key_count
 record.save()
 parsed.add(j)
 logger.info('saved data successfully ' +str(key_count))
 except Exception as e:
 logger.info('Exception ' +str(e)+' in keywords searching')
 continue
def populate_db():
 k = set()
 k.add(args.url)
 temp = time.time()
 logger.info(str(datetime.now()))
 get_all_the_urls(args.url,k,0)
 logger.info('time taken '+str(time.time()-temp))
populate_db()

Answer 1

Some of the general things I would work on:

• split the code into separate modules logically. Currently, you have all the code mixed up in single file - you have argument parsing, database interactions, web-scraping code blocks in one place
• consistent indentation. Use 4 spaces for indentation
• variable naming. Use descriptive variable names. Variable names like l, i or j are not meaningful and raise questions when reading the code

Code Style

• avoid handling broad exceptions with a bare except
• remove unused imports and re-group them based on PEP8 recommendations
• make sure to properly use whitespaces in expressions in statements
• put the main execution logic of the program into the if __name__ == '__main__':
• you don't need that pass in the get_list_of_urls() function
• depth = depth + 1 could be shortened to depth += 1

Performance

• since you are requesting the pages from the same host multiple times, consider switching to requests making use of a single session instance which allows to re-use an underlying TCP connection making subsequent requests to the same host faster
• importing SoupStrainer was actually a good idea. You can use it and scope the parsing to only the desired parts of the HTML
• since you are using regular expressions checks here and there, consider pre-compiling them and using the compiled patterns for searching and matching

Answer 2

1. Follow pep8 style guide for python's coding guide lines. This would increase readability

2. Leave a blank line after all imports.try to organize imports like first standard library, then third-party library like BeautifulSoup

3. Do not use a general Exception, catch only the specific exception you expect

4. continue is not needed