Twitter contact scraper with Tweepy & Django

Question

I created a project for academic purposes that works on Twitter data. It should:

• Get all friends and follows of the user
• Store them in MongoDB
• Display them in a table

I'm using django + plain HTML, but when trying on an account with only 230 contacts it took 3 minutes to load. The application is intended to be scalable to handle accounts with at least 100k contacts.

How can I improve the scalability of this program?

from twitter_auth.apiKeys import *
import tweepy
from manager.models import Contact
def get_api(user):
 access_tokens = user.social_auth \
 .filter(provider='twitter')[0] \
 .extra_data['access_token']
 auth = tweepy.OAuthHandler(
 SOCIAL_AUTH_TWITTER_KEY,
 SOCIAL_AUTH_TWITTER_SECRET
 )
 auth.set_access_token(
 access_tokens['oauth_token'],
 access_tokens['oauth_token_secret']
 )
 api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
 return api
def get_user_contacts(user):
 # Setup Twitter API instance of logged User
 api = get_api(user)
 me = api.me()
 # Set number of contacts fetched per call Max:200
 count = 200
 # Fetch logged User's friends into a list
 friends = tweepy.Cursor(api.friends, me.screen_name, count=count).items()
 # friends = api.friends(me.screen_name, count=10)
 friend_list = []
 for i in friends:
 friend_list.append(i)
 # Fetch logged User's followers into a list
 followers = tweepy.Cursor(api.followers, me.screen_name, count=count).items()
 # followers = api.followers(me.screen_name, count=10)
 follower_list = []
 for j in followers:
 follower_list.append(j)
 existing_contacts = Contact.objects.filter(user=user)
 for contact in existing_contacts:
 if contact not in friend_list and contact not in follower_list:
 contact.delete()
 # Start iterating friend list fetched from Twitter API
 for friend in friend_list:
 # Initialize Contact Object
 temp_friend = Contact(
 twitter_id=friend.id,
 profile_image_url=friend.profile_image_url_https,
 screen_name=friend.screen_name,
 name=friend.name,
 followers_count=friend.followers_count,
 friends_count=friend.friends_count,
 statuses_count=friend.statuses_count,
 description=friend.description,
 location=friend.location,
 friendship_status=1,
 protected_status=friend.protected,
 user=user
 )
 # Check if entry is just Friend (1) or Friend & Follower (3)
 if friend in follower_list:
 temp_friend.friendship_status = 3
 # Check if current friend already exists in DB
 existing_friend = Contact.objects.filter(screen_name=friend.screen_name, user=user)
 if existing_friend:
 # Update Contact info
 existing_friend.update(
 profile_image_url=friend.profile_image_url_https,
 name=friend.name,
 followers_count=friend.followers_count,
 friends_count=friend.friends_count,
 statuses_count=friend.statuses_count,
 description=friend.description,
 location=friend.location,
 protected_status=friend.protected,
 )
 # Check if existing Contact followed back
 # Case: Have followed back so friendship status updates to (3)
 if existing_friend in follower_list:
 existing_friend.update(
 friendship_status=3
 )
 # Case: Have not followed back so friendship status remains unchanged(1)
 else:
 temp_friend.save()
 # Start iterating follower list fetched from Twitter API
 for follower in follower_list:
 # Initialize Contact Object
 temp_follower = Contact(
 twitter_id=follower.id,
 profile_image_url=follower.profile_image_url_https,
 screen_name=follower.screen_name,
 name=follower.name,
 followers_count=follower.followers_count,
 friends_count=follower.friends_count,
 statuses_count=follower.statuses_count,
 description=follower.description,
 location=follower.location,
 friendship_status=2,
 protected_status=follower.protected,
 user=user
 )
 # Check if current follower already exists in DB
 existing_follower = Contact.objects.filter(twitter_id=follower.id, user=user)
 if existing_follower:
 # Update Contact info
 existing_follower.update(
 profile_image_url=follower.profile_image_url_https,
 name=follower.name,
 followers_count=follower.followers_count,
 friends_count=follower.friends_count,
 statuses_count=follower.statuses_count,
 description=follower.description,
 location=follower.location,
 protected_status=follower.protected,
 )
 # Check if user followed back existing the existing follower
 # Case: Have followed back so friendship status updates to (3)
 if existing_follower in friend_list:
 existing_follower.update(
 friendship_status=3
 )
 # Case: Have not followed back so friendship status remains unchanged(2)
 else:
 temp_follower.save()
def get_user_tweets(user, id):
 api = get_api(user)
 tweet_list = api.user_timeline(id=id, count=2)
 return tweet_list

Answer 1

Line continuations

They're possible but discouraged.

access_tokens = user.social_auth \
 .filter(provider='twitter')[0] \
 .extra_data['access_token']

would be better, according to most linters, as

access_tokens = (
 user.social_auth
 .filter(provider='twitter')[0]
 .extra_data['access_token']
)

List formation

friend_list = []
for i in friends:
 friend_list.append(i)

can just be

friend_list = list(friends)

However, given your usage:

for contact in existing_contacts:
 if contact not in friend_list and contact not in follower_list:
 contact.delete()

you're better off using sets:

friend_list = set(friends)
follower_list = set(followers)
existing_contacts = set(Contact.objects.filter(user=user))
to_delete = existing_contacts - friend_list - follower_list
for contact in to_delete:
 contact.delete()